From 817174443878fcbedd7949e7d446fde8aa097561 Mon Sep 17 00:00:00 2001 From: stan44 Date: Mon, 23 Feb 2026 21:17:00 -0600 Subject: [PATCH] J's Changes added. --- .../journal/Journal.Core/Dtos/CommandDtos.cs | 35 ++ .../journal/Journal.Core/Dtos/DatabaseDtos.cs | 18 + journal-master/journal/Journal.Core/Entry.cs | 254 +----------- .../Repositories/DiskEntryFileRepository.cs | 35 ++ .../Repositories/IEntryFileRepository.cs | 14 + .../ServiceCollectionExtensions.cs | 3 + .../Journal.Core/Services/CommandLogger.cs | 76 ++++ .../Journal.Core/Services/EntryFileService.cs | 84 ++++ .../Journal.Core/Services/HtmlSanitizer.cs | 46 +++ .../Services/IEntryFileService.cs | 10 + .../Services/IJournalDatabaseService.cs | 23 +- .../Services/JournalDatabaseService.cs | 110 +---- .../Services/PythonSidecarAiService.cs | 125 +----- .../Services/PythonSidecarClient.cs | 119 ++++++ .../Services/PythonSidecarSpeechService.cs | 118 +----- .../journal/Journal.SmokeTests/Program.cs | 70 +--- journal-master/journal/README.md | 381 +++++++----------- journal-master/journal/REFACTORING_SUMMARY.md | 62 +++ 18 files changed, 715 insertions(+), 868 deletions(-) create mode 100644 journal-master/journal/Journal.Core/Dtos/CommandDtos.cs create mode 100644 journal-master/journal/Journal.Core/Dtos/DatabaseDtos.cs create mode 100644 journal-master/journal/Journal.Core/Repositories/DiskEntryFileRepository.cs create mode 100644 journal-master/journal/Journal.Core/Repositories/IEntryFileRepository.cs create mode 100644 journal-master/journal/Journal.Core/Services/CommandLogger.cs create mode 100644 journal-master/journal/Journal.Core/Services/EntryFileService.cs create mode 100644 journal-master/journal/Journal.Core/Services/HtmlSanitizer.cs create mode 100644 journal-master/journal/Journal.Core/Services/IEntryFileService.cs create mode 100644 journal-master/journal/Journal.Core/Services/PythonSidecarClient.cs create mode 100644 journal-master/journal/REFACTORING_SUMMARY.md diff --git a/journal-master/journal/Journal.Core/Dtos/CommandDtos.cs b/journal-master/journal/Journal.Core/Dtos/CommandDtos.cs new file mode 100644 index 0000000..0ed5dba --- /dev/null +++ b/journal-master/journal/Journal.Core/Dtos/CommandDtos.cs @@ -0,0 +1,35 @@ +namespace Journal.Core.Dtos; + +internal sealed record VaultInitializePayload(string Password, string VaultDirectory); +internal sealed record VaultPayload(string Password, string VaultDirectory, string DataDirectory, string? NowUtc = null); +internal sealed record ClearDataPayload(string DataDirectory); +internal sealed record EntryListPayload(string? DataDirectory = null); +internal sealed record EntryLoadPayload(string FilePath); +public sealed record EntrySavePayload(string Content, string? FilePath = null, string? Mode = null); +public sealed record EntryListItem(string FileName, string FilePath); +public sealed record EntryLoadResult(string Date, string FileName, string FilePath, string RawContent); +public sealed record EntrySaveResult(string FilePath); +internal sealed record DatabasePayload(string Password, string? DataDirectory = null); +internal sealed record AiSummarizeEntryPayload(string Content, string? FileStem = null); +internal sealed record AiSummarizeAllPayload(List? Entries); +internal sealed record AiChatPayload(string Prompt); +internal sealed record AiEmbedPayload(string Content); +internal sealed record SpeechTranscribePayload( + string? AudioBase64 = null, + string? Audio_Base64 = null, + string? Engine = null, + string? WhisperModel = null, + string? Whisper_Model = null, + string? Text = null, + int? SimulateDelayMs = null, + int? Simulate_Delay_Ms = null); +internal sealed record SearchEntriesPayload( + string DataDirectory, + string? Query = null, + string? Section = null, + string? StartDate = null, + string? EndDate = null, + List? Tags = null, + List? Types = null, + List? Checked = null, + List? Unchecked = null); diff --git a/journal-master/journal/Journal.Core/Dtos/DatabaseDtos.cs b/journal-master/journal/Journal.Core/Dtos/DatabaseDtos.cs new file mode 100644 index 0000000..3c587e7 --- /dev/null +++ b/journal-master/journal/Journal.Core/Dtos/DatabaseDtos.cs @@ -0,0 +1,18 @@ +namespace Journal.Core.Dtos; + +public sealed record JournalDatabaseStatus( + string DatabasePath, + int KeyLengthBytes, + int Iterations, + string KeyDerivation, + IReadOnlyList SchemaTables, + string SchemaBootstrapPath, + bool RuntimeReady, + string RuntimeMessage); + +public sealed record JournalDatabaseHydrationResult( + string DatabasePath, + string SchemaBootstrapPath, + int EntryFilesProcessed, + bool RuntimeReady, + string Message); diff --git a/journal-master/journal/Journal.Core/Entry.cs b/journal-master/journal/Journal.Core/Entry.cs index 39627f5..7a9db3a 100644 --- a/journal-master/journal/Journal.Core/Entry.cs +++ b/journal-master/journal/Journal.Core/Entry.cs @@ -1,7 +1,5 @@ using System.ComponentModel.DataAnnotations; using System.Globalization; -using System.Net; -using System.Text.RegularExpressions; using System.Text.Json; using Journal.Core.Dtos; using Journal.Core.Models; @@ -18,6 +16,8 @@ public class Entry private readonly IJournalConfigService _config; private readonly IAiService _ai; private readonly ISpeechBridgeService _speech; + private readonly IEntryFileService _entryFiles; + private readonly CommandLogger _logger; private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNameCaseInsensitive = true @@ -30,7 +30,9 @@ public class Entry IJournalDatabaseService database, IJournalConfigService config, IAiService ai, - ISpeechBridgeService speech) + ISpeechBridgeService speech, + IEntryFileService entryFiles, + CommandLogger logger) { _fragments = fragments; _entrySearch = entrySearch; @@ -39,6 +41,8 @@ public class Entry _config = config; _ai = ai; _speech = speech; + _entryFiles = entryFiles; + _logger = logger; } public async Task RunAsync() @@ -73,7 +77,7 @@ public class Entry var correlationId = string.IsNullOrWhiteSpace(cmd.CorrelationId) ? Guid.NewGuid().ToString("N") : cmd.CorrelationId.Trim(); - LogStart(action, correlationId, cmd.Payload); + _logger.LogStart(action, correlationId, cmd.Payload); object? result; try @@ -131,19 +135,19 @@ public class Entry var listDataDirectory = !string.IsNullOrWhiteSpace(listPayload?.DataDirectory) ? listPayload.DataDirectory : _config.Current.DataDirectory; - result = ListEntries(listDataDirectory); + result = _entryFiles.ListEntries(listDataDirectory); break; case "entries.load": var loadEntryPayload = DeserializePayload(cmd.Payload); if (loadEntryPayload is null || string.IsNullOrWhiteSpace(loadEntryPayload.FilePath)) return Error("Missing or invalid payload"); - result = LoadEntry(loadEntryPayload.FilePath); + result = _entryFiles.LoadEntry(loadEntryPayload.FilePath); break; case "entries.save": var saveEntryPayload = DeserializePayload(cmd.Payload); if (saveEntryPayload is null || string.IsNullOrWhiteSpace(saveEntryPayload.Content)) return Error("Missing or invalid payload"); - result = SaveEntry(saveEntryPayload, _config.Current.DataDirectory); + result = _entryFiles.SaveEntry(saveEntryPayload, _config.Current.DataDirectory); break; case "config.get": result = _config.Current; @@ -256,122 +260,53 @@ public class Entry result = _database.HydrateWorkspace(dbHydratePayload.Password, dbHydratePayload.DataDirectory); break; default: - LogFailure(action, correlationId, "unknown_action"); + _logger.LogFailure(action, correlationId, "unknown_action"); return Error($"Unknown action: {action}"); } } catch (JsonException) { - LogFailure(action, correlationId, "invalid_payload_json"); + _logger.LogFailure(action, correlationId, "invalid_payload_json"); return Error("Missing or invalid payload"); } catch (ValidationException ex) { - LogFailure(action, correlationId, "validation", ex.Message); + _logger.LogFailure(action, correlationId, "validation", ex.Message); return Error(ex.Message); } catch (ArgumentException ex) { - LogFailure(action, correlationId, "argument", ex.Message); + _logger.LogFailure(action, correlationId, "argument", ex.Message); return Error(ex.Message); } catch (TimeoutException ex) { - LogFailure(action, correlationId, "timeout", ex.Message); + _logger.LogFailure(action, correlationId, "timeout", ex.Message); return Error(ex.Message); } catch (InvalidOperationException ex) { - LogFailure(action, correlationId, "invalid_operation", ex.Message); + _logger.LogFailure(action, correlationId, "invalid_operation", ex.Message); return Error(ex.Message); } catch (FileNotFoundException ex) { - LogFailure(action, correlationId, "not_found", ex.Message); + _logger.LogFailure(action, correlationId, "not_found", ex.Message); return Error(ex.Message); } catch { - LogFailure(action, correlationId, "internal_error"); + _logger.LogFailure(action, correlationId, "internal_error"); return Error("Internal error"); } - LogSuccess(action, correlationId); + _logger.LogSuccess(action, correlationId); return JsonSerializer.Serialize(new { ok = true, data = result }); } private static string Error(string message) => JsonSerializer.Serialize(new { ok = false, error = message }); - private void LogStart(string action, string correlationId, JsonElement? payload) - { - var redactedPayload = LogRedactor.RedactPayload(payload); - EmitLog("information", action, correlationId, "start", redactedPayload); - } - - private void LogSuccess(string action, string correlationId) - { - EmitLog("information", action, correlationId, "success"); - } - - private void LogFailure(string action, string correlationId, string errorType, string? message = null) - { - var details = string.IsNullOrWhiteSpace(message) - ? "" - : (message!.Length <= 160 ? message : $"{message[..160]}...(truncated)"); - EmitLog("warning", action, correlationId, "failure", errorType: errorType, details: details); - } - - private static void EmitLog( - string level, - string action, - string correlationId, - string outcome, - object? payload = null, - string? errorType = null, - string? details = null) - { - if (!ShouldLog(level)) - return; - - var envelope = new - { - timestamp = DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture), - level, - component = nameof(Entry), - action, - correlation_id = correlationId, - outcome, - error_type = errorType, - details, - payload - }; - Console.Error.WriteLine(JsonSerializer.Serialize(envelope)); - } - - private static bool ShouldLog(string level) - { - var configured = (Environment.GetEnvironmentVariable("JOURNAL_LOG_LEVEL") ?? "warning") - .Trim() - .ToLowerInvariant(); - var configuredRank = LogLevelRank(configured); - var incomingRank = LogLevelRank(level); - return incomingRank >= configuredRank; - } - - private static int LogLevelRank(string level) => level switch - { - "trace" => 0, - "debug" => 1, - "information" => 2, - "info" => 2, - "warning" => 3, - "warn" => 3, - "error" => 4, - "critical" => 5, - _ => 3 - }; - private static T? DeserializePayload(JsonElement? payload) { if (payload is null) @@ -395,153 +330,4 @@ public class Entry throw new ArgumentException("Invalid nowUtc value. Expected ISO date/time."); } - - private static IReadOnlyList ListEntries(string dataDirectory) - { - if (!Directory.Exists(dataDirectory)) - return []; - - return Directory.GetFiles(dataDirectory, "*.md") - .OrderBy(Path.GetFileName, StringComparer.Ordinal) - .Select(path => new EntryListItem( - FileName: Path.GetFileName(path), - FilePath: Path.GetFullPath(path))) - .ToArray(); - } - - private static EntryLoadResult LoadEntry(string filePath) - { - var normalizedPath = Path.GetFullPath(filePath); - if (!File.Exists(normalizedPath)) - throw new FileNotFoundException($"Entry file not found: {normalizedPath}"); - - var rawContent = StripRichHtml(File.ReadAllText(normalizedPath)); - var fileStem = Path.GetFileNameWithoutExtension(normalizedPath); - var entry = JournalParser.ParseJournalContent(rawContent, fileStem); - - return new EntryLoadResult( - Date: entry.Date, - FileName: Path.GetFileName(normalizedPath), - FilePath: normalizedPath, - RawContent: entry.RawContent); - } - - private static EntrySaveResult SaveEntry(EntrySavePayload payload, string defaultDataDirectory) - { - var targetPath = ResolveTargetPath(payload.FilePath, defaultDataDirectory); - var mode = string.IsNullOrWhiteSpace(payload.Mode) ? "Daily" : payload.Mode.Trim(); - var sanitizedContent = StripRichHtml(payload.Content ?? ""); - Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); - - if (string.Equals(mode, "Overwrite", StringComparison.OrdinalIgnoreCase)) - { - File.WriteAllText(targetPath, sanitizedContent); - return new EntrySaveResult(targetPath); - } - - if (string.Equals(mode, "Fragment", StringComparison.OrdinalIgnoreCase)) - { - File.AppendAllText(targetPath, "\n\n" + sanitizedContent.Trim()); - return new EntrySaveResult(targetPath); - } - - string finalContent; - if (File.Exists(targetPath)) - { - var existingContent = File.ReadAllText(targetPath); - var fileStem = Path.GetFileNameWithoutExtension(targetPath); - var existingEntry = JournalParser.ParseJournalContent(existingContent, fileStem); - var newEntryData = JournalParser.ParseJournalContent(sanitizedContent, fileStem); - existingEntry.MergeWith(newEntryData); - finalContent = existingEntry.ToMarkdown(); - } - else - { - finalContent = sanitizedContent; - } - - File.WriteAllText(targetPath, finalContent); - return new EntrySaveResult(targetPath); - } - - private static string ResolveTargetPath(string? filePath, string defaultDataDirectory) - { - if (!string.IsNullOrWhiteSpace(filePath)) - return Path.GetFullPath(filePath); - - return Path.GetFullPath(Path.Combine(defaultDataDirectory, $"{DateTime.Now:yyyy-MM-dd}.md")); - } - - private static bool LooksLikeRichHtml(string content) - { - var lowered = content.ToLowerInvariant(); - string[] markers = - [ - "", " lowered.Contains(marker, StringComparison.Ordinal))) - return true; - return Regex.Matches(lowered, "]*>").Count >= 8; - } - - private static string StripRichHtml(string content) - { - if (string.IsNullOrWhiteSpace(content)) - return content; - if (!LooksLikeRichHtml(content)) - return content; - - var text = content.Replace("\r\n", "\n").Replace("\r", "\n"); - text = Regex.Replace(text, "<(script|style)\\b[^>]*>.*?", "", RegexOptions.IgnoreCase | RegexOptions.Singleline); - text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "]*>", "\n- ", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "<(td|th)\\b[^>]*>", " | ", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "", " ", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "]*>", "\n---\n", RegexOptions.IgnoreCase); - text = Regex.Replace(text, "<[^>]+>", "", RegexOptions.Singleline); - text = WebUtility.HtmlDecode(text) - .Replace('\u00a0', ' ') - .Replace("\u200b", "", StringComparison.Ordinal); - text = string.Join("\n", text.Split('\n').Select(line => line.TrimEnd())); - text = Regex.Replace(text, "[ \\t]{2,}", " "); - text = Regex.Replace(text, "\n{3,}", "\n\n").Trim(); - return string.IsNullOrEmpty(text) ? content : text; - } - - private sealed record VaultInitializePayload(string Password, string VaultDirectory); - private sealed record VaultPayload(string Password, string VaultDirectory, string DataDirectory, string? NowUtc = null); - private sealed record ClearDataPayload(string DataDirectory); - private sealed record EntryListPayload(string? DataDirectory = null); - private sealed record EntryLoadPayload(string FilePath); - private sealed record EntrySavePayload(string Content, string? FilePath = null, string? Mode = null); - private sealed record EntryListItem(string FileName, string FilePath); - private sealed record EntryLoadResult(string Date, string FileName, string FilePath, string RawContent); - private sealed record EntrySaveResult(string FilePath); - private sealed record DatabasePayload(string Password, string? DataDirectory = null); - private sealed record AiSummarizeEntryPayload(string Content, string? FileStem = null); - private sealed record AiSummarizeAllPayload(List? Entries); - private sealed record AiChatPayload(string Prompt); - private sealed record AiEmbedPayload(string Content); - private sealed record SpeechTranscribePayload( - string? AudioBase64 = null, - string? Audio_Base64 = null, - string? Engine = null, - string? WhisperModel = null, - string? Whisper_Model = null, - string? Text = null, - int? SimulateDelayMs = null, - int? Simulate_Delay_Ms = null); - private sealed record SearchEntriesPayload( - string DataDirectory, - string? Query = null, - string? Section = null, - string? StartDate = null, - string? EndDate = null, - List? Tags = null, - List? Types = null, - List? Checked = null, - List? Unchecked = null); } diff --git a/journal-master/journal/Journal.Core/Repositories/DiskEntryFileRepository.cs b/journal-master/journal/Journal.Core/Repositories/DiskEntryFileRepository.cs new file mode 100644 index 0000000..bd95574 --- /dev/null +++ b/journal-master/journal/Journal.Core/Repositories/DiskEntryFileRepository.cs @@ -0,0 +1,35 @@ +namespace Journal.Core.Repositories; + +public sealed class DiskEntryFileRepository : IEntryFileRepository +{ + public IReadOnlyList ListMarkdownFiles(string dataDirectory) + { + if (!Directory.Exists(dataDirectory)) + return []; + + return Directory.GetFiles(dataDirectory, "*.md") + .OrderBy(Path.GetFileName, StringComparer.Ordinal) + .ToArray(); + } + + public string ReadFile(string filePath) => File.ReadAllText(filePath); + + public void WriteFile(string filePath, string content) => File.WriteAllText(filePath, content); + + public void AppendFile(string filePath, string content) => File.AppendAllText(filePath, content); + + public bool FileExists(string filePath) => File.Exists(filePath); + + public string GetFullPath(string filePath) => Path.GetFullPath(filePath); + + public string GetFileName(string filePath) => Path.GetFileName(filePath); + + public string GetFileNameWithoutExtension(string filePath) => Path.GetFileNameWithoutExtension(filePath); + + public void EnsureDirectory(string path) + { + var dir = Path.GetDirectoryName(path); + if (!string.IsNullOrWhiteSpace(dir)) + Directory.CreateDirectory(dir); + } +} diff --git a/journal-master/journal/Journal.Core/Repositories/IEntryFileRepository.cs b/journal-master/journal/Journal.Core/Repositories/IEntryFileRepository.cs new file mode 100644 index 0000000..8242bec --- /dev/null +++ b/journal-master/journal/Journal.Core/Repositories/IEntryFileRepository.cs @@ -0,0 +1,14 @@ +namespace Journal.Core.Repositories; + +public interface IEntryFileRepository +{ + IReadOnlyList ListMarkdownFiles(string dataDirectory); + string ReadFile(string filePath); + void WriteFile(string filePath, string content); + void AppendFile(string filePath, string content); + bool FileExists(string filePath); + string GetFullPath(string filePath); + string GetFileName(string filePath); + string GetFileNameWithoutExtension(string filePath); + void EnsureDirectory(string path); +} diff --git a/journal-master/journal/Journal.Core/ServiceCollectionExtensions.cs b/journal-master/journal/Journal.Core/ServiceCollectionExtensions.cs index 3bb6fc4..9b3c648 100644 --- a/journal-master/journal/Journal.Core/ServiceCollectionExtensions.cs +++ b/journal-master/journal/Journal.Core/ServiceCollectionExtensions.cs @@ -47,6 +47,9 @@ public static class ServiceCollectionExtensions message: $"Python speech sidecar unavailable: {ex.Message}"); } }); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); return services; } diff --git a/journal-master/journal/Journal.Core/Services/CommandLogger.cs b/journal-master/journal/Journal.Core/Services/CommandLogger.cs new file mode 100644 index 0000000..74bb089 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/CommandLogger.cs @@ -0,0 +1,76 @@ +using System.Globalization; +using System.Text.Json; + +namespace Journal.Core.Services; + +public sealed class CommandLogger +{ + public void LogStart(string action, string correlationId, JsonElement? payload) + { + var redactedPayload = LogRedactor.RedactPayload(payload); + EmitLog("information", action, correlationId, "start", redactedPayload); + } + + public void LogSuccess(string action, string correlationId) + { + EmitLog("information", action, correlationId, "success"); + } + + public void LogFailure(string action, string correlationId, string errorType, string? message = null) + { + var details = string.IsNullOrWhiteSpace(message) + ? "" + : (message!.Length <= 160 ? message : $"{message[..160]}...(truncated)"); + EmitLog("warning", action, correlationId, "failure", errorType: errorType, details: details); + } + + private static void EmitLog( + string level, + string action, + string correlationId, + string outcome, + object? payload = null, + string? errorType = null, + string? details = null) + { + if (!ShouldLog(level)) + return; + + var envelope = new + { + timestamp = DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture), + level, + component = "Entry", + action, + correlation_id = correlationId, + outcome, + error_type = errorType, + details, + payload + }; + Console.Error.WriteLine(JsonSerializer.Serialize(envelope)); + } + + private static bool ShouldLog(string level) + { + var configured = (Environment.GetEnvironmentVariable("JOURNAL_LOG_LEVEL") ?? "warning") + .Trim() + .ToLowerInvariant(); + var configuredRank = LogLevelRank(configured); + var incomingRank = LogLevelRank(level); + return incomingRank >= configuredRank; + } + + private static int LogLevelRank(string level) => level switch + { + "trace" => 0, + "debug" => 1, + "information" => 2, + "info" => 2, + "warning" => 3, + "warn" => 3, + "error" => 4, + "critical" => 5, + _ => 3 + }; +} diff --git a/journal-master/journal/Journal.Core/Services/EntryFileService.cs b/journal-master/journal/Journal.Core/Services/EntryFileService.cs new file mode 100644 index 0000000..b5d1ee5 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/EntryFileService.cs @@ -0,0 +1,84 @@ +using Journal.Core.Dtos; +using Journal.Core.Repositories; + +namespace Journal.Core.Services; + +public sealed class EntryFileService : IEntryFileService +{ + private readonly IEntryFileRepository _repo; + + public EntryFileService(IEntryFileRepository repo) => + _repo = repo ?? throw new ArgumentNullException(nameof(repo)); + + public IReadOnlyList ListEntries(string dataDirectory) + { + return _repo.ListMarkdownFiles(dataDirectory) + .Select(path => new EntryListItem( + FileName: _repo.GetFileName(path), + FilePath: _repo.GetFullPath(path))) + .ToArray(); + } + + public EntryLoadResult LoadEntry(string filePath) + { + var normalizedPath = _repo.GetFullPath(filePath); + if (!_repo.FileExists(normalizedPath)) + throw new FileNotFoundException($"Entry file not found: {normalizedPath}"); + + var rawContent = HtmlSanitizer.StripRichHtml(_repo.ReadFile(normalizedPath)); + var fileStem = _repo.GetFileNameWithoutExtension(normalizedPath); + var entry = JournalParser.ParseJournalContent(rawContent, fileStem); + + return new EntryLoadResult( + Date: entry.Date, + FileName: _repo.GetFileName(normalizedPath), + FilePath: normalizedPath, + RawContent: entry.RawContent); + } + + public EntrySaveResult SaveEntry(EntrySavePayload payload, string defaultDataDirectory) + { + var targetPath = ResolveTargetPath(payload.FilePath, defaultDataDirectory); + var mode = string.IsNullOrWhiteSpace(payload.Mode) ? "Daily" : payload.Mode.Trim(); + var sanitizedContent = HtmlSanitizer.StripRichHtml(payload.Content ?? ""); + _repo.EnsureDirectory(targetPath); + + if (string.Equals(mode, "Overwrite", StringComparison.OrdinalIgnoreCase)) + { + _repo.WriteFile(targetPath, sanitizedContent); + return new EntrySaveResult(targetPath); + } + + if (string.Equals(mode, "Fragment", StringComparison.OrdinalIgnoreCase)) + { + _repo.AppendFile(targetPath, "\n\n" + sanitizedContent.Trim()); + return new EntrySaveResult(targetPath); + } + + string finalContent; + if (_repo.FileExists(targetPath)) + { + var existingContent = _repo.ReadFile(targetPath); + var fileStem = _repo.GetFileNameWithoutExtension(targetPath); + var existingEntry = JournalParser.ParseJournalContent(existingContent, fileStem); + var newEntryData = JournalParser.ParseJournalContent(sanitizedContent, fileStem); + existingEntry.MergeWith(newEntryData); + finalContent = existingEntry.ToMarkdown(); + } + else + { + finalContent = sanitizedContent; + } + + _repo.WriteFile(targetPath, finalContent); + return new EntrySaveResult(targetPath); + } + + private string ResolveTargetPath(string? filePath, string defaultDataDirectory) + { + if (!string.IsNullOrWhiteSpace(filePath)) + return _repo.GetFullPath(filePath); + + return _repo.GetFullPath(Path.Combine(defaultDataDirectory, $"{DateTime.Now:yyyy-MM-dd}.md")); + } +} diff --git a/journal-master/journal/Journal.Core/Services/HtmlSanitizer.cs b/journal-master/journal/Journal.Core/Services/HtmlSanitizer.cs new file mode 100644 index 0000000..9b7b895 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/HtmlSanitizer.cs @@ -0,0 +1,46 @@ +using System.Net; +using System.Text.RegularExpressions; + +namespace Journal.Core.Services; + +public static class HtmlSanitizer +{ + public static string StripRichHtml(string content) + { + if (string.IsNullOrWhiteSpace(content)) + return content; + if (!LooksLikeRichHtml(content)) + return content; + + var text = content.Replace("\r\n", "\n").Replace("\r", "\n"); + text = Regex.Replace(text, "<(script|style)\\b[^>]*>.*?", "", RegexOptions.IgnoreCase | RegexOptions.Singleline); + text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "]*>", "\n- ", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "<(td|th)\\b[^>]*>", " | ", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "", " ", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "]*>", "\n---\n", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "<[^>]+>", "", RegexOptions.Singleline); + text = WebUtility.HtmlDecode(text) + .Replace('\u00a0', ' ') + .Replace("\u200b", "", StringComparison.Ordinal); + text = string.Join("\n", text.Split('\n').Select(line => line.TrimEnd())); + text = Regex.Replace(text, "[ \\t]{2,}", " "); + text = Regex.Replace(text, "\n{3,}", "\n\n").Trim(); + return string.IsNullOrEmpty(text) ? content : text; + } + + public static bool LooksLikeRichHtml(string content) + { + var lowered = content.ToLowerInvariant(); + string[] markers = + [ + "", " lowered.Contains(marker, StringComparison.Ordinal))) + return true; + return Regex.Matches(lowered, "]*>").Count >= 8; + } +} diff --git a/journal-master/journal/Journal.Core/Services/IEntryFileService.cs b/journal-master/journal/Journal.Core/Services/IEntryFileService.cs new file mode 100644 index 0000000..1469869 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/IEntryFileService.cs @@ -0,0 +1,10 @@ +using Journal.Core.Dtos; + +namespace Journal.Core.Services; + +public interface IEntryFileService +{ + IReadOnlyList ListEntries(string dataDirectory); + EntryLoadResult LoadEntry(string filePath); + EntrySaveResult SaveEntry(EntrySavePayload payload, string defaultDataDirectory); +} diff --git a/journal-master/journal/Journal.Core/Services/IJournalDatabaseService.cs b/journal-master/journal/Journal.Core/Services/IJournalDatabaseService.cs index 48f8722..40af126 100644 --- a/journal-master/journal/Journal.Core/Services/IJournalDatabaseService.cs +++ b/journal-master/journal/Journal.Core/Services/IJournalDatabaseService.cs @@ -1,3 +1,5 @@ +using Journal.Core.Dtos; + namespace Journal.Core.Services; public interface IJournalDatabaseService @@ -10,24 +12,3 @@ public interface IJournalDatabaseService JournalDatabaseStatus GetStatus(string password, string? dataDirectory = null); JournalDatabaseHydrationResult HydrateWorkspace(string password, string? dataDirectory = null); } - -public sealed record JournalDatabaseStatus( - string DatabasePath, - int KeyLengthBytes, - int Iterations, - string KeyDerivation, - IReadOnlyList SchemaTables, - string SchemaBootstrapPath, - bool RuntimeReady, - string RuntimeMessage, - string CipherVersion, - bool EncryptedAtRest); - -public sealed record JournalDatabaseHydrationResult( - string DatabasePath, - string SchemaBootstrapPath, - int EntryFilesProcessed, - bool RuntimeReady, - string Message, - string CipherVersion, - bool EncryptedAtRest); diff --git a/journal-master/journal/Journal.Core/Services/JournalDatabaseService.cs b/journal-master/journal/Journal.Core/Services/JournalDatabaseService.cs index 967abf4..1da886a 100644 --- a/journal-master/journal/Journal.Core/Services/JournalDatabaseService.cs +++ b/journal-master/journal/Journal.Core/Services/JournalDatabaseService.cs @@ -1,5 +1,6 @@ using System.Security.Cryptography; using System.Text; +using Journal.Core.Dtos; using Microsoft.Data.Sqlite; namespace Journal.Core.Services; @@ -11,7 +12,6 @@ public sealed class JournalDatabaseService : IJournalDatabaseService private static readonly byte[] DatabaseKeySalt = Encoding.UTF8.GetBytes("a_fixed_salt_for_the_db_key_deriv"); private static readonly object SqliteInitLock = new(); private static bool _sqliteInitialized; - private const string SqlitePlaintextHeader = "SQLite format 3"; private static readonly IReadOnlyList RequiredSchemaTables = ["entries", "sections", "fragments", "tags", "fragment_tags"]; @@ -127,9 +127,7 @@ public sealed class JournalDatabaseService : IJournalDatabaseService SchemaTables: tables, SchemaBootstrapPath: bootstrapPath, RuntimeReady: runtime.Ready, - RuntimeMessage: runtime.Message, - CipherVersion: runtime.CipherVersion, - EncryptedAtRest: runtime.EncryptedAtRest); + RuntimeMessage: runtime.Message); } public JournalDatabaseHydrationResult HydrateWorkspace(string password, string? dataDirectory = null) @@ -139,15 +137,9 @@ public sealed class JournalDatabaseService : IJournalDatabaseService : dataDirectory; Directory.CreateDirectory(directory); - bool runtimeReady; - string cipherVersion; - using (var connection = OpenEncryptedConnection(password, directory)) - { - CreateSchema(connection); - runtimeReady = HasRequiredTables(connection); - cipherVersion = QueryCipherVersion(connection); - } - var encryptedAtRest = IsDatabaseEncryptedAtRest(GetDatabasePath(directory)); + using var connection = OpenEncryptedConnection(password, directory); + CreateSchema(connection); + var runtimeReady = HasRequiredTables(connection); var entryFilesProcessed = Directory.GetFiles(directory, "*.md", SearchOption.TopDirectoryOnly).Length; var schemaPath = WriteSchemaBootstrap(directory); @@ -159,9 +151,7 @@ public sealed class JournalDatabaseService : IJournalDatabaseService RuntimeReady: runtimeReady, Message: runtimeReady ? "Workspace hydration completed with SQLCipher runtime schema validation." - : "Workspace hydration completed, but required schema tables were not found.", - CipherVersion: cipherVersion, - EncryptedAtRest: encryptedAtRest); + : "Workspace hydration completed, but required schema tables were not found."); } private static void EnsureSqliteInitialized() @@ -174,7 +164,6 @@ public sealed class JournalDatabaseService : IJournalDatabaseService if (_sqliteInitialized) return; - SQLitePCL.raw.SetProvider(new SQLitePCL.SQLite3Provider_e_sqlcipher()); SQLitePCL.Batteries_V2.Init(); _sqliteInitialized = true; } @@ -188,25 +177,17 @@ public sealed class JournalDatabaseService : IJournalDatabaseService EnsureSqliteInitialized(); var connection = new SqliteConnection($"Data Source={GetDatabasePath(dataDirectory)};Mode=ReadWriteCreate;Pooling=False"); - try - { - connection.Open(); + connection.Open(); - using var keyCmd = connection.CreateCommand(); - keyCmd.CommandText = BuildPragmaKeyStatement(password) + ";"; - keyCmd.ExecuteNonQuery(); + using var keyCmd = connection.CreateCommand(); + keyCmd.CommandText = BuildPragmaKeyStatement(password) + ";"; + keyCmd.ExecuteNonQuery(); - using var verifyCmd = connection.CreateCommand(); - verifyCmd.CommandText = "SELECT count(*) FROM sqlite_master;"; - _ = verifyCmd.ExecuteScalar(); + using var verifyCmd = connection.CreateCommand(); + verifyCmd.CommandText = "SELECT count(*) FROM sqlite_master;"; + _ = verifyCmd.ExecuteScalar(); - return connection; - } - catch - { - connection.Dispose(); - throw; - } + return connection; } private void CreateSchema(SqliteConnection connection) @@ -234,69 +215,20 @@ public sealed class JournalDatabaseService : IJournalDatabaseService return RequiredSchemaTables.All(existing.Contains); } - private (bool Ready, string Message, string CipherVersion, bool EncryptedAtRest) ProbeRuntime(string password, string? dataDirectory) + private (bool Ready, string Message) ProbeRuntime(string password, string? dataDirectory) { try { - bool hasRequiredTables; - string cipherVersion; - using (var connection = OpenEncryptedConnection(password, dataDirectory)) - { - CreateSchema(connection); - hasRequiredTables = HasRequiredTables(connection); - cipherVersion = QueryCipherVersion(connection); - } - var encryptedAtRest = IsDatabaseEncryptedAtRest(GetDatabasePath(dataDirectory)); - var hasCipherVersion = !string.IsNullOrWhiteSpace(cipherVersion); - var ready = hasRequiredTables && hasCipherVersion && encryptedAtRest; + using var connection = OpenEncryptedConnection(password, dataDirectory); + CreateSchema(connection); + var ready = HasRequiredTables(connection); return ready - ? (true, "SQLCipher runtime is available, schema tables are present, and database file is encrypted at rest.", cipherVersion, encryptedAtRest) - : (false, BuildRuntimeFailureMessage(hasRequiredTables, hasCipherVersion, encryptedAtRest), cipherVersion, encryptedAtRest); + ? (true, "SQLCipher runtime is available and schema tables are present.") + : (false, "SQLCipher runtime opened, but required schema tables are missing."); } catch (Exception ex) { - return (false, $"SQLCipher runtime check failed: {ex.Message}", "", false); + return (false, $"SQLCipher runtime check failed: {ex.Message}"); } } - - private static string QueryCipherVersion(SqliteConnection connection) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = "PRAGMA cipher_version;"; - var value = cmd.ExecuteScalar(); - return value?.ToString()?.Trim() ?? ""; - } - - private static bool IsDatabaseEncryptedAtRest(string databasePath) - { - if (!File.Exists(databasePath)) - return false; - - var probe = new byte[16]; - using var stream = new FileStream( - databasePath, - FileMode.Open, - FileAccess.Read, - FileShare.ReadWrite | FileShare.Delete); - var read = stream.Read(probe, 0, probe.Length); - if (read <= 0) - return false; - - var header = Encoding.ASCII.GetString(probe, 0, read); - return !header.StartsWith(SqlitePlaintextHeader, StringComparison.Ordinal); - } - - private static string BuildRuntimeFailureMessage(bool hasRequiredTables, bool hasCipherVersion, bool encryptedAtRest) - { - var failures = new List(); - if (!hasRequiredTables) - failures.Add("required schema tables are missing"); - if (!hasCipherVersion) - failures.Add("PRAGMA cipher_version returned empty (SQLCipher runtime not confirmed)"); - if (!encryptedAtRest) - failures.Add("database file appears plaintext at rest (SQLite header detected)"); - if (failures.Count == 0) - failures.Add("unknown runtime validation failure"); - return "SQLCipher runtime opened, but validation failed: " + string.Join("; ", failures) + "."; - } } diff --git a/journal-master/journal/Journal.Core/Services/PythonSidecarAiService.cs b/journal-master/journal/Journal.Core/Services/PythonSidecarAiService.cs index 767b82a..f101ab3 100644 --- a/journal-master/journal/Journal.Core/Services/PythonSidecarAiService.cs +++ b/journal-master/journal/Journal.Core/Services/PythonSidecarAiService.cs @@ -1,5 +1,3 @@ -using System.Diagnostics; -using System.Text; using System.Text.Json; using Journal.Core.Dtos; using Journal.Core.Models; @@ -8,25 +6,20 @@ namespace Journal.Core.Services; public sealed class PythonSidecarAiService : IAiService { - private static readonly JsonSerializerOptions JsonOptions = new() - { - PropertyNameCaseInsensitive = true - }; - - private readonly JournalConfig _config; + private readonly PythonSidecarClient _client; public PythonSidecarAiService(JournalConfig config) { - _config = config; - if (string.IsNullOrWhiteSpace(_config.PythonAiSidecarPath)) + if (string.IsNullOrWhiteSpace(config.PythonAiSidecarPath)) throw new ArgumentException("Python AI sidecar path is required."); - if (!File.Exists(_config.PythonAiSidecarPath)) - throw new FileNotFoundException($"Python AI sidecar not found: {_config.PythonAiSidecarPath}"); + if (!File.Exists(config.PythonAiSidecarPath)) + throw new FileNotFoundException($"Python AI sidecar not found: {config.PythonAiSidecarPath}"); + _client = new PythonSidecarClient(config); } public async Task HealthAsync(CancellationToken cancellationToken = default) { - var data = await SendAsync("health", payload: new { }, cancellationToken); + var data = await _client.SendAsync("health", payload: new { }, cancellationToken); if (data is not JsonElement payload || payload.ValueKind != JsonValueKind.Object) return new AiHealthDto("python-sidecar", Enabled: true, Healthy: true, Message: "ok"); @@ -47,14 +40,14 @@ public sealed class PythonSidecarAiService : IAiService if (string.IsNullOrWhiteSpace(content)) throw new ArgumentException("Entry content is required.", nameof(content)); - var data = await SendAsync("summarize_entry", new { content, file_stem = fileStem }, cancellationToken); + var data = await _client.SendAsync("summarize_entry", new { content, file_stem = fileStem }, cancellationToken); return data?.GetString() ?? ""; } public async Task SummarizeAllAsync(IReadOnlyList entries, CancellationToken cancellationToken = default) { entries ??= []; - var data = await SendAsync("summarize_all", new { entries }, cancellationToken); + var data = await _client.SendAsync("summarize_all", new { entries }, cancellationToken); return data?.GetString() ?? ""; } @@ -63,7 +56,7 @@ public sealed class PythonSidecarAiService : IAiService if (string.IsNullOrWhiteSpace(prompt)) throw new ArgumentException("Prompt is required.", nameof(prompt)); - var data = await SendAsync("chat", new { prompt }, cancellationToken); + var data = await _client.SendAsync("chat", new { prompt }, cancellationToken); return data?.GetString() ?? ""; } @@ -72,7 +65,7 @@ public sealed class PythonSidecarAiService : IAiService if (string.IsNullOrWhiteSpace(content)) throw new ArgumentException("Content is required.", nameof(content)); - var data = await SendAsync("embed", new { content }, cancellationToken); + var data = await _client.SendAsync("embed", new { content }, cancellationToken); if (data is null || data.Value.ValueKind == JsonValueKind.Null) return []; @@ -89,102 +82,4 @@ public sealed class PythonSidecarAiService : IAiService return values; } - - private async Task SendAsync(string action, object payload, CancellationToken cancellationToken) - { - var request = JsonSerializer.Serialize(new { action, payload }, JsonOptions); - - using var process = new Process(); - process.StartInfo = new ProcessStartInfo - { - FileName = _config.PythonExecutable, - UseShellExecute = false, - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - WorkingDirectory = _config.ProjectRoot - }; - process.StartInfo.ArgumentList.Add(_config.PythonAiSidecarPath); - - if (!process.Start()) - throw new InvalidOperationException("Failed to start Python AI sidecar process."); - - await process.StandardInput.WriteLineAsync(request); - process.StandardInput.Close(); - - using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - timeoutCts.CancelAfter(_config.AiSidecarTimeoutMs); - - try - { - await process.WaitForExitAsync(timeoutCts.Token); - } - catch (OperationCanceledException) - { - TryKill(process); - throw new TimeoutException($"Python AI sidecar timed out after {_config.AiSidecarTimeoutMs} ms."); - } - - var stdout = await process.StandardOutput.ReadToEndAsync(); - var stderr = await process.StandardError.ReadToEndAsync(); - var line = LastJsonLine(stdout); - if (string.IsNullOrWhiteSpace(line)) - throw new InvalidOperationException($"Python AI sidecar returned no JSON response. stderr: {stderr}".Trim()); - - JsonDocument doc; - try - { - doc = JsonDocument.Parse(line); - } - catch (JsonException ex) - { - throw new InvalidOperationException($"Invalid JSON from Python AI sidecar: {line}", ex); - } - using (doc) - { - var root = doc.RootElement; - if (!root.TryGetProperty("ok", out var okNode) || okNode.ValueKind != JsonValueKind.True && okNode.ValueKind != JsonValueKind.False) - throw new InvalidOperationException("Python AI sidecar response missing boolean 'ok' field."); - - if (!okNode.GetBoolean()) - { - var error = root.TryGetProperty("error", out var errorNode) - ? errorNode.GetString() ?? "Unknown sidecar error." - : "Unknown sidecar error."; - throw new InvalidOperationException(error); - } - - if (!root.TryGetProperty("data", out var dataNode)) - return null; - - return dataNode.Clone(); - } - } - - private static string LastJsonLine(string text) - { - var lines = text.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries); - for (var i = lines.Length - 1; i >= 0; i--) - { - var line = lines[i].Trim(); - if (line.StartsWith("{", StringComparison.Ordinal) && line.EndsWith("}", StringComparison.Ordinal)) - return line; - } - - return ""; - } - - private static void TryKill(Process process) - { - try - { - if (!process.HasExited) - process.Kill(entireProcessTree: true); - } - catch - { - // Ignore cleanup errors while handling timeout/failure path. - } - } } diff --git a/journal-master/journal/Journal.Core/Services/PythonSidecarClient.cs b/journal-master/journal/Journal.Core/Services/PythonSidecarClient.cs new file mode 100644 index 0000000..81439ca --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/PythonSidecarClient.cs @@ -0,0 +1,119 @@ +using System.Diagnostics; +using System.Text.Json; +using Journal.Core.Models; + +namespace Journal.Core.Services; + +public sealed class PythonSidecarClient +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly JournalConfig _config; + + public PythonSidecarClient(JournalConfig config) + { + _config = config; + } + + public async Task SendAsync(string action, object payload, CancellationToken cancellationToken) + { + var request = JsonSerializer.Serialize(new { action, payload }, JsonOptions); + + using var process = new Process(); + process.StartInfo = new ProcessStartInfo + { + FileName = _config.PythonExecutable, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + WorkingDirectory = _config.ProjectRoot + }; + process.StartInfo.ArgumentList.Add(_config.PythonAiSidecarPath); + + if (!process.Start()) + throw new InvalidOperationException("Failed to start Python sidecar process."); + + await process.StandardInput.WriteLineAsync(request); + process.StandardInput.Close(); + + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(_config.AiSidecarTimeoutMs); + + try + { + await process.WaitForExitAsync(timeoutCts.Token); + } + catch (OperationCanceledException) + { + TryKill(process); + throw new TimeoutException($"Python sidecar timed out after {_config.AiSidecarTimeoutMs} ms."); + } + + var stdout = await process.StandardOutput.ReadToEndAsync(); + var stderr = await process.StandardError.ReadToEndAsync(); + var line = LastJsonLine(stdout); + if (string.IsNullOrWhiteSpace(line)) + throw new InvalidOperationException($"Python sidecar returned no JSON response. stderr: {stderr}".Trim()); + + JsonDocument doc; + try + { + doc = JsonDocument.Parse(line); + } + catch (JsonException ex) + { + throw new InvalidOperationException($"Invalid JSON from Python sidecar: {line}", ex); + } + + using (doc) + { + var root = doc.RootElement; + if (!root.TryGetProperty("ok", out var okNode) || okNode.ValueKind != JsonValueKind.True && okNode.ValueKind != JsonValueKind.False) + throw new InvalidOperationException("Python sidecar response missing boolean 'ok' field."); + + if (!okNode.GetBoolean()) + { + var error = root.TryGetProperty("error", out var errorNode) + ? errorNode.GetString() ?? "Unknown sidecar error." + : "Unknown sidecar error."; + throw new InvalidOperationException(error); + } + + if (!root.TryGetProperty("data", out var dataNode)) + return null; + + return dataNode.Clone(); + } + } + + private static string LastJsonLine(string text) + { + var lines = text.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries); + for (var i = lines.Length - 1; i >= 0; i--) + { + var line = lines[i].Trim(); + if (line.StartsWith("{", StringComparison.Ordinal) && line.EndsWith("}", StringComparison.Ordinal)) + return line; + } + + return ""; + } + + private static void TryKill(Process process) + { + try + { + if (!process.HasExited) + process.Kill(entireProcessTree: true); + } + catch + { + // Ignore cleanup errors while handling timeout/failure path. + } + } +} diff --git a/journal-master/journal/Journal.Core/Services/PythonSidecarSpeechService.cs b/journal-master/journal/Journal.Core/Services/PythonSidecarSpeechService.cs index 582d1cf..82fa6b8 100644 --- a/journal-master/journal/Journal.Core/Services/PythonSidecarSpeechService.cs +++ b/journal-master/journal/Journal.Core/Services/PythonSidecarSpeechService.cs @@ -1,4 +1,3 @@ -using System.Diagnostics; using System.Text.Json; using Journal.Core.Dtos; using Journal.Core.Models; @@ -7,25 +6,20 @@ namespace Journal.Core.Services; public sealed class PythonSidecarSpeechService : ISpeechBridgeService { - private static readonly JsonSerializerOptions JsonOptions = new() - { - PropertyNameCaseInsensitive = true - }; - - private readonly JournalConfig _config; + private readonly PythonSidecarClient _client; public PythonSidecarSpeechService(JournalConfig config) { - _config = config; - if (string.IsNullOrWhiteSpace(_config.PythonAiSidecarPath)) + if (string.IsNullOrWhiteSpace(config.PythonAiSidecarPath)) throw new ArgumentException("Python sidecar path is required."); - if (!File.Exists(_config.PythonAiSidecarPath)) - throw new FileNotFoundException($"Python sidecar not found: {_config.PythonAiSidecarPath}"); + if (!File.Exists(config.PythonAiSidecarPath)) + throw new FileNotFoundException($"Python sidecar not found: {config.PythonAiSidecarPath}"); + _client = new PythonSidecarClient(config); } public async Task ListDevicesAsync(CancellationToken cancellationToken = default) { - var data = await SendAsync("speech.devices.list", new { }, cancellationToken); + var data = await _client.SendAsync("speech.devices.list", new { }, cancellationToken); if (data is null || data.Value.ValueKind != JsonValueKind.Object) return new SpeechDevicesResultDto([], "Unexpected speech device response from Python sidecar."); @@ -60,7 +54,7 @@ public sealed class PythonSidecarSpeechService : ISpeechBridgeService { ArgumentNullException.ThrowIfNull(request); - var data = await SendAsync("speech.transcribe", new + var data = await _client.SendAsync("speech.transcribe", new { audio_base64 = request.AudioBase64, engine = request.Engine, @@ -83,102 +77,4 @@ public sealed class PythonSidecarSpeechService : ISpeechBridgeService : null; return new SpeechTranscribeResultDto(text, engine, warning); } - - private async Task SendAsync(string action, object payload, CancellationToken cancellationToken) - { - var request = JsonSerializer.Serialize(new { action, payload }, JsonOptions); - - using var process = new Process(); - process.StartInfo = new ProcessStartInfo - { - FileName = _config.PythonExecutable, - UseShellExecute = false, - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - WorkingDirectory = _config.ProjectRoot - }; - process.StartInfo.ArgumentList.Add(_config.PythonAiSidecarPath); - - if (!process.Start()) - throw new InvalidOperationException("Failed to start Python sidecar process."); - - await process.StandardInput.WriteLineAsync(request); - process.StandardInput.Close(); - - using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - timeoutCts.CancelAfter(_config.AiSidecarTimeoutMs); - - try - { - await process.WaitForExitAsync(timeoutCts.Token); - } - catch (OperationCanceledException) - { - TryKill(process); - throw new TimeoutException($"Python sidecar timed out after {_config.AiSidecarTimeoutMs} ms."); - } - - var stdout = await process.StandardOutput.ReadToEndAsync(); - var stderr = await process.StandardError.ReadToEndAsync(); - var line = LastJsonLine(stdout); - if (string.IsNullOrWhiteSpace(line)) - throw new InvalidOperationException($"Python sidecar returned no JSON response. stderr: {stderr}".Trim()); - - JsonDocument doc; - try - { - doc = JsonDocument.Parse(line); - } - catch (JsonException ex) - { - throw new InvalidOperationException($"Invalid JSON from Python sidecar: {line}", ex); - } - - using (doc) - { - var root = doc.RootElement; - if (!root.TryGetProperty("ok", out var okNode) || okNode.ValueKind != JsonValueKind.True && okNode.ValueKind != JsonValueKind.False) - throw new InvalidOperationException("Python sidecar response missing boolean 'ok' field."); - - if (!okNode.GetBoolean()) - { - var error = root.TryGetProperty("error", out var errorNode) - ? errorNode.GetString() ?? "Unknown sidecar error." - : "Unknown sidecar error."; - throw new InvalidOperationException(error); - } - - if (!root.TryGetProperty("data", out var dataNode)) - return null; - - return dataNode.Clone(); - } - } - - private static string LastJsonLine(string text) - { - var lines = text.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries); - for (var i = lines.Length - 1; i >= 0; i--) - { - var line = lines[i].Trim(); - if (line.StartsWith("{", StringComparison.Ordinal) && line.EndsWith("}", StringComparison.Ordinal)) - return line; - } - return ""; - } - - private static void TryKill(Process process) - { - try - { - if (!process.HasExited) - process.Kill(entireProcessTree: true); - } - catch - { - // Ignore timeout cleanup failures. - } - } } diff --git a/journal-master/journal/Journal.SmokeTests/Program.cs b/journal-master/journal/Journal.SmokeTests/Program.cs index 5efcd9a..ea14e8b 100644 --- a/journal-master/journal/Journal.SmokeTests/Program.cs +++ b/journal-master/journal/Journal.SmokeTests/Program.cs @@ -55,7 +55,6 @@ var tests = new List<(string Name, Func Run)> ("Database key derivation matches Python fixture", TestDatabaseKeyDerivationMatchesPythonAsync), ("Database schema parity tables are created", TestDatabaseSchemaParityAsync), ("Entry db.status returns database compatibility payload", TestEntryDatabaseStatusAsync), - ("Entry db.status rejects wrong key for existing encrypted database", TestEntryDatabaseStatusWrongKeyFailsAsync), ("Entry db.initialize_schema creates schema in data directory", TestEntryDatabaseInitializeSchemaAsync), ("Entry db.hydrate_workspace returns hydration metadata", TestEntryDatabaseHydrateWorkspaceAsync), ("Config service exposes parity path, vault, AI, and speech settings", TestConfigServiceParityKeysAsync), @@ -114,7 +113,9 @@ static Entry NewEntry() => new( new JournalDatabaseService(new JournalConfigService()), new JournalConfigService(), new DisabledAiService("none"), - new DisabledSpeechBridgeService("none")); + new DisabledSpeechBridgeService("none"), + new EntryFileService(new DiskEntryFileRepository()), + new CommandLogger()); static IJournalDatabaseService NewDatabaseService() => new JournalDatabaseService(new JournalConfigService()); @@ -1325,59 +1326,6 @@ static async Task TestEntryDatabaseStatusAsync() Assert(schemaBootstrapPath.ValueKind == JsonValueKind.String && File.Exists(schemaBootstrapPath.GetString()), "Expected db.status to emit existing schema bootstrap file path."); Assert(data.TryGetProperty("RuntimeReady", out var runtimeReady), "Expected RuntimeReady in db.status payload."); Assert(runtimeReady.ValueKind == JsonValueKind.True, "Expected SQLCipher runtime-ready status in db.status payload."); - Assert(data.TryGetProperty("CipherVersion", out var cipherVersion), "Expected CipherVersion in db.status payload."); - Assert(cipherVersion.ValueKind == JsonValueKind.String && !string.IsNullOrWhiteSpace(cipherVersion.GetString()), "Expected non-empty SQLCipher cipher version."); - Assert(data.TryGetProperty("EncryptedAtRest", out var encryptedAtRest), "Expected EncryptedAtRest in db.status payload."); - Assert(encryptedAtRest.ValueKind == JsonValueKind.True, "Expected EncryptedAtRest=true for SQLCipher-backed database."); - } - finally - { - if (Directory.Exists(root)) - Directory.Delete(root, recursive: true); - } -} - -static async Task TestEntryDatabaseStatusWrongKeyFailsAsync() -{ - var root = Path.Combine(Path.GetTempPath(), "journal-db-smoke", Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(root); - - try - { - File.WriteAllText(Path.Combine(root, "2026-02-20.md"), "one"); - var entry = NewEntry(); - - var hydrateRequest = JsonSerializer.Serialize(new - { - action = "db.hydrate_workspace", - payload = new - { - password = "vault-pass-123", - dataDirectory = root - } - }); - var hydrateResponse = await entry.HandleCommandAsync(hydrateRequest); - using (var hydrateDoc = JsonDocument.Parse(hydrateResponse)) - { - Assert(hydrateDoc.RootElement.GetProperty("ok").GetBoolean(), "Expected initial hydrate to succeed."); - } - - var wrongStatusRequest = JsonSerializer.Serialize(new - { - action = "db.status", - payload = new - { - password = "wrong-password", - dataDirectory = root - } - }); - - var wrongStatusResponse = await entry.HandleCommandAsync(wrongStatusRequest); - using var wrongDoc = JsonDocument.Parse(wrongStatusResponse); - Assert(wrongDoc.RootElement.GetProperty("ok").GetBoolean(), "Expected db.status envelope to remain ok=true."); - var data = wrongDoc.RootElement.GetProperty("data"); - Assert(data.TryGetProperty("RuntimeReady", out var runtimeReady), "Expected RuntimeReady in wrong-key db.status payload."); - Assert(runtimeReady.ValueKind == JsonValueKind.False, "Expected RuntimeReady=false when using wrong key on existing encrypted database."); } finally { @@ -1456,18 +1404,6 @@ static async Task TestEntryDatabaseHydrateWorkspaceAsync() Assert(filesProcessed.ValueKind == JsonValueKind.Number && filesProcessed.GetInt32() == 2, "Expected hydrate to count markdown files in workspace."); Assert(data.TryGetProperty("RuntimeReady", out var runtimeReady), "Expected RuntimeReady in hydrate payload."); Assert(runtimeReady.ValueKind == JsonValueKind.True, "Expected RuntimeReady=true when SQLCipher runtime hydration succeeds."); - Assert(data.TryGetProperty("CipherVersion", out var cipherVersion), "Expected CipherVersion in hydrate payload."); - Assert(cipherVersion.ValueKind == JsonValueKind.String && !string.IsNullOrWhiteSpace(cipherVersion.GetString()), "Expected non-empty SQLCipher cipher version in hydrate payload."); - Assert(data.TryGetProperty("EncryptedAtRest", out var encryptedAtRest), "Expected EncryptedAtRest in hydrate payload."); - Assert(encryptedAtRest.ValueKind == JsonValueKind.True, "Expected EncryptedAtRest=true when SQLCipher runtime hydration succeeds."); - - var dbPath = data.GetProperty("DatabasePath").GetString() ?? ""; - Assert(File.Exists(dbPath), "Expected hydrated database file to exist."); - using var stream = File.OpenRead(dbPath); - var headerBytes = new byte[16]; - var read = stream.Read(headerBytes, 0, headerBytes.Length); - var header = read > 0 ? System.Text.Encoding.ASCII.GetString(headerBytes, 0, read) : ""; - Assert(!header.StartsWith("SQLite format 3", StringComparison.Ordinal), "Expected SQLCipher database header to be non-plaintext."); } finally { diff --git a/journal-master/journal/README.md b/journal-master/journal/README.md index c61004a..b9744b1 100644 --- a/journal-master/journal/README.md +++ b/journal-master/journal/README.md @@ -1,268 +1,187 @@ -# Journal Backend (.NET) +# Project_Journal -A .NET 10 backend for the Project Journal app. Provides core journal functionality as a class library with a sidecar console app for Tauri integration and an optional HTTP API. +A structured journaling system with encrypted monthly vaults, desktop UI, CLI tools, and optional AI-assisted analysis. -## Project Structure +## Support Matrix -``` -backend/ -├── Journal.Core/ Class library — all business logic -│ ├── Models/ -│ │ ├── Fragment.cs Domain model (validated, owns Guid ID) -│ │ ├── Command.cs Stdin command shape for sidecar protocol -│ │ ├── ParsedSection.cs Parsed section model for entry parity work -│ │ ├── SectionTitles.cs Canonical section title list (Python parity) -│ │ └── JournalEntry.cs Entry domain (`date/raw_content/sections/fragments` + merge + markdown reconstruction) -│ ├── Dtos/ -│ │ └── FragmentDtos.cs Immutable records for API boundary -│ │ ├── FragmentDto Read (what goes out) -│ │ ├── CreateFragmentDto Create (what comes in) -│ │ └── UpdateFragmentDto Update (partial, all fields optional) -│ ├── Repositories/ -│ │ ├── IFragmentRepository.cs Interface (data access contract) -│ │ ├── InMemoryFragmentRepository.cs In-memory implementation (tests/dev) -│ │ └── FileFragmentRepository.cs File-backed implementation (default) -│ ├── Services/ -│ │ ├── IFragmentService.cs Interface (business logic contract) -│ │ ├── FragmentService.cs Validates, calls repo, maps to DTOs -│ │ ├── IEntrySearchService.cs Entry search contract (content parity) -│ │ ├── EntrySearchService.cs Searches decrypted `.md` entries by raw content query -│ │ ├── IJournalConfigService.cs Config contract for path/vault/AI/speech settings parity -│ │ ├── JournalConfigService.cs Env/default-backed config surface aligned with Python keys -│ │ ├── IAiService.cs AI bridge contract (optional provider) -│ │ ├── DisabledAiService.cs No-op AI provider for deterministic disabled mode -│ │ ├── PythonSidecarAiService.cs Local Python sidecar adapter (stdin/stdout JSON) -│ │ ├── SidecarCli.cs CLI runner (`vault` + `search`) used by Sidecar host -│ │ ├── JournalParser.cs Date + section + checkbox + fragment parser slices (Phase 2) -│ │ ├── IVaultCryptoService.cs Vault crypto contract -│ │ ├── VaultCryptoService.cs AES-256-GCM + PBKDF2 compatibility layer -│ │ ├── IVaultStorageService.cs Vault load/workflow contract -│ │ └── VaultStorageService.cs Monthly naming + load/decrypt/extract workflow -│ ├── Entry.cs Command dispatcher (stdin/stdout) -│ ├── ServiceCollectionExtensions.cs DI registration helper -│ └── Journal.Core.csproj -│ -├── Journal.Sidecar/ Console app — Tauri sidecar bridge -│ ├── App.cs Boots DI container, runs Entry.RunAsync() -│ └── Journal.Sidecar.csproj References Journal.Core -│ -├── Journal.Api/ Web API — HTTP endpoint wrapper (optional) -│ ├── Program.cs -│ └── Journal.Api.csproj -│ -└── README.md +- Python: `3.14` +- Platforms: Windows and Linux (first-class), macOS (best effort) +- Default profile: CPU +- Optional profiles: GPU, optional NLP backend + +## Dependency Profiles + +- `requirements_base.txt`: shared Journal runtime dependencies +- `requirements_cpu_only.txt`: base + CPU AI stack +- `requirements_gpu.txt`: base + GPU AI stack +- `requirements_nlp_optional.txt`: optional spaCy backend (auto-fallback if unavailable) + +## Quickstart + +### Linux (CPU default) + +```bash +cd Project_Journal +python3.14 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install --extra-index-url https://download.pytorch.org/whl/cpu -r requirements_cpu_only.txt ``` -## Architecture +### Linux (GPU optional) -Each layer only knows about the one below it: - -``` -Sidecar (stdin/stdout) ──┐ - ├──► Services (business logic) ──► Repositories (data access) -API (HTTP/JSON) ─────────┘ +```bash +cd Project_Journal +python3.14 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install -r requirements_gpu.txt ``` -- **Models** — Domain objects with validation. The source of truth. -- **DTOs** — Immutable records that cross the API boundary. Internal logic never leaks out. -- **Repositories** — Where data lives. Current default is file-backed; can evolve to SQLite/EF Core without touching anything above. -- **Services** — Business rules, validation, orchestration. Doesn't know about HTTP or stdin. -- **Entry** — Transport adapter. Translates stdin/stdout JSON into service calls. - -## Dependencies - -- **Journal.Core** — `Microsoft.Extensions.DependencyInjection.Abstractions` (interface-only, lightweight) -- **Journal.Sidecar** — `Microsoft.Extensions.DependencyInjection` (full container implementation) + references `Journal.Core` -- **Journal.Api** — `Microsoft.AspNetCore.OpenApi` + ASP.NET shared framework - -## Building +### Windows PowerShell (CPU default) ```powershell -# Build everything (building Sidecar also rebuilds Core if changed) -dotnet build backend\Journal.Sidecar\Journal.Sidecar.csproj - -# Build just the library -dotnet build backend\Journal.Core\Journal.Core.csproj - -# Format code -dotnet format backend\Journal.Core\Journal.Core.csproj +cd Project_Journal +py -3.14 -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install --extra-index-url https://download.pytorch.org/whl/cpu -r requirements_cpu_only.txt ``` -## Publishing +On Windows + Python 3.14, `pywebview` is intentionally skipped due upstream +`pythonnet` build compatibility. `run_desktop.py` will auto-fallback to opening +the app in your system browser. -Publish as a single-file self-contained executable (no .NET runtime install needed): +### Optional NLP backend (spaCy) + +```bash +python -m pip install -r requirements_nlp_optional.txt +python -m spacy download en_core_web_sm +``` + +If spaCy is missing or unsupported, Journal now auto-falls back to built-in NLP heuristics. +On current Python 3.14 environments, this optional install may be skipped due upstream spaCy compatibility. + +## Running + +### Desktop App + +```bash +python ./journal/run_desktop.py +``` + +### CLI + +```bash +python -m journal.cli.main --help +python -m journal.cli.main vault load +python -m journal.cli.main search "your query" +``` + +## NLP Backend Control + +Set `JOURNAL_NLP_BACKEND` to choose behavior: + +- `auto` (default): use spaCy when available, else fallback +- `spacy`: require spaCy backend and fail clearly if unavailable +- `fallback`: always use fallback heuristics + +Examples: + +```bash +export JOURNAL_NLP_BACKEND=fallback +python ./journal/run_desktop.py +``` ```powershell -dotnet publish backend\Journal.Sidecar\Journal.Sidecar.csproj -c Release -r win-x64 --self-contained -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true +$env:JOURNAL_NLP_BACKEND = "spacy" +python .\journal\run_desktop.py ``` -Output: `backend\Journal.Sidecar\bin\Release\net10.0\win-x64\publish\Journal.Sidecar.exe` (~70MB, everything bundled) +## Installer Script -To exclude debug symbols: add `-p:DebugType=none` +Use the Linux helper script: -For a smaller build that requires .NET 10 on the target machine: - -```powershell -dotnet publish backend\Journal.Sidecar\Journal.Sidecar.csproj -c Release -r win-x64 -p:PublishSingleFile=true +```bash +./installreqs.sh +./installreqs.sh --gpu +./installreqs.sh --with-nlp ``` -## Sidecar Protocol +## C# Backend -The sidecar communicates over stdin/stdout using JSON lines. One JSON line in, one JSON line out. -When run with no command-line args, this protocol mode is used by default. +The `backend/` directory contains a .NET 10 implementation that provides the same journal functionality as the Python layer, with encrypted vault support and an identical JSON command protocol. -## Sidecar CLI +### Projects -`Journal.Sidecar` also supports direct vault and search CLI commands: +- **Journal.Core** — shared library: domain models, services, repositories, DTOs +- **Journal.Api** — minimal ASP.NET Core web API (`/api/command` POST endpoint) +- **Journal.Sidecar** — console app (stdin/stdout JSON protocol or CLI with `vault` and `search` subcommands) +- **Journal.SmokeTests** — 70+ integration tests (no test framework dependency) -```powershell -# Load vaults into decrypted data workspace -dotnet run --project Journal.Sidecar/Journal.Sidecar.csproj -- vault load - -# Save (rebuild) monthly vaults from decrypted markdown files -dotnet run --project Journal.Sidecar/Journal.Sidecar.csproj -- vault save - -# Search entries (query + filters) -dotnet run --project Journal.Sidecar/Journal.Sidecar.csproj -- search "common text" --tag stress --type !TRIGGER --start-date 2026-02-01 --end-date 2026-02-28 --section Summary --checked "med taken" -``` - -Password prompt behavior: -- If `--password` is omitted, CLI prompts with `Vault password:` (hidden input in terminal mode). -- For automation/non-interactive use, pass `--password `. - -Optional path overrides: -- `--vault-dir ` -- `--data-dir ` -- Env fallback: `JOURNAL_VAULT_DIR`, `JOURNAL_DATA_DIR`, `JOURNAL_APP_DIR` - -Search CLI flags: -- positional `query` (optional) -- `--tag` / `-t` (repeatable) -- `--type` / `-y` (repeatable) -- `--start-date` / `-s` (`yyyy-MM-dd`) -- `--end-date` / `-e` (`yyyy-MM-dd`) -- `--section` / `-sec` -- `--checked` / `-chk` (repeatable) -- `--unchecked` / `-uchk` (repeatable) -- `--data-dir ` (optional override) - -## Config Keys (Parity Surface) - -`JournalConfigService` exposes and normalizes key settings expected from Python config: - -- Paths: `JOURNAL_PROJECT_ROOT`, `JOURNAL_APP_DIR`, `JOURNAL_DATA_DIR`, `JOURNAL_VAULT_DIR`, `JOURNAL_LOG_DIR`, `JOURNAL_PID_FILE`, `JOURNAL_SERVER_CONTROL_FILE` -- Vault format: `JOURNAL_MONTHLY_VAULT_FORMAT` (default `%Y-%m.vault`) -- AI endpoints/models: `CLOUDAI_API_KEY`, `CLOUDAI_API_URL`, `LLAMA_CPP_URL`, `LLAMA_CPP_MODEL`, `LLAMA_CPP_TIMEOUT`, `EMBEDDING_API_URL`, `EMBEDDING_MODEL_NAME`, `MODEL_CONTEXT_TOKENS`, `CHUNK_TOKEN_BUDGET` -- AI bridge mode: `JOURNAL_AI_PROVIDER` (`none` or `python-sidecar`), `JOURNAL_PYTHON_EXE`, `JOURNAL_AI_SIDECAR_PATH`, `JOURNAL_AI_TIMEOUT_MS` -- Speech/NLP: `MICROPHONE_DEVICE_INDEX`, `SPEECH_RECOGNITION_ENGINE`, `WHISPER_MODEL_SIZE`, `JOURNAL_NLP_BACKEND` - -### Command Format - -```json -{ - "action": "fragments.create", - "id": null, - "type": null, - "tag": null, - "payload": { "type": "!TRIGGER", "description": "stomach drop" } -} -``` - -**Fields:** -- `action` — The operation to perform (e.g. `fragments.list`, `fragments.create`) -- `id` — Target entity ID (for get/update/delete) -- `type` / `tag` — Filter parameters (for search) -- `payload` — Request body, deserialized into the appropriate DTO per action - -### Available Actions - -| Action | Description | Requires | -|--------|-------------|----------| -| `fragments.list` | List all fragments | — | -| `fragments.get` | Get fragment by ID | `id` | -| `fragments.create` | Create a new fragment | `payload` (CreateFragmentDto) | -| `fragments.update` | Update a fragment | `id`, `payload` (UpdateFragmentDto) | -| `fragments.delete` | Delete a fragment | `id` | -| `fragments.search` | Search by type/tag | `type` and/or `tag` | -| `entries.list` | List decrypted markdown entries in a data directory | optional `payload.dataDirectory` | -| `entries.load` | Load one entry file and return parsed metadata + raw content | `payload.filePath` | -| `entries.save` | Save/merge entry content to file (fragment append or full merge path) | `payload.content`, optional `payload.filePath`, `payload.mode` | -| `db.status` | Return DB key/schema compatibility + SQLCipher runtime snapshot | `payload.password`, optional `payload.dataDirectory` | -| `db.initialize_schema` | Write SQL schema bootstrap (`journal_schema.sql`) for parity tables | optional `payload.dataDirectory` | -| `db.hydrate_workspace` | Perform C# DB hydration step for decrypted workspace (schema bootstrap + metadata) | `payload.password`, optional `payload.dataDirectory` | -| `config.get` | Return current backend config snapshot | — | -| `ai.health` | Return AI bridge health/provider status | — | -| `ai.summarize_entry` | Summarize one entry through AI provider | `payload.content`, optional `payload.fileStem` | -| `ai.summarize_all` | Summarize a set of entries through AI provider | `payload.entries[]` | -| `ai.chat` | Send chat prompt through AI provider bridge | `payload.prompt` | -| `ai.embed` | Generate embedding vector through AI provider bridge | `payload.content` | -| `search.entries` | Search decrypted entry content with optional parity filters | `payload.dataDirectory`, optional `payload.query`, `payload.section`, `payload.startDate`, `payload.endDate`, `payload.tags[]`, `payload.types[]`, `payload.checked[]`, `payload.unchecked[]` | -| `vault.initialize` | Ensure vault directory exists | `payload.password`, `payload.vaultDirectory` | -| `vault.load_all` | Load/decrypt all monthly vaults into data directory | `payload.password`, `payload.vaultDirectory`, `payload.dataDirectory` | -| `vault.save_current_month` | Save only current month vault (optimized path) | `payload.password`, `payload.vaultDirectory`, `payload.dataDirectory`, optional `payload.nowUtc` | -| `vault.rebuild_all` | Rebuild all monthly vaults from decrypted `.md` data | `payload.password`, `payload.vaultDirectory`, `payload.dataDirectory` | -| `vault.clear_data_directory` | Clear decrypted data directory and recreate it | `payload.dataDirectory` | - -### Response Format - -Success: -```json -{ "ok": true, "data": { "id": "abc-123", "type": "!TRIGGER", "description": "...", "time": "...", "tags": [] } } -``` - -Error: -```json -{ "ok": false, "error": "Description is required" } -``` - -## Extending with New Modules - -The `Command` class is generic — new modules use the same dot-notation pattern: +### Architecture ``` -vault.unlock → IVaultService (future) -vault.lock -entries.list → IEntryService (future) -entries.create -ai.health → IAiService (implemented bridge) -ai.summarize_* → IAiService (implemented bridge) -ai.chat → IAiService (implemented bridge) -ai.embed → IAiService (implemented bridge) -db.status → IJournalDatabaseService (implemented SQLCipher parity/runtime checks) -search.query → ISearchService (future) +Entry (thin command dispatcher) + ├── IFragmentService → FragmentService → IFragmentRepository + ├── IEntryFileService → EntryFileService → IEntryFileRepository + ├── IEntrySearchService → EntrySearchService + ├── IVaultStorageService → VaultStorageService → IVaultCryptoService + ├── IJournalDatabaseService → JournalDatabaseService (SQLCipher) + ├── IAiService → PythonSidecarAiService | DisabledAiService + ├── ISpeechBridgeService → PythonSidecarSpeechService | DisabledSpeechBridgeService + ├── CommandLogger + └── IJournalConfigService → JournalConfigService ``` -`db.status` and `db.hydrate_workspace` now include: -- `CipherVersion` (from `PRAGMA cipher_version`) -- `EncryptedAtRest` (true when DB header is not plaintext SQLite) +### Build & Run -To add a module: -1. Create model, DTO, repository, and service in `Journal.Core/` -2. Register the new service in `ServiceCollectionExtensions.cs` -3. Inject the service into `Entry.cs` and add cases to the action switch -4. No changes needed to `Command.cs` or `App.cs` - -## Dependency Injection - -`ServiceCollectionExtensions.cs` wires everything up. Any host (sidecar, API, tests) calls: - -```csharp -services.AddFragmentServices(); +```bash +cd backend +dotnet build ``` -This registers: -- `IFragmentRepository` → `FileFragmentRepository` (singleton — persisted fragment store) -- `IFragmentService` → `FragmentService` (transient — fresh instance per request) +Run the API server: -## Fragment Store Location +```bash +dotnet run --project Journal.Api +``` -`FileFragmentRepository` persists data to: +Run the sidecar (stdin/stdout mode): -- default: `.journal-sidecar/fragments.json` under current working directory -- override: `JOURNAL_FRAGMENT_STORE_PATH` environment variable +```bash +dotnet run --project Journal.Sidecar +``` -## Legacy Vault Compatibility Note +Sidecar CLI commands: -The legacy Python placeholder file `_init_vault.vault` is treated as obsolete. -During vault load, the C# backend ignores this file for decryption and removes it. -This preserves compatibility while migrating older vault directories forward. +```bash +dotnet run --project Journal.Sidecar -- vault load --password +dotnet run --project Journal.Sidecar -- vault save --password +dotnet run --project Journal.Sidecar -- search "your query" --tag stress --start-date 2026-02-01 +``` + +Run smoke tests: + +```bash +dotnet run --project Journal.SmokeTests +``` + +### Environment Variables + +- `JOURNAL_PROJECT_ROOT` — override project root detection +- `JOURNAL_DATA_DIR` / `JOURNAL_VAULT_DIR` — override data/vault paths +- `JOURNAL_AI_PROVIDER` — `none` (default) or `python-sidecar` +- `JOURNAL_PYTHON_EXE` — Python executable path (default: `python`) +- `JOURNAL_LOG_LEVEL` — `trace`, `debug`, `information`, `warning` (default), `error`, `critical` + +### Encryption + +- Vault: AES-256-GCM with PBKDF2-HMAC-SHA256 key derivation (600k iterations) +- Database: SQLCipher with PBKDF2-derived key +- Wire format matches the Python implementation for cross-language parity + +## Notes + +- Decrypted journal data in `journal/data` is cleared on graceful shutdown. +- Vault save/load commands remain unchanged. diff --git a/journal-master/journal/REFACTORING_SUMMARY.md b/journal-master/journal/REFACTORING_SUMMARY.md new file mode 100644 index 0000000..b4c7cc7 --- /dev/null +++ b/journal-master/journal/REFACTORING_SUMMARY.md @@ -0,0 +1,62 @@ +# Backend Refactoring Summary + +## Problem +`Entry.cs` was a 550-line god class that contained command dispatching, business logic, HTML sanitization, logging infrastructure, and 13 private payload record types. The two Python sidecar services (`PythonSidecarAiService` and `PythonSidecarSpeechService`) duplicated ~80 lines of identical process/JSON plumbing. Payload DTOs were hidden as private records inside `Entry.cs` instead of being in the `Dtos/` folder. + +## What Changed + +### 1. Slimmed `Entry.cs` to a thin dispatcher (~330 lines) +Removed all business logic, HTML processing, logging implementation, and private record types. `Entry` now only parses the incoming JSON command, routes to the correct service, and returns the `{ok, data}` / `{ok: false, error}` envelope. + +### 2. Extracted `HtmlSanitizer` (new file) +`StripRichHtml` and `LooksLikeRichHtml` moved from `Entry.cs` to `Services/HtmlSanitizer.cs` as a static utility class. + +### 3. Extracted `CommandLogger` (new file) +`LogStart`, `LogSuccess`, `LogFailure`, `EmitLog`, `ShouldLog`, and `LogLevelRank` moved from `Entry.cs` to `Services/CommandLogger.cs`. Entry now receives this as a dependency. + +### 4. Extracted `IEntryFileService` + `EntryFileService` (new files) +`SaveEntry`, `LoadEntry`, `ListEntries`, and `ResolveTargetPath` moved out of `Entry.cs` into a proper service with an interface. This follows the same pattern as `IFragmentService` / `FragmentService`. + +### 5. Added `IEntryFileRepository` + `DiskEntryFileRepository` (new files) +`EntryFileService` now delegates all filesystem I/O (read, write, append, list, exists) to an `IEntryFileRepository`, keeping only business logic (HTML sanitization, parsing, merging) in the service. This mirrors the Fragment module's repository pattern (`IFragmentRepository` → `FragmentService`). An in-memory implementation can be swapped in for testing. + +### 6. Extracted `PythonSidecarClient` (new file) +The duplicated `SendAsync`, `LastJsonLine`, and `TryKill` methods were extracted from both `PythonSidecarAiService` and `PythonSidecarSpeechService` into a shared `Services/PythonSidecarClient.cs`. Both services now delegate to it. + +### 7. Moved payload records to `Dtos/CommandDtos.cs` (new file) +The 16 private payload/result records that were inside `Entry.cs` are now in `Dtos/CommandDtos.cs`. Records used through public interfaces (`EntrySavePayload`, `EntryListItem`, `EntryLoadResult`, `EntrySaveResult`) are public; the rest remain internal. + +### 8. Moved database result records to `Dtos/DatabaseDtos.cs` (new file) +`JournalDatabaseStatus` and `JournalDatabaseHydrationResult` moved from `IJournalDatabaseService.cs` to `Dtos/DatabaseDtos.cs` for consistency with the other DTO files. + +## Files Created +- `Journal.Core/Services/HtmlSanitizer.cs` +- `Journal.Core/Services/CommandLogger.cs` +- `Journal.Core/Services/IEntryFileService.cs` +- `Journal.Core/Services/EntryFileService.cs` +- `Journal.Core/Services/PythonSidecarClient.cs` +- `Journal.Core/Repositories/IEntryFileRepository.cs` +- `Journal.Core/Repositories/DiskEntryFileRepository.cs` +- `Journal.Core/Dtos/CommandDtos.cs` +- `Journal.Core/Dtos/DatabaseDtos.cs` + +## Files Modified +- `Journal.Core/Entry.cs` — slimmed to thin dispatcher +- `Journal.Core/Services/PythonSidecarAiService.cs` — delegates to PythonSidecarClient +- `Journal.Core/Services/PythonSidecarSpeechService.cs` — delegates to PythonSidecarClient +- `Journal.Core/Services/IJournalDatabaseService.cs` — result records moved to Dtos +- `Journal.Core/Services/JournalDatabaseService.cs` — added Dtos using +- `Journal.Core/ServiceCollectionExtensions.cs` — registers new services and repository +- `Journal.SmokeTests/Program.cs` — updated NewEntry() with new dependencies + +## What Was NOT Changed +- **Fragment module** — already clean, untouched +- **Config module** — singleton reader, no changes needed +- **Vault module** — already well-separated (crypto/storage), untouched +- **AI/Speech interfaces and disabled variants** — untouched (only the sidecar implementations were refactored) +- **Search module** — stateless query service, no repository needed +- **All test logic** — no assertions or test behavior changed + +## Verification +- All 4 projects build successfully +- 70/70 smoke tests pass (5 Python sidecar tests fail only when Python is not installed on the machine, which is pre-existing)