diff --git a/.gitignore b/.gitignore index ff2a231..df0104f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ .nicegui .obsidian/ /lyricflow/ -/journal-master/ +#/journal-master/ /srczip/ # Cache @@ -13,6 +13,7 @@ data/ .mypy_cache/ .ruff_cache/ *.pyc +.cache/ # Ignore the encrypted vault and directorys journal/vault/ @@ -27,3 +28,11 @@ logs/ /setup_swap.sh /system_tune.sh +.pydeps/ +.dotnet* +.tmp/ +.pip*/ +.journal-sidecar/ +.nuget +_hybrid_tmp*/ +journal-master/journal/tls_registry_backup_before_fix.txt diff --git a/KANBAN_BOARD.md b/KANBAN_BOARD.md new file mode 100644 index 0000000..a474b21 --- /dev/null +++ b/KANBAN_BOARD.md @@ -0,0 +1,72 @@ +--- + +kanban-plugin: board + +--- + +# Migration Kanban Board + +Last updated: 2026-02-24 + +## Architecture Lock +- AI remains Python backend (local sidecar/service). C# AI surface is bridge-only (`IAiService` adapter + contracts), not model runtime. +- Non-AI backend migration target remains C# system-of-record (vault/entries/search/fragments/config/transport). + +## Doing +- `PH6-001` (Cutover Prep): Keep `migration-gate.ps1` green while running hybrid soak/pilot. + +## Ready Next + +## Blocked + +## Done +- `PH4-009` (DB-001/DB-002): C# SQLCipher runtime parity completed with NuGet packages (`Microsoft.Data.Sqlite.Core`, `SQLitePCLRaw.bundle_e_sqlcipher`) and sidecar DB actions (`db.status`, `db.initialize_schema`, `db.hydrate_workspace`). +- `PH4-010` (DB Cutover slice): Hybrid vault-load now routes DB hydration handoff to C# sidecar action `db.hydrate_workspace` (schema bootstrap + hydration metadata), removing Python-side post-load hydration in hybrid mode. +- `CFG-002` (UX): Settings UI exposes runtime-configurable backend/AI/speech values (backend mode, sidecar path, NLP backend, model endpoints, model names, timeouts, speech engine/device/compute). +- `PH1-001` (DOM-001): Fragment model/service parity with validation + trim behavior. +- `PH1-002` (Persistence): File-backed fragment repository replaces in-memory default. +- `PH1-003` (API-001): Sidecar transport stability with fixture-backed line-delimited JSON envelope tests. +- `PH2-001` (DOM-002): C# `JournalEntry`/`ParsedSection` domain skeleton with smoke coverage. +- `PH2-002` (PAR-001): Date extraction parity for `**Date:**` / `Date:` with file-stem fallback. +- `PH2-003` (PAR-002): Canonical section parsing parity with section content capture. +- `PH2-004` (PAR-003): Checkbox parsing parity with per-section checked state capture. +- `PH2-005` (PAR-004): Fragment block parsing parity (`!TYPE @time #tags` + multiline boundaries). +- `PH2-006` (MRG-001): Merge semantics parity for section updates and fragment de-dup append. +- `PH2-007` (MRG-002): Markdown reconstruction parity (`to_markdown()` equivalent). +- `PH3-001` (VLT-001): Vault crypto format compatibility (`salt + nonce + tag + ciphertext`, AES-256-GCM). +- `PH3-002` (VLT-002): PBKDF2-HMAC-SHA256 compatibility (salt/key size + iterations). +- `PH3-003` (VLT-003): Monthly vault naming parity (`YYYY-MM.vault`). +- `PH3-004` (VLT-004): Load-all-vault workflow parity. +- `PH3-005` (VLT-005): Wrong-password handling parity with no vault-file corruption. +- `PH3-006` (VLT-007): Current-month optimized save path parity. +- `PH3-007` (VLT-008): Full monthly rebuild save flow parity. +- `PH3-008` (DAT-001): Decrypted workspace cleanup parity. +- `PH3-009` (VLT-006): Legacy `_init_vault.vault` compatibility handling. +- `PH4-001` (SCH-001): Entry-content search parity. +- `PH4-002` (SCH-002): Search filter parity (date/section/tag/type/checkbox). +- `PH4-003` (CLI-001): Vault CLI parity actions and password UX. +- `PH4-004` (CLI-002): Search CLI parity actions/options. +- `PH4-005` (CFG-001): Config parity surface for backend/runtime keys. +- `PH4-006` (Hybrid): Route Python app non-AI backend paths (`entries/*`, `vault/*`, `search.entries`) to C# sidecar in `csharp-hybrid` mode. +- `PH4-007` (Hybrid): Add Python CLI fragment commands routed to C# `fragments.*` actions for user-side backend validation. +- `PH5-001` (AI-001): Summarize bridge parity via optional Python sidecar contract. +- `PH5-002` (AI-003): Cloud chat request/response parity via Python sidecar bridge (`ai.chat`). +- `PH5-003` (AI-002): Embeddings parity contract via Python sidecar bridge (`ai.embed`). +- `PH4-008` (API-002): HTTP API parity contract lock completed (`/api/command` envelope + `/health`/`/healthz` + API contract tests). +- `FIX-001` (Global): Shared fixture corpus completed with generated vault binaries + manifest (`fixtures/vaults/manifest.json`) and expanded entry/search cases. +- `HAR-001` (Global): Parity harness expanded to search expected IDs, vault hash matrix, wrong-password invariants, and sanitizer parity with report output. +- `PH5-004` (SPC-001): Speech parity completed with Python-executed speech bridge actions (`speech.devices.list`, `speech.transcribe`) orchestrated by C#. +- `OBS-001` (Observability): Structured redacted logging envelope implemented in C# sidecar (`timestamp`, `level`, `component`, `action`, `correlation_id`, `outcome`) with redaction tests. + +## Notes +- Source criteria: `MIGRATION_ACCEPTANCE_CRITERIA.md` +- C# migration path: `journal-master/journal/` + + + + +%% kanban:settings +``` +{"kanban-plugin":"board","list-collapse":[false]} +``` +%% diff --git a/MIGRATION_ACCEPTANCE_CRITERIA.md b/MIGRATION_ACCEPTANCE_CRITERIA.md new file mode 100644 index 0000000..73045ad --- /dev/null +++ b/MIGRATION_ACCEPTANCE_CRITERIA.md @@ -0,0 +1,185 @@ +--- +kanban-plugin: table +--- + +# C# Backend Parity Acceptance Criteria (Python Twin) + +## Goal +Create a C# backend that is behaviorally equivalent to the current Python backend for core journaling workflows, with zero data-loss regressions and compatible vault handling. + +## Parity Rule +Behavior parity is required. Internal implementation can differ. + +## Architecture Lock (Decision Record) +1. Non-AI backend remains C# system-of-record for migration scope: +- entry I/O, vault crypto/load/save, search, fragments, config surface, and transport layers. +2. AI inference/execution remains Python: +- C# AI code is bridge-only (`IAiService` + Python sidecar adapter), with timeout/error envelope handling. +- No model inference logic or model runtime will be introduced in C# during this migration plan. +3. Speech remains intentionally Python-side delegated during hybrid and cutover phases. +4. Any change to this lock requires a written RFC, benchmark evidence, and explicit maintainer approval. + +## Source of Truth (Current Python) +- `journal/core/storage.py` +- `journal/core/encryption.py` +- `journal/core/parser.py` +- `journal/core/models.py` +- `journal/core/database.py` +- `journal/cli/main.py` +- `journal/ai/analysis.py` +- `journal/ai/chat.py` +- `journal/core/speech.py` + +## Definition of Done (Release Gate) +1. All `P0` criteria below are marked `Pass`. +2. Existing Python-created vault files load successfully in C# backend. +3. Side-by-side output comparison on shared fixture corpus shows no functional mismatch for `P0` flows. +4. No plaintext journal data remains after graceful shutdown flow. + +## Shared Fixture Corpus (Required) +1. `fixtures/vaults/` +- Multiple months (`YYYY-MM.vault`) generated by Python app. +- Includes at least one wrong-password test case. +2. `fixtures/entries/` +- Daily, deep, recovery, fragment-heavy entries. +- Includes multiline fragments, tags, checkboxes, unusual spacing. +3. `fixtures/search/` +- Queries for text, section, tag, type, checked, unchecked, date ranges. +4. `fixtures/ai/` +- Stubbed LLM/embedding responses for deterministic comparison. + +## Acceptance Matrix +Use status values: `Not Started`, `In Progress`, `Blocked`, `Pass`, `Fail`. + +| ID | Priority | Feature | Acceptance Criteria | Status | Owner | Evidence | +| ------- | -------- | --------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | ----------- | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| DOM-001 | P0 | Fragment model parity | Fragment has `type`, `description`, `time`, `tags`; rejects empty `type`/`description`; trims values. | Pass | Stan44/Codex/J. Schmidt | `Journal.Core/Models/Fragment.cs`, `Journal.Core/Services/FragmentService.cs`, smoke test `CreateAsync trims fields` + `UpdateAsync rejects whitespace type` | +| DOM-002 | P0 | Journal entry domain parity | Entry model supports `date`, `raw_content`, `sections`, `fragments` equivalent to Python semantics. | Pass | Stan44/Gemini | `Journal.Core/Models/JournalEntry.cs`, `Journal.Core/Models/ParsedSection.cs`, `Journal.Core/Entry.cs`, smoke tests `JournalEntry model stores parity fields` + `Entry entries.save writes and merges content` + `Entry entries.load returns raw content payload` | +| PAR-001 | P0 | Date extraction parity | Parser reads `**Date:**` or `Date:`; falls back to filename stem when missing. | Pass | Stan44/Gemini | `Journal.Core/Services/JournalParser.cs`, smoke tests `Parser extracts date from **Date:** marker` + `Parser falls back to file stem when date missing` | +| PAR-002 | P0 | Section parsing parity | Recognizes canonical section titles and captures section content lines. | Pass | Stan44/Codex | `Journal.Core/Services/JournalParser.cs`, smoke tests `Parser captures canonical sections and content` + `Parser ignores non-canonical section headers` | +| PAR-003 | P0 | Checkbox parsing parity | Parses markdown checkboxes (`- [ ]`, `- [x]`) and preserves checked state by checkbox text. | Pass | Stan44/Codex | `Journal.Core/Services/JournalParser.cs`, smoke test `Parser captures checkbox states per section` | +| PAR-004 | P0 | Fragment parsing parity | Parses `!TYPE @time #tags` plus multiline description blocks with same boundary behavior as Python regex parser. | Pass | Stan44 | `Journal.Core/Services/JournalParser.cs`, smoke tests `Parser captures multiline fragment blocks` + `Parser fragment boundary follows header lines` | +| MRG-001 | P0 | Merge behavior parity | Saving existing entry merges non-empty section updates and appends only non-duplicate fragments by description. | Pass | Stan44/Codex | `Journal.Core/Models/JournalEntry.cs`, smoke tests `MergeWith overwrites section when new content is meaningful` + `MergeWith appends non-duplicate fragments by description` | +| MRG-002 | P0 | Markdown reconstruction parity | Entry serialization writes canonical section order and fragment block formatting equivalent to Python `to_markdown()`. | Pass | Stan44 | `Journal.Core/Models/JournalEntry.cs`, smoke tests `ToMarkdown writes canonical section order` + `ToMarkdown writes fragment blocks` | +| VLT-001 | P0 | Vault crypto format compatibility | Uses AES-256-GCM with payload layout compatible with Python (`salt + nonce + tag + ciphertext`). | Pass | Stan44/Codex | `Journal.Core/Services/VaultCryptoService.cs`, smoke tests `Vault crypto roundtrip preserves data and layout` + `Vault crypto decrypts Python payload fixture` | +| VLT-002 | P0 | KDF compatibility | Uses PBKDF2-HMAC-SHA256 with matching salt/key sizes and iteration count for Python vault compatibility. | Pass | Stan44/Codex | `Journal.Core/Services/VaultCryptoService.cs`, smoke test `Vault key derivation matches Python fixture` | +| VLT-003 | P0 | Monthly vault naming parity | Vault filename format exactly `YYYY-MM.vault`. | Pass | Stan44/Codex | `Journal.Core/Services/VaultStorageService.cs`, smoke test `Vault monthly filename matches parity format` | +| VLT-004 | P0 | Load workflow parity | `load all vaults` clears decrypted workspace first, then decrypts/extracts monthly vaults. | Pass | Stan44/Codex | `Journal.Core/Services/VaultStorageService.cs`, smoke test `Vault load clears workspace and extracts decrypted files` | +| VLT-005 | P0 | Wrong password behavior | Wrong password returns explicit failure and does not corrupt existing vault files. | Pass | Stan44/Codex | `Journal.Core/Services/VaultStorageService.cs`, smoke test `Vault load wrong password does not modify vault files` | +| VLT-006 | P1 | Legacy vault handling | Legacy `_init_vault.vault` handling behavior is preserved or intentionally migrated with backward-compat note. | Pass | Stan44/Codex | `journal-master/journal/Journal.Core/Services/VaultStorageService.cs`, smoke test `Vault load ignores and removes legacy _init_vault.vault`; migration note: file is ignored+deleted on load | +| VLT-007 | P0 | Current-month optimized save | Supports current-month save path equivalent to Python optimization. | Pass | Stan44 | `journal-master/journal/Journal.Core/Services/VaultStorageService.cs`, smoke test `Vault current-month save writes only current month and skips unchanged state` | +| VLT-008 | P0 | Full rebuild save | Supports full monthly regroup/rebuild save flow from decrypted `.md` files. | Pass | Stan44/Codex | `journal-master/journal/Journal.Core/Services/VaultStorageService.cs`, smoke test `Vault rebuild saves grouped monthly archives from decrypted files` | +| DAT-001 | P0 | Decrypted data cleanup | Graceful shutdown removes decrypted workspace artifacts. | Pass | Stan44/Codex | `journal-master/journal/Journal.Core/Services/VaultStorageService.cs`, smoke test `Vault clear data directory removes decrypted workspace artifacts` | +| DB-001 | P1 | SQLCipher compatibility | Database key derivation and SQLCipher connection behavior are compatible with Python expectations. | Pass | Stan44/Codex | `journal-master/journal/Journal.Core/Services/JournalDatabaseService.cs`, `journal-master/journal/Journal.Core/Services/IJournalDatabaseService.cs`, smoke tests `Database key derivation matches Python fixture` + `Entry db.status returns database compatibility payload` + `Entry db.hydrate_workspace returns hydration metadata` | +| DB-002 | P1 | Schema parity | `entries`, `sections`, `fragments`, `tags`, `fragment_tags` schema exists with compatible constraints. | Pass | Stan44/Codex | `journal-master/journal/Journal.Core/Services/JournalDatabaseService.cs`, smoke tests `Database schema parity tables are created` + `Entry db.initialize_schema creates schema in data directory` + `Entry db.hydrate_workspace returns hydration metadata` | +| SCH-001 | P0 | Search parity (content) | Search supports text query over full entry content. | Pass | Stan44/Codex | `journal-master/journal/Journal.Core/Services/EntrySearchService.cs`, `journal-master/journal/Journal.Core/Entry.cs`, smoke tests `Entry search.entries matches query against full raw content` + `Entry search.entries without query returns all markdown entries` | +| SCH-002 | P0 | Search parity (filters) | Search supports date range, section filter, tag filter, fragment type filter, checked/unchecked filters. | Pass | Stan44 | `journal-master/journal/Journal.Core/Services/EntrySearchService.cs`, `journal-master/journal/Journal.Core/Entry.cs`, smoke tests for date/section/tag+type/checked+unchecked filters + invalid date handling | +| CLI-001 | P0 | Vault CLI parity | CLI supports vault load/save with password prompt semantics matching Python UX. | Pass | Stan44/Codex | `journal-master/journal/Journal.Core/Services/SidecarCli.cs`, `journal-master/journal/Journal.Sidecar/App.cs`, smoke tests `Sidecar vault CLI load succeeds with --password` + `Sidecar vault CLI save writes monthly vault with --password` | +| CLI-002 | P0 | Search CLI parity | CLI options match existing Python capabilities sufficiently for drop-in replacement. | Pass | Stan44/Codex | `journal-master/journal/Journal.Core/Services/SidecarCli.cs`, smoke tests `Sidecar search CLI returns matching entries with filters` + `Sidecar search CLI warns when no decrypted entries exist` | +| API-001 | P0 | Sidecar protocol stability | Stdin/stdout contract is line-delimited JSON with `{ok,data}` or `{ok:false,error}` and strict action routing. | Pass | Stan44/Codex | `Journal.Core/Entry.cs`, `Journal.SmokeTests/Program.cs`, `Journal.SmokeTests/Fixtures/transport_cases.json` | +| API-002 | P1 | HTTP API parity | API exposes endpoints equivalent to supported sidecar actions; no template-only endpoints in parity mode. | Pass | Stan44/Codex | `journal-master/journal/Journal.Api/Program.cs`, `journal-master/journal/Journal.Api/Journal.Api.http`, `tests/test_api_contract.py`, `scripts/migration-gate.ps1` | +| AI-001 | P1 | LLM summarize parity | Entry/all-entry summarize workflow callable with equivalent behavior and timeout controls. | Pass | Stan44/Codex | `journal-master/journal/Journal.Core/Services/IAiService.cs`, `journal-master/journal/Journal.Core/Services/PythonSidecarAiService.cs`, `journal/ai/sidecar.py`, `journal-master/journal/Journal.Core/Entry.cs`, smoke tests for `ai.health` + summarize bridge/error handling | +| AI-002 | P2 | Embedding parity | Embedding endpoint integration available with equivalent request/response contract. | Pass | Stan44/Codex | `journal-master/journal/Journal.Core/Services/IAiService.cs`, `journal-master/journal/Journal.Core/Services/PythonSidecarAiService.cs`, `journal/ai/sidecar.py`, `journal-master/journal/Journal.Core/Entry.cs`, smoke tests for `ai.embed` bridge/disabled parsing | +| AI-003 | P1 | Cloud chat parity | Cloud chat request/response flow available and configurable. | Pass | Stan44 | `journal-master/journal/Journal.Core/Services/IAiService.cs`, `journal-master/journal/Journal.Core/Services/PythonSidecarAiService.cs`, `journal/ai/sidecar.py`, `journal/ai/chat.py`, `journal-master/journal/Journal.Core/Entry.cs`, smoke tests for `ai.chat` bridge/disabled/error handling | +| SPC-001 | P2 | Speech parity | Engine-selectable speech flow (whisper/google/sphinx-like modes) retained or intentionally delegated. | Pass | Stan44/Codex | `journal-master/journal/Journal.Core/Services/ISpeechBridgeService.cs`, `journal-master/journal/Journal.Core/Services/PythonSidecarSpeechService.cs`, `journal-master/journal/Journal.Core/Entry.cs`, `journal/ai/sidecar.py`, smoke tests for speech bridge empty-device/error/timeout paths | +| CFG-001 | P0 | Config parity | Equivalent configuration keys exist for paths, vault format, AI endpoints, and speech settings. | Pass | Stan44 | `journal-master/journal/Journal.Core/Services/JournalConfigService.cs`, `journal-master/journal/Journal.Core/Models/JournalConfig.cs`, smoke tests `Config service exposes parity path, vault, AI, and speech settings` + `Entry config.get returns config payload` | +| OBS-001 | P1 | Logging/error parity | Failures return actionable messages without leaking secrets or plaintext journal data. | Pass | Stan44/Codex | `journal-master/journal/Journal.Core/Entry.cs`, `journal-master/journal/Journal.Core/Services/LogRedactor.cs`, smoke tests `Log redactor scrubs sensitive payload fields` + `Log redactor preserves non-sensitive payload fields` | + +## Mandatory Gate Tests (P0) +1. `Vault Compatibility` +- Given existing Python vault fixtures, C# load succeeds with correct password. +- Wrong password consistently fails without partial corruption. +2. `Parser + Merge` +- Given fixture entries, C# parser output matches Python parser output for sections/fragments/checkboxes. +- Saving edited entry preserves merge semantics. +3. `Search` +- For shared corpus, C# and Python return the same entry set for all P0 search filters. +4. `Transport` +- Sidecar commands for core actions produce stable JSON success/error envelopes. +5. `Cleanup` +- After graceful shutdown sequence, decrypted workspace is empty. + +## Migration Phases (Recommended) +1. `Phase 1: Fragment Vertical Slice` +- Complete `DOM-001`, `API-001`, persistence beyond in-memory. +2. `Phase 2: Entry/Parser/Merge Twin` +- Complete `DOM-002`, `PAR-*`, `MRG-*`. +3. `Phase 3: Vault/Crypto Twin` +- Complete `VLT-*`, `DAT-001`. +4. `Phase 4: Search + CLI Twin` +- Complete `SCH-*`, `CLI-*`, `CFG-001`. +5. `Phase 5: AI/Speech` +- Complete `AI-*` as Python-bridge parity (not C# model parity), plus `SPC-001`. +6. `Phase 6: Frontend Cutover` +- Tauri frontend switches to C# backend only after all P0 criteria pass. + +## Rules for Change Requests +1. Any intentional divergence from Python behavior requires: +- Written rationale. +- New acceptance criterion replacing old one. +- Migration note describing user-visible impact. +2. No removal of a `P0` criterion without both maintainers approving in writing. + +## Status Snapshot (2026-02-24, Phase 5 Complete / Phase 6 Ready) +This section is a proposed status/evidence snapshot based on current code in: +- Python source: `Project_Journal/` +- C# migration: `Project_Journal/journal-master/journal/` + +It does not change release gates or acceptance definitions. + +### Phase Summary +| Phase | Current State | Evidence | +| -------------------------------- | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Phase 1: Fragment Vertical Slice | Pass (complete) | Fragment model/service/sidecar + persisted repository + fixture-backed transport smoke tests: `journal-master/journal/Journal.Core/Models/Fragment.cs`, `journal-master/journal/Journal.Core/Entry.cs`, `journal-master/journal/Journal.Core/Repositories/FileFragmentRepository.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs`, `journal-master/journal/Journal.SmokeTests/Fixtures/transport_cases.json` | +| Phase 2: Entry/Parser/Merge Twin | Pass | C# `JournalEntry` domain, parser slices (`PAR-001..004`), and merge/reconstruction slices (`MRG-001..002`) are implemented with smoke evidence. | +| Phase 3: Vault/Crypto Twin | Pass | Vault parity slices (`VLT-001..008`) and decrypted cleanup (`DAT-001`) are implemented with smoke evidence. | +| Phase 4: Search + CLI Twin | Pass | Search parity (`SCH-001`, `SCH-002`), CLI parity (`CLI-001`, `CLI-002`), and config parity (`CFG-001`) are implemented with smoke evidence. | +| Phase 5: AI/Speech | Pass | `AI-001`/`AI-002`/`AI-003` bridge parity implemented and `SPC-001` completed with Python-side speech execution + C# orchestration contract (`speech.devices.list`, `speech.transcribe`). | +| Phase 6: Frontend Cutover | Ready | Release-gate fixture corpus, parity harness, and API contract tests are implemented and callable through `scripts/migration-gate.ps1`. | + +### Acceptance Status Snapshot +| ID | Proposed Status | Evidence | Notes | +| ------- | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| DOM-001 | Pass | `journal-master/journal/Journal.Core/Models/Fragment.cs`, `journal-master/journal/Journal.Core/Services/FragmentService.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Fields/validation/trim behavior implemented and smoke-tested. | +| DOM-002 | Pass | `journal-master/journal/Journal.Core/Models/JournalEntry.cs`, `journal-master/journal/Journal.Core/Models/ParsedSection.cs`, `journal-master/journal/Journal.Core/Entry.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Entry parity is implemented for model and runtime save/load/list flows with smoke coverage (`entries.save`/`entries.load`/`entries.list`). | +| PAR-001 | Pass | `journal-master/journal/Journal.Core/Services/JournalParser.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Date extraction supports `**Date:**` and `Date:` with file-stem fallback and smoke coverage. | +| PAR-002 | Pass | `journal-master/journal/Journal.Core/Services/JournalParser.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Canonical section headers are recognized and section content lines are captured with smoke coverage. | +| PAR-003 | Pass | `journal-master/journal/Journal.Core/Services/JournalParser.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Checkbox state extraction for `- [ ]`/`- [x]` is implemented with per-section lookup coverage. | +| PAR-004 | Pass | `journal-master/journal/Journal.Core/Services/JournalParser.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Fragment header metadata and multiline description boundaries are parsed with smoke coverage. | +| MRG-001 | Pass | `journal-master/journal/Journal.Core/Models/JournalEntry.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Merge overwrites only meaningful section updates and appends non-duplicate fragments by description. | +| MRG-002 | Pass | `journal-master/journal/Journal.Core/Models/JournalEntry.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Markdown reconstruction emits canonical section order and fragment blocks with smoke coverage. | +| VLT-001 | Pass | `journal-master/journal/Journal.Core/Services/VaultCryptoService.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Payload layout (`salt+nonce+tag+ciphertext`) is validated and can decrypt Python-generated fixture data. | +| VLT-002 | Pass | `journal-master/journal/Journal.Core/Services/VaultCryptoService.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | PBKDF2-HMAC-SHA256 derivation matches Python fixture output and parameterization. | +| VLT-003 | Pass | `journal-master/journal/Journal.Core/Services/VaultStorageService.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Monthly vault filename format is emitted as `YYYY-MM.vault`. | +| VLT-004 | Pass | `journal-master/journal/Journal.Core/Services/VaultStorageService.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Load workflow clears decrypted workspace before decrypting and extracting vault files. | +| VLT-005 | Pass | `journal-master/journal/Journal.Core/Services/VaultStorageService.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Wrong password returns failure and leaves existing vault bytes unchanged. | +| VLT-006 | Pass | `journal-master/journal/Journal.Core/Services/VaultStorageService.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Legacy `_init_vault.vault` is treated as obsolete, ignored for decryption, and deleted during load for compatibility. | +| VLT-007 | Pass | `journal-master/journal/Journal.Core/Services/VaultStorageService.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Current-month optimized save path is implemented with fingerprint-based skip behavior and smoke coverage. | +| VLT-008 | Pass | `journal-master/journal/Journal.Core/Services/VaultStorageService.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Full monthly rebuild flow from decrypted `.md` files is implemented with smoke coverage. | +| DAT-001 | Pass | `journal-master/journal/Journal.Core/Services/VaultStorageService.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Decrypted workspace cleanup path removes artifacts and recreates an empty data directory with smoke coverage. | +| DB-001 | Pass | `journal-master/journal/Journal.Core/Services/JournalDatabaseService.cs`, `journal-master/journal/Journal.Core/Services/IJournalDatabaseService.cs`, `journal-master/journal/Journal.Core/Entry.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | PBKDF2-HMAC-SHA256 key derivation parity plus runtime SQLCipher connection/key validation are implemented and surfaced by `db.status` (`RuntimeReady=true`) and `db.hydrate_workspace`. | +| DB-002 | Pass | `journal-master/journal/Journal.Core/Services/JournalDatabaseService.cs`, `journal-master/journal/Journal.Core/Entry.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Schema parity DDL for `entries/sections/fragments/tags/fragment_tags` is implemented and validated through runtime schema creation and table checks in `db.hydrate_workspace`. | +| SCH-001 | Pass | `journal-master/journal/Journal.Core/Services/EntrySearchService.cs`, `journal-master/journal/Journal.Core/Entry.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | `search.entries` performs case-insensitive full raw-content query across decrypted `.md` entries with smoke coverage. | +| SCH-002 | Pass | `journal-master/journal/Journal.Core/Services/EntrySearchService.cs`, `journal-master/journal/Journal.Core/Entry.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Date range, section-scoped query, tag/type, and checked/unchecked filters are implemented for `search.entries`. | +| CLI-001 | Pass | `journal-master/journal/Journal.Core/Services/SidecarCli.cs`, `journal-master/journal/Journal.Sidecar/App.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Sidecar supports `vault load/save` CLI mode with password prompt semantics and non-interactive `--password` support. | +| CLI-002 | Pass | `journal-master/journal/Journal.Core/Services/SidecarCli.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Sidecar supports search CLI flags (query/date/section/tag/type/checked/unchecked) and parity-style output messaging. | +| API-001 | Pass | `journal-master/journal/Journal.Core/Entry.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs`, `journal-master/journal/Journal.SmokeTests/Fixtures/transport_cases.json` | Line-delimited JSON envelope behavior and strict action routing are covered by smoke tests and transport fixtures. | +| API-002 | Pass | `journal-master/journal/Journal.Api/Program.cs`, `journal-master/journal/Journal.Api/Journal.Api.http`, `tests/test_api_contract.py`, `scripts/migration-gate.ps1` | HTTP parity contract is locked (`POST /api/command`, `GET /health`, `GET /healthz`) with malformed/missing/unknown action coverage. | +| AI-001 | Pass | `journal-master/journal/Journal.Core/Services/IAiService.cs`, `journal-master/journal/Journal.Core/Services/PythonSidecarAiService.cs`, `journal/ai/sidecar.py`, `journal-master/journal/Journal.Core/Entry.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | `ai.health` + summarize bridge parity implemented with provider modes (`none`, `python-sidecar`) and timeout/error envelopes. | +| AI-002 | Pass | `journal-master/journal/Journal.Core/Services/IAiService.cs`, `journal-master/journal/Journal.Core/Services/PythonSidecarAiService.cs`, `journal/ai/sidecar.py`, `journal-master/journal/Journal.Core/Entry.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | `ai.embed` bridge parity is implemented with provider-mode fallback (`none`/`python-sidecar`) and smoke coverage. | +| AI-003 | Pass | `journal-master/journal/Journal.Core/Services/IAiService.cs`, `journal-master/journal/Journal.Core/Services/PythonSidecarAiService.cs`, `journal/ai/sidecar.py`, `journal/ai/chat.py`, `journal-master/journal/Journal.Core/Entry.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | `ai.chat` bridge parity is implemented with provider-mode fallback (`none`/`python-sidecar`) and smoke coverage. | +| SPC-001 | Pass | `journal-master/journal/Journal.Core/Services/ISpeechBridgeService.cs`, `journal-master/journal/Journal.Core/Services/PythonSidecarSpeechService.cs`, `journal-master/journal/Journal.Core/Entry.cs`, `journal/ai/sidecar.py`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Speech delegation is locked as Python execution + C# orchestration (`speech.devices.list`, `speech.transcribe`) with timeout/error/empty-device coverage. | +| CFG-001 | Pass | `journal-master/journal/Journal.Core/Services/JournalConfigService.cs`, `journal-master/journal/Journal.Core/Models/JournalConfig.cs`, `journal-master/journal/Journal.Core/Entry.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | C# config surface now includes parity keys for paths, vault format, AI endpoints, and speech settings via `config.get`. | +| OBS-001 | Pass | `journal-master/journal/Journal.Core/Entry.cs`, `journal-master/journal/Journal.Core/Services/LogRedactor.cs`, `journal-master/journal/Journal.SmokeTests/Program.cs` | Structured logging envelope and secret redaction are implemented and covered by smoke tests. | + +### Snapshot Risks/Blockers +1. Remaining migration risk is operational only: maintain parity by running `scripts/migration-gate.ps1` before cutover merges. +2. DB runtime parity is active in C# with SQLCipher via NuGet packages (`Microsoft.Data.Sqlite.Core` + `SQLitePCLRaw.bundle_e_sqlcipher`) and smoke validation. +3. Hybrid DB hydration handoff remains routed through C# sidecar (`db.hydrate_workspace`) from `journal/core/storage.py`; Python-side post-load DB hydration remains disabled in hybrid mode. + + +%% kanban:settings +``` +{"kanban-plugin":"board","list-collapse":[false]} +``` +%% diff --git a/README.md b/README.md index 9fe01ab..9e0ce4e 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,21 @@ On Windows + Python 3.14, `pywebview` is intentionally skipped due upstream `pythonnet` build compatibility. `run_desktop.py` will auto-fallback to opening the app in your system browser. +### Windows Minimal Wrapper Commands + +If your shell has broken proxy/no-index env vars or restricted profile paths: + +```powershell +cd Project_Journal +.\scripts\dev-shell.ps1 +.\scripts\dotnet-min.ps1 restore journal-master/journal/Journal.Sidecar/Journal.Sidecar.csproj +.\scripts\pip-min.ps1 install --index-url https://pypi.org/simple faster-whisper +``` + +`pip-min.ps1` installs to repo-local `.pydeps/py314` by default to avoid user site-packages permission issues. +On Windows, it also maps `pyaudio` to `pyaudiowpatch` to avoid PortAudio source-build failures. +The dev shell also sets Hugging Face cache to repo-local `.cache/huggingface` and suppresses the Windows symlink warning. + ### Optional NLP backend (spaCy) ```bash @@ -72,14 +87,42 @@ On current Python 3.14 environments, this optional install may be skipped due up python ./journal/run_desktop.py ``` +### Hybrid Backend Mode (Python UI + C# backend bridge) + +Build the sidecar once: + +```powershell +cd .\journal-master\journal +.\scripts\dotnet-min.ps1 build Journal.Sidecar/Journal.Sidecar.csproj +``` + +Run the Python app with hybrid mode enabled: + +```powershell +cd ..\.. +$env:JOURNAL_BACKEND_MODE = "csharp-hybrid" +python .\journal\run_desktop.py +``` + +Optional override if the sidecar executable is in a custom path: + +```powershell +$env:JOURNAL_CSHARP_SIDECAR_PATH = "E:\path\to\Journal.Sidecar.exe" +``` + ### CLI ```bash python -m journal.cli.main --help python -m journal.cli.main vault load python -m journal.cli.main search "your query" +python -m journal.cli.main fragments list +python -m journal.cli.main fragments create --type !NOTE --description "example" --tag testing ``` +Fragment CLI commands use C# sidecar actions and require: +- `JOURNAL_BACKEND_MODE=csharp-hybrid` + ## NLP Backend Control Set `JOURNAL_NLP_BACKEND` to choose behavior: @@ -114,3 +157,6 @@ Use the Linux helper script: - Decrypted journal data in `journal/data` is cleared on graceful shutdown. - Vault save/load commands remain unchanged. +- In `csharp-hybrid` mode, UI entry load/save, vault operations, and CLI search use C# sidecar actions. +- AI remains Python-local by design in the Python UI runtime. +- Settings dialog now exposes backend/AI/speech runtime fields (endpoints, models, timeouts, engine/device/compute), with restart controls for changes that require reinitialization. diff --git a/fixtures/README.md b/fixtures/README.md new file mode 100644 index 0000000..b387ff6 --- /dev/null +++ b/fixtures/README.md @@ -0,0 +1,17 @@ +# Shared Fixture Corpus + +This corpus is the common input set for Python-vs-C# parity checks. + +## Layout + +- `fixtures/entries/`: Canonical markdown entries used by parser/merge/search parity tests. +- `fixtures/search/queries.json`: Search cases consumed by parity harness. +- `fixtures/ai/stubs.json`: Deterministic AI request/response samples for bridge-level contract tests. +- `fixtures/vaults/`: Python-generated monthly vault fixtures plus `manifest.json` (hashes + metadata). + +## Usage Contract + +- Treat files here as stable test vectors. +- Any behavior change that affects outputs must update this corpus and migration notes. +- Keep content synthetic and non-sensitive. +- Regenerate vault fixtures only with `python fixtures/vaults/generate_vault_fixtures.py` and commit updated hashes. diff --git a/fixtures/ai/stubs.json b/fixtures/ai/stubs.json new file mode 100644 index 0000000..6130950 --- /dev/null +++ b/fixtures/ai/stubs.json @@ -0,0 +1,24 @@ +{ + "health": { + "provider": "python-local", + "enabled": true, + "healthy": true + }, + "summarize_entry": { + "request": { + "content": "Entry content for deterministic summary.", + "file_stem": "2026-02-22" + }, + "response": "Deterministic summary fixture response." + }, + "summarize_all": { + "request": { + "entries": [ + "Entry one fixture text.", + "Entry two fixture text." + ] + }, + "response": "Deterministic aggregate summary fixture response." + } +} + diff --git a/fixtures/entries/2026-01-05.md b/fixtures/entries/2026-01-05.md new file mode 100644 index 0000000..0ff4532 --- /dev/null +++ b/fixtures/entries/2026-01-05.md @@ -0,0 +1,22 @@ +**Date:** 2026-01-05 + +## Morning Check-in +- [x] took meds +- [ ] call therapist +Felt stable after breakfast. + +## Triggers +Crowded grocery store caused mild panic. + +## Reflections +I used grounding exercises and improved within 20 minutes. + +!TRIGGER @08:15 #anxiety #crowd +Crowded aisle led to elevated heart rate. + +!COPING @08:42 #breathing #recovery +Used box breathing: +in for 4 +hold for 4 +out for 4 + diff --git a/fixtures/entries/2026-01-12.md b/fixtures/entries/2026-01-12.md new file mode 100644 index 0000000..ff7b52b --- /dev/null +++ b/fixtures/entries/2026-01-12.md @@ -0,0 +1,18 @@ +Date: 2026-01-12 + +## Morning Check-in +- [x] took meds +Sleep quality was poor. + +## Triggers +Conflict at work raised stress. + +## Reflections +Wrote down three things I can control today. + +!TRIGGER @14:10 #stress #work +Unexpected escalation in a team meeting. + +!INSIGHT @22:01 #sleep #routine +Late caffeine made it harder to unwind. + diff --git a/fixtures/entries/2026-02-03.md b/fixtures/entries/2026-02-03.md new file mode 100644 index 0000000..728fddf --- /dev/null +++ b/fixtures/entries/2026-02-03.md @@ -0,0 +1,17 @@ +## Morning Check-in +- [ ] took meds +- [x] call therapist +Low-energy morning with racing thoughts. + +## Triggers +News cycle doomscrolling increased anxiety. + +## Reflections +Set a 20-minute timer and stepped away from screens. + +!TRIGGER @09:33 #anxiety #news +Doomscrolling increased rumination quickly. + +!COPING @09:55 #walk #recovery +Short outdoor walk reduced panic symptoms. + diff --git a/fixtures/entries/2026-02-19.md b/fixtures/entries/2026-02-19.md new file mode 100644 index 0000000..d833ec9 --- /dev/null +++ b/fixtures/entries/2026-02-19.md @@ -0,0 +1,15 @@ +**Date:** 2026-02-19 + +## Morning Check-in +- [x] took meds +Clear focus before noon. + +## Triggers +No major triggers today. + +## Reflections +Finished backlog tasks and felt more in control. + +!INSIGHT @21:30 #routine #sleep +Going offline after 9PM improved sleep onset. + diff --git a/fixtures/entries/2026-02-21.md b/fixtures/entries/2026-02-21.md new file mode 100644 index 0000000..e5d048d --- /dev/null +++ b/fixtures/entries/2026-02-21.md @@ -0,0 +1,15 @@ +**Date:** 2026-02-21 + +## Morning Check-in +- [x] hydration logged +- [ ] inbox zero +Started steady and focused. + +## Reflections +Maintained momentum with short, timed sessions. + +!NOTE @10:12 #focus #planning +Used a 25-minute planning cycle before implementation. + +!MEMORY @18:40 #routine +Evening review helped close open loops. diff --git a/fixtures/entries/2026-02-22.md b/fixtures/entries/2026-02-22.md new file mode 100644 index 0000000..1b1bc27 --- /dev/null +++ b/fixtures/entries/2026-02-22.md @@ -0,0 +1,14 @@ +Date: 2026-02-22 + +## Summary +Legacy spacing and punctuation test. + +## Cognitive State +- [X] focus window +- [ ] task switch overload + +## Notes +This file intentionally uses a non-canonical section for parser tolerance checks. + +!NOTE @14:45 #legacy #format +Header spacing : should still remain stable in storage. diff --git a/fixtures/entries/2026-02-24.md b/fixtures/entries/2026-02-24.md new file mode 100644 index 0000000..53f5823 --- /dev/null +++ b/fixtures/entries/2026-02-24.md @@ -0,0 +1,12 @@ +**Date:** 2026-02-24 + +## Morning Check-in +- [ ] follow up email +- [x] backup complete +- [ ] backup complete + +## Reflections +Duplicate checkbox labels are intentional edge cases. + +!NOTE @20:11 #checkbox #edge +Validate checked/unchecked filtering handles repeated labels. diff --git a/fixtures/entries/2026-02-25.md b/fixtures/entries/2026-02-25.md new file mode 100644 index 0000000..694eb86 --- /dev/null +++ b/fixtures/entries/2026-02-25.md @@ -0,0 +1,11 @@ +**Date:** 2026-02-25 + +## Summary +

Clipboard paste normalization sample.

+ + +## Reflections +This fixture verifies rich HTML cleanup parity before persistence. + +!NOTE @16:05 #sanitize #html +Rich paste should normalize to plain markdown-compatible text. diff --git a/fixtures/search/queries.json b/fixtures/search/queries.json new file mode 100644 index 0000000..d101cb7 --- /dev/null +++ b/fixtures/search/queries.json @@ -0,0 +1,66 @@ +[ + { + "name": "text_query_panic", + "payload": { + "query": "panic" + }, + "expected_file_names": [ + "2026-01-05.md", + "2026-02-03.md" + ] + }, + { + "name": "section_query_triggers_anxiety", + "payload": { + "query": "anxiety", + "section": "Triggers" + }, + "expected_file_names": [] + }, + { + "name": "fragment_tag_sleep", + "payload": { + "tags": ["sleep"] + }, + "expected_file_names": [ + "2026-01-12.md", + "2026-02-19.md" + ] + }, + { + "name": "fragment_type_trigger", + "payload": { + "types": ["!TRIGGER"] + }, + "expected_file_names": [ + "2026-01-05.md", + "2026-01-12.md", + "2026-02-03.md" + ] + }, + { + "name": "checked_took_meds", + "payload": { + "checked": ["took meds"] + }, + "expected_file_names": [] + }, + { + "name": "unchecked_call_therapist", + "payload": { + "unchecked": ["call therapist"] + }, + "expected_file_names": [] + }, + { + "name": "date_range_january", + "payload": { + "startDate": "2026-01-01", + "endDate": "2026-01-31" + }, + "expected_file_names": [ + "2026-01-05.md", + "2026-01-12.md" + ] + } +] diff --git a/fixtures/vaults/2026-01.vault b/fixtures/vaults/2026-01.vault new file mode 100644 index 0000000..7180d38 Binary files /dev/null and b/fixtures/vaults/2026-01.vault differ diff --git a/fixtures/vaults/2026-02.vault b/fixtures/vaults/2026-02.vault new file mode 100644 index 0000000..f9662a2 Binary files /dev/null and b/fixtures/vaults/2026-02.vault differ diff --git a/fixtures/vaults/README.md b/fixtures/vaults/README.md new file mode 100644 index 0000000..06522dc --- /dev/null +++ b/fixtures/vaults/README.md @@ -0,0 +1,24 @@ +# Vault Fixture Contract + +This directory holds vault compatibility fixtures shared between Python and C# tests. + +## Required cases + +1. `YYYY-MM.vault` files generated by Python encryption flow from `fixtures/entries/`. +2. At least one wrong-password case where decrypt fails and source vault bytes remain unchanged. +3. Optional legacy case: `_init_vault.vault` for compatibility cleanup behavior. + +## Current status + +- Vault fixtures are generated from `fixtures/entries/*.md` using Python crypto flow: + - `python fixtures/vaults/generate_vault_fixtures.py` +- Committed artifacts: + - `2026-01.vault` + - `2026-02.vault` + - `manifest.json` with generation metadata, fixture password, wrong-password case, and SHA-256 hashes. + +## Validation contract + +1. Load with `manifest.password` must extract all `expected_entries`. +2. Each extracted entry SHA-256 must match manifest values. +3. Load with `manifest.wrong_password` must fail and keep vault bytes unchanged. diff --git a/fixtures/vaults/generate_vault_fixtures.py b/fixtures/vaults/generate_vault_fixtures.py new file mode 100644 index 0000000..770406a --- /dev/null +++ b/fixtures/vaults/generate_vault_fixtures.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import hashlib +import json +import os +import platform +import shutil +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +ENTRY_FIXTURES = PROJECT_ROOT / "fixtures" / "entries" +VAULT_FIXTURES = PROJECT_ROOT / "fixtures" / "vaults" +MANIFEST_PATH = VAULT_FIXTURES / "manifest.json" +FIXTURE_PASSWORD = "fixture-pass-123" +WRONG_PASSWORD = "fixture-pass-wrong" + + +def _sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while True: + chunk = handle.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + + +def _load_encrypt_data(): + import sys + + if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + from journal.core.encryption import encrypt_data + + return encrypt_data + + +def _group_entries_by_month(entries: list[Path]) -> dict[str, list[Path]]: + grouped: dict[str, list[Path]] = {} + for entry in entries: + month_key = entry.stem[:7] + grouped.setdefault(month_key, []).append(entry) + return grouped + + +def _build_zip_payload(month_entries: list[Path]) -> bytes: + staging_root = VAULT_FIXTURES / f".staging-{uuid4().hex}" + staging_root.mkdir(parents=True, exist_ok=True) + try: + for source in sorted(month_entries, key=lambda p: p.name): + shutil.copy2(source, staging_root / source.name) + zip_path = staging_root / "payload.zip" + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as archive: + for staged in sorted(staging_root.glob("*.md"), key=lambda p: p.name): + archive.write(staged, arcname=staged.name) + return zip_path.read_bytes() + finally: + shutil.rmtree(staging_root, ignore_errors=True) + + +def generate() -> None: + encrypt_data = _load_encrypt_data() + VAULT_FIXTURES.mkdir(parents=True, exist_ok=True) + + for old_vault in VAULT_FIXTURES.glob("*.vault"): + old_vault.unlink() + + entries = sorted(ENTRY_FIXTURES.glob("*.md"), key=lambda p: p.name) + if not entries: + raise RuntimeError("No entry fixtures found under fixtures/entries.") + + grouped = _group_entries_by_month(entries) + + manifest: dict[str, object] = { + "format_version": 1, + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "generator": "fixtures/vaults/generate_vault_fixtures.py", + "python_version": platform.python_version(), + "password": FIXTURE_PASSWORD, + "wrong_password": WRONG_PASSWORD, + "vaults": [], + } + + vault_rows: list[dict[str, object]] = [] + for month_key in sorted(grouped.keys()): + month_entries = grouped[month_key] + zip_payload = _build_zip_payload(month_entries) + encrypted = encrypt_data(zip_payload, FIXTURE_PASSWORD) + + vault_name = f"{month_key}.vault" + vault_path = VAULT_FIXTURES / vault_name + vault_path.write_bytes(encrypted) + + expected_entries = [] + for entry in sorted(month_entries, key=lambda p: p.name): + expected_entries.append( + { + "file_name": entry.name, + "sha256": _sha256_file(entry), + } + ) + + vault_rows.append( + { + "vault_file": vault_name, + "sha256": _sha256_file(vault_path), + "entry_count": len(expected_entries), + "expected_entries": expected_entries, + } + ) + + manifest["vaults"] = vault_rows + MANIFEST_PATH.write_text(json.dumps(manifest, indent=2) + os.linesep, encoding="utf-8") + print(f"Generated {len(vault_rows)} vault fixture(s) at {VAULT_FIXTURES}") + + +if __name__ == "__main__": + generate() diff --git a/fixtures/vaults/manifest.json b/fixtures/vaults/manifest.json new file mode 100644 index 0000000..b6b1805 --- /dev/null +++ b/fixtures/vaults/manifest.json @@ -0,0 +1,56 @@ +{ + "format_version": 1, + "generated_at_utc": "2026-02-24T01:12:43.837001+00:00", + "generator": "fixtures/vaults/generate_vault_fixtures.py", + "python_version": "3.14.2", + "password": "fixture-pass-123", + "wrong_password": "fixture-pass-wrong", + "vaults": [ + { + "vault_file": "2026-01.vault", + "sha256": "e915082ae2a8b0c53c644eecc7893486528d993a1b47170a7b9990649138b0da", + "entry_count": 2, + "expected_entries": [ + { + "file_name": "2026-01-05.md", + "sha256": "381ec7625ca120365ce4206be43e0f0c9b99b3aa73c7e5d5b3f8e523dddbaff9" + }, + { + "file_name": "2026-01-12.md", + "sha256": "df2dc47da9b1849fe564cc52ac3df7f69d0f65948cc41b89fe8b8da89febfb1f" + } + ] + }, + { + "vault_file": "2026-02.vault", + "sha256": "cf932fd3df5fb9d410461a0de59a06acaa3016ff01f66d0d37d673364bf35862", + "entry_count": 6, + "expected_entries": [ + { + "file_name": "2026-02-03.md", + "sha256": "0314026982893b9be04dcc17c694e86ab9559cf5857f9907cb922b4691fd6d83" + }, + { + "file_name": "2026-02-19.md", + "sha256": "c441b3d0fc279379b55c7298d25f51eb17dac08dfc20c22e60dc0db358d3dcbb" + }, + { + "file_name": "2026-02-21.md", + "sha256": "9bafe3b485b4599d18eac5309502e877d23abd7a12cd09c939e49980c54101ae" + }, + { + "file_name": "2026-02-22.md", + "sha256": "c37dc7e3d370d380bd94a4e3bd78cdf5fafa260466d1b58021c940c28b42092f" + }, + { + "file_name": "2026-02-24.md", + "sha256": "496a157450d027b6f1fcec455d759ffd5bc818b7cec254f314a8853281939869" + }, + { + "file_name": "2026-02-25.md", + "sha256": "14236a844e3ef53b9dee3e01c459daa91461fd76e9c1148f3976569772bbb2da" + } + ] + } + ] +} diff --git a/journal-master/journal/.editorconfig b/journal-master/journal/.editorconfig new file mode 100644 index 0000000..960e0be --- /dev/null +++ b/journal-master/journal/.editorconfig @@ -0,0 +1,5 @@ +[*.cs] +# Prefer expression body for single-line constructors/methods/properties +csharp_style_expression_bodied_constructors = when_on_single_line:suggestion +csharp_style_expression_bodied_methods = when_on_single_line:suggestion +csharp_style_expression_bodied_properties = true:suggestion \ No newline at end of file diff --git a/journal-master/journal/.gitignore b/journal-master/journal/.gitignore new file mode 100644 index 0000000..023d554 --- /dev/null +++ b/journal-master/journal/.gitignore @@ -0,0 +1,40 @@ +# Build output +bin/ +obj/ + +# Visual Studio +.vs/ +*.user +*.suo +*.userosscache +*.sln.docstates + +# Rider +.idea/ +*.sln.iml + +# VS Code +.vscode/ + +# NuGet +*.nupkg +**/packages/ +project.lock.json +project.fragment.lock.json +.nuget/ +.dotnet_home/ +.journal-sidecar/ + +# Publish output +publish/ + +# User secrets +secrets.json + +# Windows +Thumbs.db +desktop.ini + +# macOS +.DS_Store +journal-master\journal\tls_registry_backup_before_fix.txt \ No newline at end of file diff --git a/journal-master/journal/Journal.Api/Journal.Api.csproj b/journal-master/journal/Journal.Api/Journal.Api.csproj new file mode 100644 index 0000000..17d3d94 --- /dev/null +++ b/journal-master/journal/Journal.Api/Journal.Api.csproj @@ -0,0 +1,17 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/journal-master/journal/Journal.Api/Journal.Api.http b/journal-master/journal/Journal.Api/Journal.Api.http new file mode 100644 index 0000000..4ca4b1b --- /dev/null +++ b/journal-master/journal/Journal.Api/Journal.Api.http @@ -0,0 +1,24 @@ +@Journal.Api_HostAddress = http://localhost:5014 + +GET {{Journal.Api_HostAddress}}/health +Accept: application/json + +### + +POST {{Journal.Api_HostAddress}}/api/command +Content-Type: application/json + +{ + "action": "config.get", + "payload": {} +} + +### + +POST {{Journal.Api_HostAddress}}/api/command +Content-Type: application/json + +{ + "action": + +### diff --git a/journal-master/journal/Journal.Api/Program.cs b/journal-master/journal/Journal.Api/Program.cs new file mode 100644 index 0000000..d1f9cae --- /dev/null +++ b/journal-master/journal/Journal.Api/Program.cs @@ -0,0 +1,29 @@ +using System.Text.Json; +using Journal.Core; + +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddOpenApi(); +builder.Services.AddFragmentServices(); +builder.Services.AddSingleton(); + +var app = builder.Build(); + +if (app.Environment.IsDevelopment()) + app.MapOpenApi(); + +app.MapGet("/health", () => Results.Json(new { ok = true, data = "healthy" })); +app.MapGet("/healthz", () => Results.Json(new { ok = true, data = "healthy" })); +app.MapGet("/api/health", () => Results.Json(new { ok = true, data = "healthy" })); + +// Mirrors sidecar transport semantics over HTTP. +// request body is passed directly to Entry.HandleCommandAsync so malformed JSON +// and payload errors return the same {ok:false,error} envelope as sidecar mode. +app.MapPost("/api/command", async (HttpRequest request, Entry entry) => +{ + using var reader = new StreamReader(request.Body); + var body = await reader.ReadToEndAsync(); + var response = await entry.HandleCommandAsync(body); + return Results.Content(response, "application/json"); +}); + +app.Run(); diff --git a/journal-master/journal/Journal.Api/Properties/launchSettings.json b/journal-master/journal/Journal.Api/Properties/launchSettings.json new file mode 100644 index 0000000..76f7662 --- /dev/null +++ b/journal-master/journal/Journal.Api/Properties/launchSettings.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": false, + "applicationUrl": "http://localhost:5014", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": false, + "applicationUrl": "https://localhost:7086;http://localhost:5014", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/journal-master/journal/Journal.Api/appsettings.Development.json b/journal-master/journal/Journal.Api/appsettings.Development.json new file mode 100644 index 0000000..0c208ae --- /dev/null +++ b/journal-master/journal/Journal.Api/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/journal-master/journal/Journal.Api/appsettings.json b/journal-master/journal/Journal.Api/appsettings.json new file mode 100644 index 0000000..10f68b8 --- /dev/null +++ b/journal-master/journal/Journal.Api/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} diff --git a/journal-master/journal/Journal.Core/Dtos/AiDtos.cs b/journal-master/journal/Journal.Core/Dtos/AiDtos.cs new file mode 100644 index 0000000..964498e --- /dev/null +++ b/journal-master/journal/Journal.Core/Dtos/AiDtos.cs @@ -0,0 +1,7 @@ +namespace Journal.Core.Dtos; + +public sealed record AiHealthDto( + string Provider, + bool Enabled, + bool Healthy, + string Message); diff --git a/journal-master/journal/Journal.Core/Dtos/EntrySearchDtos.cs b/journal-master/journal/Journal.Core/Dtos/EntrySearchDtos.cs new file mode 100644 index 0000000..2ac363b --- /dev/null +++ b/journal-master/journal/Journal.Core/Dtos/EntrySearchDtos.cs @@ -0,0 +1,17 @@ +namespace Journal.Core.Dtos; + +public sealed record EntrySearchRequestDto( + string DataDirectory, + string? Query = null, + string? Section = null, + string? StartDate = null, + string? EndDate = null, + IReadOnlyList? Tags = null, + IReadOnlyList? Types = null, + IReadOnlyList? Checked = null, + IReadOnlyList? Unchecked = null); + +public sealed record EntrySearchResultDto( + string Date, + string FileName, + string RawContent); diff --git a/journal-master/journal/Journal.Core/Dtos/FragmentDtos.cs b/journal-master/journal/Journal.Core/Dtos/FragmentDtos.cs new file mode 100644 index 0000000..aace939 --- /dev/null +++ b/journal-master/journal/Journal.Core/Dtos/FragmentDtos.cs @@ -0,0 +1,24 @@ +using System.ComponentModel.DataAnnotations; + +namespace Journal.Core.Dtos; + +public record FragmentDto( + Guid Id, + string Type, + string Description, + DateTimeOffset Time, + List Tags +); + +public record CreateFragmentDto( + [property: Required(AllowEmptyStrings = false)] string Type, + [property: Required(AllowEmptyStrings = false)] string Description, + List? Tags = null +); + +public record UpdateFragmentDto( + string? Type = null, + string? Description = null, + List? Tags = null, + DateTimeOffset? Time = null +); diff --git a/journal-master/journal/Journal.Core/Dtos/SpeechDtos.cs b/journal-master/journal/Journal.Core/Dtos/SpeechDtos.cs new file mode 100644 index 0000000..aa3d1c9 --- /dev/null +++ b/journal-master/journal/Journal.Core/Dtos/SpeechDtos.cs @@ -0,0 +1,21 @@ +namespace Journal.Core.Dtos; + +public sealed record SpeechDeviceDto( + int Index, + string Name); + +public sealed record SpeechDevicesResultDto( + IReadOnlyList Devices, + string? Warning = null); + +public sealed record SpeechTranscribeRequestDto( + string? AudioBase64 = null, + string? Engine = null, + string? WhisperModel = null, + string? Text = null, + int? SimulateDelayMs = null); + +public sealed record SpeechTranscribeResultDto( + string Text, + string Engine, + string? Warning = null); diff --git a/journal-master/journal/Journal.Core/Entry.cs b/journal-master/journal/Journal.Core/Entry.cs new file mode 100644 index 0000000..39627f5 --- /dev/null +++ b/journal-master/journal/Journal.Core/Entry.cs @@ -0,0 +1,547 @@ +using System.ComponentModel.DataAnnotations; +using System.Globalization; +using System.Net; +using System.Text.RegularExpressions; +using System.Text.Json; +using Journal.Core.Dtos; +using Journal.Core.Models; +using Journal.Core.Services; + +namespace Journal.Core; + +public class Entry +{ + private readonly IFragmentService _fragments; + private readonly IEntrySearchService _entrySearch; + private readonly IVaultStorageService _vaultStorage; + private readonly IJournalDatabaseService _database; + private readonly IJournalConfigService _config; + private readonly IAiService _ai; + private readonly ISpeechBridgeService _speech; + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + public Entry( + IFragmentService fragments, + IEntrySearchService entrySearch, + IVaultStorageService vaultStorage, + IJournalDatabaseService database, + IJournalConfigService config, + IAiService ai, + ISpeechBridgeService speech) + { + _fragments = fragments; + _entrySearch = entrySearch; + _vaultStorage = vaultStorage; + _database = database; + _config = config; + _ai = ai; + _speech = speech; + } + + public async Task RunAsync() + { + string? line; + while ((line = Console.ReadLine()) is not null) + { + var response = await HandleCommandAsync(line); + Console.WriteLine(response); + } + } + + public async Task HandleCommandAsync(string json) + { + if (string.IsNullOrWhiteSpace(json)) + return Error("Invalid command"); + + Command? cmd; + try + { + cmd = JsonSerializer.Deserialize(json, JsonOptions); + } + catch (JsonException) + { + return Error("Invalid command JSON"); + } + + if (cmd is null || string.IsNullOrWhiteSpace(cmd.Action)) + return Error("Invalid command"); + + var action = cmd.Action.Trim(); + var correlationId = string.IsNullOrWhiteSpace(cmd.CorrelationId) + ? Guid.NewGuid().ToString("N") + : cmd.CorrelationId.Trim(); + LogStart(action, correlationId, cmd.Payload); + object? result; + + try + { + switch (action) + { + case "fragments.list": + result = await _fragments.GetAllAsync(); + break; + case "fragments.get": + if (!Guid.TryParse(cmd.Id, out var getId)) + return Error("Invalid or missing id"); + result = await _fragments.GetByIdAsync(getId); + break; + case "fragments.create": + var createDto = DeserializePayload(cmd.Payload); + if (createDto is null) + return Error("Missing or invalid payload"); + result = await _fragments.CreateAsync(createDto); + break; + case "fragments.update": + if (!Guid.TryParse(cmd.Id, out var updateId)) + return Error("Invalid or missing id"); + var updateDto = DeserializePayload(cmd.Payload); + if (updateDto is null) + return Error("Missing or invalid payload"); + result = await _fragments.UpdateAsync(updateId, updateDto); + break; + case "fragments.delete": + if (!Guid.TryParse(cmd.Id, out var deleteId)) + return Error("Invalid or missing id"); + result = await _fragments.RemoveAsync(deleteId); + break; + case "fragments.search": + result = await _fragments.SearchAsync(cmd.Type, cmd.Tag); + break; + case "search.entries": + var searchPayload = DeserializePayload(cmd.Payload); + if (searchPayload is null || string.IsNullOrWhiteSpace(searchPayload.DataDirectory)) + return Error("Missing or invalid payload"); + var searchRequest = new EntrySearchRequestDto( + DataDirectory: searchPayload.DataDirectory, + Query: searchPayload.Query, + Section: searchPayload.Section, + StartDate: searchPayload.StartDate, + EndDate: searchPayload.EndDate, + Tags: searchPayload.Tags, + Types: searchPayload.Types, + Checked: searchPayload.Checked, + Unchecked: searchPayload.Unchecked); + result = await _entrySearch.SearchEntriesAsync(searchRequest); + break; + case "entries.list": + var listPayload = DeserializePayload(cmd.Payload); + var listDataDirectory = !string.IsNullOrWhiteSpace(listPayload?.DataDirectory) + ? listPayload.DataDirectory + : _config.Current.DataDirectory; + result = ListEntries(listDataDirectory); + break; + case "entries.load": + var loadEntryPayload = DeserializePayload(cmd.Payload); + if (loadEntryPayload is null || string.IsNullOrWhiteSpace(loadEntryPayload.FilePath)) + return Error("Missing or invalid payload"); + result = LoadEntry(loadEntryPayload.FilePath); + break; + case "entries.save": + var saveEntryPayload = DeserializePayload(cmd.Payload); + if (saveEntryPayload is null || string.IsNullOrWhiteSpace(saveEntryPayload.Content)) + return Error("Missing or invalid payload"); + result = SaveEntry(saveEntryPayload, _config.Current.DataDirectory); + break; + case "config.get": + result = _config.Current; + break; + case "ai.health": + result = await _ai.HealthAsync(); + break; + case "ai.summarize_entry": + var summarizeEntryPayload = DeserializePayload(cmd.Payload); + if (summarizeEntryPayload is null || string.IsNullOrWhiteSpace(summarizeEntryPayload.Content)) + return Error("Missing or invalid payload"); + result = await _ai.SummarizeEntryAsync(summarizeEntryPayload.Content, summarizeEntryPayload.FileStem); + break; + case "ai.summarize_all": + var summarizeAllPayload = DeserializePayload(cmd.Payload); + if (summarizeAllPayload is null) + return Error("Missing or invalid payload"); + result = await _ai.SummarizeAllAsync(summarizeAllPayload.Entries ?? []); + break; + case "ai.chat": + var chatPayload = DeserializePayload(cmd.Payload); + if (chatPayload is null || string.IsNullOrWhiteSpace(chatPayload.Prompt)) + return Error("Missing or invalid payload"); + result = await _ai.ChatAsync(chatPayload.Prompt); + break; + case "ai.embed": + var embedPayload = DeserializePayload(cmd.Payload); + if (embedPayload is null || string.IsNullOrWhiteSpace(embedPayload.Content)) + return Error("Missing or invalid payload"); + result = await _ai.EmbedAsync(embedPayload.Content); + break; + case "speech.devices.list": + result = await _speech.ListDevicesAsync(); + break; + case "speech.transcribe": + var speechPayload = DeserializePayload(cmd.Payload); + if (speechPayload is null) + return Error("Missing or invalid payload"); + var audioBase64 = !string.IsNullOrWhiteSpace(speechPayload.AudioBase64) + ? speechPayload.AudioBase64 + : speechPayload.Audio_Base64; + var text = speechPayload.Text; + var whisperModel = !string.IsNullOrWhiteSpace(speechPayload.WhisperModel) + ? speechPayload.WhisperModel + : speechPayload.Whisper_Model; + var simulateDelayMs = speechPayload.SimulateDelayMs ?? speechPayload.Simulate_Delay_Ms; + if (string.IsNullOrWhiteSpace(audioBase64) && string.IsNullOrWhiteSpace(text)) + return Error("Missing or invalid payload"); + result = await _speech.TranscribeAsync(new SpeechTranscribeRequestDto( + AudioBase64: audioBase64, + Engine: speechPayload.Engine, + WhisperModel: whisperModel, + Text: text, + SimulateDelayMs: simulateDelayMs)); + break; + case "vault.initialize": + var initPayload = DeserializePayload(cmd.Payload); + if (initPayload is null || string.IsNullOrWhiteSpace(initPayload.Password) || string.IsNullOrWhiteSpace(initPayload.VaultDirectory)) + return Error("Missing or invalid payload"); + Directory.CreateDirectory(initPayload.VaultDirectory); + result = true; + break; + case "vault.load_all": + var loadPayload = DeserializePayload(cmd.Payload); + if (loadPayload is null) + return Error("Missing or invalid payload"); + result = _vaultStorage.LoadAllVaults(loadPayload.Password, loadPayload.VaultDirectory, loadPayload.DataDirectory); + break; + case "vault.save_current_month": + var saveCurrentPayload = DeserializePayload(cmd.Payload); + if (saveCurrentPayload is null) + return Error("Missing or invalid payload"); + result = _vaultStorage.SaveCurrentMonthVault( + saveCurrentPayload.Password, + saveCurrentPayload.VaultDirectory, + saveCurrentPayload.DataDirectory, + ParseNowOrDefault(saveCurrentPayload.NowUtc)); + break; + case "vault.rebuild_all": + var rebuildPayload = DeserializePayload(cmd.Payload); + if (rebuildPayload is null) + return Error("Missing or invalid payload"); + _vaultStorage.RebuildAllVaults(rebuildPayload.Password, rebuildPayload.VaultDirectory, rebuildPayload.DataDirectory); + result = true; + break; + case "vault.clear_data_directory": + var clearPayload = DeserializePayload(cmd.Payload); + if (clearPayload is null || string.IsNullOrWhiteSpace(clearPayload.DataDirectory)) + return Error("Missing or invalid payload"); + _vaultStorage.ClearDataDirectory(clearPayload.DataDirectory); + result = true; + break; + case "db.status": + var dbStatusPayload = DeserializePayload(cmd.Payload); + if (dbStatusPayload is null || string.IsNullOrWhiteSpace(dbStatusPayload.Password)) + return Error("Missing or invalid payload"); + result = _database.GetStatus(dbStatusPayload.Password, dbStatusPayload.DataDirectory); + break; + case "db.initialize_schema": + var dbInitPayload = DeserializePayload(cmd.Payload); + if (dbInitPayload is null) + return Error("Missing or invalid payload"); + var schemaPath = _database.WriteSchemaBootstrap(dbInitPayload.DataDirectory); + result = new { schemaPath }; + break; + case "db.hydrate_workspace": + var dbHydratePayload = DeserializePayload(cmd.Payload); + if (dbHydratePayload is null || string.IsNullOrWhiteSpace(dbHydratePayload.Password)) + return Error("Missing or invalid payload"); + result = _database.HydrateWorkspace(dbHydratePayload.Password, dbHydratePayload.DataDirectory); + break; + default: + LogFailure(action, correlationId, "unknown_action"); + return Error($"Unknown action: {action}"); + } + } + catch (JsonException) + { + LogFailure(action, correlationId, "invalid_payload_json"); + return Error("Missing or invalid payload"); + } + catch (ValidationException ex) + { + LogFailure(action, correlationId, "validation", ex.Message); + return Error(ex.Message); + } + catch (ArgumentException ex) + { + LogFailure(action, correlationId, "argument", ex.Message); + return Error(ex.Message); + } + catch (TimeoutException ex) + { + LogFailure(action, correlationId, "timeout", ex.Message); + return Error(ex.Message); + } + catch (InvalidOperationException ex) + { + LogFailure(action, correlationId, "invalid_operation", ex.Message); + return Error(ex.Message); + } + catch (FileNotFoundException ex) + { + LogFailure(action, correlationId, "not_found", ex.Message); + return Error(ex.Message); + } + catch + { + LogFailure(action, correlationId, "internal_error"); + return Error("Internal error"); + } + + LogSuccess(action, correlationId); + return JsonSerializer.Serialize(new { ok = true, data = result }); + } + + private static string Error(string message) + => JsonSerializer.Serialize(new { ok = false, error = message }); + + private void LogStart(string action, string correlationId, JsonElement? payload) + { + var redactedPayload = LogRedactor.RedactPayload(payload); + EmitLog("information", action, correlationId, "start", redactedPayload); + } + + private void LogSuccess(string action, string correlationId) + { + EmitLog("information", action, correlationId, "success"); + } + + private void LogFailure(string action, string correlationId, string errorType, string? message = null) + { + var details = string.IsNullOrWhiteSpace(message) + ? "" + : (message!.Length <= 160 ? message : $"{message[..160]}...(truncated)"); + EmitLog("warning", action, correlationId, "failure", errorType: errorType, details: details); + } + + private static void EmitLog( + string level, + string action, + string correlationId, + string outcome, + object? payload = null, + string? errorType = null, + string? details = null) + { + if (!ShouldLog(level)) + return; + + var envelope = new + { + timestamp = DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture), + level, + component = nameof(Entry), + action, + correlation_id = correlationId, + outcome, + error_type = errorType, + details, + payload + }; + Console.Error.WriteLine(JsonSerializer.Serialize(envelope)); + } + + private static bool ShouldLog(string level) + { + var configured = (Environment.GetEnvironmentVariable("JOURNAL_LOG_LEVEL") ?? "warning") + .Trim() + .ToLowerInvariant(); + var configuredRank = LogLevelRank(configured); + var incomingRank = LogLevelRank(level); + return incomingRank >= configuredRank; + } + + private static int LogLevelRank(string level) => level switch + { + "trace" => 0, + "debug" => 1, + "information" => 2, + "info" => 2, + "warning" => 3, + "warn" => 3, + "error" => 4, + "critical" => 5, + _ => 3 + }; + + private static T? DeserializePayload(JsonElement? payload) + { + if (payload is null) + return default; + return payload.Value.Deserialize(JsonOptions); + } + + private static DateTime ParseNowOrDefault(string? nowUtc) + { + if (string.IsNullOrWhiteSpace(nowUtc)) + return DateTime.UtcNow; + + if (DateTime.TryParse( + nowUtc, + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var parsed)) + { + return parsed; + } + + throw new ArgumentException("Invalid nowUtc value. Expected ISO date/time."); + } + + private static IReadOnlyList ListEntries(string dataDirectory) + { + if (!Directory.Exists(dataDirectory)) + return []; + + return Directory.GetFiles(dataDirectory, "*.md") + .OrderBy(Path.GetFileName, StringComparer.Ordinal) + .Select(path => new EntryListItem( + FileName: Path.GetFileName(path), + FilePath: Path.GetFullPath(path))) + .ToArray(); + } + + private static EntryLoadResult LoadEntry(string filePath) + { + var normalizedPath = Path.GetFullPath(filePath); + if (!File.Exists(normalizedPath)) + throw new FileNotFoundException($"Entry file not found: {normalizedPath}"); + + var rawContent = StripRichHtml(File.ReadAllText(normalizedPath)); + var fileStem = Path.GetFileNameWithoutExtension(normalizedPath); + var entry = JournalParser.ParseJournalContent(rawContent, fileStem); + + return new EntryLoadResult( + Date: entry.Date, + FileName: Path.GetFileName(normalizedPath), + FilePath: normalizedPath, + RawContent: entry.RawContent); + } + + private static EntrySaveResult SaveEntry(EntrySavePayload payload, string defaultDataDirectory) + { + var targetPath = ResolveTargetPath(payload.FilePath, defaultDataDirectory); + var mode = string.IsNullOrWhiteSpace(payload.Mode) ? "Daily" : payload.Mode.Trim(); + var sanitizedContent = StripRichHtml(payload.Content ?? ""); + Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); + + if (string.Equals(mode, "Overwrite", StringComparison.OrdinalIgnoreCase)) + { + File.WriteAllText(targetPath, sanitizedContent); + return new EntrySaveResult(targetPath); + } + + if (string.Equals(mode, "Fragment", StringComparison.OrdinalIgnoreCase)) + { + File.AppendAllText(targetPath, "\n\n" + sanitizedContent.Trim()); + return new EntrySaveResult(targetPath); + } + + string finalContent; + if (File.Exists(targetPath)) + { + var existingContent = File.ReadAllText(targetPath); + var fileStem = Path.GetFileNameWithoutExtension(targetPath); + var existingEntry = JournalParser.ParseJournalContent(existingContent, fileStem); + var newEntryData = JournalParser.ParseJournalContent(sanitizedContent, fileStem); + existingEntry.MergeWith(newEntryData); + finalContent = existingEntry.ToMarkdown(); + } + else + { + finalContent = sanitizedContent; + } + + File.WriteAllText(targetPath, finalContent); + return new EntrySaveResult(targetPath); + } + + private static string ResolveTargetPath(string? filePath, string defaultDataDirectory) + { + if (!string.IsNullOrWhiteSpace(filePath)) + return Path.GetFullPath(filePath); + + return Path.GetFullPath(Path.Combine(defaultDataDirectory, $"{DateTime.Now:yyyy-MM-dd}.md")); + } + + private static bool LooksLikeRichHtml(string content) + { + var lowered = content.ToLowerInvariant(); + string[] markers = + [ + "", " lowered.Contains(marker, StringComparison.Ordinal))) + return true; + return Regex.Matches(lowered, "]*>").Count >= 8; + } + + private static string StripRichHtml(string content) + { + if (string.IsNullOrWhiteSpace(content)) + return content; + if (!LooksLikeRichHtml(content)) + return content; + + var text = content.Replace("\r\n", "\n").Replace("\r", "\n"); + text = Regex.Replace(text, "<(script|style)\\b[^>]*>.*?", "", RegexOptions.IgnoreCase | RegexOptions.Singleline); + text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "]*>", "\n- ", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "", "\n", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "<(td|th)\\b[^>]*>", " | ", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "", " ", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "]*>", "\n---\n", RegexOptions.IgnoreCase); + text = Regex.Replace(text, "<[^>]+>", "", RegexOptions.Singleline); + text = WebUtility.HtmlDecode(text) + .Replace('\u00a0', ' ') + .Replace("\u200b", "", StringComparison.Ordinal); + text = string.Join("\n", text.Split('\n').Select(line => line.TrimEnd())); + text = Regex.Replace(text, "[ \\t]{2,}", " "); + text = Regex.Replace(text, "\n{3,}", "\n\n").Trim(); + return string.IsNullOrEmpty(text) ? content : text; + } + + private sealed record VaultInitializePayload(string Password, string VaultDirectory); + private sealed record VaultPayload(string Password, string VaultDirectory, string DataDirectory, string? NowUtc = null); + private sealed record ClearDataPayload(string DataDirectory); + private sealed record EntryListPayload(string? DataDirectory = null); + private sealed record EntryLoadPayload(string FilePath); + private sealed record EntrySavePayload(string Content, string? FilePath = null, string? Mode = null); + private sealed record EntryListItem(string FileName, string FilePath); + private sealed record EntryLoadResult(string Date, string FileName, string FilePath, string RawContent); + private sealed record EntrySaveResult(string FilePath); + private sealed record DatabasePayload(string Password, string? DataDirectory = null); + private sealed record AiSummarizeEntryPayload(string Content, string? FileStem = null); + private sealed record AiSummarizeAllPayload(List? Entries); + private sealed record AiChatPayload(string Prompt); + private sealed record AiEmbedPayload(string Content); + private sealed record SpeechTranscribePayload( + string? AudioBase64 = null, + string? Audio_Base64 = null, + string? Engine = null, + string? WhisperModel = null, + string? Whisper_Model = null, + string? Text = null, + int? SimulateDelayMs = null, + int? Simulate_Delay_Ms = null); + private sealed record SearchEntriesPayload( + string DataDirectory, + string? Query = null, + string? Section = null, + string? StartDate = null, + string? EndDate = null, + List? Tags = null, + List? Types = null, + List? Checked = null, + List? Unchecked = null); +} diff --git a/journal-master/journal/Journal.Core/Journal.Core.csproj b/journal-master/journal/Journal.Core/Journal.Core.csproj new file mode 100644 index 0000000..7e67750 --- /dev/null +++ b/journal-master/journal/Journal.Core/Journal.Core.csproj @@ -0,0 +1,15 @@ + + + + net10.0 + enable + enable + + + + + + + + + diff --git a/journal-master/journal/Journal.Core/Models/Command.cs b/journal-master/journal/Journal.Core/Models/Command.cs new file mode 100644 index 0000000..ac44027 --- /dev/null +++ b/journal-master/journal/Journal.Core/Models/Command.cs @@ -0,0 +1,13 @@ +using System.Text.Json; + +namespace Journal.Core.Models; + +public class Command +{ + public string Action { get; set; } = ""; + public string? CorrelationId { get; set; } + public string? Id { get; set; } + public string? Type { get; set; } + public string? Tag { get; set; } + public JsonElement? Payload { get; set; } +} diff --git a/journal-master/journal/Journal.Core/Models/Fragment.cs b/journal-master/journal/Journal.Core/Models/Fragment.cs new file mode 100644 index 0000000..6b67db7 --- /dev/null +++ b/journal-master/journal/Journal.Core/Models/Fragment.cs @@ -0,0 +1,42 @@ +namespace Journal.Core.Models; + +public class Fragment +{ + public Guid Id { get; } + public string Type { get; set; } + public string Description { get; set; } + public DateTimeOffset Time { get; set; } + public List Tags { get; set; } = []; + + public Fragment(string type, string description) + { + Validate(type, description); + + Id = Guid.NewGuid(); + Type = type.Trim(); + Description = description.Trim(); + Time = DateTimeOffset.Now; + } + + public Fragment(Guid id, string type, string description, DateTimeOffset time, IEnumerable? tags = null) + { + if (id == Guid.Empty) + throw new ArgumentException("Id is required", nameof(id)); + Validate(type, description); + + Id = id; + Type = type.Trim(); + Description = description.Trim(); + Time = time; + if (tags is not null) + Tags = [.. tags.Where(t => !string.IsNullOrWhiteSpace(t)).Select(t => t.Trim())]; + } + + private static void Validate(string type, string description) + { + if (string.IsNullOrWhiteSpace(type)) + throw new ArgumentException("Type is required", nameof(type)); + if (string.IsNullOrWhiteSpace(description)) + throw new ArgumentException("Description is required", nameof(description)); + } +} diff --git a/journal-master/journal/Journal.Core/Models/JournalConfig.cs b/journal-master/journal/Journal.Core/Models/JournalConfig.cs new file mode 100644 index 0000000..f540a41 --- /dev/null +++ b/journal-master/journal/Journal.Core/Models/JournalConfig.cs @@ -0,0 +1,29 @@ +namespace Journal.Core.Models; + +public sealed record JournalConfig( + string ProjectRoot, + string AppDirectory, + string DataDirectory, + string VaultDirectory, + string LogDirectory, + string PidFile, + string ServerControlFile, + string DatabaseFilename, + string MonthlyVaultFormat, + string CloudAiApiKey, + string CloudAiApiUrl, + string LlamaCppUrl, + string LlamaCppModel, + int LlamaCppTimeout, + string EmbeddingApiUrl, + string EmbeddingModelName, + int ModelContextTokens, + int ChunkTokenBudget, + int? MicrophoneDeviceIndex, + string SpeechRecognitionEngine, + string WhisperModelSize, + string NlpBackend, + string AiProvider, + string PythonExecutable, + string PythonAiSidecarPath, + int AiSidecarTimeoutMs); diff --git a/journal-master/journal/Journal.Core/Models/JournalEntry.cs b/journal-master/journal/Journal.Core/Models/JournalEntry.cs new file mode 100644 index 0000000..e647c55 --- /dev/null +++ b/journal-master/journal/Journal.Core/Models/JournalEntry.cs @@ -0,0 +1,98 @@ +namespace Journal.Core.Models; + +public class JournalEntry +{ + public string Date { get; set; } + public List Fragments { get; set; } + public string RawContent { get; set; } + public Dictionary Sections { get; set; } + + public JournalEntry( + string date, + IEnumerable? fragments = null, + string rawContent = "", + IDictionary? sections = null) + { + if (string.IsNullOrWhiteSpace(date)) + throw new ArgumentException("Date is required", nameof(date)); + + Date = date.Trim(); + Fragments = fragments is null ? [] : [.. fragments]; + RawContent = rawContent ?? ""; + Sections = sections is null ? [] : new Dictionary(sections); + } + + public string GetSection(string sectionTitle) + { + if (string.IsNullOrWhiteSpace(sectionTitle)) + return ""; + if (!Sections.TryGetValue(sectionTitle, out var section)) + return ""; + return string.Join("\n", section.Content); + } + + public bool? GetCheckboxState(string sectionTitle, string checkboxText) + { + if (string.IsNullOrWhiteSpace(sectionTitle) || string.IsNullOrWhiteSpace(checkboxText)) + return null; + if (!Sections.TryGetValue(sectionTitle, out var section)) + return null; + return section.Checkboxes.TryGetValue(checkboxText, out var checkedState) ? checkedState : null; + } + + public void MergeWith(JournalEntry otherEntry) + { + ArgumentNullException.ThrowIfNull(otherEntry); + + foreach (var (title, newSection) in otherEntry.Sections) + { + if (newSection.Content.Any(line => !string.IsNullOrWhiteSpace(line))) + Sections[title] = newSection; + } + + var existingFragmentDescriptions = Fragments + .Select(fragment => fragment.Description) + .ToHashSet(StringComparer.Ordinal); + + foreach (var newFragment in otherEntry.Fragments) + { + if (!existingFragmentDescriptions.Contains(newFragment.Description)) + Fragments.Add(newFragment); + } + } + + public string ToMarkdown() + { + var lines = new List + { + "---", + "type: journal", + "---", + $"**Date:** {Date}\n" + }; + + foreach (var title in SectionTitles.Canonical) + { + if (!Sections.TryGetValue(title, out var section)) + continue; + + lines.Add($"## {section.Title}\n"); + lines.AddRange(section.Content); + lines.Add(""); + } + + if (Fragments.Count > 0) + { + lines.Add("# Fragments\n"); + foreach (var fragment in Fragments) + { + var timeStr = fragment.Time != default ? $"@{fragment.Time:O}" : ""; + var tagsStr = string.Join(" ", fragment.Tags.Select(tag => $"#{tag}")); + var header = $"{fragment.Type} {timeStr} {tagsStr}".Trim(); + lines.Add($"{header}\n{fragment.Description}\n"); + } + } + + return string.Join("\n", lines); + } +} diff --git a/journal-master/journal/Journal.Core/Models/ParsedSection.cs b/journal-master/journal/Journal.Core/Models/ParsedSection.cs new file mode 100644 index 0000000..bc9ff3c --- /dev/null +++ b/journal-master/journal/Journal.Core/Models/ParsedSection.cs @@ -0,0 +1,21 @@ +namespace Journal.Core.Models; + +public class ParsedSection +{ + public string Title { get; set; } + public List Content { get; set; } + public Dictionary Checkboxes { get; set; } + + public ParsedSection( + string title, + IEnumerable? content = null, + IDictionary? checkboxes = null) + { + if (string.IsNullOrWhiteSpace(title)) + throw new ArgumentException("Section title is required", nameof(title)); + + Title = title.Trim(); + Content = content is null ? [] : [.. content]; + Checkboxes = checkboxes is null ? [] : new Dictionary(checkboxes); + } +} diff --git a/journal-master/journal/Journal.Core/Models/SectionTitles.cs b/journal-master/journal/Journal.Core/Models/SectionTitles.cs new file mode 100644 index 0000000..3aaf666 --- /dev/null +++ b/journal-master/journal/Journal.Core/Models/SectionTitles.cs @@ -0,0 +1,20 @@ +namespace Journal.Core.Models; + +public static class SectionTitles +{ + public static readonly IReadOnlyList Canonical = + [ + "Summary", + "Cognitive State", + "Mental / Emotional Snapshot", + "Memory / Mind Failures", + "Events / Triggers", + "Communication / Expression Log", + "Coping / Tools Used", + "Reflection", + "Core Events or Memories", + "Autism/ADHD-Related Elements", + "Emotional & Bodily Reactions", + "Truth to Anchor Myself To", + ]; +} diff --git a/journal-master/journal/Journal.Core/Repositories/FileFragmentRepository.cs b/journal-master/journal/Journal.Core/Repositories/FileFragmentRepository.cs new file mode 100644 index 0000000..71e107d --- /dev/null +++ b/journal-master/journal/Journal.Core/Repositories/FileFragmentRepository.cs @@ -0,0 +1,228 @@ +using System.Text.Json; +using Journal.Core.Models; + +namespace Journal.Core.Repositories; + +public class FileFragmentRepository : IFragmentRepository +{ + private readonly Lock _lock = new(); + private readonly string _storagePath; + private readonly JsonSerializerOptions _jsonOptions = new() + { + WriteIndented = true + }; + private readonly List _store; + + public FileFragmentRepository() : this(storagePath: null) + { + } + + public FileFragmentRepository(string? storagePath) + { + _storagePath = ResolveStoragePath(storagePath); + _store = LoadStore(_storagePath); + } + + public Task> GetAllAsync() + { + lock (_lock) + { + return Task.FromResult(_store.ToList()); + } + } + + public Task GetByIdAsync(Guid id) + { + lock (_lock) + { + return Task.FromResult(_store.FirstOrDefault(f => f.Id == id)); + } + } + + public Task AddAsync(Fragment fragment) + { + ArgumentNullException.ThrowIfNull(fragment); + lock (_lock) + { + Normalize(fragment); + _store.Add(fragment); + SaveStoreLocked(); + } + return Task.CompletedTask; + } + + public Task RemoveAsync(Guid id) + { + lock (_lock) + { + var item = _store.FirstOrDefault(f => f.Id == id); + if (item is null) + return Task.FromResult(false); + + var removed = _store.Remove(item); + if (removed) + SaveStoreLocked(); + return Task.FromResult(removed); + } + } + + public Task UpdateAsync( + Guid id, + string? type = null, + string? description = null, + IEnumerable? tags = null, + DateTimeOffset? time = null) + { + lock (_lock) + { + var item = _store.FirstOrDefault(f => f.Id == id); + if (item is null) + return Task.FromResult(false); + + if (type != null) + { + if (string.IsNullOrWhiteSpace(type)) + throw new ArgumentException("Type cannot be empty", nameof(type)); + item.Type = type.Trim(); + } + + if (description != null) + { + if (string.IsNullOrWhiteSpace(description)) + throw new ArgumentException("Description cannot be empty", nameof(description)); + item.Description = description.Trim(); + } + + if (tags != null) + { + item.Tags = [.. + tags.Where(t => !string.IsNullOrWhiteSpace(t)) + .Select(t => t.Trim())]; + } + + if (time.HasValue) + item.Time = time.Value; + + SaveStoreLocked(); + return Task.FromResult(true); + } + } + + public Task> GetByTagAsync(string tag) + { + var q = tag?.Trim(); + if (string.IsNullOrWhiteSpace(q)) + return Task.FromResult(new List()); + + lock (_lock) + { + var items = _store + .Where(f => f.Tags.Any(t => t.Contains(q, StringComparison.OrdinalIgnoreCase))) + .ToList(); + return Task.FromResult(items); + } + } + + public Task> GetByTypeAsync(string type) + { + var q = type?.Trim(); + if (string.IsNullOrWhiteSpace(q)) + return Task.FromResult(new List()); + + lock (_lock) + { + var items = _store + .Where(f => f.Type.Contains(q, StringComparison.OrdinalIgnoreCase)) + .ToList(); + return Task.FromResult(items); + } + } + + public Task> SearchAsync(string? type = null, string? tag = null, DateTimeOffset? timeAfter = null) + { + var qType = type?.Trim(); + var qTag = tag?.Trim(); + + lock (_lock) + { + IEnumerable results = _store; + + if (!string.IsNullOrWhiteSpace(qType)) + results = results.Where(f => f.Type.Contains(qType, StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrWhiteSpace(qTag)) + results = results.Where(f => f.Tags.Any(t => t.Contains(qTag, StringComparison.OrdinalIgnoreCase))); + if (timeAfter.HasValue) + results = results.Where(f => f.Time > timeAfter.Value); + + return Task.FromResult(results.ToList()); + } + } + + private static string ResolveStoragePath(string? storagePath) + { + var configured = storagePath; + if (string.IsNullOrWhiteSpace(configured)) + configured = Environment.GetEnvironmentVariable("JOURNAL_FRAGMENT_STORE_PATH"); + if (string.IsNullOrWhiteSpace(configured)) + configured = Path.Combine(Environment.CurrentDirectory, ".journal-sidecar", "fragments.json"); + + return Path.GetFullPath(configured); + } + + private List LoadStore(string path) + { + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrWhiteSpace(directory)) + Directory.CreateDirectory(directory); + + if (!File.Exists(path)) + return []; + + var json = File.ReadAllText(path); + if (string.IsNullOrWhiteSpace(json)) + return []; + + var docs = JsonSerializer.Deserialize>(json, _jsonOptions) ?? []; + return docs.Select(d => new Fragment(d.Id, d.Type, d.Description, d.Time, d.Tags)).ToList(); + } + + private void SaveStoreLocked() + { + var directory = Path.GetDirectoryName(_storagePath); + if (!string.IsNullOrWhiteSpace(directory)) + Directory.CreateDirectory(directory); + + var docs = _store.Select(f => new FragmentDocument + { + Id = f.Id, + Type = f.Type, + Description = f.Description, + Time = f.Time, + Tags = [.. f.Tags] + }).ToList(); + var json = JsonSerializer.Serialize(docs, _jsonOptions); + + var tempPath = _storagePath + ".tmp"; + File.WriteAllText(tempPath, json); + File.Copy(tempPath, _storagePath, overwrite: true); + File.Delete(tempPath); + } + + private static void Normalize(Fragment fragment) + { + fragment.Type = fragment.Type.Trim(); + fragment.Description = fragment.Description.Trim(); + fragment.Tags = [.. + fragment.Tags.Where(t => !string.IsNullOrWhiteSpace(t)) + .Select(t => t.Trim())]; + } + + private sealed class FragmentDocument + { + public Guid Id { get; init; } + public string Type { get; init; } = ""; + public string Description { get; init; } = ""; + public DateTimeOffset Time { get; init; } + public List Tags { get; init; } = []; + } +} diff --git a/journal-master/journal/Journal.Core/Repositories/IFragmentRepository.cs b/journal-master/journal/Journal.Core/Repositories/IFragmentRepository.cs new file mode 100644 index 0000000..54011c1 --- /dev/null +++ b/journal-master/journal/Journal.Core/Repositories/IFragmentRepository.cs @@ -0,0 +1,15 @@ +using Journal.Core.Models; + +namespace Journal.Core.Repositories; + +public interface IFragmentRepository +{ + Task> GetAllAsync(); + Task GetByIdAsync(Guid id); + Task AddAsync(Fragment fragment); + Task RemoveAsync(Guid id); + Task UpdateAsync(Guid id, string? type = null, string? description = null, IEnumerable? tags = null, DateTimeOffset? time = null); + Task> GetByTagAsync(string tag); + Task> GetByTypeAsync(string type); + Task> SearchAsync(string? type = null, string? tag = null, DateTimeOffset? timeAfter = null); +} diff --git a/journal-master/journal/Journal.Core/Repositories/InMemoryFragmentRepository.cs b/journal-master/journal/Journal.Core/Repositories/InMemoryFragmentRepository.cs new file mode 100644 index 0000000..d283645 --- /dev/null +++ b/journal-master/journal/Journal.Core/Repositories/InMemoryFragmentRepository.cs @@ -0,0 +1,126 @@ +using Journal.Core.Models; + +namespace Journal.Core.Repositories; + +public class InMemoryFragmentRepository : IFragmentRepository +{ + private readonly List _store = []; + private readonly Lock _lock = new(); + + public Task> GetAllAsync() + { + lock (_lock) + { + return Task.FromResult(_store.ToList()); + } + } + + public Task GetByIdAsync(Guid id) + { + lock (_lock) + { + return Task.FromResult(_store.FirstOrDefault(f => f.Id == id)); + } + } + + public Task AddAsync(Fragment fragment) + { + if (fragment is null) throw new ArgumentNullException(nameof(fragment)); + lock (_lock) + { + if (fragment.Tags != null) + { + fragment.Tags = [.. fragment.Tags + .Where(t => !string.IsNullOrWhiteSpace(t)) + .Select(t => t!.Trim())]; + } + if (!string.IsNullOrWhiteSpace(fragment.Type)) fragment.Type = fragment.Type.Trim(); + if (!string.IsNullOrWhiteSpace(fragment.Description)) fragment.Description = fragment.Description.Trim(); + + _store.Add(fragment); + } + return Task.CompletedTask; + } + + public Task RemoveAsync(Guid id) + { + lock (_lock) + { + var item = _store.FirstOrDefault(f => f.Id == id); + if (item is null) return Task.FromResult(false); + return Task.FromResult(_store.Remove(item)); + } + } + + public Task UpdateAsync(Guid id, string? type = null, string? description = null, IEnumerable? tags = null, DateTimeOffset? time = null) + { + lock (_lock) + { + var item = _store.FirstOrDefault(f => f.Id == id); + if (item is null) return Task.FromResult(false); + + if (type != null) + { + if (string.IsNullOrWhiteSpace(type)) throw new ArgumentException("Type cannot be empty", nameof(type)); + item.Type = type.Trim(); + } + + if (description != null) + { + if (string.IsNullOrWhiteSpace(description)) throw new ArgumentException("Description cannot be empty", nameof(description)); + item.Description = description.Trim(); + } + + if (tags != null) + { + item.Tags = [.. tags + .Where(t => !string.IsNullOrWhiteSpace(t)) + .Select(t => t!.Trim())]; + } + + if (time.HasValue) + item.Time = time.Value; + + return Task.FromResult(true); + } + } + + public Task> GetByTagAsync(string tag) + { + var q = tag?.Trim(); + if (string.IsNullOrWhiteSpace(q)) return Task.FromResult(new List()); + lock (_lock) + { + return Task.FromResult(_store.Where(f => f.Tags?.Any(t => !string.IsNullOrWhiteSpace(t) && t.Contains(q, StringComparison.OrdinalIgnoreCase)) == true).ToList()); + } + } + + public Task> GetByTypeAsync(string type) + { + var q = type?.Trim(); + if (string.IsNullOrWhiteSpace(q)) return Task.FromResult(new List()); + lock (_lock) + { + return Task.FromResult(_store.Where(f => !string.IsNullOrWhiteSpace(f.Type) && f.Type.Trim().Contains(q, StringComparison.OrdinalIgnoreCase)).ToList()); + } + } + + public Task> SearchAsync(string? type = null, string? tag = null, DateTimeOffset? timeAfter = null) + { + var results = _store.AsEnumerable(); + var qType = type?.Trim(); + var qTag = tag?.Trim(); + + lock (_lock) + { + if (!string.IsNullOrWhiteSpace(qType)) + results = results.Where(f => !string.IsNullOrWhiteSpace(f.Type) && f.Type.Trim().Contains(qType, StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrWhiteSpace(qTag)) + results = results.Where(f => f.Tags?.Any(t => !string.IsNullOrWhiteSpace(t) && t.Contains(qTag, StringComparison.OrdinalIgnoreCase)) == true); + if (timeAfter.HasValue) + results = results.Where(f => f.Time > timeAfter.Value); + + return Task.FromResult(results.ToList()); + } + } +} diff --git a/journal-master/journal/Journal.Core/ServiceCollectionExtensions.cs b/journal-master/journal/Journal.Core/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..3bb6fc4 --- /dev/null +++ b/journal-master/journal/Journal.Core/ServiceCollectionExtensions.cs @@ -0,0 +1,53 @@ +using Microsoft.Extensions.DependencyInjection; +using Journal.Core.Repositories; +using Journal.Core.Services; + +namespace Journal.Core; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddFragmentServices(this IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddTransient(); + services.AddTransient(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(provider => + { + var config = provider.GetRequiredService().Current; + if (!string.Equals(config.AiProvider, "python-sidecar", StringComparison.OrdinalIgnoreCase)) + return new DisabledAiService(config.AiProvider); + + try + { + return new PythonSidecarAiService(config); + } + catch (Exception ex) + { + return new DisabledAiService( + provider: "python-sidecar", + message: $"Python AI sidecar unavailable: {ex.Message}", + healthy: false); + } + }); + services.AddSingleton(provider => + { + var config = provider.GetRequiredService().Current; + try + { + return new PythonSidecarSpeechService(config); + } + catch (Exception ex) + { + return new DisabledSpeechBridgeService( + provider: "python-sidecar", + message: $"Python speech sidecar unavailable: {ex.Message}"); + } + }); + services.AddSingleton(); + return services; + } +} diff --git a/journal-master/journal/Journal.Core/Services/DisabledAiService.cs b/journal-master/journal/Journal.Core/Services/DisabledAiService.cs new file mode 100644 index 0000000..cc5b8ac --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/DisabledAiService.cs @@ -0,0 +1,32 @@ +using Journal.Core.Dtos; + +namespace Journal.Core.Services; + +public sealed class DisabledAiService : IAiService +{ + private readonly string _provider; + private readonly string _message; + private readonly bool _healthy; + + public DisabledAiService(string provider, string message = "AI provider disabled.", bool healthy = true) + { + _provider = string.IsNullOrWhiteSpace(provider) ? "none" : provider.Trim(); + _message = string.IsNullOrWhiteSpace(message) ? "AI provider disabled." : message.Trim(); + _healthy = healthy; + } + + public Task HealthAsync(CancellationToken cancellationToken = default) => + Task.FromResult(new AiHealthDto(_provider, Enabled: false, Healthy: _healthy, Message: _message)); + + public Task SummarizeEntryAsync(string content, string? fileStem = null, CancellationToken cancellationToken = default) => + Task.FromResult(_message); + + public Task SummarizeAllAsync(IReadOnlyList entries, CancellationToken cancellationToken = default) => + Task.FromResult(_message); + + public Task ChatAsync(string prompt, CancellationToken cancellationToken = default) => + Task.FromResult(_message); + + public Task> EmbedAsync(string content, CancellationToken cancellationToken = default) => + Task.FromResult>([]); +} diff --git a/journal-master/journal/Journal.Core/Services/DisabledSpeechBridgeService.cs b/journal-master/journal/Journal.Core/Services/DisabledSpeechBridgeService.cs new file mode 100644 index 0000000..9f9a658 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/DisabledSpeechBridgeService.cs @@ -0,0 +1,32 @@ +using Journal.Core.Dtos; + +namespace Journal.Core.Services; + +public sealed class DisabledSpeechBridgeService : ISpeechBridgeService +{ + private readonly string _provider; + private readonly string _message; + + public DisabledSpeechBridgeService(string provider = "none", string message = "Speech bridge is disabled.") + { + _provider = string.IsNullOrWhiteSpace(provider) ? "none" : provider.Trim(); + _message = string.IsNullOrWhiteSpace(message) ? "Speech bridge is disabled." : message.Trim(); + } + + public Task ListDevicesAsync(CancellationToken cancellationToken = default) + { + var warning = $"{_message} (provider={_provider})"; + return Task.FromResult(new SpeechDevicesResultDto([], warning)); + } + + public Task TranscribeAsync( + SpeechTranscribeRequestDto request, + CancellationToken cancellationToken = default) + { + if (request is null) + throw new ArgumentNullException(nameof(request)); + var engine = string.IsNullOrWhiteSpace(request.Engine) ? "none" : request.Engine.Trim(); + var warning = $"{_message} (provider={_provider})"; + return Task.FromResult(new SpeechTranscribeResultDto("", engine, warning)); + } +} diff --git a/journal-master/journal/Journal.Core/Services/EntrySearchService.cs b/journal-master/journal/Journal.Core/Services/EntrySearchService.cs new file mode 100644 index 0000000..e2ff6dc --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/EntrySearchService.cs @@ -0,0 +1,108 @@ +using Journal.Core.Dtos; +using System.Globalization; + +namespace Journal.Core.Services; + +public class EntrySearchService : IEntrySearchService +{ + public Task> SearchEntriesAsync(EntrySearchRequestDto request) + { + ArgumentNullException.ThrowIfNull(request); + if (string.IsNullOrWhiteSpace(request.DataDirectory)) + throw new ArgumentException("Data directory is required.", nameof(request.DataDirectory)); + + if (!Directory.Exists(request.DataDirectory)) + return Task.FromResult>([]); + + var hasQuery = !string.IsNullOrWhiteSpace(request.Query); + var query = request.Query?.Trim() ?? ""; + var hasSectionFilter = !string.IsNullOrWhiteSpace(request.Section); + var section = request.Section?.Trim() ?? ""; + + var typeSet = NormalizeSet(request.Types); + var tagSet = NormalizeSet(request.Tags); + var checkedSet = NormalizeSet(request.Checked); + var uncheckedSet = NormalizeSet(request.Unchecked); + var hasFragmentFilters = typeSet.Count > 0 || tagSet.Count > 0; + var hasCheckboxFilters = checkedSet.Count > 0 || uncheckedSet.Count > 0; + + var startDate = ParseOptionalDate(request.StartDate, nameof(request.StartDate)); + var endDate = ParseOptionalDate(request.EndDate, nameof(request.EndDate)); + if (startDate.HasValue && endDate.HasValue && startDate.Value > endDate.Value) + throw new ArgumentException("startDate cannot be after endDate."); + + var results = new List(); + foreach (var filePath in Directory.GetFiles(request.DataDirectory, "*.md") + .OrderBy(Path.GetFileName, StringComparer.Ordinal)) + { + var fileName = Path.GetFileName(filePath); + var fileStem = Path.GetFileNameWithoutExtension(filePath); + var rawContent = File.ReadAllText(filePath); + var entry = JournalParser.ParseJournalContent(rawContent, fileStem); + + if (startDate.HasValue || endDate.HasValue) + { + if (!DateOnly.TryParseExact(entry.Date, "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var entryDate)) + continue; + + if (startDate.HasValue && entryDate < startDate.Value) + continue; + if (endDate.HasValue && entryDate > endDate.Value) + continue; + } + + var contentMatch = true; + if (hasQuery) + { + var haystack = hasSectionFilter ? entry.GetSection(section) : entry.RawContent; + contentMatch = haystack.IndexOf(query, StringComparison.OrdinalIgnoreCase) >= 0; + } + if (!contentMatch) + continue; + + var fragmentMatch = !hasFragmentFilters || entry.Fragments.Any(fragment => + (typeSet.Count == 0 || typeSet.Contains(fragment.Type)) && + (tagSet.Count == 0 || fragment.Tags.Any(tagSet.Contains))); + if (!fragmentMatch) + continue; + + var checkboxMatch = !hasCheckboxFilters || entry.Sections.Values.Any(sectionValue => + sectionValue.Checkboxes.Any(checkbox => + (checkedSet.Count > 0 && checkbox.Value && checkedSet.Contains(checkbox.Key)) || + (uncheckedSet.Count > 0 && !checkbox.Value && uncheckedSet.Contains(checkbox.Key)))); + if (!checkboxMatch) + continue; + + results.Add(new EntrySearchResultDto(entry.Date, fileName, entry.RawContent)); + } + + return Task.FromResult>(results); + } + + private static HashSet NormalizeSet(IReadOnlyList? values) + { + if (values is null || values.Count == 0) + return []; + + var set = new HashSet(StringComparer.Ordinal); + foreach (var value in values) + { + if (string.IsNullOrWhiteSpace(value)) + continue; + set.Add(value.Trim()); + } + + return set; + } + + private static DateOnly? ParseOptionalDate(string? raw, string argumentName) + { + if (string.IsNullOrWhiteSpace(raw)) + return null; + + if (DateOnly.TryParseExact(raw.Trim(), "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var date)) + return date; + + throw new ArgumentException($"Invalid {argumentName} value. Expected yyyy-MM-dd."); + } +} diff --git a/journal-master/journal/Journal.Core/Services/FragmentService.cs b/journal-master/journal/Journal.Core/Services/FragmentService.cs new file mode 100644 index 0000000..fef24b7 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/FragmentService.cs @@ -0,0 +1,83 @@ +using System.ComponentModel.DataAnnotations; +using Journal.Core.Dtos; +using Journal.Core.Models; +using Journal.Core.Repositories; + +namespace Journal.Core.Services; + +public class FragmentService : IFragmentService +{ + private readonly IFragmentRepository _repo; + + public FragmentService(IFragmentRepository repo) => _repo = repo ?? throw new ArgumentNullException(nameof(repo)); + + private static FragmentDto Map(Fragment f) => new( + f.Id, + f.Type, + f.Description, + f.Time, + f.Tags != null ? [.. f.Tags] : [] + ); + + public async Task CreateAsync(CreateFragmentDto dto) + { + ArgumentNullException.ThrowIfNull(dto); + + var ctx = new ValidationContext(dto); + Validator.ValidateObject(dto, ctx, validateAllProperties: true); + + var f = new Fragment(dto.Type, dto.Description); + if (dto.Tags != null) + f.Tags = [.. dto.Tags.Where(t => !string.IsNullOrWhiteSpace(t)).Select(t => t!.Trim())]; + + await _repo.AddAsync(f); + return Map(f); + } + + public async Task UpdateAsync(Guid id, UpdateFragmentDto dto) + { + ArgumentNullException.ThrowIfNull(dto); + if (dto.Type != null && string.IsNullOrWhiteSpace(dto.Type)) + throw new ValidationException("Type cannot be empty"); + if (dto.Description != null && string.IsNullOrWhiteSpace(dto.Description)) + throw new ValidationException("Description cannot be empty"); + + var type = dto.Type?.Trim(); + var description = dto.Description?.Trim(); + var tags = dto.Tags?.Where(t => !string.IsNullOrWhiteSpace(t)).Select(t => t!.Trim()).ToList(); + + return await _repo.UpdateAsync(id, type, description, tags, dto.Time); + } + + public Task RemoveAsync(Guid id) => _repo.RemoveAsync(id); + + public async Task> SearchAsync(string? type = null, string? tag = null, DateTimeOffset? timeAfter = null) + { + var items = await _repo.SearchAsync(type, tag, timeAfter); + return [.. items.Select(Map)]; + } + + public async Task> GetByTagAsync(string tag) + { + var items = await _repo.GetByTagAsync(tag); + return [.. items.Select(Map)]; + } + + public async Task> GetByTypeAsync(string type) + { + var items = await _repo.GetByTypeAsync(type); + return [.. items.Select(Map)]; + } + + public async Task> GetAllAsync() + { + var items = await _repo.GetAllAsync(); + return [.. items.Select(Map)]; + } + + public async Task GetByIdAsync(Guid id) + { + var f = await _repo.GetByIdAsync(id); + return f is null ? null : Map(f); + } +} diff --git a/journal-master/journal/Journal.Core/Services/IAiService.cs b/journal-master/journal/Journal.Core/Services/IAiService.cs new file mode 100644 index 0000000..791873b --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/IAiService.cs @@ -0,0 +1,12 @@ +using Journal.Core.Dtos; + +namespace Journal.Core.Services; + +public interface IAiService +{ + Task HealthAsync(CancellationToken cancellationToken = default); + Task SummarizeEntryAsync(string content, string? fileStem = null, CancellationToken cancellationToken = default); + Task SummarizeAllAsync(IReadOnlyList entries, CancellationToken cancellationToken = default); + Task ChatAsync(string prompt, CancellationToken cancellationToken = default); + Task> EmbedAsync(string content, CancellationToken cancellationToken = default); +} diff --git a/journal-master/journal/Journal.Core/Services/IEntrySearchService.cs b/journal-master/journal/Journal.Core/Services/IEntrySearchService.cs new file mode 100644 index 0000000..e9bfede --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/IEntrySearchService.cs @@ -0,0 +1,8 @@ +using Journal.Core.Dtos; + +namespace Journal.Core.Services; + +public interface IEntrySearchService +{ + Task> SearchEntriesAsync(EntrySearchRequestDto request); +} diff --git a/journal-master/journal/Journal.Core/Services/IFragmentService.cs b/journal-master/journal/Journal.Core/Services/IFragmentService.cs new file mode 100644 index 0000000..2bde778 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/IFragmentService.cs @@ -0,0 +1,15 @@ +using Journal.Core.Dtos; + +namespace Journal.Core.Services; + +public interface IFragmentService +{ + Task CreateAsync(CreateFragmentDto dto); + Task UpdateAsync(Guid id, UpdateFragmentDto dto); + Task RemoveAsync(Guid id); + Task> SearchAsync(string? type = null, string? tag = null, DateTimeOffset? timeAfter = null); + Task> GetByTagAsync(string tag); + Task> GetByTypeAsync(string type); + Task> GetAllAsync(); + Task GetByIdAsync(Guid id); +} diff --git a/journal-master/journal/Journal.Core/Services/IJournalConfigService.cs b/journal-master/journal/Journal.Core/Services/IJournalConfigService.cs new file mode 100644 index 0000000..3e0a7b5 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/IJournalConfigService.cs @@ -0,0 +1,8 @@ +using Journal.Core.Models; + +namespace Journal.Core.Services; + +public interface IJournalConfigService +{ + JournalConfig Current { get; } +} diff --git a/journal-master/journal/Journal.Core/Services/IJournalDatabaseService.cs b/journal-master/journal/Journal.Core/Services/IJournalDatabaseService.cs new file mode 100644 index 0000000..54b86bc --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/IJournalDatabaseService.cs @@ -0,0 +1,29 @@ +namespace Journal.Core.Services; + +public interface IJournalDatabaseService +{ + string GetDatabasePath(string? dataDirectory = null); + byte[] DeriveDatabaseKey(string password); + string BuildPragmaKeyStatement(string password); + IReadOnlyDictionary GetSchemaStatements(); + string WriteSchemaBootstrap(string? dataDirectory = null); + JournalDatabaseStatus GetStatus(string password, string? dataDirectory = null); + JournalDatabaseHydrationResult HydrateWorkspace(string password, string? dataDirectory = null); +} + +public sealed record JournalDatabaseStatus( + string DatabasePath, + int KeyLengthBytes, + int Iterations, + string KeyDerivation, + IReadOnlyList SchemaTables, + string SchemaBootstrapPath, + bool RuntimeReady, + string RuntimeMessage); + +public sealed record JournalDatabaseHydrationResult( + string DatabasePath, + string SchemaBootstrapPath, + int EntryFilesProcessed, + bool RuntimeReady, + string Message); diff --git a/journal-master/journal/Journal.Core/Services/ISpeechBridgeService.cs b/journal-master/journal/Journal.Core/Services/ISpeechBridgeService.cs new file mode 100644 index 0000000..0294722 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/ISpeechBridgeService.cs @@ -0,0 +1,9 @@ +using Journal.Core.Dtos; + +namespace Journal.Core.Services; + +public interface ISpeechBridgeService +{ + Task ListDevicesAsync(CancellationToken cancellationToken = default); + Task TranscribeAsync(SpeechTranscribeRequestDto request, CancellationToken cancellationToken = default); +} diff --git a/journal-master/journal/Journal.Core/Services/IVaultCryptoService.cs b/journal-master/journal/Journal.Core/Services/IVaultCryptoService.cs new file mode 100644 index 0000000..85418e5 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/IVaultCryptoService.cs @@ -0,0 +1,8 @@ +namespace Journal.Core.Services; + +public interface IVaultCryptoService +{ + byte[] DeriveKey(string password, byte[] salt); + byte[] EncryptData(byte[] data, string password); + byte[] DecryptData(byte[] encryptedData, string password); +} diff --git a/journal-master/journal/Journal.Core/Services/IVaultStorageService.cs b/journal-master/journal/Journal.Core/Services/IVaultStorageService.cs new file mode 100644 index 0000000..525c1f3 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/IVaultStorageService.cs @@ -0,0 +1,10 @@ +namespace Journal.Core.Services; + +public interface IVaultStorageService +{ + string GetMonthlyVaultFileName(DateTime date); + bool LoadAllVaults(string password, string vaultDirectory, string dataDirectory); + bool SaveCurrentMonthVault(string password, string vaultDirectory, string dataDirectory, DateTime now); + void RebuildAllVaults(string password, string vaultDirectory, string dataDirectory); + void ClearDataDirectory(string dataDirectory); +} diff --git a/journal-master/journal/Journal.Core/Services/JournalConfigService.cs b/journal-master/journal/Journal.Core/Services/JournalConfigService.cs new file mode 100644 index 0000000..bce9602 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/JournalConfigService.cs @@ -0,0 +1,107 @@ +using Journal.Core.Models; + +namespace Journal.Core.Services; + +public sealed class JournalConfigService : IJournalConfigService +{ + public JournalConfig Current { get; } = BuildConfig(); + + private static JournalConfig BuildConfig() + { + var projectRoot = ResolveProjectRoot(); + var appDirectory = ResolvePath("JOURNAL_APP_DIR", Path.Combine(projectRoot, "journal")); + + var dataDirectory = ResolvePath("JOURNAL_DATA_DIR", Path.Combine(appDirectory, "data")); + var vaultDirectory = ResolvePath("JOURNAL_VAULT_DIR", Path.Combine(appDirectory, "vault")); + var logDirectory = ResolvePath("JOURNAL_LOG_DIR", Path.Combine(projectRoot, "logs")); + + var pidFile = ResolvePath("JOURNAL_PID_FILE", Path.Combine(logDirectory, "nicegui_server.pid")); + var serverControlFile = ResolvePath("JOURNAL_SERVER_CONTROL_FILE", Path.Combine(logDirectory, "server_control.action")); + + var nlpBackend = (Environment.GetEnvironmentVariable("JOURNAL_NLP_BACKEND") ?? "auto").Trim().ToLowerInvariant(); + if (nlpBackend is not ("auto" or "spacy" or "fallback")) + nlpBackend = "auto"; + + var aiProvider = (Environment.GetEnvironmentVariable("JOURNAL_AI_PROVIDER") ?? "none").Trim().ToLowerInvariant(); + if (aiProvider is not ("none" or "python-sidecar")) + aiProvider = "none"; + + var pythonExecutable = Environment.GetEnvironmentVariable("JOURNAL_PYTHON_EXE"); + if (string.IsNullOrWhiteSpace(pythonExecutable)) + pythonExecutable = "python"; + + var defaultAiSidecarPath = Path.Combine(projectRoot, "journal", "ai", "sidecar.py"); + var pythonAiSidecarPath = ResolvePath("JOURNAL_AI_SIDECAR_PATH", defaultAiSidecarPath); + var aiSidecarTimeoutMs = ParseInt("JOURNAL_AI_TIMEOUT_MS", 45000); + + return new JournalConfig( + ProjectRoot: projectRoot, + AppDirectory: appDirectory, + DataDirectory: dataDirectory, + VaultDirectory: vaultDirectory, + LogDirectory: logDirectory, + PidFile: pidFile, + ServerControlFile: serverControlFile, + DatabaseFilename: Environment.GetEnvironmentVariable("JOURNAL_DATABASE_FILENAME") ?? "journal_cache.db", + MonthlyVaultFormat: Environment.GetEnvironmentVariable("JOURNAL_MONTHLY_VAULT_FORMAT") ?? "%Y-%m.vault", + CloudAiApiKey: Environment.GetEnvironmentVariable("CLOUDAI_API_KEY") ?? "", + CloudAiApiUrl: Environment.GetEnvironmentVariable("CLOUDAI_API_URL") ?? "", + LlamaCppUrl: Environment.GetEnvironmentVariable("LLAMA_CPP_URL") ?? "http://127.0.0.1:8085/v1/completions", + LlamaCppModel: Environment.GetEnvironmentVariable("LLAMA_CPP_MODEL") ?? "qwen/qwen3-4b", + LlamaCppTimeout: ParseInt("LLAMA_CPP_TIMEOUT", 6000), + EmbeddingApiUrl: Environment.GetEnvironmentVariable("EMBEDDING_API_URL") ?? "http://127.0.0.1:8086/v1/embeddings", + EmbeddingModelName: Environment.GetEnvironmentVariable("EMBEDDING_MODEL_NAME") ?? "text-embedding-nomic-embed-text-v2-moe", + ModelContextTokens: ParseInt("MODEL_CONTEXT_TOKENS", 131072), + ChunkTokenBudget: ParseInt("CHUNK_TOKEN_BUDGET", 120000), + MicrophoneDeviceIndex: ParseNullableInt("MICROPHONE_DEVICE_INDEX"), + SpeechRecognitionEngine: Environment.GetEnvironmentVariable("SPEECH_RECOGNITION_ENGINE") ?? "whisper", + WhisperModelSize: Environment.GetEnvironmentVariable("WHISPER_MODEL_SIZE") ?? "base", + NlpBackend: nlpBackend, + AiProvider: aiProvider, + PythonExecutable: pythonExecutable, + PythonAiSidecarPath: pythonAiSidecarPath, + AiSidecarTimeoutMs: aiSidecarTimeoutMs); + } + + private static string ResolveProjectRoot() + { + var envRoot = Environment.GetEnvironmentVariable("JOURNAL_PROJECT_ROOT"); + if (!string.IsNullOrWhiteSpace(envRoot)) + return Path.GetFullPath(envRoot); + + var cwd = Directory.GetCurrentDirectory(); + if (Directory.Exists(Path.Combine(cwd, "journal"))) + return Path.GetFullPath(cwd); + + var upOne = Path.GetFullPath(Path.Combine(cwd, "..")); + if (Directory.Exists(Path.Combine(upOne, "journal"))) + return upOne; + + var upTwo = Path.GetFullPath(Path.Combine(cwd, "..", "..")); + if (Directory.Exists(Path.Combine(upTwo, "journal"))) + return upTwo; + + return Path.GetFullPath(cwd); + } + + private static string ResolvePath(string envVar, string defaultPath) + { + var value = Environment.GetEnvironmentVariable(envVar); + var raw = string.IsNullOrWhiteSpace(value) ? defaultPath : value; + return Path.GetFullPath(raw); + } + + private static int ParseInt(string envVar, int defaultValue) + { + var value = Environment.GetEnvironmentVariable(envVar); + return int.TryParse(value, out var parsed) ? parsed : defaultValue; + } + + private static int? ParseNullableInt(string envVar) + { + var value = Environment.GetEnvironmentVariable(envVar); + if (string.IsNullOrWhiteSpace(value)) + return null; + return int.TryParse(value, out var parsed) ? parsed : null; + } +} diff --git a/journal-master/journal/Journal.Core/Services/JournalDatabaseService.cs b/journal-master/journal/Journal.Core/Services/JournalDatabaseService.cs new file mode 100644 index 0000000..73c0657 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/JournalDatabaseService.cs @@ -0,0 +1,233 @@ +using System.Security.Cryptography; +using System.Text; +using Microsoft.Data.Sqlite; + +namespace Journal.Core.Services; + +public sealed class JournalDatabaseService : IJournalDatabaseService +{ + public const int KeySize = 32; + public const int Iterations = 600_000; + private static readonly byte[] DatabaseKeySalt = Encoding.UTF8.GetBytes("a_fixed_salt_for_the_db_key_deriv"); + private static readonly object SqliteInitLock = new(); + private static bool _sqliteInitialized; + private static readonly IReadOnlyList RequiredSchemaTables = + ["entries", "sections", "fragments", "tags", "fragment_tags"]; + + private readonly IJournalConfigService _config; + + public JournalDatabaseService(IJournalConfigService config) + { + _config = config; + } + + public string GetDatabasePath(string? dataDirectory = null) + { + var directory = string.IsNullOrWhiteSpace(dataDirectory) + ? _config.Current.DataDirectory + : dataDirectory; + + Directory.CreateDirectory(directory); + return Path.GetFullPath(Path.Combine(directory, _config.Current.DatabaseFilename)); + } + + public byte[] DeriveDatabaseKey(string password) + { + if (string.IsNullOrWhiteSpace(password)) + throw new ArgumentException("Password cannot be empty.", nameof(password)); + + return Rfc2898DeriveBytes.Pbkdf2( + Encoding.UTF8.GetBytes(password), + DatabaseKeySalt, + Iterations, + HashAlgorithmName.SHA256, + KeySize); + } + + public string BuildPragmaKeyStatement(string password) + { + var dbKeyHex = Convert.ToHexString(DeriveDatabaseKey(password)).ToLowerInvariant(); + return $"PRAGMA key = \"x'{dbKeyHex}'\""; + } + + public IReadOnlyDictionary GetSchemaStatements() + { + return new Dictionary(StringComparer.Ordinal) + { + ["entries"] = """ + CREATE TABLE IF NOT EXISTS entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + date TEXT NOT NULL UNIQUE + ); + """, + ["sections"] = """ + CREATE TABLE IF NOT EXISTS sections ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entry_id INTEGER NOT NULL, + title TEXT NOT NULL, + content TEXT, + FOREIGN KEY (entry_id) REFERENCES entries (id) + ); + """, + ["fragments"] = """ + CREATE TABLE IF NOT EXISTS fragments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entry_id INTEGER NOT NULL, + type TEXT NOT NULL, + description TEXT, + time TEXT, + FOREIGN KEY (entry_id) REFERENCES entries (id) + ); + """, + ["tags"] = """ + CREATE TABLE IF NOT EXISTS tags ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE + ); + """, + ["fragment_tags"] = """ + CREATE TABLE IF NOT EXISTS fragment_tags ( + fragment_id INTEGER NOT NULL, + tag_id INTEGER NOT NULL, + PRIMARY KEY (fragment_id, tag_id), + FOREIGN KEY (fragment_id) REFERENCES fragments (id), + FOREIGN KEY (tag_id) REFERENCES tags (id) + ); + """ + }; + } + + public string WriteSchemaBootstrap(string? dataDirectory = null) + { + var directory = string.IsNullOrWhiteSpace(dataDirectory) + ? _config.Current.DataDirectory + : dataDirectory; + Directory.CreateDirectory(directory); + + var bootstrapPath = Path.GetFullPath(Path.Combine(directory, "journal_schema.sql")); + var statements = GetSchemaStatements() + .Select(pair => $"-- {pair.Key}\n{pair.Value.Trim()}") + .ToArray(); + var content = string.Join("\n\n", statements) + "\n"; + File.WriteAllText(bootstrapPath, content); + return bootstrapPath; + } + + public JournalDatabaseStatus GetStatus(string password, string? dataDirectory = null) + { + var tables = GetSchemaStatements().Keys.OrderBy(x => x, StringComparer.Ordinal).ToArray(); + var bootstrapPath = WriteSchemaBootstrap(dataDirectory); + var runtime = ProbeRuntime(password, dataDirectory); + return new JournalDatabaseStatus( + DatabasePath: GetDatabasePath(dataDirectory), + KeyLengthBytes: DeriveDatabaseKey(password).Length, + Iterations: Iterations, + KeyDerivation: "PBKDF2-HMAC-SHA256", + SchemaTables: tables, + SchemaBootstrapPath: bootstrapPath, + RuntimeReady: runtime.Ready, + RuntimeMessage: runtime.Message); + } + + public JournalDatabaseHydrationResult HydrateWorkspace(string password, string? dataDirectory = null) + { + var directory = string.IsNullOrWhiteSpace(dataDirectory) + ? _config.Current.DataDirectory + : dataDirectory; + Directory.CreateDirectory(directory); + + using var connection = OpenEncryptedConnection(password, directory); + CreateSchema(connection); + var runtimeReady = HasRequiredTables(connection); + + var entryFilesProcessed = Directory.GetFiles(directory, "*.md", SearchOption.TopDirectoryOnly).Length; + var schemaPath = WriteSchemaBootstrap(directory); + + return new JournalDatabaseHydrationResult( + DatabasePath: GetDatabasePath(directory), + SchemaBootstrapPath: schemaPath, + EntryFilesProcessed: entryFilesProcessed, + RuntimeReady: runtimeReady, + Message: runtimeReady + ? "Workspace hydration completed with SQLCipher runtime schema validation." + : "Workspace hydration completed, but required schema tables were not found."); + } + + private static void EnsureSqliteInitialized() + { + if (_sqliteInitialized) + return; + + lock (SqliteInitLock) + { + if (_sqliteInitialized) + return; + + SQLitePCL.Batteries_V2.Init(); + _sqliteInitialized = true; + } + } + + private SqliteConnection OpenEncryptedConnection(string password, string? dataDirectory = null) + { + if (string.IsNullOrWhiteSpace(password)) + throw new ArgumentException("Password cannot be empty.", nameof(password)); + + EnsureSqliteInitialized(); + + var connection = new SqliteConnection($"Data Source={GetDatabasePath(dataDirectory)};Mode=ReadWriteCreate;Pooling=False"); + connection.Open(); + + using var keyCmd = connection.CreateCommand(); + keyCmd.CommandText = BuildPragmaKeyStatement(password) + ";"; + keyCmd.ExecuteNonQuery(); + + using var verifyCmd = connection.CreateCommand(); + verifyCmd.CommandText = "SELECT count(*) FROM sqlite_master;"; + _ = verifyCmd.ExecuteScalar(); + + return connection; + } + + private void CreateSchema(SqliteConnection connection) + { + foreach (var statement in GetSchemaStatements().Values) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = statement; + cmd.ExecuteNonQuery(); + } + } + + private static bool HasRequiredTables(SqliteConnection connection) + { + var existing = new HashSet(StringComparer.OrdinalIgnoreCase); + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT name FROM sqlite_master WHERE type='table'"; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + if (!reader.IsDBNull(0)) + existing.Add(reader.GetString(0)); + } + + return RequiredSchemaTables.All(existing.Contains); + } + + private (bool Ready, string Message) ProbeRuntime(string password, string? dataDirectory) + { + try + { + using var connection = OpenEncryptedConnection(password, dataDirectory); + CreateSchema(connection); + var ready = HasRequiredTables(connection); + return ready + ? (true, "SQLCipher runtime is available and schema tables are present.") + : (false, "SQLCipher runtime opened, but required schema tables are missing."); + } + catch (Exception ex) + { + return (false, $"SQLCipher runtime check failed: {ex.Message}"); + } + } +} diff --git a/journal-master/journal/Journal.Core/Services/JournalParser.cs b/journal-master/journal/Journal.Core/Services/JournalParser.cs new file mode 100644 index 0000000..ff00634 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/JournalParser.cs @@ -0,0 +1,175 @@ +using System.Text.RegularExpressions; +using Journal.Core.Models; + +namespace Journal.Core.Services; + +public static partial class JournalParser +{ + [GeneratedRegex(@"(?:\*\*Date:\*\*|\*\*Date:|Date:)\s*(.+)")] + private static partial Regex DatePattern(); + [GeneratedRegex(@"^\#\#+\s*(.*)$")] + private static partial Regex SectionHeaderPattern(); + [GeneratedRegex(@"^\s*[-*]\s*\[([xX ])\]\s*(.*)$")] + private static partial Regex CheckboxPattern(); + [GeneratedRegex(@"^(!\w+)\s*((?:@\S+\s*)?)(?:\s*((?:#\S+\s*)*))?\s*$")] + private static partial Regex FragmentHeaderPattern(); + [GeneratedRegex(@"^!\w+\s*")] + private static partial Regex FragmentBoundaryPattern(); + + public static JournalEntry ParseJournalContent(string content, string fileStem) + { + ArgumentNullException.ThrowIfNull(content); + return new JournalEntry( + date: ExtractDate(content, fileStem), + rawContent: content, + sections: ParseSections(content), + fragments: ParseFragments(content)); + } + + public static string ExtractDate(string content, string fileStem) + { + ArgumentNullException.ThrowIfNull(content); + if (string.IsNullOrWhiteSpace(fileStem)) + throw new ArgumentException("File stem is required", nameof(fileStem)); + + var match = DatePattern().Match(content); + if (match.Success) + { + var parsed = match.Groups[1].Value.Trim(); + if (!string.IsNullOrWhiteSpace(parsed)) + return parsed; + } + + return fileStem.Trim(); + } + + public static Dictionary ParseSections(string content) + { + ArgumentNullException.ThrowIfNull(content); + + var parsedSections = new Dictionary(); + string? currentSectionTitle = null; + var currentSectionContent = new List(); + var currentSectionCheckboxes = new Dictionary(); + + var lines = content.Split(["\r\n", "\n", "\r"], StringSplitOptions.None); + foreach (var line in lines) + { + var sectionHeaderMatch = SectionHeaderPattern().Match(line.Trim()); + if (sectionHeaderMatch.Success) + { + if (currentSectionTitle is not null) + { + parsedSections[currentSectionTitle] = new ParsedSection( + currentSectionTitle, + currentSectionContent, + currentSectionCheckboxes); + } + + var headerText = sectionHeaderMatch.Groups[1].Value.Trim(); + var foundTitle = FindCanonicalSectionTitle(headerText); + + if (foundTitle is not null) + { + currentSectionTitle = foundTitle; + currentSectionContent = []; + currentSectionCheckboxes = []; + } + else + { + currentSectionTitle = null; + currentSectionContent = []; + currentSectionCheckboxes = []; + } + + continue; + } + + if (currentSectionTitle is not null) + { + var checkboxMatch = CheckboxPattern().Match(line); + if (checkboxMatch.Success) + { + var isChecked = checkboxMatch.Groups[1].Value.Trim().Equals("x", StringComparison.OrdinalIgnoreCase); + var checkboxText = checkboxMatch.Groups[2].Value.Trim(); + currentSectionCheckboxes[checkboxText] = isChecked; + } + + currentSectionContent.Add(line); + } + } + + if (currentSectionTitle is not null) + { + parsedSections[currentSectionTitle] = new ParsedSection( + currentSectionTitle, + currentSectionContent, + currentSectionCheckboxes); + } + + return parsedSections; + } + + public static List ParseFragments(string content) + { + ArgumentNullException.ThrowIfNull(content); + + var fragments = new List(); + var lines = content.Split(["\r\n", "\n", "\r"], StringSplitOptions.None); + + for (var i = 0; i < lines.Length; i++) + { + var headerMatch = FragmentHeaderPattern().Match(lines[i]); + if (!headerMatch.Success) + continue; + + var type = headerMatch.Groups[1].Value.Trim(); + var timeToken = headerMatch.Groups[2].Value.Trim().TrimStart('@'); + var tagsToken = headerMatch.Groups[3].Value.Trim(); + + var descriptionLines = new List(); + var j = i + 1; + while (j < lines.Length && !FragmentBoundaryPattern().IsMatch(lines[j])) + { + descriptionLines.Add(lines[j]); + j++; + } + + var description = string.Join("\n", descriptionLines).Trim(); + if (!string.IsNullOrWhiteSpace(description)) + { + var fragment = new Fragment(type, description); + if (!string.IsNullOrWhiteSpace(timeToken) && DateTimeOffset.TryParse(timeToken, out var parsedTime)) + fragment.Time = parsedTime; + + if (!string.IsNullOrWhiteSpace(tagsToken)) + { + fragment.Tags = + [ + .. tagsToken.Split(' ', StringSplitOptions.RemoveEmptyEntries) + .Where(t => t.StartsWith('#')) + .Select(t => t.Trim().TrimStart('#')) + .Where(t => !string.IsNullOrWhiteSpace(t)) + ]; + } + + fragments.Add(fragment); + } + + i = j - 1; + } + + return fragments; + } + + private static string? FindCanonicalSectionTitle(string headerText) + { + foreach (var title in SectionTitles.Canonical) + { + if (headerText.Contains(title, StringComparison.OrdinalIgnoreCase)) + return title; + } + + return null; + } +} diff --git a/journal-master/journal/Journal.Core/Services/LogRedactor.cs b/journal-master/journal/Journal.Core/Services/LogRedactor.cs new file mode 100644 index 0000000..4554174 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/LogRedactor.cs @@ -0,0 +1,73 @@ +using System.Text.Json; + +namespace Journal.Core.Services; + +public static class LogRedactor +{ + private static readonly HashSet SensitiveKeys = new(StringComparer.OrdinalIgnoreCase) + { + "password", + "passphrase", + "secret", + "token", + "apiKey", + "api_key", + "cloudAiApiKey", + "content", + "rawContent", + "prompt", + "audioBase64", + "audio_base64", + "text" + }; + + public static object? RedactPayload(JsonElement? payload) + { + if (payload is null) + return null; + return RedactElement(payload.Value, parentKey: null); + } + + private static object? RedactElement(JsonElement element, string? parentKey) + { + if (parentKey is not null && SensitiveKeys.Contains(parentKey)) + return "[REDACTED]"; + + return element.ValueKind switch + { + JsonValueKind.Object => RedactObject(element), + JsonValueKind.Array => RedactArray(element), + JsonValueKind.String => RedactString(element.GetString() ?? "", parentKey), + JsonValueKind.Number => element.GetRawText(), + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Null => null, + _ => element.GetRawText() + }; + } + + private static Dictionary RedactObject(JsonElement element) + { + var output = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var property in element.EnumerateObject()) + output[property.Name] = RedactElement(property.Value, property.Name); + return output; + } + + private static List RedactArray(JsonElement element) + { + var output = new List(); + foreach (var item in element.EnumerateArray()) + output.Add(RedactElement(item, parentKey: null)); + return output; + } + + private static object RedactString(string value, string? key) + { + if (key is not null && SensitiveKeys.Contains(key)) + return "[REDACTED]"; + if (value.Length <= 128) + return value; + return $"{value[..128]}...(truncated)"; + } +} diff --git a/journal-master/journal/Journal.Core/Services/PythonSidecarAiService.cs b/journal-master/journal/Journal.Core/Services/PythonSidecarAiService.cs new file mode 100644 index 0000000..767b82a --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/PythonSidecarAiService.cs @@ -0,0 +1,190 @@ +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using Journal.Core.Dtos; +using Journal.Core.Models; + +namespace Journal.Core.Services; + +public sealed class PythonSidecarAiService : IAiService +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly JournalConfig _config; + + public PythonSidecarAiService(JournalConfig config) + { + _config = config; + if (string.IsNullOrWhiteSpace(_config.PythonAiSidecarPath)) + throw new ArgumentException("Python AI sidecar path is required."); + if (!File.Exists(_config.PythonAiSidecarPath)) + throw new FileNotFoundException($"Python AI sidecar not found: {_config.PythonAiSidecarPath}"); + } + + public async Task HealthAsync(CancellationToken cancellationToken = default) + { + var data = await SendAsync("health", payload: new { }, cancellationToken); + if (data is not JsonElement payload || payload.ValueKind != JsonValueKind.Object) + return new AiHealthDto("python-sidecar", Enabled: true, Healthy: true, Message: "ok"); + + var provider = payload.TryGetProperty("provider", out var providerNode) + ? providerNode.GetString() ?? "python-sidecar" + : "python-sidecar"; + var message = payload.TryGetProperty("message", out var messageNode) + ? messageNode.GetString() ?? "ok" + : "ok"; + var healthy = !payload.TryGetProperty("healthy", out var healthyNode) || + healthyNode.ValueKind is JsonValueKind.True || + (healthyNode.ValueKind is JsonValueKind.False ? false : true); + return new AiHealthDto(provider, Enabled: true, Healthy: healthy, Message: message); + } + + public async Task SummarizeEntryAsync(string content, string? fileStem = null, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(content)) + throw new ArgumentException("Entry content is required.", nameof(content)); + + var data = await SendAsync("summarize_entry", new { content, file_stem = fileStem }, cancellationToken); + return data?.GetString() ?? ""; + } + + public async Task SummarizeAllAsync(IReadOnlyList entries, CancellationToken cancellationToken = default) + { + entries ??= []; + var data = await SendAsync("summarize_all", new { entries }, cancellationToken); + return data?.GetString() ?? ""; + } + + public async Task ChatAsync(string prompt, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(prompt)) + throw new ArgumentException("Prompt is required.", nameof(prompt)); + + var data = await SendAsync("chat", new { prompt }, cancellationToken); + return data?.GetString() ?? ""; + } + + public async Task> EmbedAsync(string content, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(content)) + throw new ArgumentException("Content is required.", nameof(content)); + + var data = await SendAsync("embed", new { content }, cancellationToken); + if (data is null || data.Value.ValueKind == JsonValueKind.Null) + return []; + + if (data.Value.ValueKind != JsonValueKind.Array) + throw new InvalidOperationException("Python AI sidecar embed response must be a numeric array."); + + var values = new List(); + foreach (var item in data.Value.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Number) + throw new InvalidOperationException("Python AI sidecar embed response contains a non-numeric value."); + values.Add(item.GetDouble()); + } + + return values; + } + + private async Task SendAsync(string action, object payload, CancellationToken cancellationToken) + { + var request = JsonSerializer.Serialize(new { action, payload }, JsonOptions); + + using var process = new Process(); + process.StartInfo = new ProcessStartInfo + { + FileName = _config.PythonExecutable, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + WorkingDirectory = _config.ProjectRoot + }; + process.StartInfo.ArgumentList.Add(_config.PythonAiSidecarPath); + + if (!process.Start()) + throw new InvalidOperationException("Failed to start Python AI sidecar process."); + + await process.StandardInput.WriteLineAsync(request); + process.StandardInput.Close(); + + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(_config.AiSidecarTimeoutMs); + + try + { + await process.WaitForExitAsync(timeoutCts.Token); + } + catch (OperationCanceledException) + { + TryKill(process); + throw new TimeoutException($"Python AI sidecar timed out after {_config.AiSidecarTimeoutMs} ms."); + } + + var stdout = await process.StandardOutput.ReadToEndAsync(); + var stderr = await process.StandardError.ReadToEndAsync(); + var line = LastJsonLine(stdout); + if (string.IsNullOrWhiteSpace(line)) + throw new InvalidOperationException($"Python AI sidecar returned no JSON response. stderr: {stderr}".Trim()); + + JsonDocument doc; + try + { + doc = JsonDocument.Parse(line); + } + catch (JsonException ex) + { + throw new InvalidOperationException($"Invalid JSON from Python AI sidecar: {line}", ex); + } + using (doc) + { + var root = doc.RootElement; + if (!root.TryGetProperty("ok", out var okNode) || okNode.ValueKind != JsonValueKind.True && okNode.ValueKind != JsonValueKind.False) + throw new InvalidOperationException("Python AI sidecar response missing boolean 'ok' field."); + + if (!okNode.GetBoolean()) + { + var error = root.TryGetProperty("error", out var errorNode) + ? errorNode.GetString() ?? "Unknown sidecar error." + : "Unknown sidecar error."; + throw new InvalidOperationException(error); + } + + if (!root.TryGetProperty("data", out var dataNode)) + return null; + + return dataNode.Clone(); + } + } + + private static string LastJsonLine(string text) + { + var lines = text.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries); + for (var i = lines.Length - 1; i >= 0; i--) + { + var line = lines[i].Trim(); + if (line.StartsWith("{", StringComparison.Ordinal) && line.EndsWith("}", StringComparison.Ordinal)) + return line; + } + + return ""; + } + + private static void TryKill(Process process) + { + try + { + if (!process.HasExited) + process.Kill(entireProcessTree: true); + } + catch + { + // Ignore cleanup errors while handling timeout/failure path. + } + } +} diff --git a/journal-master/journal/Journal.Core/Services/PythonSidecarSpeechService.cs b/journal-master/journal/Journal.Core/Services/PythonSidecarSpeechService.cs new file mode 100644 index 0000000..582d1cf --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/PythonSidecarSpeechService.cs @@ -0,0 +1,184 @@ +using System.Diagnostics; +using System.Text.Json; +using Journal.Core.Dtos; +using Journal.Core.Models; + +namespace Journal.Core.Services; + +public sealed class PythonSidecarSpeechService : ISpeechBridgeService +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; + + private readonly JournalConfig _config; + + public PythonSidecarSpeechService(JournalConfig config) + { + _config = config; + if (string.IsNullOrWhiteSpace(_config.PythonAiSidecarPath)) + throw new ArgumentException("Python sidecar path is required."); + if (!File.Exists(_config.PythonAiSidecarPath)) + throw new FileNotFoundException($"Python sidecar not found: {_config.PythonAiSidecarPath}"); + } + + public async Task ListDevicesAsync(CancellationToken cancellationToken = default) + { + var data = await SendAsync("speech.devices.list", new { }, cancellationToken); + if (data is null || data.Value.ValueKind != JsonValueKind.Object) + return new SpeechDevicesResultDto([], "Unexpected speech device response from Python sidecar."); + + var warning = data.Value.TryGetProperty("warning", out var warningNode) + ? warningNode.GetString() + : null; + + var devices = new List(); + if (data.Value.TryGetProperty("devices", out var devicesNode) && devicesNode.ValueKind == JsonValueKind.Array) + { + foreach (var device in devicesNode.EnumerateArray()) + { + if (device.ValueKind != JsonValueKind.Object) + continue; + + var index = device.TryGetProperty("index", out var indexNode) && indexNode.ValueKind == JsonValueKind.Number + ? indexNode.GetInt32() + : -1; + var name = device.TryGetProperty("name", out var nameNode) + ? nameNode.GetString() ?? "" + : ""; + devices.Add(new SpeechDeviceDto(index, name)); + } + } + + return new SpeechDevicesResultDto(devices, warning); + } + + public async Task TranscribeAsync( + SpeechTranscribeRequestDto request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var data = await SendAsync("speech.transcribe", new + { + audio_base64 = request.AudioBase64, + engine = request.Engine, + whisper_model = request.WhisperModel, + text = request.Text, + simulate_delay_ms = request.SimulateDelayMs + }, cancellationToken); + + if (data is null || data.Value.ValueKind != JsonValueKind.Object) + throw new InvalidOperationException("Python sidecar speech response must be a JSON object."); + + var text = data.Value.TryGetProperty("text", out var textNode) + ? textNode.GetString() ?? "" + : ""; + var engine = data.Value.TryGetProperty("engine", out var engineNode) + ? engineNode.GetString() ?? (request.Engine ?? "whisper") + : (request.Engine ?? "whisper"); + var warning = data.Value.TryGetProperty("warning", out var warningNode) + ? warningNode.GetString() + : null; + return new SpeechTranscribeResultDto(text, engine, warning); + } + + private async Task SendAsync(string action, object payload, CancellationToken cancellationToken) + { + var request = JsonSerializer.Serialize(new { action, payload }, JsonOptions); + + using var process = new Process(); + process.StartInfo = new ProcessStartInfo + { + FileName = _config.PythonExecutable, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + WorkingDirectory = _config.ProjectRoot + }; + process.StartInfo.ArgumentList.Add(_config.PythonAiSidecarPath); + + if (!process.Start()) + throw new InvalidOperationException("Failed to start Python sidecar process."); + + await process.StandardInput.WriteLineAsync(request); + process.StandardInput.Close(); + + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(_config.AiSidecarTimeoutMs); + + try + { + await process.WaitForExitAsync(timeoutCts.Token); + } + catch (OperationCanceledException) + { + TryKill(process); + throw new TimeoutException($"Python sidecar timed out after {_config.AiSidecarTimeoutMs} ms."); + } + + var stdout = await process.StandardOutput.ReadToEndAsync(); + var stderr = await process.StandardError.ReadToEndAsync(); + var line = LastJsonLine(stdout); + if (string.IsNullOrWhiteSpace(line)) + throw new InvalidOperationException($"Python sidecar returned no JSON response. stderr: {stderr}".Trim()); + + JsonDocument doc; + try + { + doc = JsonDocument.Parse(line); + } + catch (JsonException ex) + { + throw new InvalidOperationException($"Invalid JSON from Python sidecar: {line}", ex); + } + + using (doc) + { + var root = doc.RootElement; + if (!root.TryGetProperty("ok", out var okNode) || okNode.ValueKind != JsonValueKind.True && okNode.ValueKind != JsonValueKind.False) + throw new InvalidOperationException("Python sidecar response missing boolean 'ok' field."); + + if (!okNode.GetBoolean()) + { + var error = root.TryGetProperty("error", out var errorNode) + ? errorNode.GetString() ?? "Unknown sidecar error." + : "Unknown sidecar error."; + throw new InvalidOperationException(error); + } + + if (!root.TryGetProperty("data", out var dataNode)) + return null; + + return dataNode.Clone(); + } + } + + private static string LastJsonLine(string text) + { + var lines = text.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries); + for (var i = lines.Length - 1; i >= 0; i--) + { + var line = lines[i].Trim(); + if (line.StartsWith("{", StringComparison.Ordinal) && line.EndsWith("}", StringComparison.Ordinal)) + return line; + } + return ""; + } + + private static void TryKill(Process process) + { + try + { + if (!process.HasExited) + process.Kill(entireProcessTree: true); + } + catch + { + // Ignore timeout cleanup failures. + } + } +} diff --git a/journal-master/journal/Journal.Core/Services/SidecarCli.cs b/journal-master/journal/Journal.Core/Services/SidecarCli.cs new file mode 100644 index 0000000..86cf444 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/SidecarCli.cs @@ -0,0 +1,385 @@ +using System.Text; +using Journal.Core.Dtos; + +namespace Journal.Core.Services; + +public sealed class SidecarCli +{ + private readonly IVaultStorageService _vaultStorage; + private readonly IEntrySearchService _entrySearch; + private readonly IJournalConfigService _config; + + public SidecarCli(IVaultStorageService vaultStorage, IEntrySearchService entrySearch, IJournalConfigService config) + { + _vaultStorage = vaultStorage; + _entrySearch = entrySearch; + _config = config; + } + + public async Task RunAsync(string[] args, Entry entry) + { + ArgumentNullException.ThrowIfNull(args); + ArgumentNullException.ThrowIfNull(entry); + + if (args.Length == 0) + { + await entry.RunAsync(); + return 0; + } + + if (IsHelp(args[0])) + { + PrintUsage(); + return 0; + } + + if (string.Equals(args[0], "vault", StringComparison.OrdinalIgnoreCase)) + return RunVaultCommand(args.Skip(1).ToArray()); + if (string.Equals(args[0], "search", StringComparison.OrdinalIgnoreCase)) + return RunSearchCommand(args.Skip(1).ToArray()); + + Console.Error.WriteLine($"Unknown command: {args[0]}"); + PrintUsage(); + return 2; + } + + public int RunVaultCommand(string[] args) + { + ArgumentNullException.ThrowIfNull(args); + if (args.Length == 0 || IsHelp(args[0])) + { + PrintVaultUsage(); + return 2; + } + + var action = args[0].Trim().ToLowerInvariant(); + if (action is not ("load" or "save")) + { + Console.Error.WriteLine($"Unknown vault action: {args[0]}"); + PrintVaultUsage(); + return 2; + } + + if (!TryParseVaultOptions(args.Skip(1).ToArray(), out var options, out var parseError)) + { + Console.Error.WriteLine(parseError); + PrintVaultUsage(); + return 2; + } + + var password = options.Password; + if (string.IsNullOrWhiteSpace(password)) + password = PromptPassword(); + + if (string.IsNullOrWhiteSpace(password)) + { + Console.Error.WriteLine("Vault password cannot be empty."); + return 2; + } + + var (vaultDirectory, dataDirectory) = ResolveDirectories(options.VaultDirectory, options.DataDirectory); + + try + { + if (action == "load") + { + var ok = _vaultStorage.LoadAllVaults(password, vaultDirectory, dataDirectory); + if (!ok) + { + Console.Error.WriteLine("Incorrect password."); + return 1; + } + + Console.WriteLine($"Vault loaded. Decrypted files are in {dataDirectory}"); + return 0; + } + + _vaultStorage.RebuildAllVaults(password, vaultDirectory, dataDirectory); + Console.WriteLine($"Vault saved from decrypted files in {dataDirectory}"); + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine($"Vault command failed: {ex.Message}"); + return 1; + } + } + + public int RunSearchCommand(string[] args) + { + ArgumentNullException.ThrowIfNull(args); + if (args.Length > 0 && IsHelp(args[0])) + { + PrintSearchUsage(); + return 0; + } + + if (!TryParseSearchOptions(args, out var options, out var parseError)) + { + Console.Error.WriteLine(parseError); + PrintSearchUsage(); + return 2; + } + + var (_, dataDirectory) = ResolveDirectories(vaultOverride: null, options.DataDirectory); + if (!Directory.Exists(dataDirectory) || Directory.GetFiles(dataDirectory, "*.md").Length == 0) + { + Console.WriteLine("No decrypted journal entries found. Please load the vault first: journal vault load"); + return 0; + } + + try + { + var request = new EntrySearchRequestDto( + DataDirectory: dataDirectory, + Query: options.Query, + Section: options.Section, + StartDate: options.StartDate, + EndDate: options.EndDate, + Tags: options.Tags, + Types: options.Types, + Checked: options.Checked, + Unchecked: options.Unchecked); + + var results = _entrySearch.SearchEntriesAsync(request).GetAwaiter().GetResult(); + if (results.Count == 0) + { + Console.WriteLine("No entries found matching the criteria."); + return 0; + } + + foreach (var result in results) + { + Console.WriteLine($"--- {result.Date} ---"); + Console.WriteLine(result.RawContent); + Console.WriteLine(); + } + + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine($"Search command failed: {ex.Message}"); + return 1; + } + } + + private static bool TryParseVaultOptions(string[] args, out VaultOptions options, out string error) + { + var parsed = new VaultOptions(); + for (var i = 0; i < args.Length; i++) + { + var token = args[i]; + if (IsHelp(token)) + { + options = parsed; + error = ""; + return false; + } + + if (i + 1 >= args.Length) + { + options = parsed; + error = $"Missing value for option '{token}'."; + return false; + } + + var value = args[i + 1]; + switch (token) + { + case "--password": + case "-p": + parsed.Password = value; + break; + case "--vault-dir": + parsed.VaultDirectory = value; + break; + case "--data-dir": + parsed.DataDirectory = value; + break; + default: + options = parsed; + error = $"Unknown option '{token}'."; + return false; + } + + i++; + } + + options = parsed; + error = ""; + return true; + } + + private static bool TryParseSearchOptions(string[] args, out SearchOptions options, out string error) + { + var parsed = new SearchOptions(); + for (var i = 0; i < args.Length; i++) + { + var token = args[i]; + if (IsHelp(token)) + { + options = parsed; + error = ""; + return false; + } + + if (!token.StartsWith("-", StringComparison.Ordinal)) + { + if (parsed.Query is null) + { + parsed.Query = token; + continue; + } + + options = parsed; + error = $"Unexpected positional argument '{token}'."; + return false; + } + + if (i + 1 >= args.Length) + { + options = parsed; + error = $"Missing value for option '{token}'."; + return false; + } + + var value = args[i + 1]; + switch (token) + { + case "--data-dir": + parsed.DataDirectory = value; + break; + case "--tag": + case "-t": + parsed.Tags.Add(value); + break; + case "--type": + case "-y": + parsed.Types.Add(value); + break; + case "--start-date": + case "-s": + parsed.StartDate = value; + break; + case "--end-date": + case "-e": + parsed.EndDate = value; + break; + case "--section": + case "-sec": + parsed.Section = value; + break; + case "--checked": + case "-chk": + parsed.Checked.Add(value); + break; + case "--unchecked": + case "-uchk": + parsed.Unchecked.Add(value); + break; + default: + options = parsed; + error = $"Unknown option '{token}'."; + return false; + } + + i++; + } + + options = parsed; + error = ""; + return true; + } + + private (string VaultDirectory, string DataDirectory) ResolveDirectories(string? vaultOverride, string? dataOverride) + { + var envVault = Environment.GetEnvironmentVariable("JOURNAL_VAULT_DIR"); + var envData = Environment.GetEnvironmentVariable("JOURNAL_DATA_DIR"); + var defaults = _config.Current; + + var vault = FirstNonEmpty(vaultOverride, envVault) ?? defaults.VaultDirectory; + var data = FirstNonEmpty(dataOverride, envData) ?? defaults.DataDirectory; + return (Path.GetFullPath(vault), Path.GetFullPath(data)); + } + + private static string? FirstNonEmpty(params string?[] values) => + values.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value)); + + private static string PromptPassword() + { + if (Console.IsInputRedirected) + return Console.ReadLine() ?? ""; + + Console.Write("Vault password: "); + var builder = new StringBuilder(); + while (true) + { + var keyInfo = Console.ReadKey(intercept: true); + if (keyInfo.Key == ConsoleKey.Enter) + { + Console.WriteLine(); + break; + } + + if (keyInfo.Key == ConsoleKey.Backspace) + { + if (builder.Length > 0) + builder.Length--; + continue; + } + + if (!char.IsControl(keyInfo.KeyChar)) + builder.Append(keyInfo.KeyChar); + } + + return builder.ToString(); + } + + private static bool IsHelp(string token) => + string.Equals(token, "--help", StringComparison.OrdinalIgnoreCase) || + string.Equals(token, "-h", StringComparison.OrdinalIgnoreCase) || + string.Equals(token, "help", StringComparison.OrdinalIgnoreCase); + + private static void PrintUsage() + { + Console.WriteLine("Usage:"); + Console.WriteLine(" Journal.Sidecar # sidecar stdin/stdout mode"); + Console.WriteLine(" Journal.Sidecar vault load [--password ] [--vault-dir ] [--data-dir ]"); + Console.WriteLine(" Journal.Sidecar vault save [--password ] [--vault-dir ] [--data-dir ]"); + Console.WriteLine(" Journal.Sidecar search [query] [--tag ] [--type ] [--start-date ] [--end-date ] [--section ] [--checked <text>] [--unchecked <text>] [--data-dir <path>]"); + } + + private static void PrintVaultUsage() + { + Console.WriteLine("Vault usage:"); + Console.WriteLine(" Journal.Sidecar vault load [--password <value>] [--vault-dir <path>] [--data-dir <path>]"); + Console.WriteLine(" Journal.Sidecar vault save [--password <value>] [--vault-dir <path>] [--data-dir <path>]"); + } + + private static void PrintSearchUsage() + { + Console.WriteLine("Search usage:"); + Console.WriteLine(" Journal.Sidecar search [query] [--tag <value>] [--type <value>] [--start-date <yyyy-MM-dd>] [--end-date <yyyy-MM-dd>] [--section <title>] [--checked <text>] [--unchecked <text>] [--data-dir <path>]"); + } + + private sealed class VaultOptions + { + public string? Password { get; set; } + public string? VaultDirectory { get; set; } + public string? DataDirectory { get; set; } + } + + private sealed class SearchOptions + { + public string? Query { get; set; } + public string? DataDirectory { get; set; } + public string? StartDate { get; set; } + public string? EndDate { get; set; } + public string? Section { get; set; } + public List<string> Tags { get; } = []; + public List<string> Types { get; } = []; + public List<string> Checked { get; } = []; + public List<string> Unchecked { get; } = []; + } +} diff --git a/journal-master/journal/Journal.Core/Services/VaultCryptoService.cs b/journal-master/journal/Journal.Core/Services/VaultCryptoService.cs new file mode 100644 index 0000000..e6c7df7 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/VaultCryptoService.cs @@ -0,0 +1,83 @@ +using System.Security.Cryptography; +using System.Text; + +namespace Journal.Core.Services; + +public class VaultCryptoService : IVaultCryptoService +{ + public const int SaltSize = 16; + public const int KeySize = 32; + public const int NonceSize = 12; + public const int TagSize = 16; + public const int Iterations = 600_000; + + public byte[] DeriveKey(string password, byte[] salt) + { + if (string.IsNullOrWhiteSpace(password)) + throw new ArgumentException("Password cannot be empty.", nameof(password)); + ArgumentNullException.ThrowIfNull(salt); + if (salt.Length != SaltSize) + throw new ArgumentException($"Salt must be {SaltSize} bytes.", nameof(salt)); + + return Rfc2898DeriveBytes.Pbkdf2( + Encoding.UTF8.GetBytes(password), + salt, + Iterations, + HashAlgorithmName.SHA256, + KeySize); + } + + public byte[] EncryptData(byte[] data, string password) + { + ArgumentNullException.ThrowIfNull(data); + + var salt = RandomNumberGenerator.GetBytes(SaltSize); + var nonce = RandomNumberGenerator.GetBytes(NonceSize); + return EncryptData(data, password, salt, nonce); + } + + public byte[] DecryptData(byte[] encryptedData, string password) + { + ArgumentNullException.ThrowIfNull(encryptedData); + + var minLength = SaltSize + NonceSize + TagSize; + if (encryptedData.Length < minLength) + throw new ArgumentException("Encrypted payload is too short.", nameof(encryptedData)); + + var salt = encryptedData.AsSpan(0, SaltSize).ToArray(); + var nonce = encryptedData.AsSpan(SaltSize, NonceSize).ToArray(); + var tag = encryptedData.AsSpan(SaltSize + NonceSize, TagSize).ToArray(); + var ciphertext = encryptedData.AsSpan(SaltSize + NonceSize + TagSize).ToArray(); + + var key = DeriveKey(password, salt); + var plaintext = new byte[ciphertext.Length]; + using var aes = new AesGcm(key, TagSize); + aes.Decrypt(nonce, ciphertext, tag, plaintext); + return plaintext; + } + + public byte[] EncryptData(byte[] data, string password, byte[] salt, byte[] nonce) + { + ArgumentNullException.ThrowIfNull(data); + ArgumentNullException.ThrowIfNull(salt); + ArgumentNullException.ThrowIfNull(nonce); + if (salt.Length != SaltSize) + throw new ArgumentException($"Salt must be {SaltSize} bytes.", nameof(salt)); + if (nonce.Length != NonceSize) + throw new ArgumentException($"Nonce must be {NonceSize} bytes.", nameof(nonce)); + + var key = DeriveKey(password, salt); + var ciphertext = new byte[data.Length]; + var tag = new byte[TagSize]; + + using var aes = new AesGcm(key, TagSize); + aes.Encrypt(nonce, data, ciphertext, tag); + + var payload = new byte[SaltSize + NonceSize + TagSize + ciphertext.Length]; + Buffer.BlockCopy(salt, 0, payload, 0, SaltSize); + Buffer.BlockCopy(nonce, 0, payload, SaltSize, NonceSize); + Buffer.BlockCopy(tag, 0, payload, SaltSize + NonceSize, TagSize); + Buffer.BlockCopy(ciphertext, 0, payload, SaltSize + NonceSize + TagSize, ciphertext.Length); + return payload; + } +} diff --git a/journal-master/journal/Journal.Core/Services/VaultStorageService.cs b/journal-master/journal/Journal.Core/Services/VaultStorageService.cs new file mode 100644 index 0000000..2ccf253 --- /dev/null +++ b/journal-master/journal/Journal.Core/Services/VaultStorageService.cs @@ -0,0 +1,276 @@ +using System.IO.Compression; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Threading; + +namespace Journal.Core.Services; + +public class VaultStorageService : IVaultStorageService +{ + private readonly IVaultCryptoService _crypto; + private readonly Dictionary<string, string> _monthFingerprintCache = new(StringComparer.Ordinal); + private readonly object _vaultIoLock = new(); + + public VaultStorageService(IVaultCryptoService crypto) => _crypto = crypto; + + public string GetMonthlyVaultFileName(DateTime date) => date.ToString("yyyy-MM") + ".vault"; + + public bool LoadAllVaults(string password, string vaultDirectory, string dataDirectory) + { + EnsureRequiredArguments(password, vaultDirectory, dataDirectory); + + lock (_vaultIoLock) + { + _monthFingerprintCache.Clear(); + PrepareDataDirectory(dataDirectory); + + if (!Directory.Exists(vaultDirectory)) + return true; + + var vaultFiles = Directory.GetFiles(vaultDirectory, "*.vault") + .OrderBy(Path.GetFileName, StringComparer.Ordinal) + .ToArray(); + if (vaultFiles.Length == 0) + return true; + + var anyDecrypted = false; + var anyVaultFiles = false; + foreach (var vaultFile in vaultFiles) + { + var fileName = Path.GetFileName(vaultFile); + if (string.Equals(fileName, "_init_vault.vault", StringComparison.OrdinalIgnoreCase)) + { + try + { + File.Delete(vaultFile); + } + catch + { + // Legacy file cleanup should never block loading. + } + continue; + } + + anyVaultFiles = true; + try + { + var encrypted = File.ReadAllBytes(vaultFile); + var decryptedZip = _crypto.DecryptData(encrypted, password); + ExtractZipContent(decryptedZip, dataDirectory); + anyDecrypted = true; + } + catch (CryptographicException) + { + // Wrong password for this vault file; continue trying others. + } + catch + { + // Non-password vault read/decrypt/extract error; continue loading others. + } + } + + if (!anyDecrypted && anyVaultFiles) + return false; + + return true; + } + } + + public bool SaveCurrentMonthVault(string password, string vaultDirectory, string dataDirectory, DateTime now) + { + EnsureRequiredArguments(password, vaultDirectory, dataDirectory); + + lock (_vaultIoLock) + { + Directory.CreateDirectory(vaultDirectory); + if (!Directory.Exists(dataDirectory)) + return false; + + var monthKey = now.ToString("yyyy-MM", CultureInfo.InvariantCulture); + var filesInMonth = Directory.GetFiles(dataDirectory, "*.md") + .Where(path => Path.GetFileNameWithoutExtension(path).StartsWith(monthKey, StringComparison.Ordinal)) + .OrderBy(Path.GetFileName, StringComparer.Ordinal) + .ToList(); + + if (filesInMonth.Count == 0) + return false; + + var currentFingerprint = ComputeMonthFingerprint(filesInMonth); + if (_monthFingerprintCache.TryGetValue(monthKey, out var cachedFingerprint) && + string.Equals(cachedFingerprint, currentFingerprint, StringComparison.Ordinal)) + { + return false; + } + + SaveMonth(password, monthKey, filesInMonth, vaultDirectory); + return true; + } + } + + public void RebuildAllVaults(string password, string vaultDirectory, string dataDirectory) + { + EnsureRequiredArguments(password, vaultDirectory, dataDirectory); + + lock (_vaultIoLock) + { + Directory.CreateDirectory(vaultDirectory); + if (!Directory.Exists(dataDirectory)) + return; + + var monthlyFiles = new Dictionary<string, List<string>>(StringComparer.Ordinal); + foreach (var filePath in Directory.GetFiles(dataDirectory, "*.md")) + { + var stem = Path.GetFileNameWithoutExtension(filePath); + if (!DateTime.TryParseExact(stem, "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var fileDate)) + continue; + + var monthKey = fileDate.ToString("yyyy-MM", CultureInfo.InvariantCulture); + if (!monthlyFiles.TryGetValue(monthKey, out var files)) + { + files = []; + monthlyFiles[monthKey] = files; + } + + files.Add(filePath); + } + + foreach (var (monthKey, filesInMonth) in monthlyFiles.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + SaveMonth(password, monthKey, filesInMonth, vaultDirectory); + } + } + + public void ClearDataDirectory(string dataDirectory) + { + if (string.IsNullOrWhiteSpace(dataDirectory)) + throw new ArgumentException("Data directory is required.", nameof(dataDirectory)); + + lock (_vaultIoLock) + { + PrepareDataDirectory(dataDirectory); + _monthFingerprintCache.Clear(); + } + } + + private static void PrepareDataDirectory(string dataDirectory) + { + DeleteDirectoryWithRetries(dataDirectory); + Directory.CreateDirectory(dataDirectory); + } + + private static void DeleteDirectoryWithRetries(string dataDirectory, int retries = 5, int delayMs = 200) + { + if (!Directory.Exists(dataDirectory)) + return; + + for (var attempt = 0; attempt < retries; attempt++) + { + try + { + Directory.Delete(dataDirectory, recursive: true); + return; + } + catch (IOException) when (attempt < retries - 1) + { + Thread.Sleep(delayMs); + } + catch (UnauthorizedAccessException) when (attempt < retries - 1) + { + Thread.Sleep(delayMs); + } + } + + // Final attempt should throw with the underlying exception if deletion still fails. + Directory.Delete(dataDirectory, recursive: true); + } + + private static void EnsureRequiredArguments(string password, string vaultDirectory, string dataDirectory) + { + if (string.IsNullOrWhiteSpace(password)) + throw new ArgumentException("Password cannot be empty.", nameof(password)); + if (string.IsNullOrWhiteSpace(vaultDirectory)) + throw new ArgumentException("Vault directory is required.", nameof(vaultDirectory)); + if (string.IsNullOrWhiteSpace(dataDirectory)) + throw new ArgumentException("Data directory is required.", nameof(dataDirectory)); + } + + private void SaveMonth(string password, string monthKey, List<string> filesInMonth, string vaultDirectory) + { + var monthDate = DateTime.ParseExact(monthKey, "yyyy-MM", CultureInfo.InvariantCulture); + var monthlyVaultPath = Path.Combine(vaultDirectory, GetMonthlyVaultFileName(monthDate)); + + var zipBytes = CreateMonthlyArchive(filesInMonth); + var encryptedPayload = _crypto.EncryptData(zipBytes, password); + File.WriteAllBytes(monthlyVaultPath, encryptedPayload); + + _monthFingerprintCache[monthKey] = ComputeMonthFingerprint(filesInMonth); + } + + private static byte[] CreateMonthlyArchive(List<string> filesInMonth) + { + using var memoryStream = new MemoryStream(); + using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var filePath in filesInMonth.OrderBy(Path.GetFileName, StringComparer.Ordinal)) + { + var fileName = Path.GetFileName(filePath); + if (string.IsNullOrWhiteSpace(fileName)) + continue; + + var entry = archive.CreateEntry(fileName, CompressionLevel.Optimal); + using var entryStream = entry.Open(); + using var sourceStream = File.OpenRead(filePath); + sourceStream.CopyTo(entryStream); + } + } + + return memoryStream.ToArray(); + } + + private static string ComputeMonthFingerprint(List<string> files) + { + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + + foreach (var filePath in files.OrderBy(Path.GetFileName, StringComparer.Ordinal)) + { + var fileInfo = new FileInfo(filePath); + if (!fileInfo.Exists) + continue; + + AppendUtf8(hash, fileInfo.Name); + AppendAscii(hash, fileInfo.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture)); + AppendAscii(hash, fileInfo.Length.ToString(CultureInfo.InvariantCulture)); + } + + return Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); + } + + private static void AppendUtf8(IncrementalHash hash, string value) => hash.AppendData(Encoding.UTF8.GetBytes(value)); + private static void AppendAscii(IncrementalHash hash, string value) => hash.AppendData(Encoding.ASCII.GetBytes(value)); + + private static void ExtractZipContent(byte[] zipBytes, string dataDirectory) + { + using var stream = new MemoryStream(zipBytes); + using var archive = new ZipArchive(stream, ZipArchiveMode.Read); + + var dataRoot = Path.GetFullPath(dataDirectory); + if (!dataRoot.EndsWith(Path.DirectorySeparatorChar)) + dataRoot += Path.DirectorySeparatorChar; + + foreach (var entry in archive.Entries) + { + if (string.IsNullOrEmpty(entry.Name)) + continue; + + var destinationPath = Path.GetFullPath(Path.Combine(dataDirectory, entry.FullName)); + if (!destinationPath.StartsWith(dataRoot, StringComparison.OrdinalIgnoreCase)) + throw new InvalidDataException("Zip entry path escapes target data directory."); + + var destinationDir = Path.GetDirectoryName(destinationPath); + if (!string.IsNullOrWhiteSpace(destinationDir)) + Directory.CreateDirectory(destinationDir); + + entry.ExtractToFile(destinationPath, overwrite: true); + } + } +} diff --git a/journal-master/journal/Journal.Sidecar/App.cs b/journal-master/journal/Journal.Sidecar/App.cs new file mode 100644 index 0000000..cb0c2f1 --- /dev/null +++ b/journal-master/journal/Journal.Sidecar/App.cs @@ -0,0 +1,13 @@ +using Microsoft.Extensions.DependencyInjection; +using Journal.Core; +using Journal.Core.Services; + +var services = new ServiceCollection(); +services.AddFragmentServices(); +services.AddSingleton<Entry>(); +var provider = services.BuildServiceProvider(); + +var entry = provider.GetRequiredService<Entry>(); +var cli = provider.GetRequiredService<SidecarCli>(); +var exitCode = await cli.RunAsync(args, entry); +Environment.ExitCode = exitCode; diff --git a/journal-master/journal/Journal.Sidecar/Journal.Sidecar.csproj b/journal-master/journal/Journal.Sidecar/Journal.Sidecar.csproj new file mode 100644 index 0000000..d9b4688 --- /dev/null +++ b/journal-master/journal/Journal.Sidecar/Journal.Sidecar.csproj @@ -0,0 +1,18 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <OutputType>Exe</OutputType> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="..\Journal.Core\Journal.Core.csproj" /> + </ItemGroup> + + <ItemGroup> + <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.3" /> + </ItemGroup> + +</Project> diff --git a/journal-master/journal/Journal.SmokeTests/Fixtures/transport_cases.json b/journal-master/journal/Journal.SmokeTests/Fixtures/transport_cases.json new file mode 100644 index 0000000..b6c84dd --- /dev/null +++ b/journal-master/journal/Journal.SmokeTests/Fixtures/transport_cases.json @@ -0,0 +1,50 @@ +[ + { + "name": "List returns array envelope", + "request": "{\"action\":\"fragments.list\"}", + "expectOk": true, + "dataKind": "array" + }, + { + "name": "Create returns object envelope", + "request": "{\"action\":\"fragments.create\",\"payload\":{\"type\":\"!NOTE\",\"description\":\"fixture create\"}}", + "expectOk": true, + "dataKind": "object" + }, + { + "name": "Get missing id returns null data", + "request": "{\"action\":\"fragments.get\",\"id\":\"00000000-0000-0000-0000-000000000001\"}", + "expectOk": true, + "dataKind": "null" + }, + { + "name": "Create missing payload fails", + "request": "{\"action\":\"fragments.create\"}", + "expectOk": false, + "errorContains": "payload" + }, + { + "name": "AI health returns object envelope", + "request": "{\"action\":\"ai.health\"}", + "expectOk": true, + "dataKind": "object" + }, + { + "name": "AI summarize entry returns string envelope", + "request": "{\"action\":\"ai.summarize_entry\",\"payload\":{\"content\":\"transport test\"}}", + "expectOk": true, + "dataKind": "string" + }, + { + "name": "Unknown action fails", + "request": "{\"action\":\"unknown.action\"}", + "expectOk": false, + "errorContains": "Unknown action" + }, + { + "name": "Malformed JSON fails", + "request": "{\"action\":\"fragments.list\"", + "expectOk": false, + "errorContains": "Invalid command JSON" + } +] diff --git a/journal-master/journal/Journal.SmokeTests/Journal.SmokeTests.csproj b/journal-master/journal/Journal.SmokeTests/Journal.SmokeTests.csproj new file mode 100644 index 0000000..a81680c --- /dev/null +++ b/journal-master/journal/Journal.SmokeTests/Journal.SmokeTests.csproj @@ -0,0 +1,20 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <OutputType>Exe</OutputType> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="..\Journal.Core\Journal.Core.csproj" /> + </ItemGroup> + + <ItemGroup> + <None Update="Fixtures\*.json"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + </ItemGroup> + +</Project> diff --git a/journal-master/journal/Journal.SmokeTests/Program.cs b/journal-master/journal/Journal.SmokeTests/Program.cs new file mode 100644 index 0000000..55eeddf --- /dev/null +++ b/journal-master/journal/Journal.SmokeTests/Program.cs @@ -0,0 +1,2129 @@ +using System.ComponentModel.DataAnnotations; +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text.Json; +using Journal.Core; +using Journal.Core.Dtos; +using Journal.Core.Models; +using Journal.Core.Repositories; +using Journal.Core.Services; + +var tests = new List<(string Name, Func<Task> Run)> +{ + ("CreateAsync trims fields", TestCreateTrimsAsync), + ("UpdateAsync accepts valid type updates", TestUpdateAcceptsTypeAsync), + ("UpdateAsync rejects whitespace type", TestUpdateRejectsWhitespaceTypeAsync), + ("JournalEntry model stores parity fields", TestJournalEntryModelAsync), + ("MergeWith overwrites section when new content is meaningful", TestMergeOverwritesMeaningfulSectionAsync), + ("MergeWith ignores whitespace-only section updates", TestMergeIgnoresWhitespaceOnlySectionAsync), + ("MergeWith appends non-duplicate fragments by description", TestMergeAppendsNonDuplicateFragmentsAsync), + ("ToMarkdown writes canonical section order", TestToMarkdownCanonicalSectionOrderAsync), + ("ToMarkdown writes fragment blocks", TestToMarkdownFragmentFormattingAsync), + ("Vault crypto roundtrip preserves data and layout", TestVaultCryptoRoundtripAsync), + ("Vault crypto decrypts Python payload fixture", TestVaultCryptoDecryptsPythonFixtureAsync), + ("Vault key derivation matches Python fixture", TestVaultKeyDerivationMatchesPythonAsync), + ("Vault monthly filename matches parity format", TestVaultMonthlyFilenameParityAsync), + ("Vault load clears workspace and extracts decrypted files", TestVaultLoadClearsAndExtractsAsync), + ("Vault load wrong password does not modify vault files", TestVaultLoadWrongPasswordPreservesVaultAsync), + ("Vault load ignores and removes legacy _init_vault.vault", TestVaultLoadLegacyInitVaultHandlingAsync), + ("Vault current-month save writes only current month and skips unchanged state", TestVaultCurrentMonthSaveOptimizedAsync), + ("Vault rebuild saves grouped monthly archives from decrypted files", TestVaultRebuildAllVaultsAsync), + ("Vault clear data directory removes decrypted workspace artifacts", TestVaultClearDataDirectoryAsync), + ("Parser extracts date from **Date:** marker", TestParserExtractsBoldDateAsync), + ("Parser extracts date from Date: marker", TestParserExtractsPlainDateAsync), + ("Parser falls back to file stem when date missing", TestParserFallsBackToFileStemAsync), + ("Parser captures canonical sections and content", TestParserCapturesSectionsAsync), + ("Parser ignores non-canonical section headers", TestParserIgnoresNonCanonicalHeadersAsync), + ("Parser captures checkbox states per section", TestParserCapturesCheckboxStatesAsync), + ("Parser captures multiline fragment blocks", TestParserCapturesMultilineFragmentsAsync), + ("Parser fragment boundary follows header lines", TestParserFragmentBoundaryBehaviorAsync), + ("File repository persists fragments", TestFileRepositoryPersistsAsync), + ("Entry invalid JSON returns error envelope", TestEntryInvalidJsonAsync), + ("Entry unknown action returns error envelope", TestEntryUnknownActionAsync), + ("Entry get missing id returns ok with null data", TestEntryGetMissingReturnsNullDataAsync), + ("Entry create without payload returns error envelope", TestEntryCreateMissingPayloadAsync), + ("Entry entries.save writes and merges content", TestEntryEntriesSaveMergeAsync), + ("Entry entries.load returns raw content payload", TestEntryEntriesLoadAsync), + ("Entry entries.list returns markdown files", TestEntryEntriesListAsync), + ("Entry search.entries matches query against full raw content", TestEntrySearchEntriesMatchesRawContentAsync), + ("Entry search.entries without query returns all markdown entries", TestEntrySearchEntriesWithoutQueryReturnsAllAsync), + ("Entry search.entries applies date range filter", TestEntrySearchEntriesDateRangeFilterAsync), + ("Entry search.entries applies section-scoped query filter", TestEntrySearchEntriesSectionFilterAsync), + ("Entry search.entries applies fragment tag and type filters", TestEntrySearchEntriesTagTypeFilterAsync), + ("Entry search.entries applies checkbox checked and unchecked filters", TestEntrySearchEntriesCheckboxFilterAsync), + ("Entry search.entries rejects invalid date filter format", TestEntrySearchEntriesRejectsInvalidDateAsync), + ("Database key derivation matches Python fixture", TestDatabaseKeyDerivationMatchesPythonAsync), + ("Database schema parity tables are created", TestDatabaseSchemaParityAsync), + ("Entry db.status returns database compatibility payload", TestEntryDatabaseStatusAsync), + ("Entry db.initialize_schema creates schema in data directory", TestEntryDatabaseInitializeSchemaAsync), + ("Entry db.hydrate_workspace returns hydration metadata", TestEntryDatabaseHydrateWorkspaceAsync), + ("Config service exposes parity path, vault, AI, and speech settings", TestConfigServiceParityKeysAsync), + ("Entry config.get returns config payload", TestEntryConfigGetAsync), + ("Log redactor scrubs sensitive payload fields", TestLogRedactorScrubsSensitiveFieldsAsync), + ("Log redactor preserves non-sensitive payload fields", TestLogRedactorPreservesNonSensitiveFieldsAsync), + ("Entry ai.health returns disabled by default", TestEntryAiHealthDefaultAsync), + ("Entry ai.summarize_entry succeeds when disabled", TestEntryAiSummarizeEntryDisabledAsync), + ("Entry ai.summarize_all succeeds when disabled", TestEntryAiSummarizeAllDisabledAsync), + ("Entry ai.chat succeeds when disabled", TestEntryAiChatDisabledAsync), + ("Entry ai.embed returns empty vector when disabled", TestEntryAiEmbedDisabledAsync), + ("Entry speech.devices.list returns envelope when disabled", TestEntrySpeechDevicesListDisabledAsync), + ("Entry speech.transcribe returns envelope when disabled", TestEntrySpeechTranscribeDisabledAsync), + ("Python sidecar AI service parses last JSON line", TestPythonSidecarAiServiceJsonLineAsync), + ("Python sidecar AI service surfaces sidecar errors", TestPythonSidecarAiServiceErrorAsync), + ("Python sidecar speech service handles empty devices payload", TestPythonSidecarSpeechServiceNoDevicesAsync), + ("Python sidecar speech service surfaces unavailable engine errors", TestPythonSidecarSpeechServiceErrorAsync), + ("Python sidecar speech service times out deterministically", TestPythonSidecarSpeechServiceTimeoutAsync), + ("Entry vault.load_all succeeds for empty vault directory", TestEntryVaultLoadAllEmptyAsync), + ("Entry vault.clear_data_directory removes files", TestEntryVaultClearDataDirectoryAsync), + ("Sidecar vault CLI load succeeds with --password", TestSidecarVaultCliLoadAsync), + ("Sidecar vault CLI save writes monthly vault with --password", TestSidecarVaultCliSaveAsync), + ("Sidecar search CLI returns matching entries with filters", TestSidecarSearchCliFilteredAsync), + ("Sidecar search CLI warns when no decrypted entries exist", TestSidecarSearchCliEmptyDataAsync), + ("Transport fixtures produce stable envelopes", TestTransportFixturesAsync), +}; + +var passed = 0; +foreach (var (name, run) in tests) +{ + try + { + await run(); + Console.WriteLine($"PASS {name}"); + passed++; + } + catch (Exception ex) + { + Console.WriteLine($"FAIL {name}: {ex.Message}"); + } +} + +Console.WriteLine($"Summary: {passed}/{tests.Count} passed."); +Environment.ExitCode = passed == tests.Count ? 0 : 1; + +static FragmentService NewService() +{ + IFragmentRepository repo = new InMemoryFragmentRepository(); + return new FragmentService(repo); +} + +static Entry NewEntry() => new( + NewService(), + new EntrySearchService(), + new VaultStorageService(new VaultCryptoService()), + new JournalDatabaseService(new JournalConfigService()), + new JournalConfigService(), + new DisabledAiService("none"), + new DisabledSpeechBridgeService("none")); + +static IJournalDatabaseService NewDatabaseService() => new JournalDatabaseService(new JournalConfigService()); + +static async Task TestCreateTrimsAsync() +{ + var service = NewService(); + var created = await service.CreateAsync(new CreateFragmentDto(" !TRIGGER ", " stomach drop ", [" stress ", "", " body "])); + + Assert(created.Type == "!TRIGGER", "Type should be trimmed."); + Assert(created.Description == "stomach drop", "Description should be trimmed."); + Assert(created.Tags.Count == 2, "Expected two normalized tags."); + Assert(created.Tags[0] == "stress" && created.Tags[1] == "body", "Tags should be trimmed and preserved."); +} + +static async Task TestUpdateAcceptsTypeAsync() +{ + var service = NewService(); + var created = await service.CreateAsync(new CreateFragmentDto("!TRIGGER", "one")); + var ok = await service.UpdateAsync(created.Id, new UpdateFragmentDto(Type: " !FLASHBACK ", Description: " two ", Tags: [" memory "])); + + Assert(ok, "Expected update to succeed."); + var updated = await service.GetByIdAsync(created.Id); + Assert(updated is not null, "Updated fragment should exist."); + Assert(updated!.Type == "!FLASHBACK", "Updated type should be trimmed and stored."); + Assert(updated.Description == "two", "Updated description should be trimmed and stored."); + Assert(updated.Tags.Count == 1 && updated.Tags[0] == "memory", "Updated tags should be normalized."); +} + +static async Task TestUpdateRejectsWhitespaceTypeAsync() +{ + var service = NewService(); + var created = await service.CreateAsync(new CreateFragmentDto("!TRIGGER", "desc")); + + try + { + _ = await service.UpdateAsync(created.Id, new UpdateFragmentDto(Type: " ")); + } + catch (ValidationException) + { + return; + } + + throw new InvalidOperationException("Expected ValidationException for whitespace type update."); +} + +static async Task TestFileRepositoryPersistsAsync() +{ + var tempRoot = Path.Combine(Path.GetTempPath(), "journal-smoke", Guid.NewGuid().ToString("N")); + var storePath = Path.Combine(tempRoot, "fragments.json"); + + try + { + IFragmentRepository repo1 = new FileFragmentRepository(storePath); + var service1 = new FragmentService(repo1); + var created = await service1.CreateAsync(new CreateFragmentDto("!TRIGGER", "persist me", ["tag1"])); + + IFragmentRepository repo2 = new FileFragmentRepository(storePath); + var service2 = new FragmentService(repo2); + var loaded = await service2.GetByIdAsync(created.Id); + + Assert(loaded is not null, "Expected fragment to persist across repository instances."); + Assert(loaded!.Description == "persist me", "Persisted fragment description mismatch."); + Assert(loaded.Tags.Count == 1 && loaded.Tags[0] == "tag1", "Persisted tags mismatch."); + } + finally + { + if (Directory.Exists(tempRoot)) + Directory.Delete(tempRoot, recursive: true); + } +} + +static Task TestJournalEntryModelAsync() +{ + var fragment = new Fragment("!TRIGGER", "test fragment"); + var section = new ParsedSection( + "Summary", + content: ["line one", "- [x] completed thing"], + checkboxes: new Dictionary<string, bool> { ["completed thing"] = true }); + + var entry = new JournalEntry( + date: "2026-02-22", + fragments: [fragment], + rawContent: "raw markdown content", + sections: new Dictionary<string, ParsedSection> { ["Summary"] = section }); + + Assert(entry.Date == "2026-02-22", "JournalEntry date mismatch."); + Assert(entry.RawContent == "raw markdown content", "JournalEntry raw content mismatch."); + Assert(entry.Fragments.Count == 1, "JournalEntry fragment count mismatch."); + Assert(entry.Sections.Count == 1, "JournalEntry section count mismatch."); + Assert(entry.GetSection("Summary").Contains("line one"), "JournalEntry section content mismatch."); + Assert(entry.GetCheckboxState("Summary", "completed thing") is true, "JournalEntry checkbox state mismatch."); + Assert(entry.GetCheckboxState("Summary", "missing") is null, "JournalEntry checkbox should return null when missing."); + + return Task.CompletedTask; +} + +static Task TestMergeOverwritesMeaningfulSectionAsync() +{ + var current = new JournalEntry( + date: "2026-02-22", + sections: new Dictionary<string, ParsedSection> + { + ["Summary"] = new ParsedSection("Summary", ["old content"]) + }); + + var incoming = new JournalEntry( + date: "2026-02-22", + sections: new Dictionary<string, ParsedSection> + { + ["Summary"] = new ParsedSection( + "Summary", + [" ", "new content line"], + new Dictionary<string, bool> { ["new check"] = true }), + ["Reflection"] = new ParsedSection("Reflection", ["reflective note"]) + }); + + current.MergeWith(incoming); + + Assert(current.GetSection("Summary").Contains("new content line"), "Meaningful section update should overwrite existing section."); + Assert(!current.GetSection("Summary").Contains("old content"), "Old section content should be replaced."); + Assert(current.GetCheckboxState("Summary", "new check") is true, "Overwritten section checkbox state should come from incoming section."); + Assert(current.GetSection("Reflection").Contains("reflective note"), "Meaningful new section should be added."); + + return Task.CompletedTask; +} + +static Task TestMergeIgnoresWhitespaceOnlySectionAsync() +{ + var current = new JournalEntry( + date: "2026-02-22", + sections: new Dictionary<string, ParsedSection> + { + ["Summary"] = new ParsedSection("Summary", ["keep existing"]) + }); + + var incoming = new JournalEntry( + date: "2026-02-22", + sections: new Dictionary<string, ParsedSection> + { + ["Summary"] = new ParsedSection("Summary", [" ", "\t", ""]) + }); + + current.MergeWith(incoming); + + Assert(current.GetSection("Summary").Contains("keep existing"), "Whitespace-only section update should be ignored."); + + return Task.CompletedTask; +} + +static Task TestMergeAppendsNonDuplicateFragmentsAsync() +{ + var current = new JournalEntry( + date: "2026-02-22", + fragments: + [ + new Fragment("!TRIGGER", "duplicate description") + ]); + + var incoming = new JournalEntry( + date: "2026-02-22", + fragments: + [ + new Fragment("!NOTE", "duplicate description"), + new Fragment("!NOTE", "new description") + ]); + + current.MergeWith(incoming); + + Assert(current.Fragments.Count == 2, "Expected only one new fragment to be appended."); + Assert(current.Fragments.Count(fragment => fragment.Description == "duplicate description") == 1, "Duplicate description should not be appended."); + Assert(current.Fragments.Any(fragment => fragment.Description == "new description"), "New fragment description should be appended."); + + return Task.CompletedTask; +} + +static Task TestToMarkdownCanonicalSectionOrderAsync() +{ + var entry = new JournalEntry( + date: "2026-02-22", + sections: new Dictionary<string, ParsedSection> + { + ["Reflection"] = new ParsedSection("Reflection", ["reflection body"]), + ["Summary"] = new ParsedSection("Summary", ["summary body"]) + }); + + var markdown = entry.ToMarkdown(); + var summaryIdx = markdown.IndexOf("## Summary", StringComparison.Ordinal); + var reflectionIdx = markdown.IndexOf("## Reflection", StringComparison.Ordinal); + + Assert(summaryIdx >= 0, "Summary header should be emitted."); + Assert(reflectionIdx >= 0, "Reflection header should be emitted."); + Assert(summaryIdx < reflectionIdx, "Sections should be emitted in canonical order."); + + return Task.CompletedTask; +} + +static Task TestToMarkdownFragmentFormattingAsync() +{ + var fragment = new Fragment("!TRIGGER", "fragment body") + { + Time = default, + Tags = ["stress", "body"] + }; + var entry = new JournalEntry( + date: "2026-02-22", + fragments: [fragment]); + + var markdown = entry.ToMarkdown(); + + Assert(markdown.Contains("# Fragments\n", StringComparison.Ordinal), "Fragments header should be present."); + Assert(markdown.Contains("!TRIGGER #stress #body\nfragment body\n", StringComparison.Ordinal), "Fragment block format should match parity shape."); + Assert(markdown.Contains("**Date:** 2026-02-22", StringComparison.Ordinal), "Date frontmatter line should be present."); + + return Task.CompletedTask; +} + +static Task TestVaultCryptoRoundtripAsync() +{ + var crypto = new VaultCryptoService(); + var plaintext = "sample vault payload"; + var payload = crypto.EncryptData(System.Text.Encoding.UTF8.GetBytes(plaintext), "vault-pass-123"); + + Assert(payload.Length == VaultCryptoService.SaltSize + VaultCryptoService.NonceSize + VaultCryptoService.TagSize + plaintext.Length, "Vault payload length should match salt+nonce+tag+ciphertext layout."); + var decrypted = crypto.DecryptData(payload, "vault-pass-123"); + Assert(System.Text.Encoding.UTF8.GetString(decrypted) == plaintext, "Vault roundtrip decrypt should return original plaintext."); + + return Task.CompletedTask; +} + +static Task TestVaultCryptoDecryptsPythonFixtureAsync() +{ + var crypto = new VaultCryptoService(); + + var payload = Convert.FromBase64String("AAECAwQFBgcICQoLDA0ODwABAgMEBQYHCAkKC6AErhDEMERBl7OFkG4L4oZ2JZckS0VzhxaZoVLckF7VXE+NIYXILsJ8f1I="); + var expectedPlaintext = Convert.FromBase64String("dmF1bHQgcGF5bG9hZCBleGFtcGxlCmxpbmUy"); + var decrypted = crypto.DecryptData(payload, "vault-pass-123"); + + Assert(decrypted.SequenceEqual(expectedPlaintext), "C# decrypt should match Python-generated payload plaintext."); + + return Task.CompletedTask; +} + +static Task TestVaultKeyDerivationMatchesPythonAsync() +{ + var crypto = new VaultCryptoService(); + var salt = Enumerable.Range(0, VaultCryptoService.SaltSize).Select(i => (byte)i).ToArray(); + var key = crypto.DeriveKey("vault-pass-123", salt); + var expectedKeyHex = "b29f523f28bf178f6815c6ca9ee2a588d79b3bd9a822c92a2f0dde5bc853bb52"; + var actualKeyHex = Convert.ToHexString(key).ToLowerInvariant(); + + Assert(actualKeyHex == expectedKeyHex, "Derived key should match Python PBKDF2 fixture key."); + + return Task.CompletedTask; +} + +static Task TestVaultMonthlyFilenameParityAsync() +{ + IVaultStorageService vaultStorage = new VaultStorageService(new VaultCryptoService()); + var name = vaultStorage.GetMonthlyVaultFileName(new DateTime(2026, 2, 7)); + Assert(name == "2026-02.vault", "Monthly vault filename must match yyyy-MM.vault format."); + return Task.CompletedTask; +} + +static Task TestVaultLoadClearsAndExtractsAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-vault-smoke", Guid.NewGuid().ToString("N")); + var vaultDir = Path.Combine(root, "vault"); + var dataDir = Path.Combine(root, "data"); + Directory.CreateDirectory(vaultDir); + Directory.CreateDirectory(dataDir); + + try + { + File.WriteAllText(Path.Combine(dataDir, "old_file.md"), "stale"); + + var zipBytes = CreateZipBytes(new Dictionary<string, string> + { + ["2026-02-01.md"] = "hello from vault" + }); + var crypto = new VaultCryptoService(); + var encrypted = crypto.EncryptData(zipBytes, "vault-pass-123"); + File.WriteAllBytes(Path.Combine(vaultDir, "2026-02.vault"), encrypted); + + IVaultStorageService storage = new VaultStorageService(crypto); + var ok = storage.LoadAllVaults("vault-pass-123", vaultDir, dataDir); + + Assert(ok, "Expected vault load success with correct password."); + Assert(!File.Exists(Path.Combine(dataDir, "old_file.md")), "Data directory should be cleared before extraction."); + var extractedPath = Path.Combine(dataDir, "2026-02-01.md"); + Assert(File.Exists(extractedPath), "Expected markdown file extracted from vault archive."); + Assert(File.ReadAllText(extractedPath) == "hello from vault", "Extracted file content mismatch."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + + return Task.CompletedTask; +} + +static Task TestVaultLoadWrongPasswordPreservesVaultAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-vault-smoke", Guid.NewGuid().ToString("N")); + var vaultDir = Path.Combine(root, "vault"); + var dataDir = Path.Combine(root, "data"); + Directory.CreateDirectory(vaultDir); + Directory.CreateDirectory(dataDir); + + try + { + var zipBytes = CreateZipBytes(new Dictionary<string, string> + { + ["2026-02-01.md"] = "hello from vault" + }); + var crypto = new VaultCryptoService(); + var encrypted = crypto.EncryptData(zipBytes, "vault-pass-123"); + var vaultPath = Path.Combine(vaultDir, "2026-02.vault"); + File.WriteAllBytes(vaultPath, encrypted); + var before = File.ReadAllBytes(vaultPath); + + IVaultStorageService storage = new VaultStorageService(crypto); + var ok = storage.LoadAllVaults("wrong-password", vaultDir, dataDir); + var after = File.ReadAllBytes(vaultPath); + + Assert(!ok, "Expected vault load failure with wrong password."); + Assert(before.SequenceEqual(after), "Vault file bytes should remain unchanged on wrong password."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + + return Task.CompletedTask; +} + +static Task TestVaultLoadLegacyInitVaultHandlingAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-vault-smoke", Guid.NewGuid().ToString("N")); + var vaultDir = Path.Combine(root, "vault"); + var dataDir = Path.Combine(root, "data"); + Directory.CreateDirectory(vaultDir); + Directory.CreateDirectory(dataDir); + + try + { + var legacyPath = Path.Combine(vaultDir, "_init_vault.vault"); + File.WriteAllBytes(legacyPath, [1, 2, 3, 4]); + + IVaultStorageService storage = new VaultStorageService(new VaultCryptoService()); + var ok = storage.LoadAllVaults("vault-pass-123", vaultDir, dataDir); + + Assert(ok, "Legacy-only vault directory should still be treated as successful load state."); + Assert(!File.Exists(legacyPath), "Legacy _init_vault.vault should be removed during load."); + Assert(Directory.Exists(dataDir), "Data directory should exist after load workflow."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + + return Task.CompletedTask; +} + +static Task TestVaultCurrentMonthSaveOptimizedAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-vault-smoke", Guid.NewGuid().ToString("N")); + var vaultDir = Path.Combine(root, "vault"); + var dataDir = Path.Combine(root, "data"); + Directory.CreateDirectory(vaultDir); + Directory.CreateDirectory(dataDir); + + try + { + File.WriteAllText(Path.Combine(dataDir, "2026-02-01.md"), "feb one"); + File.WriteAllText(Path.Combine(dataDir, "2026-02-18.md"), "feb two"); + File.WriteAllText(Path.Combine(dataDir, "2026-01-31.md"), "jan one"); + + IVaultStorageService storage = new VaultStorageService(new VaultCryptoService()); + var now = new DateTime(2026, 2, 22, 12, 0, 0, DateTimeKind.Utc); + + var firstSaved = storage.SaveCurrentMonthVault("vault-pass-123", vaultDir, dataDir, now); + Assert(firstSaved, "Expected first current-month save to write vault data."); + + var febVaultPath = Path.Combine(vaultDir, "2026-02.vault"); + var janVaultPath = Path.Combine(vaultDir, "2026-01.vault"); + Assert(File.Exists(febVaultPath), "Expected current-month vault file to be created."); + Assert(!File.Exists(janVaultPath), "Current-month save should not write non-current month vault files."); + + var entries = ReadVaultEntryTexts(febVaultPath, "vault-pass-123"); + Assert(entries.Count == 2, "Current-month vault should include only current-month markdown files."); + Assert(entries.ContainsKey("2026-02-01.md"), "Missing first current-month entry in vault archive."); + Assert(entries.ContainsKey("2026-02-18.md"), "Missing second current-month entry in vault archive."); + Assert(!entries.ContainsKey("2026-01-31.md"), "Current-month vault must not include previous-month files."); + + var beforeSkipBytes = File.ReadAllBytes(febVaultPath); + var secondSaved = storage.SaveCurrentMonthVault("vault-pass-123", vaultDir, dataDir, now); + var afterSkipBytes = File.ReadAllBytes(febVaultPath); + Assert(!secondSaved, "Expected unchanged current-month save to skip write."); + Assert(beforeSkipBytes.SequenceEqual(afterSkipBytes), "Vault bytes should remain unchanged when save is skipped."); + + File.WriteAllText(Path.Combine(dataDir, "2026-02-18.md"), "feb two changed"); + var thirdSaved = storage.SaveCurrentMonthVault("vault-pass-123", vaultDir, dataDir, now); + Assert(thirdSaved, "Expected save to run after current-month file change."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + + return Task.CompletedTask; +} + +static Task TestVaultRebuildAllVaultsAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-vault-smoke", Guid.NewGuid().ToString("N")); + var vaultDir = Path.Combine(root, "vault"); + var dataDir = Path.Combine(root, "data"); + Directory.CreateDirectory(vaultDir); + Directory.CreateDirectory(dataDir); + + try + { + File.WriteAllText(Path.Combine(dataDir, "2026-01-31.md"), "jan body"); + File.WriteAllText(Path.Combine(dataDir, "2026-02-01.md"), "feb body"); + File.WriteAllText(Path.Combine(dataDir, "not-a-journal.md"), "should be ignored"); + + IVaultStorageService storage = new VaultStorageService(new VaultCryptoService()); + storage.RebuildAllVaults("vault-pass-123", vaultDir, dataDir); + + var janVaultPath = Path.Combine(vaultDir, "2026-01.vault"); + var febVaultPath = Path.Combine(vaultDir, "2026-02.vault"); + Assert(File.Exists(janVaultPath), "Expected January vault from rebuild flow."); + Assert(File.Exists(febVaultPath), "Expected February vault from rebuild flow."); + + var janEntries = ReadVaultEntryTexts(janVaultPath, "vault-pass-123"); + var febEntries = ReadVaultEntryTexts(febVaultPath, "vault-pass-123"); + + Assert(janEntries.Count == 1 && janEntries.ContainsKey("2026-01-31.md"), "January vault contents mismatch."); + Assert(febEntries.Count == 1 && febEntries.ContainsKey("2026-02-01.md"), "February vault contents mismatch."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + + return Task.CompletedTask; +} + +static Task TestVaultClearDataDirectoryAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-vault-smoke", Guid.NewGuid().ToString("N")); + var dataDir = Path.Combine(root, "data"); + Directory.CreateDirectory(dataDir); + Directory.CreateDirectory(Path.Combine(dataDir, "nested")); + + try + { + File.WriteAllText(Path.Combine(dataDir, "2026-02-01.md"), "decrypted content"); + File.WriteAllText(Path.Combine(dataDir, "journal_cache.db"), "cache"); + File.WriteAllText(Path.Combine(dataDir, "nested", "tmp.txt"), "temp"); + + IVaultStorageService storage = new VaultStorageService(new VaultCryptoService()); + storage.ClearDataDirectory(dataDir); + + Assert(Directory.Exists(dataDir), "Data directory should be recreated after cleanup."); + Assert(!Directory.EnumerateFileSystemEntries(dataDir).Any(), "Data directory should be empty after cleanup."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + + return Task.CompletedTask; +} + +static Dictionary<string, string> ReadVaultEntryTexts(string vaultPath, string password) +{ + var crypto = new VaultCryptoService(); + var encrypted = File.ReadAllBytes(vaultPath); + var zipBytes = crypto.DecryptData(encrypted, password); + + using var stream = new MemoryStream(zipBytes); + using var archive = new ZipArchive(stream, ZipArchiveMode.Read); + var result = new Dictionary<string, string>(StringComparer.Ordinal); + foreach (var entry in archive.Entries) + { + if (string.IsNullOrEmpty(entry.Name)) + continue; + + using var reader = new StreamReader(entry.Open()); + result[entry.Name] = reader.ReadToEnd(); + } + + return result; +} + +static byte[] CreateZipBytes(Dictionary<string, string> files) +{ + using var stream = new MemoryStream(); + using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var (name, content) in files) + { + var entry = archive.CreateEntry(name); + using var writer = new StreamWriter(entry.Open()); + writer.Write(content); + } + } + return stream.ToArray(); +} + +static Task TestParserExtractsBoldDateAsync() +{ + var content = """ + --- + type: journal + --- + **Date:** 2026-02-22 + ## Summary + hello + """; + + var entry = JournalParser.ParseJournalContent(content, "2026-02-01"); + Assert(entry.Date == "2026-02-22", "Parser should read date from **Date:** marker."); + return Task.CompletedTask; +} + +static Task TestParserExtractsPlainDateAsync() +{ + var content = """ + Date: 2026-02-23 + ## Summary + hello + """; + + var entry = JournalParser.ParseJournalContent(content, "2026-02-01"); + Assert(entry.Date == "2026-02-23", "Parser should read date from Date: marker."); + return Task.CompletedTask; +} + +static Task TestParserFallsBackToFileStemAsync() +{ + var content = """ + ## Summary + no explicit date + """; + + var entry = JournalParser.ParseJournalContent(content, "2026-02-24"); + Assert(entry.Date == "2026-02-24", "Parser should fall back to file stem when no date marker is present."); + return Task.CompletedTask; +} + +static Task TestParserCapturesSectionsAsync() +{ + var content = """ + Date: 2026-02-25 + ## Summary + line one + line two + ### Events / Triggers - Work + trigger line + ## reflection notes + anchor line + """; + + var entry = JournalParser.ParseJournalContent(content, "2026-02-01"); + + Assert(entry.Sections.ContainsKey("Summary"), "Parser should capture Summary section."); + Assert(entry.Sections.ContainsKey("Events / Triggers"), "Parser should capture Events / Triggers section."); + Assert(entry.Sections.ContainsKey("Reflection"), "Parser should match canonical section title by substring."); + Assert(entry.GetSection("Summary").Contains("line one"), "Summary section content mismatch."); + Assert(entry.GetSection("Events / Triggers").Contains("trigger line"), "Events / Triggers section content mismatch."); + Assert(entry.GetSection("Reflection").Contains("anchor line"), "Reflection section content mismatch."); + + return Task.CompletedTask; +} + +static Task TestParserIgnoresNonCanonicalHeadersAsync() +{ + var content = """ + ## Summary + keep this + ## Totally Custom Header + should not be captured + ### Events / Triggers + keep this too + """; + + var entry = JournalParser.ParseJournalContent(content, "2026-02-01"); + + Assert(entry.GetSection("Summary").Contains("keep this"), "Summary section should be captured."); + Assert(!entry.GetSection("Summary").Contains("should not be captured"), "Non-canonical section content should not bleed into previous section."); + Assert(entry.GetSection("Events / Triggers").Contains("keep this too"), "Canonical section after custom header should be captured."); + + return Task.CompletedTask; +} + +static Task TestParserCapturesCheckboxStatesAsync() +{ + var content = """ + ## Summary + - [x] took medication + - [ ] drank water + * [X] wrote reflection + ## Events / Triggers + - [ ] talked to manager + """; + + var entry = JournalParser.ParseJournalContent(content, "2026-02-01"); + + Assert(entry.GetCheckboxState("Summary", "took medication") is true, "Expected checked state for '- [x]' checkbox."); + Assert(entry.GetCheckboxState("Summary", "drank water") is false, "Expected unchecked state for '- [ ]' checkbox."); + Assert(entry.GetCheckboxState("Summary", "wrote reflection") is true, "Expected checked state for '* [X]' checkbox."); + Assert(entry.GetCheckboxState("Events / Triggers", "talked to manager") is false, "Expected unchecked state in Events / Triggers section."); + Assert(entry.GetCheckboxState("Summary", "missing item") is null, "Missing checkbox text should return null."); + + return Task.CompletedTask; +} + +static Task TestParserCapturesMultilineFragmentsAsync() +{ + var content = """ + Date: 2026-02-26 + ## Summary + text + !TRIGGER @2026-02-26T10:15:00Z #stress #body + first line + second line + !NOTE #daily + short note + """; + + var entry = JournalParser.ParseJournalContent(content, "2026-02-01"); + + Assert(entry.Fragments.Count == 2, "Expected two parsed fragments."); + Assert(entry.Fragments[0].Type == "!TRIGGER", "First fragment type mismatch."); + Assert(entry.Fragments[0].Description == "first line\nsecond line", "First fragment multiline description mismatch."); + Assert(entry.Fragments[0].Tags.Count == 2, "First fragment tag count mismatch."); + Assert(entry.Fragments[0].Tags[0] == "stress" && entry.Fragments[0].Tags[1] == "body", "First fragment tags mismatch."); + Assert(entry.Fragments[1].Type == "!NOTE", "Second fragment type mismatch."); + Assert(entry.Fragments[1].Description == "short note", "Second fragment description mismatch."); + Assert(entry.Fragments[1].Tags.Count == 1 && entry.Fragments[1].Tags[0] == "daily", "Second fragment tags mismatch."); + + return Task.CompletedTask; +} + +static Task TestParserFragmentBoundaryBehaviorAsync() +{ + var content = """ + !TRIGGER #a + line one + !NOTE this starts another fragment header + line two + """; + + var fragments = JournalParser.ParseFragments(content); + Assert(fragments.Count == 1, "Expected one parsed fragment because second boundary line is not a valid fragment header."); + Assert(fragments[0].Description == "line one", "First fragment boundary capture mismatch."); + return Task.CompletedTask; +} + +static async Task TestEntryUnknownActionAsync() +{ + var entry = NewEntry(); + var response = await entry.HandleCommandAsync("""{"action":"unknown.action"}"""); + using var doc = JsonDocument.Parse(response); + + Assert(!doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=false."); + Assert(doc.RootElement.GetProperty("error").GetString()!.Contains("Unknown action"), "Expected unknown action error."); +} + +static async Task TestEntryInvalidJsonAsync() +{ + var entry = NewEntry(); + var response = await entry.HandleCommandAsync("{\"action\":\"fragments.list\""); + using var doc = JsonDocument.Parse(response); + + Assert(!doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=false."); + Assert(doc.RootElement.GetProperty("error").GetString()!.Contains("Invalid command JSON"), "Expected invalid JSON error."); +} + +static async Task TestEntryGetMissingReturnsNullDataAsync() +{ + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "fragments.get", + id = Guid.NewGuid().ToString(), + }); + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true."); + Assert(doc.RootElement.GetProperty("data").ValueKind == JsonValueKind.Null, "Expected data=null for missing fragment."); +} + +static async Task TestEntryCreateMissingPayloadAsync() +{ + var entry = NewEntry(); + var response = await entry.HandleCommandAsync("""{"action":"fragments.create"}"""); + using var doc = JsonDocument.Parse(response); + + Assert(!doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=false."); + Assert(doc.RootElement.GetProperty("error").GetString()!.Contains("payload", StringComparison.OrdinalIgnoreCase), "Expected payload validation error."); +} + +static async Task TestEntryEntriesSaveMergeAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-entry-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + var filePath = Path.Combine(root, "2026-02-22.md"); + File.WriteAllText(filePath, """ +Date: 2026-02-22 +## Summary +old summary text +"""); + + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "entries.save", + payload = new + { + filePath, + mode = "Daily", + content = """ +Date: 2026-02-22 +## Summary +new summary text +## Reflection +new reflection text +""" + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for entries.save."); + + var saved = File.ReadAllText(filePath); + Assert(saved.Contains("new summary text", StringComparison.Ordinal), "Expected merged file to contain new summary text."); + Assert(!saved.Contains("old summary text", StringComparison.Ordinal), "Expected merged file to replace old summary section."); + Assert(saved.Contains("new reflection text", StringComparison.Ordinal), "Expected merged file to contain new reflection section."); + + var fragmentSaveRequest = JsonSerializer.Serialize(new + { + action = "entries.save", + payload = new + { + filePath, + mode = "Fragment", + content = "!NOTE\nfragment append text" + } + }); + + var fragmentResponse = await entry.HandleCommandAsync(fragmentSaveRequest); + using var fragmentDoc = JsonDocument.Parse(fragmentResponse); + Assert(fragmentDoc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for entries.save fragment mode."); + var appended = File.ReadAllText(filePath); + Assert(appended.Contains("fragment append text", StringComparison.Ordinal), "Expected fragment append text in saved file."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestEntryEntriesLoadAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-entry-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + var filePath = Path.Combine(root, "2026-02-22.md"); + var content = """ +Date: 2026-02-22 +## Summary +hello world +"""; + File.WriteAllText(filePath, content); + + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "entries.load", + payload = new + { + filePath + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for entries.load."); + + var data = doc.RootElement.GetProperty("data"); + Assert(data.GetProperty("Date").GetString() == "2026-02-22", "Expected parsed date from entries.load."); + Assert(data.GetProperty("RawContent").GetString() == content, "Expected raw content from entries.load."); + Assert(data.GetProperty("FileName").GetString() == "2026-02-22.md", "Expected file name from entries.load."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestEntryEntriesListAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-entry-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + File.WriteAllText(Path.Combine(root, "2026-02-03.md"), "c"); + File.WriteAllText(Path.Combine(root, "2026-02-01.md"), "a"); + File.WriteAllText(Path.Combine(root, "ignore.txt"), "x"); + + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "entries.list", + payload = new + { + dataDirectory = root + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for entries.list."); + + var data = doc.RootElement.GetProperty("data"); + Assert(data.ValueKind == JsonValueKind.Array, "Expected entries.list data array."); + Assert(data.GetArrayLength() == 2, "Expected entries.list to return only markdown files."); + Assert(data[0].GetProperty("FileName").GetString() == "2026-02-01.md", "Expected entries.list sort order by file name."); + Assert(data[1].GetProperty("FileName").GetString() == "2026-02-03.md", "Expected entries.list sort order by file name."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestEntrySearchEntriesMatchesRawContentAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-search-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + File.WriteAllText(Path.Combine(root, "2026-02-01.md"), "## Summary\nAlpha line\ncommon token"); + File.WriteAllText(Path.Combine(root, "2026-02-02.md"), "## Summary\nbeta line\nCOMMON token"); + File.WriteAllText(Path.Combine(root, "2026-02-03.md"), "## Summary\ngamma only"); + + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "search.entries", + payload = new + { + dataDirectory = root, + query = "common token", + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for search.entries."); + var data = doc.RootElement.GetProperty("data"); + Assert(data.ValueKind == JsonValueKind.Array, "Expected data array from search.entries."); + Assert(data.GetArrayLength() == 2, "Expected two entries matching query across raw content."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestEntrySearchEntriesWithoutQueryReturnsAllAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-search-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + File.WriteAllText(Path.Combine(root, "2026-02-01.md"), "one"); + File.WriteAllText(Path.Combine(root, "2026-02-02.md"), "two"); + File.WriteAllText(Path.Combine(root, "ignore.txt"), "not markdown"); + + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "search.entries", + payload = new + { + dataDirectory = root, + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for search.entries without query."); + var data = doc.RootElement.GetProperty("data"); + Assert(data.ValueKind == JsonValueKind.Array, "Expected data array from search.entries."); + Assert(data.GetArrayLength() == 2, "Expected all markdown files to be returned when query is omitted."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestEntrySearchEntriesDateRangeFilterAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-search-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + WriteSearchFixtureFiles(root); + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "search.entries", + payload = new + { + dataDirectory = root, + startDate = "2026-02-02", + endDate = "2026-02-28", + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for date-range filtered search."); + var data = doc.RootElement.GetProperty("data"); + Assert(data.GetArrayLength() == 1, "Expected one result for filtered date range."); + Assert(data[0].GetProperty("FileName").GetString() == "2026-02-05.md", "Date-range result mismatch."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestEntrySearchEntriesSectionFilterAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-search-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + WriteSearchFixtureFiles(root); + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "search.entries", + payload = new + { + dataDirectory = root, + query = "focus area", + section = "Reflection", + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for section-scoped search."); + var data = doc.RootElement.GetProperty("data"); + Assert(data.GetArrayLength() == 1, "Expected one section-scoped result."); + Assert(data[0].GetProperty("FileName").GetString() == "2026-02-01.md", "Section filter result mismatch."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestEntrySearchEntriesTagTypeFilterAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-search-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + WriteSearchFixtureFiles(root); + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "search.entries", + payload = new + { + dataDirectory = root, + tags = new[] { "stress" }, + types = new[] { "!TRIGGER" }, + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for fragment tag/type filtered search."); + var data = doc.RootElement.GetProperty("data"); + Assert(data.GetArrayLength() == 1, "Expected one result for fragment tag/type filters."); + Assert(data[0].GetProperty("FileName").GetString() == "2026-02-01.md", "Tag/type filter result mismatch."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestEntrySearchEntriesCheckboxFilterAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-search-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + WriteSearchFixtureFiles(root); + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "search.entries", + payload = new + { + dataDirectory = root, + @checked = new[] { "med taken" }, + @unchecked = new[] { "drink water" }, + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for checkbox filtered search."); + var data = doc.RootElement.GetProperty("data"); + Assert(data.GetArrayLength() == 2, "Expected OR-style checkbox match across checked/unchecked filters."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestEntrySearchEntriesRejectsInvalidDateAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-search-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + WriteSearchFixtureFiles(root); + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "search.entries", + payload = new + { + dataDirectory = root, + startDate = "2026/02/01", + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(!doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=false for invalid date format."); + var error = doc.RootElement.GetProperty("error").GetString() ?? ""; + Assert(error.Contains("invalid startdate value", StringComparison.OrdinalIgnoreCase), "Expected invalid startDate error."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static void WriteSearchFixtureFiles(string root) +{ + File.WriteAllText(Path.Combine(root, "2026-02-01.md"), """ +Date: 2026-02-01 +## Summary +Alpha common +## Reflection +focus area +- [x] med taken +!TRIGGER #stress +fragment one +"""); + + File.WriteAllText(Path.Combine(root, "2026-02-05.md"), """ +Date: 2026-02-05 +## Summary +Beta common +## Reflection +other notes +- [ ] drink water +!NOTE #daily +fragment two +"""); + + File.WriteAllText(Path.Combine(root, "2026-03-01.md"), """ +Date: 2026-03-01 +## Summary +Gamma unique +## Reflection +nothing related +!NOTE #other +fragment three +"""); +} + +static Task TestDatabaseKeyDerivationMatchesPythonAsync() +{ + var service = NewDatabaseService(); + var keyHex = Convert.ToHexString(service.DeriveDatabaseKey("vault-pass-123")).ToLowerInvariant(); + var expected = "6a9de08e13357aa8f14e7eb0ccde119e7b4d277c60aaaca6493d9a1e1eaa5b04"; + Assert(keyHex == expected, "Database key derivation should match Python PBKDF2 fixture."); + return Task.CompletedTask; +} + +static Task TestDatabaseSchemaParityAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-db-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + var service = NewDatabaseService(); + var schemaPath = service.WriteSchemaBootstrap(root); + var statements = service.GetSchemaStatements(); + var tableNames = new HashSet<string>(statements.Keys, StringComparer.OrdinalIgnoreCase); + + Assert(tableNames.Contains("entries"), "Schema should contain entries table."); + Assert(tableNames.Contains("sections"), "Schema should contain sections table."); + Assert(tableNames.Contains("fragments"), "Schema should contain fragments table."); + Assert(tableNames.Contains("tags"), "Schema should contain tags table."); + Assert(tableNames.Contains("fragment_tags"), "Schema should contain fragment_tags table."); + + Assert(File.Exists(schemaPath), "Schema bootstrap file should be written."); + var fragmentTagsSql = statements["fragment_tags"]; + Assert(fragmentTagsSql.Contains("PRIMARY KEY (fragment_id, tag_id)", StringComparison.OrdinalIgnoreCase), "fragment_tags should enforce composite primary key parity."); + Assert(fragmentTagsSql.Contains("FOREIGN KEY (fragment_id) REFERENCES fragments (id)", StringComparison.OrdinalIgnoreCase), "fragment_tags should contain fragment foreign key parity."); + Assert(fragmentTagsSql.Contains("FOREIGN KEY (tag_id) REFERENCES tags (id)", StringComparison.OrdinalIgnoreCase), "fragment_tags should contain tag foreign key parity."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + + return Task.CompletedTask; +} + +static async Task TestEntryDatabaseStatusAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-db-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "db.status", + payload = new + { + password = "vault-pass-123", + dataDirectory = root + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for db.status."); + var data = doc.RootElement.GetProperty("data"); + Assert(data.TryGetProperty("DatabasePath", out var databasePath), "Expected DatabasePath in db.status payload."); + Assert(databasePath.ValueKind == JsonValueKind.String && !string.IsNullOrWhiteSpace(databasePath.GetString()), "Expected non-empty DatabasePath."); + Assert(data.TryGetProperty("KeyDerivation", out var keyDerivation), "Expected KeyDerivation in db.status payload."); + Assert(string.Equals(keyDerivation.GetString(), "PBKDF2-HMAC-SHA256", StringComparison.Ordinal), "Expected PBKDF2-HMAC-SHA256 key derivation."); + Assert(data.TryGetProperty("SchemaTables", out var schemaTables), "Expected SchemaTables list in db.status payload."); + Assert(schemaTables.ValueKind == JsonValueKind.Array && schemaTables.GetArrayLength() >= 5, "Expected schema table list in db.status payload."); + Assert(data.TryGetProperty("SchemaBootstrapPath", out var schemaBootstrapPath), "Expected SchemaBootstrapPath in db.status payload."); + Assert(schemaBootstrapPath.ValueKind == JsonValueKind.String && File.Exists(schemaBootstrapPath.GetString()), "Expected db.status to emit existing schema bootstrap file path."); + Assert(data.TryGetProperty("RuntimeReady", out var runtimeReady), "Expected RuntimeReady in db.status payload."); + Assert(runtimeReady.ValueKind == JsonValueKind.True, "Expected SQLCipher runtime-ready status in db.status payload."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestEntryDatabaseInitializeSchemaAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-db-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "db.initialize_schema", + payload = new + { + password = "vault-pass-123", + dataDirectory = root + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for db.initialize_schema."); + var data = doc.RootElement.GetProperty("data"); + Assert(data.TryGetProperty("schemaPath", out var schemaPath), "Expected schemaPath in db.initialize_schema response."); + Assert(schemaPath.ValueKind == JsonValueKind.String, "Expected string schemaPath value."); + var resolvedPath = schemaPath.GetString() ?? ""; + Assert(File.Exists(resolvedPath), "db.initialize_schema should write schema bootstrap file."); + var schemaText = File.ReadAllText(resolvedPath); + Assert(schemaText.Contains("CREATE TABLE IF NOT EXISTS entries", StringComparison.OrdinalIgnoreCase), "schema bootstrap should include entries table."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestEntryDatabaseHydrateWorkspaceAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-db-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + File.WriteAllText(Path.Combine(root, "2026-02-20.md"), "one"); + File.WriteAllText(Path.Combine(root, "2026-02-21.md"), "two"); + + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "db.hydrate_workspace", + payload = new + { + password = "vault-pass-123", + dataDirectory = root + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for db.hydrate_workspace."); + var data = doc.RootElement.GetProperty("data"); + + Assert(data.TryGetProperty("DatabasePath", out var databasePath), "Expected DatabasePath in hydrate payload."); + Assert(databasePath.ValueKind == JsonValueKind.String && !string.IsNullOrWhiteSpace(databasePath.GetString()), "Expected non-empty DatabasePath."); + Assert(data.TryGetProperty("SchemaBootstrapPath", out var schemaPath), "Expected SchemaBootstrapPath in hydrate payload."); + Assert(schemaPath.ValueKind == JsonValueKind.String && File.Exists(schemaPath.GetString()), "Expected hydrate to write schema bootstrap file."); + Assert(data.TryGetProperty("EntryFilesProcessed", out var filesProcessed), "Expected EntryFilesProcessed in hydrate payload."); + Assert(filesProcessed.ValueKind == JsonValueKind.Number && filesProcessed.GetInt32() == 2, "Expected hydrate to count markdown files in workspace."); + Assert(data.TryGetProperty("RuntimeReady", out var runtimeReady), "Expected RuntimeReady in hydrate payload."); + Assert(runtimeReady.ValueKind == JsonValueKind.True, "Expected RuntimeReady=true when SQLCipher runtime hydration succeeds."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static Task TestConfigServiceParityKeysAsync() +{ + IJournalConfigService config = new JournalConfigService(); + var current = config.Current; + + Assert(!string.IsNullOrWhiteSpace(current.ProjectRoot), "Config ProjectRoot should not be empty."); + Assert(!string.IsNullOrWhiteSpace(current.AppDirectory), "Config AppDirectory should not be empty."); + Assert(!string.IsNullOrWhiteSpace(current.DataDirectory), "Config DataDirectory should not be empty."); + Assert(!string.IsNullOrWhiteSpace(current.VaultDirectory), "Config VaultDirectory should not be empty."); + Assert(current.MonthlyVaultFormat == "%Y-%m.vault", "Config MonthlyVaultFormat should match Python format token."); + + Assert(current.LlamaCppUrl == "http://127.0.0.1:8085/v1/completions", "Config LlamaCppUrl default mismatch."); + Assert(current.LlamaCppModel == "qwen/qwen3-4b", "Config LlamaCppModel default mismatch."); + Assert(current.EmbeddingApiUrl == "http://127.0.0.1:8086/v1/embeddings", "Config EmbeddingApiUrl default mismatch."); + Assert(current.SpeechRecognitionEngine == "whisper", "Config SpeechRecognitionEngine default mismatch."); + Assert(current.WhisperModelSize == "base", "Config WhisperModelSize default mismatch."); + Assert(current.AiProvider == "none", "Config AiProvider default mismatch."); + Assert(current.PythonExecutable == "python", "Config PythonExecutable default mismatch."); + Assert(current.AiSidecarTimeoutMs == 45000, "Config AiSidecarTimeoutMs default mismatch."); + Assert(current.PythonAiSidecarPath.EndsWith(Path.Combine("journal", "ai", "sidecar.py"), StringComparison.OrdinalIgnoreCase), "Config PythonAiSidecarPath default mismatch."); + + return Task.CompletedTask; +} + +static async Task TestEntryConfigGetAsync() +{ + var entry = NewEntry(); + var response = await entry.HandleCommandAsync("""{"action":"config.get"}"""); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for config.get."); + var data = doc.RootElement.GetProperty("data"); + Assert(data.ValueKind == JsonValueKind.Object, "Expected object payload for config.get."); + + Assert(data.TryGetProperty("DataDirectory", out var dataDirectory), "Expected DataDirectory in config payload."); + Assert(dataDirectory.ValueKind == JsonValueKind.String && !string.IsNullOrWhiteSpace(dataDirectory.GetString()), "Expected non-empty DataDirectory value."); + Assert(data.TryGetProperty("MonthlyVaultFormat", out var monthlyVaultFormat), "Expected MonthlyVaultFormat in config payload."); + Assert(monthlyVaultFormat.GetString() == "%Y-%m.vault", "Expected Python-compatible MonthlyVaultFormat value."); + Assert(data.TryGetProperty("LlamaCppUrl", out _), "Expected LlamaCppUrl in config payload."); + Assert(data.TryGetProperty("SpeechRecognitionEngine", out _), "Expected SpeechRecognitionEngine in config payload."); +} + +static Task TestLogRedactorScrubsSensitiveFieldsAsync() +{ + var payload = JsonSerializer.SerializeToElement(new + { + password = "vault-pass-123", + content = "private journal body", + prompt = "private ai prompt", + nested = new + { + token = "abc123" + } + }); + + var redacted = LogRedactor.RedactPayload(payload); + var serialized = JsonSerializer.Serialize(redacted); + + Assert(!serialized.Contains("vault-pass-123", StringComparison.Ordinal), "Password should be redacted."); + Assert(!serialized.Contains("private journal body", StringComparison.Ordinal), "Entry content should be redacted."); + Assert(!serialized.Contains("private ai prompt", StringComparison.Ordinal), "Prompt should be redacted."); + Assert(!serialized.Contains("abc123", StringComparison.Ordinal), "Nested token should be redacted."); + Assert(serialized.Contains("[REDACTED]", StringComparison.Ordinal), "Redacted marker should be present."); + + return Task.CompletedTask; +} + +static Task TestLogRedactorPreservesNonSensitiveFieldsAsync() +{ + var payload = JsonSerializer.SerializeToElement(new + { + action = "entries.save", + mode = "Daily", + filePath = "E:/journal/2026-02-24.md" + }); + + var redacted = LogRedactor.RedactPayload(payload); + var serialized = JsonSerializer.Serialize(redacted); + + Assert(serialized.Contains("entries.save", StringComparison.Ordinal), "Non-sensitive action field should be preserved."); + Assert(serialized.Contains("Daily", StringComparison.Ordinal), "Non-sensitive mode field should be preserved."); + Assert(serialized.Contains("2026-02-24.md", StringComparison.Ordinal), "Non-sensitive path field should be preserved."); + + return Task.CompletedTask; +} + +static async Task TestEntryAiHealthDefaultAsync() +{ + var entry = NewEntry(); + var response = await entry.HandleCommandAsync("""{"action":"ai.health"}"""); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for ai.health."); + var data = doc.RootElement.GetProperty("data"); + Assert(data.GetProperty("Enabled").GetBoolean() is false, "Expected AI disabled by default."); + Assert(string.Equals(data.GetProperty("Provider").GetString(), "none", StringComparison.OrdinalIgnoreCase), "Expected default provider 'none'."); +} + +static async Task TestEntryAiSummarizeEntryDisabledAsync() +{ + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "ai.summarize_entry", + payload = new + { + content = "sample entry" + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for disabled ai.summarize_entry."); + var data = doc.RootElement.GetProperty("data").GetString() ?? ""; + Assert(data.Contains("disabled", StringComparison.OrdinalIgnoreCase), "Expected disabled provider message for ai.summarize_entry."); +} + +static async Task TestEntryAiSummarizeAllDisabledAsync() +{ + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "ai.summarize_all", + payload = new + { + entries = new[] { "entry one", "entry two" } + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for disabled ai.summarize_all."); + var data = doc.RootElement.GetProperty("data").GetString() ?? ""; + Assert(data.Contains("disabled", StringComparison.OrdinalIgnoreCase), "Expected disabled provider message for ai.summarize_all."); +} + +static async Task TestEntryAiChatDisabledAsync() +{ + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "ai.chat", + payload = new + { + prompt = "hello cloud" + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for disabled ai.chat."); + var data = doc.RootElement.GetProperty("data").GetString() ?? ""; + Assert(data.Contains("disabled", StringComparison.OrdinalIgnoreCase), "Expected disabled provider message for ai.chat."); +} + +static async Task TestEntryAiEmbedDisabledAsync() +{ + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "ai.embed", + payload = new + { + content = "embedding source text" + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for disabled ai.embed."); + var data = doc.RootElement.GetProperty("data"); + Assert(data.ValueKind == JsonValueKind.Array, "Expected ai.embed response to be a JSON array."); + Assert(data.GetArrayLength() == 0, "Expected disabled ai.embed to return an empty vector."); +} + +static async Task TestEntrySpeechDevicesListDisabledAsync() +{ + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "speech.devices.list", + payload = new { } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for speech.devices.list when disabled."); + var data = doc.RootElement.GetProperty("data"); + Assert(data.ValueKind == JsonValueKind.Object, "Expected speech.devices.list data to be an object."); +} + +static async Task TestEntrySpeechTranscribeDisabledAsync() +{ + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "speech.transcribe", + payload = new + { + text = "fixture transcript", + engine = "whisper" + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for speech.transcribe when disabled."); + var data = doc.RootElement.GetProperty("data"); + Assert(data.ValueKind == JsonValueKind.Object, "Expected speech.transcribe data to be an object."); + var warning = data.TryGetProperty("Warning", out var warningNode) ? warningNode.GetString() ?? "" : ""; + Assert(warning.Contains("disabled", StringComparison.OrdinalIgnoreCase), "Expected disabled speech warning."); +} + +static async Task TestPythonSidecarAiServiceJsonLineAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-ai-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var scriptPath = Path.Combine(root, "fake_ai_sidecar.py"); + File.WriteAllText(scriptPath, """ +import json, sys +request = json.loads(sys.stdin.readline()) +action = request.get("action", "") +print("DEBUG prelude") +if action == "health": + print(json.dumps({"ok": True, "data": {"provider": "python-sidecar", "healthy": True, "message": "ok"}})) +elif action == "summarize_entry": + payload = request.get("payload") or {} + print(json.dumps({"ok": True, "data": "ENTRY::" + str(payload.get("content", ""))})) +elif action == "summarize_all": + payload = request.get("payload") or {} + entries = payload.get("entries") or [] + print(json.dumps({"ok": True, "data": "ALL::" + str(len(entries))})) +elif action == "chat": + payload = request.get("payload") or {} + print(json.dumps({"ok": True, "data": "CHAT::" + str(payload.get("prompt", ""))})) +elif action == "embed": + payload = request.get("payload") or {} + text = str(payload.get("content", "")) + print(json.dumps({"ok": True, "data": [float(len(text)), 2.5, -1.0]})) +else: + print(json.dumps({"ok": False, "error": "unknown action"})) +"""); + + try + { + var config = BuildAiConfig(scriptPath, timeoutMs: 4000); + IAiService service = new PythonSidecarAiService(config); + + var health = await service.HealthAsync(); + Assert(health.Enabled, "Expected enabled=true for python-sidecar health."); + Assert(health.Healthy, "Expected healthy=true from fake sidecar health."); + + var one = await service.SummarizeEntryAsync("hello"); + Assert(one == "ENTRY::hello", "Unexpected summarize_entry response."); + + var all = await service.SummarizeAllAsync(["a", "b", "c"]); + Assert(all == "ALL::3", "Unexpected summarize_all response."); + + var chat = await service.ChatAsync("hello"); + Assert(chat == "CHAT::hello", "Unexpected chat response."); + + var vector = await service.EmbedAsync("hello"); + Assert(vector.Count == 3, "Unexpected embed vector length."); + Assert(Math.Abs(vector[0] - 5d) < 0.0001d, "Unexpected embed vector first value."); + Assert(Math.Abs(vector[1] - 2.5d) < 0.0001d, "Unexpected embed vector second value."); + Assert(Math.Abs(vector[2] + 1.0d) < 0.0001d, "Unexpected embed vector third value."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestPythonSidecarAiServiceErrorAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-ai-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var scriptPath = Path.Combine(root, "fake_ai_sidecar_error.py"); + File.WriteAllText(scriptPath, """ +import json, sys +_ = json.loads(sys.stdin.readline()) +print(json.dumps({"ok": False, "error": "simulated failure"})) +"""); + + try + { + var config = BuildAiConfig(scriptPath, timeoutMs: 4000); + IAiService service = new PythonSidecarAiService(config); + + try + { + _ = await service.SummarizeEntryAsync("hello"); + } + catch (InvalidOperationException ex) when (ex.Message.Contains("simulated failure", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + throw new InvalidOperationException("Expected summarize_entry to surface sidecar error."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestPythonSidecarSpeechServiceNoDevicesAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-speech-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var scriptPath = Path.Combine(root, "fake_speech_sidecar_nodes.py"); + File.WriteAllText(scriptPath, """ +import json, sys +request = json.loads(sys.stdin.readline()) +action = request.get("action", "") +if action == "speech.devices.list": + print(json.dumps({"ok": True, "data": {"devices": [], "warning": "no devices"}})) +elif action == "speech.transcribe": + payload = request.get("payload") or {} + print(json.dumps({"ok": True, "data": {"text": payload.get("text", ""), "engine": payload.get("engine", "whisper")}})) +else: + print(json.dumps({"ok": False, "error": "unknown action"})) +"""); + + try + { + var config = BuildAiConfig(scriptPath, timeoutMs: 4000); + ISpeechBridgeService service = new PythonSidecarSpeechService(config); + + var devices = await service.ListDevicesAsync(); + Assert(devices.Devices.Count == 0, "Expected empty devices list."); + Assert((devices.Warning ?? "").Contains("no devices", StringComparison.OrdinalIgnoreCase), "Expected no-devices warning."); + + var transcript = await service.TranscribeAsync(new SpeechTranscribeRequestDto(Text: "fixture text", Engine: "whisper")); + Assert(transcript.Text == "fixture text", "Expected passthrough transcript text."); + Assert(transcript.Engine == "whisper", "Expected passthrough transcript engine."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestPythonSidecarSpeechServiceErrorAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-speech-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var scriptPath = Path.Combine(root, "fake_speech_sidecar_error.py"); + File.WriteAllText(scriptPath, """ +import json, sys +request = json.loads(sys.stdin.readline()) +action = request.get("action", "") +if action == "speech.transcribe": + print(json.dumps({"ok": False, "error": "engine unavailable"})) +else: + print(json.dumps({"ok": True, "data": {"devices": []}})) +"""); + + try + { + var config = BuildAiConfig(scriptPath, timeoutMs: 4000); + ISpeechBridgeService service = new PythonSidecarSpeechService(config); + + try + { + _ = await service.TranscribeAsync(new SpeechTranscribeRequestDto(Text: "fixture", Engine: "faster-whisper")); + } + catch (InvalidOperationException ex) when (ex.Message.Contains("engine unavailable", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + throw new InvalidOperationException("Expected speech transcribe to surface sidecar engine error."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestPythonSidecarSpeechServiceTimeoutAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-speech-smoke", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + var scriptPath = Path.Combine(root, "fake_speech_sidecar_timeout.py"); + File.WriteAllText(scriptPath, """ +import json, sys, time +request = json.loads(sys.stdin.readline()) +payload = request.get("payload") or {} +sleep_ms = int(payload.get("simulate_delay_ms") or 0) +time.sleep(max(0, sleep_ms) / 1000.0) +print(json.dumps({"ok": True, "data": {"text": "", "engine": "whisper"}})) +"""); + + try + { + var config = BuildAiConfig(scriptPath, timeoutMs: 100); + ISpeechBridgeService service = new PythonSidecarSpeechService(config); + + try + { + _ = await service.TranscribeAsync(new SpeechTranscribeRequestDto(Text: "fixture", SimulateDelayMs: 500)); + } + catch (TimeoutException) + { + return; + } + + throw new InvalidOperationException("Expected speech transcribe timeout path."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestEntryVaultLoadAllEmptyAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-sidecar-smoke", Guid.NewGuid().ToString("N")); + var vaultDir = Path.Combine(root, "vault"); + var dataDir = Path.Combine(root, "data"); + Directory.CreateDirectory(vaultDir); + + try + { + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "vault.load_all", + payload = new + { + password = "vault-pass-123", + vaultDirectory = vaultDir, + dataDirectory = dataDir, + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for empty vault directory load."); + Assert(doc.RootElement.GetProperty("data").GetBoolean(), "Expected vault.load_all data=true for empty vault directory."); + Assert(Directory.Exists(dataDir), "Expected data directory to be created by load workflow."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static async Task TestEntryVaultClearDataDirectoryAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-sidecar-smoke", Guid.NewGuid().ToString("N")); + var dataDir = Path.Combine(root, "data"); + Directory.CreateDirectory(dataDir); + File.WriteAllText(Path.Combine(dataDir, "tmp.md"), "x"); + + try + { + var entry = NewEntry(); + var request = JsonSerializer.Serialize(new + { + action = "vault.clear_data_directory", + payload = new + { + dataDirectory = dataDir, + } + }); + + var response = await entry.HandleCommandAsync(request); + using var doc = JsonDocument.Parse(response); + + Assert(doc.RootElement.GetProperty("ok").GetBoolean(), "Expected ok=true for clear_data_directory."); + Assert(doc.RootElement.GetProperty("data").GetBoolean(), "Expected clear_data_directory result=true."); + Assert(Directory.Exists(dataDir), "Expected data directory to exist after clear."); + Assert(!Directory.EnumerateFileSystemEntries(dataDir).Any(), "Expected data directory to be empty after clear."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } +} + +static Task TestSidecarVaultCliLoadAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-sidecar-cli-smoke", Guid.NewGuid().ToString("N")); + var vaultDir = Path.Combine(root, "vault"); + var dataDir = Path.Combine(root, "data"); + Directory.CreateDirectory(vaultDir); + + try + { + var cli = new SidecarCli(new VaultStorageService(new VaultCryptoService()), new EntrySearchService(), new JournalConfigService()); + var exitCode = cli.RunVaultCommand(["load", "--password", "vault-pass-123", "--vault-dir", vaultDir, "--data-dir", dataDir]); + + Assert(exitCode == 0, "Expected vault load CLI command to succeed on empty vault directory."); + Assert(Directory.Exists(dataDir), "Expected data directory to be created by vault load CLI command."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + + return Task.CompletedTask; +} + +static Task TestSidecarVaultCliSaveAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-sidecar-cli-smoke", Guid.NewGuid().ToString("N")); + var vaultDir = Path.Combine(root, "vault"); + var dataDir = Path.Combine(root, "data"); + Directory.CreateDirectory(vaultDir); + Directory.CreateDirectory(dataDir); + + try + { + File.WriteAllText(Path.Combine(dataDir, "2026-02-22.md"), "entry body"); + + var cli = new SidecarCli(new VaultStorageService(new VaultCryptoService()), new EntrySearchService(), new JournalConfigService()); + var exitCode = cli.RunVaultCommand(["save", "--password", "vault-pass-123", "--vault-dir", vaultDir, "--data-dir", dataDir]); + + Assert(exitCode == 0, "Expected vault save CLI command to succeed."); + Assert(File.Exists(Path.Combine(vaultDir, "2026-02.vault")), "Expected monthly vault file to be written by save CLI command."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + + return Task.CompletedTask; +} + +static Task TestSidecarSearchCliFilteredAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-sidecar-cli-smoke", Guid.NewGuid().ToString("N")); + var dataDir = Path.Combine(root, "data"); + Directory.CreateDirectory(dataDir); + + try + { + WriteSearchFixtureFiles(dataDir); + var cli = new SidecarCli(new VaultStorageService(new VaultCryptoService()), new EntrySearchService(), new JournalConfigService()); + + var (exitCode, stdout, stderr) = CaptureConsole(() => cli.RunSearchCommand( + [ + "common", + "--data-dir", dataDir, + "--start-date", "2026-02-01", + "--end-date", "2026-02-28", + "--tag", "stress", + "--type", "!TRIGGER", + "--checked", "med taken", + "--section", "Summary" + ])); + + Assert(exitCode == 0, "Expected search CLI command to succeed."); + Assert(string.IsNullOrWhiteSpace(stderr), "Expected no stderr output for successful search CLI command."); + Assert(stdout.Contains("--- 2026-02-01 ---", StringComparison.Ordinal), "Expected matching entry header in search CLI output."); + Assert(!stdout.Contains("--- 2026-02-05 ---", StringComparison.Ordinal), "Unexpected non-matching entry in filtered search CLI output."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + + return Task.CompletedTask; +} + +static Task TestSidecarSearchCliEmptyDataAsync() +{ + var root = Path.Combine(Path.GetTempPath(), "journal-sidecar-cli-smoke", Guid.NewGuid().ToString("N")); + var dataDir = Path.Combine(root, "data"); + Directory.CreateDirectory(dataDir); + + try + { + var cli = new SidecarCli(new VaultStorageService(new VaultCryptoService()), new EntrySearchService(), new JournalConfigService()); + var (exitCode, stdout, _) = CaptureConsole(() => cli.RunSearchCommand(["--data-dir", dataDir])); + + Assert(exitCode == 0, "Expected search CLI command to return success for empty data directory."); + Assert(stdout.Contains("No decrypted journal entries found", StringComparison.OrdinalIgnoreCase), "Expected empty-data guidance message."); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + + return Task.CompletedTask; +} + +static (int ExitCode, string Stdout, string Stderr) CaptureConsole(Func<int> action) +{ + var originalOut = Console.Out; + var originalError = Console.Error; + using var stdout = new StringWriter(); + using var stderr = new StringWriter(); + + try + { + Console.SetOut(stdout); + Console.SetError(stderr); + var exitCode = action(); + return (exitCode, stdout.ToString(), stderr.ToString()); + } + finally + { + Console.SetOut(originalOut); + Console.SetError(originalError); + } +} + +static JournalConfig BuildAiConfig(string sidecarScriptPath, int timeoutMs) +{ + var baseConfig = new JournalConfigService().Current; + var pythonExe = Environment.GetEnvironmentVariable("JOURNAL_PYTHON_EXE"); + if (string.IsNullOrWhiteSpace(pythonExe)) + pythonExe = "python"; + + return baseConfig with + { + AiProvider = "python-sidecar", + PythonExecutable = pythonExe, + PythonAiSidecarPath = sidecarScriptPath, + AiSidecarTimeoutMs = timeoutMs + }; +} + +static async Task TestTransportFixturesAsync() +{ + var fixtures = await LoadTransportFixturesAsync(); + Assert(fixtures.Count > 0, "Transport fixtures should not be empty."); + + foreach (var fixture in fixtures) + { + var entry = NewEntry(); + var response = await entry.HandleCommandAsync(fixture.Request); + + Assert(!response.Contains('\n') && !response.Contains('\r'), $"Fixture '{fixture.Name}' returned multiline output."); + + using var doc = JsonDocument.Parse(response); + var ok = doc.RootElement.GetProperty("ok").GetBoolean(); + Assert(ok == fixture.ExpectOk, $"Fixture '{fixture.Name}' expected ok={fixture.ExpectOk} but got ok={ok}."); + + if (fixture.ExpectOk) + { + Assert(doc.RootElement.TryGetProperty("data", out var data), $"Fixture '{fixture.Name}' expected data field."); + if (!string.IsNullOrWhiteSpace(fixture.DataKind)) + { + var expectedKind = ParseValueKind(fixture.DataKind!); + Assert(data.ValueKind == expectedKind, $"Fixture '{fixture.Name}' expected data kind {expectedKind} but got {data.ValueKind}."); + } + continue; + } + + Assert(doc.RootElement.TryGetProperty("error", out var error), $"Fixture '{fixture.Name}' expected error field."); + if (!string.IsNullOrWhiteSpace(fixture.ErrorContains)) + { + var message = error.GetString() ?? ""; + Assert(message.Contains(fixture.ErrorContains!, StringComparison.OrdinalIgnoreCase), $"Fixture '{fixture.Name}' expected error containing '{fixture.ErrorContains}'."); + } + } +} + +static async Task<List<TransportFixture>> LoadTransportFixturesAsync() +{ + var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", "transport_cases.json"); + if (!File.Exists(path)) + throw new FileNotFoundException($"Transport fixture file not found: {path}"); + + var json = await File.ReadAllTextAsync(path); + return JsonSerializer.Deserialize<List<TransportFixture>>(json, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true + }) ?? []; +} + +static JsonValueKind ParseValueKind(string value) => value.Trim().ToLowerInvariant() switch +{ + "array" => JsonValueKind.Array, + "object" => JsonValueKind.Object, + "null" => JsonValueKind.Null, + "string" => JsonValueKind.String, + "number" => JsonValueKind.Number, + "true" => JsonValueKind.True, + "false" => JsonValueKind.False, + _ => throw new InvalidOperationException($"Unsupported JsonValueKind '{value}' in transport fixture.") +}; + +static void Assert(bool condition, string message) +{ + if (!condition) + throw new InvalidOperationException(message); +} + +sealed class TransportFixture +{ + public string Name { get; init; } = ""; + public string Request { get; init; } = ""; + public bool ExpectOk { get; init; } + public string? DataKind { get; init; } + public string? ErrorContains { get; init; } +} diff --git a/journal-master/journal/Journal.slnx b/journal-master/journal/Journal.slnx new file mode 100644 index 0000000..a5a68e3 --- /dev/null +++ b/journal-master/journal/Journal.slnx @@ -0,0 +1,5 @@ +<Solution> + <Project Path="Journal.Api/Journal.Api.csproj" /> + <Project Path="Journal.Core/Journal.Core.csproj" /> + <Project Path="Journal.Sidecar/Journal.Sidecar.csproj" /> +</Solution> diff --git a/journal-master/journal/MINIMAL_MACHINE_SETUP.md b/journal-master/journal/MINIMAL_MACHINE_SETUP.md new file mode 100644 index 0000000..f93f888 --- /dev/null +++ b/journal-master/journal/MINIMAL_MACHINE_SETUP.md @@ -0,0 +1,139 @@ +# Minimal Machine Setup (CLI-Only, No Visual Studio) + +This project can be developed with a small footprint: +- .NET SDK only (`dotnet --version` should work) +- PowerShell +- No Visual Studio required + +## Why This Exists + +This repo uses NuGet package references. The helper script keeps all dotnet and +NuGet artifacts local to this repo, clears proxy env vars for each command, and +uses restore flags that tolerate offline/unreachable remote sources. +This keeps footprint small and avoids host-level environment drift. + +Current status target: +- `./scripts/dotnet-min.ps1 restore ...` succeeds in this workspace. +- If remote NuGet TLS is flaky, restores still work when package cache is present. + +## One-Time Check + +```powershell +dotnet --version +./scripts/dotnet-min.ps1 --info +``` + +## Standard Workflow + +Run commands from `journal-master/journal`: + +```powershell +./scripts/dotnet-min.ps1 restore Journal.Sidecar/Journal.Sidecar.csproj +./scripts/dotnet-min.ps1 restore Journal.Api/Journal.Api.csproj +./scripts/dotnet-min.ps1 build Journal.Core/Journal.Core.csproj +./scripts/dotnet-min.ps1 build Journal.Sidecar/Journal.Sidecar.csproj +./scripts/dotnet-min.ps1 build Journal.Api/Journal.Api.csproj +./scripts/dotnet-min.ps1 run --project Journal.SmokeTests/Journal.SmokeTests.csproj +``` + +## If Restore Fails In A New Shell + +Quick checks: + +```powershell +./scripts/dotnet-min.ps1 nuget list source +./scripts/dotnet-min.ps1 restore Journal.Sidecar/Journal.Sidecar.csproj +``` + +If your shell has proxy env vars set, the helper script already clears them per run. +It also applies: +- `--ignore-failed-sources` on restore +- `-p:RestoreIgnoreFailedSources=true` +- `-p:NuGetAudit=false` + +## Admin TLS Repair Attempt (Host-Level) + +These commands must be run in an **Administrator PowerShell** outside this agent session. + +1. Backup the current key: + +```powershell +reg export "HKLM\SYSTEM\CurrentControlSet\Control\SecurityProviders" "$env:USERPROFILE\Desktop\SecurityProviders_backup.reg" /y +``` + +2. Restore standard provider list for Schannel: + +```powershell +reg add "HKLM\SYSTEM\CurrentControlSet\Control\SecurityProviders" /v SecurityProviders /t REG_SZ /d "credssp.dll, schannel.dll, digest.dll, msnsspc.dll" /f +``` + +3. Backup and repair LSA security packages (required if `SEC_E_NO_CREDENTIALS` persists): + +```powershell +reg export "HKLM\SYSTEM\CurrentControlSet\Control\Lsa" "$env:USERPROFILE\Desktop\Lsa_backup.reg" /y +Set-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\Lsa" -Name "Security Packages" -Type MultiString -Value @("kerberos","msv1_0","schannel","wdigest","tspkg","pku2u") +Set-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\Lsa" -Name "Authentication Packages" -Type MultiString -Value @("msv1_0") +``` + +4. Reboot, then test: + +```powershell +curl.exe -I https://api.nuget.org/v3/index.json +./scripts/dotnet-min.ps1 restore Journal.Sidecar/Journal.Sidecar.csproj +``` + +5. If still broken, run system repair and reboot: + +```powershell +DISM /Online /Cleanup-Image /RestoreHealth +sfc /scannow +``` + +## Friend Machine Export -> This Machine Import (Offline NuGet Cache) + +Use this when your friend can restore packages on a healthy machine. + +On friend machine (from `journal-master/journal`): + +```powershell +./scripts/nuget-export-cache.ps1 -OutputZip .\nuget-cache-export.zip +``` + +Copy `nuget-cache-export.zip` to this machine, then from `journal-master/journal`: + +```powershell +./scripts/nuget-import-cache.ps1 -InputZip .\nuget-cache-export.zip +``` + +If restore still reports remote source errors after import, run one more pass: + +```powershell +./scripts/dotnet-min.ps1 restore Journal.Sidecar/Journal.Sidecar.csproj --ignore-failed-sources +./scripts/dotnet-min.ps1 build Journal.Sidecar/Journal.Sidecar.csproj +``` + +### What the export/import scripts do + +- `nuget-export-cache.ps1` + - runs restores to prime cache + - bundles `.nuget/` into a zip + - writes a small manifest file in the bundle +- `nuget-import-cache.ps1` + - expands the bundle into this repo + - runs restores against local cache + +## Local Footprint Paths + +The helper script keeps artifacts under this repo: +- `.dotnet_home/` +- `.nuget/packages/` +- `.nuget/http-cache/` +- normal build outputs (`bin/`, `obj/`) + +To clean everything created for this machine: + +```powershell +Remove-Item -Recurse -Force .dotnet_home,.nuget -ErrorAction SilentlyContinue +Get-ChildItem -Recurse -Directory -Filter bin | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue +Get-ChildItem -Recurse -Directory -Filter obj | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue +``` diff --git a/journal-master/journal/README.md b/journal-master/journal/README.md new file mode 100644 index 0000000..b07d408 --- /dev/null +++ b/journal-master/journal/README.md @@ -0,0 +1,264 @@ +# Journal Backend (.NET) + +A .NET 10 backend for the Project Journal app. Provides core journal functionality as a class library with a sidecar console app for Tauri integration and an optional HTTP API. + +## Project Structure + +``` +backend/ +├── Journal.Core/ Class library — all business logic +│ ├── Models/ +│ │ ├── Fragment.cs Domain model (validated, owns Guid ID) +│ │ ├── Command.cs Stdin command shape for sidecar protocol +│ │ ├── ParsedSection.cs Parsed section model for entry parity work +│ │ ├── SectionTitles.cs Canonical section title list (Python parity) +│ │ └── JournalEntry.cs Entry domain (`date/raw_content/sections/fragments` + merge + markdown reconstruction) +│ ├── Dtos/ +│ │ └── FragmentDtos.cs Immutable records for API boundary +│ │ ├── FragmentDto Read (what goes out) +│ │ ├── CreateFragmentDto Create (what comes in) +│ │ └── UpdateFragmentDto Update (partial, all fields optional) +│ ├── Repositories/ +│ │ ├── IFragmentRepository.cs Interface (data access contract) +│ │ ├── InMemoryFragmentRepository.cs In-memory implementation (tests/dev) +│ │ └── FileFragmentRepository.cs File-backed implementation (default) +│ ├── Services/ +│ │ ├── IFragmentService.cs Interface (business logic contract) +│ │ ├── FragmentService.cs Validates, calls repo, maps to DTOs +│ │ ├── IEntrySearchService.cs Entry search contract (content parity) +│ │ ├── EntrySearchService.cs Searches decrypted `.md` entries by raw content query +│ │ ├── IJournalConfigService.cs Config contract for path/vault/AI/speech settings parity +│ │ ├── JournalConfigService.cs Env/default-backed config surface aligned with Python keys +│ │ ├── IAiService.cs AI bridge contract (optional provider) +│ │ ├── DisabledAiService.cs No-op AI provider for deterministic disabled mode +│ │ ├── PythonSidecarAiService.cs Local Python sidecar adapter (stdin/stdout JSON) +│ │ ├── SidecarCli.cs CLI runner (`vault` + `search`) used by Sidecar host +│ │ ├── JournalParser.cs Date + section + checkbox + fragment parser slices (Phase 2) +│ │ ├── IVaultCryptoService.cs Vault crypto contract +│ │ ├── VaultCryptoService.cs AES-256-GCM + PBKDF2 compatibility layer +│ │ ├── IVaultStorageService.cs Vault load/workflow contract +│ │ └── VaultStorageService.cs Monthly naming + load/decrypt/extract workflow +│ ├── Entry.cs Command dispatcher (stdin/stdout) +│ ├── ServiceCollectionExtensions.cs DI registration helper +│ └── Journal.Core.csproj +│ +├── Journal.Sidecar/ Console app — Tauri sidecar bridge +│ ├── App.cs Boots DI container, runs Entry.RunAsync() +│ └── Journal.Sidecar.csproj References Journal.Core +│ +├── Journal.Api/ Web API — HTTP endpoint wrapper (optional) +│ ├── Program.cs +│ └── Journal.Api.csproj +│ +└── README.md +``` + +## Architecture + +Each layer only knows about the one below it: + +``` +Sidecar (stdin/stdout) ──┐ + ├──► Services (business logic) ──► Repositories (data access) +API (HTTP/JSON) ─────────┘ +``` + +- **Models** — Domain objects with validation. The source of truth. +- **DTOs** — Immutable records that cross the API boundary. Internal logic never leaks out. +- **Repositories** — Where data lives. Current default is file-backed; can evolve to SQLite/EF Core without touching anything above. +- **Services** — Business rules, validation, orchestration. Doesn't know about HTTP or stdin. +- **Entry** — Transport adapter. Translates stdin/stdout JSON into service calls. + +## Dependencies + +- **Journal.Core** — `Microsoft.Extensions.DependencyInjection.Abstractions` (interface-only, lightweight) +- **Journal.Sidecar** — `Microsoft.Extensions.DependencyInjection` (full container implementation) + references `Journal.Core` +- **Journal.Api** — `Microsoft.AspNetCore.OpenApi` + ASP.NET shared framework + +## Building + +```powershell +# Build everything (building Sidecar also rebuilds Core if changed) +dotnet build backend\Journal.Sidecar\Journal.Sidecar.csproj + +# Build just the library +dotnet build backend\Journal.Core\Journal.Core.csproj + +# Format code +dotnet format backend\Journal.Core\Journal.Core.csproj +``` + +## Publishing + +Publish as a single-file self-contained executable (no .NET runtime install needed): + +```powershell +dotnet publish backend\Journal.Sidecar\Journal.Sidecar.csproj -c Release -r win-x64 --self-contained -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true +``` + +Output: `backend\Journal.Sidecar\bin\Release\net10.0\win-x64\publish\Journal.Sidecar.exe` (~70MB, everything bundled) + +To exclude debug symbols: add `-p:DebugType=none` + +For a smaller build that requires .NET 10 on the target machine: + +```powershell +dotnet publish backend\Journal.Sidecar\Journal.Sidecar.csproj -c Release -r win-x64 -p:PublishSingleFile=true +``` + +## Sidecar Protocol + +The sidecar communicates over stdin/stdout using JSON lines. One JSON line in, one JSON line out. +When run with no command-line args, this protocol mode is used by default. + +## Sidecar CLI + +`Journal.Sidecar` also supports direct vault and search CLI commands: + +```powershell +# Load vaults into decrypted data workspace +dotnet run --project Journal.Sidecar/Journal.Sidecar.csproj -- vault load + +# Save (rebuild) monthly vaults from decrypted markdown files +dotnet run --project Journal.Sidecar/Journal.Sidecar.csproj -- vault save + +# Search entries (query + filters) +dotnet run --project Journal.Sidecar/Journal.Sidecar.csproj -- search "common text" --tag stress --type !TRIGGER --start-date 2026-02-01 --end-date 2026-02-28 --section Summary --checked "med taken" +``` + +Password prompt behavior: +- If `--password` is omitted, CLI prompts with `Vault password:` (hidden input in terminal mode). +- For automation/non-interactive use, pass `--password <value>`. + +Optional path overrides: +- `--vault-dir <path>` +- `--data-dir <path>` +- Env fallback: `JOURNAL_VAULT_DIR`, `JOURNAL_DATA_DIR`, `JOURNAL_APP_DIR` + +Search CLI flags: +- positional `query` (optional) +- `--tag` / `-t` (repeatable) +- `--type` / `-y` (repeatable) +- `--start-date` / `-s` (`yyyy-MM-dd`) +- `--end-date` / `-e` (`yyyy-MM-dd`) +- `--section` / `-sec` +- `--checked` / `-chk` (repeatable) +- `--unchecked` / `-uchk` (repeatable) +- `--data-dir <path>` (optional override) + +## Config Keys (Parity Surface) + +`JournalConfigService` exposes and normalizes key settings expected from Python config: + +- Paths: `JOURNAL_PROJECT_ROOT`, `JOURNAL_APP_DIR`, `JOURNAL_DATA_DIR`, `JOURNAL_VAULT_DIR`, `JOURNAL_LOG_DIR`, `JOURNAL_PID_FILE`, `JOURNAL_SERVER_CONTROL_FILE` +- Vault format: `JOURNAL_MONTHLY_VAULT_FORMAT` (default `%Y-%m.vault`) +- AI endpoints/models: `CLOUDAI_API_KEY`, `CLOUDAI_API_URL`, `LLAMA_CPP_URL`, `LLAMA_CPP_MODEL`, `LLAMA_CPP_TIMEOUT`, `EMBEDDING_API_URL`, `EMBEDDING_MODEL_NAME`, `MODEL_CONTEXT_TOKENS`, `CHUNK_TOKEN_BUDGET` +- AI bridge mode: `JOURNAL_AI_PROVIDER` (`none` or `python-sidecar`), `JOURNAL_PYTHON_EXE`, `JOURNAL_AI_SIDECAR_PATH`, `JOURNAL_AI_TIMEOUT_MS` +- Speech/NLP: `MICROPHONE_DEVICE_INDEX`, `SPEECH_RECOGNITION_ENGINE`, `WHISPER_MODEL_SIZE`, `JOURNAL_NLP_BACKEND` + +### Command Format + +```json +{ + "action": "fragments.create", + "id": null, + "type": null, + "tag": null, + "payload": { "type": "!TRIGGER", "description": "stomach drop" } +} +``` + +**Fields:** +- `action` — The operation to perform (e.g. `fragments.list`, `fragments.create`) +- `id` — Target entity ID (for get/update/delete) +- `type` / `tag` — Filter parameters (for search) +- `payload` — Request body, deserialized into the appropriate DTO per action + +### Available Actions + +| Action | Description | Requires | +|--------|-------------|----------| +| `fragments.list` | List all fragments | — | +| `fragments.get` | Get fragment by ID | `id` | +| `fragments.create` | Create a new fragment | `payload` (CreateFragmentDto) | +| `fragments.update` | Update a fragment | `id`, `payload` (UpdateFragmentDto) | +| `fragments.delete` | Delete a fragment | `id` | +| `fragments.search` | Search by type/tag | `type` and/or `tag` | +| `entries.list` | List decrypted markdown entries in a data directory | optional `payload.dataDirectory` | +| `entries.load` | Load one entry file and return parsed metadata + raw content | `payload.filePath` | +| `entries.save` | Save/merge entry content to file (fragment append or full merge path) | `payload.content`, optional `payload.filePath`, `payload.mode` | +| `db.status` | Return DB key/schema compatibility status snapshot | `payload.password`, optional `payload.dataDirectory` | +| `db.initialize_schema` | Write SQL schema bootstrap (`journal_schema.sql`) for parity tables | optional `payload.dataDirectory` | +| `db.hydrate_workspace` | Perform C# DB hydration step for decrypted workspace (schema bootstrap + metadata) | `payload.password`, optional `payload.dataDirectory` | +| `config.get` | Return current backend config snapshot | — | +| `ai.health` | Return AI bridge health/provider status | — | +| `ai.summarize_entry` | Summarize one entry through AI provider | `payload.content`, optional `payload.fileStem` | +| `ai.summarize_all` | Summarize a set of entries through AI provider | `payload.entries[]` | +| `ai.chat` | Send chat prompt through AI provider bridge | `payload.prompt` | +| `ai.embed` | Generate embedding vector through AI provider bridge | `payload.content` | +| `search.entries` | Search decrypted entry content with optional parity filters | `payload.dataDirectory`, optional `payload.query`, `payload.section`, `payload.startDate`, `payload.endDate`, `payload.tags[]`, `payload.types[]`, `payload.checked[]`, `payload.unchecked[]` | +| `vault.initialize` | Ensure vault directory exists | `payload.password`, `payload.vaultDirectory` | +| `vault.load_all` | Load/decrypt all monthly vaults into data directory | `payload.password`, `payload.vaultDirectory`, `payload.dataDirectory` | +| `vault.save_current_month` | Save only current month vault (optimized path) | `payload.password`, `payload.vaultDirectory`, `payload.dataDirectory`, optional `payload.nowUtc` | +| `vault.rebuild_all` | Rebuild all monthly vaults from decrypted `.md` data | `payload.password`, `payload.vaultDirectory`, `payload.dataDirectory` | +| `vault.clear_data_directory` | Clear decrypted data directory and recreate it | `payload.dataDirectory` | + +### Response Format + +Success: +```json +{ "ok": true, "data": { "id": "abc-123", "type": "!TRIGGER", "description": "...", "time": "...", "tags": [] } } +``` + +Error: +```json +{ "ok": false, "error": "Description is required" } +``` + +## Extending with New Modules + +The `Command` class is generic — new modules use the same dot-notation pattern: + +``` +vault.unlock → IVaultService (future) +vault.lock +entries.list → IEntryService (future) +entries.create +ai.health → IAiService (implemented bridge) +ai.summarize_* → IAiService (implemented bridge) +ai.chat → IAiService (implemented bridge) +ai.embed → IAiService (implemented bridge) +db.status → IJournalDatabaseService (in-progress DB parity) +search.query → ISearchService (future) +``` + +To add a module: +1. Create model, DTO, repository, and service in `Journal.Core/` +2. Register the new service in `ServiceCollectionExtensions.cs` +3. Inject the service into `Entry.cs` and add cases to the action switch +4. No changes needed to `Command.cs` or `App.cs` + +## Dependency Injection + +`ServiceCollectionExtensions.cs` wires everything up. Any host (sidecar, API, tests) calls: + +```csharp +services.AddFragmentServices(); +``` + +This registers: +- `IFragmentRepository` → `FileFragmentRepository` (singleton — persisted fragment store) +- `IFragmentService` → `FragmentService` (transient — fresh instance per request) + +## Fragment Store Location + +`FileFragmentRepository` persists data to: + +- default: `.journal-sidecar/fragments.json` under current working directory +- override: `JOURNAL_FRAGMENT_STORE_PATH` environment variable + +## Legacy Vault Compatibility Note + +The legacy Python placeholder file `_init_vault.vault` is treated as obsolete. +During vault load, the C# backend ignores this file for decryption and removes it. +This preserves compatibility while migrating older vault directories forward. diff --git a/journal-master/journal/nuget-cache-manifest.txt b/journal-master/journal/nuget-cache-manifest.txt new file mode 100644 index 0000000..d996ea2 --- /dev/null +++ b/journal-master/journal/nuget-cache-manifest.txt @@ -0,0 +1,4 @@ +exported_utc=2026-02-22T16:34:35.9385707Z +repo_root=E:\stansshit\Project_Journal\journal-master\journal +include_dotnet_home=False +note=Copy this zip to target machine and run scripts/nuget-import-cache.ps1 diff --git a/journal-master/journal/scripts/dotnet-min.ps1 b/journal-master/journal/scripts/dotnet-min.ps1 new file mode 100644 index 0000000..9b36e3b --- /dev/null +++ b/journal-master/journal/scripts/dotnet-min.ps1 @@ -0,0 +1,62 @@ +param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$DotnetArgs +) + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path + +# Keep dotnet and NuGet artifacts local to the repo for easy cleanup. +$env:DOTNET_CLI_HOME = Join-Path $repoRoot ".dotnet_home" +$env:NUGET_PACKAGES = Join-Path $repoRoot ".nuget\packages" +$env:NUGET_HTTP_CACHE_PATH = Join-Path $repoRoot ".nuget\http-cache" + +# Keep setup minimal and non-interactive. +$env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = "1" +$env:DOTNET_ADD_GLOBAL_TOOLS_TO_PATH = "0" +$env:DOTNET_GENERATE_ASPNET_CERTIFICATE = "0" +$env:DOTNET_CLI_TELEMETRY_OPTOUT = "1" + +# Clear proxy env vars for this process. The host machine currently points them +# to 127.0.0.1:9, which breaks NuGet restore. +Remove-Item Env:HTTP_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:HTTPS_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:ALL_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:http_proxy -ErrorAction SilentlyContinue +Remove-Item Env:https_proxy -ErrorAction SilentlyContinue +Remove-Item Env:all_proxy -ErrorAction SilentlyContinue +Remove-Item Env:GIT_HTTP_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:GIT_HTTPS_PROXY -ErrorAction SilentlyContinue + +# Prefer offline cert revocation checks to reduce flaky TLS behavior on constrained hosts. +$env:NUGET_CERT_REVOCATION_MODE = "offline" + +New-Item -ItemType Directory -Force -Path $env:DOTNET_CLI_HOME | Out-Null +New-Item -ItemType Directory -Force -Path $env:NUGET_PACKAGES | Out-Null +New-Item -ItemType Directory -Force -Path $env:NUGET_HTTP_CACHE_PATH | Out-Null + +if (-not $DotnetArgs -or $DotnetArgs.Count -eq 0) { + Write-Host "Usage: ./scripts/dotnet-min.ps1 <dotnet args>" + Write-Host "Example: ./scripts/dotnet-min.ps1 build Journal.Sidecar/Journal.Sidecar.csproj" + exit 2 +} + +$firstArg = $DotnetArgs[0].ToLowerInvariant() +$effectiveArgs = @($DotnetArgs) + +if ($firstArg -in @("restore", "build", "run", "test", "publish", "pack")) { + if (-not ($effectiveArgs -contains "-p:RestoreIgnoreFailedSources=true")) { + $effectiveArgs += "-p:RestoreIgnoreFailedSources=true" + } + if (-not ($effectiveArgs -contains "-p:NuGetAudit=false")) { + $effectiveArgs += "-p:NuGetAudit=false" + } +} + +if ($firstArg -eq "restore") { + if (-not ($effectiveArgs -contains "--ignore-failed-sources")) { + $effectiveArgs += "--ignore-failed-sources" + } +} + +& dotnet @effectiveArgs +exit $LASTEXITCODE diff --git a/journal-master/journal/scripts/nuget-export-cache.ps1 b/journal-master/journal/scripts/nuget-export-cache.ps1 new file mode 100644 index 0000000..4c9cc61 --- /dev/null +++ b/journal-master/journal/scripts/nuget-export-cache.ps1 @@ -0,0 +1,57 @@ +param( + [string]$OutputZip = "nuget-cache-export.zip", + [switch]$IncludeDotnetHome +) + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +$outputPath = if ([System.IO.Path]::IsPathRooted($OutputZip)) { $OutputZip } else { Join-Path $repoRoot $OutputZip } +$outputDir = Split-Path -Parent $outputPath +if (-not (Test-Path $outputDir)) { + New-Item -ItemType Directory -Force -Path $outputDir | Out-Null +} + +Write-Host "Priming restore cache..." +& (Join-Path $PSScriptRoot "dotnet-min.ps1") restore "Journal.Sidecar/Journal.Sidecar.csproj" +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +& (Join-Path $PSScriptRoot "dotnet-min.ps1") restore "Journal.Api/Journal.Api.csproj" +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +& (Join-Path $PSScriptRoot "dotnet-min.ps1") restore "Journal.SmokeTests/Journal.SmokeTests.csproj" +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + +$staging = Join-Path $repoRoot ".nuget-export-staging" +if (Test-Path $staging) { + Remove-Item -Recurse -Force $staging +} +New-Item -ItemType Directory -Force -Path $staging | Out-Null + +$nugetRoot = Join-Path $repoRoot ".nuget" +if (-not (Test-Path $nugetRoot)) { + Write-Error "No .nuget directory found under $repoRoot" + exit 1 +} + +Copy-Item -Recurse -Force -Path $nugetRoot -Destination (Join-Path $staging ".nuget") +if ($IncludeDotnetHome) { + $dotnetHome = Join-Path $repoRoot ".dotnet_home" + if (Test-Path $dotnetHome) { + Copy-Item -Recurse -Force -Path $dotnetHome -Destination (Join-Path $staging ".dotnet_home") + } +} + +$manifest = @( + "exported_utc=$([DateTime]::UtcNow.ToString("o"))" + "repo_root=$repoRoot" + "include_dotnet_home=$($IncludeDotnetHome.IsPresent)" + "note=Copy this zip to target machine and run scripts/nuget-import-cache.ps1" +) +$manifest | Set-Content -Encoding UTF8 -Path (Join-Path $staging "nuget-cache-manifest.txt") + +if (Test-Path $outputPath) { + Remove-Item -Force $outputPath +} + +Compress-Archive -Path (Join-Path $staging "*") -DestinationPath $outputPath -Force +Remove-Item -Recurse -Force $staging + +Write-Host "NuGet cache export created at: $outputPath" + diff --git a/journal-master/journal/scripts/nuget-import-cache.ps1 b/journal-master/journal/scripts/nuget-import-cache.ps1 new file mode 100644 index 0000000..aced5e8 --- /dev/null +++ b/journal-master/journal/scripts/nuget-import-cache.ps1 @@ -0,0 +1,25 @@ +param( + [string]$InputZip = "nuget-cache-export.zip" +) + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +$inputPath = if ([System.IO.Path]::IsPathRooted($InputZip)) { $InputZip } else { Join-Path $repoRoot $InputZip } + +if (-not (Test-Path $inputPath)) { + Write-Error "Input zip not found: $inputPath" + exit 1 +} + +Write-Host "Importing cache from: $inputPath" +Expand-Archive -Path $inputPath -DestinationPath $repoRoot -Force + +Write-Host "Running restore with local cache..." +& (Join-Path $PSScriptRoot "dotnet-min.ps1") restore "Journal.Sidecar/Journal.Sidecar.csproj" +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +& (Join-Path $PSScriptRoot "dotnet-min.ps1") restore "Journal.Api/Journal.Api.csproj" +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +& (Join-Path $PSScriptRoot "dotnet-min.ps1") restore "Journal.SmokeTests/Journal.SmokeTests.csproj" +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + +Write-Host "Cache import complete." + diff --git a/journal/ai/analysis.py b/journal/ai/analysis.py index 7839b29..72bf820 100644 --- a/journal/ai/analysis.py +++ b/journal/ai/analysis.py @@ -1,6 +1,7 @@ from collections import Counter import re from typing import Any +import os import requests @@ -27,6 +28,7 @@ _VALID_BACKENDS = {_BACKEND_AUTO, _BACKEND_SPACY, _BACKEND_FALLBACK} _backend_name: str | None = None _spacy_nlp: Any | None = None _fallback_warning_printed = False +_backend_requested: str | None = None _STOP_WORDS = { "about", @@ -79,12 +81,18 @@ _STOP_WORDS = { def _resolve_backend() -> str: - global _backend_name, _spacy_nlp, _fallback_warning_printed + global _backend_name, _spacy_nlp, _fallback_warning_printed, _backend_requested - if _backend_name is not None: + requested_raw = os.getenv("JOURNAL_NLP_BACKEND", NLP_BACKEND).strip().lower() + requested = requested_raw if requested_raw in _VALID_BACKENDS else _BACKEND_AUTO + + # Re-resolve if the requested backend changed at runtime via settings. + if _backend_name is not None and requested == _backend_requested: return _backend_name - requested = NLP_BACKEND if NLP_BACKEND in _VALID_BACKENDS else _BACKEND_AUTO + _backend_name = None + _spacy_nlp = None + _backend_requested = requested if requested == _BACKEND_FALLBACK: _backend_name = _BACKEND_FALLBACK return _backend_name @@ -125,12 +133,22 @@ def count_tokens(text: str) -> int: def llama_cpp_generate( prompt: str, - model: str = LLAMA_CPP_MODEL, + model: str | None = None, temperature: float = 0.7, max_tokens: int = 2048, ) -> str: + llama_url = os.getenv("JOURNAL_LLAMA_CPP_URL", LLAMA_CPP_URL).strip() or LLAMA_CPP_URL + llama_model = model or os.getenv("JOURNAL_LLAMA_CPP_MODEL", LLAMA_CPP_MODEL).strip() or LLAMA_CPP_MODEL + timeout_raw = os.getenv("JOURNAL_LLAMA_CPP_TIMEOUT", str(LLAMA_CPP_TIMEOUT)).strip() + try: + llama_timeout = int(timeout_raw) + except ValueError: + llama_timeout = LLAMA_CPP_TIMEOUT + if llama_timeout <= 0: + llama_timeout = LLAMA_CPP_TIMEOUT + payload = { - "model": model, + "model": llama_model, "prompt": prompt, "max_tokens": max_tokens, "temperature": temperature, @@ -138,7 +156,7 @@ def llama_cpp_generate( "stream": False, } try: - response = requests.post(LLAMA_CPP_URL, json=payload, timeout=LLAMA_CPP_TIMEOUT) + response = requests.post(llama_url, json=payload, timeout=llama_timeout) response.raise_for_status() data = response.json() # llama.cpp returns choices array with text field @@ -160,13 +178,26 @@ def generate_embedding(text: str) -> list[float]: """ Generates an embedding for the given text using the configured embedding model. """ + embedding_url = os.getenv("JOURNAL_EMBEDDING_API_URL", EMBEDDING_API_URL).strip() or EMBEDDING_API_URL + embedding_model = ( + os.getenv("JOURNAL_EMBEDDING_MODEL_NAME", EMBEDDING_MODEL_NAME).strip() + or EMBEDDING_MODEL_NAME + ) + timeout_raw = os.getenv("JOURNAL_LLAMA_CPP_TIMEOUT", str(LLAMA_CPP_TIMEOUT)).strip() + try: + llama_timeout = int(timeout_raw) + except ValueError: + llama_timeout = LLAMA_CPP_TIMEOUT + if llama_timeout <= 0: + llama_timeout = LLAMA_CPP_TIMEOUT + payload = { - "model": EMBEDDING_MODEL_NAME, + "model": embedding_model, "input": text, } try: response = requests.post( - EMBEDDING_API_URL, json=payload, timeout=LLAMA_CPP_TIMEOUT + embedding_url, json=payload, timeout=llama_timeout ) # Reusing LLAMA_CPP_TIMEOUT for now response.raise_for_status() data = response.json() @@ -337,8 +368,17 @@ def identify_patterns(entries: list[JournalEntry]) -> list[str]: def chunk_journal_entries( - entries: list[JournalEntry], token_budget: int = CHUNK_TOKEN_BUDGET + entries: list[JournalEntry], token_budget: int | None = None ) -> list[list[JournalEntry]]: + if token_budget is None: + budget_raw = os.getenv("JOURNAL_CHUNK_TOKEN_BUDGET", str(CHUNK_TOKEN_BUDGET)).strip() + try: + token_budget = int(budget_raw) + except ValueError: + token_budget = CHUNK_TOKEN_BUDGET + if token_budget <= 0: + token_budget = CHUNK_TOKEN_BUDGET + chunks = [] current_chunk = [] current_tokens = 0 diff --git a/journal/ai/bridge.py b/journal/ai/bridge.py new file mode 100644 index 0000000..f62815d --- /dev/null +++ b/journal/ai/bridge.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from collections.abc import Sequence + +from journal.ai import analysis +from journal.core.models import JournalEntry + + +def ai_health() -> dict[str, object]: + backend = analysis.get_nlp_backend() + return { + "provider": "python-local", + "enabled": True, + "healthy": True, + "message": f"ok ({backend})", + } + + +def summarize_entry(entry: JournalEntry) -> str: + return analysis.summarize_entry(entry) + + +def summarize_all_entries(entries: Sequence[JournalEntry]) -> str: + return analysis.summarize_all_entries(list(entries)) diff --git a/journal/ai/chat.py b/journal/ai/chat.py index faaa22a..4d1c351 100644 --- a/journal/ai/chat.py +++ b/journal/ai/chat.py @@ -1,18 +1,29 @@ import requests -from journal.core.config import CLOUDAI_API_KEY, CLOUDAI_API_URL +import os +from journal.core.config import CLOUDAI_API_KEY, CLOUDAI_API_URL, CLOUDAI_TIMEOUT def get_cloud_ai_response(prompt: str) -> str: """ Gets a response from the cloud AI service. """ + api_key = os.getenv("JOURNAL_CLOUDAI_API_KEY", CLOUDAI_API_KEY).strip() + api_url = os.getenv("JOURNAL_CLOUDAI_API_URL", CLOUDAI_API_URL).strip() + timeout_raw = os.getenv("JOURNAL_CLOUDAI_TIMEOUT", str(CLOUDAI_TIMEOUT)).strip() + try: + timeout_seconds = int(timeout_raw) + except ValueError: + timeout_seconds = CLOUDAI_TIMEOUT + if timeout_seconds <= 0: + timeout_seconds = CLOUDAI_TIMEOUT + headers = { - "Authorization": f"Bearer {CLOUDAI_API_KEY}", + "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", } payload = {"prompt": prompt} try: - response = requests.post(CLOUDAI_API_URL, headers=headers, json=payload) + response = requests.post(api_url, headers=headers, json=payload, timeout=timeout_seconds) response.raise_for_status() return response.json().get("response", "No response from AI.") except requests.exceptions.RequestException as e: diff --git a/journal/ai/sidecar.py b/journal/ai/sidecar.py new file mode 100644 index 0000000..4c79be4 --- /dev/null +++ b/journal/ai/sidecar.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +import contextlib +import io +import json +import sys +import base64 +import time +from pathlib import Path +from typing import Any + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from journal.ai import analysis, chat +from journal.core.models import JournalEntry + + +def _emit(payload: dict[str, Any]) -> None: + print(json.dumps(payload, ensure_ascii=True), flush=True) + + +def _ok(data: Any) -> None: + _emit({"ok": True, "data": data}) + + +def _error(message: str) -> None: + _emit({"ok": False, "error": message}) + + +def _read_request() -> dict[str, Any]: + line = sys.stdin.readline() + if not line: + raise ValueError("Missing request body.") + + try: + payload = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError("Invalid JSON request.") from exc + + if not isinstance(payload, dict): + raise ValueError("Request must be a JSON object.") + return payload + + +def _safe_call(fn, *args, **kwargs): + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + return fn(*args, **kwargs) + + +def _to_entry(item: Any) -> JournalEntry: + if isinstance(item, str): + return JournalEntry(date="", raw_content=item) + + if isinstance(item, dict): + raw_content = str(item.get("raw_content") or item.get("content") or "") + date = str(item.get("date") or item.get("file_stem") or "") + return JournalEntry(date=date, raw_content=raw_content) + + raise ValueError("Each entry must be a string or object.") + + +def _normalize_entries(payload: dict[str, Any]) -> list[JournalEntry]: + entries = payload.get("entries") + if entries is None: + return [] + if not isinstance(entries, list): + raise ValueError("payload.entries must be an array.") + return [_to_entry(item) for item in entries] + + +def _run_action(action: str, payload: dict[str, Any]) -> Any: + if action == "health": + backend = _safe_call(analysis.get_nlp_backend) + return { + "provider": "python-sidecar", + "healthy": True, + "message": f"ok ({backend})", + } + + if action == "summarize_entry": + content = payload.get("content") + if not isinstance(content, str) or not content.strip(): + raise ValueError("payload.content is required.") + file_stem = payload.get("file_stem") + date = str(file_stem) if file_stem is not None else str(payload.get("date") or "") + entry = JournalEntry(date=date, raw_content=content) + return _safe_call(analysis.summarize_entry, entry) + + if action == "summarize_all": + entries = _normalize_entries(payload) + return _safe_call(analysis.summarize_all_entries, entries) + + if action == "chat": + prompt = payload.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + raise ValueError("payload.prompt is required.") + return _safe_call(chat.get_cloud_ai_response, prompt) + + if action == "embed": + content = payload.get("content") + if not isinstance(content, str) or not content.strip(): + raise ValueError("payload.content is required.") + return _safe_call(analysis.generate_embedding, content) + + if action == "speech.devices.list": + try: + import speech_recognition as sr + + names = sr.Microphone.list_microphone_names() + devices = [ + {"index": index, "name": name} + for index, name in enumerate(names) + ] + warning = "No microphone devices detected." if not devices else None + return {"devices": devices, "warning": warning} + except Exception as exc: + return {"devices": [], "warning": f"Speech device listing unavailable: {exc}"} + + if action == "speech.transcribe": + engine = str(payload.get("engine") or "whisper").strip().lower() or "whisper" + whisper_model = str(payload.get("whisper_model") or "base").strip() or "base" + + simulate_delay_ms = payload.get("simulate_delay_ms") + if isinstance(simulate_delay_ms, int) and simulate_delay_ms > 0: + time.sleep(simulate_delay_ms / 1000.0) + + passthrough = payload.get("text") + if isinstance(passthrough, str) and passthrough.strip(): + return {"text": passthrough, "engine": engine} + + audio_base64 = payload.get("audio_base64") + if not isinstance(audio_base64, str) or not audio_base64.strip(): + raise ValueError("payload.audio_base64 is required.") + + try: + audio_bytes = base64.b64decode(audio_base64, validate=True) + except Exception as exc: + raise ValueError("payload.audio_base64 must be valid base64.") from exc + + import speech_recognition as sr + + recognizer = sr.Recognizer() + with sr.AudioFile(io.BytesIO(audio_bytes)) as source: + audio = recognizer.record(source) + + if engine == "google": + text = recognizer.recognize_google(audio) + elif engine == "whisper": + text = recognizer.recognize_whisper(audio, model=whisper_model) + elif engine == "faster-whisper": + if not hasattr(recognizer, "recognize_faster_whisper"): + raise ValueError("faster-whisper engine unavailable in current SpeechRecognition build.") + text = recognizer.recognize_faster_whisper(audio, model=whisper_model) + elif engine == "sphinx": + text = recognizer.recognize_sphinx(audio) + else: + raise ValueError(f"Unsupported speech engine: {engine}") + + return {"text": text, "engine": engine} + + raise ValueError(f"Unknown action: {action}") + + +def main() -> int: + try: + request = _read_request() + action = request.get("action") + payload = request.get("payload") or {} + + if not isinstance(action, str) or not action.strip(): + raise ValueError("Request action is required.") + if not isinstance(payload, dict): + raise ValueError("Request payload must be an object.") + + result = _run_action(action.strip(), payload) + _ok(result) + return 0 + except Exception as exc: + _error(str(exc)) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/journal/cli/main.py b/journal/cli/main.py index 6421aef..b356b4a 100644 --- a/journal/cli/main.py +++ b/journal/cli/main.py @@ -1,6 +1,7 @@ import argparse from argparse import Namespace import getpass +import json import subprocess import sys import os @@ -14,7 +15,8 @@ sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) from journal.core.storage import rebuild_all_vaults, load_all_vaults from journal.core.parser import parse_journal_file from journal.core.models import JournalEntry -from journal.core.config import DATA_DIR, PID_FILE, PROJECT_ROOT +from journal.core.config import DATA_DIR, PID_FILE, PROJECT_ROOT, BACKEND_MODE +from journal.core.csharp_sidecar import call_sidecar_action class Args(Namespace): @@ -38,6 +40,16 @@ class Args(Namespace): # Chat prompt: str | None = None + # Fragments + fragments_action: str | None = None + fragment_id: str | None = None + fragment_type: str | None = None + fragment_tag: str | None = None + fragment_description: str | None = None + fragment_time: str | None = None + fragment_tags: list[str] | None = None + fragment_clear_tags: bool = False + def main(): parser = argparse.ArgumentParser( @@ -96,6 +108,59 @@ def main(): help="Filter by unchecked checkbox text (can be used multiple times).", ) + # Fragment commands + fragments_parser = subparsers.add_parser( + "fragments", + help="Manage fragment records (hybrid C# backend mode).", + ) + fragment_subparsers = fragments_parser.add_subparsers( + dest="fragments_action", + required=True, + ) + _ = fragment_subparsers.add_parser("list", help="List all fragments.") + + fragment_create = fragment_subparsers.add_parser("create", help="Create a fragment.") + _ = fragment_create.add_argument("--type", dest="fragment_type", required=True) + _ = fragment_create.add_argument("--description", dest="fragment_description", required=True) + _ = fragment_create.add_argument("--time", dest="fragment_time") + _ = fragment_create.add_argument( + "--tag", + dest="fragment_tags", + action="append", + help="Tag value (repeat for multiple tags).", + ) + + fragment_get = fragment_subparsers.add_parser("get", help="Get one fragment by id.") + _ = fragment_get.add_argument("--id", dest="fragment_id", required=True) + + fragment_update = fragment_subparsers.add_parser("update", help="Update one fragment by id.") + _ = fragment_update.add_argument("--id", dest="fragment_id", required=True) + _ = fragment_update.add_argument("--type", dest="fragment_type") + _ = fragment_update.add_argument("--description", dest="fragment_description") + _ = fragment_update.add_argument("--time", dest="fragment_time") + _ = fragment_update.add_argument( + "--tag", + dest="fragment_tags", + action="append", + help="Replace tags with provided values (repeat for multiple tags).", + ) + _ = fragment_update.add_argument( + "--clear-tags", + dest="fragment_clear_tags", + action="store_true", + help="Clear all tags.", + ) + + fragment_delete = fragment_subparsers.add_parser("delete", help="Delete one fragment by id.") + _ = fragment_delete.add_argument("--id", dest="fragment_id", required=True) + + fragment_search = fragment_subparsers.add_parser( + "search", + help="Search fragments by type/tag.", + ) + _ = fragment_search.add_argument("--type", dest="fragment_type") + _ = fragment_search.add_argument("--tag", dest="fragment_tag") + # Server commands server_parser = subparsers.add_parser("server", help="Manage the NiceGUI server.") _ = server_parser.add_argument( @@ -129,6 +194,56 @@ def main(): ) return + if BACKEND_MODE == "csharp-hybrid": + payload: dict[str, object] = { + "dataDirectory": str(DATA_DIR), + } + if args.query: + payload["query"] = args.query + if args.section: + payload["section"] = args.section + if args.start_date: + payload["startDate"] = args.start_date.strftime("%Y-%m-%d") + if args.end_date: + payload["endDate"] = args.end_date.strftime("%Y-%m-%d") + if args.tag: + payload["tags"] = args.tag + if args.type: + payload["types"] = args.type + if args.checked: + payload["checked"] = args.checked + if args.unchecked: + payload["unchecked"] = args.unchecked + + try: + results = call_sidecar_action( + "search.entries", + payload=payload, + timeout_seconds=180, + ) + except Exception as e: + print(f"Hybrid search failed: {e}") + return + + if not isinstance(results, list) or not results: + print("No entries found matching the criteria.") + return + + found_any = False + for item in results: + if not isinstance(item, dict): + continue + date_value = item.get("Date") or item.get("date") or "unknown-date" + raw_content = item.get("RawContent") or item.get("rawContent") or "" + print(f"--- {date_value} ---") + print(raw_content) + print("\n") + found_any = True + + if not found_any: + print("No entries found matching the criteria.") + return + found_entries: list[JournalEntry] = [] for filepath in DATA_DIR.glob("*.md"): entry = parse_journal_file(str(filepath)) @@ -208,6 +323,95 @@ def main(): else: print("No entries found matching the criteria.") + elif args.command == "fragments": + if BACKEND_MODE != "csharp-hybrid": + print("Fragments CLI requires JOURNAL_BACKEND_MODE=csharp-hybrid.") + return + + try: + if args.fragments_action == "list": + result = call_sidecar_action("fragments.list") + _print_fragments(result) + return + + if args.fragments_action == "create": + payload: dict[str, object] = { + "type": args.fragment_type or "", + "description": args.fragment_description or "", + } + if args.fragment_time: + payload["time"] = args.fragment_time + if args.fragment_tags: + payload["tags"] = args.fragment_tags + created = call_sidecar_action("fragments.create", payload=payload) + print("Fragment created.") + _print_json(created) + return + + if args.fragments_action == "get": + fragment = call_sidecar_action( + "fragments.get", + command_fields={"id": args.fragment_id}, + ) + if fragment is None: + print("Fragment not found.") + return + _print_json(fragment) + return + + if args.fragments_action == "update": + payload: dict[str, object] = {} + if args.fragment_type is not None: + payload["type"] = args.fragment_type + if args.fragment_description is not None: + payload["description"] = args.fragment_description + if args.fragment_time is not None: + payload["time"] = args.fragment_time + if args.fragment_clear_tags: + payload["tags"] = [] + elif args.fragment_tags is not None: + payload["tags"] = args.fragment_tags + + if not payload: + print("No update fields provided. Use --type/--description/--time/--tag/--clear-tags.") + return + + updated = call_sidecar_action( + "fragments.update", + payload=payload, + command_fields={"id": args.fragment_id}, + ) + if updated: + print("Fragment updated.") + else: + print("Fragment not found.") + return + + if args.fragments_action == "delete": + deleted = call_sidecar_action( + "fragments.delete", + command_fields={"id": args.fragment_id}, + ) + if deleted: + print("Fragment deleted.") + else: + print("Fragment not found.") + return + + if args.fragments_action == "search": + results = call_sidecar_action( + "fragments.search", + command_fields={ + "type": args.fragment_type, + "tag": args.fragment_tag, + }, + ) + _print_fragments(results) + return + except Exception as e: + print(f"Fragment command failed: {e}") + return + elif args.command == "chat": from journal.ai.chat import get_cloud_ai_response @@ -295,5 +499,17 @@ def main(): print("Server stop command finished.") +def _print_json(value: object) -> None: + print(json.dumps(value, indent=2, ensure_ascii=False)) + + +def _print_fragments(value: object) -> None: + if not isinstance(value, list) or not value: + print("No fragments found.") + return + for item in value: + _print_json(item) + + if __name__ == "__main__": main() diff --git a/journal/core/config.py b/journal/core/config.py index ef0a154..667e43e 100644 --- a/journal/core/config.py +++ b/journal/core/config.py @@ -5,6 +5,25 @@ from pathlib import Path sys.path.append(str(Path(__file__).resolve().parent.parent)) # --- Directories --- PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent + +# Optional local dependency target used by scripts/pip-min.ps1 on constrained hosts. +LOCAL_PYDEPS_DIR = PROJECT_ROOT / ".pydeps" / f"py{sys.version_info.major}{sys.version_info.minor}" +if LOCAL_PYDEPS_DIR.exists(): + local_pydeps = str(LOCAL_PYDEPS_DIR) + if local_pydeps not in sys.path: + sys.path.insert(0, local_pydeps) + +# --- Local model/cache directories --- +HF_HOME = os.getenv("JOURNAL_HF_HOME", str(PROJECT_ROOT / ".cache" / "huggingface")).strip() +HF_HUB_CACHE = os.getenv("JOURNAL_HF_HUB_CACHE", str(Path(HF_HOME) / "hub")).strip() +HF_HUB_DISABLE_SYMLINKS_WARNING = ( + os.getenv("HF_HUB_DISABLE_SYMLINKS_WARNING", "1").strip() or "1" +) + +os.environ.setdefault("HF_HOME", HF_HOME) +os.environ.setdefault("HUGGINGFACE_HUB_CACHE", HF_HUB_CACHE) +os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", HF_HUB_DISABLE_SYMLINKS_WARNING) + DATA_DIR = ( PROJECT_ROOT / "journal" / "data" ) # This will become the temporary decrypted data directory @@ -25,16 +44,35 @@ ITERATIONS = 600_000 MONTHLY_VAULT_FORMAT = "%Y-%m.vault" # e.g., 2025-07.vault # --- AI Configuration --- -CLOUDAI_API_KEY = "" -CLOUDAI_API_URL = "" -LLAMA_CPP_URL = "http://127.0.0.1:8085/v1/completions" -LLAMA_CPP_MODEL = "qwen/qwen3-4b" -LLAMA_CPP_TIMEOUT = 6000 +CLOUDAI_API_KEY = os.getenv("JOURNAL_CLOUDAI_API_KEY", "").strip() +CLOUDAI_API_URL = os.getenv("JOURNAL_CLOUDAI_API_URL", "").strip() +CLOUDAI_TIMEOUT = int(os.getenv("JOURNAL_CLOUDAI_TIMEOUT", "30").strip() or "30") +if CLOUDAI_TIMEOUT <= 0: + CLOUDAI_TIMEOUT = 30 -EMBEDDING_API_URL = "http://127.0.0.1:8086/v1/embeddings" -EMBEDDING_MODEL_NAME = "text-embedding-nomic-embed-text-v2-moe" -MODEL_CONTEXT_TOKENS = 131072 -CHUNK_TOKEN_BUDGET = 120000 +LLAMA_CPP_URL = ( + os.getenv("JOURNAL_LLAMA_CPP_URL", "http://127.0.0.1:8085/v1/completions").strip() + or "http://127.0.0.1:8085/v1/completions" +) +LLAMA_CPP_MODEL = os.getenv("JOURNAL_LLAMA_CPP_MODEL", "qwen/qwen3-4b").strip() or "qwen/qwen3-4b" +LLAMA_CPP_TIMEOUT = int(os.getenv("JOURNAL_LLAMA_CPP_TIMEOUT", "6000").strip() or "6000") +if LLAMA_CPP_TIMEOUT <= 0: + LLAMA_CPP_TIMEOUT = 6000 + +EMBEDDING_API_URL = ( + os.getenv("JOURNAL_EMBEDDING_API_URL", "http://127.0.0.1:8086/v1/embeddings").strip() + or "http://127.0.0.1:8086/v1/embeddings" +) +EMBEDDING_MODEL_NAME = ( + os.getenv("JOURNAL_EMBEDDING_MODEL_NAME", "text-embedding-nomic-embed-text-v2-moe").strip() + or "text-embedding-nomic-embed-text-v2-moe" +) +MODEL_CONTEXT_TOKENS = int(os.getenv("JOURNAL_MODEL_CONTEXT_TOKENS", "131072").strip() or "131072") +if MODEL_CONTEXT_TOKENS <= 0: + MODEL_CONTEXT_TOKENS = 131072 +CHUNK_TOKEN_BUDGET = int(os.getenv("JOURNAL_CHUNK_TOKEN_BUDGET", "120000").strip() or "120000") +if CHUNK_TOKEN_BUDGET <= 0: + CHUNK_TOKEN_BUDGET = 120000 # --- Hardware Configuration --- # Set this to a specific index from `journal devices list` to force a microphone. @@ -42,11 +80,47 @@ CHUNK_TOKEN_BUDGET = 120000 MICROPHONE_DEVICE_INDEX: int | None = None # --- Speech Recognition --- -# "whisper" is local, private, and highly accurate (recommended). Downloads a model on first use. -# "google" is online, accurate, but sends data to Google. -# "sphinx" is offline, fast, but much less accurate. -SPEECH_RECOGNITION_ENGINE: str = "whisper" -WHISPER_MODEL_SIZE: str = "base" # Options: "tiny", "base", "small", "medium", "large" +# "faster-whisper" and "whisper" are local/private options. +# "google" is online and sends audio to Google. +# "sphinx" is offline and fast but less accurate. +_VALID_SPEECH_ENGINES = {"faster-whisper", "whisper", "google", "sphinx"} +_VALID_WHISPER_MODELS = {"tiny", "base", "small", "medium", "large"} +_VALID_FASTER_WHISPER_DEVICES = {"auto", "cpu", "cuda"} +_VALID_FASTER_WHISPER_COMPUTE_TYPES = { + "auto", + "default", + "int8", + "int8_float32", + "int8_float16", + "float16", + "float32", +} + +SPEECH_RECOGNITION_ENGINE: str = ( + os.getenv("JOURNAL_SPEECH_ENGINE", "faster-whisper").strip().lower() + or "faster-whisper" +) +if SPEECH_RECOGNITION_ENGINE not in _VALID_SPEECH_ENGINES: + SPEECH_RECOGNITION_ENGINE = "faster-whisper" + +WHISPER_MODEL_SIZE: str = ( + os.getenv("JOURNAL_WHISPER_MODEL", "base").strip().lower() or "base" +) +if WHISPER_MODEL_SIZE not in _VALID_WHISPER_MODELS: + WHISPER_MODEL_SIZE = "base" + +FASTER_WHISPER_DEVICE: str = ( + os.getenv("JOURNAL_FASTER_WHISPER_DEVICE", "auto").strip().lower() or "auto" +) +if FASTER_WHISPER_DEVICE not in _VALID_FASTER_WHISPER_DEVICES: + FASTER_WHISPER_DEVICE = "auto" + +FASTER_WHISPER_COMPUTE_TYPE: str = ( + os.getenv("JOURNAL_FASTER_WHISPER_COMPUTE_TYPE", "float32").strip().lower() + or "float32" +) +if FASTER_WHISPER_COMPUTE_TYPE not in _VALID_FASTER_WHISPER_COMPUTE_TYPES: + FASTER_WHISPER_COMPUTE_TYPE = "float32" # NLP backend selection: # - auto: use spaCy when available, otherwise fallback heuristics. @@ -60,3 +134,14 @@ if NLP_BACKEND not in {"auto", "spacy", "fallback"}: DATA_DIR.mkdir(exist_ok=True) VAULT_DIR.mkdir(exist_ok=True) LOG_DIR.mkdir(exist_ok=True) +Path(HF_HUB_CACHE).mkdir(parents=True, exist_ok=True) + +# --- Backend selection --- +# python: pure Python backend (C# is Default) +# csharp-hybrid: Python UI + C# sidecar for vault, entry, and search operations +BACKEND_MODE: str = os.getenv("JOURNAL_BACKEND_MODE", "csharp-hybrid").strip().lower() or "csharp-hybrid" +if BACKEND_MODE not in {"python", "csharp-hybrid"}: + BACKEND_MODE = "csharp-hybrid" + +# Optional absolute/relative override for the C# sidecar executable path. +CSHARP_SIDECAR_PATH: str = os.getenv("JOURNAL_CSHARP_SIDECAR_PATH", "").strip() diff --git a/journal/core/csharp_sidecar.py b/journal/core/csharp_sidecar.py new file mode 100644 index 0000000..4b3c547 --- /dev/null +++ b/journal/core/csharp_sidecar.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +from .config import CSHARP_SIDECAR_PATH + + +def call_sidecar_action( + action: str, + payload: dict[str, Any] | None = None, + *, + command_fields: dict[str, Any] | None = None, + timeout_seconds: int = 120, +) -> Any: + request = { + "action": action, + "payload": payload or {}, + } + if command_fields: + for key, value in command_fields.items(): + if key in {"action", "payload"}: + continue + request[key] = value + command, cwd = _resolve_sidecar_command() + + result = subprocess.run( + command, + input=json.dumps(request) + "\n", + text=True, + capture_output=True, + cwd=str(cwd), + timeout=timeout_seconds, + check=False, + ) + + stdout_line = _first_non_empty_line(result.stdout) + if not stdout_line: + stderr_line = _first_non_empty_line(result.stderr) + details = stderr_line or f"sidecar exited with code {result.returncode}" + raise RuntimeError(f"C# sidecar did not return JSON response: {details}") + + try: + response = json.loads(stdout_line) + except json.JSONDecodeError as exc: + raise RuntimeError(f"Invalid JSON from C# sidecar: {stdout_line}") from exc + + if not isinstance(response, dict): + raise RuntimeError("Unexpected sidecar response shape.") + + if bool(response.get("ok")): + return response.get("data") + + error = str(response.get("error", "Unknown sidecar error")) + raise RuntimeError(error) + + +def _resolve_sidecar_command() -> tuple[list[str], Path]: + project_root = Path(__file__).resolve().parents[2] + sidecar_root = project_root / "journal-master" / "journal" + + override = _resolve_override_path(project_root) + if override is not None: + return [str(override)], sidecar_root + + for candidate in _candidate_sidecar_paths(sidecar_root): + if candidate.exists(): + return [str(candidate)], sidecar_root + + csproj = sidecar_root / "Journal.Sidecar" / "Journal.Sidecar.csproj" + return ["dotnet", "run", "--project", str(csproj)], sidecar_root + + +def _resolve_override_path(project_root: Path) -> Path | None: + if not CSHARP_SIDECAR_PATH: + return None + override = Path(CSHARP_SIDECAR_PATH) + if not override.is_absolute(): + override = project_root / override + return override + + +def _candidate_sidecar_paths(sidecar_root: Path) -> list[Path]: + sidecar_dir = sidecar_root / "Journal.Sidecar" + candidates: list[Path] = [] + + if sys.platform == "win32": + candidates.extend( + [ + sidecar_dir / "bin" / "Debug" / "net10.0" / "Journal.Sidecar.exe", + sidecar_dir / "bin" / "Release" / "net10.0" / "Journal.Sidecar.exe", + ] + ) + else: + candidates.extend( + [ + sidecar_dir / "bin" / "Debug" / "net10.0" / "Journal.Sidecar", + sidecar_dir / "bin" / "Release" / "net10.0" / "Journal.Sidecar", + ] + ) + + return candidates + + +def _first_non_empty_line(text: str) -> str: + for line in text.splitlines(): + stripped = line.strip() + if stripped: + return stripped + return "" diff --git a/journal/core/parser.py b/journal/core/parser.py index b743209..ba4dbaa 100644 --- a/journal/core/parser.py +++ b/journal/core/parser.py @@ -15,7 +15,7 @@ def parse_journal_file(file_path: str) -> JournalEntry: def parse_journal_content(content: str, file_stem: str) -> JournalEntry: """Parses the raw text content of a journal entry.""" - date_match = re.search(r"(?:\*\*Date:|Date:)\s*(.+)", content) + date_match = re.search(r"(?:\*\*Date:\*\*|Date:)\s*(.+)", content) date = date_match.group(1).strip() if date_match else file_stem parsed_sections: dict[str, ParsedSection] = {} diff --git a/journal/core/speech.py b/journal/core/speech.py index 8c38a6e..d825bc3 100644 --- a/journal/core/speech.py +++ b/journal/core/speech.py @@ -1,7 +1,24 @@ import speech_recognition as sr from typing import Protocol import queue -from .config import MICROPHONE_DEVICE_INDEX +import sys +from .config import ( + MICROPHONE_DEVICE_INDEX, + FASTER_WHISPER_DEVICE, + FASTER_WHISPER_COMPUTE_TYPE, +) + +# Windows fallback: pyaudiowpatch ships wheels for newer Python versions +# where PyAudio source builds can fail due missing PortAudio headers. +try: + import pyaudio # type: ignore # noqa: F401 +except Exception: + try: + import pyaudiowpatch as _pyaudio_patch # type: ignore + + sys.modules.setdefault("pyaudio", _pyaudio_patch) + except Exception: + pass class Stoppable(Protocol): @@ -14,8 +31,46 @@ class Stoppable(Protocol): background_listener_stop: Stoppable | None = None +def _recognize_local_whisper( + recognizer: sr.Recognizer, + audio: sr.AudioData, + engine: str, + whisper_model: str, + faster_whisper_device: str | None = None, + faster_whisper_compute_type: str | None = None, +) -> str: + if engine == "faster-whisper": + if hasattr(recognizer, "recognize_faster_whisper"): + init_options: dict[str, str] = {} + active_device = ( + (faster_whisper_device or "").strip().lower() or FASTER_WHISPER_DEVICE + ) + active_compute = ( + (faster_whisper_compute_type or "").strip().lower() + or FASTER_WHISPER_COMPUTE_TYPE + ) + if active_device != "auto": + init_options["device"] = active_device + if active_compute not in {"auto", "default"}: + init_options["compute_type"] = active_compute + + return recognizer.recognize_faster_whisper( + audio, + model=whisper_model, + init_options=init_options or None, + ) + # Older/newer SpeechRecognition builds may not expose this wrapper. + return recognizer.recognize_whisper(audio, model=whisper_model) + + return recognizer.recognize_whisper(audio, model=whisper_model) + + def start_background_listening( - message_queue: queue.Queue[tuple[str, str]], engine: str, whisper_model: str + message_queue: queue.Queue[tuple[str, str]], + engine: str, + whisper_model: str, + faster_whisper_device: str | None = None, + faster_whisper_compute_type: str | None = None, ): """ Starts listening to the microphone in a background thread. @@ -44,10 +99,26 @@ def start_background_listening( try: if engine == "google": text = recognizer.recognize_google(audio) - elif engine == "whisper": - # Use local Whisper for high accuracy and privacy. - # The model will be downloaded automatically on first use. - text = recognizer.recognize_whisper(audio, model=whisper_model) + elif engine in {"whisper", "faster-whisper"}: + # Use local Whisper-family inference for privacy. + # Models are downloaded automatically on first use. + if engine == "faster-whisper" and not hasattr( + recognizer, "recognize_faster_whisper" + ): + message_queue.put( + ( + "status", + "faster-whisper not exposed by this SpeechRecognition build; falling back to whisper.", + ) + ) + text = _recognize_local_whisper( + recognizer, + audio, + engine, + whisper_model, + faster_whisper_device=faster_whisper_device, + faster_whisper_compute_type=faster_whisper_compute_type, + ) else: # Default to the fast, offline, but less accurate sphinx text = recognizer.recognize_sphinx(audio) diff --git a/journal/core/storage.py b/journal/core/storage.py index eb84867..3dc7939 100644 --- a/journal/core/storage.py +++ b/journal/core/storage.py @@ -2,6 +2,8 @@ import sys import hashlib import threading import time +import html +import re from cryptography.exceptions import InvalidTag import shutil import zipfile @@ -12,17 +14,70 @@ sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) from .parser import parse_journal_content, parse_journal_file from .database import get_db_connection, hydrate_database from .encryption import encrypt_data, decrypt_data +from .csharp_sidecar import call_sidecar_action from .config import ( DATA_DIR, VAULT_DIR, MONTHLY_VAULT_FORMAT, + BACKEND_MODE, ) _month_fingerprint_cache: dict[str, str] = {} _vault_io_lock = threading.RLock() +def _using_csharp_hybrid() -> bool: + return BACKEND_MODE == "csharp-hybrid" + + +def _looks_like_rich_html(content: str) -> bool: + lowered = content.lower() + html_markers = ( + "<p", + "</p>", + "<div", + "<span", + "<table", + "<tr", + "<td", + "<li", + "<ul", + "<ol", + "style=", + "font-family:", + "-webkit-text-stroke", + ) + if any(marker in lowered for marker in html_markers): + return True + return len(re.findall(r"</?[a-z][^>]*>", lowered)) >= 8 + + +def _strip_rich_html(content: str) -> str: + if not _looks_like_rich_html(content): + return content + + text = content.replace("\r\n", "\n").replace("\r", "\n") + text = re.sub(r"(?is)<(script|style)\b[^>]*>.*?</\1>", "", text) + text = re.sub(r"(?i)<br\s*/?>", "\n", text) + text = re.sub(r"(?i)</(p|div|h[1-6]|tr|table|ul|ol|blockquote)>", "\n", text) + text = re.sub(r"(?i)<li\b[^>]*>", "\n- ", text) + text = re.sub(r"(?i)</li>", "\n", text) + text = re.sub(r"(?i)<(td|th)\b[^>]*>", " | ", text) + text = re.sub(r"(?i)</(td|th)>", " ", text) + text = re.sub(r"(?i)<hr\b[^>]*>", "\n---\n", text) + text = re.sub(r"(?is)<[^>]+>", "", text) + text = html.unescape(text) + text = text.replace("\u00a0", " ").replace("\u200b", "") + text = "\n".join(line.rstrip() for line in text.splitlines()) + text = re.sub(r"[ \t]{2,}", " ", text) + text = re.sub(r"\n{3,}", "\n\n", text).strip() + + if text: + return text + return content + + # --- Monthly Vault Management --- @@ -100,29 +155,89 @@ def get_today_filename() -> Path: return DATA_DIR / f"{datetime.now().strftime('%Y-%m-%d')}.md" +def list_journal_files() -> list[tuple[str, str]]: + """Lists decrypted markdown entries as (file_name, absolute_path).""" + if _using_csharp_hybrid(): + results = call_sidecar_action( + "entries.list", + payload={"dataDirectory": str(DATA_DIR)}, + ) + if not isinstance(results, list): + return [] + + files: list[tuple[str, str]] = [] + for item in results: + if not isinstance(item, dict): + continue + name = item.get("FileName") or item.get("fileName") + path = item.get("FilePath") or item.get("filePath") + if isinstance(name, str) and isinstance(path, str): + files.append((name, path)) + return files + + files = sorted(DATA_DIR.glob("*.md")) + return [(f.name, str(f)) for f in files] + + +def load_entry_content(file_path: str | Path) -> str: + """Loads one journal entry and returns the raw markdown content.""" + normalized_path = str(file_path) + if _using_csharp_hybrid(): + data = call_sidecar_action( + "entries.load", + payload={"filePath": normalized_path}, + ) + if isinstance(data, str): + return _strip_rich_html(data) + if isinstance(data, dict): + raw = data.get("RawContent") or data.get("rawContent") + if isinstance(raw, str): + return _strip_rich_html(raw) + raise RuntimeError("Unexpected entries.load response shape from C# sidecar.") + + entry = parse_journal_file(normalized_path) + return _strip_rich_html(entry.raw_content) + + def save_entry_content( content: str, file_path: Path | None = None, mode: str = "Daily" ): + sanitized_content = _strip_rich_html(content) target_file = file_path or get_today_filename() target_file.parent.mkdir(parents=True, exist_ok=True) + if _using_csharp_hybrid(): + _ = call_sidecar_action( + "entries.save", + payload={ + "content": sanitized_content, + "filePath": str(target_file), + "mode": mode, + }, + ) + return + + if mode == "Overwrite": + _ = target_file.write_text(sanitized_content, encoding="utf-8") + return + if mode == "Fragment": print(f"Appending fragment to {target_file.name}...") with open(target_file, "a", encoding="utf-8") as f: # Ensure there's a newline before the new content - _ = f.write("\n\n" + content.strip()) + _ = f.write("\n\n" + sanitized_content.strip()) return # For Daily, Deep, etc., perform a merge if target_file.exists(): print(f"Merging content into existing file: {target_file.name}") existing_entry = parse_journal_file(str(target_file)) - new_entry_data = parse_journal_content(content, target_file.stem) + new_entry_data = parse_journal_content(sanitized_content, target_file.stem) existing_entry.merge_with(new_entry_data) final_content = existing_entry.to_markdown() else: print(f"Creating new entry: {target_file.name}") - final_content = content + final_content = sanitized_content _ = target_file.write_text(final_content, encoding="utf-8") @@ -139,53 +254,79 @@ def load_all_vaults(password: str) -> bool: with _vault_io_lock: _month_fingerprint_cache.clear() - # Clear DATA_DIR first - _clear_data_dir_with_retries() - DATA_DIR.mkdir(parents=True, exist_ok=True) - - if not VAULT_DIR.exists() or not any(VAULT_DIR.iterdir()): - print("Vault directory is empty or does not exist. Assuming new vault.") - return True # No vaults to load, so it's a success (new vault) - - decryption_successful = False - for vault_file in VAULT_DIR.glob("*.vault"): - if vault_file.name == "_init_vault.vault": - print(f"Deleting old dummy vault file: {vault_file.name}") - vault_file.unlink() - continue + if _using_csharp_hybrid(): + load_success = bool( + call_sidecar_action( + "vault.load_all", + payload={ + "password": password, + "vaultDirectory": str(VAULT_DIR), + "dataDirectory": str(DATA_DIR), + }, + ) + ) + if not load_success: + return False try: - with open(vault_file, "rb") as f_in: - encrypted_data = f_in.read() - - decrypted_zip_content = decrypt_data(encrypted_data, password) - - # Write decrypted content to a temporary zip file - temp_zip_path = VAULT_DIR / f"temp_{vault_file.name}.zip" - with open(temp_zip_path, "wb") as f_out: - _ = f_out.write(decrypted_zip_content) - - _extract_monthly_archive(temp_zip_path, DATA_DIR) - temp_zip_path.unlink() # Clean up temp zip - decryption_successful = True - print(f"Successfully loaded {vault_file.name}") - print( - f"Contents of DATA_DIR after loading {vault_file.name}: {list(DATA_DIR.iterdir())}" + _ = call_sidecar_action( + "db.hydrate_workspace", + payload={ + "password": password, + "dataDirectory": str(DATA_DIR), + }, ) - except InvalidTag: - print( - f"Warning: Could not decrypt '{vault_file.name}'. Invalid password for this file." - ) - # Do not set decryption_successful to True if only some files fail except Exception as e: - print(f"Error loading vault '{vault_file.name}': {e}") - # If any other error occurs, it's not necessarily a password issue + print(f"Fatal error during C# workspace hydration: {e}") + return False + return True + else: + # Clear DATA_DIR first + _clear_data_dir_with_retries() + DATA_DIR.mkdir(parents=True, exist_ok=True) - if not decryption_successful and any(VAULT_DIR.iterdir()): - # If there are vault files, but none could be decrypted, password is wrong - print("Error: No vault files could be decrypted with the provided password.") - return False + if not VAULT_DIR.exists() or not any(VAULT_DIR.iterdir()): + print("Vault directory is empty or does not exist. Assuming new vault.") + return True # No vaults to load, so it's a success (new vault) - # --- Database Hydration --- + decryption_successful = False + for vault_file in VAULT_DIR.glob("*.vault"): + if vault_file.name == "_init_vault.vault": + print(f"Deleting old dummy vault file: {vault_file.name}") + vault_file.unlink() + continue + try: + with open(vault_file, "rb") as f_in: + encrypted_data = f_in.read() + + decrypted_zip_content = decrypt_data(encrypted_data, password) + + # Write decrypted content to a temporary zip file + temp_zip_path = VAULT_DIR / f"temp_{vault_file.name}.zip" + with open(temp_zip_path, "wb") as f_out: + _ = f_out.write(decrypted_zip_content) + + _extract_monthly_archive(temp_zip_path, DATA_DIR) + temp_zip_path.unlink() # Clean up temp zip + decryption_successful = True + print(f"Successfully loaded {vault_file.name}") + print( + f"Contents of DATA_DIR after loading {vault_file.name}: {list(DATA_DIR.iterdir())}" + ) + except InvalidTag: + print( + f"Warning: Could not decrypt '{vault_file.name}'. Invalid password for this file." + ) + # Do not set decryption_successful to True if only some files fail + except Exception as e: + print(f"Error loading vault '{vault_file.name}': {e}") + # If any other error occurs, it's not necessarily a password issue + + if not decryption_successful and any(VAULT_DIR.iterdir()): + # If there are vault files, but none could be decrypted, password is wrong + print("Error: No vault files could be decrypted with the provided password.") + return False + + # --- Database Hydration (Python mode only) --- # After successfully decrypting files, hydrate the live, encrypted database. conn = None try: @@ -216,6 +357,17 @@ def rebuild_all_vaults(password: str): if not password: raise ValueError("Password cannot be empty.") + if _using_csharp_hybrid(): + _ = call_sidecar_action( + "vault.rebuild_all", + payload={ + "password": password, + "vaultDirectory": str(VAULT_DIR), + "dataDirectory": str(DATA_DIR), + }, + ) + return + with _vault_io_lock: # Group files by month monthly_files: dict[str, list[Path]] = {} @@ -248,6 +400,18 @@ def save_current_month_vault(password: str): if not password: raise ValueError("Password cannot be empty.") + if _using_csharp_hybrid(): + _ = call_sidecar_action( + "vault.save_current_month", + payload={ + "password": password, + "vaultDirectory": str(VAULT_DIR), + "dataDirectory": str(DATA_DIR), + "nowUtc": datetime.utcnow().isoformat() + "Z", + }, + ) + return + with _vault_io_lock: # Determine current month now = datetime.now() @@ -279,6 +443,16 @@ def initialize_vault(password: str): if not password: raise ValueError("Password cannot be empty.") + if _using_csharp_hybrid(): + _ = call_sidecar_action( + "vault.initialize", + payload={ + "password": password, + "vaultDirectory": str(VAULT_DIR), + }, + ) + return + VAULT_DIR.mkdir(parents=True, exist_ok=True) print("Vault directory ensured to exist.") @@ -288,6 +462,14 @@ def clear_data_directory(): Clears the DATA_DIR. This should only be called on application shutdown. """ print("Clearing DATA_DIR...") + if _using_csharp_hybrid(): + _ = call_sidecar_action( + "vault.clear_data_directory", + payload={"dataDirectory": str(DATA_DIR)}, + ) + print("DATA_DIR cleared.") + return + with _vault_io_lock: # The encrypted database file lives in DATA_DIR, so this function # will securely delete it along with all the decrypted .md files. diff --git a/journal/run_desktop.py b/journal/run_desktop.py index fb56575..f5968e5 100644 --- a/journal/run_desktop.py +++ b/journal/run_desktop.py @@ -25,13 +25,23 @@ _process_lock = threading.Lock() _watchdog_stop = threading.Event() _WATCHDOG_INTERVAL_SECONDS = 10.0 _WATCHDOG_MAX_HEALTH_FAILURES = 3 +_WATCHDOG_HEALTHCHECK_TIMEOUT_SECONDS = 2.5 _WATCHDOG_MIN_RESTART_INTERVAL_SECONDS = 5.0 _WATCHDOG_MAX_FAILED_RESTARTS = 5 +_WATCHDOG_MAX_CONSECUTIVE_CRASH_RESTARTS = 3 +_WATCHDOG_STARTUP_GRACE_SECONDS = 45.0 +_WATCHDOG_RESTART_GRACE_SECONDS = 30.0 _watchdog_failed_restarts = 0 _last_restart_monotonic = 0.0 +_watchdog_grace_until_monotonic = 0.0 SERVER_URL = "http://localhost:8080" HEALTHCHECK_URL = f"{SERVER_URL}/_health" VALID_SERVER_ACTIONS = {"restart", "shutdown"} +DISABLE_WEBVIEW = os.getenv("JOURNAL_DISABLE_WEBVIEW", "").strip().lower() in { + "1", + "true", + "yes", +} def wait_for_server(url: str, timeout_seconds: float = 20.0) -> bool: @@ -72,20 +82,45 @@ def _clear_server_action() -> None: def get_python_executable() -> str: """ - Determines the correct Python executable path by searching the system's PATH. + Returns the current interpreter path for child server startup. - This function uses `shutil.which('python')` to locate the `python` executable. - When a virtual environment is active, its `bin` directory is at the front of the - PATH, so this will correctly return the path to the venv's interpreter. + Prefer `sys.executable` so wrapper and child use the same runtime (e.g., 3.14t). + Fall back to PATH lookup only if `sys.executable` is missing/unusable. Returns: The absolute path to the Python executable. """ - python_executable = shutil.which('python') + if sys.executable: + return sys.executable + + python_executable = shutil.which("python") if python_executable: return python_executable - # Fallback to sys.executable if shutil.which fails for some reason. - return sys.executable + + raise RuntimeError("Could not determine Python executable path.") + + +def can_use_webview() -> bool: + """ + Returns True when pywebview backend is available on this host. + On Windows, pywebview requires pythonnet (`clr`) for WinForms backend. + """ + if DISABLE_WEBVIEW: + print("Webview disabled via JOURNAL_DISABLE_WEBVIEW; using browser mode.") + return False + + if webview is None: + return False + + if sys.platform != "win32": + return True + + try: + import clr # type: ignore # noqa: F401 + return True + except Exception: + print("pywebview backend unavailable on Windows (pythonnet `clr` is missing).") + return False def is_server_running(): @@ -178,7 +213,7 @@ def _stop_process(process: Optional[subprocess.Popen], timeout_seconds: float = def _restart_nicegui(reason: str) -> None: - global _watchdog_failed_restarts, _last_restart_monotonic + global _watchdog_failed_restarts, _last_restart_monotonic, _watchdog_grace_until_monotonic if _watchdog_stop.is_set(): return elapsed = time.monotonic() - _last_restart_monotonic @@ -190,6 +225,9 @@ def _restart_nicegui(reason: str) -> None: _safe_set_process(None) _last_restart_monotonic = time.monotonic() start_nicegui() + _watchdog_grace_until_monotonic = ( + time.monotonic() + _WATCHDOG_RESTART_GRACE_SECONDS + ) if wait_for_server(HEALTHCHECK_URL, timeout_seconds=20.0): _watchdog_failed_restarts = 0 print("Watchdog restart completed: server is healthy.") @@ -205,7 +243,9 @@ def _restart_nicegui(reason: str) -> None: def _watchdog_loop() -> None: + global _watchdog_grace_until_monotonic consecutive_health_failures = 0 + consecutive_crash_restarts = 0 while not _watchdog_stop.wait(_WATCHDOG_INTERVAL_SECONDS): process = _safe_get_process() if process is None: @@ -221,13 +261,33 @@ def _watchdog_loop() -> None: _watchdog_stop.set() break if action == "restart": + consecutive_crash_restarts = 0 _restart_nicegui("server restart requested from UI") else: + if exit_code != 0: + consecutive_crash_restarts += 1 + else: + consecutive_crash_restarts = 0 + if consecutive_crash_restarts >= _WATCHDOG_MAX_CONSECUTIVE_CRASH_RESTARTS: + print( + "Server crashed repeatedly during startup. " + "Common cause: launching with free-threaded Python (python3.14t) while " + "binary wheels (orjson/pydantic_core) are installed for regular CPython. " + "Use python.exe for this project runtime." + ) + _watchdog_stop.set() + break _restart_nicegui(f"server process exited with code {exit_code}") consecutive_health_failures = 0 continue - is_healthy = wait_for_server(HEALTHCHECK_URL, timeout_seconds=1.0) + if time.monotonic() < _watchdog_grace_until_monotonic: + consecutive_health_failures = 0 + continue + + is_healthy = wait_for_server( + HEALTHCHECK_URL, timeout_seconds=_WATCHDOG_HEALTHCHECK_TIMEOUT_SECONDS + ) if is_healthy: consecutive_health_failures = 0 continue @@ -242,6 +302,7 @@ def _watchdog_loop() -> None: def run(): + global _watchdog_grace_until_monotonic started_by_wrapper = False watchdog_thread: Optional[threading.Thread] = None _clear_server_action() @@ -253,6 +314,9 @@ def run(): # Start NiceGUI server managed by this wrapper. print("Starting NiceGUI server...") start_nicegui() + _watchdog_grace_until_monotonic = ( + time.monotonic() + _WATCHDOG_STARTUP_GRACE_SECONDS + ) started_by_wrapper = True if started_by_wrapper: @@ -264,7 +328,7 @@ def run(): try: # Open desktop shell if available; otherwise use browser fallback. - if webview is not None: + if can_use_webview(): try: print("Opening webview window...") _ = webview.create_window("Project Journal", SERVER_URL) diff --git a/journal/ui/components/calendar.py b/journal/ui/components/calendar.py index 5e46acd..669d05a 100644 --- a/journal/ui/components/calendar.py +++ b/journal/ui/components/calendar.py @@ -2,12 +2,12 @@ from nicegui import ui from typing import Callable, cast -def calendar_view(on_select: Callable[[str], None]) -> None: - with ui.card().tight().classes("bg-gray-800 text-white"): - with ui.row().classes("w-full items-center px-4"): +def calendar_view(on_select: Callable[[str | None], None]) -> None: + with ui.card().classes("bg-gray-800 text-white w-full journal-calendar-card"): + with ui.row().classes("w-full items-center px-3"): _ = ui.label("Journal Calendar").classes("text-lg font-bold") - with ui.row().classes("w-full items-center px-4"): + with ui.row().classes("w-full items-center px-3"): # Calendar view _ = ui.date( - on_change=lambda e: on_select(cast(str, e.value)) - ).classes("bg-gray-800 text-white") + on_change=lambda e: on_select(cast(str | None, e.value)) + ).props("dark").classes("bg-gray-800 text-white journal-calendar") diff --git a/journal/ui/components/editor.py b/journal/ui/components/editor.py index e21e4fd..e703acf 100644 --- a/journal/ui/components/editor.py +++ b/journal/ui/components/editor.py @@ -136,3 +136,13 @@ def rich_text_editor() -> ui.editor: _ = editor.on("vue-mounted", attach_paste_handler) return editor + + +def markdown_editor() -> ui.textarea: + """A markdown-native editor with stable scrolling and no HTML conversion.""" + return ( + ui.textarea(placeholder="Write markdown here...") + .props("outlined autogrow=false") + .classes("bg-gray-800 text-white journal-markdown-editor") + .style("flex: 1; width: 100%; min-height: 0;") + ) diff --git a/journal/ui/components/settings.py b/journal/ui/components/settings.py index a0169b0..7f62d2c 100644 --- a/journal/ui/components/settings.py +++ b/journal/ui/components/settings.py @@ -1,16 +1,119 @@ import asyncio +import os +from typing import Any, cast + +from nicegui import app, ui -from nicegui import ui, app from journal.core.config import ( + BACKEND_MODE, + CHUNK_TOKEN_BUDGET, + CLOUDAI_API_KEY, + CLOUDAI_API_URL, + CLOUDAI_TIMEOUT, + CSHARP_SIDECAR_PATH, + EMBEDDING_API_URL, + EMBEDDING_MODEL_NAME, + FASTER_WHISPER_COMPUTE_TYPE, + FASTER_WHISPER_DEVICE, + LLAMA_CPP_MODEL, + LLAMA_CPP_TIMEOUT, + LLAMA_CPP_URL, + MODEL_CONTEXT_TOKENS, + NLP_BACKEND, + SERVER_CONTROL_FILE, SPEECH_RECOGNITION_ENGINE, WHISPER_MODEL_SIZE, - SERVER_CONTROL_FILE, ) -from typing import cast + +_DEFAULT_SETTINGS: dict[str, Any] = { + "backend_mode": BACKEND_MODE, + "csharp_sidecar_path": CSHARP_SIDECAR_PATH, + "nlp_backend": NLP_BACKEND, + "speech_engine": SPEECH_RECOGNITION_ENGINE, + "whisper_model": WHISPER_MODEL_SIZE, + "faster_whisper_device": FASTER_WHISPER_DEVICE, + "faster_whisper_compute_type": FASTER_WHISPER_COMPUTE_TYPE, + "llama_cpp_url": LLAMA_CPP_URL, + "llama_cpp_model": LLAMA_CPP_MODEL, + "llama_cpp_timeout": LLAMA_CPP_TIMEOUT, + "embedding_api_url": EMBEDDING_API_URL, + "embedding_model_name": EMBEDDING_MODEL_NAME, + "cloudai_api_url": CLOUDAI_API_URL, + "cloudai_api_key": CLOUDAI_API_KEY, + "cloudai_timeout": CLOUDAI_TIMEOUT, + "model_context_tokens": MODEL_CONTEXT_TOKENS, + "chunk_token_budget": CHUNK_TOKEN_BUDGET, +} + +_ENV_KEYS: dict[str, str] = { + "backend_mode": "JOURNAL_BACKEND_MODE", + "csharp_sidecar_path": "JOURNAL_CSHARP_SIDECAR_PATH", + "nlp_backend": "JOURNAL_NLP_BACKEND", + "speech_engine": "JOURNAL_SPEECH_ENGINE", + "whisper_model": "JOURNAL_WHISPER_MODEL", + "faster_whisper_device": "JOURNAL_FASTER_WHISPER_DEVICE", + "faster_whisper_compute_type": "JOURNAL_FASTER_WHISPER_COMPUTE_TYPE", + "llama_cpp_url": "JOURNAL_LLAMA_CPP_URL", + "llama_cpp_model": "JOURNAL_LLAMA_CPP_MODEL", + "llama_cpp_timeout": "JOURNAL_LLAMA_CPP_TIMEOUT", + "embedding_api_url": "JOURNAL_EMBEDDING_API_URL", + "embedding_model_name": "JOURNAL_EMBEDDING_MODEL_NAME", + "cloudai_api_url": "JOURNAL_CLOUDAI_API_URL", + "cloudai_api_key": "JOURNAL_CLOUDAI_API_KEY", + "cloudai_timeout": "JOURNAL_CLOUDAI_TIMEOUT", + "model_context_tokens": "JOURNAL_MODEL_CONTEXT_TOKENS", + "chunk_token_budget": "JOURNAL_CHUNK_TOKEN_BUDGET", +} + + +def _ensure_defaults() -> None: + for key, value in _DEFAULT_SETTINGS.items(): + app.storage.user.setdefault(key, value) + + +def _apply_env_from_storage() -> None: + for key, env_key in _ENV_KEYS.items(): + value = app.storage.user.get(key, _DEFAULT_SETTINGS.get(key, "")) + os.environ[env_key] = str(value).strip() + + +def _on_text_change(user_key: str, env_key: str, label: str, restart_required: bool = False): + def handler(e: Any) -> None: + value = str(e.value or "").strip() + app.storage.user[user_key] = value + os.environ[env_key] = value + suffix = " Restart server to apply." if restart_required else "" + ui.notify(f"{label} updated.{suffix}") + + return handler + + +def _on_int_change(user_key: str, env_key: str, label: str, restart_required: bool = False): + def handler(e: Any) -> None: + try: + value = int(float(cast(Any, e.value))) + except (TypeError, ValueError): + ui.notify(f"{label} must be a valid number.", type="negative") + return + + if value <= 0: + ui.notify(f"{label} must be greater than 0.", type="negative") + return + + app.storage.user[user_key] = value + os.environ[env_key] = str(value) + suffix = " Restart server to apply." if restart_required else "" + ui.notify(f"{label} updated.{suffix}") + + return handler def settings_dialog(): - """Creates a dialog for application settings.""" + """Creates a dialog for runtime/application settings.""" + + _ensure_defaults() + _apply_env_from_storage() + async def request_server_action(action: str) -> None: try: SERVER_CONTROL_FILE.parent.mkdir(parents=True, exist_ok=True) @@ -24,7 +127,6 @@ def settings_dialog(): else: ui.notify("Server shutdown requested...", type="warning") - # Give the notification a moment to flush before shutdown. await asyncio.sleep(0.2) app.shutdown() @@ -34,56 +136,238 @@ def settings_dialog(): async def shutdown_server() -> None: await request_server_action("shutdown") - with ui.dialog() as dialog, ui.card().classes("w-80"): + with ui.dialog() as dialog, ui.card().classes("w-full max-w-[38rem] max-h-[85vh] overflow-y-auto"): _ = ui.label("Settings").classes("text-xl font-bold") + _ = ui.markdown( + "Most values apply immediately. " + "Some runtime/module values are marked with restart hints." + ).classes("text-xs text-gray-500") - _ = ui.label("Speech to Text").classes("text-lg font-medium mt-4") - - if "speech_engine" in app.storage.user: - initial_engine_value = cast(str, app.storage.user["speech_engine"]) - else: - initial_engine_value = SPEECH_RECOGNITION_ENGINE - + _ = ui.separator().classes("my-2") + _ = ui.label("Backend").classes("text-lg font-medium") _ = ( ui.select( - ["whisper", "google", "sphinx"], - label="Recognition Engine", - value=initial_engine_value, - on_change=lambda e: ui.notify( - f"Engine set to {cast(str, e.value)}. Takes effect on next recording." + ["csharp-hybrid", "python"], + label="Backend Mode", + value=cast(str, app.storage.user["backend_mode"]), + on_change=_on_text_change( + "backend_mode", + "JOURNAL_BACKEND_MODE", + "Backend mode", + restart_required=True, + ), + ) + .bind_value(app.storage.user, "backend_mode") + .classes("w-full") + ) + _ = ( + ui.input( + label="C# Sidecar Path (optional override)", + value=cast(str, app.storage.user["csharp_sidecar_path"]), + on_change=_on_text_change( + "csharp_sidecar_path", + "JOURNAL_CSHARP_SIDECAR_PATH", + "C# sidecar path", + restart_required=True, + ), + ) + .bind_value(app.storage.user, "csharp_sidecar_path") + .classes("w-full") + ) + + _ = ui.separator().classes("my-2") + _ = ui.label("AI: Local Model Endpoints").classes("text-lg font-medium") + _ = ( + ui.input( + label="Llama.cpp URL", + value=cast(str, app.storage.user["llama_cpp_url"]), + on_change=_on_text_change( + "llama_cpp_url", "JOURNAL_LLAMA_CPP_URL", "Llama.cpp URL" + ), + ) + .bind_value(app.storage.user, "llama_cpp_url") + .classes("w-full") + ) + _ = ( + ui.input( + label="Llama.cpp Model", + value=cast(str, app.storage.user["llama_cpp_model"]), + on_change=_on_text_change( + "llama_cpp_model", "JOURNAL_LLAMA_CPP_MODEL", "Llama.cpp model" + ), + ) + .bind_value(app.storage.user, "llama_cpp_model") + .classes("w-full") + ) + _ = ( + ui.number( + label="Llama.cpp Timeout (ms)", + value=float(cast(str, app.storage.user["llama_cpp_timeout"])), + on_change=_on_int_change( + "llama_cpp_timeout", "JOURNAL_LLAMA_CPP_TIMEOUT", "Llama timeout" + ), + ) + .bind_value(app.storage.user, "llama_cpp_timeout") + .classes("w-full") + ) + _ = ( + ui.input( + label="Embedding API URL", + value=cast(str, app.storage.user["embedding_api_url"]), + on_change=_on_text_change( + "embedding_api_url", "JOURNAL_EMBEDDING_API_URL", "Embedding API URL" + ), + ) + .bind_value(app.storage.user, "embedding_api_url") + .classes("w-full") + ) + _ = ( + ui.input( + label="Embedding Model Name", + value=cast(str, app.storage.user["embedding_model_name"]), + on_change=_on_text_change( + "embedding_model_name", + "JOURNAL_EMBEDDING_MODEL_NAME", + "Embedding model name", + ), + ) + .bind_value(app.storage.user, "embedding_model_name") + .classes("w-full") + ) + _ = ( + ui.number( + label="Model Context Tokens", + value=float(cast(str, app.storage.user["model_context_tokens"])), + on_change=_on_int_change( + "model_context_tokens", + "JOURNAL_MODEL_CONTEXT_TOKENS", + "Model context tokens", + ), + ) + .bind_value(app.storage.user, "model_context_tokens") + .classes("w-full") + ) + _ = ( + ui.number( + label="Chunk Token Budget", + value=float(cast(str, app.storage.user["chunk_token_budget"])), + on_change=_on_int_change( + "chunk_token_budget", + "JOURNAL_CHUNK_TOKEN_BUDGET", + "Chunk token budget", + ), + ) + .bind_value(app.storage.user, "chunk_token_budget") + .classes("w-full") + ) + + _ = ui.separator().classes("my-2") + _ = ui.label("AI: Cloud Endpoint").classes("text-lg font-medium") + _ = ( + ui.input( + label="Cloud AI URL", + value=cast(str, app.storage.user["cloudai_api_url"]), + on_change=_on_text_change( + "cloudai_api_url", "JOURNAL_CLOUDAI_API_URL", "Cloud AI URL" + ), + ) + .bind_value(app.storage.user, "cloudai_api_url") + .classes("w-full") + ) + _ = ( + ui.input( + label="Cloud AI API Key", + password=True, + password_toggle_button=True, + value=cast(str, app.storage.user["cloudai_api_key"]), + on_change=_on_text_change( + "cloudai_api_key", "JOURNAL_CLOUDAI_API_KEY", "Cloud AI API key" + ), + ) + .bind_value(app.storage.user, "cloudai_api_key") + .classes("w-full") + ) + _ = ( + ui.number( + label="Cloud AI Timeout (s)", + value=float(cast(str, app.storage.user["cloudai_timeout"])), + on_change=_on_int_change( + "cloudai_timeout", "JOURNAL_CLOUDAI_TIMEOUT", "Cloud AI timeout" + ), + ) + .bind_value(app.storage.user, "cloudai_timeout") + .classes("w-full") + ) + + _ = ui.separator().classes("my-2") + _ = ui.label("NLP + Speech").classes("text-lg font-medium") + _ = ( + ui.select( + ["auto", "spacy", "fallback"], + label="NLP Backend", + value=cast(str, app.storage.user["nlp_backend"]), + on_change=_on_text_change( + "nlp_backend", + "JOURNAL_NLP_BACKEND", + "NLP backend", + restart_required=True, + ), + ) + .bind_value(app.storage.user, "nlp_backend") + .classes("w-full") + ) + _ = ( + ui.select( + ["faster-whisper", "whisper", "google", "sphinx"], + label="Speech Recognition Engine", + value=cast(str, app.storage.user["speech_engine"]), + on_change=_on_text_change( + "speech_engine", "JOURNAL_SPEECH_ENGINE", "Speech engine" ), ) .bind_value(app.storage.user, "speech_engine") .classes("w-full") ) - _ = ui.markdown( - "`whisper` is local & private (recommended).\n`google` is online.\n`sphinx` is offline & fast." - ).classes("text-xs text-gray-500") - - # Add whisper model select - _ = ui.separator().classes("my-2") - _ = ui.label("Whisper Model Size").classes("font-medium") - - if "whisper_model" in app.storage.user: - initial_whisper_model = cast(str, app.storage.user["whisper_model"]) - else: - initial_whisper_model = WHISPER_MODEL_SIZE - _ = ( ui.select( ["tiny", "base", "small", "medium", "large"], - label="Accuracy vs. Speed", - value=initial_whisper_model, - on_change=lambda e: ui.notify( - f"Whisper model set to {cast(str, e.value)}. Takes effect on next recording." + label="Whisper Model Size", + value=cast(str, app.storage.user["whisper_model"]), + on_change=_on_text_change( + "whisper_model", "JOURNAL_WHISPER_MODEL", "Whisper model" ), ) .bind_value(app.storage.user, "whisper_model") .classes("w-full") ) - _ = ui.markdown( - "`base` is a good balance. Larger models are more accurate but slower." - ).classes("text-xs text-gray-500") + _ = ( + ui.select( + ["auto", "cpu", "cuda"], + label="faster-whisper Device", + value=cast(str, app.storage.user["faster_whisper_device"]), + on_change=_on_text_change( + "faster_whisper_device", + "JOURNAL_FASTER_WHISPER_DEVICE", + "faster-whisper device", + ), + ) + .bind_value(app.storage.user, "faster_whisper_device") + .classes("w-full") + ) + _ = ( + ui.select( + ["float32", "int8", "default", "auto", "float16", "int8_float32", "int8_float16"], + label="faster-whisper Compute Type", + value=cast(str, app.storage.user["faster_whisper_compute_type"]), + on_change=_on_text_change( + "faster_whisper_compute_type", + "JOURNAL_FASTER_WHISPER_COMPUTE_TYPE", + "faster-whisper compute type", + ), + ) + .bind_value(app.storage.user, "faster_whisper_compute_type") + .classes("w-full") + ) _ = ui.separator().classes("my-2") _ = ui.label("Server Controls").classes("font-medium") diff --git a/journal/ui/components/speech.py b/journal/ui/components/speech.py index c57fd03..37c7814 100644 --- a/journal/ui/components/speech.py +++ b/journal/ui/components/speech.py @@ -6,7 +6,12 @@ import queue # Add project root to sys.path to allow for absolute imports sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) -from journal.core.config import SPEECH_RECOGNITION_ENGINE, WHISPER_MODEL_SIZE +from journal.core.config import ( + SPEECH_RECOGNITION_ENGINE, + WHISPER_MODEL_SIZE, + FASTER_WHISPER_DEVICE, + FASTER_WHISPER_COMPUTE_TYPE, +) from journal.core.speech import start_background_listening, stop_background_listening @@ -55,10 +60,27 @@ def speech_to_text(on_result: Callable[[str], None]) -> None: else: whisper_model = WHISPER_MODEL_SIZE + if "faster_whisper_device" in app.storage.user: + faster_whisper_device = cast(str, app.storage.user["faster_whisper_device"]) + else: + faster_whisper_device = FASTER_WHISPER_DEVICE + + if "faster_whisper_compute_type" in app.storage.user: + faster_whisper_compute_type = cast( + str, app.storage.user["faster_whisper_compute_type"] + ) + else: + faster_whisper_compute_type = FASTER_WHISPER_COMPUTE_TYPE + if queue_timer is not None: queue_timer.active = True await run.io_bound( - start_background_listening, message_queue, engine, whisper_model + start_background_listening, + message_queue, + engine, + whisper_model, + faster_whisper_device, + faster_whisper_compute_type, ) async def stop_listening_task(): diff --git a/journal/ui/main.py b/journal/ui/main.py index 90ddc6f..a8d4341 100644 --- a/journal/ui/main.py +++ b/journal/ui/main.py @@ -1,12 +1,17 @@ import asyncio import sys import warnings +import os +import time -if sys.platform == "win32" and hasattr(asyncio, "WindowsSelectorEventLoopPolicy"): +if sys.platform == "win32": # Avoid noisy Proactor transport resets on long-running Windows sessions. + # Python 3.14+ marks this policy deprecated, so probe/set under warning suppression. with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + selector_policy = getattr(asyncio, "WindowsSelectorEventLoopPolicy", None) + if selector_policy is not None: + asyncio.set_event_loop_policy(selector_policy()) from nicegui import ui, app, run from typing import cast @@ -30,22 +35,153 @@ from journal.core.storage import ( save_current_month_vault, initialize_vault, clear_data_directory, + list_journal_files as list_journal_files_backend, + load_entry_content, ) from journal.core.config import ( DATA_DIR, VAULT_DIR, + BACKEND_MODE, + CSHARP_SIDECAR_PATH, + NLP_BACKEND, SPEECH_RECOGNITION_ENGINE, WHISPER_MODEL_SIZE, + FASTER_WHISPER_DEVICE, + FASTER_WHISPER_COMPUTE_TYPE, + LLAMA_CPP_URL, + LLAMA_CPP_MODEL, + LLAMA_CPP_TIMEOUT, + EMBEDDING_API_URL, + EMBEDDING_MODEL_NAME, + CLOUDAI_API_URL, + CLOUDAI_API_KEY, + CLOUDAI_TIMEOUT, + MODEL_CONTEXT_TOKENS, + CHUNK_TOKEN_BUDGET, ) -from journal.core.parser import parse_journal_file -from journal.ai.analysis import summarize_entry, summarize_all_entries +from journal.core.models import JournalEntry +from journal.ai.bridge import summarize_entry, summarize_all_entries from journal.ai.chat import get_cloud_ai_response from journal.ui.components.speech import speech_to_text from journal.ui.components.calendar import calendar_view -from journal.ui.components.editor import rich_text_editor +from journal.ui.components.editor import markdown_editor from journal.ui.components.settings import settings_dialog ui.dark_mode = True +_ = ui.add_head_html( + """ +<style> +.journal-drawer { + width: clamp(18rem, 30vw, 22rem) !important; + max-width: 94vw; +} +.journal-sidebar { + height: 100%; + overflow-y: auto; + overflow-x: hidden; + padding-bottom: 1rem; +} +.journal-main-content { + height: calc(100vh - 64px); + overflow: hidden; +} +.journal-root { + width: 100%; + height: 100%; + min-height: 0; + overflow: hidden; +} +.journal-header-title { + font-size: 1.55rem; + font-weight: 700; + max-width: 60vw; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} +.journal-tab-panels, +.journal-tab-panel { + min-height: 0; +} +.journal-tab-panels { + overflow: hidden; +} +.journal-tab-panel { + overflow: hidden; +} +.journal-action-row { + flex-wrap: wrap; +} +.journal-action-row .q-btn { + flex: 1 1 10rem; +} +.journal-calendar-card { + width: 100%; + max-width: 100%; + overflow-x: auto; + overflow-y: hidden; +} +.journal-calendar { + max-width: 100%; +} +.journal-sidebar .q-date { + min-width: 17.5rem; + max-width: 100%; +} +.journal-editor-wrapper { + flex: 1 1 auto; + min-height: 0; + overflow: hidden; +} +.journal-editor { + height: 100%; + min-height: 0; +} +.journal-edit-actions { + flex: 0 0 auto; + border-top: 1px solid rgba(255, 255, 255, 0.08); +} +.journal-markdown-editor .q-field__native, +.journal-markdown-editor .q-field__input, +.journal-markdown-editor textarea { + color: #e8f1ff !important; + caret-color: #e8f1ff !important; + font-size: 1.04rem; + line-height: 1.7; +} +.journal-markdown-editor .q-field__native::placeholder, +.journal-markdown-editor .q-field__input::placeholder, +.journal-markdown-editor textarea::placeholder { + color: #9ab0d6 !important; + opacity: 1; +} +.journal-markdown-editor .q-field__label { + color: #b9c8e6 !important; +} +.journal-markdown-editor textarea, +.journal-large-textarea textarea { + min-height: 58vh !important; + height: 58vh !important; + resize: vertical !important; +} +@media (max-width: 768px) { + .journal-main-content { + height: calc(100vh - 56px); + } + .journal-header-title { + font-size: 1.1rem; + max-width: 56vw; + } + .journal-markdown-editor textarea, + .journal-large-textarea textarea { + min-height: 42vh !important; + height: 42vh !important; + } +} +</style> +""", + shared=True, +) # Global variables @@ -55,9 +191,9 @@ main_tab: ui.tabs | None = None edit_tab: ui.tab | None = None ai_tab: ui.tab | None = None new_tab: ui.tab | None = None -entry_content_box: ui.editor | None = None +entry_content_box: ui.textarea | None = None analysis_box: ui.textarea | None = None -new_entry_box: ui.editor | None = None +new_entry_box: ui.textarea | None = None file_map: dict[str, str] = {} vault_password: str | None = None # Global to store the password for the shutdown hook sidebar_content: ui.column | None = None @@ -65,6 +201,56 @@ content_container: ui.column | None = None # Must be created inside page/layout scope for NiceGUI. all_analysis_dialog: ui.dialog | None = None vault_load_lock = asyncio.Lock() +_sidebar_retry_scheduled = False + +_USER_SETTINGS_DEFAULTS: dict[str, str] = { + "backend_mode": BACKEND_MODE, + "csharp_sidecar_path": CSHARP_SIDECAR_PATH, + "nlp_backend": NLP_BACKEND, + "speech_engine": SPEECH_RECOGNITION_ENGINE, + "whisper_model": WHISPER_MODEL_SIZE, + "faster_whisper_device": FASTER_WHISPER_DEVICE, + "faster_whisper_compute_type": FASTER_WHISPER_COMPUTE_TYPE, + "llama_cpp_url": LLAMA_CPP_URL, + "llama_cpp_model": LLAMA_CPP_MODEL, + "llama_cpp_timeout": str(LLAMA_CPP_TIMEOUT), + "embedding_api_url": EMBEDDING_API_URL, + "embedding_model_name": EMBEDDING_MODEL_NAME, + "cloudai_api_url": CLOUDAI_API_URL, + "cloudai_api_key": CLOUDAI_API_KEY, + "cloudai_timeout": str(CLOUDAI_TIMEOUT), + "model_context_tokens": str(MODEL_CONTEXT_TOKENS), + "chunk_token_budget": str(CHUNK_TOKEN_BUDGET), +} + +_USER_SETTINGS_ENV_KEYS: dict[str, str] = { + "backend_mode": "JOURNAL_BACKEND_MODE", + "csharp_sidecar_path": "JOURNAL_CSHARP_SIDECAR_PATH", + "nlp_backend": "JOURNAL_NLP_BACKEND", + "speech_engine": "JOURNAL_SPEECH_ENGINE", + "whisper_model": "JOURNAL_WHISPER_MODEL", + "faster_whisper_device": "JOURNAL_FASTER_WHISPER_DEVICE", + "faster_whisper_compute_type": "JOURNAL_FASTER_WHISPER_COMPUTE_TYPE", + "llama_cpp_url": "JOURNAL_LLAMA_CPP_URL", + "llama_cpp_model": "JOURNAL_LLAMA_CPP_MODEL", + "llama_cpp_timeout": "JOURNAL_LLAMA_CPP_TIMEOUT", + "embedding_api_url": "JOURNAL_EMBEDDING_API_URL", + "embedding_model_name": "JOURNAL_EMBEDDING_MODEL_NAME", + "cloudai_api_url": "JOURNAL_CLOUDAI_API_URL", + "cloudai_api_key": "JOURNAL_CLOUDAI_API_KEY", + "cloudai_timeout": "JOURNAL_CLOUDAI_TIMEOUT", + "model_context_tokens": "JOURNAL_MODEL_CONTEXT_TOKENS", + "chunk_token_budget": "JOURNAL_CHUNK_TOKEN_BUDGET", +} + + +def _initialize_runtime_user_settings() -> None: + for key, value in _USER_SETTINGS_DEFAULTS.items(): + app.storage.user.setdefault(key, value) + + for key, env_key in _USER_SETTINGS_ENV_KEYS.items(): + value = app.storage.user.get(key, _USER_SETTINGS_DEFAULTS.get(key, "")) + os.environ[env_key] = str(value).strip() def _is_client_active(client: object | None) -> bool: @@ -76,11 +262,63 @@ def _is_deleted_client_error(error: RuntimeError) -> bool: return "client this element belongs to has been deleted" in text or "parent slot of the element has been deleted" in text +def _render_authenticated_layout() -> bool: + global content_container + if content_container is None: + print("Authenticated layout skipped: content container is not ready.") + return False + try: + content_container.clear() + with content_container: + journal_ui_layout() + if not update_sidebar(): + _schedule_sidebar_retry("layout render") + return True + except RuntimeError as error: + if _is_deleted_client_error(error): + print("Skipping layout update because the client was deleted.") + _schedule_sidebar_retry("deleted layout client") + return False + raise + + +def _has_runtime_auth() -> bool: + return cast(bool, app.storage.user.get("authenticated", False)) and bool(vault_password) + + +def _schedule_sidebar_retry(reason: str, delay_seconds: float = 0.45) -> None: + global _sidebar_retry_scheduled + if _sidebar_retry_scheduled: + return + + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + + _sidebar_retry_scheduled = True + + async def _retry() -> None: + global _sidebar_retry_scheduled + try: + await asyncio.sleep(delay_seconds) + if not _has_runtime_auth(): + return + if update_sidebar(): + return + await asyncio.sleep(0.9) + update_sidebar() + finally: + _sidebar_retry_scheduled = False + + print(f"Scheduling sidebar retry ({reason}).") + loop.create_task(_retry()) + + def list_journal_files(): - # List files directly from the DATA_DIR (which is the decrypted workspace) - files = sorted(DATA_DIR.glob("*.md")) - print(f"list_journal_files found: {[f.name for f in files]}") - return [(f.name, str(f)) for f in files] + files = list_journal_files_backend() + print(f"list_journal_files found: {[f[0] for f in files]}") + return files def save_entry(content: str, mode: str, password: str | None) -> str: @@ -93,26 +331,25 @@ def save_entry(content: str, mode: str, password: str | None) -> str: def load_entry(file_path: str) -> str: try: - entry = parse_journal_file(file_path) - return entry.raw_content + return load_entry_content(file_path) except Exception as e: return f"Error loading file: {e}" def save_existing_entry(file_path: str, content: str, password: str | None) -> str: try: - # Overwriting is the correct behavior when editing a full, existing entry. - save_entry_content(content, file_path=Path(file_path), mode="Daily") + # Existing-entry editing should persist exactly what the user sees in editor. + save_entry_content(content, file_path=Path(file_path), mode="Overwrite") if password: save_current_month_vault(password) - return f"Appended changes to {file_path}" + return f"Saved changes to {Path(file_path).name}" except Exception as e: return f"Error saving file: {e}" def analyze_entry(file_path: str) -> str: try: - entry = parse_journal_file(file_path) + entry = JournalEntry(date=Path(file_path).stem, raw_content=load_entry_content(file_path)) analysis = summarize_entry(entry) return analysis except Exception as e: @@ -121,17 +358,20 @@ def analyze_entry(file_path: str) -> str: def analyze_all_entries() -> str: try: - journal_files = sorted(DATA_DIR.glob("*.md")) + journal_files = list_journal_files() if not journal_files: return "No journal files found." - entries = [parse_journal_file(str(f)) for f in journal_files] + entries = [ + JournalEntry(date=Path(path).stem, raw_content=load_entry_content(path)) + for _, path in journal_files + ] analysis = summarize_all_entries(entries) return analysis except Exception as e: return f"Error analyzing all entries: {e}" -def update_sidebar(): +def update_sidebar() -> bool: global \ file_map, \ drawer, \ @@ -141,88 +381,111 @@ def update_sidebar(): analysis_box, \ sidebar_content print("update_sidebar called.") - if sidebar_content: - try: - sidebar_content.clear() - except RuntimeError as error: - if _is_deleted_client_error(error): - print("Skipping sidebar update because the client was deleted.") - return - raise - try: - with sidebar_content: - _ = ui.label("📅 Calendar").classes("text-md font-bold mb-4") + if not sidebar_content: + print("Sidebar content is not ready yet; retry needed.") + return False - def on_date_select(date: str): - global selected_file_name - selected_file_name = f"{date}.md" - if main_tab is not None and edit_tab is not None: - main_tab.set_value(edit_tab) + try: + sidebar_content.clear() + except RuntimeError as error: + if _is_deleted_client_error(error): + print("Skipping sidebar update because the client was deleted.") + return False + raise + try: + with sidebar_content: + _ = ui.label("📅 Calendar").classes("text-md font-bold mb-4") + + def on_date_select(date: str | None): + global selected_file_name + if not date: + return + + selected_file_name = f"{date}.md" + if main_tab is not None and edit_tab is not None: + main_tab.set_value(edit_tab) + selected_path = file_map.get(selected_file_name) + if selected_path is None: if entry_content_box is not None: - entry_content_box.set_value( - load_entry(file_map[selected_file_name]) - ) + entry_content_box.set_value("") if analysis_box is not None: analysis_box.set_value("") - if drawer: - drawer.set_value(False) - - calendar_view(on_select=on_date_select) - _ = ui.separator().classes("my-4") - _ = ui.label("📁 Journal Files").classes("text-lg font-bold mb-4") - - files = list_journal_files() - file_names = [f[0] for f in files] - file_map = dict(files) - print(f"file_map in update_sidebar: {file_map}") - - def on_file_select(fname: str): - global selected_file_name - selected_file_name = fname - if main_tab is not None and edit_tab is not None: - main_tab.set_value(edit_tab) - if entry_content_box is not None: - entry_content_box.set_value(load_entry(file_map[fname])) - if analysis_box is not None: - analysis_box.set_value("") - if drawer: - drawer.set_value(False) - - for fname in file_names: - _ = ( - ui.button(fname, on_click=partial(on_file_select, fname)) - .classes("w-full justify-start mb-1") - .props("flat") + ui.notify( + f"No entry exists for {date}. Use New Entry to create one.", + type="info", ) - - _ = ui.separator().classes("my-4") - - def open_new_tab(): - if main_tab is not None and new_tab is not None: - main_tab.set_value(new_tab) if drawer: drawer.set_value(False) + return + if entry_content_box is not None: + entry_content_box.set_value(load_entry(selected_path)) + if analysis_box is not None: + analysis_box.set_value("") + if drawer: + drawer.set_value(False) + calendar_view(on_select=on_date_select) + _ = ui.separator().classes("my-4") + _ = ui.label("📁 Journal Files").classes("text-lg font-bold mb-4") + + files = list_journal_files() + file_names = [f[0] for f in files] + file_map = dict(files) + print(f"file_map in update_sidebar: {file_map}") + + def on_file_select(fname: str): + global selected_file_name + selected_file_name = fname + selected_path = file_map.get(fname) + if selected_path is None: + ui.notify("Selected file is no longer available.", type="warning") + return + if main_tab is not None and edit_tab is not None: + main_tab.set_value(edit_tab) + if entry_content_box is not None: + entry_content_box.set_value(load_entry(selected_path)) + if analysis_box is not None: + analysis_box.set_value("") + if drawer: + drawer.set_value(False) + + for fname in file_names: _ = ( - ui.button("New Entry", on_click=open_new_tab) - .props("color=primary") - .classes("w-full mb-2") + ui.button(fname, on_click=partial(on_file_select, fname)) + .classes("w-full justify-start mb-1") + .props("flat") ) - def open_all_analysis_dialog(): - if all_analysis_dialog is not None: - all_analysis_dialog.open() + _ = ui.separator().classes("my-4") - _ = ( - ui.button("Analyze All Entries", on_click=open_all_analysis_dialog) - .props("color=accent") - .classes("w-full") - ) - except RuntimeError as error: - if _is_deleted_client_error(error): - print("Sidebar update aborted because client elements were deleted.") - return - raise + def open_new_tab(): + if main_tab is not None and new_tab is not None: + main_tab.set_value(new_tab) + if drawer: + drawer.set_value(False) + + _ = ( + ui.button("New Entry", on_click=open_new_tab) + .props("color=primary") + .classes("w-full mb-2") + ) + + def open_all_analysis_dialog(): + if all_analysis_dialog is not None: + all_analysis_dialog.open() + + _ = ( + ui.button("Analyze All Entries", on_click=open_all_analysis_dialog) + .props("color=accent") + .classes("w-full") + ) + except RuntimeError as error: + if _is_deleted_client_error(error): + print("Sidebar update aborted because client elements were deleted.") + return False + raise + + return True # --- Main UI Layout Function (Inner Content) --- @@ -237,14 +500,14 @@ def journal_ui_layout(): new_entry_box # --- Main Content - Full Screen --- - with ui.element("div").style( - "width: 100vw; height: calc(100vh - 64px); overflow: hidden; padding: 0; margin: 0;" + with ui.element("div").classes("journal-root").style( + "padding: 0; margin: 0;" ): with ui.element("div").style( - "width: 100%; height: 100%; padding: 16px; box-sizing: border-box; display: flex; flex-direction: column;" + "width: 100%; height: 100%; padding: clamp(10px, 2vw, 16px); box-sizing: border-box; display: flex; flex-direction: column; min-height: 0;" ): # Tabs - with ui.tabs().classes("w-full mb-4") as main_tab_instance: + with ui.tabs().props("outside-arrows mobile-arrows inline-label").classes("w-full mb-2") as main_tab_instance: global main_tab main_tab = main_tab_instance edit_tab = ui.tab("Edit Entry") @@ -255,16 +518,81 @@ def journal_ui_layout(): # Tab Panels - Full Height with ui.tab_panels(main_tab, value=edit_tab).classes( - "w-full flex-grow bg-gray-800" + "w-full flex-grow bg-gray-800 journal-tab-panels" ): # --- Edit Entry Tab --- with ui.tab_panel(edit_tab).style( "flex: 1; display: flex; flex-direction: column; padding: 0;" ): - entry_content_box = rich_text_editor().classes("w-full h-full") + with ui.column().classes("journal-editor-wrapper w-full"): + entry_content_box = markdown_editor().classes("w-full journal-editor") + with ui.row().classes("gap-2 p-3 bg-gray-900 journal-action-row journal-edit-actions").style("flex-shrink: 0;"): + + async def save_selected_wrapper(): + client = ui.context.client + if not selected_file_name or selected_file_name not in file_map: + ui.notify("No entry selected to save.", type="warning") + return + if entry_content_box is None: + ui.notify("Editor is not ready.", type="warning") + return + + content = cast(str, entry_content_box.value or "") + password = vault_password + if not content.strip(): + ui.notify("Entry content is empty.", type="warning") + return + + try: + msg = await run.io_bound( + save_existing_entry, + file_map[selected_file_name], + content, + password, + ) + except Exception as error: + if _is_client_active(client): + ui.notify(f"Save failed: {error}", type="negative") + return + + if not _is_client_active(client): + return + + if msg.startswith("Error"): + ui.notify(msg, type="negative") + else: + ui.notify(msg, type="positive") + if not update_sidebar(): + _schedule_sidebar_retry("existing entry save") + + async def reload_selected_wrapper(): + client = ui.context.client + if not selected_file_name or selected_file_name not in file_map: + ui.notify("No entry selected to reload.", type="warning") + return + if entry_content_box is None: + ui.notify("Editor is not ready.", type="warning") + return + try: + content = await run.io_bound(load_entry, file_map[selected_file_name]) + except Exception as error: + if _is_client_active(client): + ui.notify(f"Reload failed: {error}", type="negative") + return + if not _is_client_active(client): + return + entry_content_box.set_value(content) + ui.notify("Entry reloaded from disk.", type="info") + + _ = ui.button("Save Changes", on_click=save_selected_wrapper).props( + "color=primary" + ) + _ = ui.button("Reload From Disk", on_click=reload_selected_wrapper).props( + "color=secondary flat" + ) # --- AI Analysis Tab --- - with ui.tab_panel(ai_tab).style( + with ui.tab_panel(ai_tab).classes("journal-tab-panel").style( "flex: 1; display: flex; flex-direction: column; padding: 16px;" ): _ = ( @@ -279,10 +607,11 @@ def journal_ui_layout(): placeholder='Click "Analyze Entry" to get AI analysis of the selected journal entry...', ) .props("readonly outlined") + .classes("journal-large-textarea") .style("flex: 1; width: 100%; min-height: 0;") ) - with ui.row().classes("gap-4 mt-4").style("flex-shrink: 0;"): + with ui.row().classes("gap-4 mt-4 journal-action-row").style("flex-shrink: 0;"): async def analyze_selected_wrapper(): client = ui.context.client @@ -304,10 +633,10 @@ def journal_ui_layout(): _ = ui.button( "Analyze Entry", on_click=analyze_selected_wrapper - ).props("color=primary size=lg") + ).props("color=primary").style("min-width: 10rem;") # --- AI Chat Tab --- - with ui.tab_panel(chat_tab).style( + with ui.tab_panel(chat_tab).classes("journal-tab-panel").style( "flex: 1; display: flex; flex-direction: column; padding: 16px;" ): _ = ( @@ -322,6 +651,7 @@ def journal_ui_layout(): placeholder="Chat with the AI...", ) .props("readonly outlined") + .classes("journal-large-textarea") .style("flex: 1; width: 100%; min-height: 0;") ) @@ -348,7 +678,7 @@ def journal_ui_layout(): _ = chat_input.on("keydown.enter", send_chat_message) # --- New Entry Tab --- - with ui.tab_panel(new_tab).style( + with ui.tab_panel(new_tab).classes("journal-tab-panel").style( "flex: 1; display: flex; flex-direction: column; padding: 16px;" ): _ = ( @@ -363,11 +693,12 @@ def journal_ui_layout(): label="Entry Type", value="Daily", ) - .classes("mb-4 w-64") + .classes("mb-4 w-full max-w-sm") .style("flex-shrink: 0;") ) - new_entry_box = rich_text_editor() + with ui.column().classes("w-full flex-grow min-h-0"): + new_entry_box = markdown_editor().classes("w-full h-full") def update_new_entry_box_on_mode_change(): if new_entry_box is None: @@ -382,10 +713,11 @@ def journal_ui_layout(): and mode_val in ["Daily", "Deep Recovery", "Deep Entry"] and today_file_path.exists() ): - new_entry_box.set_value( - today_file_path.read_text(encoding="utf-8") - ) - ui.notify("Loaded today's existing entry.", type="info") + try: + new_entry_box.set_value(load_entry_content(str(today_file_path))) + ui.notify("Loaded today's existing entry.", type="info") + except Exception as error: + ui.notify(f"Error loading today's entry: {error}", type="negative") else: new_entry_box.set_value("") @@ -393,7 +725,7 @@ def journal_ui_layout(): "update:model-value", update_new_entry_box_on_mode_change ) - with ui.row().classes("gap-4 mt-4").style("flex-shrink: 0;"): + with ui.row().classes("gap-4 mt-4 journal-action-row").style("flex-shrink: 0;"): def fill_template(): mode_val = cast(str | None, mode.value) @@ -414,38 +746,42 @@ def journal_ui_layout(): new_entry_box.value if new_entry_box else None, ) mode_val = cast(str, mode.value) - password = cast( - str | None, getattr(app.storage.user, "vault_password", None) - ) + password = vault_password if new_entry_value and new_entry_value.strip(): - # For new entries, always append to today's file - msg = await run.io_bound( - save_entry, new_entry_value, mode_val, password - ) + try: + # For new entries, always append to today's file + msg = await run.io_bound( + save_entry, new_entry_value, mode_val, password + ) + except Exception as error: + if _is_client_active(client): + ui.notify(f"Save failed: {error}", type="negative") + return if not _is_client_active(client): return ui.notify(msg, type="positive") if new_entry_box: new_entry_box.set_value("") - update_sidebar() # Refresh the sidebar + if not update_sidebar(): + _schedule_sidebar_retry("new entry save") else: ui.notify("Entry content is empty", type="warning") _ = ui.button("Create Template", on_click=fill_template).props( - "color=secondary size=lg" - ) + "color=secondary" + ).style("min-width: 10rem;") _ = ui.button("Save Entry", on_click=save_new_wrapper).props( - "color=primary size=lg" - ) + "color=primary" + ).style("min-width: 10rem;") # --- Speech to Text Tab --- - with ui.tab_panel(speech_tab).style( + with ui.tab_panel(speech_tab).classes("journal-tab-panel").style( "flex: 1; display: flex; flex-direction: column; padding: 16px;" ): # Add a text area to display the live transcription speech_output_box = ( ui.textarea(label="Live Transcription") .props("outlined") - .classes("w-full flex-grow") + .classes("w-full flex-grow journal-large-textarea") ) def on_speech_result(text: str): @@ -454,7 +790,7 @@ def journal_ui_layout(): speech_output_box.set_value(current_text + text) # The speech component itself, which now updates the local text area - with ui.row().classes("w-full justify-between mt-4"): + with ui.row().classes("w-full mt-4 journal-action-row"): speech_to_text(on_result=on_speech_result) def append_to_new_entry(): @@ -492,7 +828,7 @@ def journal_ui_layout(): .classes("h-96 mb-4 w-full") ) - with ui.row().classes("gap-4"): + with ui.row().classes("gap-4 journal-action-row"): async def do_analyze_all_wrapper(): client = ui.context.client @@ -519,20 +855,19 @@ async def index_page(): with ( ui.left_drawer(value=False) .props("overlay") - .classes("w-64 bg-gray-900 text-white") as drawer_instance + .classes("journal-drawer bg-gray-900 text-white") as drawer_instance ): drawer = drawer_instance - sidebar_content = ui.column().classes("p-4 gap-2 h-full") + sidebar_content = ui.column().classes("journal-sidebar p-3 gap-2 h-full") # Sidebar content will be updated after vault load with ui.header(elevated=True).classes( - "items-center justify-between px-4 bg-gray-900" + "items-center justify-between px-3 py-2 bg-gray-900" ): - # Add a None check inside the lambda to satisfy the type checker _ = ui.button( icon="menu", on_click=lambda: drawer.toggle() if drawer else None ).props("flat color=white") - _ = ui.label("📓 Project Journal").classes("text-2xl font-bold") + _ = ui.label("📓 Project Journal").classes("journal-header-title") _ = ui.space() # Instantiate and open the settings dialog settings = settings_dialog() @@ -540,11 +875,16 @@ async def index_page(): # Main content area (dynamically populated) with ui.column().classes( - "w-full h-full bg-gray-900 text-white" + "w-full bg-gray-900 text-white journal-main-content" ) as content_container_instance: content_container = content_container_instance - if not getattr(app.storage.user, "authenticated", False): - with ui.card().classes("absolute-center bg-gray-800 text-white"): + if vault_password is None and cast(bool, app.storage.user.get("authenticated", False)): + # Stale browser/session flag from a previous process start; require password again. + app.storage.user["authenticated"] = False + app.storage.user.pop("vault_password", None) + + if not _has_runtime_auth(): + with ui.card().classes("absolute-center bg-gray-800 text-white w-full max-w-md"): _ = ui.label("Welcome to Your Journal").classes("text-2xl font-bold") password_input = ( ui.input( @@ -562,10 +902,9 @@ async def index_page(): ), ) else: - # This branch will be executed if the page reloads and password is already set - # or if we navigate back to '/' after successful login. - journal_ui_layout() - update_sidebar() # Ensure sidebar is populated on reload + _initialize_runtime_user_settings() + if not _render_authenticated_layout(): + _schedule_sidebar_retry("index page auth render") async def process_password(password: str | None): @@ -588,6 +927,8 @@ async def process_password(password: str | None): app.storage.user["auth_in_progress"] = True try: async with vault_load_lock: + if not _is_client_active(client): + return # Check if vault is empty to determine if we're setting a new password is_new_vault = not any(VAULT_DIR.iterdir()) @@ -605,6 +946,8 @@ async def process_password(password: str | None): if _is_client_active(client): ui.notify("Incorrect password.", type="negative") vault_password = None # Reset password if loading fails + app.storage.user["authenticated"] = False + app.storage.user.pop("vault_password", None) return # Store password globally for shutdown hook, and in session for this client @@ -613,33 +956,34 @@ async def process_password(password: str | None): vault_password = password app.storage.user["authenticated"] = True - app.storage.user["vault_password"] = password - # Initialize user settings in session storage - app.storage.user["speech_engine"] = SPEECH_RECOGNITION_ENGINE - app.storage.user["whisper_model"] = WHISPER_MODEL_SIZE - ui.notify("Vault loaded successfully!", type="positive") - - # Clear the password input area and render the main UI - if content_container is not None: - try: - content_container.clear() - with content_container: - journal_ui_layout() - except RuntimeError as error: - if _is_deleted_client_error(error): - print("Skipping layout update because the client was deleted.") - return - raise - update_sidebar() # Populate sidebar after vault is loaded + app.storage.user.pop("vault_password", None) + _initialize_runtime_user_settings() + if _is_client_active(client): + ui.notify("Vault loaded successfully! Loading workspace...", type="positive") + # Deterministically rebuild page state; avoids stale-container login hangs. + await asyncio.sleep(0.05) + if _is_client_active(client): + target = f"/?auth={int(time.time() * 1000)}" + ui.navigate.to(target) + await asyncio.sleep(0.12) + if _is_client_active(client): + _ = ui.run_javascript( + f"if (window.location.pathname + window.location.search !== '{target}') " + + "{ window.location.replace('" + target + "'); }" + ) except ValueError as e: if _is_client_active(client): ui.notify(f"Error: {e}", type="negative") vault_password = None + app.storage.user["authenticated"] = False + app.storage.user.pop("vault_password", None) except Exception as e: if _is_client_active(client): ui.notify(f"Error loading vault: {e}", type="negative") vault_password = None # Reset password if loading fails + app.storage.user["authenticated"] = False + app.storage.user.pop("vault_password", None) finally: try: app.storage.user["auth_in_progress"] = False diff --git a/originalprojectplan.md b/originalprojectplan.md deleted file mode 100644 index 493175f..0000000 --- a/originalprojectplan.md +++ /dev/null @@ -1,290 +0,0 @@ -# 🧠🗂️ JOURNAL SYSTEM OVERVIEW ("Mind Prosthetic Log") - -## 🛍️ PURPOSE - -To help externalize memory, track psychological patterns, log important events, and record internal states in a structured, searchable way—compensating for: - -- Rapid memory loss or forgetting what you were just thinking/saying - -- Difficulty organizing and verbalizing complex emotions - -- Inability to track patterns over time without external structure - -- Need for logs to aid therapy, legal documentation, co-parenting disputes - -- Cross-platform native app (Linux/Windows/macOS) with mobile access via Tailscale + NiceGUI - -- Designed for neurodivergent daily use with structured, low-friction interfaces - -- Allows fragment logging, full journal entry templates, tagging, and search - -- Uses Python where powerful NLP/AI tasks are needed - -- Expandable to Android (ideal) and iOS (via browser or shortcuts) - ---- - -## 🧱 COMPONENTS - -### 1. Templates & Daily Entries - -Multiple modular templates are available, including: - -- Full daily entry (see below) - -- Meltdown logs - -- Shutdown summaries - -- Therapy prep and recap - -- Legal event summaries - -#### Daily Entry Format Example: - -```markdown -📅 Date: YYYY-MM-DD - -## 🧠 Cognitive State - -- [ ] Masking -- [ ] Shutdown -- [ ] Meltdown -- [ ] Freeze -- [ ] Flow state - -- Notes: - -## 🧠 Mental / Emotional Snapshot - -- Internal monologue or silence? -- Thought loops or rumination? -- Anxiety level: (0–10) -- Depression level: (0–10) -- Suicidal ideation: (Y/N, passive/active) -- Emotional state(s): Angry / Tired / Hopeful / Numb / Triggered / Overwhelmed / Energized / etc. -- Notes: - -## ⚡ Memory / Mind Failures - -- Forgot something mid-sentence? -- Lost train of thought? -- Couldn't speak thoughts? -- Time blindness / lost hours? -- Notes: - -## 📜 Events / Triggers - -- Interactions (e.g., with co-parent, child, officials) -- Flashbacks / trauma triggers -- Physical symptoms -- Legal / medical events -- Notes: - -## 💬 Communication / Expression Log - -- Messages I didn’t send -- Things I forgot to say -- Things I said that I didn’t mean -- Verbal conflicts / miscommunication - -## 🧰 Coping / Tools Used - -- Breathing -- Music -- Walking -- Writing -- AI journaling -- Hiding / Isolation -- Notes: - -## 🧠 Reflection - -- What do I wish I’d done differently? -- What patterns am I noticing? -- Is this getting better or worse? -- Notes: -``` - ---- - -### 2. Modular "Insert Blocks" - -Quick journal fragments for when you're too overwhelmed to fill out a full template. - -#### Example block types: - -- `!FLASHBACK:` description of what triggered it - -- `!FORGOT:` mid-thought freeze or sentence drop - -- `!QUOTE:` something I wish I'd said - -- `!TRIGGER:` encounter that caused a somatic or shutdown response - -- `!LOOP:` thought pattern or obsession - -- `!VIOLATION:` emotional harm from another person (e.g., co-parent) - -- `!SOMATIC:` physical response (shaking, tears, tight chest) - -These can be dropped into a daily log or used stand-alone. - ---- - -### 3. Indexing / Metadata System - -To make logs searchable: - -- **Tagging**: `#shutdown`, `#CPTSD`, `#co-parent`, `#legal`, etc. - -- **Timestamps**: `@HH:MM` - -- **Sources**: `> from text convo`, `> from therapy session`, `> from memory`, etc. - -- **Priority markers**: - - - `‼️` = urgent - - - `🔁` = recurring pattern - - - `🧩` = unexplained moment - ---- - -### 4. Use Cases - -This system supports: - -- **Therapy**: structure logs showing memory gaps, trauma patterns, breakdowns - -- **Legal**: document co-parenting issues and harmful behavior neutrally and time-stamped - -- **Internal Growth**: recognize cycles, triggers, and patterns - -- **Compensation**: catch memory failures before they damage communication or safety - ---- - -### 5. Capture Modes (Tools) - -**Currently Available:** - -- **Desktop UI** (NiceGUI) - -- **Mobile browser access** via Tailscale - -- **CLI tools**: `jfrag`, `vault`, `server`, `search` - -- **ChatGPT log syntax** (can copy-paste into assistant) - -- **Encrypted Vaults**: Journals saved as monthly `.vault` files - -- **Automatic data cleanup**: Decrypted data auto-cleared on shutdown - - -- **Voice-to-text input**: (desktop + mobile) - -- **Calendar and rich Markdown**: preview in UI - -- **SQcypher backend**: Enrypted database backend. - ---- - -## 🛍️ Syntax Format - -```markdown -!TYPE @time #tags -Description of the event, thought, or experience. -``` - -### 📦 Example Fragments: - -```markdown -!FLASHBACK @15:20 #CPTSD #shutdown -Smelled her shampoo in the hallway and got hit with a memory of the hospital visit. Heart raced, froze completely. - -!FORGOT @16:45 #aphantasia #mindblank -Mid-sentence memory drop while trying to explain Phaylynn’s school schedule. Just froze and couldn’t finish. Felt ashamed. - -!TRIGGER @email #co-parent #legal -Kathryn’s message today saying “you never do anything for her” triggered a whole-body tension + tears. Completely false. - -!QUOTE @walk #unsaid -What I *wanted* to say was: “You act like you want control more than peace.” Didn’t say it. - -!LOOP #rumination -Keep repeating: “What if I’m the problem? What if it *is* all my fault?” over and over. - -!SOMATIC @22:05 #CPTSD -Shaking in both arms, vision blurring, and that sharp ice-feeling in my chest. No obvious trigger identified yet. -``` - ---- - -## 🔢 Fragment Insert Interfaces - -### 📱 Mobile (Planned) - -- Shortcut or PWA access - -- Prompts: Type, Time, Tags, Description - -- Appends to `YYYY-MM-DD.md` securely via NiceGUI interface - -### 💻 CLI / Bash - -```bash -jfrag "!TRIGGER" "Person texted me 'you don’t do anything for her'..." "#co-parent #shutdown" -``` - -Appends to journal or vault. - -### 🤖 AI Session Logging - -Say: - -``` -!QUOTE "I wish you would just work with me instead of against me." -``` - -Assistant logs it into today's entry using proper syntax. - ---- - -## ✅ Summary: Current & Future Tasks - -### 🔹 Completed - -- Vault encryption and cleanup - -- UI (NiceGUI), cross-platform and Tailscale-accessible - -- CLI tooling (vault, jfrag, search) - -- Metadata and tag system - -- AI summarization and pattern detection - -- Documentation and structured templates - -- Voice-to-text input (desktop & mobile) - -- Calendar view + richer Markdown preview in UI - -- Advanced NLP (sentiment, NER, topic modeling) - -- SQLcypher backend for fast structured search - -- Entry merging logic (into existing sections) - - -### 🔹 In Progress / Planned - -- Export therapy-ready summaries - -- Weekly/monthly summary generator - -- AI tag suggestion - -- In-memory decrypted vault reading (no full file extraction) \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 52bd987..16411ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "uvicorn>=0.38,<1", "pywebview>=6.1,<7; python_version < '3.14' or platform_system != 'Windows'", "SpeechRecognition>=3.14,<4", + "pyaudiowpatch>=0.2.12.8,<0.3; platform_system == 'Windows'", "pocketsphinx>=5,<6", "soundfile>=0.13,<1", "sounddevice>=0.5,<1", @@ -29,11 +30,13 @@ nlp = [ cpu-ai = [ "torch>=2.9,<3", "openai-whisper>=20250625", + "faster-whisper>=1.2,<2", ] gpu-ai = [ "torch>=2.9,<3", "triton>=3,<4; platform_system != 'Windows'", "openai-whisper>=20250625", + "faster-whisper>=1.2,<2", ] [project.urls] diff --git a/requirements_base.txt b/requirements_base.txt index 0643350..865d9e2 100644 --- a/requirements_base.txt +++ b/requirements_base.txt @@ -13,6 +13,7 @@ sqlcipher3-binary>=0.5,<1 ; platform_system != "Windows" # Speech and audio SpeechRecognition>=3.14,<4 +pyaudiowpatch>=0.2.12.8,<0.3 ; platform_system == "Windows" pocketsphinx>=5,<6 soundfile>=0.13,<1 sounddevice>=0.5,<1 diff --git a/requirements_cpu_only.txt b/requirements_cpu_only.txt index c74d023..01b5aa4 100644 --- a/requirements_cpu_only.txt +++ b/requirements_cpu_only.txt @@ -4,6 +4,7 @@ # AI (CPU) torch>=2.9,<3 openai-whisper>=20250625 +faster-whisper>=1.2,<2 # Optional NLP backend: # pip install -r requirements_nlp_optional.txt diff --git a/requirements_gpu.txt b/requirements_gpu.txt index 250c2a7..a343af1 100644 --- a/requirements_gpu.txt +++ b/requirements_gpu.txt @@ -5,6 +5,7 @@ torch>=2.9,<3 triton>=3,<4 ; platform_system != "Windows" openai-whisper>=20250625 +faster-whisper>=1.2,<2 # Optional NLP backend: # pip install -r requirements_nlp_optional.txt diff --git a/scripts/dev-shell.ps1 b/scripts/dev-shell.ps1 new file mode 100644 index 0000000..999f3ce --- /dev/null +++ b/scripts/dev-shell.ps1 @@ -0,0 +1,42 @@ +# Run this in PowerShell before development commands: +# . ./scripts/dev-shell.ps1 + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path + +# Clear dead proxy overrides and offline-only pip mode in current shell. +Remove-Item Env:HTTP_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:HTTPS_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:ALL_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:http_proxy -ErrorAction SilentlyContinue +Remove-Item Env:https_proxy -ErrorAction SilentlyContinue +Remove-Item Env:all_proxy -ErrorAction SilentlyContinue +Remove-Item Env:GIT_HTTP_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:GIT_HTTPS_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:PIP_NO_INDEX -ErrorAction SilentlyContinue + +# Keep .NET artifacts local to repo to avoid restricted user-profile paths. +$env:DOTNET_CLI_HOME = Join-Path $repoRoot ".dotnet_home" +$env:NUGET_PACKAGES = Join-Path $repoRoot ".nuget\packages" +$env:NUGET_HTTP_CACHE_PATH = Join-Path $repoRoot ".nuget\http-cache" +$env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = "1" +$env:DOTNET_ADD_GLOBAL_TOOLS_TO_PATH = "0" +$env:DOTNET_GENERATE_ASPNET_CERTIFICATE = "0" +$env:DOTNET_CLI_TELEMETRY_OPTOUT = "1" +$env:NUGET_CERT_REVOCATION_MODE = "offline" + +# Keep pip artifacts local to repo. +$env:PIP_CACHE_DIR = Join-Path $repoRoot ".pip\cache" +$env:TEMP = Join-Path $repoRoot ".tmp\pip-temp" +$env:TMP = $env:TEMP +$env:PIP_DISABLE_PIP_VERSION_CHECK = "1" +$env:PIP_DEFAULT_TIMEOUT = "30" +$env:PIP_RETRIES = "2" + +# Keep Hugging Face cache local and silence symlink-only warning on Windows. +$env:HF_HOME = Join-Path $repoRoot ".cache\huggingface" +$env:HUGGINGFACE_HUB_CACHE = Join-Path $env:HF_HOME "hub" +$env:HF_HUB_DISABLE_SYMLINKS_WARNING = "1" + +New-Item -ItemType Directory -Force -Path $env:DOTNET_CLI_HOME,$env:NUGET_PACKAGES,$env:NUGET_HTTP_CACHE_PATH,$env:PIP_CACHE_DIR,$env:TEMP,$env:HUGGINGFACE_HUB_CACHE | Out-Null + +Write-Host "Development shell initialized for repo-local dotnet/pip paths." diff --git a/scripts/dotnet-min.ps1 b/scripts/dotnet-min.ps1 new file mode 100644 index 0000000..a35844d --- /dev/null +++ b/scripts/dotnet-min.ps1 @@ -0,0 +1,16 @@ +param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$DotnetArgs +) + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +$innerScript = Join-Path $repoRoot "journal-master\journal\scripts\dotnet-min.ps1" + +if (-not (Test-Path $innerScript)) { + Write-Host "Missing script: $innerScript" + exit 2 +} + +& $innerScript @DotnetArgs +exit $LASTEXITCODE + diff --git a/scripts/migration-gate.ps1 b/scripts/migration-gate.ps1 new file mode 100644 index 0000000..46781bc --- /dev/null +++ b/scripts/migration-gate.ps1 @@ -0,0 +1,46 @@ +param( + [switch]$SkipSmoke, + [switch]$SkipApi +) + +$ErrorActionPreference = "Stop" +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +$parityReport = Join-Path $repoRoot "logs\parity_harness_results.json" + +Write-Host "migration-gate: repo root = $repoRoot" + +Push-Location $repoRoot +try { + Write-Host "migration-gate: building sidecar and api binaries..." + & "$repoRoot\scripts\dotnet-min.ps1" build journal-master/journal/Journal.Sidecar/Journal.Sidecar.csproj + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & "$repoRoot\scripts\dotnet-min.ps1" build journal-master/journal/Journal.Api/Journal.Api.csproj + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + if (-not $SkipSmoke) { + Write-Host "migration-gate: running csharp smoke tests..." + & "$repoRoot\scripts\dotnet-min.ps1" run --project journal-master/journal/Journal.SmokeTests/Journal.SmokeTests.csproj + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } else { + Write-Host "migration-gate: skipping smoke tests (--SkipSmoke)." + } + + Write-Host "migration-gate: running parity harness + fixture matrix..." + $env:PARITY_HARNESS_REPORT = $parityReport + & python -m unittest discover -s tests -p "test_parity_harness.py" -v + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + if (-not $SkipApi) { + Write-Host "migration-gate: running API contract tests..." + & python -m unittest discover -s tests -p "test_api_contract.py" -v + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } else { + Write-Host "migration-gate: skipping API contract tests (--SkipApi)." + } + + Write-Host "migration-gate: PASS" + Write-Host "migration-gate: parity report => $parityReport" +} +finally { + Pop-Location +} diff --git a/scripts/pip-min.ps1 b/scripts/pip-min.ps1 new file mode 100644 index 0000000..8ee36fb --- /dev/null +++ b/scripts/pip-min.ps1 @@ -0,0 +1,70 @@ +param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$PipArgs +) + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path + +# Keep pip artifacts local for easy cleanup on minimal/portable machines. +$env:PIP_CACHE_DIR = Join-Path $repoRoot ".pip\cache" +$env:TEMP = Join-Path $repoRoot ".tmp\pip-temp" +$env:TMP = $env:TEMP + +New-Item -ItemType Directory -Force -Path $env:PIP_CACHE_DIR | Out-Null +New-Item -ItemType Directory -Force -Path $env:TEMP | Out-Null + +# Clear proxy/no-index env vars that commonly break package fetch. +Remove-Item Env:HTTP_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:HTTPS_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:ALL_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:http_proxy -ErrorAction SilentlyContinue +Remove-Item Env:https_proxy -ErrorAction SilentlyContinue +Remove-Item Env:all_proxy -ErrorAction SilentlyContinue +Remove-Item Env:GIT_HTTP_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:GIT_HTTPS_PROXY -ErrorAction SilentlyContinue +Remove-Item Env:PIP_NO_INDEX -ErrorAction SilentlyContinue + +# Keep behavior deterministic and avoid interactive/network hangs. +$env:PIP_DISABLE_PIP_VERSION_CHECK = "1" +$env:PIP_DEFAULT_TIMEOUT = "30" +$env:PIP_RETRIES = "2" + +if (-not $PipArgs -or $PipArgs.Count -eq 0) { + Write-Host "Usage: ./scripts/pip-min.ps1 <pip args>" + Write-Host "Example: ./scripts/pip-min.ps1 install --index-url https://pypi.org/simple faster-whisper" + exit 2 +} + +# Default install target to a repo-local directory so installs do not require +# user/site-packages write access on constrained hosts. +$effectiveArgs = @($PipArgs) +$firstArg = $effectiveArgs[0].ToLowerInvariant() +if ($firstArg -eq "install") { + # On Windows, map PyAudio to pyaudiowpatch (wheel available for newer CPython), + # avoiding source builds that require PortAudio headers/toolchain wiring. + for ($i = 0; $i -lt $effectiveArgs.Count; $i++) { + $arg = $effectiveArgs[$i] + if ($arg -match '^(?i)pyaudio($|[<>=!~].*)') { + $suffix = $arg.Substring(7) + $effectiveArgs[$i] = "pyaudiowpatch$suffix" + Write-Host "pip-min: mapped '$arg' -> '$($effectiveArgs[$i])' on Windows." + } + } + + $hasTarget = $effectiveArgs -contains "--target" -or $effectiveArgs -contains "-t" -or $effectiveArgs -contains "--prefix" + if (-not $hasTarget) { + $effectiveArgs = $effectiveArgs | Where-Object { $_ -ne "--user" } + $localTarget = Join-Path $repoRoot ".pydeps\py314" + New-Item -ItemType Directory -Force -Path $localTarget | Out-Null + $effectiveArgs += @("--target", $localTarget) + Write-Host "pip-min: using local target $localTarget" + } +} + +$pipWrapper = Join-Path $PSScriptRoot "pip_safe.py" +if (Test-Path $pipWrapper) { + & python $pipWrapper @effectiveArgs +} else { + & python -m pip @effectiveArgs +} +exit $LASTEXITCODE diff --git a/scripts/pip_safe.py b/scripts/pip_safe.py new file mode 100644 index 0000000..a02a307 --- /dev/null +++ b/scripts/pip_safe.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import os +import tempfile +from typing import Callable + + +def _mkdtemp_compat( + suffix: str | None = None, + prefix: str | None = None, + dir: str | None = None, +) -> str: + # Python 3.14 on some Windows hosts creates mkdtemp dirs that are + # immediately non-writable by the same process when mode=0o700 is used. + # pip relies heavily on tempfile; force 0o777 for compatibility. + if dir is None: + dir = tempfile.gettempdir() + if prefix is None: + prefix = tempfile.template + if suffix is None: + suffix = "" + + names = tempfile._get_candidate_names() + for _ in range(tempfile.TMP_MAX): + name = next(names) + path = os.path.join(dir, f"{prefix}{name}{suffix}") + try: + os.mkdir(path, 0o777) + return path + except FileExistsError: + continue + + raise FileExistsError("No usable temporary directory name found.") + + +def main(argv: list[str]) -> int: + tempfile.mkdtemp = _mkdtemp_compat # type: ignore[assignment] + + from pip._internal.cli.main import main as pip_main + + return int(pip_main(argv)) + + +if __name__ == "__main__": + raise SystemExit(main(__import__("sys").argv[1:])) + diff --git a/tests/test_ai_hybrid_bridge.py b/tests/test_ai_hybrid_bridge.py new file mode 100644 index 0000000..d80cebd --- /dev/null +++ b/tests/test_ai_hybrid_bridge.py @@ -0,0 +1,51 @@ +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from journal.core.models import JournalEntry +from journal.ai import bridge + + +class AiHybridBridgeTests(unittest.TestCase): + def test_summarize_entry_uses_local_analysis_in_hybrid_mode(self): + entry = JournalEntry(date="2026-02-22", raw_content="sample content") + with ( + patch("journal.ai.bridge.analysis.summarize_entry", return_value="local result") as mock_local, + ): + result = bridge.summarize_entry(entry) + + self.assertEqual(result, "local result") + mock_local.assert_called_once_with(entry) + + def test_summarize_all_uses_local_analysis_in_hybrid_mode(self): + entries = [ + JournalEntry(date="2026-02-21", raw_content="entry one"), + JournalEntry(date="2026-02-22", raw_content="entry two"), + ] + with ( + patch("journal.ai.bridge.analysis.summarize_all_entries", return_value="all local result") as mock_local, + ): + result = bridge.summarize_all_entries(entries) + + self.assertEqual(result, "all local result") + mock_local.assert_called_once() + + def test_summarize_entry_uses_local_analysis_in_python_mode(self): + entry = JournalEntry(date="2026-02-22", raw_content="sample content") + with ( + patch("journal.ai.bridge.analysis.summarize_entry", return_value="local result") as mock_local, + ): + result = bridge.summarize_entry(entry) + + self.assertEqual(result, "local result") + mock_local.assert_called_once_with(entry) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_api_contract.py b/tests/test_api_contract.py new file mode 100644 index 0000000..db27880 --- /dev/null +++ b/tests/test_api_contract.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import json +import os +import socket +import subprocess +import time +import unittest +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +API_WORKDIR = PROJECT_ROOT / "journal-master" / "journal" +API_PROJECT = API_WORKDIR / "Journal.Api" / "Journal.Api.csproj" +API_DLL = API_WORKDIR / "Journal.Api" / "bin" / "Debug" / "net10.0" / "Journal.Api.dll" + + +def _pick_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _http_json(method: str, url: str, body: str | None = None) -> tuple[int, dict]: + headers = {"Accept": "application/json"} + data = None + if body is not None: + headers["Content-Type"] = "application/json" + data = body.encode("utf-8") + + request = Request(url=url, method=method, data=data, headers=headers) + try: + with urlopen(request, timeout=5.0) as response: + payload = response.read().decode("utf-8") + return int(response.status), json.loads(payload) + except HTTPError as ex: + payload = ex.read().decode("utf-8") + try: + parsed = json.loads(payload) + except json.JSONDecodeError: + parsed = {"raw": payload} + return int(ex.code), parsed + + +class ApiContractTests(unittest.TestCase): + process: subprocess.Popen[str] | None = None + base_url: str = "" + + @classmethod + def setUpClass(cls) -> None: + if not API_DLL.exists(): + raise FileNotFoundError( + f"Missing API binary: {API_DLL}. Build Journal.Api first." + ) + + port = _pick_free_port() + cls.base_url = f"http://127.0.0.1:{port}" + env = os.environ.copy() + env["DOTNET_CLI_HOME"] = str(PROJECT_ROOT / ".dotnet_home") + env["NUGET_PACKAGES"] = str(PROJECT_ROOT / ".nuget" / "packages") + env["NUGET_HTTP_CACHE_PATH"] = str(PROJECT_ROOT / ".nuget" / "http-cache") + env["DOTNET_SKIP_FIRST_TIME_EXPERIENCE"] = "1" + env["DOTNET_GENERATE_ASPNET_CERTIFICATE"] = "0" + env["DOTNET_CLI_TELEMETRY_OPTOUT"] = "1" + env["ASPNETCORE_URLS"] = cls.base_url + + cls.process = subprocess.Popen( + [ + "dotnet", + str(API_DLL), + ], + cwd=str(API_WORKDIR), + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + text=True, + ) + + deadline = time.time() + 60 + while time.time() < deadline: + if cls.process.poll() is not None: + raise RuntimeError("Journal.Api exited during startup.") + try: + status, payload = _http_json("GET", f"{cls.base_url}/health") + if status == 200 and payload.get("ok") is True: + return + except (URLError, TimeoutError): + pass + time.sleep(0.5) + + raise TimeoutError("Timed out waiting for Journal.Api /health endpoint.") + + @classmethod + def tearDownClass(cls) -> None: + if cls.process is None: + return + if cls.process.poll() is None: + cls.process.terminate() + try: + cls.process.wait(timeout=10) + except subprocess.TimeoutExpired: + cls.process.kill() + cls.process = None + + def test_health_endpoint_envelope(self): + status, payload = _http_json("GET", f"{self.base_url}/health") + self.assertEqual(status, 200) + self.assertEqual(payload, {"ok": True, "data": "healthy"}) + + def test_command_success_path(self): + status, payload = _http_json( + "POST", + f"{self.base_url}/api/command", + body=json.dumps({"action": "config.get", "payload": {}}), + ) + self.assertEqual(status, 200) + self.assertTrue(payload.get("ok")) + self.assertIsInstance(payload.get("data"), dict) + + def test_command_unknown_action_error_envelope(self): + status, payload = _http_json( + "POST", + f"{self.base_url}/api/command", + body=json.dumps({"action": "unknown.action", "payload": {}}), + ) + self.assertEqual(status, 200) + self.assertFalse(payload.get("ok")) + self.assertIn("unknown action", str(payload.get("error", "")).lower()) + + def test_command_missing_payload_error_envelope(self): + status, payload = _http_json( + "POST", + f"{self.base_url}/api/command", + body=json.dumps({"action": "entries.save"}), + ) + self.assertEqual(status, 200) + self.assertFalse(payload.get("ok")) + self.assertIn("payload", str(payload.get("error", "")).lower()) + + def test_command_malformed_json_error_envelope(self): + status, payload = _http_json( + "POST", + f"{self.base_url}/api/command", + body='{"action":', + ) + self.assertEqual(status, 200) + self.assertFalse(payload.get("ok")) + self.assertIn("invalid command json", str(payload.get("error", "")).lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli_fragments_hybrid.py b/tests/test_cli_fragments_hybrid.py new file mode 100644 index 0000000..cf3b46b --- /dev/null +++ b/tests/test_cli_fragments_hybrid.py @@ -0,0 +1,94 @@ +import io +import sys +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from unittest.mock import patch + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from journal.cli import main as cli_main + + +class CliFragmentsHybridTests(unittest.TestCase): + def test_fragments_list_calls_sidecar(self): + with ( + patch.object(sys, "argv", ["journal", "fragments", "list"]), + patch("journal.cli.main.BACKEND_MODE", "csharp-hybrid"), + patch( + "journal.cli.main.call_sidecar_action", + return_value=[{"Id": "1", "Type": "!NOTE", "Description": "desc", "Tags": []}], + ) as mock_call, + redirect_stdout(io.StringIO()) as stdout, + ): + cli_main.main() + + mock_call.assert_called_once_with("fragments.list") + self.assertIn("!NOTE", stdout.getvalue()) + + def test_fragments_create_calls_sidecar(self): + with ( + patch.object( + sys, + "argv", + [ + "journal", + "fragments", + "create", + "--type", + "!TRIGGER", + "--description", + "flashback", + "--tag", + "stress", + ], + ), + patch("journal.cli.main.BACKEND_MODE", "csharp-hybrid"), + patch("journal.cli.main.call_sidecar_action", return_value={"Id": "1"}) as mock_call, + redirect_stdout(io.StringIO()) as stdout, + ): + cli_main.main() + + mock_call.assert_called_once_with( + "fragments.create", + payload={"type": "!TRIGGER", "description": "flashback", "tags": ["stress"]}, + ) + self.assertIn("Fragment created.", stdout.getvalue()) + + def test_fragments_search_calls_sidecar_with_command_fields(self): + with ( + patch.object( + sys, + "argv", + ["journal", "fragments", "search", "--type", "!NOTE", "--tag", "daily"], + ), + patch("journal.cli.main.BACKEND_MODE", "csharp-hybrid"), + patch("journal.cli.main.call_sidecar_action", return_value=[]) as mock_call, + redirect_stdout(io.StringIO()) as stdout, + ): + cli_main.main() + + mock_call.assert_called_once_with( + "fragments.search", + command_fields={"type": "!NOTE", "tag": "daily"}, + ) + self.assertIn("No fragments found.", stdout.getvalue()) + + def test_fragments_command_requires_hybrid(self): + with ( + patch.object(sys, "argv", ["journal", "fragments", "list"]), + patch("journal.cli.main.BACKEND_MODE", "python"), + patch("journal.cli.main.call_sidecar_action") as mock_call, + redirect_stdout(io.StringIO()) as stdout, + ): + cli_main.main() + + mock_call.assert_not_called() + self.assertIn("requires JOURNAL_BACKEND_MODE=csharp-hybrid", stdout.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parity_harness.py b/tests/test_parity_harness.py new file mode 100644 index 0000000..2dd47b7 --- /dev/null +++ b/tests/test_parity_harness.py @@ -0,0 +1,401 @@ +import difflib +import hashlib +import json +import os +import shutil +import unittest +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from journal.core import storage +from journal.core.csharp_sidecar import call_sidecar_action +from journal.core.parser import parse_journal_content, parse_journal_file + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +FIXTURES_ROOT = PROJECT_ROOT / "fixtures" +ENTRY_FIXTURES = FIXTURES_ROOT / "entries" +SEARCH_FIXTURES = FIXTURES_ROOT / "search" / "queries.json" +VAULT_MANIFEST = FIXTURES_ROOT / "vaults" / "manifest.json" +PARITY_REPORT: list[dict[str, Any]] = [] + + +def _load_queries() -> list[dict[str, Any]]: + return json.loads(SEARCH_FIXTURES.read_text(encoding="utf-8")) + + +def _load_vault_manifest() -> dict[str, Any]: + return json.loads(VAULT_MANIFEST.read_text(encoding="utf-8")) + + +def _copy_entry_fixtures(target_dir: Path) -> None: + target_dir.mkdir(parents=True, exist_ok=True) + for source in sorted(ENTRY_FIXTURES.glob("*.md")): + shutil.copy2(source, target_dir / source.name) + + +def _copy_vault_fixtures(manifest: dict[str, Any], target_dir: Path) -> None: + target_dir.mkdir(parents=True, exist_ok=True) + for vault_row in manifest.get("vaults", []): + if not isinstance(vault_row, dict): + continue + name = vault_row.get("vault_file") + if not isinstance(name, str): + continue + source = FIXTURES_ROOT / "vaults" / name + shutil.copy2(source, target_dir / name) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while True: + chunk = handle.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + + +@contextmanager +def _workspace(): + root = PROJECT_ROOT / ".tmp" / "parity-tests" / uuid4().hex + root.mkdir(parents=True, exist_ok=True) + try: + yield root + finally: + shutil.rmtree(root, ignore_errors=True) + + +def _normalize_for_json(value: Any) -> Any: + if isinstance(value, dict): + return {str(k): _normalize_for_json(v) for k, v in sorted(value.items(), key=lambda item: str(item[0]))} + if isinstance(value, list): + return [_normalize_for_json(item) for item in value] + if isinstance(value, tuple): + return [_normalize_for_json(item) for item in value] + return value + + +def _record_parity(name: str, python_result: Any, csharp_result: Any) -> dict[str, Any]: + normalized_python = _normalize_for_json(python_result) + normalized_csharp = _normalize_for_json(csharp_result) + python_json = json.dumps(normalized_python, indent=2, ensure_ascii=True, sort_keys=True) + csharp_json = json.dumps(normalized_csharp, indent=2, ensure_ascii=True, sort_keys=True) + match = python_json == csharp_json + diff = "" + if not match: + diff = "\n".join( + difflib.unified_diff( + python_json.splitlines(), + csharp_json.splitlines(), + fromfile="python_result", + tofile="csharp_result", + lineterm="", + ) + ) + row = { + "name": name, + "python_result": normalized_python, + "csharp_result": normalized_csharp, + "match": match, + "diff": diff, + } + PARITY_REPORT.append(row) + return row + + +def _normalize_search_results(results: list[dict[str, Any]]) -> list[tuple[str, str]]: + normalized: list[tuple[str, str]] = [] + for item in results: + date_value = item.get("Date") or item.get("date") + file_name = item.get("FileName") or item.get("fileName") + if isinstance(date_value, str) and isinstance(file_name, str): + normalized.append((date_value, file_name)) + return sorted(normalized, key=lambda row: row[1]) + + +def _python_search(data_dir: Path, payload: dict[str, Any]) -> list[tuple[str, str]]: + query = (payload.get("query") or "").strip() + section = (payload.get("section") or "").strip() + tags = {v.strip() for v in payload.get("tags", []) if isinstance(v, str) and v.strip()} + types = {v.strip() for v in payload.get("types", []) if isinstance(v, str) and v.strip()} + checked = {v.strip() for v in payload.get("checked", []) if isinstance(v, str) and v.strip()} + unchecked = {v.strip() for v in payload.get("unchecked", []) if isinstance(v, str) and v.strip()} + + start_date = _parse_optional_date(payload.get("startDate")) + end_date = _parse_optional_date(payload.get("endDate")) + if start_date and end_date and start_date > end_date: + raise ValueError("startDate cannot be after endDate.") + + results: list[tuple[str, str]] = [] + for file_path in sorted(data_dir.glob("*.md"), key=lambda p: p.name): + entry = parse_journal_file(str(file_path)) + entry_date = _parse_optional_date(entry.date) + + if (start_date or end_date) and entry_date is None: + continue + if start_date and entry_date and entry_date < start_date: + continue + if end_date and entry_date and entry_date > end_date: + continue + + if query: + haystack = entry.get_section(section) if section else entry.raw_content + if query.lower() not in haystack.lower(): + continue + + if tags or types: + matched_fragment = False + for fragment in entry.fragments: + type_ok = not types or fragment.type in types + tag_ok = not tags or any(tag in tags for tag in fragment.tags) + if type_ok and tag_ok: + matched_fragment = True + break + if not matched_fragment: + continue + + if checked or unchecked: + matched_checkbox = False + for parsed_section in entry.sections.values(): + for checkbox_text, is_checked in parsed_section.checkboxes.items(): + if checked and is_checked and checkbox_text in checked: + matched_checkbox = True + break + if unchecked and (not is_checked) and checkbox_text in unchecked: + matched_checkbox = True + break + if matched_checkbox: + break + if not matched_checkbox: + continue + + results.append((entry.date, file_path.name)) + + return sorted(results, key=lambda row: row[1]) + + +def _parse_optional_date(value: str | None): + if not value or not isinstance(value, str): + return None + try: + return datetime.strptime(value.strip(), "%Y-%m-%d").date() + except ValueError: + return None + + +class ParityHarnessTests(unittest.TestCase): + @classmethod + def tearDownClass(cls) -> None: + report_path = Path( + os.environ.get( + "PARITY_HARNESS_REPORT", + str(PROJECT_ROOT / "logs" / "parity_harness_results.json"), + ) + ) + report_path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "total_cases": len(PARITY_REPORT), + "passed_cases": sum(1 for case in PARITY_REPORT if case["match"]), + "failed_cases": sum(1 for case in PARITY_REPORT if not case["match"]), + "cases": PARITY_REPORT, + } + report_path.write_text(json.dumps(payload, indent=2, ensure_ascii=True) + "\n", encoding="utf-8") + + def test_entries_list_load_parity(self): + with _workspace() as root: + data_dir = root / "data" + _copy_entry_fixtures(data_dir) + + csharp_list = call_sidecar_action( + "entries.list", + payload={"dataDirectory": str(data_dir)}, + ) + self.assertIsInstance(csharp_list, list) + + csharp_names = sorted( + [ + item.get("FileName") or item.get("fileName") + for item in csharp_list + if isinstance(item, dict) + ] + ) + python_names = sorted([path.name for path in data_dir.glob("*.md")]) + row = _record_parity("entries.list", python_names, csharp_names) + self.assertTrue(row["match"], row["diff"]) + + for name in python_names: + file_path = data_dir / name + csharp_loaded = call_sidecar_action( + "entries.load", + payload={"filePath": str(file_path)}, + ) + self.assertIsInstance(csharp_loaded, dict) + csharp_raw = csharp_loaded.get("RawContent") or csharp_loaded.get("rawContent") + python_raw = storage._strip_rich_html(file_path.read_text(encoding="utf-8")) # pylint: disable=protected-access + row = _record_parity(f"entries.load::{name}", python_raw, csharp_raw) + self.assertTrue(row["match"], row["diff"]) + + def test_entries_save_merge_parity(self): + with _workspace() as root: + data_dir = root / "data" + _copy_entry_fixtures(data_dir) + target = data_dir / "2026-01-05.md" + original = target.read_text(encoding="utf-8") + + new_content = ( + "**Date:** 2026-01-05\n\n" + "## Triggers\n" + "Crowded grocery store caused severe panic.\n\n" + "## Reflections\n" + "Added one new thought after grounding.\n" + ) + + python_existing = parse_journal_content(original, target.stem) + python_incoming = parse_journal_content(new_content, target.stem) + python_existing.merge_with(python_incoming) + python_markdown = python_existing.to_markdown() + + _ = call_sidecar_action( + "entries.save", + payload={ + "content": new_content, + "filePath": str(target), + "mode": "Daily", + }, + ) + + csharp_markdown = target.read_text(encoding="utf-8") + python_entry = parse_journal_content(python_markdown, target.stem) + csharp_entry = parse_journal_content(csharp_markdown, target.stem) + row = _record_parity( + "entries.save::merge", + { + "date": python_entry.date, + "triggers": python_entry.get_section("Triggers").strip(), + "reflections": python_entry.get_section("Reflections").strip(), + }, + { + "date": csharp_entry.date, + "triggers": csharp_entry.get_section("Triggers").strip(), + "reflections": csharp_entry.get_section("Reflections").strip(), + }, + ) + self.assertTrue(row["match"], row["diff"]) + + def test_search_parity_against_python_and_expected_ids(self): + with _workspace() as root: + data_dir = root / "data" + _copy_entry_fixtures(data_dir) + queries = _load_queries() + + for case in queries: + case_name = str(case.get("name", "unnamed")) + payload = dict(case.get("payload", {})) + payload["dataDirectory"] = str(data_dir) + + python_result = _python_search(data_dir, payload) + csharp_result = call_sidecar_action("search.entries", payload=payload) + self.assertIsInstance(csharp_result, list) + csharp_normalized = _normalize_search_results(csharp_result) + + parity_row = _record_parity(f"search.entries::{case_name}", python_result, csharp_normalized) + self.assertTrue(parity_row["match"], parity_row["diff"]) + + expected_file_names = sorted(case.get("expected_file_names", [])) + expected_row = _record_parity( + f"search.expected::{case_name}", + expected_file_names, + [item[1] for item in csharp_normalized], + ) + self.assertTrue(expected_row["match"], expected_row["diff"]) + + def test_sanitizer_parity_for_html_heavy_input(self): + with _workspace() as root: + data_dir = root / "data" + data_dir.mkdir(parents=True, exist_ok=True) + target = data_dir / "2026-02-26.md" + html_input = ( + '<p style="font-family: Times New Roman;">Hello <b>World</b></p>' + "<ul><li>alpha</li><li>beta</li></ul>" + ) + python_sanitized = storage._strip_rich_html(html_input) # pylint: disable=protected-access + _ = call_sidecar_action( + "entries.save", + payload={ + "content": html_input, + "filePath": str(target), + "mode": "Overwrite", + }, + ) + csharp_saved = target.read_text(encoding="utf-8") + row = _record_parity("sanitizer.rich_html", python_sanitized, csharp_saved) + self.assertTrue(row["match"], row["diff"]) + + def test_vault_manifest_load_and_hash_integrity(self): + manifest = _load_vault_manifest() + fixture_password = manifest.get("password") + self.assertIsInstance(fixture_password, str) + self.assertTrue(fixture_password) + + with _workspace() as root: + vault_dir = root / "vault" + data_dir = root / "data" + _copy_vault_fixtures(manifest, vault_dir) + + expected_hashes: dict[str, str] = {} + for vault_row in manifest.get("vaults", []): + for entry_row in vault_row.get("expected_entries", []): + expected_hashes[str(entry_row["file_name"])] = str(entry_row["sha256"]) + + loaded = call_sidecar_action( + "vault.load_all", + payload={ + "password": fixture_password, + "vaultDirectory": str(vault_dir), + "dataDirectory": str(data_dir), + }, + ) + self.assertTrue(bool(loaded), "Expected fixture vaults to load with manifest password.") + + actual_hashes: dict[str, str] = {} + for file_path in sorted(data_dir.glob("*.md"), key=lambda p: p.name): + actual_hashes[file_path.name] = _sha256_file(file_path) + + row = _record_parity("vault.load_all::hashes", expected_hashes, actual_hashes) + self.assertTrue(row["match"], row["diff"]) + + def test_vault_wrong_password_preserves_bytes(self): + manifest = _load_vault_manifest() + wrong_password = manifest.get("wrong_password") + self.assertIsInstance(wrong_password, str) + self.assertTrue(wrong_password) + + with _workspace() as root: + vault_dir = root / "vault" + data_dir = root / "data" + _copy_vault_fixtures(manifest, vault_dir) + + before_hashes = {path.name: _sha256_file(path) for path in sorted(vault_dir.glob("*.vault"), key=lambda p: p.name)} + loaded = call_sidecar_action( + "vault.load_all", + payload={ + "password": wrong_password, + "vaultDirectory": str(vault_dir), + "dataDirectory": str(data_dir), + }, + ) + self.assertFalse(bool(loaded), "Wrong password should fail vault.load_all.") + after_hashes = {path.name: _sha256_file(path) for path in sorted(vault_dir.glob("*.vault"), key=lambda p: p.name)} + + row = _record_parity("vault.load_all::wrong_password_invariant", before_hashes, after_hashes) + self.assertTrue(row["match"], row["diff"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_storage_hybrid_bridge.py b/tests/test_storage_hybrid_bridge.py new file mode 100644 index 0000000..631ff43 --- /dev/null +++ b/tests/test_storage_hybrid_bridge.py @@ -0,0 +1,132 @@ +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from journal.core import storage + + +class StorageHybridBridgeTests(unittest.TestCase): + def test_save_entry_content_uses_entries_save_in_hybrid_mode(self): + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / "2026-02-22.md" + with ( + patch("journal.core.storage.BACKEND_MODE", "csharp-hybrid"), + patch("journal.core.storage.call_sidecar_action", return_value={"FilePath": str(target)}) as mock_call, + ): + storage.save_entry_content("hello world", file_path=target, mode="Daily") + + mock_call.assert_called_once_with( + "entries.save", + payload={ + "content": "hello world", + "filePath": str(target), + "mode": "Daily", + }, + ) + + def test_load_entry_content_uses_entries_load_in_hybrid_mode(self): + fake_path = "E:/tmp/2026-02-22.md" + with ( + patch("journal.core.storage.BACKEND_MODE", "csharp-hybrid"), + patch( + "journal.core.storage.call_sidecar_action", + return_value={"RawContent": "entry content"}, + ) as mock_call, + ): + result = storage.load_entry_content(fake_path) + + self.assertEqual(result, "entry content") + mock_call.assert_called_once_with( + "entries.load", + payload={"filePath": fake_path}, + ) + + def test_save_entry_content_strips_rich_html_before_hybrid_save(self): + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / "2026-02-22.md" + nasty_html = ( + '<p style="font-family: Times New Roman;">Hello <b>World</b></p>' + "<ul><li>A</li><li>B</li></ul>" + ) + with ( + patch("journal.core.storage.BACKEND_MODE", "csharp-hybrid"), + patch("journal.core.storage.call_sidecar_action", return_value={"FilePath": str(target)}) as mock_call, + ): + storage.save_entry_content(nasty_html, file_path=target, mode="Overwrite") + + sent_payload = mock_call.call_args.kwargs["payload"] + sent_content = sent_payload["content"] + self.assertNotIn("<p", sent_content.lower()) + self.assertNotIn("style=", sent_content.lower()) + self.assertIn("Hello World", sent_content) + self.assertIn("- A", sent_content) + + def test_load_entry_content_strips_rich_html_in_hybrid_mode(self): + fake_path = "E:/tmp/2026-02-22.md" + nasty_html = ( + '<p style="font-family: Times New Roman;">Top</p>' + '<p><span>Body</span></p>' + ) + with ( + patch("journal.core.storage.BACKEND_MODE", "csharp-hybrid"), + patch( + "journal.core.storage.call_sidecar_action", + return_value={"RawContent": nasty_html}, + ), + ): + result = storage.load_entry_content(fake_path) + + self.assertEqual(result, "Top\nBody") + + def test_list_journal_files_uses_entries_list_in_hybrid_mode(self): + with ( + patch("journal.core.storage.BACKEND_MODE", "csharp-hybrid"), + patch( + "journal.core.storage.call_sidecar_action", + return_value=[ + {"FileName": "2026-02-01.md", "FilePath": "E:/tmp/2026-02-01.md"}, + {"FileName": "2026-02-02.md", "FilePath": "E:/tmp/2026-02-02.md"}, + ], + ) as mock_call, + ): + result = storage.list_journal_files() + + self.assertEqual( + result, + [ + ("2026-02-01.md", "E:/tmp/2026-02-01.md"), + ("2026-02-02.md", "E:/tmp/2026-02-02.md"), + ], + ) + mock_call.assert_called_once() + + def test_load_all_vaults_uses_csharp_workspace_hydration_in_hybrid_mode(self): + with tempfile.TemporaryDirectory() as tmp: + with ( + patch("journal.core.storage.BACKEND_MODE", "csharp-hybrid"), + patch("journal.core.storage.VAULT_DIR", Path(tmp) / "vault"), + patch("journal.core.storage.DATA_DIR", Path(tmp) / "data"), + patch( + "journal.core.storage.call_sidecar_action", + side_effect=[True, {"EntryFilesProcessed": 2}], + ) as mock_call, + ): + result = storage.load_all_vaults("vault-pass-123") + + self.assertTrue(result) + self.assertEqual(mock_call.call_count, 2) + first_call = mock_call.call_args_list[0] + second_call = mock_call.call_args_list[1] + self.assertEqual(first_call.args[0], "vault.load_all") + self.assertEqual(second_call.args[0], "db.hydrate_workspace") + + +if __name__ == "__main__": + unittest.main()