plasalid 0.3.5 → 0.5.0

package/dist/scanner/pipeline.js

@@ -1,64 +1,73 @@
 import { randomUUID } from "crypto";
 import { getDb } from "../db/connection.js";
+import { countOpenConcerns, } from "../db/queries/concerns.js";
+import { correlatePairs } from "../db/queries/transactions.js";
 import { runScanAgent } from "../ai/agent.js";
-import { statusSpinner, makePromptUser, makeAgentOnProgress, } from "../cli/ux.js";
-import { readPdf, buildDocumentBlock } from "./pdf.js";
+import { buildDocumentBlock } from "./pdf.js";
 import { buildScanUserMessage } from "./prompts.js";
 import { scanDataDir } from "./walker.js";
-import { unlockIfNeeded, persistUnlockOutcome } from "./unlock.js";
-// ── DB helpers ──────────────────────────────────────────────────────────────
-function findScannedByHash(db, hash) {
-    return db
-        .prepare(`SELECT id FROM scanned_files WHERE file_hash = ?`)
-        .get(hash) ?? null;
-}
-function deleteScannedFile(db, id) {
-    db.prepare(`DELETE FROM scanned_files WHERE id = ?`).run(id);
-}
-function insertScannedFile(db, args) {
-    const id = `sf:${randomUUID()}`;
-    db.prepare(`INSERT INTO scanned_files (id, path, file_hash, mime, status)
-     VALUES (?, ?, ?, ?, 'pending')`).run(id, args.path, args.hash, args.mime);
-    return id;
-}
-function countPendingQuestions(db, fileId) {
-    const row = db
-        .prepare(`SELECT COUNT(*) as n FROM pending_questions WHERE file_id = ? AND resolved_at IS NULL`)
-        .get(fileId);
-    return row.n;
-}
-function setFileStatus(db, id, status, fields = {}) {
-    db.prepare(`UPDATE scanned_files
-     SET status = ?, scanned_at = datetime('now'), error = ?, raw_text = COALESCE(?, raw_text)
-     WHERE id = ?`).run(status, fields.error ?? null, fields.raw_text ?? null, id);
+import { BufferedWriteContext } from "./buffer.js";
+import { runWithConcurrency } from "./concurrency.js";
+import { decryptQueue, confirmProceedAfterFailures, } from "./decrypt_queue.js";
+export function compileMatcher(input) {
+    return new RegExp(input, "i");
 }
-// ── Per-file scan ───────────────────────────────────────────────────────────
-export async function scanFile(filePath, opts = {}) {
+/** Orchestration */
+export async function runScan(opts = {}) {
     const db = getDb();
-    const file = readPdf(filePath);
-    const existing = findScannedByHash(db, file.hash);
-    if (existing && !opts.force) {
-        return { fileId: existing.id, status: "skipped", pendingQuestions: countPendingQuestions(db, existing.id) };
-    }
-    const wasReplaced = !!existing;
-    if (existing) {
-        deleteScannedFile(db, existing.id);
-    }
-    let unlocked;
-    try {
-        unlocked = await unlockIfNeeded({
-            db,
-            filePath,
-            bytes: file.bytes,
-            interactive: opts.interactive ?? true,
-        });
-    }
-    catch (err) {
-        return { fileId: null, status: "failed", error: err.message, pendingQuestions: 0 };
+    const matcher = opts.regex ? compileMatcher(opts.regex) : null;
+    const allFiles = scanDataDir().filter(f => (matcher ? matcher.test(f.relPath) : true));
+    const concurrency = Math.min(8, Math.max(1, opts.concurrency ?? 3));
+    const interactive = opts.interactive ?? true;
+    const events = opts.events;
+    // Phase 1 — decrypt all
+    events?.decryptStart?.(allFiles.length);
+    const decryptResult = await decryptQueue(db, allFiles, {
+        force: !!opts.force,
+        interactive,
+        onProgress: events?.decryptProgress,
+    });
+    events?.decryptDone?.({
+        decrypted: decryptResult.decrypted.length,
+        skipped: decryptResult.skipped.length,
+        failed: decryptResult.failed.length,
+    });
+    const proceed = await confirmProceedAfterFailures(decryptResult, interactive);
+    if (!proceed) {
+        return buildAbortedSummary(allFiles.length, decryptResult);
     }
-    persistUnlockOutcome(db, filePath, unlocked.outcome);
-    const fileId = insertScannedFile(db, { path: filePath, hash: file.hash, mime: file.mime });
-    const block = buildDocumentBlock(unlocked.decrypted, file.fileName, file.mime);
+    // Phase 2 — parallel scan with buffered writes
+    const scanResults = await scanInParallel(db, decryptResult.decrypted, { concurrency, events });
+    // Phase 3 — cross-file correlation pre-commit
+    const pairCount = applyCrossFileCorrelations(scanResults);
+    events?.correlating?.(pairCount);
+    // Phase 4 — per-file commit
+    events?.committing?.();
+    const fileResults = commitAll(db, decryptResult, scanResults);
+    return buildSummary(allFiles.length, fileResults, decryptResult);
+}
+async function scanInParallel(db, files, opts) {
+    const tasks = files.map(f => () => scanOneFile(db, f, opts.events));
+    const settled = await runWithConcurrency(tasks, opts.concurrency);
+    // Worker errors are captured per-slot by runWithConcurrency. scanOneFile
+    // itself catches LLM errors and returns a ScanWorkResult with `error` set,
+    // so the `{error}` branch only fires for truly unexpected throws.
+    return settled.map((r, i) => {
+        if (r && typeof r === "object" && "error" in r && !("buffer" in r)) {
+            return {
+                decryptedFile: files[i],
+                buffer: new BufferedWriteContext(files[i].fileName),
+                error: String(r.error),
+                agentText: "",
+            };
+        }
+        return r;
+    });
+}
+async function scanOneFile(db, file, events) {
+    const buffer = new BufferedWriteContext(file.fileName);
+    events?.scanStart?.({ fileName: file.fileName });
+    const block = buildDocumentBlock(file.decryptedBytes, file.fileName, file.mime);
     const messages = [
         {
             role: "user",
@@ -68,70 +77,211 @@ export async function scanFile(filePath, opts = {}) {
             ],
         },
     ];
-    const spinner = statusSpinner(`Scanning ${file.fileName}...`);
-    let summary = "";
     try {
         const text = await runScanAgent({
             db,
             initialMessages: messages,
             prompt: { fileName: file.fileName },
             agentCtx: {
-                fileId,
-                interactive: opts.interactive ?? true,
-                promptUser: opts.interactive === false ? undefined : makePromptUser(spinner),
-                onComplete: (s) => { summary = s; },
+                interactive: false,
+                buffer,
             },
-            onProgress: makeAgentOnProgress(spinner, file.fileName),
+            onProgress: (event) => {
+                if (event.phase === "tool" && event.toolName) {
+                    events?.scanProgress?.({ fileName: file.fileName, step: event.toolName });
+                }
+                else if (event.phase === "responding") {
+                    events?.scanProgress?.({ fileName: file.fileName, step: "thinking" });
+                }
+            },
+        });
+        events?.scanEnd?.({
+            fileName: file.fileName,
+            status: "scanned",
+            transactions: buffer.transactions.length,
+            concerns: buffer.concerns.length,
         });
-        const stillPending = countPendingQuestions(db, fileId);
-        if (stillPending > 0) {
-            setFileStatus(db, fileId, "needs_input", { raw_text: text });
-            spinner.info(`${file.fileName} needs input (${stillPending} pending).`);
-            return { fileId, status: "needs_input", summary: summary || text, pendingQuestions: stillPending };
+        return { decryptedFile: file, buffer, agentText: text };
+    }
+    catch (err) {
+        const message = err?.message ?? "agent error";
+        events?.scanEnd?.({
+            fileName: file.fileName,
+            status: "failed",
+            transactions: 0,
+            concerns: 0,
+            error: message,
+        });
+        return { decryptedFile: file, buffer, error: message, agentText: "" };
+    }
+}
+/** Phase 3: cross-file correlation */
+/**
+ * For every pair of buffered entries that look like the same money movement
+ * across two different files, append a mirror concern to each side's buffer.
+ * Returns the number of pairs detected so the CLI can report it.
+ */
+function applyCrossFileCorrelations(results) {
+    const all = [];
+    for (const res of results) {
+        if (res.error)
+            continue;
+        for (const bt of res.buffer.transactions) {
+            all.push({
+                file: res,
+                transactionId: bt.transaction_id,
+                postings: bt.input.postings,
+                date: bt.input.date,
+                description: bt.input.description,
+            });
         }
-        setFileStatus(db, fileId, "scanned", { raw_text: text });
-        spinner.succeed(`Scanned ${file.fileName}.`);
+    }
+    const candidates = all.map(e => {
+        const debit = e.postings.reduce((s, p) => s + (p.debit ?? 0), 0);
+        const currency = e.postings.find(p => p.currency)?.currency ?? "THB";
+        const ids = Array.from(new Set(e.postings.map(p => p.account_id)));
         return {
-            fileId,
-            status: wasReplaced ? "replaced" : "scanned",
-            summary: summary || text,
-            pendingQuestions: 0,
+            id: e.transactionId,
+            date: e.date,
+            description: e.description,
+            amount: Math.round(debit * 100) / 100,
+            currency,
+            account_ids: ids,
+            account_names: ids,
         };
+    });
+    const pairs = correlatePairs(candidates, { toleranceDays: 3 });
+    const byTransaction = new Map(all.map(a => [a.transactionId, a]));
+    for (const pair of pairs) {
+        const a = byTransaction.get(pair.a.id);
+        const b = byTransaction.get(pair.b.id);
+        if (!a || !b)
+            continue;
+        if (a.file === b.file)
+            continue;
+        const amountStr = `฿${pair.amount.toLocaleString("en-US", { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`;
+        a.file.buffer.appendConcern({
+            transaction_id: a.transactionId,
+            account_id: null,
+            prompt: `Looks like the matching half of this ${amountStr} movement on ${a.date} was also recorded in ${b.file.decryptedFile.fileName} on ${b.date}. Merge during review?`,
+            options: ["Yes — merge into one transaction", "No — these are two real events", "Skip — leave as is"],
+        });
+        b.file.buffer.appendConcern({
+            transaction_id: b.transactionId,
+            account_id: null,
+            prompt: `Looks like the matching half of this ${amountStr} movement on ${b.date} was also recorded in ${a.file.decryptedFile.fileName} on ${a.date}. Merge during review?`,
+            options: ["Yes — merge into one transaction", "No — these are two real events", "Skip — leave as is"],
+        });
     }
-    catch (err) {
-        setFileStatus(db, fileId, "failed", { error: err.message });
-        spinner.fail(`${file.fileName} failed: ${err.message}`);
-        return { fileId, status: "failed", error: err.message, pendingQuestions: countPendingQuestions(db, fileId) };
-    }
+    return pairs.filter(p => byTransaction.get(p.a.id)?.file !== byTransaction.get(p.b.id)?.file).length;
 }
-export function compileMatcher(input) {
-    return new RegExp(input, "i");
+/** Phase 4: commit */
+function commitAll(db, decryptResult, scanResults) {
+    const out = [];
+    for (const skipped of decryptResult.skipped) {
+        out.push({
+            name: skipped.file.name,
+            relPath: skipped.file.relPath,
+            status: "skipped",
+            transactions: 0,
+            concerns: countOpenConcerns(db, { file_id: skipped.existingScannedFileId }),
+        });
+    }
+    for (const failed of decryptResult.failed) {
+        out.push({
+            name: failed.file.name,
+            relPath: failed.file.relPath,
+            status: "failed",
+            transactions: 0,
+            concerns: 0,
+            error: failed.error,
+        });
+    }
+    for (const res of scanResults) {
+        const { decryptedFile, buffer, error, agentText } = res;
+        if (error) {
+            out.push({
+                name: decryptedFile.fileName,
+                relPath: decryptedFile.relPath,
+                status: "failed",
+                transactions: 0,
+                concerns: buffer.concerns.length,
+                error,
+            });
+            continue;
+        }
+        try {
+            if (decryptedFile.replacesPriorScannedFileId) {
+                deleteScannedFile(db, decryptedFile.replacesPriorScannedFileId);
+            }
+            const scannedFileId = insertScannedFile(db, {
+                path: decryptedFile.path,
+                hash: decryptedFile.hash,
+                mime: decryptedFile.mime,
+            });
+            const counts = buffer.commit(db, scannedFileId);
+            setFileStatus(db, scannedFileId, "scanned", { raw_text: agentText });
+            out.push({
+                name: decryptedFile.fileName,
+                relPath: decryptedFile.relPath,
+                status: decryptedFile.replacesPriorScannedFileId ? "replaced" : "scanned",
+                transactions: counts.transactions,
+                concerns: counts.concerns,
+            });
+        }
+        catch (err) {
+            out.push({
+                name: decryptedFile.fileName,
+                relPath: decryptedFile.relPath,
+                status: "failed",
+                transactions: 0,
+                concerns: buffer.concerns.length,
+                error: err?.message ?? "commit failed",
+            });
+        }
+    }
+    return out;
 }
-export async function runScan(opts = {}) {
-    const matcher = opts.regex ? compileMatcher(opts.regex) : null;
-    const files = scanDataDir().filter(f => (matcher ? matcher.test(f.relPath) : true));
+/** Summary assembly */
+function buildSummary(total, details, _decrypt) {
     const summary = {
-        total: files.length,
+        total,
         scanned: 0,
         replaced: 0,
         skipped: 0,
-        needsInput: 0,
         failed: 0,
-        details: [],
+        concerns: 0,
+        details,
     };
-    for (const f of files) {
-        const result = await scanFile(f.path, opts);
-        summary.details.push({ name: f.name, relPath: f.relPath, result });
-        if (result.status === "scanned")
-            summary.scanned++;
-        else if (result.status === "replaced")
-            summary.replaced++;
-        else if (result.status === "skipped")
-            summary.skipped++;
-        else if (result.status === "needs_input")
-            summary.needsInput++;
-        else if (result.status === "failed")
-            summary.failed++;
+    for (const d of details) {
+        summary[d.status]++;
+        summary.concerns += d.concerns;
     }
     return summary;
 }
+function buildAbortedSummary(total, decrypt) {
+    const details = [
+        ...decrypt.skipped.map(s => ({
+            name: s.file.name, relPath: s.file.relPath, status: "skipped", transactions: 0, concerns: 0,
+        })),
+        ...decrypt.failed.map(f => ({
+            name: f.file.name, relPath: f.file.relPath, status: "failed", transactions: 0, concerns: 0, error: f.error,
+        })),
+    ];
+    return buildSummary(total, details, decrypt);
+}
+/** Low-level DB helpers */
+function deleteScannedFile(db, id) {
+    db.prepare(`DELETE FROM scanned_files WHERE id = ?`).run(id);
+}
+function insertScannedFile(db, args) {
+    const id = `sf:${randomUUID()}`;
+    db.prepare(`INSERT INTO scanned_files (id, path, file_hash, mime, status)
+     VALUES (?, ?, ?, ?, 'pending')`).run(id, args.path, args.hash, args.mime);
+    return id;
+}
+function setFileStatus(db, id, status, fields = {}) {
+    db.prepare(`UPDATE scanned_files
+     SET status = ?, scanned_at = datetime('now'), error = ?, raw_text = COALESCE(?, raw_text)
+     WHERE id = ?`).run(status, fields.error ?? null, fields.raw_text ?? null, id);
+}

package/dist/scanner/prompts.js

@@ -11,10 +11,10 @@ export function buildScanUserMessage(opts) {
         `Steps:`,
         `1. Call list_accounts to see what already exists.`,
         `2. Infer the primary account type (asset / liability / income / expense) from the document's header, account type field, and transaction patterns.`,
-        `3. If this document references an account that isn't yet in the chart, call create_account once. Mask the account number to the last 4 digits.`,
+        `3. If this document references an account that isn't yet in the chart, call create_account once (pass parent_id under the matching top-level type root). Mask the account number to the last 4 digits.`,
         `4. Persist any document-level metadata you find (statement_day, due_day, points_balance, etc.) using update_account_metadata.`,
-        `5. For every transaction in the document, call record_journal_entry with balanced debit/credit lines. Use existing accounts where possible; create expense/income accounts as needed.`,
-        `6. If a row is ambiguous, call ask_user before guessing.`,
+        `5. For every transaction in the document, call record_transaction with balanced debit/credit postings. Attach a merchant block (canonical_name + alias + default_account_id when categorization is confident) for any external counter-party. Reuse existing accounts; create expense categories under their parent (e.g. expense:food before expense:food:groceries) as needed. When you cannot categorize confidently, post the expense side to expense:uncategorized and call note_concern with kind="uncategorized_expense".`,
+        `6. Never pause to ask the user. If a row is ambiguous, post your best-guess transaction first, then call note_concern with details and the new transaction_id. If a row is truly unparseable, skip it and call note_concern with the raw row text (no transaction_id). A missing row is better than a wrong row.`,
         `7. When you are done, call mark_file_scanned with a short summary.`,
     ].join("\n");
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "plasalid",
-  "version": "0.3.5",
-  "description": "A local-first AI that reads every line of your transactions and coaches you the best move.",
+  "version": "0.5.0",
+  "description": "Plasalid — The Harness for Personal Finance",
   "keywords": [
     "finance",
     "personal-finance",