plasalid 0.5.8 → 0.6.1

package/dist/scanner/inspectors/similarities.js

@@ -0,0 +1,73 @@
+import { findSimilarAccounts } from "../../db/queries/account-balance.js";
+/**
+ * Flag pairs of accounts whose names are near-identical (Levenshtein ≥ 0.85).
+ * Runs whenever a scan committed at least one transaction — the assumption is
+ * that the scanner may have created a new account this run, so it's worth a
+ * fresh similarity sweep. Idempotent against existing open unknowns: a pair
+ * already flagged is not flagged again. The resolver applies "Merge A into B"
+ * via merge_accounts.
+ */
+function inspect(db, scope) {
+    if (scope.fileIds.length === 0)
+        return [];
+    const pairs = findSimilarAccounts(db);
+    if (pairs.length === 0)
+        return [];
+    const alreadyFlagged = loadAlreadyFlaggedAccountPairs(db);
+    const out = [];
+    for (const pair of pairs) {
+        const key = pairKey(pair.a.id, pair.b.id);
+        if (alreadyFlagged.has(key))
+            continue;
+        out.push({
+            file_id: null,
+            transaction_id: null,
+            account_id: pair.a.id,
+            kind: "similar_accounts",
+            prompt: `These two accounts look like the same thing (similarity ${pair.similarity}):\n  ${pair.a.id} — ${pair.a.name}\n  ${pair.b.id} — ${pair.b.name}`,
+            options: [
+                `Merge ${pair.b.id} into ${pair.a.id}`,
+                `Merge ${pair.a.id} into ${pair.b.id}`,
+                "Keep separate",
+                "Skip",
+            ],
+        });
+    }
+    return out;
+}
+/**
+ * `similar_accounts` unknowns (open OR resolved) embed the other account's id
+ * in their options strings ("Merge X into Y"). Parse those out so we don't
+ * re-flag a pair the user has already seen — including pairs they've already
+ * answered "Keep separate" on a prior run.
+ */
+function loadAlreadyFlaggedAccountPairs(db) {
+    const rows = db
+        .prepare(`SELECT account_id, options_json FROM unknowns
+       WHERE kind = 'similar_accounts' AND account_id IS NOT NULL`)
+        .all();
+    const out = new Set();
+    for (const row of rows) {
+        if (!row.options_json)
+            continue;
+        try {
+            const options = JSON.parse(row.options_json);
+            for (const opt of options) {
+                const match = opt.match(/Merge (\S+) into (\S+)/);
+                if (match)
+                    out.add(pairKey(match[1], match[2]));
+            }
+        }
+        catch {
+            // skip malformed options_json
+        }
+    }
+    return out;
+}
+function pairKey(a, b) {
+    return [a, b].sort().join("|");
+}
+export const similarAccountsInspector = {
+    name: "similar_accounts",
+    inspect,
+};

package/dist/scanner/inspectors/types.d.ts

@@ -0,0 +1,16 @@
+import type Database from "libsql";
+import type { RecordUnknownInput } from "../../db/queries/unknowns.js";
+/**
+ * Scope passed to every inspector by the scanner's Phase 5. Inspectors emit
+ * unknowns for transactions whose `source_file_id` is in `fileIds` (or for
+ * cross-pair findings where at least one side lives in that set). Inspectors
+ * are free to read the wider DB for context — the scope is a filter for what
+ * to surface, not a limit on what to read.
+ */
+export interface InspectorScope {
+    readonly fileIds: readonly string[];
+}
+export interface Inspector {
+    readonly name: string;
+    inspect(db: Database.Database, scope: InspectorScope): RecordUnknownInput[];
+}

package/dist/scanner/inspectors/types.js

@@ -0,0 +1 @@
+export {};

package/dist/scanner/pipeline.d.ts

@@ -1,10 +1,11 @@
+import { type InspectionRunResult } from "./inspectors/index.js";
 export type ScanFileStatus = "scanned" | "replaced" | "failed" | "skipped";
 export interface ScanFileResult {
     name: string;
     relPath: string;
     status: ScanFileStatus;
     transactions: number;
-    concerns: number;
+    unknowns: number;
     error?: string;
 }
 export interface ScanSummary {
@@ -13,7 +14,7 @@ export interface ScanSummary {
     replaced: number;
     skipped: number;
     failed: number;
-    concerns: number;
+    unknowns: number;
     details: ScanFileResult[];
 }
 /** Event hooks the CLI subscribes to. All callbacks are best-effort and ignored if absent. */
@@ -41,11 +42,12 @@ export interface ScanRunEvents {
         fileName: string;
         status: "scanned" | "failed";
         transactions: number;
-        concerns: number;
+        unknowns: number;
         error?: string;
     }) => void;
-    correlating?: (pairs: number) => void;
     committing?: () => void;
+    /** Post-commit inspector pass. `result.total` is the count of unknowns emitted by all inspectors combined. */
+    inspecting?: (result: InspectionRunResult) => void;
 }
 export interface RunScanOptions {
     regex?: string;

package/dist/scanner/pipeline.js

@@ -1,14 +1,14 @@
 import { randomUUID } from "crypto";
 import { getDb } from "../db/connection.js";
-import { countOpenConcerns } from "../db/queries/concerns.js";
-import { correlatePairs } from "../db/queries/transactions.js";
+import { countOpenUnknowns } from "../db/queries/unknowns.js";
+import { runInspectors } from "./inspectors/index.js";
 import { runScanAgent } from "../ai/agent.js";
 import { buildDocumentBlock } from "./pdf.js";
 import { buildScanUserMessage } from "./prompts.js";
 import { scanDataDir } from "./walker.js";
 import { BufferedWriteContext } from "./buffer.js";
 import { runWithConcurrency } from "./concurrency.js";
-import { decryptQueue, confirmProceedAfterFailures, } from "./decrypt_queue.js";
+import { decryptQueue, confirmProceedAfterFailures, } from "./decrypt-queue.js";
 export function compileMatcher(input) {
     return new RegExp(input, "i");
 }
@@ -38,13 +38,35 @@ export async function runScan(opts = {}) {
     }
     // Phase 2 — parallel scan with buffered writes
     const scanResults = await scanInParallel(db, decryptResult.decrypted, { concurrency, events });
-    // Phase 3 — cross-file correlation pre-commit
-    const pairCount = applyCrossFileCorrelations(scanResults);
-    events?.correlating?.(pairCount);
-    // Phase 4 — per-file commit
+    // Phase 3 — per-file commit
     events?.committing?.();
-    const fileResults = commitAll(db, decryptResult, scanResults);
-    return buildSummary(allFiles.length, fileResults);
+    const { details, committedFileIds } = commitAll(db, decryptResult, scanResults);
+    // Phase 4 — post-commit inspector sweep (duplicates, correlations, recurrences, similar accounts)
+    if (committedFileIds.length > 0) {
+        const inspectionResult = runInspectors(db, { fileIds: committedFileIds });
+        events?.inspecting?.(inspectionResult);
+        addInspectionUnknownsToSummary(details, committedFileIds, inspectionResult.total);
+    }
+    return buildSummary(allFiles.length, details);
+}
+/**
+ * Inspector unknowns were inserted after the per-file commit, so the per-file
+ * `unknowns` counters in `details` don't see them. Spread the total across the
+ * files that participated in this run so the summary's `unknowns` line stays
+ * truthful. Distribution is per-file proportional — good enough for a summary,
+ * not a load-bearing fact.
+ */
+function addInspectionUnknownsToSummary(details, committedFileIds, total) {
+    if (total === 0 || committedFileIds.length === 0)
+        return;
+    const scannedDetails = details.filter(d => d.status === "scanned" || d.status === "replaced");
+    if (scannedDetails.length === 0)
+        return;
+    const perFile = Math.floor(total / scannedDetails.length);
+    const remainder = total - perFile * scannedDetails.length;
+    for (let i = 0; i < scannedDetails.length; i++) {
+        scannedDetails[i].unknowns += perFile + (i < remainder ? 1 : 0);
+    }
 }
 async function scanInParallel(db, files, opts) {
     const tasks = files.map(f => () => scanOneFile(db, f, opts.events));
@@ -97,7 +119,7 @@ async function scanOneFile(db, file, events) {
             fileName: file.fileName,
             status: "scanned",
             transactions: buffer.transactions.length,
-            concerns: buffer.concerns.length,
+            unknowns: buffer.unknowns.length,
         });
         return { decryptedFile: file, buffer, agentText: text };
     }
@@ -107,103 +129,43 @@ async function scanOneFile(db, file, events) {
             fileName: file.fileName,
             status: "failed",
             transactions: 0,
-            concerns: 0,
+            unknowns: 0,
             error: message,
         });
         return { decryptedFile: file, buffer, error: message, agentText: "" };
     }
 }
-/** Phase 3: cross-file correlation */
-/**
- * For every pair of buffered entries that look like the same money movement
- * across two different files, append a mirror concern to each side's buffer.
- * Returns the number of pairs detected so the CLI can report it.
- */
-function applyCrossFileCorrelations(results) {
-    const all = [];
-    for (const res of results) {
-        if (res.error)
-            continue;
-        for (const bt of res.buffer.transactions) {
-            all.push({
-                file: res,
-                transactionId: bt.transaction_id,
-                postings: bt.input.postings,
-                date: bt.input.date,
-                description: bt.input.description,
-            });
-        }
-    }
-    const candidates = all.map(e => {
-        const debit = e.postings.reduce((s, p) => s + (p.debit ?? 0), 0);
-        const currency = e.postings.find(p => p.currency)?.currency ?? "THB";
-        const ids = Array.from(new Set(e.postings.map(p => p.account_id)));
-        return {
-            id: e.transactionId,
-            date: e.date,
-            description: e.description,
-            amount: Math.round(debit * 100) / 100,
-            currency,
-            account_ids: ids,
-            account_names: ids,
-        };
-    });
-    const pairs = correlatePairs(candidates, { toleranceDays: 3 });
-    const byTransaction = new Map(all.map(a => [a.transactionId, a]));
-    for (const pair of pairs) {
-        const a = byTransaction.get(pair.a.id);
-        const b = byTransaction.get(pair.b.id);
-        if (!a || !b)
-            continue;
-        if (a.file === b.file)
-            continue;
-        const amountStr = `฿${pair.amount.toLocaleString("en-US", { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`;
-        a.file.buffer.appendConcern({
-            transaction_id: a.transactionId,
-            account_id: null,
-            prompt: `Looks like the matching half of this ${amountStr} movement on ${a.date} was also recorded in ${b.file.decryptedFile.fileName} on ${b.date}. Merge during review?`,
-            options: ["Yes — merge into one transaction", "No — these are two real events", "Skip — leave as is"],
-        });
-        b.file.buffer.appendConcern({
-            transaction_id: b.transactionId,
-            account_id: null,
-            prompt: `Looks like the matching half of this ${amountStr} movement on ${b.date} was also recorded in ${a.file.decryptedFile.fileName} on ${a.date}. Merge during review?`,
-            options: ["Yes — merge into one transaction", "No — these are two real events", "Skip — leave as is"],
-        });
-    }
-    return pairs.filter(p => byTransaction.get(p.a.id)?.file !== byTransaction.get(p.b.id)?.file).length;
-}
-/** Phase 4: commit */
 function commitAll(db, decryptResult, scanResults) {
-    const out = [];
+    const details = [];
+    const committedFileIds = [];
     for (const skipped of decryptResult.skipped) {
-        out.push({
+        details.push({
             name: skipped.file.name,
             relPath: skipped.file.relPath,
             status: "skipped",
             transactions: 0,
-            concerns: countOpenConcerns(db, { file_id: skipped.existingScannedFileId }),
+            unknowns: countOpenUnknowns(db, { file_id: skipped.existingScannedFileId }),
         });
     }
     for (const failed of decryptResult.failed) {
-        out.push({
+        details.push({
             name: failed.file.name,
             relPath: failed.file.relPath,
             status: "failed",
             transactions: 0,
-            concerns: 0,
+            unknowns: 0,
             error: failed.error,
         });
     }
     for (const res of scanResults) {
         const { decryptedFile, buffer, error, agentText } = res;
         if (error) {
-            out.push({
+            details.push({
                 name: decryptedFile.fileName,
                 relPath: decryptedFile.relPath,
                 status: "failed",
                 transactions: 0,
-                concerns: buffer.concerns.length,
+                unknowns: buffer.unknowns.length,
                 error,
             });
             continue;
@@ -219,26 +181,27 @@ function commitAll(db, decryptResult, scanResults) {
             });
             const counts = buffer.commit(db, scannedFileId);
             setFileStatus(db, scannedFileId, "scanned", { raw_text: agentText });
-            out.push({
+            committedFileIds.push(scannedFileId);
+            details.push({
                 name: decryptedFile.fileName,
                 relPath: decryptedFile.relPath,
                 status: decryptedFile.replacesPriorScannedFileId ? "replaced" : "scanned",
                 transactions: counts.transactions,
-                concerns: counts.concerns,
+                unknowns: counts.unknowns,
             });
         }
         catch (err) {
-            out.push({
+            details.push({
                 name: decryptedFile.fileName,
                 relPath: decryptedFile.relPath,
                 status: "failed",
                 transactions: 0,
-                concerns: buffer.concerns.length,
+                unknowns: buffer.unknowns.length,
                 error: err?.message ?? "commit failed",
             });
         }
     }
-    return out;
+    return { details, committedFileIds };
 }
 /** Summary assembly */
 function buildSummary(total, details) {
@@ -248,22 +211,22 @@ function buildSummary(total, details) {
         replaced: 0,
         skipped: 0,
         failed: 0,
-        concerns: 0,
+        unknowns: 0,
         details,
     };
     for (const d of details) {
         summary[d.status]++;
-        summary.concerns += d.concerns;
+        summary.unknowns += d.unknowns;
     }
     return summary;
 }
 function buildAbortedSummary(total, decrypt) {
     const details = [
         ...decrypt.skipped.map(s => ({
-            name: s.file.name, relPath: s.file.relPath, status: "skipped", transactions: 0, concerns: 0,
+            name: s.file.name, relPath: s.file.relPath, status: "skipped", transactions: 0, unknowns: 0,
         })),
         ...decrypt.failed.map(f => ({
-            name: f.file.name, relPath: f.file.relPath, status: "failed", transactions: 0, concerns: 0, error: f.error,
+            name: f.file.name, relPath: f.file.relPath, status: "failed", transactions: 0, unknowns: 0, error: f.error,
         })),
     ];
     return buildSummary(total, details);

package/dist/scanner/prompts.js

@@ -13,8 +13,8 @@ export function buildScanUserMessage(opts) {
         `2. Infer the primary account type (asset / liability / income / expense) from the document's header, account type field, and transaction patterns.`,
         `3. If this document references an account that isn't yet in the chart, call create_account once (pass parent_id under the matching top-level type root). Mask the account number to the last 4 digits.`,
         `4. Persist any document-level metadata you find (statement_day, due_day, points_balance, etc.) using update_account_metadata.`,
-        `5. For every transaction in the document, call record_transaction with balanced debit/credit postings. Attach a merchant block (canonical_name + alias + default_account_id when categorization is confident) for any external counter-party. Reuse existing accounts; create expense categories under their parent (e.g. expense:food before expense:food:groceries) as needed. When you cannot categorize confidently, post the expense side to expense:uncategorized and call note_concern with kind="uncategorized_expense".`,
-        `6. Never pause to ask the user. If a row is ambiguous, post your best-guess transaction first, then call note_concern with details and the new transaction_id. If a row is truly unparseable, skip it and call note_concern with the raw row text (no transaction_id). A missing row is better than a wrong row.`,
+        `5. For every transaction in the document, call record_transaction with balanced debit/credit postings. Attach a merchant block (canonical_name + alias + default_account_id when categorization is confident) for any external counter-party. Reuse existing accounts; create expense categories under their parent (e.g. expense:food before expense:food:groceries) as needed. When you cannot categorize confidently, post the expense side to expense:uncategorized and call note_unknown with kind="uncategorized_expense".`,
+        `6. Never pause to ask the user. If a row is ambiguous, post your best-guess transaction first, then call note_unknown with details and the new transaction_id. If a row is truly unparseable, skip it and call note_unknown with the raw row text (no transaction_id). A missing row is better than a wrong row.`,
         `7. When you are done, call mark_file_scanned with a short summary.`,
     ].join("\n");
 }

package/package.json

@@ -1,9 +1,10 @@
 {
   "name": "plasalid",
-  "version": "0.5.8",
-  "description": "Plasalid — AI Harness for Personal Finance",
+  "version": "0.6.1",
+  "description": "Plasalid — The Harness Layer for Personal Finance",
   "keywords": [
     "finance",
+    "harness",
     "personal-finance",
     "aggregator",
     "parser",