plasalid 0.7.0 → 0.7.2

package/dist/scanner/buffer.d.ts

@@ -1,51 +1,58 @@
-import type Database from "libsql";
-import { type TransactionInput } from "../db/queries/transactions.js";
+import type { TransactionInput } from "../db/queries/transactions.js";
+import type { Bus } from "./bus.js";
 /**
- * One scan agent's pending writes. Transactions and unknowns accumulate here
- * while the LLM works; nothing hits the DB until `commit()` runs inside a
- * single SQLite transaction. If `commit()` throws, the transaction rolls back
- * and the DB stays exactly as it was before this file's scan began.
- *
- * Account writes (`create_account`, `update_account_metadata`) and merchant
- * writes deliberately bypass the buffer — they go directly to the DB through
- * their own mutexes so concurrent agents see each other's creates and don't
- * duplicate.
+ * One in-flight transaction held by the buffer. The id is synthesized on
+ * append so the agent can refer to it in the same turn (e.g. to attach a
+ * note_unknown). `chunkId` records who created it — audit passes use this
+ * to dedup across chunks and merge boundary continuations.
  */
+export interface BufferedTransaction {
+    readonly transaction_id: string;
+    readonly chunkId: string;
+    input: TransactionInput;
+}
 export interface BufferedUnknown {
-    /** Synthesized when the LLM called note_unknown with a buffered transaction_id. */
+    readonly unknown_id: string;
+    readonly chunkId: string | null;
     transaction_id: string | null;
     account_id: string | null;
-    kind?: string | null;
+    kind: string | null;
     prompt: string;
     options?: string[];
+    /** Null until the auditor or review TUI closes it. */
+    answer: string | null;
 }
-export interface BufferedTransaction {
-    /** Synthesized at queue-time so unknowns can reference this transaction. */
-    transaction_id: string;
-    input: TransactionInput;
+export interface BufferSnapshot {
+    readonly transactions: readonly BufferedTransaction[];
+    readonly unknowns: readonly BufferedUnknown[];
+    readonly accountsCreated: readonly string[];
+    readonly merchantsCreated: readonly string[];
+    readonly fileIds: readonly string[];
 }
-export declare class BufferedWriteContext {
-    readonly fileName: string;
-    readonly transactions: BufferedTransaction[];
-    readonly unknowns: BufferedUnknown[];
-    doneSummary: string | null;
-    constructor(fileName: string);
-    /**
-     * Queue a transaction. Returns the synthesized transaction id so the agent
-     * can use it in subsequent note_unknown calls inside the same file.
-     */
-    appendTransaction(input: TransactionInput): string;
-    appendUnknown(unknown: BufferedUnknown): void;
-    markDone(summary: string): void;
-    get isDone(): boolean;
-    /**
-     * Replay all buffered writes inside one DB transaction. `scannedFileId` is
-     * stamped onto every transaction and unknown so they're attributable to this
-     * file. Returns `{ transactions, unknowns }` counts so the caller can report
-     * them.
-     */
-    commit(db: Database.Database, scannedFileId: string): {
+/**
+ * The shared buffer every chunk worker writes to during a scan run. Mutations
+ * serialize through an internal mutex; reads are lock-free. Every mutation
+ * publishes a typed event on the bus so the auditor can react in flight.
+ */
+export interface ScanBuffer {
+    readonly scanId: string;
+    appendTransaction(input: TransactionInput, chunkId: string): Promise<string>;
+    updateTransaction(id: string, mut: (current: BufferedTransaction) => BufferedTransaction): Promise<boolean>;
+    removeTransaction(id: string, reason: string): Promise<boolean>;
+    appendUnknown(input: Omit<BufferedUnknown, "unknown_id" | "answer">): Promise<string>;
+    closeUnknown(id: string, answer: string): Promise<boolean>;
+    /** Returns true if newly recorded; false if already present. */
+    recordFile(fileId: string): boolean;
+    recordAccountCreated(accountId: string): boolean;
+    recordMerchantCreated(merchantId: string): boolean;
+    snapshot(): BufferSnapshot;
+    size(): {
         transactions: number;
         unknowns: number;
+        openUnknowns: number;
     };
+    getTransaction(id: string): BufferedTransaction | undefined;
+    filterTransactions(predicate: (tx: BufferedTransaction) => boolean): IterableIterator<BufferedTransaction>;
+    openUnknowns(): BufferedUnknown[];
 }
+export declare function createBuffer(scanId: string, bus: Bus): ScanBuffer;

package/dist/scanner/buffer.js

@@ -1,63 +1,95 @@
 import { randomUUID } from "crypto";
-import { insertTransactionRows, validateTransaction, } from "../db/queries/transactions.js";
-import { recordUnknown } from "../db/queries/unknowns.js";
-export class BufferedWriteContext {
-    fileName;
-    transactions = [];
-    unknowns = [];
-    doneSummary = null;
-    constructor(fileName) {
-        this.fileName = fileName;
-    }
-    /**
-     * Queue a transaction. Returns the synthesized transaction id so the agent
-     * can use it in subsequent note_unknown calls inside the same file.
-     */
-    appendTransaction(input) {
-        const transactionId = `tx:${randomUUID()}`;
-        this.transactions.push({ transaction_id: transactionId, input });
-        return transactionId;
-    }
-    appendUnknown(unknown) {
-        this.unknowns.push(unknown);
-    }
-    markDone(summary) {
-        this.doneSummary = summary;
-    }
-    get isDone() {
-        return this.doneSummary !== null;
-    }
-    /**
-     * Replay all buffered writes inside one DB transaction. `scannedFileId` is
-     * stamped onto every transaction and unknown so they're attributable to this
-     * file. Returns `{ transactions, unknowns }` counts so the caller can report
-     * them.
-     */
-    commit(db, scannedFileId) {
-        const validated = this.transactions.map(b => ({
-            buffered: b,
-            validated: validateTransaction({
-                ...b.input,
-                id: b.transaction_id,
-                source_file_id: scannedFileId,
-            }),
-        }));
-        const tx = db.transaction(() => {
-            for (const { validated: v } of validated) {
-                insertTransactionRows(db, v);
-            }
-            for (const u of this.unknowns) {
-                recordUnknown(db, {
-                    file_id: scannedFileId,
-                    transaction_id: u.transaction_id,
-                    account_id: u.account_id,
-                    kind: u.kind ?? null,
-                    prompt: u.prompt,
-                    options: u.options,
-                });
-            }
-        });
-        tx();
-        return { transactions: this.transactions.length, unknowns: this.unknowns.length };
-    }
+function createMutex() {
+    let chain = Promise.resolve();
+    return {
+        runExclusive(fn) {
+            const next = chain.then(() => fn());
+            chain = next.catch(() => undefined);
+            return next;
+        },
+    };
+}
+function emptyState(scanId) {
+    return {
+        scanId,
+        transactions: new Map(),
+        unknowns: new Map(),
+        accountsCreated: new Set(),
+        merchantsCreated: new Set(),
+        fileIds: new Set(),
+    };
+}
+function addIfNew(set, value) {
+    if (set.has(value))
+        return false;
+    set.add(value);
+    return true;
+}
+export function createBuffer(scanId, bus) {
+    const state = emptyState(scanId);
+    const mutex = createMutex();
+    return {
+        scanId,
+        appendTransaction: (input, chunkId) => mutex.runExclusive(() => {
+            const id = `tx:${randomUUID()}`;
+            const tx = { transaction_id: id, chunkId, input };
+            state.transactions.set(id, tx);
+            bus.publish({ kind: "transaction_appended", transaction: tx, chunkId });
+            return id;
+        }),
+        updateTransaction: (id, mut) => mutex.runExclusive(() => {
+            const before = state.transactions.get(id);
+            if (!before)
+                return false;
+            const after = mut(before);
+            state.transactions.set(id, after);
+            bus.publish({ kind: "transaction_updated", transactionId: id, before, after });
+            return true;
+        }),
+        removeTransaction: (id, reason) => mutex.runExclusive(() => {
+            if (!state.transactions.delete(id))
+                return false;
+            bus.publish({ kind: "transaction_removed", transactionId: id, reason });
+            return true;
+        }),
+        appendUnknown: (input) => mutex.runExclusive(() => {
+            const id = `bu:${randomUUID()}`;
+            const u = { ...input, unknown_id: id, answer: null };
+            state.unknowns.set(id, u);
+            bus.publish({ kind: "unknown_appended", unknown: u, chunkId: u.chunkId });
+            return id;
+        }),
+        closeUnknown: (id, answer) => mutex.runExclusive(() => {
+            const u = state.unknowns.get(id);
+            if (!u || u.answer !== null)
+                return false;
+            state.unknowns.set(id, { ...u, answer });
+            bus.publish({ kind: "unknown_closed", unknownId: id, answer, chunkId: u.chunkId });
+            return true;
+        }),
+        recordFile: (fileId) => addIfNew(state.fileIds, fileId),
+        recordAccountCreated: (accountId) => addIfNew(state.accountsCreated, accountId),
+        recordMerchantCreated: (merchantId) => addIfNew(state.merchantsCreated, merchantId),
+        snapshot: () => ({
+            transactions: Array.from(state.transactions.values()),
+            unknowns: Array.from(state.unknowns.values()),
+            accountsCreated: Array.from(state.accountsCreated),
+            merchantsCreated: Array.from(state.merchantsCreated),
+            fileIds: Array.from(state.fileIds),
+        }),
+        size: () => {
+            let open = 0;
+            for (const u of state.unknowns.values())
+                if (u.answer === null)
+                    open++;
+            return { transactions: state.transactions.size, unknowns: state.unknowns.size, openUnknowns: open };
+        },
+        getTransaction: (id) => state.transactions.get(id),
+        *filterTransactions(predicate) {
+            for (const tx of state.transactions.values())
+                if (predicate(tx))
+                    yield tx;
+        },
+        openUnknowns: () => Array.from(state.unknowns.values()).filter(u => u.answer === null),
+    };
 }

package/dist/scanner/bus/engine.d.ts

@@ -0,0 +1,11 @@
+import type { Bus } from "./types.js";
+/**
+ * Pub/sub bus factory. Closes over a Set of listeners + an event history. No
+ * class, no `this` — just a record of callbacks the scanner engine wires up
+ * once per scan run.
+ *
+ * Errors thrown synchronously by a listener are caught and logged. Rejected
+ * promises from async listeners are also caught. A misbehaving subscriber
+ * never silences the bus for the rest.
+ */
+export declare function createBus(): Bus;

package/dist/scanner/bus/engine.js

@@ -0,0 +1,42 @@
+/**
+ * Pub/sub bus factory. Closes over a Set of listeners + an event history. No
+ * class, no `this` — just a record of callbacks the scanner engine wires up
+ * once per scan run.
+ *
+ * Errors thrown synchronously by a listener are caught and logged. Rejected
+ * promises from async listeners are also caught. A misbehaving subscriber
+ * never silences the bus for the rest.
+ */
+export function createBus() {
+    const listeners = new Set();
+    const history = [];
+    const safelyInvoke = (fn, event) => {
+        try {
+            const result = fn(event);
+            if (result && typeof result.catch === "function") {
+                result.catch(err => console.error(`[bus listener] ${err.message}`));
+            }
+        }
+        catch (err) {
+            console.error(`[bus listener] ${err.message}`);
+        }
+    };
+    return {
+        subscribe(fn) {
+            listeners.add(fn);
+            return () => { listeners.delete(fn); };
+        },
+        publish(event) {
+            history.push(event);
+            for (const fn of listeners)
+                safelyInvoke(fn, event);
+        },
+        history() {
+            return history;
+        },
+        reset() {
+            listeners.clear();
+            history.length = 0;
+        },
+    };
+}

package/dist/scanner/bus/types.d.ts

@@ -0,0 +1,53 @@
+import type { BufferedTransaction, BufferedUnknown } from "../buffer/types.js";
+/**
+ * The typed event stream every subdomain in the scanner communicates over.
+ * The Buffer publishes mutation events; the Auditor and the CLI dashboard
+ * subscribe. Producers and consumers never reference each other directly —
+ * everyone talks through the Bus.
+ */
+export type BufferEvent = {
+    kind: "transaction_appended";
+    transaction: BufferedTransaction;
+    chunkId: string;
+} | {
+    kind: "transaction_updated";
+    transactionId: string;
+    before: BufferedTransaction;
+    after: BufferedTransaction;
+} | {
+    kind: "transaction_removed";
+    transactionId: string;
+    reason: string;
+} | {
+    kind: "unknown_appended";
+    unknown: BufferedUnknown;
+    chunkId: string | null;
+} | {
+    kind: "unknown_closed";
+    unknownId: string;
+    answer: string;
+} | {
+    kind: "chunk_started";
+    chunkId: string;
+    fileId: string;
+    pageNumber: number;
+} | {
+    kind: "chunk_completed";
+    chunkId: string;
+    fileId: string;
+    pageNumber: number;
+} | {
+    kind: "worker_completed";
+    workerId: string;
+    chunkId: string;
+    transactionsAdded: number;
+    unknownsAdded: number;
+};
+export type EventKind = BufferEvent["kind"];
+export type EventListener = (event: BufferEvent) => void | Promise<void>;
+export interface Bus {
+    subscribe(fn: EventListener): () => void;
+    publish(event: BufferEvent): void;
+    history(): readonly BufferEvent[];
+    reset(): void;
+}

package/dist/scanner/bus/types.js

@@ -0,0 +1 @@
+export {};

package/dist/scanner/bus.d.ts

@@ -0,0 +1,38 @@
+import type { BufferedTransaction, BufferedUnknown } from "./buffer.js";
+/**
+ * Typed pub/sub bus every scanner subdomain talks over. The buffer publishes
+ * mutation events; the audit engine and CLI dashboard subscribe. Producers
+ * never reference consumers — everyone goes through the bus.
+ */
+export type BufferEvent = {
+    kind: "transaction_appended";
+    transaction: BufferedTransaction;
+    chunkId: string;
+} | {
+    kind: "transaction_updated";
+    transactionId: string;
+    before: BufferedTransaction;
+    after: BufferedTransaction;
+} | {
+    kind: "transaction_removed";
+    transactionId: string;
+    reason: string;
+} | {
+    kind: "unknown_appended";
+    unknown: BufferedUnknown;
+    chunkId: string | null;
+} | {
+    kind: "unknown_closed";
+    unknownId: string;
+    answer: string;
+    chunkId: string | null;
+};
+export type EventKind = BufferEvent["kind"];
+export type EventListener = (event: BufferEvent) => void | Promise<void>;
+export interface Bus {
+    subscribe(fn: EventListener): () => void;
+    publish(event: BufferEvent): void;
+    history(): readonly BufferEvent[];
+    reset(): void;
+}
+export declare function createBus(): Bus;

package/dist/scanner/bus.js

@@ -0,0 +1,37 @@
+function logListenerError(err) {
+    console.error(`[bus listener] ${err instanceof Error ? err.message : String(err)}`);
+}
+function safelyInvoke(fn, event) {
+    try {
+        const result = fn(event);
+        if (result instanceof Promise)
+            result.catch(logListenerError);
+    }
+    catch (err) {
+        logListenerError(err);
+    }
+}
+export function createBus() {
+    const listeners = new Set();
+    const history = [];
+    return {
+        subscribe(fn) {
+            listeners.add(fn);
+            return () => {
+                listeners.delete(fn);
+            };
+        },
+        publish(event) {
+            history.push(event);
+            for (const fn of listeners)
+                safelyInvoke(fn, event);
+        },
+        history() {
+            return history;
+        },
+        reset() {
+            listeners.clear();
+            history.length = 0;
+        },
+    };
+}

package/dist/scanner/chunk-worker.d.ts

@@ -0,0 +1,19 @@
+import type Database from "libsql";
+import type { Chunk } from "./engine.js";
+import type { ScanHooks } from "./hooks.js";
+import type { ScanProgress } from "./progress.js";
+export interface ChunkWorkerDeps {
+    readonly db: Database.Database;
+    readonly scanId: string;
+    readonly scannedFileId: string | undefined;
+    readonly progress: ScanProgress;
+    readonly chunk: Chunk;
+}
+/**
+ * Process one chunk: run the LLM scan agent over a single-page PDF blob with
+ * scanId + progress sink + scanned_files row injected through the agent
+ * context. Agent's record_transactions / note_unknown calls write directly to
+ * the DB; per-row ticks fan out via `progress.emit`. Failures land in the DB
+ * as a `chunk_failed` unknown so the resolver can pick them up.
+ */
+export declare function runChunkWorker(deps: ChunkWorkerDeps, hooks: ScanHooks): Promise<void>;

package/dist/scanner/chunk-worker.js

@@ -0,0 +1,67 @@
+import { randomUUID } from "crypto";
+import { runScanAgent } from "../ai/agent.js";
+import { recordUnknown } from "../db/queries/unknowns.js";
+import { buildDocumentBlock } from "./pdf/pdf.js";
+import { tryExecute } from "./result.js";
+/**
+ * Process one chunk: run the LLM scan agent over a single-page PDF blob with
+ * scanId + progress sink + scanned_files row injected through the agent
+ * context. Agent's record_transactions / note_unknown calls write directly to
+ * the DB; per-row ticks fan out via `progress.emit`. Failures land in the DB
+ * as a `chunk_failed` unknown so the resolver can pick them up.
+ */
+export async function runChunkWorker(deps, hooks) {
+    const workerId = `cw:${randomUUID()}`;
+    hooks.onWorkerStart?.(workerId, deps.chunk);
+    const outcome = await tryExecute(() => runScanAgent({
+        db: deps.db,
+        initialMessages: [
+            {
+                role: "user",
+                content: [
+                    buildDocumentBlock(deps.chunk.bytes, deps.chunk.fileName, deps.chunk.mime),
+                    { type: "text", text: buildChunkPrompt(deps.chunk) },
+                ],
+            },
+        ],
+        prompt: { fileName: deps.chunk.fileName },
+        agentCtx: {
+            interactive: false,
+            scanId: deps.scanId,
+            fileId: deps.scannedFileId,
+            chunkId: deps.chunk.chunkId,
+            progress: deps.progress,
+        },
+    }));
+    hooks.onWorkerEnd?.(workerId, deps.chunk, outcome.ok);
+    if (!outcome.ok)
+        recordChunkFailure(deps, outcome.error);
+}
+function recordChunkFailure(deps, error) {
+    try {
+        recordUnknown(deps.db, {
+            file_id: deps.scannedFileId ?? null,
+            scan_id: deps.scanId,
+            transaction_id: null,
+            account_id: null,
+            kind: "chunk_failed",
+            prompt: `Chunk ${deps.chunk.fileName} p${deps.chunk.pageNumber} failed to parse: ${error}.`,
+        });
+        deps.progress.emit({ chunkId: deps.chunk.chunkId, kind: "unknown" });
+    }
+    catch {
+        // failure to record a failure shouldn't crash the file worker
+    }
+}
+function buildChunkPrompt(chunk) {
+    return [
+        `You are parsing page ${chunk.pageNumber} of ${chunk.totalPages} of ${chunk.fileName}.`,
+        ``,
+        `Steps:`,
+        `1. Call list_accounts to see what already exists.`,
+        `2. If this page reveals an account that isn't in the chart yet, call create_account once.`,
+        `3. For every transaction on this page, call record_transactions (plural) with all rows in one batch.`,
+        `4. If the first or last row looks incomplete (no date, or no amount column visible — the row likely continues onto an adjacent page), call note_unknown with kind="boundary_continuation" and the raw row text. Do NOT invent missing fields.`,
+        `5. When done with this page, call mark_file_scanned with a short summary.`,
+    ].join("\n");
+}

package/dist/scanner/chunkWorker.d.ts

@@ -0,0 +1,20 @@
+import type Database from "libsql";
+import type { ScanBuffer } from "./buffer.js";
+import type { Chunk } from "./engine.js";
+import type { ScanHooks } from "./hooks.js";
+export interface ChunkWorkerDeps {
+    readonly db: Database.Database;
+    readonly buffer: ScanBuffer;
+    readonly chunk: Chunk;
+}
+export interface ChunkWorkerResult {
+    readonly workerId: string;
+    readonly ok: boolean;
+    readonly error?: string;
+}
+/**
+ * Process one chunk: run the LLM scan agent over a single-page PDF blob with
+ * the shared buffer + chunkId injected. Agent's `record_transactions` calls
+ * land in the shared buffer; events fan out to the audit engine + dashboard.
+ */
+export declare function runChunkWorker(deps: ChunkWorkerDeps, hooks: ScanHooks): Promise<ChunkWorkerResult>;

package/dist/scanner/chunkWorker.js

@@ -0,0 +1,59 @@
+import { randomUUID } from "crypto";
+import { runScanAgent } from "../ai/agent.js";
+import { buildDocumentBlock } from "./pdf.js";
+import { tryExecute } from "./result.js";
+/**
+ * Process one chunk: run the LLM scan agent over a single-page PDF blob with
+ * the shared buffer + chunkId injected. Agent's `record_transactions` calls
+ * land in the shared buffer; events fan out to the audit engine + dashboard.
+ */
+export async function runChunkWorker(deps, hooks) {
+    const workerId = `cw:${randomUUID()}`;
+    hooks.onWorkerStart?.(workerId, deps.chunk);
+    const outcome = await tryExecute(() => runScanAgent({
+        db: deps.db,
+        initialMessages: [
+            {
+                role: "user",
+                content: [
+                    buildDocumentBlock(deps.chunk.bytes, deps.chunk.fileName, deps.chunk.mime),
+                    { type: "text", text: buildChunkPrompt(deps.chunk) },
+                ],
+            },
+        ],
+        prompt: { fileName: deps.chunk.fileName },
+        agentCtx: {
+            interactive: false,
+            buffer: deps.buffer,
+            chunkId: deps.chunk.chunkId,
+        },
+        onProgress: ev => hooks.onWorkerProgress?.(workerId, deps.chunk, { phase: ev.phase, toolName: ev.toolName }),
+    }));
+    hooks.onWorkerEnd?.(workerId, deps.chunk, outcome.ok);
+    if (!outcome.ok) {
+        await recordChunkFailure(deps.buffer, deps.chunk, outcome.error);
+        return { workerId, ok: false, error: outcome.error };
+    }
+    return { workerId, ok: true };
+}
+async function recordChunkFailure(buffer, chunk, error) {
+    await buffer.appendUnknown({
+        chunkId: chunk.chunkId,
+        transaction_id: null,
+        account_id: null,
+        kind: "chunk_failed",
+        prompt: `Chunk ${chunk.fileName} p${chunk.pageNumber} failed to parse: ${error}.`,
+    });
+}
+function buildChunkPrompt(chunk) {
+    return [
+        `You are parsing page ${chunk.pageNumber} of ${chunk.totalPages} of ${chunk.fileName}.`,
+        ``,
+        `Steps:`,
+        `1. Call list_accounts to see what already exists.`,
+        `2. If this page reveals an account that isn't in the chart yet, call create_account once.`,
+        `3. For every transaction on this page, call record_transactions (plural) with all rows in one batch.`,
+        `4. If the first or last row looks incomplete (no date, or no amount column visible — the row likely continues onto an adjacent page), call note_unknown with kind="boundary_continuation" and the raw row text. Do NOT invent missing fields.`,
+        `5. When done with this page, call mark_file_scanned with a short summary.`,
+    ].join("\n");
+}

package/dist/scanner/chunker/chunker.d.ts

@@ -0,0 +1,7 @@
+import type { Chunk, DecryptedFile } from "../engine/types.js";
+/**
+ * Split one decrypted PDF into N single-page Chunks. Each chunk is a
+ * standalone, valid PDF so the per-chunk LLM agent gets a clean document
+ * without siblings.
+ */
+export declare function chunkPdf(file: DecryptedFile): Promise<Chunk[]>;