plasalid 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -4
- package/dist/ai/agent.d.ts +6 -7
- package/dist/ai/agent.js +27 -11
- package/dist/ai/personas.js +48 -46
- package/dist/ai/system-prompt.js +1 -1
- package/dist/ai/tools/account-mutex.d.ts +1 -0
- package/dist/ai/tools/account-mutex.js +16 -0
- package/dist/ai/tools/index.js +4 -12
- package/dist/ai/tools/ingest.d.ts +1 -1
- package/dist/ai/tools/ingest.js +282 -242
- package/dist/ai/tools/merchants.js +1 -28
- package/dist/ai/tools/read.js +8 -8
- package/dist/ai/tools/record.js +3 -36
- package/dist/ai/tools/resolve.js +25 -22
- package/dist/ai/tools/scan.js +0 -1
- package/dist/ai/tools/types.d.ts +14 -21
- package/dist/cli/commands/record.js +1 -82
- package/dist/cli/commands/resolve.d.ts +5 -2
- package/dist/cli/commands/resolve.js +36 -5
- package/dist/cli/commands/revert.js +4 -2
- package/dist/cli/commands/rules.js +2 -2
- package/dist/cli/commands/scan.js +199 -128
- package/dist/cli/commands/status.js +5 -5
- package/dist/cli/index.js +8 -29
- package/dist/cli/ink/ScanDashboard.d.ts +49 -0
- package/dist/cli/ink/ScanDashboard.js +214 -0
- package/dist/cli/ink/scan_dashboard.d.ts +40 -25
- package/dist/cli/ink/scan_dashboard.js +139 -44
- package/dist/db/queries/account-balance.d.ts +1 -1
- package/dist/db/queries/questions.d.ts +62 -0
- package/dist/db/queries/questions.js +110 -0
- package/dist/db/queries/transactions.d.ts +1 -1
- package/dist/db/queries/unknowns.d.ts +17 -15
- package/dist/db/queries/unknowns.js +35 -39
- package/dist/db/schema.js +6 -28
- package/dist/scanner/audit/auditor.d.ts +31 -0
- package/dist/scanner/audit/auditor.js +72 -0
- package/dist/scanner/audit/engine.d.ts +10 -0
- package/dist/scanner/audit/engine.js +98 -0
- package/dist/scanner/audit/eventBus.d.ts +60 -0
- package/dist/scanner/audit/eventBus.js +35 -0
- package/dist/scanner/audit/passes/index.d.ts +11 -0
- package/dist/scanner/audit/passes/index.js +9 -0
- package/dist/scanner/audit/passes/types.d.ts +23 -0
- package/dist/scanner/audit/passes/types.js +1 -0
- package/dist/scanner/audit/types.d.ts +27 -0
- package/dist/scanner/audit/types.js +1 -0
- package/dist/scanner/auditor.d.ts +51 -0
- package/dist/scanner/auditor.js +80 -0
- package/dist/scanner/buffer/engine.d.ts +9 -0
- package/dist/scanner/buffer/engine.js +110 -0
- package/dist/scanner/buffer/sharedBuffer.d.ts +78 -0
- package/dist/scanner/buffer/sharedBuffer.js +130 -0
- package/dist/scanner/buffer/types.d.ts +67 -0
- package/dist/scanner/buffer/types.js +1 -0
- package/dist/scanner/buffer.d.ts +45 -38
- package/dist/scanner/buffer.js +93 -61
- package/dist/scanner/bus/engine.d.ts +11 -0
- package/dist/scanner/bus/engine.js +42 -0
- package/dist/scanner/bus/types.d.ts +53 -0
- package/dist/scanner/bus/types.js +1 -0
- package/dist/scanner/bus.d.ts +38 -0
- package/dist/scanner/bus.js +37 -0
- package/dist/scanner/chunk-worker.d.ts +19 -0
- package/dist/scanner/chunk-worker.js +67 -0
- package/dist/scanner/chunkWorker.d.ts +20 -0
- package/dist/scanner/chunkWorker.js +59 -0
- package/dist/scanner/chunker/chunker.d.ts +7 -0
- package/dist/scanner/chunker/chunker.js +60 -0
- package/dist/scanner/chunker.d.ts +7 -0
- package/dist/scanner/chunker.js +60 -0
- package/dist/scanner/converge.d.ts +29 -0
- package/dist/scanner/converge.js +15 -0
- package/dist/scanner/decrypt.d.ts +10 -0
- package/dist/scanner/decrypt.js +80 -0
- package/dist/scanner/engine/scanEngine.d.ts +24 -0
- package/dist/scanner/engine/scanEngine.js +87 -0
- package/dist/scanner/engine/types.d.ts +90 -0
- package/dist/scanner/engine/types.js +1 -0
- package/dist/scanner/engine.d.ts +90 -0
- package/dist/scanner/engine.js +84 -0
- package/dist/scanner/file-worker.d.ts +33 -0
- package/dist/scanner/file-worker.js +28 -0
- package/dist/scanner/fileWorker.d.ts +33 -0
- package/dist/scanner/fileWorker.js +22 -0
- package/dist/scanner/hooks/types.d.ts +25 -0
- package/dist/scanner/hooks/types.js +1 -0
- package/dist/scanner/hooks.d.ts +23 -0
- package/dist/scanner/hooks.js +1 -0
- package/dist/scanner/parse.d.ts +10 -0
- package/dist/scanner/parse.js +47 -0
- package/dist/scanner/passes/index.d.ts +8 -0
- package/dist/scanner/passes/index.js +6 -0
- package/dist/scanner/passes/types.d.ts +22 -0
- package/dist/scanner/passes/types.js +1 -0
- package/dist/scanner/pdf/chunker.d.ts +7 -0
- package/dist/scanner/pdf/chunker.js +60 -0
- package/dist/scanner/pdf/password-store.d.ts +34 -0
- package/dist/scanner/pdf/password-store.js +83 -0
- package/dist/scanner/pdf/pdf-unlock.d.ts +17 -0
- package/dist/scanner/pdf/pdf-unlock.js +50 -0
- package/dist/scanner/pdf/pdf.d.ts +17 -0
- package/dist/scanner/pdf/pdf.js +36 -0
- package/dist/scanner/pdf/state-machine.d.ts +60 -0
- package/dist/scanner/pdf/state-machine.js +64 -0
- package/dist/scanner/pdf/unlock.d.ts +22 -0
- package/dist/scanner/pdf/unlock.js +121 -0
- package/dist/scanner/phase-decrypt.d.ts +10 -0
- package/dist/scanner/phase-decrypt.js +80 -0
- package/dist/scanner/phase-parse.d.ts +10 -0
- package/dist/scanner/phase-parse.js +46 -0
- package/dist/scanner/phases/chunk.d.ts +8 -0
- package/dist/scanner/phases/chunk.js +13 -0
- package/dist/scanner/phases/commit.d.ts +12 -0
- package/dist/scanner/phases/commit.js +140 -0
- package/dist/scanner/phases/decrypt.d.ts +10 -0
- package/dist/scanner/phases/decrypt.js +80 -0
- package/dist/scanner/phases/parse.d.ts +10 -0
- package/dist/scanner/phases/parse.js +46 -0
- package/dist/scanner/phases/resolve.d.ts +10 -0
- package/dist/scanner/phases/resolve.js +17 -0
- package/dist/scanner/phases/review.d.ts +10 -0
- package/dist/scanner/phases/review.js +12 -0
- package/dist/scanner/progress.d.ts +14 -0
- package/dist/scanner/progress.js +21 -0
- package/dist/scanner/resolver-memory.d.ts +8 -0
- package/dist/scanner/resolver-memory.js +24 -0
- package/dist/scanner/resolver.d.ts +39 -0
- package/dist/scanner/resolver.js +196 -0
- package/dist/scanner/result.d.ts +17 -0
- package/dist/scanner/result.js +19 -0
- package/dist/scanner/run-passes.d.ts +30 -0
- package/dist/scanner/run-passes.js +15 -0
- package/dist/scanner/unlock.js +1 -1
- package/dist/scanner/worker.d.ts +19 -0
- package/dist/scanner/worker.js +67 -0
- package/dist/scanner/workers/chunkWorker.d.ts +20 -0
- package/dist/scanner/workers/chunkWorker.js +65 -0
- package/dist/scanner/workers/fileWorker.d.ts +32 -0
- package/dist/scanner/workers/fileWorker.js +22 -0
- package/package.json +1 -1
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { randomUUID } from "crypto";
|
|
2
|
+
/**
|
|
3
|
+
* Insert a new questions row and flip the `has_question` boolean on whichever
|
|
4
|
+
* target (transaction / account) was named. Returns the new id. The id keeps
|
|
5
|
+
* the historical `cn:` prefix — it's opaque and nothing else references it,
|
|
6
|
+
* so the prefix is a no-op detail.
|
|
7
|
+
*/
|
|
8
|
+
export function recordQuestion(db, input) {
|
|
9
|
+
const id = `cn:${randomUUID()}`;
|
|
10
|
+
db.prepare(`INSERT INTO questions (id, scan_id, file_id, transaction_id, account_id, kind, prompt, options_json, context_json)
|
|
11
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, input.scan_id ?? null, input.file_id, input.transaction_id, input.account_id, input.kind ?? null, input.prompt, input.options ? JSON.stringify(input.options) : null, input.context ? JSON.stringify(input.context) : null);
|
|
12
|
+
if (input.transaction_id) {
|
|
13
|
+
db.prepare(`UPDATE transactions SET has_question = 1 WHERE id = ?`).run(input.transaction_id);
|
|
14
|
+
}
|
|
15
|
+
if (input.account_id) {
|
|
16
|
+
db.prepare(`UPDATE accounts SET has_question = 1 WHERE id = ?`).run(input.account_id);
|
|
17
|
+
}
|
|
18
|
+
return id;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Close a question by capturing its (prompt, kind, answer) tuple and
|
|
22
|
+
* deleting the row outright. Returns the captured tuple so callers can
|
|
23
|
+
* synthesize memory rules; returns null when the id doesn't exist.
|
|
24
|
+
*/
|
|
25
|
+
export function closeQuestion(db, id, answer) {
|
|
26
|
+
const row = db
|
|
27
|
+
.prepare(`SELECT prompt, kind, transaction_id, account_id FROM questions WHERE id = ?`)
|
|
28
|
+
.get(id);
|
|
29
|
+
if (!row)
|
|
30
|
+
return null;
|
|
31
|
+
db.prepare(`DELETE FROM questions WHERE id = ?`).run(id);
|
|
32
|
+
maybeClearHasQuestionFlags(db, {
|
|
33
|
+
transaction_id: row.transaction_id,
|
|
34
|
+
account_id: row.account_id,
|
|
35
|
+
});
|
|
36
|
+
return { prompt: row.prompt, kind: row.kind, answer };
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Look up the transaction/account a question is attached to. Returns null when
|
|
40
|
+
* the question id doesn't exist.
|
|
41
|
+
*/
|
|
42
|
+
export function getQuestionTarget(db, id) {
|
|
43
|
+
const row = db
|
|
44
|
+
.prepare(`SELECT transaction_id, account_id FROM questions WHERE id = ?`)
|
|
45
|
+
.get(id);
|
|
46
|
+
return row ?? null;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Clear `has_question` on the named transaction / account if no other
|
|
50
|
+
* questions still reference it. Safe to call after any resolution; idempotent.
|
|
51
|
+
*/
|
|
52
|
+
function maybeClearHasQuestionFlags(db, target) {
|
|
53
|
+
if (target.transaction_id) {
|
|
54
|
+
const open = db
|
|
55
|
+
.prepare(`SELECT 1 FROM questions WHERE transaction_id = ? LIMIT 1`)
|
|
56
|
+
.get(target.transaction_id);
|
|
57
|
+
if (!open)
|
|
58
|
+
db.prepare(`UPDATE transactions SET has_question = 0 WHERE id = ?`).run(target.transaction_id);
|
|
59
|
+
}
|
|
60
|
+
if (target.account_id) {
|
|
61
|
+
const open = db
|
|
62
|
+
.prepare(`SELECT 1 FROM questions WHERE account_id = ? LIMIT 1`)
|
|
63
|
+
.get(target.account_id);
|
|
64
|
+
if (!open)
|
|
65
|
+
db.prepare(`UPDATE accounts SET has_question = 0 WHERE id = ?`).run(target.account_id);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
export function countQuestions(db, scope = {}) {
|
|
69
|
+
const conditions = [];
|
|
70
|
+
const params = [];
|
|
71
|
+
if (scope.file_id) {
|
|
72
|
+
conditions.push("file_id = ?");
|
|
73
|
+
params.push(scope.file_id);
|
|
74
|
+
}
|
|
75
|
+
if (scope.transaction_id) {
|
|
76
|
+
conditions.push("transaction_id = ?");
|
|
77
|
+
params.push(scope.transaction_id);
|
|
78
|
+
}
|
|
79
|
+
if (scope.account_id) {
|
|
80
|
+
conditions.push("account_id = ?");
|
|
81
|
+
params.push(scope.account_id);
|
|
82
|
+
}
|
|
83
|
+
if (scope.kind) {
|
|
84
|
+
conditions.push("kind = ?");
|
|
85
|
+
params.push(scope.kind);
|
|
86
|
+
}
|
|
87
|
+
if (scope.scan_id) {
|
|
88
|
+
conditions.push("scan_id = ?");
|
|
89
|
+
params.push(scope.scan_id);
|
|
90
|
+
}
|
|
91
|
+
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
92
|
+
const row = db
|
|
93
|
+
.prepare(`SELECT COUNT(*) AS n FROM questions ${where}`)
|
|
94
|
+
.get(...params);
|
|
95
|
+
return row.n;
|
|
96
|
+
}
|
|
97
|
+
export function listQuestions(db, opts = {}) {
|
|
98
|
+
const capped = Math.min(Math.max(opts.limit ?? 200, 1), 1000);
|
|
99
|
+
if (opts.scanId) {
|
|
100
|
+
return db.prepare(`SELECT id, scan_id, file_id, transaction_id, account_id, kind, prompt, options_json, context_json, created_at
|
|
101
|
+
FROM questions
|
|
102
|
+
WHERE scan_id = ?
|
|
103
|
+
ORDER BY created_at ASC
|
|
104
|
+
LIMIT ?`).all(opts.scanId, capped);
|
|
105
|
+
}
|
|
106
|
+
return db.prepare(`SELECT id, scan_id, file_id, transaction_id, account_id, kind, prompt, options_json, context_json, created_at
|
|
107
|
+
FROM questions
|
|
108
|
+
ORDER BY created_at ASC
|
|
109
|
+
LIMIT ?`).all(capped);
|
|
110
|
+
}
|
|
@@ -9,7 +9,7 @@ export interface PostingInput {
|
|
|
9
9
|
pii_flag?: boolean;
|
|
10
10
|
}
|
|
11
11
|
export interface TransactionInput {
|
|
12
|
-
/** Optional pre-assigned id. Used by the buffered-write path so
|
|
12
|
+
/** Optional pre-assigned id. Used by the buffered-write path so questions recorded mid-scan can reference the transaction before commit. */
|
|
13
13
|
id?: string;
|
|
14
14
|
date: string;
|
|
15
15
|
description: string;
|
|
@@ -5,6 +5,7 @@ export interface UnknownTarget {
|
|
|
5
5
|
}
|
|
6
6
|
export interface RecordUnknownInput extends UnknownTarget {
|
|
7
7
|
file_id: string | null;
|
|
8
|
+
scan_id?: string | null;
|
|
8
9
|
kind?: string | null;
|
|
9
10
|
prompt: string;
|
|
10
11
|
options?: string[];
|
|
@@ -13,6 +14,7 @@ export interface RecordUnknownInput extends UnknownTarget {
|
|
|
13
14
|
}
|
|
14
15
|
export interface OpenUnknownRow {
|
|
15
16
|
id: string;
|
|
17
|
+
scan_id: string | null;
|
|
16
18
|
file_id: string | null;
|
|
17
19
|
transaction_id: string | null;
|
|
18
20
|
account_id: string | null;
|
|
@@ -22,6 +24,11 @@ export interface OpenUnknownRow {
|
|
|
22
24
|
context_json: string | null;
|
|
23
25
|
created_at: string;
|
|
24
26
|
}
|
|
27
|
+
export interface ClosedUnknown {
|
|
28
|
+
prompt: string;
|
|
29
|
+
kind: string | null;
|
|
30
|
+
answer: string;
|
|
31
|
+
}
|
|
25
32
|
/**
|
|
26
33
|
* Insert a new unknowns row and flip the `has_unknown` boolean on whichever
|
|
27
34
|
* target (transaction / account) was named. Returns the new id. The id keeps
|
|
@@ -30,11 +37,11 @@ export interface OpenUnknownRow {
|
|
|
30
37
|
*/
|
|
31
38
|
export declare function recordUnknown(db: Database.Database, input: RecordUnknownInput): string;
|
|
32
39
|
/**
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
40
|
+
* Close an open unknown by capturing its (prompt, kind, answer) tuple and
|
|
41
|
+
* deleting the row outright. Returns the captured tuple so callers can
|
|
42
|
+
* synthesize memory rules; returns null when the unknown id is unknown.
|
|
36
43
|
*/
|
|
37
|
-
export declare function
|
|
44
|
+
export declare function closeUnknown(db: Database.Database, id: string, answer: string): ClosedUnknown | null;
|
|
38
45
|
/**
|
|
39
46
|
* Look up the transaction/account an unknown is attached to. Returns null when
|
|
40
47
|
* the unknown id doesn't exist.
|
|
@@ -45,16 +52,11 @@ export interface CountOpenUnknownsScope {
|
|
|
45
52
|
transaction_id?: string;
|
|
46
53
|
account_id?: string;
|
|
47
54
|
kind?: string;
|
|
55
|
+
scan_id?: string;
|
|
48
56
|
}
|
|
49
57
|
export declare function countOpenUnknowns(db: Database.Database, scope?: CountOpenUnknownsScope): number;
|
|
50
|
-
export
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
*
|
|
56
|
-
* `kind` is free-text TEXT in the schema; canonical values used by built-ins:
|
|
57
|
-
* uncategorized, duplicate, correlation, recurrence_candidate,
|
|
58
|
-
* similar_accounts, file_password
|
|
59
|
-
*/
|
|
60
|
-
export declare function listOpenUnknownsByKind(db: Database.Database, kinds: string[], limit?: number): OpenUnknownRow[];
|
|
58
|
+
export interface ListOpenUnknownsOptions {
|
|
59
|
+
limit?: number;
|
|
60
|
+
scanId?: string;
|
|
61
|
+
}
|
|
62
|
+
export declare function listOpenUnknowns(db: Database.Database, opts?: ListOpenUnknownsOptions): OpenUnknownRow[];
|
|
@@ -7,7 +7,8 @@ import { randomUUID } from "crypto";
|
|
|
7
7
|
*/
|
|
8
8
|
export function recordUnknown(db, input) {
|
|
9
9
|
const id = `cn:${randomUUID()}`;
|
|
10
|
-
db.prepare(`INSERT INTO unknowns (id, file_id, transaction_id, account_id, kind, prompt, options_json, context_json)
|
|
10
|
+
db.prepare(`INSERT INTO unknowns (id, scan_id, file_id, transaction_id, account_id, kind, prompt, options_json, context_json)
|
|
11
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, input.scan_id ?? null, input.file_id, input.transaction_id, input.account_id, input.kind ?? null, input.prompt, input.options ? JSON.stringify(input.options) : null, input.context ? JSON.stringify(input.context) : null);
|
|
11
12
|
if (input.transaction_id) {
|
|
12
13
|
db.prepare(`UPDATE transactions SET has_unknown = 1 WHERE id = ?`).run(input.transaction_id);
|
|
13
14
|
}
|
|
@@ -17,17 +18,22 @@ export function recordUnknown(db, input) {
|
|
|
17
18
|
return id;
|
|
18
19
|
}
|
|
19
20
|
/**
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
21
|
+
* Close an open unknown by capturing its (prompt, kind, answer) tuple and
|
|
22
|
+
* deleting the row outright. Returns the captured tuple so callers can
|
|
23
|
+
* synthesize memory rules; returns null when the unknown id is unknown.
|
|
23
24
|
*/
|
|
24
|
-
export function
|
|
25
|
-
const
|
|
26
|
-
|
|
25
|
+
export function closeUnknown(db, id, answer) {
|
|
26
|
+
const row = db
|
|
27
|
+
.prepare(`SELECT prompt, kind, transaction_id, account_id FROM unknowns WHERE id = ?`)
|
|
28
|
+
.get(id);
|
|
29
|
+
if (!row)
|
|
27
30
|
return null;
|
|
28
|
-
db.prepare(`
|
|
29
|
-
maybeClearHasUnknownFlags(db,
|
|
30
|
-
|
|
31
|
+
db.prepare(`DELETE FROM unknowns WHERE id = ?`).run(id);
|
|
32
|
+
maybeClearHasUnknownFlags(db, {
|
|
33
|
+
transaction_id: row.transaction_id,
|
|
34
|
+
account_id: row.account_id,
|
|
35
|
+
});
|
|
36
|
+
return { prompt: row.prompt, kind: row.kind, answer };
|
|
31
37
|
}
|
|
32
38
|
/**
|
|
33
39
|
* Look up the transaction/account an unknown is attached to. Returns null when
|
|
@@ -46,21 +52,21 @@ export function getUnknownTarget(db, id) {
|
|
|
46
52
|
function maybeClearHasUnknownFlags(db, target) {
|
|
47
53
|
if (target.transaction_id) {
|
|
48
54
|
const open = db
|
|
49
|
-
.prepare(`SELECT 1 FROM unknowns WHERE transaction_id = ?
|
|
55
|
+
.prepare(`SELECT 1 FROM unknowns WHERE transaction_id = ? LIMIT 1`)
|
|
50
56
|
.get(target.transaction_id);
|
|
51
57
|
if (!open)
|
|
52
58
|
db.prepare(`UPDATE transactions SET has_unknown = 0 WHERE id = ?`).run(target.transaction_id);
|
|
53
59
|
}
|
|
54
60
|
if (target.account_id) {
|
|
55
61
|
const open = db
|
|
56
|
-
.prepare(`SELECT 1 FROM unknowns WHERE account_id = ?
|
|
62
|
+
.prepare(`SELECT 1 FROM unknowns WHERE account_id = ? LIMIT 1`)
|
|
57
63
|
.get(target.account_id);
|
|
58
64
|
if (!open)
|
|
59
65
|
db.prepare(`UPDATE accounts SET has_unknown = 0 WHERE id = ?`).run(target.account_id);
|
|
60
66
|
}
|
|
61
67
|
}
|
|
62
68
|
export function countOpenUnknowns(db, scope = {}) {
|
|
63
|
-
const conditions = [
|
|
69
|
+
const conditions = [];
|
|
64
70
|
const params = [];
|
|
65
71
|
if (scope.file_id) {
|
|
66
72
|
conditions.push("file_id = ?");
|
|
@@ -78,37 +84,27 @@ export function countOpenUnknowns(db, scope = {}) {
|
|
|
78
84
|
conditions.push("kind = ?");
|
|
79
85
|
params.push(scope.kind);
|
|
80
86
|
}
|
|
87
|
+
if (scope.scan_id) {
|
|
88
|
+
conditions.push("scan_id = ?");
|
|
89
|
+
params.push(scope.scan_id);
|
|
90
|
+
}
|
|
91
|
+
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
81
92
|
const row = db
|
|
82
|
-
.prepare(`SELECT COUNT(*) AS n FROM unknowns
|
|
93
|
+
.prepare(`SELECT COUNT(*) AS n FROM unknowns ${where}`)
|
|
83
94
|
.get(...params);
|
|
84
95
|
return row.n;
|
|
85
96
|
}
|
|
86
|
-
export function listOpenUnknowns(db,
|
|
87
|
-
const capped = Math.min(Math.max(limit, 1),
|
|
88
|
-
|
|
97
|
+
export function listOpenUnknowns(db, opts = {}) {
|
|
98
|
+
const capped = Math.min(Math.max(opts.limit ?? 200, 1), 1000);
|
|
99
|
+
if (opts.scanId) {
|
|
100
|
+
return db.prepare(`SELECT id, scan_id, file_id, transaction_id, account_id, kind, prompt, options_json, context_json, created_at
|
|
101
|
+
FROM unknowns
|
|
102
|
+
WHERE scan_id = ?
|
|
103
|
+
ORDER BY created_at ASC
|
|
104
|
+
LIMIT ?`).all(opts.scanId, capped);
|
|
105
|
+
}
|
|
106
|
+
return db.prepare(`SELECT id, scan_id, file_id, transaction_id, account_id, kind, prompt, options_json, context_json, created_at
|
|
89
107
|
FROM unknowns
|
|
90
|
-
WHERE resolved_at IS NULL
|
|
91
108
|
ORDER BY created_at ASC
|
|
92
109
|
LIMIT ?`).all(capped);
|
|
93
110
|
}
|
|
94
|
-
/**
|
|
95
|
-
* Open unknowns filtered by `kind`, ordered by the position of the kind in the
|
|
96
|
-
* input array (priority) then by created_at. Pass `["uncategorized","duplicate"]`
|
|
97
|
-
* to drain uncategorized rows before duplicates.
|
|
98
|
-
*
|
|
99
|
-
* `kind` is free-text TEXT in the schema; canonical values used by built-ins:
|
|
100
|
-
* uncategorized, duplicate, correlation, recurrence_candidate,
|
|
101
|
-
* similar_accounts, file_password
|
|
102
|
-
*/
|
|
103
|
-
export function listOpenUnknownsByKind(db, kinds, limit = 50) {
|
|
104
|
-
if (kinds.length === 0)
|
|
105
|
-
return [];
|
|
106
|
-
const capped = Math.min(Math.max(limit, 1), 200);
|
|
107
|
-
const placeholders = kinds.map(() => "?").join(",");
|
|
108
|
-
const cases = kinds.map((_, i) => `WHEN ? THEN ${i}`).join(" ");
|
|
109
|
-
return db.prepare(`SELECT id, file_id, transaction_id, account_id, kind, prompt, options_json, context_json, created_at
|
|
110
|
-
FROM unknowns
|
|
111
|
-
WHERE resolved_at IS NULL AND kind IN (${placeholders})
|
|
112
|
-
ORDER BY CASE kind ${cases} ELSE ${kinds.length} END, created_at ASC
|
|
113
|
-
LIMIT ?`).all(...kinds, ...kinds, capped);
|
|
114
|
-
}
|
package/dist/db/schema.js
CHANGED
|
@@ -14,7 +14,7 @@ export function migrate(db) {
|
|
|
14
14
|
points_balance REAL,
|
|
15
15
|
metadata_json TEXT,
|
|
16
16
|
pii_flag INTEGER NOT NULL DEFAULT 0,
|
|
17
|
-
|
|
17
|
+
has_question INTEGER NOT NULL DEFAULT 0,
|
|
18
18
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
19
19
|
);
|
|
20
20
|
|
|
@@ -74,7 +74,7 @@ export function migrate(db) {
|
|
|
74
74
|
source_file_id TEXT REFERENCES scanned_files(id) ON DELETE CASCADE,
|
|
75
75
|
source_page INTEGER,
|
|
76
76
|
recurrence_id TEXT REFERENCES recurrences(id) ON DELETE SET NULL,
|
|
77
|
-
|
|
77
|
+
has_question INTEGER NOT NULL DEFAULT 0,
|
|
78
78
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
79
79
|
);
|
|
80
80
|
|
|
@@ -98,8 +98,9 @@ export function migrate(db) {
|
|
|
98
98
|
CREATE INDEX IF NOT EXISTS postings_transaction_idx ON postings(transaction_id);
|
|
99
99
|
CREATE INDEX IF NOT EXISTS postings_account_idx ON postings(account_id);
|
|
100
100
|
|
|
101
|
-
CREATE TABLE IF NOT EXISTS
|
|
101
|
+
CREATE TABLE IF NOT EXISTS questions (
|
|
102
102
|
id TEXT PRIMARY KEY,
|
|
103
|
+
scan_id TEXT,
|
|
103
104
|
file_id TEXT REFERENCES scanned_files(id) ON DELETE CASCADE,
|
|
104
105
|
transaction_id TEXT REFERENCES transactions(id) ON DELETE CASCADE,
|
|
105
106
|
account_id TEXT REFERENCES accounts(id) ON DELETE CASCADE,
|
|
@@ -112,6 +113,8 @@ export function migrate(db) {
|
|
|
112
113
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
113
114
|
);
|
|
114
115
|
|
|
116
|
+
CREATE INDEX IF NOT EXISTS questions_scan_idx ON questions(scan_id);
|
|
117
|
+
|
|
115
118
|
CREATE TABLE IF NOT EXISTS conversation_history (
|
|
116
119
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
117
120
|
role TEXT NOT NULL,
|
|
@@ -139,30 +142,5 @@ export function migrate(db) {
|
|
|
139
142
|
use_count INTEGER NOT NULL DEFAULT 0,
|
|
140
143
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
141
144
|
);
|
|
142
|
-
|
|
143
|
-
CREATE TABLE IF NOT EXISTS action_log (
|
|
144
|
-
id TEXT PRIMARY KEY,
|
|
145
|
-
correlation_id TEXT NOT NULL,
|
|
146
|
-
command TEXT NOT NULL,
|
|
147
|
-
user_input TEXT,
|
|
148
|
-
action_type TEXT NOT NULL CHECK(action_type IN (
|
|
149
|
-
'create_account','update_account_metadata','record_transaction','adjust_balance',
|
|
150
|
-
'create_merchant','update_merchant_default'
|
|
151
|
-
)),
|
|
152
|
-
target_id TEXT NOT NULL,
|
|
153
|
-
payload_json TEXT NOT NULL,
|
|
154
|
-
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
155
|
-
reverted_at TEXT
|
|
156
|
-
);
|
|
157
|
-
|
|
158
|
-
CREATE INDEX IF NOT EXISTS action_log_correlation_idx ON action_log(correlation_id);
|
|
159
|
-
CREATE INDEX IF NOT EXISTS action_log_created_idx ON action_log(created_at);
|
|
160
145
|
`);
|
|
161
|
-
ensureColumn(db, "unknowns", "context_json", "TEXT");
|
|
162
|
-
}
|
|
163
|
-
function ensureColumn(db, table, column, type) {
|
|
164
|
-
const cols = db.prepare(`PRAGMA table_info(${table})`).all();
|
|
165
|
-
if (cols.some((c) => c.name === column))
|
|
166
|
-
return;
|
|
167
|
-
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${type}`);
|
|
168
146
|
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type Database from "libsql";
|
|
2
|
+
import type { EventBus } from "./eventBus.js";
|
|
3
|
+
import type { SharedBuffer } from "../buffer/sharedBuffer.js";
|
|
4
|
+
import type { AuditPass } from "./passes/types.js";
|
|
5
|
+
/**
|
|
6
|
+
* The Auditor subscribes to the EventBus and dispatches every event through
|
|
7
|
+
* a registered list of AuditPass instances. Events queue serially so passes
|
|
8
|
+
* don't race each other — within one event, every pass runs to completion
|
|
9
|
+
* before the next event is processed. This makes pass ordering deterministic
|
|
10
|
+
* (memory rule before merchant default, etc.).
|
|
11
|
+
*
|
|
12
|
+
* `drain()` resolves only when the queue is empty AND no pass is in flight —
|
|
13
|
+
* the parse phase calls it after all chunk workers complete so we don't
|
|
14
|
+
* commit while the auditor still has work pending.
|
|
15
|
+
*/
|
|
16
|
+
export declare class Auditor {
|
|
17
|
+
private readonly ctx;
|
|
18
|
+
private readonly passes;
|
|
19
|
+
private readonly bus;
|
|
20
|
+
private queue;
|
|
21
|
+
private running;
|
|
22
|
+
private unsubscribe;
|
|
23
|
+
private idleResolvers;
|
|
24
|
+
constructor(db: Database.Database, buffer: SharedBuffer, bus: EventBus, passes: readonly AuditPass[]);
|
|
25
|
+
get tally(): Readonly<Record<string, number>>;
|
|
26
|
+
start(): void;
|
|
27
|
+
stop(): void;
|
|
28
|
+
/** Resolves once the queue empties and no pass is currently in flight. */
|
|
29
|
+
drain(): Promise<void>;
|
|
30
|
+
private tick;
|
|
31
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Auditor subscribes to the EventBus and dispatches every event through
|
|
3
|
+
* a registered list of AuditPass instances. Events queue serially so passes
|
|
4
|
+
* don't race each other — within one event, every pass runs to completion
|
|
5
|
+
* before the next event is processed. This makes pass ordering deterministic
|
|
6
|
+
* (memory rule before merchant default, etc.).
|
|
7
|
+
*
|
|
8
|
+
* `drain()` resolves only when the queue is empty AND no pass is in flight —
|
|
9
|
+
* the parse phase calls it after all chunk workers complete so we don't
|
|
10
|
+
* commit while the auditor still has work pending.
|
|
11
|
+
*/
|
|
12
|
+
export class Auditor {
|
|
13
|
+
ctx;
|
|
14
|
+
passes;
|
|
15
|
+
bus;
|
|
16
|
+
queue = [];
|
|
17
|
+
running = false;
|
|
18
|
+
unsubscribe = null;
|
|
19
|
+
idleResolvers = [];
|
|
20
|
+
constructor(db, buffer, bus, passes) {
|
|
21
|
+
this.bus = bus;
|
|
22
|
+
this.passes = passes;
|
|
23
|
+
this.ctx = { db, buffer, tally: {} };
|
|
24
|
+
}
|
|
25
|
+
get tally() {
|
|
26
|
+
return this.ctx.tally;
|
|
27
|
+
}
|
|
28
|
+
start() {
|
|
29
|
+
if (this.unsubscribe)
|
|
30
|
+
return;
|
|
31
|
+
this.unsubscribe = this.bus.subscribe(event => {
|
|
32
|
+
this.queue.push(event);
|
|
33
|
+
void this.tick();
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
stop() {
|
|
37
|
+
this.unsubscribe?.();
|
|
38
|
+
this.unsubscribe = null;
|
|
39
|
+
}
|
|
40
|
+
/** Resolves once the queue empties and no pass is currently in flight. */
|
|
41
|
+
async drain() {
|
|
42
|
+
if (!this.running && this.queue.length === 0)
|
|
43
|
+
return;
|
|
44
|
+
return new Promise(resolve => { this.idleResolvers.push(resolve); });
|
|
45
|
+
}
|
|
46
|
+
async tick() {
|
|
47
|
+
if (this.running)
|
|
48
|
+
return;
|
|
49
|
+
this.running = true;
|
|
50
|
+
try {
|
|
51
|
+
while (this.queue.length > 0) {
|
|
52
|
+
const event = this.queue.shift();
|
|
53
|
+
for (const pass of this.passes) {
|
|
54
|
+
if (!pass.handles(event))
|
|
55
|
+
continue;
|
|
56
|
+
try {
|
|
57
|
+
await pass.apply(event, this.ctx);
|
|
58
|
+
}
|
|
59
|
+
catch (err) {
|
|
60
|
+
console.error(`[auditor pass ${pass.name}] ${err.message}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
this.running = false;
|
|
67
|
+
const resolvers = this.idleResolvers.splice(0);
|
|
68
|
+
for (const r of resolvers)
|
|
69
|
+
r();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { AuditEngine, AuditEngineDeps } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* The in-flight audit engine. Subscribes to the bus, serializes events
|
|
4
|
+
* through a single async queue (deterministic pass ordering), and exposes
|
|
5
|
+
* drain() so the scanner engine can wait for the queue to empty between
|
|
6
|
+
* the parse phase and the review phase.
|
|
7
|
+
*
|
|
8
|
+
* No class, no `this` — every piece of state lives in the closure.
|
|
9
|
+
*/
|
|
10
|
+
export declare function createAuditEngine(deps: AuditEngineDeps): AuditEngine;
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Precompute event-kind → handlers so dispatch doesn't iterate every pass
|
|
3
|
+
* for every event. New event kinds added later still go through this map,
|
|
4
|
+
* via the empty-array fallback.
|
|
5
|
+
*/
|
|
6
|
+
function buildPassIndex(passes) {
|
|
7
|
+
const index = new Map();
|
|
8
|
+
const probeKinds = [
|
|
9
|
+
"transaction_appended", "transaction_updated", "transaction_removed",
|
|
10
|
+
"unknown_appended", "unknown_closed",
|
|
11
|
+
"chunk_started", "chunk_completed", "worker_completed",
|
|
12
|
+
];
|
|
13
|
+
for (const kind of probeKinds) {
|
|
14
|
+
const matching = passes.filter(p => probeHandles(p, kind));
|
|
15
|
+
if (matching.length > 0)
|
|
16
|
+
index.set(kind, matching);
|
|
17
|
+
}
|
|
18
|
+
return index;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Ask a pass if it handles a synthetic probe event for the given kind. We
|
|
22
|
+
* only care about the discriminator, so the rest of the fields can be dummy
|
|
23
|
+
* values — passes are expected to switch on `event.kind` and ignore shape.
|
|
24
|
+
*/
|
|
25
|
+
function probeHandles(pass, kind) {
|
|
26
|
+
const probe = { kind };
|
|
27
|
+
try {
|
|
28
|
+
return pass.handles(probe);
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The in-flight audit engine. Subscribes to the bus, serializes events
|
|
36
|
+
* through a single async queue (deterministic pass ordering), and exposes
|
|
37
|
+
* drain() so the scanner engine can wait for the queue to empty between
|
|
38
|
+
* the parse phase and the review phase.
|
|
39
|
+
*
|
|
40
|
+
* No class, no `this` — every piece of state lives in the closure.
|
|
41
|
+
*/
|
|
42
|
+
export function createAuditEngine(deps) {
|
|
43
|
+
const tally = {};
|
|
44
|
+
const ctx = { db: deps.db, buffer: deps.buffer, tally };
|
|
45
|
+
const passIndex = buildPassIndex(deps.passes);
|
|
46
|
+
const queue = [];
|
|
47
|
+
let unsubscribe = null;
|
|
48
|
+
let running = false;
|
|
49
|
+
let idleResolvers = [];
|
|
50
|
+
const tick = async () => {
|
|
51
|
+
if (running)
|
|
52
|
+
return;
|
|
53
|
+
running = true;
|
|
54
|
+
try {
|
|
55
|
+
while (queue.length > 0) {
|
|
56
|
+
const event = queue.shift();
|
|
57
|
+
const handlers = passIndex.get(event.kind);
|
|
58
|
+
if (!handlers)
|
|
59
|
+
continue;
|
|
60
|
+
for (const pass of handlers) {
|
|
61
|
+
try {
|
|
62
|
+
await pass.apply(event, ctx);
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
console.error(`[audit pass ${pass.name}] ${err.message}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
finally {
|
|
71
|
+
running = false;
|
|
72
|
+
const resolvers = idleResolvers;
|
|
73
|
+
idleResolvers = [];
|
|
74
|
+
for (const r of resolvers)
|
|
75
|
+
r();
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
return {
|
|
79
|
+
start() {
|
|
80
|
+
if (unsubscribe)
|
|
81
|
+
return;
|
|
82
|
+
unsubscribe = deps.bus.subscribe(event => {
|
|
83
|
+
queue.push(event);
|
|
84
|
+
void tick();
|
|
85
|
+
});
|
|
86
|
+
},
|
|
87
|
+
stop() {
|
|
88
|
+
unsubscribe?.();
|
|
89
|
+
unsubscribe = null;
|
|
90
|
+
},
|
|
91
|
+
drain() {
|
|
92
|
+
if (!running && queue.length === 0)
|
|
93
|
+
return Promise.resolve();
|
|
94
|
+
return new Promise(resolve => { idleResolvers.push(resolve); });
|
|
95
|
+
},
|
|
96
|
+
get tally() { return tally; },
|
|
97
|
+
};
|
|
98
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import type { BufferedTransaction, BufferedUnknown, BufferSnapshot } from "../buffer/sharedBuffer.js";
|
|
2
|
+
/**
|
|
3
|
+
* Typed event stream the SharedBuffer publishes and the Auditor subscribes to.
|
|
4
|
+
* Every buffer mutation produces an event so audit passes can react in flight
|
|
5
|
+
* instead of running as a post-commit sweep.
|
|
6
|
+
*/
|
|
7
|
+
export type BufferEvent = {
|
|
8
|
+
kind: "transaction_appended";
|
|
9
|
+
transaction: BufferedTransaction;
|
|
10
|
+
chunkId: string;
|
|
11
|
+
} | {
|
|
12
|
+
kind: "transaction_updated";
|
|
13
|
+
transactionId: string;
|
|
14
|
+
before: BufferedTransaction;
|
|
15
|
+
after: BufferedTransaction;
|
|
16
|
+
} | {
|
|
17
|
+
kind: "transaction_removed";
|
|
18
|
+
transactionId: string;
|
|
19
|
+
reason: string;
|
|
20
|
+
} | {
|
|
21
|
+
kind: "unknown_appended";
|
|
22
|
+
unknown: BufferedUnknown;
|
|
23
|
+
chunkId: string;
|
|
24
|
+
} | {
|
|
25
|
+
kind: "unknown_closed";
|
|
26
|
+
unknownId: string;
|
|
27
|
+
answer: string;
|
|
28
|
+
} | {
|
|
29
|
+
kind: "chunk_started";
|
|
30
|
+
chunkId: string;
|
|
31
|
+
fileId: string;
|
|
32
|
+
pageNumber: number;
|
|
33
|
+
} | {
|
|
34
|
+
kind: "chunk_completed";
|
|
35
|
+
chunkId: string;
|
|
36
|
+
fileId: string;
|
|
37
|
+
pageNumber: number;
|
|
38
|
+
} | {
|
|
39
|
+
kind: "worker_completed";
|
|
40
|
+
workerId: string;
|
|
41
|
+
chunkId: string;
|
|
42
|
+
transactionsAdded: number;
|
|
43
|
+
unknownsAdded: number;
|
|
44
|
+
};
|
|
45
|
+
export type EventListener = (event: BufferEvent) => void | Promise<void>;
|
|
46
|
+
export declare class EventBus {
|
|
47
|
+
private listeners;
|
|
48
|
+
private buffered;
|
|
49
|
+
subscribe(fn: EventListener): () => void;
|
|
50
|
+
/**
|
|
51
|
+
* Publish an event to every listener. Listener errors are caught and logged —
|
|
52
|
+
* a misbehaving pass never silences the bus for the rest.
|
|
53
|
+
*/
|
|
54
|
+
publish(event: BufferEvent): void;
|
|
55
|
+
/** All events seen so far, in publish order. Useful for the review TUI summary. */
|
|
56
|
+
history(): readonly BufferEvent[];
|
|
57
|
+
/** Test helper. */
|
|
58
|
+
reset(): void;
|
|
59
|
+
}
|
|
60
|
+
export type { BufferSnapshot };
|