pi-mega-compact 0.16.3 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/dist/config/vector-cortex.js +91 -0
  2. package/dist/dedup/digest.js +30 -0
  3. package/dist/dedup/l1-lsh-v2.js +60 -0
  4. package/dist/dedup/l1-lsh.js +52 -0
  5. package/dist/dedup/l1-minhash-v2.js +193 -0
  6. package/dist/dedup/l1-minhash.js +91 -0
  7. package/dist/dedup/l1-verify.js +54 -0
  8. package/dist/dedup/mmr.js +45 -0
  9. package/dist/dedup/normalize.js +39 -0
  10. package/dist/dedup/raptor/buildHistory.js +164 -0
  11. package/dist/dedup/raptor/guardrails.js +83 -0
  12. package/dist/dedup/raptor/incremental.js +304 -0
  13. package/dist/dedup/raptor/index.js +190 -0
  14. package/dist/dedup/raptor/kmeans.js +152 -0
  15. package/dist/dedup/raptor/multilevel.js +184 -0
  16. package/dist/dedup/raptor/retrieval.js +93 -0
  17. package/dist/dedup/raptor/summarizer.js +86 -0
  18. package/dist/dedup/raptor/tree.js +202 -0
  19. package/dist/dedup/topk.js +60 -0
  20. package/dist/extensions/dashboard-server/api-contracts/endpoints/registry-ext.js +30 -0
  21. package/dist/extensions/dashboard-server/api-contracts/endpoints/registry.js +4 -14
  22. package/dist/extensions/dashboard-server/api-contracts/vector-cortex.js +12 -0
  23. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +11 -0
  24. package/dist/extensions/dashboard-server/routes-vector-cortex.js +292 -0
  25. package/dist/extensions/dashboard-server/routes.js +1 -0
  26. package/dist/extensions/dashboard-server/server.js +9 -1
  27. package/dist/extensions/mega-events/context-handler.js +11 -0
  28. package/dist/extensions/mega-runtime/vector-cortex-ledger.js +131 -0
  29. package/dist/extensions/mega-runtime/vector-cortex-safety.js +100 -0
  30. package/dist/src/config/vector-cortex.js +91 -0
  31. package/dist/src/config.js +6 -0
  32. package/dist/src/dedup/l1-lsh-v2.js +60 -0
  33. package/dist/src/dedup/l1-minhash-v2.js +193 -0
  34. package/dist/src/dedup/normalize.js +1 -1
  35. package/dist/src/vector-cortex/conformance/emit.js +46 -0
  36. package/dist/src/vector-cortex/conformance/manifest.js +235 -0
  37. package/dist/src/vector-cortex/conformance/runner.js +111 -0
  38. package/dist/src/vector-cortex/conformance/triadB-reader.js +95 -0
  39. package/dist/src/vector-cortex/eval/annotations.js +41 -0
  40. package/dist/src/vector-cortex/eval/metrics.js +86 -0
  41. package/dist/src/vector-cortex/eval/observer.js +61 -0
  42. package/dist/src/vector-cortex/eval/persist.js +67 -0
  43. package/dist/src/vector-cortex/eval/reader.js +29 -0
  44. package/dist/src/vector-cortex/eval/types.js +33 -0
  45. package/dist/src/vector-cortex/ledger/adapter.js +55 -0
  46. package/dist/src/vector-cortex/ledger/compat-journal.js +206 -0
  47. package/dist/src/vector-cortex/ledger/emit.js +53 -0
  48. package/dist/src/vector-cortex/ledger/event-codec.js +72 -0
  49. package/dist/src/vector-cortex/ledger/event-codecB.js +76 -0
  50. package/dist/src/vector-cortex/ledger/sqlite.js +200 -0
  51. package/dist/src/vector-cortex/ledger/store.js +184 -0
  52. package/dist/src/vector-cortex/ledger/types.js +52 -0
  53. package/dist/src/vector-cortex/ledger/validator.js +117 -0
  54. package/dist/src/vector-cortex/migrations/effective-cut-v2.js +103 -0
  55. package/dist/src/vector-cortex/migrations/minhash-v2.js +209 -0
  56. package/dist/src/vector-cortex/migrations/occurrence-v2.js +139 -0
  57. package/dist/src/vector-cortex/replay/cut.js +123 -0
  58. package/dist/src/vector-cortex/replay/emit.js +58 -0
  59. package/dist/src/vector-cortex/replay/replay.js +260 -0
  60. package/dist/src/vector-cortex/replay/replayB.js +73 -0
  61. package/dist/src/vector-cortex/replay/types.js +52 -0
  62. package/dist/src/vector-cortex/resilience/breaker-core.js +359 -0
  63. package/dist/src/vector-cortex/resilience/breaker.js +12 -0
  64. package/dist/src/vector-cortex/resilience/emit.js +56 -0
  65. package/dist/src/vector-cortex/resilience/spool-core.js +347 -0
  66. package/dist/src/vector-cortex/resilience/spool.js +12 -0
  67. package/dist/src/vector-cortex/resilience/types.js +32 -0
  68. package/dist/vector-cortex/conformance/emit.js +46 -0
  69. package/dist/vector-cortex/conformance/manifest.js +235 -0
  70. package/dist/vector-cortex/conformance/runner.js +111 -0
  71. package/dist/vector-cortex/conformance/triadB-reader.js +95 -0
  72. package/dist/vector-cortex/eval/annotations.js +41 -0
  73. package/dist/vector-cortex/eval/metrics.js +86 -0
  74. package/dist/vector-cortex/eval/observer.js +61 -0
  75. package/dist/vector-cortex/eval/persist.js +67 -0
  76. package/dist/vector-cortex/eval/reader.js +29 -0
  77. package/dist/vector-cortex/eval/types.js +33 -0
  78. package/dist/vector-cortex/ledger/adapter.js +55 -0
  79. package/dist/vector-cortex/ledger/compat-journal.js +206 -0
  80. package/dist/vector-cortex/ledger/emit.js +53 -0
  81. package/dist/vector-cortex/ledger/event-codec.js +72 -0
  82. package/dist/vector-cortex/ledger/event-codecB.js +76 -0
  83. package/dist/vector-cortex/ledger/sqlite.js +200 -0
  84. package/dist/vector-cortex/ledger/store.js +184 -0
  85. package/dist/vector-cortex/ledger/types.js +52 -0
  86. package/dist/vector-cortex/ledger/validator.js +117 -0
  87. package/dist/vector-cortex/migrations/effective-cut-v2.js +103 -0
  88. package/dist/vector-cortex/migrations/minhash-v2.js +209 -0
  89. package/dist/vector-cortex/migrations/occurrence-v2.js +139 -0
  90. package/dist/vector-cortex/replay/cut.js +123 -0
  91. package/dist/vector-cortex/replay/emit.js +58 -0
  92. package/dist/vector-cortex/replay/replay.js +260 -0
  93. package/dist/vector-cortex/replay/replayB.js +73 -0
  94. package/dist/vector-cortex/replay/types.js +52 -0
  95. package/dist/vector-cortex/resilience/breaker-core.js +359 -0
  96. package/dist/vector-cortex/resilience/breaker.js +12 -0
  97. package/dist/vector-cortex/resilience/emit.js +56 -0
  98. package/dist/vector-cortex/resilience/spool-core.js +347 -0
  99. package/dist/vector-cortex/resilience/spool.js +12 -0
  100. package/dist/vector-cortex/resilience/types.js +32 -0
  101. package/extensions/dashboard-client/dist/assets/{AreaChart-Y_X0Bbjd.js → AreaChart-BQXFF7UI.js} +2 -2
  102. package/extensions/dashboard-client/dist/assets/{AreaChart-Y_X0Bbjd.js.map → AreaChart-BQXFF7UI.js.map} +1 -1
  103. package/extensions/dashboard-client/dist/assets/{BarChart-B9XDb-Ob.js → BarChart-BVtoz9Eg.js} +2 -2
  104. package/extensions/dashboard-client/dist/assets/{BarChart-B9XDb-Ob.js.map → BarChart-BVtoz9Eg.js.map} +1 -1
  105. package/extensions/dashboard-client/dist/assets/{CacheTab-BUGVb1G1.js → CacheTab-trI2TIGy.js} +2 -2
  106. package/extensions/dashboard-client/dist/assets/{CacheTab-BUGVb1G1.js.map → CacheTab-trI2TIGy.js.map} +1 -1
  107. package/extensions/dashboard-client/dist/assets/{EventsTab-dOqLYTsy.js → EventsTab-CNUbk9__.js} +2 -2
  108. package/extensions/dashboard-client/dist/assets/{EventsTab-dOqLYTsy.js.map → EventsTab-CNUbk9__.js.map} +1 -1
  109. package/extensions/dashboard-client/dist/assets/{HealthTab-c2vezKg1.js → HealthTab-K-bd_CM-.js} +2 -2
  110. package/extensions/dashboard-client/dist/assets/{HealthTab-c2vezKg1.js.map → HealthTab-K-bd_CM-.js.map} +1 -1
  111. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-bQYHIcdx.js → MaintenanceTab-CoEioKzx.js} +2 -2
  112. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-bQYHIcdx.js.map → MaintenanceTab-CoEioKzx.js.map} +1 -1
  113. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-2L8pH6y2.js → MemoryMapTab-CJo24ulv.js} +2 -2
  114. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-2L8pH6y2.js.map → MemoryMapTab-CJo24ulv.js.map} +1 -1
  115. package/extensions/dashboard-client/dist/assets/{MetricsTab-CzUx1jpo.js → MetricsTab-DCNlQWWP.js} +2 -2
  116. package/extensions/dashboard-client/dist/assets/{MetricsTab-CzUx1jpo.js.map → MetricsTab-DCNlQWWP.js.map} +1 -1
  117. package/extensions/dashboard-client/dist/assets/{OverviewTab-eCNoaclv.js → OverviewTab-CQQlyIQ6.js} +2 -2
  118. package/extensions/dashboard-client/dist/assets/{OverviewTab-eCNoaclv.js.map → OverviewTab-CQQlyIQ6.js.map} +1 -1
  119. package/extensions/dashboard-client/dist/assets/{ReposTab-tb0IW3NF.js → ReposTab-Cqm82_yn.js} +2 -2
  120. package/extensions/dashboard-client/dist/assets/{ReposTab-tb0IW3NF.js.map → ReposTab-Cqm82_yn.js.map} +1 -1
  121. package/extensions/dashboard-client/dist/assets/{SessionsTab-C-IoLw5h.js → SessionsTab-A8xTAdET.js} +2 -2
  122. package/extensions/dashboard-client/dist/assets/{SessionsTab-C-IoLw5h.js.map → SessionsTab-A8xTAdET.js.map} +1 -1
  123. package/extensions/dashboard-client/dist/assets/{SetupTab-BvmFi8gf.js → SetupTab-CqaSgaZl.js} +2 -2
  124. package/extensions/dashboard-client/dist/assets/{SetupTab-BvmFi8gf.js.map → SetupTab-CqaSgaZl.js.map} +1 -1
  125. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-DBH-cOlA.js → TimeSavedCard-C_f3q8GR.js} +2 -2
  126. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-DBH-cOlA.js.map → TimeSavedCard-C_f3q8GR.js.map} +1 -1
  127. package/extensions/dashboard-client/dist/assets/{TurnsTab-DBMgmvnX.js → TurnsTab-BEsafACo.js} +2 -2
  128. package/extensions/dashboard-client/dist/assets/{TurnsTab-DBMgmvnX.js.map → TurnsTab-BEsafACo.js.map} +1 -1
  129. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DD1CMK80.js +2 -0
  130. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DD1CMK80.js.map +1 -0
  131. package/extensions/dashboard-client/dist/assets/{WikiTab-1LcQrXk3.js → WikiTab-Bf01M2_Z.js} +2 -2
  132. package/extensions/dashboard-client/dist/assets/{WikiTab-1LcQrXk3.js.map → WikiTab-Bf01M2_Z.js.map} +1 -1
  133. package/extensions/dashboard-client/dist/assets/{button-DtEsZTUE.js → button-BEJPTk_E.js} +2 -2
  134. package/extensions/dashboard-client/dist/assets/{button-DtEsZTUE.js.map → button-BEJPTk_E.js.map} +1 -1
  135. package/extensions/dashboard-client/dist/assets/{card-DcfTmw-4.js → card-DK1V382s.js} +2 -2
  136. package/extensions/dashboard-client/dist/assets/{card-DcfTmw-4.js.map → card-DK1V382s.js.map} +1 -1
  137. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-BsIxUDyw.js → generateCategoricalChart-CRtaPphL.js} +2 -2
  138. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-BsIxUDyw.js.map → generateCategoricalChart-CRtaPphL.js.map} +1 -1
  139. package/extensions/dashboard-client/dist/assets/{index-B93cewiX.js → index-CB6BslE4.js} +34 -29
  140. package/extensions/dashboard-client/dist/assets/index-CB6BslE4.js.map +1 -0
  141. package/extensions/dashboard-client/dist/assets/index-CfvziyBi.css +1 -0
  142. package/extensions/dashboard-client/dist/assets/{switch-Bowd7jCD.js → switch-C4FV6B9k.js} +2 -2
  143. package/extensions/dashboard-client/dist/assets/{switch-Bowd7jCD.js.map → switch-C4FV6B9k.js.map} +1 -1
  144. package/extensions/dashboard-client/dist/assets/{toggle-Bo7kJZ4u.js → toggle-GBmeENMd.js} +2 -2
  145. package/extensions/dashboard-client/dist/assets/{toggle-Bo7kJZ4u.js.map → toggle-GBmeENMd.js.map} +1 -1
  146. package/extensions/dashboard-client/dist/assets/{useSSE-zNhb0c3u.js → useSSE-BOKZGSn1.js} +2 -2
  147. package/extensions/dashboard-client/dist/assets/{useSSE-zNhb0c3u.js.map → useSSE-BOKZGSn1.js.map} +1 -1
  148. package/extensions/dashboard-client/dist/index.html +2 -2
  149. package/extensions/dashboard-client/src/App.tsx +2 -0
  150. package/extensions/dashboard-client/src/api/vector-cortex.ts +54 -0
  151. package/extensions/dashboard-client/src/tabs/VectorCortexTab.tsx +272 -0
  152. package/extensions/dashboard-client/src/tabs/registry.ts +4 -1
  153. package/extensions/dashboard-client/src/types/vector-cortex.ts +84 -0
  154. package/extensions/dashboard-client/tsconfig.json +1 -1
  155. package/extensions/dashboard-server/api-contracts/endpoints/registry-ext.ts +67 -0
  156. package/extensions/dashboard-server/api-contracts/endpoints/registry.ts +4 -21
  157. package/extensions/dashboard-server/api-contracts/index.ts +3 -0
  158. package/extensions/dashboard-server/api-contracts/vector-cortex.ts +136 -0
  159. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +41 -0
  160. package/extensions/dashboard-server/routes-vector-cortex.ts +332 -0
  161. package/extensions/dashboard-server/routes.ts +6 -0
  162. package/extensions/dashboard-server/server.ts +8 -0
  163. package/extensions/mega-events/context-handler.ts +16 -0
  164. package/extensions/mega-runtime/vector-cortex-ledger.ts +165 -0
  165. package/extensions/mega-runtime/vector-cortex-safety.ts +181 -0
  166. package/package.json +2 -1
  167. package/src/config/vector-cortex.ts +98 -0
  168. package/src/config.ts +28 -0
  169. package/src/dedup/l1-lsh-v2.ts +78 -0
  170. package/src/dedup/l1-minhash-v2.ts +221 -0
  171. package/src/dedup/normalize.ts +1 -1
  172. package/src/store/sqlite/game-scores.ts +2 -1
  173. package/src/vector-cortex/conformance/emit.ts +56 -0
  174. package/src/vector-cortex/conformance/manifest.ts +298 -0
  175. package/src/vector-cortex/conformance/runner.ts +165 -0
  176. package/src/vector-cortex/conformance/triadB-reader.ts +110 -0
  177. package/src/vector-cortex/eval/annotations.ts +70 -0
  178. package/src/vector-cortex/eval/metrics.ts +113 -0
  179. package/src/vector-cortex/eval/observer.ts +86 -0
  180. package/src/vector-cortex/eval/persist.ts +73 -0
  181. package/src/vector-cortex/eval/reader.ts +45 -0
  182. package/src/vector-cortex/eval/types.ts +89 -0
  183. package/src/vector-cortex/ledger/adapter.ts +71 -0
  184. package/src/vector-cortex/ledger/compat-journal.ts +234 -0
  185. package/src/vector-cortex/ledger/emit.ts +86 -0
  186. package/src/vector-cortex/ledger/event-codec.ts +79 -0
  187. package/src/vector-cortex/ledger/event-codecB.ts +87 -0
  188. package/src/vector-cortex/ledger/sqlite.ts +265 -0
  189. package/src/vector-cortex/ledger/store.ts +380 -0
  190. package/src/vector-cortex/ledger/types.ts +141 -0
  191. package/src/vector-cortex/ledger/validator.ts +124 -0
  192. package/src/vector-cortex/migrations/effective-cut-v2.ts +149 -0
  193. package/src/vector-cortex/migrations/minhash-v2.ts +263 -0
  194. package/src/vector-cortex/migrations/occurrence-v2.ts +192 -0
  195. package/src/vector-cortex/replay/cut.ts +169 -0
  196. package/src/vector-cortex/replay/emit.ts +89 -0
  197. package/src/vector-cortex/replay/replay.ts +308 -0
  198. package/src/vector-cortex/replay/replayB.ts +97 -0
  199. package/src/vector-cortex/replay/types.ts +145 -0
  200. package/src/vector-cortex/resilience/breaker-core.ts +444 -0
  201. package/src/vector-cortex/resilience/breaker.ts +22 -0
  202. package/src/vector-cortex/resilience/emit.ts +99 -0
  203. package/src/vector-cortex/resilience/spool-core.ts +411 -0
  204. package/src/vector-cortex/resilience/spool.ts +22 -0
  205. package/src/vector-cortex/resilience/types.ts +158 -0
  206. package/extensions/dashboard-client/dist/assets/index-B93cewiX.js.map +0 -1
  207. package/extensions/dashboard-client/dist/assets/index-C_tCbOI0.css +0 -1
@@ -0,0 +1,265 @@
1
+ /**
2
+ * vector-cortex/ledger/sqlite.ts — occurrence-v2 SQLite store, Mode A (VC1B).
3
+ *
4
+ * A self-contained, isolated SQLite store (`node:sqlite` `DatabaseSync`) over its
5
+ * OWN database file — NOT the host `sqlite.db` — so the neutral v2 ledger is a
6
+ * true independent authority, not a host table. Holds the canonical occurrence
7
+ * rows plus per-session monotonic seq and the durable contiguous high-water.
8
+ *
9
+ * The defect M2 fixes: the host `raw_transcript` table keys on
10
+ * `(content_hash, session_id)`, so repeated equal bytes at a later turn are
11
+ * silently dropped. Here the ledger keys occurrences by `(event_id, digest)`
12
+ * ONLY — equal bytes at a distinct seq/event-id are preserved as separate
13
+ * occurrences (M2-DUP-001).
14
+ *
15
+ * Append enforces, per occurrence: (1) strictly monotonic per-session seq
16
+ * (EVT_SEQ_REGRESSION otherwise); (2) a tool RESULT names exactly one earlier
17
+ * call (EVT_TOOL_CALL_MISSING otherwise); (3) exact `(event_id,digest)`
18
+ * re-appends are acknowledged idempotently, never duplicated.
19
+ *
20
+ * Mode A of the VC1B triad: transactional SQLite append. Mode B (`./spool.ts`)
21
+ * is an independent fsync spool; mode C leaves the host transcript unchanged.
22
+ *
23
+ * PREVENT-002: every query is parameterized. PREVENT-011: no `any`.
24
+ * PREVENT-PI-004: local filesystem only, no network. No console.log.
25
+ */
26
+
27
+ import { DatabaseSync } from "node:sqlite";
28
+ import { existsSync, mkdirSync } from "node:fs";
29
+ import { dirname } from "node:path";
30
+ import { createHash } from "node:crypto";
31
+ import type { LedgerAppendResult, LedgerOccurrence } from "./store.js";
32
+
33
+ /** Failure codes (re-export of the store contract union, kept local for rows). */
34
+ export type SqliteAppendCode = "EVT_TOOL_CALL_MISSING" | "EVT_SEQ_REGRESSION";
35
+
36
+ /** DB row shape for occurrence_v2 (snake_case columns). */
37
+ interface OccurrenceDBRow {
38
+ session: string;
39
+ seq: number;
40
+ event_id: string;
41
+ digest: string;
42
+ kind: string;
43
+ tool_call_id: string | null;
44
+ source_bytes: Uint8Array | null;
45
+ }
46
+
47
+ const SCHEMA = `
48
+ CREATE TABLE IF NOT EXISTS occurrence_v2 (
49
+ session TEXT NOT NULL,
50
+ seq INTEGER NOT NULL,
51
+ event_id TEXT NOT NULL,
52
+ digest TEXT NOT NULL,
53
+ kind TEXT NOT NULL,
54
+ tool_call_id TEXT,
55
+ source_bytes BLOB,
56
+ PRIMARY KEY (session, seq),
57
+ UNIQUE (session, event_id, digest)
58
+ ) STRICT;
59
+
60
+ CREATE TABLE IF NOT EXISTS ledger_high_water (
61
+ session TEXT PRIMARY KEY,
62
+ seq INTEGER NOT NULL
63
+ ) STRICT;
64
+ `;
65
+
66
+ /** Default digest for a row when the caller omits one (sha256 over bytes). */
67
+ export function ledgerDigest(bytes: Uint8Array): string {
68
+ const hex = createHash("sha256").update(bytes).digest("hex");
69
+ return `sha256:${hex}`;
70
+ }
71
+
72
+ /** Open (or reuse) the isolated occurrence-v2 ledger DB handle. */
73
+ export function openOccurrenceStore(dbPath: string): DatabaseSync {
74
+ const dir = dirname(dbPath);
75
+ if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
76
+ const db = new DatabaseSync(dbPath);
77
+ db.exec("PRAGMA journal_mode = WAL");
78
+ db.exec("PRAGMA foreign_keys = ON");
79
+ db.exec(SCHEMA);
80
+ return db;
81
+ }
82
+
83
+ function rowToOccurrence(row: OccurrenceDBRow): LedgerOccurrence {
84
+ return {
85
+ session: row.session,
86
+ seq: BigInt(row.seq),
87
+ eventId: row.event_id,
88
+ digest: row.digest,
89
+ kind: row.kind,
90
+ toolCallId: row.tool_call_id ?? undefined,
91
+ sourceBytes: row.source_bytes ? Buffer.from(row.source_bytes) : new Uint8Array(0),
92
+ };
93
+ }
94
+
95
+ /** Current durable contiguous high-water for a session (0 when none). */
96
+ export function ledgerHighWater(db: DatabaseSync, session: string): bigint {
97
+ const row = db
98
+ .prepare(`SELECT seq FROM ledger_high_water WHERE session = @session`)
99
+ .get({ "@session": session }) as { seq: number } | undefined;
100
+ return row ? BigInt(row.seq) : 0n;
101
+ }
102
+
103
+ /** Read all accepted occurrences for a session in ascending (seq,eventId). */
104
+ export function readSessionOccurrences(db: DatabaseSync, session: string): LedgerOccurrence[] {
105
+ const rows = db
106
+ .prepare(
107
+ `SELECT session, seq, event_id, digest, kind, tool_call_id, source_bytes
108
+ FROM occurrence_v2 WHERE session = @session
109
+ ORDER BY seq ASC, event_id ASC`,
110
+ )
111
+ .all({ "@session": session }) as unknown as OccurrenceDBRow[];
112
+ return rows.map(rowToOccurrence);
113
+ }
114
+
115
+ /** Occurrences at or above `fromSeq` (inclusive), ascending. */
116
+ export function readFromSeq(db: DatabaseSync, session: string, fromSeq: bigint): LedgerOccurrence[] {
117
+ const rows = db
118
+ .prepare(
119
+ `SELECT session, seq, event_id, digest, kind, tool_call_id, source_bytes
120
+ FROM occurrence_v2 WHERE session = @session AND seq >= @from
121
+ ORDER BY seq ASC, event_id ASC`,
122
+ )
123
+ .all({ "@session": session, "@from": Number(fromSeq) }) as unknown as OccurrenceDBRow[];
124
+ return rows.map(rowToOccurrence);
125
+ }
126
+
127
+ /** Count accepted occurrences for a session. */
128
+ export function countOccurrences(db: DatabaseSync, session: string): number {
129
+ const row = db
130
+ .prepare(`SELECT COUNT(*) AS cnt FROM occurrence_v2 WHERE session = @session`)
131
+ .get({ "@session": session }) as { cnt: number };
132
+ return row.cnt;
133
+ }
134
+
135
+ /** Whether a `(event_id, digest)` pair is already accepted in a session. */
136
+ export function hasOccurrence(db: DatabaseSync, session: string, eventId: string, digest: string): boolean {
137
+ const row = db
138
+ .prepare(
139
+ `SELECT 1 AS hit FROM occurrence_v2
140
+ WHERE session = @session AND event_id = @event_id AND digest = @digest LIMIT 1`,
141
+ )
142
+ .get({ "@session": session, "@event_id": eventId, "@digest": digest }) as { hit: number } | undefined;
143
+ return row !== undefined;
144
+ }
145
+
146
+ /** The stored occurrence matching a `(event_id, digest)` pair, or undefined. */
147
+ function findOccurrence(
148
+ db: DatabaseSync,
149
+ session: string,
150
+ eventId: string,
151
+ digest: string,
152
+ ): LedgerOccurrence | undefined {
153
+ const row = db
154
+ .prepare(
155
+ `SELECT session, seq, event_id, digest, kind, tool_call_id, source_bytes
156
+ FROM occurrence_v2
157
+ WHERE session = @session AND event_id = @event_id AND digest = @digest LIMIT 1`,
158
+ )
159
+ .get({ "@session": session, "@event_id": eventId, "@digest": digest }) as OccurrenceDBRow | undefined;
160
+ return row ? rowToOccurrence(row) : undefined;
161
+ }
162
+
163
+ /** Whether a prior call row with the given eventId exists in the session. */
164
+ function hasCallRef(db: DatabaseSync, session: string, eventId: string): boolean {
165
+ const row = db
166
+ .prepare(
167
+ `SELECT 1 AS hit FROM occurrence_v2
168
+ WHERE session = @session AND event_id = @event_id LIMIT 1`,
169
+ )
170
+ .get({ "@session": session, "@event_id": eventId }) as { hit: number } | undefined;
171
+ return row !== undefined;
172
+ }
173
+
174
+ /**
175
+ * Append one occurrence within a transaction. Enforces monotonic seq, tool-call
176
+ * reference completeness, and `(event_id,digest)` idempotent ack. Returns the
177
+ * accepted row or a deterministic failure code.
178
+ */
179
+ export function appendOccurrence(
180
+ db: DatabaseSync,
181
+ input: {
182
+ readonly session: string;
183
+ readonly seq: bigint;
184
+ readonly eventId: string;
185
+ readonly kind: string;
186
+ readonly toolCallId?: string;
187
+ readonly sourceBytes: Uint8Array;
188
+ readonly digest?: string;
189
+ },
190
+ ): LedgerAppendResult {
191
+ const digest = input.digest ?? ledgerDigest(input.sourceBytes);
192
+ const occurrence: LedgerOccurrence = {
193
+ session: input.session,
194
+ seq: input.seq,
195
+ eventId: input.eventId,
196
+ digest,
197
+ kind: input.kind,
198
+ toolCallId: input.toolCallId,
199
+ sourceBytes: input.sourceBytes,
200
+ };
201
+
202
+ // (3) exact (event_id,digest) re-append is acknowledged idempotently, WITHOUT
203
+ // inserting a new row. Return the EXISTING stored occurrence (its real seq) —
204
+ // not the caller's input — so a journal record written from this result carries
205
+ // the existing seq, which always corresponds to a real occurrence_v2 row (Q01:
206
+ // never a phantom journal row at a seq with no backing occurrence).
207
+ const existing = findOccurrence(db, input.session, input.eventId, digest);
208
+ if (existing) {
209
+ return { ok: true, occurrence: existing };
210
+ }
211
+
212
+ // (1) monotonic seq: must be exactly the next expected seq (highWater+1).
213
+ const hw = ledgerHighWater(db, input.session);
214
+ if (input.seq !== hw + 1n) {
215
+ return { ok: false, code: "EVT_SEQ_REGRESSION", rejected: occurrence };
216
+ }
217
+
218
+ // (2) a tool RESULT names exactly one earlier call in the same session.
219
+ if (input.toolCallId !== undefined && input.toolCallId !== "") {
220
+ if (!hasCallRef(db, input.session, input.toolCallId)) {
221
+ return { ok: false, code: "EVT_TOOL_CALL_MISSING", rejected: occurrence };
222
+ }
223
+ }
224
+
225
+ // Accept: insert the row and advance the per-session high-water atomically.
226
+ db.exec("SAVEPOINT mc_ledger");
227
+ try {
228
+ db.prepare(
229
+ `INSERT INTO occurrence_v2 (session, seq, event_id, digest, kind, tool_call_id, source_bytes)
230
+ VALUES (@session, @seq, @event_id, @digest, @kind, @tool_call_id, @source_bytes)`,
231
+ ).run({
232
+ "@session": input.session,
233
+ "@seq": Number(input.seq),
234
+ "@event_id": input.eventId,
235
+ "@digest": digest,
236
+ "@kind": input.kind,
237
+ "@tool_call_id": input.toolCallId ?? null,
238
+ "@source_bytes": Buffer.from(input.sourceBytes),
239
+ });
240
+ db.prepare(
241
+ `INSERT INTO ledger_high_water (session, seq) VALUES (@session, @seq)
242
+ ON CONFLICT(session) DO UPDATE SET seq = excluded.seq`,
243
+ ).run({ "@session": input.session, "@seq": Number(input.seq) });
244
+ db.exec("RELEASE mc_ledger");
245
+ } catch (e) {
246
+ db.exec("ROLLBACK TO mc_ledger");
247
+ db.exec("RELEASE mc_ledger");
248
+ throw e;
249
+ }
250
+ return { ok: true, occurrence };
251
+ }
252
+
253
+ /** Append a batch; each occurrence reports its own result. */
254
+ export function appendOccurrenceBatch(
255
+ db: DatabaseSync,
256
+ inputs: ReadonlyArray<Parameters<typeof appendOccurrence>[1]>,
257
+ ): LedgerAppendResult[] {
258
+ return inputs.map((input) => appendOccurrence(db, input));
259
+ }
260
+
261
+ /** Diagnostic: total accepted occurrences across all sessions. */
262
+ export function countAllOccurrences(db: DatabaseSync): number {
263
+ const row = db.prepare(`SELECT COUNT(*) AS cnt FROM occurrence_v2`).get() as { cnt: number };
264
+ return row.cnt;
265
+ }
@@ -0,0 +1,380 @@
1
+ /**
2
+ * vector-cortex/ledger/store.ts — VC1B capability-separated ledger contracts.
3
+ *
4
+ * Owns `LedgerReader` / `LedgerWriter` / `LedgerAdmin` and `CompatJournalV1`.
5
+ * The ledger is append-only, seq is monotonic per session, a tool RESULT names
6
+ * exactly one earlier call in the same session, and duplicates are identified by
7
+ * `(eventId, digest)` only (two occurrences may share bytes at distinct seq).
8
+ *
9
+ * Capability gating mirrors the host `asReader/asWriter/asAdmin` pattern: a
10
+ * consumer receives ONLY what it needs. Dashboard GET receives `LedgerReader`;
11
+ * only ingestion receives `LedgerWriter`; only the migration coordinator
12
+ * receives `LedgerAdmin`. This keeps the neutral byte-authority ledger closed to
13
+ * anything but the sanctioned channels (CONTRACTS §Store and migration
14
+ * contracts).
15
+ *
16
+ * Pure type/schema definitions + small pure predicates. No storage, no console,
17
+ * no network, no side effects (PREVENT-PI-004 / PREVENT-011).
18
+ */
19
+
20
+ /**
21
+ * An accepted occurrence in the canonical v2 ledger. This is the durable row
22
+ * shape: the neutral byte authority plus the reference envelope. `digest` is the
23
+ * authoritative `sha256:${string}` of `sourceBytes`; `sourceBytes` round-trips
24
+ * exactly (never reconstructed from normalized text).
25
+ */
26
+ export interface LedgerOccurrence {
27
+ /** The source session the occurrence belongs to (ledger scopes by session). */
28
+ readonly session: string;
29
+ /** Monotonic per-session sequence (never regresses for an accepted row). */
30
+ readonly seq: bigint;
31
+ /** Stable occurrence identity within the session (the EventV2 eventId). */
32
+ readonly eventId: string;
33
+ /** SHA-256 over sourceBytes (authoritative, `sha256:<hex>`). */
34
+ readonly digest: string;
35
+ /** Neutral occurrence kind (policy/user/assistant/tool/message...). */
36
+ readonly kind: string;
37
+ /** On a tool RESULT, references exactly one earlier call in this session. */
38
+ readonly toolCallId?: string;
39
+ /** Authoritative original bytes (byte authority). */
40
+ readonly sourceBytes: Uint8Array;
41
+ }
42
+
43
+ /**
44
+ * Identifier of a tool pair for a RESULT row: names exactly one earlier CALL's
45
+ * (session-local) eventId. Absent on non-result rows.
46
+ */
47
+ export interface LedgerToolRef {
48
+ /** The session-local eventId of the CALL this RESULT closes. */
49
+ readonly callEventId: string;
50
+ }
51
+
52
+ /** Failure codes the writer surfaces on a rejected append (write path). */
53
+ export type LedgerAppendCode = "EVT_TOOL_CALL_MISSING" | "EVT_SEQ_REGRESSION";
54
+
55
+ /** Result of a single append attempt. */
56
+ export type LedgerAppendResult =
57
+ | { ok: true; occurrence: LedgerOccurrence }
58
+ | { ok: false; code: LedgerAppendCode; rejected: LedgerOccurrence };
59
+
60
+ /**
61
+ * Reader capability: read-access only. The dashboard's `GET
62
+ * /api/vector-cortex/ledger` is built on exactly this surface and nothing more —
63
+ * it can read counts/order/digests but never mutate.
64
+ */
65
+ export interface LedgerReader {
66
+ readonly kind: "LedgerReader";
67
+ /** Accept the latest contiguous high-water (durable authority), or 0. */
68
+ highWater(session: string): bigint;
69
+ /** Contiguous accepted occurrences for a session in ascending `(seq,eventId)`. */
70
+ readSession(session: string): readonly LedgerOccurrence[];
71
+ /** Occurrences at or above `fromSeq` (inclusive) for a session. */
72
+ readFrom(session: string, fromSeq: bigint): readonly LedgerOccurrence[];
73
+ /** Count of accepted occurrences for a session. */
74
+ count(session: string): number;
75
+ /** Whether a `(eventId, digest)` pair is already accepted (unique key). */
76
+ hasOccurrence(session: string, eventId: string, digest: string): boolean;
77
+ }
78
+
79
+ /**
80
+ * Writer capability: append-only ingestion. Can create occurrences and advance
81
+ * the per-session seq — but cannot read arbitrary history, run migrations, or
82
+ * touch the compat journal. Enforces monotonic seq and tool-call reference
83
+ * completeness on every append.
84
+ */
85
+ export interface LedgerWriter {
86
+ readonly kind: "LedgerWriter";
87
+ /** Append one occurrence; rejects on seq regression or a dangling tool ref. */
88
+ append(input: {
89
+ readonly session: string;
90
+ readonly seq: bigint;
91
+ readonly eventId: string;
92
+ readonly kind: string;
93
+ readonly toolCallId?: string;
94
+ readonly sourceBytes: Uint8Array;
95
+ readonly digest?: string;
96
+ }): LedgerAppendResult;
97
+ /** Append a batch, all-or-nothing per occurrence (each returns its outcome). */
98
+ appendBatch(inputs: ReadonlyArray<Parameters<LedgerWriter["append"]>[0]>): LedgerAppendResult[];
99
+ }
100
+
101
+ /**
102
+ * Admin capability: maintenance + migration coordination only. The sole surface
103
+ * with access to the compatibility journal switch and to run the M2
104
+ * copy/validate/switch. Never exposed to ingestion or to the dashboard reader.
105
+ */
106
+ export interface LedgerAdmin {
107
+ readonly kind: "LedgerAdmin";
108
+ /** The compatibility journal (downgrade safety). */
109
+ readonly compat: CompatJournalV1;
110
+ /** Prepare + validate + switch the v2 ledger (M2), idempotently. */
111
+ migrateOccurrenceV2(): { ok: boolean; codes: readonly string[] };
112
+ }
113
+
114
+ /**
115
+ * CompatJournalV1 — the downgrade-safety journal. Every accepted v2 append
116
+ * atomically appends a journal record holding the original bytes, IDs, and a
117
+ * legacy projection or an explicit `unrepresentable` marker. The journal records
118
+ * its own lifecycle state (prepared/copied/validated/switched) so M2 is
119
+ * resumable and a stop mid-journal never corrupts v2.
120
+ */
121
+ export interface CompatJournalV1 {
122
+ /** Whether any v2 append has activated the journal (journal present). */
123
+ active(): boolean;
124
+ /**
125
+ * Record a new v2 occurrence into the journal. `legacyProjection` holds the
126
+ * lossless legacy copy, or null when the row has no lossless legacy form — in
127
+ * which case the row is marked `unrepresentable` (e.g. invalid UTF-8 bytes).
128
+ */
129
+ record(input: {
130
+ readonly occurrence: LedgerOccurrence;
131
+ readonly legacyProjection: string | null;
132
+ }): void;
133
+ /** True when a row could not be projected to legacy and is marked. */
134
+ isUnrepresentable(eventId: string, digest: string): boolean;
135
+ /**
136
+ * Prepare a downgrade export: snapshot the journal for the copy/validate/switch
137
+ * lifecycle. Returns the list of unrepresentable rows (EVT/MIG reference).
138
+ */
139
+ prepare(): string[];
140
+ /** Mark the copy phase complete. */
141
+ copied(): void;
142
+ /** Validate the copied legacy export (sequence/digest parity). */
143
+ validate(): { ok: boolean; codes: readonly string[] };
144
+ /** Atomically switch authority to the legacy export (downgrade). */
145
+ switched(): void;
146
+ }
147
+
148
+ // ---------------------------------------------------------------------------
149
+ // Factory + capability gating (VC1B)
150
+ // ---------------------------------------------------------------------------
151
+
152
+ import type { DatabaseSync } from "node:sqlite";
153
+ import { VC1B_ENABLED } from "../../config/vector-cortex.js";
154
+ import {
155
+ openOccurrenceStore,
156
+ appendOccurrence,
157
+ ledgerHighWater,
158
+ readSessionOccurrences,
159
+ readFromSeq,
160
+ countOccurrences,
161
+ hasOccurrence,
162
+ } from "./sqlite.js";
163
+ import {
164
+ initCompatJournal,
165
+ createCompatJournal,
166
+ journalPhase,
167
+ } from "./compat-journal.js";
168
+ import {
169
+ m2Copy,
170
+ m2Validate,
171
+ m2Switch,
172
+ type M2Host,
173
+ type LegacyExportRow,
174
+ } from "../migrations/occurrence-v2.js";
175
+
176
+ /** The token that gates closures to a single capability (PREVENT-011-free). */
177
+ const _capability: unique symbol = Symbol("mc-ledger-capability");
178
+
179
+ interface ReaderToken {
180
+ readonly [_capability]: "reader";
181
+ }
182
+ interface WriterToken {
183
+ readonly [_capability]: "writer";
184
+ }
185
+ interface AdminToken {
186
+ readonly [_capability]: "admin";
187
+ }
188
+
189
+ /** A capability-gated ledger handle: access only what you were handed. */
190
+ export interface LedgerHandle {
191
+ readonly reader: () => LedgerReader & ReaderToken;
192
+ readonly writer: () => LedgerWriter & WriterToken;
193
+ readonly admin: () => LedgerAdmin & AdminToken;
194
+ /** Close the underlying DB handle (test/sandbox teardown). */
195
+ readonly close: () => void;
196
+ }
197
+
198
+ /** Optional structured-event emitter (same shape as the other VC seams). */
199
+ export type LedgerEmit = (event: string, fields: Record<string, unknown>) => void;
200
+
201
+ /**
202
+ * Create the capability-separated occurrence-v2 ledger over its OWN isolated
203
+ * SQLite DB. `emit` is optional; occurrence-appended and compat-switch-committed
204
+ * events are only emitted when VC1B_ENABLED() and an emitter is supplied (mode-C
205
+ * parity: flag OFF / no emitter => zero observability writes).
206
+ *
207
+ * Normalization: `stateDir` gives the standard daemon location
208
+ * `<stateDir>/vector-cortex/occurrence-v2.db`; a bare `dbPath` overrides it
209
+ * (tests/rehearsal isolate the ledger). The compat journal shares the same DB
210
+ * so a v2 append + its journal record commit atomically.
211
+ */
212
+ export function createLedgerStore(
213
+ opts: { readonly stateDir: string } | { readonly dbPath: string },
214
+ emit?: LedgerEmit,
215
+ ): LedgerHandle {
216
+ const dbPath =
217
+ "dbPath" in opts
218
+ ? opts.dbPath
219
+ : `${opts.stateDir}/vector-cortex/occurrence-v2.db`;
220
+ const db = openOccurrenceStore(dbPath);
221
+ initCompatJournal(db);
222
+ const compat = createCompatJournal(db);
223
+
224
+ const fire = (event: string, fields: Record<string, unknown>): void => {
225
+ if (!VC1B_ENABLED()) return;
226
+ try {
227
+ emit?.(event, fields);
228
+ } catch {
229
+ /* non-fatal observability — never break the agent loop */
230
+ }
231
+ };
232
+
233
+ const asReader = (): LedgerReader & ReaderToken => ({
234
+ kind: "LedgerReader",
235
+ [_capability]: "reader" as const,
236
+ highWater: (session) => ledgerHighWater(db, session),
237
+ readSession: (session) => readSessionOccurrences(db, session),
238
+ readFrom: (session, fromSeq) => readFromSeq(db, session, fromSeq),
239
+ count: (session) => countOccurrences(db, session),
240
+ hasOccurrence: (session, eventId, digest) =>
241
+ hasOccurrence(db, session, eventId, digest),
242
+ });
243
+
244
+ const asWriter = (): LedgerWriter & WriterToken => ({
245
+ kind: "LedgerWriter",
246
+ [_capability]: "writer" as const,
247
+ append(input) {
248
+ // The occurrence insert and its compat-journal record commit atomically
249
+ // inside one nested savepoint (CONTRACTS §Store: atomically appends).
250
+ db.exec("SAVEPOINT mc_ledger_app");
251
+ try {
252
+ const result = appendOccurrence(db, input);
253
+ if (result.ok) {
254
+ compat.record({
255
+ occurrence: result.occurrence,
256
+ legacyProjection: legacyProjectionOf(result.occurrence),
257
+ });
258
+ }
259
+ if (result.ok) {
260
+ db.exec("RELEASE mc_ledger_app");
261
+ fire("vector_cortex_occurrence_appended", {
262
+ session: result.occurrence.session,
263
+ seq: result.occurrence.seq.toString(),
264
+ eventId: result.occurrence.eventId,
265
+ digest: result.occurrence.digest,
266
+ kind: result.occurrence.kind,
267
+ });
268
+ } else {
269
+ db.exec("ROLLBACK TO mc_ledger_app");
270
+ db.exec("RELEASE mc_ledger_app");
271
+ }
272
+ return result;
273
+ } catch (e) {
274
+ db.exec("ROLLBACK TO mc_ledger_app");
275
+ db.exec("RELEASE mc_ledger_app");
276
+ throw e;
277
+ }
278
+ },
279
+ appendBatch(inputs) {
280
+ const out = [];
281
+ for (const input of inputs) out.push(this.append(input));
282
+ return out;
283
+ },
284
+ });
285
+
286
+ const asAdmin = (): LedgerAdmin & AdminToken => ({
287
+ kind: "LedgerAdmin",
288
+ [_capability]: "admin" as const,
289
+ compat,
290
+ migrateOccurrenceV2() {
291
+ // copy => validate => switch; resumable & idempotent. The M2 host wraps
292
+ // the live DB + journal so every phase transition advances the SAME
293
+ // singleton journal state machine (prepared→copied→validated→switched).
294
+ const unrepresentable = compat.prepare().length;
295
+ const mig = migrateHost(db, compat);
296
+ m2Copy(mig);
297
+ const v = m2Validate(mig);
298
+ if (v.ok) {
299
+ m2Switch(mig);
300
+ fire("vector_cortex_compat_switch_committed", {
301
+ session: "all",
302
+ unrepresentable,
303
+ });
304
+ }
305
+ return { ok: v.ok, codes: v.codes };
306
+ },
307
+ });
308
+
309
+ return { reader: asReader, writer: asWriter, admin: asAdmin, close: () => db.close() };
310
+ }
311
+
312
+ /**
313
+ * Lossless legacy projection for a v2 occurrence: its identity envelope plus
314
+ * base64 source bytes. Returns null (→ journal `unrepresentable` marker) when the
315
+ * source bytes do not form valid UTF-8, which the legacy pipeline cannot round-trip
316
+ * losslessly — those rows are listed rather than silently coerced (MIG-DOWN-003).
317
+ */
318
+ function legacyProjectionOf(occ: LedgerOccurrence): string | null {
319
+ const source = Buffer.from(occ.sourceBytes);
320
+ try {
321
+ new TextDecoder("utf-8", { fatal: true }).decode(source);
322
+ } catch {
323
+ return null;
324
+ }
325
+ return JSON.stringify({
326
+ session: occ.session,
327
+ eventId: occ.eventId,
328
+ kind: occ.kind,
329
+ digest: occ.digest,
330
+ source: source.toString("base64"),
331
+ });
332
+ }
333
+
334
+ /** Build an M2Host over the live ledger DB + journal (for migrateOccurrenceV2). */
335
+ function migrateHost(
336
+ db: DatabaseSync,
337
+ journal: CompatJournalV1,
338
+ ): M2Host {
339
+ let staged: readonly LegacyExportRow[] | null = null;
340
+ return {
341
+ db,
342
+ phase: () => journalPhase(db),
343
+ journalActive: () => journal.active(),
344
+ journalRows: () => journalRowsOf(db),
345
+ writeStagedLegacy: (rows) => {
346
+ staged = rows;
347
+ journal.copied();
348
+ },
349
+ stagedLegacy: () => staged,
350
+ validateStaged: () => journal.validate(),
351
+ switchLegacy: () => journal.switched(),
352
+ };
353
+ }
354
+
355
+ /** Read the journaled occurrences as legacy-export rows. */
356
+ function journalRowsOf(db: DatabaseSync): readonly LegacyExportRow[] {
357
+ const rows = db
358
+ .prepare(
359
+ `SELECT session, seq, event_id, digest, kind, legacy_projection, unrepresentable
360
+ FROM compat_journal_v1 ORDER BY session ASC, seq ASC`,
361
+ )
362
+ .all() as unknown as Array<{
363
+ session: string;
364
+ seq: number;
365
+ event_id: string;
366
+ digest: string;
367
+ kind: string;
368
+ legacy_projection: string | null;
369
+ unrepresentable: number;
370
+ }>;
371
+ return rows.map((r) => ({
372
+ session: r.session,
373
+ seq: BigInt(r.seq),
374
+ eventId: r.event_id,
375
+ digest: r.digest,
376
+ kind: r.kind,
377
+ legacyProjection: r.legacy_projection,
378
+ unrepresentable: r.unrepresentable === 1,
379
+ }));
380
+ }