pi-mega-compact 0.20.5 → 0.20.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/dist/config/vector-cortex.js +12 -0
  2. package/dist/config.js +1 -1
  3. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +3 -31
  4. package/dist/extensions/dashboard-server/routes-rag-settings-types.js +11 -0
  5. package/dist/extensions/dashboard-server/routes-rag-settings-vector-cortex.js +43 -0
  6. package/dist/extensions/dashboard-server/routes-vector-cortex-rollout.js +43 -0
  7. package/dist/extensions/dashboard-server/routes-vector-cortex.js +1 -0
  8. package/dist/extensions/dashboard-server/routes.js +1 -1
  9. package/dist/extensions/dashboard-server/server.js +3 -1
  10. package/dist/extensions/mega-runtime/vector-cortex-live.js +105 -0
  11. package/dist/src/config/dedup.js +1 -0
  12. package/dist/src/config/vector-cortex.js +12 -0
  13. package/dist/src/config.js +1 -1
  14. package/dist/src/dedup-audit.test-helpers.js +43 -0
  15. package/dist/src/monitoring.js +9 -0
  16. package/dist/src/vector-cortex/rollout/_acceptance-fixture.js +55 -0
  17. package/dist/src/vector-cortex/rollout/_acceptance-helpers.js +8 -0
  18. package/dist/src/vector-cortex/rollout/_acceptance-scenario.js +70 -0
  19. package/dist/src/vector-cortex/rollout/assign.js +56 -0
  20. package/dist/src/vector-cortex/rollout/emit.js +50 -0
  21. package/dist/src/vector-cortex/rollout/gate.js +75 -0
  22. package/dist/src/vector-cortex/rollout/types.js +39 -0
  23. package/dist/src/vectorStore/add-l0.js +108 -0
  24. package/dist/src/vectorStore/add-l1.js +35 -0
  25. package/dist/src/vectorStore/add.js +229 -0
  26. package/dist/src/vectorStore/class.js +15 -291
  27. package/dist/src/vectorStore/dedup-audit.js +104 -0
  28. package/dist/src/vectorStore.js +1 -0
  29. package/dist/vector-cortex/rollout/_acceptance-fixture.js +55 -0
  30. package/dist/vector-cortex/rollout/_acceptance-helpers.js +8 -0
  31. package/dist/vector-cortex/rollout/_acceptance-scenario.js +70 -0
  32. package/dist/vector-cortex/rollout/assign.js +56 -0
  33. package/dist/vector-cortex/rollout/emit.js +50 -0
  34. package/dist/vector-cortex/rollout/gate.js +75 -0
  35. package/dist/vector-cortex/rollout/types.js +39 -0
  36. package/extensions/dashboard-client/dist/assets/{AreaChart-CGL29Tcl.js → AreaChart-DqTbaSRA.js} +2 -2
  37. package/extensions/dashboard-client/dist/assets/{AreaChart-CGL29Tcl.js.map → AreaChart-DqTbaSRA.js.map} +1 -1
  38. package/extensions/dashboard-client/dist/assets/{BarChart-OxhalIZa.js → BarChart-BfNBeOvF.js} +2 -2
  39. package/extensions/dashboard-client/dist/assets/{BarChart-OxhalIZa.js.map → BarChart-BfNBeOvF.js.map} +1 -1
  40. package/extensions/dashboard-client/dist/assets/{CacheTab-VcMFwL-6.js → CacheTab-C9SOMYB5.js} +2 -2
  41. package/extensions/dashboard-client/dist/assets/{CacheTab-VcMFwL-6.js.map → CacheTab-C9SOMYB5.js.map} +1 -1
  42. package/extensions/dashboard-client/dist/assets/{EventsTab-DePofXDM.js → EventsTab-GsMKP10_.js} +2 -2
  43. package/extensions/dashboard-client/dist/assets/{EventsTab-DePofXDM.js.map → EventsTab-GsMKP10_.js.map} +1 -1
  44. package/extensions/dashboard-client/dist/assets/{HealthTab-nQd1tjSF.js → HealthTab-DAOCXwRP.js} +2 -2
  45. package/extensions/dashboard-client/dist/assets/{HealthTab-nQd1tjSF.js.map → HealthTab-DAOCXwRP.js.map} +1 -1
  46. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-CzvA8-F6.js → MaintenanceTab-BtBeMkB5.js} +2 -2
  47. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-CzvA8-F6.js.map → MaintenanceTab-BtBeMkB5.js.map} +1 -1
  48. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-3lxgWBvU.js → MemoryMapTab-DzndtgkZ.js} +2 -2
  49. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-3lxgWBvU.js.map → MemoryMapTab-DzndtgkZ.js.map} +1 -1
  50. package/extensions/dashboard-client/dist/assets/{MetricsTab-D_FcA6kN.js → MetricsTab-DyxJ4eb8.js} +2 -2
  51. package/extensions/dashboard-client/dist/assets/{MetricsTab-D_FcA6kN.js.map → MetricsTab-DyxJ4eb8.js.map} +1 -1
  52. package/extensions/dashboard-client/dist/assets/{OverviewTab-BU6R4aEs.js → OverviewTab-DCiEdVWR.js} +2 -2
  53. package/extensions/dashboard-client/dist/assets/{OverviewTab-BU6R4aEs.js.map → OverviewTab-DCiEdVWR.js.map} +1 -1
  54. package/extensions/dashboard-client/dist/assets/{ReposTab-DFjy1isz.js → ReposTab-B5SIqVHA.js} +2 -2
  55. package/extensions/dashboard-client/dist/assets/{ReposTab-DFjy1isz.js.map → ReposTab-B5SIqVHA.js.map} +1 -1
  56. package/extensions/dashboard-client/dist/assets/{SessionsTab-UGIzIH25.js → SessionsTab-DLWFJ9xI.js} +2 -2
  57. package/extensions/dashboard-client/dist/assets/{SessionsTab-UGIzIH25.js.map → SessionsTab-DLWFJ9xI.js.map} +1 -1
  58. package/extensions/dashboard-client/dist/assets/{SetupTab-CECrgMKI.js → SetupTab-CvGXZyej.js} +2 -2
  59. package/extensions/dashboard-client/dist/assets/{SetupTab-CECrgMKI.js.map → SetupTab-CvGXZyej.js.map} +1 -1
  60. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-Dyy4SCDk.js → TimeSavedCard-wZuQGxLS.js} +2 -2
  61. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-Dyy4SCDk.js.map → TimeSavedCard-wZuQGxLS.js.map} +1 -1
  62. package/extensions/dashboard-client/dist/assets/{TurnsTab-B0LPcVs1.js → TurnsTab-B7aOejch.js} +2 -2
  63. package/extensions/dashboard-client/dist/assets/{TurnsTab-B0LPcVs1.js.map → TurnsTab-B7aOejch.js.map} +1 -1
  64. package/extensions/dashboard-client/dist/assets/VectorCortexTab-eHVt9yDx.js +2 -0
  65. package/extensions/dashboard-client/dist/assets/VectorCortexTab-eHVt9yDx.js.map +1 -0
  66. package/extensions/dashboard-client/dist/assets/{WikiTab-DWk9Qvut.js → WikiTab-DhWHjAtt.js} +2 -2
  67. package/extensions/dashboard-client/dist/assets/{WikiTab-DWk9Qvut.js.map → WikiTab-DhWHjAtt.js.map} +1 -1
  68. package/extensions/dashboard-client/dist/assets/{button-O59wPr5H.js → button-BrC-W87h.js} +2 -2
  69. package/extensions/dashboard-client/dist/assets/{button-O59wPr5H.js.map → button-BrC-W87h.js.map} +1 -1
  70. package/extensions/dashboard-client/dist/assets/{card-C-DxXv1-.js → card-0rgfRgr3.js} +2 -2
  71. package/extensions/dashboard-client/dist/assets/{card-C-DxXv1-.js.map → card-0rgfRgr3.js.map} +1 -1
  72. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-Dw5YnE5s.js → generateCategoricalChart-DRresOQF.js} +2 -2
  73. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-Dw5YnE5s.js.map → generateCategoricalChart-DRresOQF.js.map} +1 -1
  74. package/extensions/dashboard-client/dist/assets/{index-i9oaPE0a.js → index-DohmtZWL.js} +3 -3
  75. package/extensions/dashboard-client/dist/assets/{index-i9oaPE0a.js.map → index-DohmtZWL.js.map} +1 -1
  76. package/extensions/dashboard-client/dist/assets/{switch-B69QY3IM.js → switch-Bs8g_zB5.js} +2 -2
  77. package/extensions/dashboard-client/dist/assets/{switch-B69QY3IM.js.map → switch-Bs8g_zB5.js.map} +1 -1
  78. package/extensions/dashboard-client/dist/assets/{toggle-BJ4Kwggu.js → toggle-DVVauIo8.js} +2 -2
  79. package/extensions/dashboard-client/dist/assets/{toggle-BJ4Kwggu.js.map → toggle-DVVauIo8.js.map} +1 -1
  80. package/extensions/dashboard-client/dist/assets/{useSSE-BpuA8sPd.js → useSSE-DxH1PTAR.js} +2 -2
  81. package/extensions/dashboard-client/dist/assets/{useSSE-BpuA8sPd.js.map → useSSE-DxH1PTAR.js.map} +1 -1
  82. package/extensions/dashboard-client/dist/index.html +1 -1
  83. package/extensions/dashboard-client/src/api/vector-cortex.ts +9 -0
  84. package/extensions/dashboard-client/src/tabs/VectorCortexRolloutCard.tsx +44 -0
  85. package/extensions/dashboard-client/src/tabs/VectorCortexTab.tsx +8 -0
  86. package/extensions/dashboard-client/src/types/vector-cortex.ts +17 -0
  87. package/extensions/dashboard-server/api-contracts/vector-cortex.ts +32 -1
  88. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +11 -129
  89. package/extensions/dashboard-server/routes-rag-settings-types.ts +34 -0
  90. package/extensions/dashboard-server/routes-rag-settings-vector-cortex.ts +142 -0
  91. package/extensions/dashboard-server/routes-vector-cortex-rollout.ts +52 -0
  92. package/extensions/dashboard-server/routes-vector-cortex.ts +1 -0
  93. package/extensions/dashboard-server/routes.ts +1 -0
  94. package/extensions/dashboard-server/server.ts +2 -0
  95. package/extensions/mega-runtime/vector-cortex-live.ts +165 -0
  96. package/package.json +1 -1
  97. package/src/config/dedup.ts +9 -0
  98. package/src/config/vector-cortex.ts +13 -0
  99. package/src/config.ts +1 -0
  100. package/src/dedup-audit.test-helpers.ts +49 -0
  101. package/src/monitoring.ts +10 -0
  102. package/src/vector-cortex/rollout/_acceptance-fixture.ts +94 -0
  103. package/src/vector-cortex/rollout/_acceptance-helpers.ts +24 -0
  104. package/src/vector-cortex/rollout/_acceptance-scenario.ts +96 -0
  105. package/src/vector-cortex/rollout/assign.ts +67 -0
  106. package/src/vector-cortex/rollout/emit.ts +81 -0
  107. package/src/vector-cortex/rollout/gate.ts +119 -0
  108. package/src/vector-cortex/rollout/types.ts +111 -0
  109. package/src/vectorStore/add-l0.ts +152 -0
  110. package/src/vectorStore/add-l1.ts +57 -0
  111. package/src/vectorStore/add.ts +269 -0
  112. package/src/vectorStore/class.ts +17 -327
  113. package/src/vectorStore/dedup-audit.ts +199 -0
  114. package/src/vectorStore.ts +1 -0
  115. package/extensions/dashboard-client/dist/assets/VectorCortexTab-BWi3u_gp.js +0 -2
  116. package/extensions/dashboard-client/dist/assets/VectorCortexTab-BWi3u_gp.js.map +0 -1
@@ -1,34 +1,21 @@
1
- /**
2
- * class.ts — VectorStore implementation (extracted from vectorStore.ts).
3
- *
4
- * One store, three consumers (per PLAN.md): auto-inline on resume, on-demand
5
- * /recall-context, and the dedup sentinel. All share `add / search / dedupe`.
6
- *
7
- * Backed by the gzipped on-disk checkpoint files (store.ts). Similarity is a
8
- * linear cosine scan — checkpoint counts are small, so no ANN index is needed.
9
- */
10
- import { createHash } from "node:crypto";
11
- import { cosineSimilarity, defaultEmbedder } from "../embedder.js";
1
+ import { defaultEmbedder } from "../embedder.js";
12
2
  import { loadDedupConfig, } from "../config/dedup.js";
13
3
  import { logDecision } from "../monitoring.js";
14
4
  import { repoKey } from "../store/repoKey.js";
15
- import { getStateDir, normalizeSessionId, compressSmart } from "../store.js";
16
- import { computeContentDigest } from "../dedup/digest.js";
17
- import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES, } from "../dedup/l1-minhash.js";
18
- import { lshBands } from "../dedup/l1-lsh.js";
19
- import { isNearDuplicate } from "../dedup/l1-verify.js";
20
- import { openBloom, saveBloom } from "../store/bloom.js";
21
- import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, addTokensSaved, bumpDedupStats, } from "../store/sqlite.js";
5
+ import { getStateDir } from "../store.js";
6
+ import { openBloom } from "../store/bloom.js";
22
7
  import { migrateJsonToSqlite } from "../store/migrate.js";
23
- import { computeRegionHash } from "./hash.js";
8
+ import { addCheckpoint } from "./add.js";
24
9
  export class VectorStore {
25
10
  // These fields are `readonly` (set once in the constructor) but NOT private:
26
- // the read/search/dedup helpers split into vector-read.ts, vector-search.ts,
27
- // and vector-dedup.ts access them directly. Marking them private would force
28
- // ugly `as unknown as` casts in those modules; keeping them package-public
29
- // makes VectorStore a thin barrel whose helpers live in sibling files.
11
+ // the add/read/search/dedup helpers split into add.ts, vector-read.ts,
12
+ // vector-search.ts, and vector-dedup.ts access them directly. Marking them
13
+ // private would force ugly `as unknown as` casts in those modules; keeping
14
+ // them package-public makes VectorStore a thin barrel whose helpers live in
15
+ // sibling files.
30
16
  embedder;
31
17
  stateDir;
18
+ /** L2 semantic firing point (cfg.L2_COSINE, or a direct test override). */
32
19
  l2Threshold;
33
20
  /** Single source of truth for tier flags + thresholds (Sprint 14). */
34
21
  cfg;
@@ -82,275 +69,12 @@ export class VectorStore {
82
69
  });
83
70
  }
84
71
  /**
85
- * Add a checkpoint. Dedup cascade:
86
- * 1. regionHash exact match (legacy, backward-compat)
87
- * 2. summaryHash exact match (new: catches same-topic incremental compactions)
88
- * 3. content similarity dedupSim (catches near-identical summaries)
89
- * 4. If none match → create new checkpoint
72
+ * Add a checkpoint, deduping it against the session's existing checkpoints.
73
+ *
74
+ * The cascade itself lives in ./add.ts (sibling-helper pattern) so this class
75
+ * stays a thin shell over the fields its helpers read.
90
76
  */
91
77
  add(input) {
92
- const t0 = Date.now();
93
- const sessionId = normalizeSessionId(input.sessionId);
94
- const regionHash = computeRegionHash(input.regionText);
95
- const all = listCheckpoints(sessionId, this.stateDir);
96
- // Honest "tokens saved" base for this region. For a deduped add the whole
97
- // original region is discarded (nothing new stored); for a new checkpoint
98
- // we persist (orig − stored). Falls back to stored when orig is unknown.
99
- const origTokens = input.originalTokenEstimate ?? input.tokenEstimate ?? 0;
100
- const cfg = this.cfg;
101
- // Live per-tier progress hook (Phase 1). Sync + optional; fired at each tier
102
- // so the UI can paint "L0 ✓ → L1 ✓ → L2 0.91 → stored" during a compaction.
103
- const onTier = input.onTier;
104
- // Tracks whether a tier matched while in MARK_ONLY (record-but-don't-collapse),
105
- // and which tier.
106
- let markOnly = null;
107
- // 0. L0 content-hash dedup (Sprint 9) — catches identical content arriving
108
- // under different regionText. Normalization handles case/whitespace/ANSI so
109
- // variants collapse to one row. Dual-hash guards a single-hash collision.
110
- // Sprint 10: bloom is the accelerator — a miss means "definitely new" and
111
- // skips the scan; a hit is only a candidate, confirmed against `all` below.
112
- // Gated by L0_ENABLED (Sprint 14). MARK_ONLY_L0 records the decision but
113
- // does not collapse — the new region is still stored.
114
- onTier?.({ tier: "L0", status: "scanning" });
115
- const digest = computeContentDigest(input.regionText);
116
- const bloom = openBloom(this.stateDir);
117
- if (cfg.L0_ENABLED && bloom.maybeHas(digest.contentHash)) {
118
- const contentMatch = all.find((cp) => cp.contentHash === digest.contentHash &&
119
- cp.contentHash2 === digest.contentHash2);
120
- if (contentMatch) {
121
- if (cfg.MARK_ONLY_L0) {
122
- markOnly = "L0"; // Record-but-don't-collapse: fall through.
123
- }
124
- else {
125
- contentMatch.timestamp = input.timestamp;
126
- upsertCheckpoint(contentMatch, this.stateDir);
127
- bumpDedupStats(true, this.stateDir);
128
- // Deduped: whole original region discarded, nothing new stored.
129
- addTokensSaved(origTokens, this.stateDir);
130
- const r = {
131
- checkpoint: contentMatch,
132
- deduped: true,
133
- reason: "contentHash",
134
- };
135
- this.record("L0", "deduped", "contentHash", Date.now() - t0, 1, contentMatch.checkpointId);
136
- onTier?.({ tier: "L0", status: "deduped", detail: "contentHash" });
137
- return r;
138
- }
139
- }
140
- }
141
- // 1. Legacy regionHash dedup (backward-compat) — part of L0 tier gating.
142
- if (cfg.L0_ENABLED) {
143
- const regionMatch = all.find((cp) => cp.regionHash === regionHash);
144
- if (regionMatch) {
145
- if (cfg.MARK_ONLY_L0) {
146
- markOnly = "L0"; // fall through
147
- }
148
- else {
149
- bumpDedupStats(true, this.stateDir);
150
- // Deduped: whole original region discarded, nothing new stored.
151
- addTokensSaved(origTokens, this.stateDir);
152
- const r = {
153
- checkpoint: regionMatch,
154
- deduped: true,
155
- reason: "regionHash",
156
- };
157
- this.record("L0", "deduped", "regionHash", Date.now() - t0, 1, regionMatch.checkpointId);
158
- onTier?.({ tier: "L0", status: "deduped", detail: "regionHash" });
159
- return r;
160
- }
161
- }
162
- }
163
- // 2. SummaryHash dedup — catches same-topic incremental compactions.
164
- // Full 64-hex SHA-256 (was 16-hex in Sprint 8 — collision-prone).
165
- const summaryHash = input.topicSummary
166
- ? createHash("sha256").update(input.topicSummary).digest("hex")
167
- : undefined;
168
- if (summaryHash && cfg.L0_ENABLED) {
169
- const summaryMatch = all.find((cp) => cp.summaryHash === summaryHash);
170
- if (summaryMatch) {
171
- if (cfg.MARK_ONLY_L0) {
172
- markOnly = "L0"; // fall through
173
- }
174
- else {
175
- summaryMatch.timestamp = input.timestamp;
176
- upsertCheckpoint(summaryMatch, this.stateDir);
177
- bumpDedupStats(true, this.stateDir);
178
- // Deduped: whole original region discarded, nothing new stored.
179
- addTokensSaved(origTokens, this.stateDir);
180
- const r = {
181
- checkpoint: summaryMatch,
182
- deduped: true,
183
- reason: "summaryHash",
184
- };
185
- this.record("L0", "deduped", "summaryHash", Date.now() - t0, 1, summaryMatch.checkpointId);
186
- onTier?.({ tier: "L0", status: "deduped", detail: "summaryHash" });
187
- return r;
188
- }
189
- }
190
- }
191
- // L0 did not collapse this region.
192
- onTier?.({ tier: "L0", status: "passed" });
193
- // 2b. L1 MinHash/LSH near-duplicate dedup (Sprint 11) — catches one-word
194
- // edits / rewordings that L0's exact hash misses. Cheap LSH bucket
195
- // retrieval → trigram verification (pg_trgm-equivalent) as the final gate.
196
- // Gated by L1_ENABLED (Sprint 14); MARK_ONLY_L1 records but doesn't collapse.
197
- onTier?.({ tier: "L1", status: "scanning" });
198
- if (cfg.L1_ENABLED) {
199
- const l1 = this.findL1Duplicate(sessionId, input.regionText, all);
200
- if (l1 && !cfg.MARK_ONLY_L1) {
201
- l1.timestamp = input.timestamp;
202
- upsertCheckpoint(l1, this.stateDir);
203
- bumpDedupStats(true, this.stateDir);
204
- const r = { checkpoint: l1, deduped: true, reason: "l1MinHash" };
205
- this.record("L1", "deduped", "l1MinHash", Date.now() - t0, 1, l1.checkpointId);
206
- onTier?.({ tier: "L1", status: "deduped", detail: "l1MinHash" });
207
- return r;
208
- }
209
- if (l1 && cfg.MARK_ONLY_L1)
210
- markOnly = "L1";
211
- }
212
- onTier?.({ tier: "L1", status: "passed" });
213
- // 3. L2 semantic dedup — catches near-identical / semantically-similar regions
214
- // via cosine over the embedding. topicSummary is used for summaryHash dedup
215
- // (tier 2); the vector index is keyed on the original region for backward-
216
- // compat search semantics. Threshold from cfg (L2_COSINE trigram honest
217
- // firing point). QA #13 timeout guard: if the O(n) scan exceeds the budget,
218
- // degrade to "store without dedup this pass" so we never lose a checkpoint.
219
- // Gated by L2_ENABLED (Sprint 14); MARK_ONLY_L2 records but doesn't collapse.
220
- const SIMILARITY_BUDGET_MS = cfg.SIMILARITY_BUDGET_MS;
221
- const simThreshold = this.l2Threshold; // from cfg.L2_COSINE (default 0.85 trigram)
222
- const embedding = this.embedder.embed(input.regionText);
223
- onTier?.({ tier: "L2", status: "scanning" });
224
- if (cfg.L2_ENABLED && all.length > 0) {
225
- const start = Date.now();
226
- let timedOut = false;
227
- const nearest = all.reduce((best, cp) => {
228
- if (!timedOut && Date.now() - start > SIMILARITY_BUDGET_MS)
229
- timedOut = true;
230
- if (timedOut)
231
- return best;
232
- const sim = cosineSimilarity(embedding, cp.embedding);
233
- return sim > best.sim ? { checkpoint: cp, sim } : best;
234
- }, { checkpoint: all[0], sim: -1 });
235
- if (!timedOut && nearest.sim >= simThreshold) {
236
- if (!cfg.MARK_ONLY_L2) {
237
- // Near-identical — update timestamp on existing checkpoint
238
- nearest.checkpoint.timestamp = input.timestamp;
239
- upsertCheckpoint(nearest.checkpoint, this.stateDir);
240
- bumpDedupStats(true, this.stateDir);
241
- // Deduped: whole original region discarded, nothing new stored.
242
- addTokensSaved(origTokens, this.stateDir);
243
- const r = {
244
- checkpoint: nearest.checkpoint,
245
- deduped: true,
246
- reason: "contentSimilarity",
247
- };
248
- this.record("L2", "deduped", "contentSimilarity", Date.now() - t0, nearest.sim, nearest.checkpoint.checkpointId);
249
- onTier?.({
250
- tier: "L2",
251
- status: "deduped",
252
- detail: nearest.sim.toFixed(2),
253
- });
254
- return r;
255
- }
256
- markOnly = "L2";
257
- }
258
- onTier?.({
259
- tier: "L2",
260
- status: "passed",
261
- detail: `best ${nearest.sim.toFixed(2)}`,
262
- });
263
- }
264
- // 4. Genuinely new — create checkpoint
265
- const checkpointId = nextCheckpointId(sessionId, this.stateDir);
266
- const checkpoint = {
267
- checkpointId,
268
- sessionId,
269
- repoId: this.repoId,
270
- summary: input.summary,
271
- topicSummary: input.topicSummary,
272
- summaryHash,
273
- keyDecisions: input.keyDecisions ?? [],
274
- nextSteps: input.nextSteps ?? [],
275
- filesModified: input.filesModified ?? [],
276
- tokenEstimate: input.tokenEstimate ?? 0,
277
- originalTokenEstimate: input.originalTokenEstimate,
278
- regionHash,
279
- contentHash: digest.contentHash,
280
- contentHash2: digest.contentHash2,
281
- contentHashVersion: digest.contentHashVersion,
282
- normalizedText: digest.normalizedText,
283
- compressedOriginal: compressSmart(Buffer.from(input.regionText, "utf-8"), input.compressionPressure),
284
- embedding,
285
- timestamp: input.timestamp,
286
- };
287
- // Persistence is SQLite (store/sqlite.ts). upsertCheckpoint keeps the
288
- // idempotent-by-id semantics the old JSON append implied.
289
- upsertCheckpoint(checkpoint, this.stateDir);
290
- // Cumulative "tokens saved" counter (per-repo SQLite meta). For a NEW
291
- // checkpoint the saved amount is (original − stored); for a deduped add the
292
- // whole original region is discarded (handled in the deduped return paths
293
- // below). Survives sessions and travels with the repo.
294
- const stored = input.tokenEstimate ?? 0;
295
- addTokensSaved(Math.max(0, origTokens - stored), this.stateDir);
296
- // L1: persist this checkpoint's MinHash signature + LSH buckets so future
297
- // near-duplicate inserts can find it. Deterministic given the seed.
298
- const sig = minhashSignature(input.regionText);
299
- upsertMinhashSignature(checkpointId, sessionId, SIGNATURE_VERSION, sig, this.stateDir);
300
- insertLshBuckets(checkpointId, sessionId, SIGNATURE_VERSION, lshBands(sig, sessionId, SIGNATURE_VERSION), this.stateDir);
301
- // Bloom accelerator: record the new content_hash so a future add() can short-
302
- // circuit the scan on a hit (still confirmed by the SELECT-based `all` above).
303
- bloom.add(digest.contentHash);
304
- saveBloom(this.stateDir);
305
- // Track the region hash in session state for fast sentinel checks.
306
- const state = loadSessionState(sessionId, this.stateDir);
307
- if (!state.storedRegionHashes.includes(regionHash)) {
308
- state.storedRegionHashes.push(regionHash);
309
- saveSessionState(sessionId, state, this.stateDir);
310
- }
311
- // A new checkpoint. If a tier matched while MARK_ONLY, record that (the
312
- // decision fired but we intentionally did not collapse).
313
- if (markOnly) {
314
- this.record(markOnly, "mark_only", "mark_only", Date.now() - t0);
315
- }
316
- else {
317
- this.record("L0", "new", undefined, Date.now() - t0);
318
- }
319
- // Cumulative store-wide dedup accounting (attempt, not collapsed).
320
- bumpDedupStats(false, this.stateDir);
321
- onTier?.({ tier: "new", status: "stored" });
322
- return { checkpoint, deduped: false };
323
- }
324
- /**
325
- * L1 near-duplicate lookup: MinHash → LSH candidate retrieval → trigram verify.
326
- * Returns the matching checkpoint or undefined. Bounded by a 100-candidate cap
327
- * and a 20ms verify budget (QA #7/#15) so it never hangs a large session.
328
- */
329
- findL1Duplicate(sessionId, regionText, all) {
330
- if (all.length === 0)
331
- return undefined;
332
- const sig = minhashSignature(regionText);
333
- if (sig.length !== NUM_HASHES)
334
- return undefined;
335
- const bands = lshBands(sig, sessionId, SIGNATURE_VERSION);
336
- // Cheap candidate retrieval (single query, capped). Exclude nothing yet —
337
- // the new checkpoint has no id, so pass a sentinel that never matches.
338
- const candidateIds = lshCandidateChunks(bands, sessionId, "__new__", this.stateDir, 100);
339
- if (candidateIds.length === 0)
340
- return undefined;
341
- const byId = new Map(all.map((cp) => [cp.checkpointId, cp]));
342
- const VERIFY_BUDGET_MS = 20;
343
- const start = Date.now();
344
- for (const id of candidateIds) {
345
- if (Date.now() - start > VERIFY_BUDGET_MS)
346
- break; // QA #15: abort → "not dup"
347
- const cand = byId.get(id);
348
- if (!cand)
349
- continue;
350
- const candText = cand.normalizedText ?? cand.summary ?? "";
351
- if (isNearDuplicate(regionText, candText))
352
- return cand;
353
- }
354
- return undefined;
78
+ return addCheckpoint(this, input);
355
79
  }
356
80
  }
@@ -0,0 +1,104 @@
1
+ /**
2
+ * dedup-audit.ts — durable audit trail for dedup tier decisions
3
+ * (external-audit item #2).
4
+ *
5
+ * Before this module a tier decision existed only as the in-process `onTier`
6
+ * callback that paints the live UI; nothing survived the process, so an
7
+ * operator could not answer "which layer collapsed this region, onto what, at
8
+ * what similarity?" — the inputs needed to tune the thresholds in
9
+ * config/dedup.ts. Here each decision is appended to the repo's events.log as
10
+ * one structured JSON line (see `DedupAuditEvent` below).
11
+ *
12
+ * The event type and its append helper live HERE rather than in monitoring.ts:
13
+ * monitoring.ts already owns three concerns (decision events, the dashboard.json
14
+ * metrics snapshot, FP alerting) and sits close to its 300-line soft limit, so
15
+ * co-locating the shape with the only recorder that produces it keeps both files
16
+ * under the headroom gate. monitoring.ts re-exports both for callers (and the
17
+ * dashboard SSE tail) that treat it as the events.log barrel.
18
+ *
19
+ * Design constraints:
20
+ * - PURE INSTRUMENTATION. Nothing in this file may influence a dedup outcome.
21
+ * - Best-effort/non-fatal: `logDedupAudit` swallows IO errors, and the emitter
22
+ * itself is wrapped so a malformed field can never break add().
23
+ * - Honest fields only: a value is emitted only where the caller actually
24
+ * computed it. L0/L1 are hash/verify tiers and pass no `similarity`.
25
+ * - Signal, not chatter: callers emit on DECISIONS (a match, a scored
26
+ * candidate, the final outcome), never on every "scanning" transition.
27
+ * - Flag-gated by cfg.DEDUP_AUDIT (default ON; OFF writes nothing at all).
28
+ *
29
+ * PREVENT-PI-004: local filesystem append only, no network.
30
+ */
31
+ import { appendFileSync, mkdirSync } from "node:fs";
32
+ import { dirname } from "node:path";
33
+ import { defaultEventsPath } from "../monitoring.js";
34
+ /**
35
+ * Append one audit event to events.log (best-effort, never throws).
36
+ *
37
+ * Same append-one-JSON-line contract as monitoring.ts's logDecision — an
38
+ * unwritable path is swallowed so instrumentation can never break add().
39
+ */
40
+ export function logDedupAudit(path, ev) {
41
+ try {
42
+ mkdirSync(dirname(path), { recursive: true });
43
+ appendFileSync(path, `${JSON.stringify(ev)}\n`, "utf-8");
44
+ }
45
+ catch {
46
+ /* best-effort — never break the extension on a log failure */
47
+ }
48
+ }
49
+ /** Build a recorder bound to one add() cascade. */
50
+ export function dedupAuditRecorder(ctx, scope) {
51
+ const base = {
52
+ sessionId: scope.sessionId,
53
+ originalTokenEstimate: scope.originalTokenEstimate,
54
+ tokenEstimate: scope.tokenEstimate,
55
+ };
56
+ return {
57
+ deduped: (tier, matchedEntry, dedupReason, similarity) => emitDedupAudit(ctx, {
58
+ ...base,
59
+ tier,
60
+ status: "deduped",
61
+ matchedEntry,
62
+ dedupReason,
63
+ ...(similarity === undefined ? {} : { similarity }),
64
+ }),
65
+ passed: (tier, matchedEntry, similarity) => emitDedupAudit(ctx, {
66
+ ...base,
67
+ tier,
68
+ status: "passed",
69
+ matchedEntry,
70
+ similarity,
71
+ }),
72
+ stored: (storedEntry, dedupReason, tokenEstimate) => emitDedupAudit(ctx, {
73
+ ...base,
74
+ tier: "new",
75
+ status: "stored",
76
+ storedEntry,
77
+ dedupReason,
78
+ tokenEstimate,
79
+ }),
80
+ };
81
+ }
82
+ /**
83
+ * Append one dedup decision to events.log.
84
+ *
85
+ * Resolves the target path from the explicit `eventsPath` when a caller opted
86
+ * in (Sprint 14 monitoring / tests), otherwise from the store's own per-repo
87
+ * state dir — production never passes `eventsPath`, so defaulting is what makes
88
+ * the audit trail actually exist on a real device.
89
+ */
90
+ export function emitDedupAudit(ctx, input) {
91
+ if (!ctx.auditEnabled)
92
+ return;
93
+ try {
94
+ const path = ctx.eventsPath ?? defaultEventsPath(ctx.stateDir);
95
+ logDedupAudit(path, {
96
+ type: "dedup_audit",
97
+ ts: new Date().toISOString(),
98
+ ...input,
99
+ });
100
+ }
101
+ catch {
102
+ /* instrumentation must never break the add() path */
103
+ }
104
+ }
@@ -8,6 +8,7 @@
8
8
  * "./vectorStore.js" are unchanged.
9
9
  */
10
10
  export { VectorStore } from "./vectorStore/class.js";
11
+ export { addCheckpoint } from "./vectorStore/add.js";
11
12
  export { computeRegionHash } from "./vectorStore/hash.js";
12
13
  export { L2_ENABLED, } from "./vectorStore/types.js";
13
14
  // Re-exports (back-compat): existing call sites keep importing from "./vectorStore.js"
@@ -0,0 +1,55 @@
1
+ /**
2
+ * rollout/_acceptance-fixture.ts — conformance fixture I/O for VC5C rollout
3
+ * acceptance rows.
4
+ *
5
+ * Reads the v2 conformance manifest + the per-sprint rollout fixture files, and
6
+ * exposes the `RolloutFx` shape the acceptance aggregator drives. No
7
+ * mocks/stubs: the fixtures are the committed canonical corpus.
8
+ */
9
+ import { readFileSync, existsSync } from "node:fs";
10
+ import { join, dirname } from "node:path";
11
+ import { fileURLToPath } from "node:url";
12
+ import assert from "node:assert/strict";
13
+ const HERE = dirname(fileURLToPath(import.meta.url));
14
+ function repoRoot(from) {
15
+ let dir = from;
16
+ for (let i = 0; i < 8; i++) {
17
+ if (existsSync(join(dir, "conformance", "vector-cortex")))
18
+ return dir;
19
+ const next = dirname(dir);
20
+ if (next === dir)
21
+ break;
22
+ dir = next;
23
+ }
24
+ throw new Error("conformance corpus not found above " + from);
25
+ }
26
+ export const REPO_ROOT = repoRoot(HERE);
27
+ export const V2 = join(REPO_ROOT, "conformance", "vector-cortex", "v2");
28
+ export function readManifest() {
29
+ return JSON.parse(readFileSync(join(V2, "manifest.json"), "utf8"));
30
+ }
31
+ function readFixture(id, prefix) {
32
+ const m = readManifest();
33
+ const row = m.fixtures.find((f) => f.id === id && f.path.startsWith(`${prefix}/`));
34
+ assert.ok(row, `fixture ${id} registered under ${prefix}/ in manifest`);
35
+ return JSON.parse(readFileSync(join(V2, row.path), "utf8"));
36
+ }
37
+ export function rolloutFixture(id) {
38
+ return readFixture(id, "rollout");
39
+ }
40
+ /** Flag-pinned wrapper: VC5C gated by MEGACOMPACT_VC5C (defaults ON). */
41
+ export function withFlagsOn(fn) {
42
+ return () => {
43
+ const saved = process.env.MEGACOMPACT_VC5C;
44
+ process.env.MEGACOMPACT_VC5C = "1";
45
+ try {
46
+ fn();
47
+ }
48
+ finally {
49
+ if (saved === undefined)
50
+ delete process.env.MEGACOMPACT_VC5C;
51
+ else
52
+ process.env.MEGACOMPACT_VC5C = saved;
53
+ }
54
+ };
55
+ }
@@ -0,0 +1,8 @@
1
+ /**
2
+ * rollout/_acceptance-helpers.ts — pure barrel re-export of the VC5C rollout
3
+ * acceptance helpers (fixture I/O + REAL runner). Mirrors the render family's
4
+ * delegate-shell split so the aggregator stays under the 600-line test hard
5
+ * limit.
6
+ */
7
+ export { REPO_ROOT, V2, readManifest, rolloutFixture, withFlagsOn, } from "./_acceptance-fixture.js";
8
+ export { runRolloutScenario } from "./_acceptance-scenario.js";
@@ -0,0 +1,70 @@
1
+ /**
2
+ * rollout/_acceptance-scenario.ts — the REAL rollout runner for VC5C acceptance
3
+ * rows. Drives `assignSession` + `decideGate` (no mocks) against the fixture
4
+ * `input` and returns the result fields the fixture's `expected` block pins.
5
+ *
6
+ * Fake clocks: the gate scenario accepts an injected monotonic clock so the
7
+ * spec's unique failure injection (restart at 71h59m with a wall-clock jump +1d
8
+ * but monotonic unchanged) is reproducible. The wall-clock jump is irrelevant to
9
+ * the safety decision because eligibility uses the MONOTONIC delta only.
10
+ */
11
+ import { assignSession } from "./assign.js";
12
+ import { decideGate, } from "./gate.js";
13
+ const GATE_MIN_ELAPSED_MS = 72 * 60 * 60 * 1000;
14
+ /** Fixed monotonic clock: elapsed = now - windowStart (the only driver). */
15
+ function clockAt(elapsedMs, windowStartMs) {
16
+ return {
17
+ now: () => windowStartMs + elapsedMs, // monotonic delta is the truth
18
+ wallNow: () => new Date(1_700_000_000_000 + elapsedMs).toISOString(),
19
+ };
20
+ }
21
+ /** Run a rollout fixture through the REAL assign + gate pipeline. */
22
+ export function runRolloutScenario(fx) {
23
+ const scenario = fx.input.scenario;
24
+ if (scenario === "assign-stable") {
25
+ const a = assignSession(fx.input.sessionId ?? "");
26
+ const b = assignSession(fx.input.sessionId ?? "");
27
+ // Determinism: recomputing yields the identical bucket.
28
+ if (a.bucket !== b.bucket) {
29
+ return { ok: false };
30
+ }
31
+ return { ok: true, bucket: a.bucket };
32
+ }
33
+ if (scenario === "gate-power" || scenario === "gate-safety") {
34
+ const ev = fx.input.evidence ?? {};
35
+ const windowStartMs = ev.windowStartMs ?? 0;
36
+ const hardFaults = (ev.hardFaults ?? []).map((f) => ({ kind: f.kind, detail: f.detail }));
37
+ const evidence = {
38
+ windowStartMs,
39
+ powered: ev.powered ?? false,
40
+ events: ev.events ?? 0,
41
+ sessions: ev.sessions ?? 0,
42
+ hardFaults,
43
+ };
44
+ // The spec's unique failure injection: wall-clock jump is irrelevant; only the
45
+ // monotonic elapsed drives eligibility. For the 71h59m row we pass 71h59m; for
46
+ // the wall-jump row we pass the SAME monotonic elapsed (so it must stay blocked);
47
+ // for the >=72h row we pass a full 72h+ monotonic delta (so it advances).
48
+ const start = fx._monotonicElapsedMs;
49
+ const elapsedMs = start ?? GATE_MIN_ELAPSED_MS; // default: a full 72h window
50
+ const clock = clockAt(elapsedMs, windowStartMs);
51
+ // gate-safety with a hard fault forces pre-VC + freezes promotion regardless of
52
+ // the monotonic window; the gate index stays at the input's prior gate (0 here).
53
+ if (scenario === "gate-safety") {
54
+ const outcome = decideGate(0, evidence, clock);
55
+ return {
56
+ ok: true,
57
+ gateIndex: outcome.gateIndex,
58
+ promotionBlocked: outcome.promotionBlocked,
59
+ selectsPreVc: outcome.promotionBlocked,
60
+ };
61
+ }
62
+ const outcome = decideGate(0, evidence, clock);
63
+ return {
64
+ ok: true,
65
+ gateIndex: outcome.gateIndex,
66
+ promotionBlocked: outcome.promotionBlocked,
67
+ };
68
+ }
69
+ throw new Error(`unknown rollout scenario: ${scenario}`);
70
+ }
@@ -0,0 +1,56 @@
1
+ /**
2
+ * vector-cortex/rollout/assign.ts — deterministic stable-bucket assignment.
3
+ *
4
+ * `assignSession(sessionId)` hashes a session id into one of 10,000 stable
5
+ * buckets (0..9999). The assignment is a PURE function of the session id and
6
+ * NEVER changes across process restart: it uses a fixed-seed FNV-1a over the
7
+ * session id bytes, never Date.now / Math.random (PREVENT-PI-004-adjacent
8
+ * determinism contract; the spec's restart-invariance requirement).
9
+ *
10
+ * A bucket `b` is "in" gate `g` (one of [1,5,25,50,100]) iff
11
+ * `b < g * 100`. So gate 1% covers buckets 0..99, gate 100% covers 0..9999.
12
+ *
13
+ * Pure: no I/O, no clock, no network. PREVENT-011: no `any`.
14
+ */
15
+ import { ROLLOUT_BUCKETS, ROLLOUT_GATES, } from "./types.js";
16
+ /**
17
+ * Deterministic 32-bit FNV-1a over the session id (UTF-8 bytes). Fixed seed
18
+ * (2166136261) — restart-invariant by construction. Returns a bucket in
19
+ * 0..(ROLLOUT_BUCKETS-1).
20
+ */
21
+ export function assignSession(sessionId) {
22
+ const enc = new TextEncoder();
23
+ const bytes = enc.encode(sessionId);
24
+ let hash = 0x811c9dc5; // FNV offset basis (32-bit)
25
+ for (let i = 0; i < bytes.length; i++) {
26
+ hash ^= bytes[i];
27
+ // hash *= 16777619 (FNV prime) with 32-bit overflow wrap.
28
+ hash = Math.imul(hash, 0x01000193);
29
+ }
30
+ const bucket = (hash >>> 0) % ROLLOUT_BUCKETS;
31
+ // The bucket's qualifying gate is the HIGHEST gate whose bound it falls under.
32
+ let gateIndex = 0;
33
+ for (let i = 0; i < ROLLOUT_GATES.length; i++) {
34
+ if (bucket < ROLLOUT_GATES[i] * 100) {
35
+ gateIndex = i;
36
+ break;
37
+ }
38
+ }
39
+ return {
40
+ schema: "rollout-assignment-v1",
41
+ sessionId,
42
+ bucket,
43
+ gateIndex,
44
+ };
45
+ }
46
+ /**
47
+ * Whether a bucket qualifies for a given gate percentage. Used by the live seam
48
+ * to decide whether a session's bucket is currently exposed to the VC path.
49
+ */
50
+ export function bucketInGate(bucket, gatePct) {
51
+ return bucket < gatePct * 100;
52
+ }
53
+ /** Current gate percentage for a gate index. */
54
+ export function gatePctForIndex(gateIndex) {
55
+ return ROLLOUT_GATES[gateIndex];
56
+ }