pi-mega-compact 0.20.6 → 0.20.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/dist/config/vector-cortex.js +12 -0
  2. package/dist/config.js +1 -1
  3. package/dist/extensions/dashboard-server/api-contracts/vector-cortex-heal.js +11 -0
  4. package/dist/extensions/dashboard-server/route-dispatch.js +102 -0
  5. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +3 -32
  6. package/dist/extensions/dashboard-server/routes-rag-settings-types.js +11 -0
  7. package/dist/extensions/dashboard-server/routes-rag-settings-vector-cortex.js +44 -0
  8. package/dist/extensions/dashboard-server/routes-vector-cortex-heal.js +47 -0
  9. package/dist/extensions/dashboard-server/routes-vector-cortex.js +1 -0
  10. package/dist/extensions/dashboard-server/routes.js +1 -1
  11. package/dist/extensions/dashboard-server/server.js +3 -82
  12. package/dist/src/config/dedup.js +1 -0
  13. package/dist/src/config/vector-cortex.js +12 -0
  14. package/dist/src/config.js +1 -1
  15. package/dist/src/dedup-audit.test-helpers.js +43 -0
  16. package/dist/src/monitoring.js +9 -0
  17. package/dist/src/vector-cortex/heal/_acceptance-fixture.js +56 -0
  18. package/dist/src/vector-cortex/heal/closure-metrics.js +66 -0
  19. package/dist/src/vector-cortex/heal/closure-opt.js +242 -0
  20. package/dist/src/vector-cortex/heal/emit.js +69 -0
  21. package/dist/src/vector-cortex/heal/proof.js +210 -0
  22. package/dist/src/vector-cortex/heal/types.js +42 -0
  23. package/dist/src/vectorStore/add-l0.js +108 -0
  24. package/dist/src/vectorStore/add-l1.js +35 -0
  25. package/dist/src/vectorStore/add.js +229 -0
  26. package/dist/src/vectorStore/class.js +15 -291
  27. package/dist/src/vectorStore/dedup-audit.js +104 -0
  28. package/dist/src/vectorStore.js +1 -0
  29. package/dist/vector-cortex/heal/_acceptance-fixture.js +56 -0
  30. package/dist/vector-cortex/heal/closure-metrics.js +66 -0
  31. package/dist/vector-cortex/heal/closure-opt.js +242 -0
  32. package/dist/vector-cortex/heal/emit.js +69 -0
  33. package/dist/vector-cortex/heal/proof.js +210 -0
  34. package/dist/vector-cortex/heal/types.js +42 -0
  35. package/extensions/dashboard-client/dist/assets/{AreaChart-DqTbaSRA.js → AreaChart-5yxaI8IO.js} +2 -2
  36. package/extensions/dashboard-client/dist/assets/{AreaChart-DqTbaSRA.js.map → AreaChart-5yxaI8IO.js.map} +1 -1
  37. package/extensions/dashboard-client/dist/assets/{BarChart-BfNBeOvF.js → BarChart-BoHhCqH-.js} +2 -2
  38. package/extensions/dashboard-client/dist/assets/{BarChart-BfNBeOvF.js.map → BarChart-BoHhCqH-.js.map} +1 -1
  39. package/extensions/dashboard-client/dist/assets/{CacheTab-C9SOMYB5.js → CacheTab-Di6L3CsF.js} +2 -2
  40. package/extensions/dashboard-client/dist/assets/{CacheTab-C9SOMYB5.js.map → CacheTab-Di6L3CsF.js.map} +1 -1
  41. package/extensions/dashboard-client/dist/assets/{EventsTab-GsMKP10_.js → EventsTab-Cdff3h6s.js} +2 -2
  42. package/extensions/dashboard-client/dist/assets/{EventsTab-GsMKP10_.js.map → EventsTab-Cdff3h6s.js.map} +1 -1
  43. package/extensions/dashboard-client/dist/assets/{HealthTab-DAOCXwRP.js → HealthTab-MthYYVLw.js} +2 -2
  44. package/extensions/dashboard-client/dist/assets/{HealthTab-DAOCXwRP.js.map → HealthTab-MthYYVLw.js.map} +1 -1
  45. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-BtBeMkB5.js → MaintenanceTab-D9G4ntsG.js} +2 -2
  46. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-BtBeMkB5.js.map → MaintenanceTab-D9G4ntsG.js.map} +1 -1
  47. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-DzndtgkZ.js → MemoryMapTab-CN8Lr5Qf.js} +2 -2
  48. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-DzndtgkZ.js.map → MemoryMapTab-CN8Lr5Qf.js.map} +1 -1
  49. package/extensions/dashboard-client/dist/assets/{MetricsTab-DyxJ4eb8.js → MetricsTab-CgO32kSo.js} +2 -2
  50. package/extensions/dashboard-client/dist/assets/{MetricsTab-DyxJ4eb8.js.map → MetricsTab-CgO32kSo.js.map} +1 -1
  51. package/extensions/dashboard-client/dist/assets/{OverviewTab-DCiEdVWR.js → OverviewTab-Cb4kR0h1.js} +2 -2
  52. package/extensions/dashboard-client/dist/assets/{OverviewTab-DCiEdVWR.js.map → OverviewTab-Cb4kR0h1.js.map} +1 -1
  53. package/extensions/dashboard-client/dist/assets/{ReposTab-B5SIqVHA.js → ReposTab-Dt1FGnMR.js} +2 -2
  54. package/extensions/dashboard-client/dist/assets/{ReposTab-B5SIqVHA.js.map → ReposTab-Dt1FGnMR.js.map} +1 -1
  55. package/extensions/dashboard-client/dist/assets/{SessionsTab-DLWFJ9xI.js → SessionsTab-D7D2RW6U.js} +2 -2
  56. package/extensions/dashboard-client/dist/assets/{SessionsTab-DLWFJ9xI.js.map → SessionsTab-D7D2RW6U.js.map} +1 -1
  57. package/extensions/dashboard-client/dist/assets/{SetupTab-CvGXZyej.js → SetupTab-CgV5xCzz.js} +2 -2
  58. package/extensions/dashboard-client/dist/assets/{SetupTab-CvGXZyej.js.map → SetupTab-CgV5xCzz.js.map} +1 -1
  59. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-wZuQGxLS.js → TimeSavedCard-CCNuJjyO.js} +2 -2
  60. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-wZuQGxLS.js.map → TimeSavedCard-CCNuJjyO.js.map} +1 -1
  61. package/extensions/dashboard-client/dist/assets/{TurnsTab-B7aOejch.js → TurnsTab-Cv1OCttJ.js} +2 -2
  62. package/extensions/dashboard-client/dist/assets/{TurnsTab-B7aOejch.js.map → TurnsTab-Cv1OCttJ.js.map} +1 -1
  63. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DTabQo6h.js +2 -0
  64. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DTabQo6h.js.map +1 -0
  65. package/extensions/dashboard-client/dist/assets/{WikiTab-DhWHjAtt.js → WikiTab-B48QnQrK.js} +2 -2
  66. package/extensions/dashboard-client/dist/assets/{WikiTab-DhWHjAtt.js.map → WikiTab-B48QnQrK.js.map} +1 -1
  67. package/extensions/dashboard-client/dist/assets/{button-BrC-W87h.js → button-BBQl2MiF.js} +2 -2
  68. package/extensions/dashboard-client/dist/assets/{button-BrC-W87h.js.map → button-BBQl2MiF.js.map} +1 -1
  69. package/extensions/dashboard-client/dist/assets/{card-0rgfRgr3.js → card-BYbefUur.js} +2 -2
  70. package/extensions/dashboard-client/dist/assets/{card-0rgfRgr3.js.map → card-BYbefUur.js.map} +1 -1
  71. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-DRresOQF.js → generateCategoricalChart-swm0YZe0.js} +2 -2
  72. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-DRresOQF.js.map → generateCategoricalChart-swm0YZe0.js.map} +1 -1
  73. package/extensions/dashboard-client/dist/assets/{index-DohmtZWL.js → index-DeIky7xm.js} +3 -3
  74. package/extensions/dashboard-client/dist/assets/{index-DohmtZWL.js.map → index-DeIky7xm.js.map} +1 -1
  75. package/extensions/dashboard-client/dist/assets/{switch-Bs8g_zB5.js → switch-CctU78ww.js} +2 -2
  76. package/extensions/dashboard-client/dist/assets/{switch-Bs8g_zB5.js.map → switch-CctU78ww.js.map} +1 -1
  77. package/extensions/dashboard-client/dist/assets/{toggle-DVVauIo8.js → toggle-BRCNCoiI.js} +2 -2
  78. package/extensions/dashboard-client/dist/assets/{toggle-DVVauIo8.js.map → toggle-BRCNCoiI.js.map} +1 -1
  79. package/extensions/dashboard-client/dist/assets/{useSSE-DxH1PTAR.js → useSSE-WfOiWDH_.js} +2 -2
  80. package/extensions/dashboard-client/dist/assets/{useSSE-DxH1PTAR.js.map → useSSE-WfOiWDH_.js.map} +1 -1
  81. package/extensions/dashboard-client/dist/index.html +1 -1
  82. package/extensions/dashboard-client/src/api/vector-cortex.ts +9 -0
  83. package/extensions/dashboard-client/src/tabs/VectorCortexClosureCard.tsx +46 -0
  84. package/extensions/dashboard-client/src/tabs/VectorCortexLedgerCard.tsx +80 -0
  85. package/extensions/dashboard-client/src/tabs/VectorCortexTab.tsx +12 -164
  86. package/extensions/dashboard-client/src/tabs/VectorCortexTopologyCard.tsx +131 -0
  87. package/extensions/dashboard-client/src/types/vector-cortex.ts +14 -0
  88. package/extensions/dashboard-server/api-contracts/vector-cortex-heal.ts +43 -0
  89. package/extensions/dashboard-server/api-contracts/vector-cortex.ts +2 -0
  90. package/extensions/dashboard-server/route-dispatch.ts +112 -0
  91. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +11 -135
  92. package/extensions/dashboard-server/routes-rag-settings-types.ts +34 -0
  93. package/extensions/dashboard-server/routes-rag-settings-vector-cortex.ts +148 -0
  94. package/extensions/dashboard-server/routes-vector-cortex-heal.ts +56 -0
  95. package/extensions/dashboard-server/routes-vector-cortex.ts +1 -0
  96. package/extensions/dashboard-server/routes.ts +1 -0
  97. package/extensions/dashboard-server/server.ts +3 -86
  98. package/package.json +1 -1
  99. package/src/config/dedup.ts +9 -0
  100. package/src/config/vector-cortex.ts +13 -0
  101. package/src/config.ts +1 -0
  102. package/src/dedup-audit.test-helpers.ts +49 -0
  103. package/src/monitoring.ts +10 -0
  104. package/src/vector-cortex/heal/_acceptance-fixture.ts +108 -0
  105. package/src/vector-cortex/heal/closure-metrics.ts +73 -0
  106. package/src/vector-cortex/heal/closure-opt.ts +266 -0
  107. package/src/vector-cortex/heal/emit.ts +93 -0
  108. package/src/vector-cortex/heal/proof.ts +234 -0
  109. package/src/vector-cortex/heal/types.ts +226 -0
  110. package/src/vectorStore/add-l0.ts +152 -0
  111. package/src/vectorStore/add-l1.ts +57 -0
  112. package/src/vectorStore/add.ts +269 -0
  113. package/src/vectorStore/class.ts +17 -327
  114. package/src/vectorStore/dedup-audit.ts +199 -0
  115. package/src/vectorStore.ts +1 -0
  116. package/extensions/dashboard-client/dist/assets/VectorCortexTab-eHVt9yDx.js +0 -2
  117. package/extensions/dashboard-client/dist/assets/VectorCortexTab-eHVt9yDx.js.map +0 -1
@@ -1,34 +1,21 @@
1
- /**
2
- * class.ts — VectorStore implementation (extracted from vectorStore.ts).
3
- *
4
- * One store, three consumers (per PLAN.md): auto-inline on resume, on-demand
5
- * /recall-context, and the dedup sentinel. All share `add / search / dedupe`.
6
- *
7
- * Backed by the gzipped on-disk checkpoint files (store.ts). Similarity is a
8
- * linear cosine scan — checkpoint counts are small, so no ANN index is needed.
9
- */
10
- import { createHash } from "node:crypto";
11
- import { cosineSimilarity, defaultEmbedder } from "../embedder.js";
1
+ import { defaultEmbedder } from "../embedder.js";
12
2
  import { loadDedupConfig, } from "../config/dedup.js";
13
3
  import { logDecision } from "../monitoring.js";
14
4
  import { repoKey } from "../store/repoKey.js";
15
- import { getStateDir, normalizeSessionId, compressSmart } from "../store.js";
16
- import { computeContentDigest } from "../dedup/digest.js";
17
- import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES, } from "../dedup/l1-minhash.js";
18
- import { lshBands } from "../dedup/l1-lsh.js";
19
- import { isNearDuplicate } from "../dedup/l1-verify.js";
20
- import { openBloom, saveBloom } from "../store/bloom.js";
21
- import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, addTokensSaved, bumpDedupStats, } from "../store/sqlite.js";
5
+ import { getStateDir } from "../store.js";
6
+ import { openBloom } from "../store/bloom.js";
22
7
  import { migrateJsonToSqlite } from "../store/migrate.js";
23
- import { computeRegionHash } from "./hash.js";
8
+ import { addCheckpoint } from "./add.js";
24
9
  export class VectorStore {
25
10
  // These fields are `readonly` (set once in the constructor) but NOT private:
26
- // the read/search/dedup helpers split into vector-read.ts, vector-search.ts,
27
- // and vector-dedup.ts access them directly. Marking them private would force
28
- // ugly `as unknown as` casts in those modules; keeping them package-public
29
- // makes VectorStore a thin barrel whose helpers live in sibling files.
11
+ // the add/read/search/dedup helpers split into add.ts, vector-read.ts,
12
+ // vector-search.ts, and vector-dedup.ts access them directly. Marking them
13
+ // private would force ugly `as unknown as` casts in those modules; keeping
14
+ // them package-public makes VectorStore a thin barrel whose helpers live in
15
+ // sibling files.
30
16
  embedder;
31
17
  stateDir;
18
+ /** L2 semantic firing point (cfg.L2_COSINE, or a direct test override). */
32
19
  l2Threshold;
33
20
  /** Single source of truth for tier flags + thresholds (Sprint 14). */
34
21
  cfg;
@@ -82,275 +69,12 @@ export class VectorStore {
82
69
  });
83
70
  }
84
71
  /**
85
- * Add a checkpoint. Dedup cascade:
86
- * 1. regionHash exact match (legacy, backward-compat)
87
- * 2. summaryHash exact match (new: catches same-topic incremental compactions)
88
- * 3. content similarity dedupSim (catches near-identical summaries)
89
- * 4. If none match → create new checkpoint
72
+ * Add a checkpoint, deduping it against the session's existing checkpoints.
73
+ *
74
+ * The cascade itself lives in ./add.ts (sibling-helper pattern) so this class
75
+ * stays a thin shell over the fields its helpers read.
90
76
  */
91
77
  add(input) {
92
- const t0 = Date.now();
93
- const sessionId = normalizeSessionId(input.sessionId);
94
- const regionHash = computeRegionHash(input.regionText);
95
- const all = listCheckpoints(sessionId, this.stateDir);
96
- // Honest "tokens saved" base for this region. For a deduped add the whole
97
- // original region is discarded (nothing new stored); for a new checkpoint
98
- // we persist (orig − stored). Falls back to stored when orig is unknown.
99
- const origTokens = input.originalTokenEstimate ?? input.tokenEstimate ?? 0;
100
- const cfg = this.cfg;
101
- // Live per-tier progress hook (Phase 1). Sync + optional; fired at each tier
102
- // so the UI can paint "L0 ✓ → L1 ✓ → L2 0.91 → stored" during a compaction.
103
- const onTier = input.onTier;
104
- // Tracks whether a tier matched while in MARK_ONLY (record-but-don't-collapse),
105
- // and which tier.
106
- let markOnly = null;
107
- // 0. L0 content-hash dedup (Sprint 9) — catches identical content arriving
108
- // under different regionText. Normalization handles case/whitespace/ANSI so
109
- // variants collapse to one row. Dual-hash guards a single-hash collision.
110
- // Sprint 10: bloom is the accelerator — a miss means "definitely new" and
111
- // skips the scan; a hit is only a candidate, confirmed against `all` below.
112
- // Gated by L0_ENABLED (Sprint 14). MARK_ONLY_L0 records the decision but
113
- // does not collapse — the new region is still stored.
114
- onTier?.({ tier: "L0", status: "scanning" });
115
- const digest = computeContentDigest(input.regionText);
116
- const bloom = openBloom(this.stateDir);
117
- if (cfg.L0_ENABLED && bloom.maybeHas(digest.contentHash)) {
118
- const contentMatch = all.find((cp) => cp.contentHash === digest.contentHash &&
119
- cp.contentHash2 === digest.contentHash2);
120
- if (contentMatch) {
121
- if (cfg.MARK_ONLY_L0) {
122
- markOnly = "L0"; // Record-but-don't-collapse: fall through.
123
- }
124
- else {
125
- contentMatch.timestamp = input.timestamp;
126
- upsertCheckpoint(contentMatch, this.stateDir);
127
- bumpDedupStats(true, this.stateDir);
128
- // Deduped: whole original region discarded, nothing new stored.
129
- addTokensSaved(origTokens, this.stateDir);
130
- const r = {
131
- checkpoint: contentMatch,
132
- deduped: true,
133
- reason: "contentHash",
134
- };
135
- this.record("L0", "deduped", "contentHash", Date.now() - t0, 1, contentMatch.checkpointId);
136
- onTier?.({ tier: "L0", status: "deduped", detail: "contentHash" });
137
- return r;
138
- }
139
- }
140
- }
141
- // 1. Legacy regionHash dedup (backward-compat) — part of L0 tier gating.
142
- if (cfg.L0_ENABLED) {
143
- const regionMatch = all.find((cp) => cp.regionHash === regionHash);
144
- if (regionMatch) {
145
- if (cfg.MARK_ONLY_L0) {
146
- markOnly = "L0"; // fall through
147
- }
148
- else {
149
- bumpDedupStats(true, this.stateDir);
150
- // Deduped: whole original region discarded, nothing new stored.
151
- addTokensSaved(origTokens, this.stateDir);
152
- const r = {
153
- checkpoint: regionMatch,
154
- deduped: true,
155
- reason: "regionHash",
156
- };
157
- this.record("L0", "deduped", "regionHash", Date.now() - t0, 1, regionMatch.checkpointId);
158
- onTier?.({ tier: "L0", status: "deduped", detail: "regionHash" });
159
- return r;
160
- }
161
- }
162
- }
163
- // 2. SummaryHash dedup — catches same-topic incremental compactions.
164
- // Full 64-hex SHA-256 (was 16-hex in Sprint 8 — collision-prone).
165
- const summaryHash = input.topicSummary
166
- ? createHash("sha256").update(input.topicSummary).digest("hex")
167
- : undefined;
168
- if (summaryHash && cfg.L0_ENABLED) {
169
- const summaryMatch = all.find((cp) => cp.summaryHash === summaryHash);
170
- if (summaryMatch) {
171
- if (cfg.MARK_ONLY_L0) {
172
- markOnly = "L0"; // fall through
173
- }
174
- else {
175
- summaryMatch.timestamp = input.timestamp;
176
- upsertCheckpoint(summaryMatch, this.stateDir);
177
- bumpDedupStats(true, this.stateDir);
178
- // Deduped: whole original region discarded, nothing new stored.
179
- addTokensSaved(origTokens, this.stateDir);
180
- const r = {
181
- checkpoint: summaryMatch,
182
- deduped: true,
183
- reason: "summaryHash",
184
- };
185
- this.record("L0", "deduped", "summaryHash", Date.now() - t0, 1, summaryMatch.checkpointId);
186
- onTier?.({ tier: "L0", status: "deduped", detail: "summaryHash" });
187
- return r;
188
- }
189
- }
190
- }
191
- // L0 did not collapse this region.
192
- onTier?.({ tier: "L0", status: "passed" });
193
- // 2b. L1 MinHash/LSH near-duplicate dedup (Sprint 11) — catches one-word
194
- // edits / rewordings that L0's exact hash misses. Cheap LSH bucket
195
- // retrieval → trigram verification (pg_trgm-equivalent) as the final gate.
196
- // Gated by L1_ENABLED (Sprint 14); MARK_ONLY_L1 records but doesn't collapse.
197
- onTier?.({ tier: "L1", status: "scanning" });
198
- if (cfg.L1_ENABLED) {
199
- const l1 = this.findL1Duplicate(sessionId, input.regionText, all);
200
- if (l1 && !cfg.MARK_ONLY_L1) {
201
- l1.timestamp = input.timestamp;
202
- upsertCheckpoint(l1, this.stateDir);
203
- bumpDedupStats(true, this.stateDir);
204
- const r = { checkpoint: l1, deduped: true, reason: "l1MinHash" };
205
- this.record("L1", "deduped", "l1MinHash", Date.now() - t0, 1, l1.checkpointId);
206
- onTier?.({ tier: "L1", status: "deduped", detail: "l1MinHash" });
207
- return r;
208
- }
209
- if (l1 && cfg.MARK_ONLY_L1)
210
- markOnly = "L1";
211
- }
212
- onTier?.({ tier: "L1", status: "passed" });
213
- // 3. L2 semantic dedup — catches near-identical / semantically-similar regions
214
- // via cosine over the embedding. topicSummary is used for summaryHash dedup
215
- // (tier 2); the vector index is keyed on the original region for backward-
216
- // compat search semantics. Threshold from cfg (L2_COSINE trigram honest
217
- // firing point). QA #13 timeout guard: if the O(n) scan exceeds the budget,
218
- // degrade to "store without dedup this pass" so we never lose a checkpoint.
219
- // Gated by L2_ENABLED (Sprint 14); MARK_ONLY_L2 records but doesn't collapse.
220
- const SIMILARITY_BUDGET_MS = cfg.SIMILARITY_BUDGET_MS;
221
- const simThreshold = this.l2Threshold; // from cfg.L2_COSINE (default 0.85 trigram)
222
- const embedding = this.embedder.embed(input.regionText);
223
- onTier?.({ tier: "L2", status: "scanning" });
224
- if (cfg.L2_ENABLED && all.length > 0) {
225
- const start = Date.now();
226
- let timedOut = false;
227
- const nearest = all.reduce((best, cp) => {
228
- if (!timedOut && Date.now() - start > SIMILARITY_BUDGET_MS)
229
- timedOut = true;
230
- if (timedOut)
231
- return best;
232
- const sim = cosineSimilarity(embedding, cp.embedding);
233
- return sim > best.sim ? { checkpoint: cp, sim } : best;
234
- }, { checkpoint: all[0], sim: -1 });
235
- if (!timedOut && nearest.sim >= simThreshold) {
236
- if (!cfg.MARK_ONLY_L2) {
237
- // Near-identical — update timestamp on existing checkpoint
238
- nearest.checkpoint.timestamp = input.timestamp;
239
- upsertCheckpoint(nearest.checkpoint, this.stateDir);
240
- bumpDedupStats(true, this.stateDir);
241
- // Deduped: whole original region discarded, nothing new stored.
242
- addTokensSaved(origTokens, this.stateDir);
243
- const r = {
244
- checkpoint: nearest.checkpoint,
245
- deduped: true,
246
- reason: "contentSimilarity",
247
- };
248
- this.record("L2", "deduped", "contentSimilarity", Date.now() - t0, nearest.sim, nearest.checkpoint.checkpointId);
249
- onTier?.({
250
- tier: "L2",
251
- status: "deduped",
252
- detail: nearest.sim.toFixed(2),
253
- });
254
- return r;
255
- }
256
- markOnly = "L2";
257
- }
258
- onTier?.({
259
- tier: "L2",
260
- status: "passed",
261
- detail: `best ${nearest.sim.toFixed(2)}`,
262
- });
263
- }
264
- // 4. Genuinely new — create checkpoint
265
- const checkpointId = nextCheckpointId(sessionId, this.stateDir);
266
- const checkpoint = {
267
- checkpointId,
268
- sessionId,
269
- repoId: this.repoId,
270
- summary: input.summary,
271
- topicSummary: input.topicSummary,
272
- summaryHash,
273
- keyDecisions: input.keyDecisions ?? [],
274
- nextSteps: input.nextSteps ?? [],
275
- filesModified: input.filesModified ?? [],
276
- tokenEstimate: input.tokenEstimate ?? 0,
277
- originalTokenEstimate: input.originalTokenEstimate,
278
- regionHash,
279
- contentHash: digest.contentHash,
280
- contentHash2: digest.contentHash2,
281
- contentHashVersion: digest.contentHashVersion,
282
- normalizedText: digest.normalizedText,
283
- compressedOriginal: compressSmart(Buffer.from(input.regionText, "utf-8"), input.compressionPressure),
284
- embedding,
285
- timestamp: input.timestamp,
286
- };
287
- // Persistence is SQLite (store/sqlite.ts). upsertCheckpoint keeps the
288
- // idempotent-by-id semantics the old JSON append implied.
289
- upsertCheckpoint(checkpoint, this.stateDir);
290
- // Cumulative "tokens saved" counter (per-repo SQLite meta). For a NEW
291
- // checkpoint the saved amount is (original − stored); for a deduped add the
292
- // whole original region is discarded (handled in the deduped return paths
293
- // below). Survives sessions and travels with the repo.
294
- const stored = input.tokenEstimate ?? 0;
295
- addTokensSaved(Math.max(0, origTokens - stored), this.stateDir);
296
- // L1: persist this checkpoint's MinHash signature + LSH buckets so future
297
- // near-duplicate inserts can find it. Deterministic given the seed.
298
- const sig = minhashSignature(input.regionText);
299
- upsertMinhashSignature(checkpointId, sessionId, SIGNATURE_VERSION, sig, this.stateDir);
300
- insertLshBuckets(checkpointId, sessionId, SIGNATURE_VERSION, lshBands(sig, sessionId, SIGNATURE_VERSION), this.stateDir);
301
- // Bloom accelerator: record the new content_hash so a future add() can short-
302
- // circuit the scan on a hit (still confirmed by the SELECT-based `all` above).
303
- bloom.add(digest.contentHash);
304
- saveBloom(this.stateDir);
305
- // Track the region hash in session state for fast sentinel checks.
306
- const state = loadSessionState(sessionId, this.stateDir);
307
- if (!state.storedRegionHashes.includes(regionHash)) {
308
- state.storedRegionHashes.push(regionHash);
309
- saveSessionState(sessionId, state, this.stateDir);
310
- }
311
- // A new checkpoint. If a tier matched while MARK_ONLY, record that (the
312
- // decision fired but we intentionally did not collapse).
313
- if (markOnly) {
314
- this.record(markOnly, "mark_only", "mark_only", Date.now() - t0);
315
- }
316
- else {
317
- this.record("L0", "new", undefined, Date.now() - t0);
318
- }
319
- // Cumulative store-wide dedup accounting (attempt, not collapsed).
320
- bumpDedupStats(false, this.stateDir);
321
- onTier?.({ tier: "new", status: "stored" });
322
- return { checkpoint, deduped: false };
323
- }
324
- /**
325
- * L1 near-duplicate lookup: MinHash → LSH candidate retrieval → trigram verify.
326
- * Returns the matching checkpoint or undefined. Bounded by a 100-candidate cap
327
- * and a 20ms verify budget (QA #7/#15) so it never hangs a large session.
328
- */
329
- findL1Duplicate(sessionId, regionText, all) {
330
- if (all.length === 0)
331
- return undefined;
332
- const sig = minhashSignature(regionText);
333
- if (sig.length !== NUM_HASHES)
334
- return undefined;
335
- const bands = lshBands(sig, sessionId, SIGNATURE_VERSION);
336
- // Cheap candidate retrieval (single query, capped). Exclude nothing yet —
337
- // the new checkpoint has no id, so pass a sentinel that never matches.
338
- const candidateIds = lshCandidateChunks(bands, sessionId, "__new__", this.stateDir, 100);
339
- if (candidateIds.length === 0)
340
- return undefined;
341
- const byId = new Map(all.map((cp) => [cp.checkpointId, cp]));
342
- const VERIFY_BUDGET_MS = 20;
343
- const start = Date.now();
344
- for (const id of candidateIds) {
345
- if (Date.now() - start > VERIFY_BUDGET_MS)
346
- break; // QA #15: abort → "not dup"
347
- const cand = byId.get(id);
348
- if (!cand)
349
- continue;
350
- const candText = cand.normalizedText ?? cand.summary ?? "";
351
- if (isNearDuplicate(regionText, candText))
352
- return cand;
353
- }
354
- return undefined;
78
+ return addCheckpoint(this, input);
355
79
  }
356
80
  }
@@ -0,0 +1,104 @@
1
+ /**
2
+ * dedup-audit.ts — durable audit trail for dedup tier decisions
3
+ * (external-audit item #2).
4
+ *
5
+ * Before this module a tier decision existed only as the in-process `onTier`
6
+ * callback that paints the live UI; nothing survived the process, so an
7
+ * operator could not answer "which layer collapsed this region, onto what, at
8
+ * what similarity?" — the inputs needed to tune the thresholds in
9
+ * config/dedup.ts. Here each decision is appended to the repo's events.log as
10
+ * one structured JSON line (see `DedupAuditEvent` below).
11
+ *
12
+ * The event type and its append helper live HERE rather than in monitoring.ts:
13
+ * monitoring.ts already owns three concerns (decision events, the dashboard.json
14
+ * metrics snapshot, FP alerting) and sits close to its 300-line soft limit, so
15
+ * co-locating the shape with the only recorder that produces it keeps both files
16
+ * under the headroom gate. monitoring.ts re-exports both for callers (and the
17
+ * dashboard SSE tail) that treat it as the events.log barrel.
18
+ *
19
+ * Design constraints:
20
+ * - PURE INSTRUMENTATION. Nothing in this file may influence a dedup outcome.
21
+ * - Best-effort/non-fatal: `logDedupAudit` swallows IO errors, and the emitter
22
+ * itself is wrapped so a malformed field can never break add().
23
+ * - Honest fields only: a value is emitted only where the caller actually
24
+ * computed it. L0/L1 are hash/verify tiers and pass no `similarity`.
25
+ * - Signal, not chatter: callers emit on DECISIONS (a match, a scored
26
+ * candidate, the final outcome), never on every "scanning" transition.
27
+ * - Flag-gated by cfg.DEDUP_AUDIT (default ON; OFF writes nothing at all).
28
+ *
29
+ * PREVENT-PI-004: local filesystem append only, no network.
30
+ */
31
+ import { appendFileSync, mkdirSync } from "node:fs";
32
+ import { dirname } from "node:path";
33
+ import { defaultEventsPath } from "../monitoring.js";
34
+ /**
35
+ * Append one audit event to events.log (best-effort, never throws).
36
+ *
37
+ * Same append-one-JSON-line contract as monitoring.ts's logDecision — an
38
+ * unwritable path is swallowed so instrumentation can never break add().
39
+ */
40
+ export function logDedupAudit(path, ev) {
41
+ try {
42
+ mkdirSync(dirname(path), { recursive: true });
43
+ appendFileSync(path, `${JSON.stringify(ev)}\n`, "utf-8");
44
+ }
45
+ catch {
46
+ /* best-effort — never break the extension on a log failure */
47
+ }
48
+ }
49
+ /** Build a recorder bound to one add() cascade. */
50
+ export function dedupAuditRecorder(ctx, scope) {
51
+ const base = {
52
+ sessionId: scope.sessionId,
53
+ originalTokenEstimate: scope.originalTokenEstimate,
54
+ tokenEstimate: scope.tokenEstimate,
55
+ };
56
+ return {
57
+ deduped: (tier, matchedEntry, dedupReason, similarity) => emitDedupAudit(ctx, {
58
+ ...base,
59
+ tier,
60
+ status: "deduped",
61
+ matchedEntry,
62
+ dedupReason,
63
+ ...(similarity === undefined ? {} : { similarity }),
64
+ }),
65
+ passed: (tier, matchedEntry, similarity) => emitDedupAudit(ctx, {
66
+ ...base,
67
+ tier,
68
+ status: "passed",
69
+ matchedEntry,
70
+ similarity,
71
+ }),
72
+ stored: (storedEntry, dedupReason, tokenEstimate) => emitDedupAudit(ctx, {
73
+ ...base,
74
+ tier: "new",
75
+ status: "stored",
76
+ storedEntry,
77
+ dedupReason,
78
+ tokenEstimate,
79
+ }),
80
+ };
81
+ }
82
+ /**
83
+ * Append one dedup decision to events.log.
84
+ *
85
+ * Resolves the target path from the explicit `eventsPath` when a caller opted
86
+ * in (Sprint 14 monitoring / tests), otherwise from the store's own per-repo
87
+ * state dir — production never passes `eventsPath`, so defaulting is what makes
88
+ * the audit trail actually exist on a real device.
89
+ */
90
+ export function emitDedupAudit(ctx, input) {
91
+ if (!ctx.auditEnabled)
92
+ return;
93
+ try {
94
+ const path = ctx.eventsPath ?? defaultEventsPath(ctx.stateDir);
95
+ logDedupAudit(path, {
96
+ type: "dedup_audit",
97
+ ts: new Date().toISOString(),
98
+ ...input,
99
+ });
100
+ }
101
+ catch {
102
+ /* instrumentation must never break the add() path */
103
+ }
104
+ }
@@ -8,6 +8,7 @@
8
8
  * "./vectorStore.js" are unchanged.
9
9
  */
10
10
  export { VectorStore } from "./vectorStore/class.js";
11
+ export { addCheckpoint } from "./vectorStore/add.js";
11
12
  export { computeRegionHash } from "./vectorStore/hash.js";
12
13
  export { L2_ENABLED, } from "./vectorStore/types.js";
13
14
  // Re-exports (back-compat): existing call sites keep importing from "./vectorStore.js"
@@ -0,0 +1,56 @@
1
+ /**
2
+ * heal/_acceptance-fixture.ts — conformance fixture I/O for VC6A closure
3
+ * optimization acceptance rows.
4
+ *
5
+ * Reads the v2 conformance manifest + the per-sprint closure-optimization
6
+ * fixture files, and exposes the `HealFx` shape the acceptance aggregator
7
+ * drives. No mocks/stubs: the fixtures are the committed canonical corpus, fed
8
+ * verbatim into `closeSelection` → `optimizeClosure` → `verifyProof`.
9
+ */
10
+ import { readFileSync, existsSync } from "node:fs";
11
+ import { join, dirname } from "node:path";
12
+ import { fileURLToPath } from "node:url";
13
+ import assert from "node:assert/strict";
14
+ const HERE = dirname(fileURLToPath(import.meta.url));
15
+ function repoRoot(from) {
16
+ let dir = from;
17
+ for (let i = 0; i < 8; i++) {
18
+ if (existsSync(join(dir, "conformance", "vector-cortex")))
19
+ return dir;
20
+ const next = dirname(dir);
21
+ if (next === dir)
22
+ break;
23
+ dir = next;
24
+ }
25
+ throw new Error("conformance corpus not found above " + from);
26
+ }
27
+ export const REPO_ROOT = repoRoot(HERE);
28
+ export const V2 = join(REPO_ROOT, "conformance", "vector-cortex", "v2");
29
+ export function readManifest() {
30
+ return JSON.parse(readFileSync(join(V2, "manifest.json"), "utf8"));
31
+ }
32
+ function readFixture(id, prefix) {
33
+ const m = readManifest();
34
+ const row = m.fixtures.find((f) => f.id === id && f.path.startsWith(`${prefix}/`));
35
+ assert.ok(row, `fixture ${id} registered under ${prefix}/ in manifest`);
36
+ return JSON.parse(readFileSync(join(V2, row.path), "utf8"));
37
+ }
38
+ export function healFixture(id) {
39
+ return readFixture(id, "closure-optimization");
40
+ }
41
+ /** Flag-pinned wrapper: VC6A gated by MEGACOMPACT_VC6A (defaults ON). */
42
+ export function withFlagsOn(fn) {
43
+ return () => {
44
+ const saved = process.env.MEGACOMPACT_VC6A;
45
+ process.env.MEGACOMPACT_VC6A = "1";
46
+ try {
47
+ fn();
48
+ }
49
+ finally {
50
+ if (saved === undefined)
51
+ delete process.env.MEGACOMPACT_VC6A;
52
+ else
53
+ process.env.MEGACOMPACT_VC6A = saved;
54
+ }
55
+ };
56
+ }
@@ -0,0 +1,66 @@
1
+ /**
2
+ * vector-cortex/heal/closure-metrics.ts — VC6A closure proof metrics + handoff.
3
+ *
4
+ * Pure read-only derivations over a `ClosureProofV2`: traversal savings (the
5
+ * sprint's >=20% median acceptance bar) and the `RestoreHintV1` list VC6B
6
+ * consumes. Extracted from `closure-opt.ts` (delegate-shell) to keep the core
7
+ * reduction algorithm under the 300-line soft limit.
8
+ *
9
+ * PURE / DETERMINISTIC / LOCAL: no storage, no clock, no network (PREVENT-PI-004
10
+ * / PREVENT-011). Identity only — never source bytes or prompt text
11
+ * (SECURITY_PRIVACY).
12
+ */
13
+ /** Bytewise id comparator — matches `closure-opt.ts` / `reconstruct/closure.ts`. */
14
+ function byBytes(a, b) {
15
+ return a < b ? -1 : a > b ? 1 : 0;
16
+ }
17
+ /** Anchor id set from a graph (matches `closure-opt.ts`). */
18
+ function anchorIds(graph) {
19
+ return new Set(graph.nodes.filter((n) => n.anchor === true).map((n) => n.id));
20
+ }
21
+ /**
22
+ * Traversal savings as a fraction in [0,1]: how much of the conservative edge
23
+ * walk the optimized plan avoids. A plan with nothing to remove scores 0 (not a
24
+ * failure — a chain graph is already minimal). The sprint's acceptance bar is a
25
+ * >=20% MEDIAN over the corpus, never a per-graph floor.
26
+ */
27
+ export function traversalSavings(proof) {
28
+ if (proof.conservativeTraversals === 0)
29
+ return 0;
30
+ const saved = proof.conservativeTraversals - proof.optimizedTraversals;
31
+ return saved / proof.conservativeTraversals;
32
+ }
33
+ /**
34
+ * Derive the `RestoreHintV1` list VC6B consumes (the sprint's declared handoff:
35
+ * "VC6B receives missing-source hints and proof").
36
+ *
37
+ * One hint per selected node that a RETAINED edge reaches, naming the cheapest
38
+ * (bytewise-first) retained prerequisite and whether the node is protected.
39
+ * Identity only — never source bytes or prompt text (SECURITY_PRIVACY).
40
+ */
41
+ export function restoreHints(proof, graph) {
42
+ const anchors = anchorIds(graph);
43
+ const best = new Map();
44
+ for (const row of proof.rows) {
45
+ if (row.decision !== "retained")
46
+ continue;
47
+ const reason = row.reason;
48
+ const existing = best.get(row.to);
49
+ if (existing === undefined || byBytes(row.from, existing.from) < 0) {
50
+ best.set(row.to, { from: row.from, reason });
51
+ }
52
+ }
53
+ const hints = [];
54
+ for (const nodeId of [...proof.selected].sort(byBytes)) {
55
+ const hit = best.get(nodeId);
56
+ const isToolPair = proof.retainedEdges.some((e) => e.kind === "tool-pair" && (e.from === nodeId || e.to === nodeId));
57
+ hints.push({
58
+ schema: "restore-hint-v1",
59
+ nodeId,
60
+ viaEdgeFrom: hit?.from,
61
+ protectedNode: anchors.has(nodeId) || isToolPair,
62
+ reason: hit?.reason ?? (anchors.has(nodeId) ? "anchor" : "no-alternate-path"),
63
+ });
64
+ }
65
+ return hints;
66
+ }