pi-mega-compact 0.4.10 β†’ 0.4.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -33,6 +33,7 @@ function readSnapshot(snapshotPath) {
33
33
  store: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, injectedCount: 0, dedupHitRate: 0, storageDedupRate: 0, dedupCollapsed: 0 },
34
34
  crew: { activeAgents: 0, currentTurn: 0 },
35
35
  repo: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupAttempts: 0, dedupCollapsed: 0, storageDedupRate: 0 },
36
+ integrity: { regionsRetained: 0, compressedOriginalBytes: 0, duplicatesCollapsed: 0, bytesPermanentlyDeleted: 0 },
36
37
  };
37
38
  }
38
39
  }
@@ -64,6 +65,10 @@ function dashboardHtml(tierName) {
64
65
  h1 .tier { background: #1f6feb; color: #fff; font-size: 11px; font-weight: 700; padding: 2px 8px; border-radius: 10px; text-transform: uppercase; letter-spacing: .5px; }
65
66
  .grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; margin-bottom: 20px; }
66
67
  .card { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 16px; }
68
+ .card.safe { border-color: #238636; }
69
+ .card.safe h2 { color: #3fb950; }
70
+ .safe-note { font-size: 12px; color: #8b949e; margin: 12px 0 0; line-height: 1.5; }
71
+ .value.ok { color: #3fb950; }
67
72
  .card h2 { font-size: 12px; text-transform: uppercase; letter-spacing: .5px; color: #8b949e; margin-bottom: 12px; font-weight: 600; }
68
73
  .meter-track { background: #21262d; border-radius: 4px; height: 20px; overflow: hidden; margin: 8px 0; }
69
74
  .meter-fill { height: 100%; border-radius: 4px; transition: width .6s ease; min-width: 2px; }
@@ -144,6 +149,16 @@ function dashboardHtml(tierName) {
144
149
  <span class="label">Storage Dedup</span><span class="value" id="rp-sdedup">0%</span>
145
150
  </div>
146
151
  </div>
152
+ <div class="card safe">
153
+ <h2>πŸ›‘ Data Safety</h2>
154
+ <div class="stat-grid">
155
+ <span class="label">Regions Retained</span><span class="value" id="ig-retained">0</span>
156
+ <span class="label">Compressed-Original</span><span class="value" id="ig-bytes">0 B</span>
157
+ <span class="label">Dedup Duplicates</span><span class="value" id="ig-dupes">0</span>
158
+ <span class="label">Permanently Deleted</span><span class="value ok" id="ig-deleted">0 B</span>
159
+ </div>
160
+ <p class="safe-note">Every compacted region is kept verbatim (compressed). "Drop" = removed from the live window only. We never delete your data.</p>
161
+ </div>
147
162
  <div class="card">
148
163
  <h2>Configuration</h2>
149
164
  <div class="conf-grid">
@@ -223,6 +238,19 @@ function dashboardHtml(tierName) {
223
238
  var rsdr = repo.storageDedupRate || 0;
224
239
  document.getElementById('rp-sdedup').textContent = (rsdr * 100 >= 10 ? Math.round(rsdr * 100) : (rsdr * 100).toFixed(1)) + '%';
225
240
 
241
+ // Data-safety invariant (Phase 0 β€” trust foundation).
242
+ var ig = d.integrity || { regionsRetained: 0, compressedOriginalBytes: 0, duplicatesCollapsed: 0, bytesPermanentlyDeleted: 0 };
243
+ function fmtBytes(b) {
244
+ b = b || 0;
245
+ if (b >= 1048576) return (b / 1048576).toFixed(1) + ' MiB';
246
+ if (b >= 1024) return (b / 1024).toFixed(1) + ' KiB';
247
+ return b + ' B';
248
+ }
249
+ document.getElementById('ig-retained').textContent = (ig.regionsRetained || 0).toLocaleString();
250
+ document.getElementById('ig-bytes').textContent = fmtBytes(ig.compressedOriginalBytes);
251
+ document.getElementById('ig-dupes').textContent = (ig.duplicatesCollapsed || 0).toLocaleString();
252
+ document.getElementById('ig-deleted').textContent = fmtBytes(ig.bytesPermanentlyDeleted);
253
+
226
254
  // Crew / agents (live sub-agent activity + turn).
227
255
  var crew = d.crew || { activeAgents: 0, currentTurn: 0 };
228
256
  document.getElementById('cr-agents').textContent = crew.activeAgents || 0;
@@ -185,6 +185,7 @@ export default function (pi) {
185
185
  bindRepo(ctx.cwd);
186
186
  const st = store.stats(rt.sessionId);
187
187
  const repo = store.repoStats();
188
+ const di = store.dataInvariant();
188
189
  const armed = lastCtxPercent != null && lastCtxPercent >= config.fastGatePct;
189
190
  const ready = armed && (lastCtxTokens ?? 0) >= config.thresholdTokens;
190
191
  dashboard.snapshot({
@@ -223,6 +224,12 @@ export default function (pi) {
223
224
  dedupCollapsed: repo.dedupCollapsed,
224
225
  storageDedupRate: repo.storageDedupRate,
225
226
  },
227
+ integrity: {
228
+ regionsRetained: di.regionsRetained,
229
+ compressedOriginalBytes: di.compressedOriginalBytes,
230
+ duplicatesCollapsed: di.duplicatesCollapsed,
231
+ bytesPermanentlyDeleted: di.bytesPermanentlyDeleted,
232
+ },
226
233
  });
227
234
  // Live stats widget above the editor
228
235
  if (ctx) {
@@ -254,7 +261,16 @@ export default function (pi) {
254
261
  ];
255
262
  // Live "now processing" line β€” teal while fresh (≀4s), then the last-seen
256
263
  // action keeps the widget lively. Cleared on session reset.
257
- if (currentActivity) {
264
+ if (tierTrace) {
265
+ // Per-tier dedup progress is more relevant than the last action while a
266
+ // compaction is mid-flight, so prefer it when fresh.
267
+ const fresh = Date.now() - lastActivityAt < 4000;
268
+ if (fresh)
269
+ lines.push(` ${tierTrace}`);
270
+ else if (currentActivity)
271
+ lines.push(` ${C.dim}${currentActivity}${C.reset}`);
272
+ }
273
+ else if (currentActivity) {
258
274
  const fresh = Date.now() - lastActivityAt < 4000;
259
275
  lines.push(` ${fresh ? C.teal : C.dim}${currentActivity}${C.reset}`);
260
276
  }
@@ -285,6 +301,10 @@ export default function (pi) {
285
301
  // the widget is never blank. Cleared on session reset.
286
302
  let currentActivity;
287
303
  let lastActivityAt = 0;
304
+ // Live per-tier dedup trace (Phase 1): e.g. "L0 βœ“ β†’ L1 βœ“ β†’ L2 0.91 β†’ stored".
305
+ // Built from the store's sync onTier callback during a compaction so the user
306
+ // watches each tier evaluate in real time. Cleared once the outcome settles.
307
+ let tierTrace;
288
308
  // ANSI palette for the toolbar. The pi TUI's Text component preserves ANSI
289
309
  // escape codes (see wrapTextWithAnsi), so raw escapes render as colors. No
290
310
  // chalk dependency needed β€” these are just strings.
@@ -323,6 +343,34 @@ export default function (pi) {
323
343
  currentTurn = 0;
324
344
  currentActivity = undefined;
325
345
  lastActivityAt = 0;
346
+ tierTrace = undefined;
347
+ }
348
+ /** Build the sync onTier callback that paints the live per-tier trace. */
349
+ function makeTierCallback(ctx) {
350
+ const order = ["L0", "L1", "L2", "new"];
351
+ const seen = new Map();
352
+ const glyph = (status) => status === "deduped" ? `${C.green}βœ“${C.reset}` :
353
+ status === "passed" ? `${C.dim}β—‹${C.reset}` :
354
+ status === "scanning" ? `${C.amber}…${C.reset}` :
355
+ `${C.cyan}●${C.reset}`;
356
+ return (ev) => {
357
+ const label = ev.tier === "new"
358
+ ? `${C.cyan}stored${C.reset}`
359
+ : `${ev.tier} ${glyph(ev.status)}` +
360
+ (ev.detail ? ` ${C.gray}(${ev.detail})${C.reset}` : "");
361
+ // Show the most recent outcome per tier (collapses re-fires).
362
+ seen.set(ev.tier, label);
363
+ const show = [];
364
+ for (const t of order)
365
+ if (seen.has(t))
366
+ show.push(seen.get(t));
367
+ tierTrace = `${C.teal}βš™${C.reset} ${show.join(` ${C.gray}β†’${C.reset} `)}`;
368
+ lastActivityAt = Date.now();
369
+ try {
370
+ snapshot(ctx);
371
+ }
372
+ catch { /* non-fatal */ }
373
+ };
326
374
  }
327
375
  /** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
328
376
  function runCompact(ctx, messages, opts = {}) {
@@ -340,6 +388,7 @@ export default function (pi) {
340
388
  keepFrom,
341
389
  summary: opts.summary,
342
390
  timestamp: Date.now(),
391
+ onTier: makeTierCallback(ctx),
343
392
  }, store);
344
393
  if (result.skipped)
345
394
  return { skipped: true };
@@ -371,6 +420,9 @@ export default function (pi) {
371
420
  ? `β™» deduped ${fileLabel}`
372
421
  : `πŸ—œ compacted ${result.checkpointId} Β· ${fileLabel}`;
373
422
  lastActivityAt = Date.now();
423
+ // The per-tier trace has settled into the final outcome β€” fold it back into
424
+ // the activity line and stop showing the live trace.
425
+ tierTrace = undefined;
374
426
  // Record session activity + a daily-log entry in the per-repo SQLite store
375
427
  // (foundation for resume-sessions / daily-log features). Best-effort β€” never
376
428
  // block a compaction on bookkeeping.
@@ -608,7 +660,7 @@ export default function (pi) {
608
660
  },
609
661
  });
610
662
  pi.registerCommand("mega-status", {
611
- description: "Show mega-compact config and current context usage.",
663
+ description: "Show mega-compact config, context usage, and the data-safety invariant.",
612
664
  handler: async (_args, ctx) => {
613
665
  bindRepo(ctx.cwd);
614
666
  const usage = ctx.getContextUsage();
@@ -616,6 +668,9 @@ export default function (pi) {
616
668
  const tokens = usage?.tokens != null ? `${usage.tokens} tok` : "n/a";
617
669
  const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
618
670
  const st = store.stats(sid);
671
+ const di = store.dataInvariant();
672
+ const fmtB = (b) => b >= 1_048_576 ? `${(b / 1_048_576).toFixed(1)} MiB` :
673
+ b >= 1024 ? `${(b / 1024).toFixed(1)} KiB` : `${b} B`;
619
674
  ctx.ui.notify(`[mega-compact] pct=${pct} tokens=${tokens} tier=${config.tier} fastGate=${config.fastGatePct}% ` +
620
675
  `threshold=${config.thresholdTokens} auto=${config.auto} autoInline=${config.autoInline}\n` +
621
676
  `[mega-compact] store: ${st.checkpointCount} chkpt Β· ` +
@@ -623,6 +678,10 @@ export default function (pi) {
623
678
  `injected=${st.injectedCount} Β· dedup=${(st.dedupHitRate * 100).toFixed(0)}%\n` +
624
679
  `[mega-compact] anchor=${config.anchorUserMessages} preserveRecent=${config.preserveRecent} ` +
625
680
  `autoInlineK=${config.autoInlineK} dedupSim=${config.dedupSim} debug=${config.debug}\n` +
681
+ `[mega-compact] πŸ›‘ data-safe: ${di.regionsRetained} regions retained ` +
682
+ `(${fmtB(di.compressedOriginalBytes)} compressed-original) Β· ` +
683
+ `${di.duplicatesCollapsed} dedup-duplicates collapsed Β· ` +
684
+ `${C.green}0 bytes permanently deleted${C.reset}\n` +
626
685
  `[mega-compact] stateDir=${currentStateDir}`);
627
686
  },
628
687
  });
@@ -103,6 +103,7 @@ export function compactSession(input, store = getDefaultStore()) {
103
103
  tokenEstimate: storedTokens,
104
104
  originalTokenEstimate,
105
105
  timestamp: input.timestamp ?? 0,
106
+ onTier: input.onTier,
106
107
  });
107
108
  return {
108
109
  skipped: false,
@@ -0,0 +1,81 @@
1
+ import { test } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { mkdtempSync, rmSync } from "node:fs";
4
+ import { tmpdir } from "node:os";
5
+ import { join } from "node:path";
6
+ import { VectorStore } from "../vectorStore.js";
7
+ import { dataInvariantStats } from "./sqlite.js";
8
+ const baseTmp = mkdtempSync(join(tmpdir(), "mc-p01-"));
9
+ let counter = 0;
10
+ function store(opts = {}) {
11
+ const dir = join(baseTmp, `run-${counter++}`);
12
+ return { s: new VectorStore({ dedupSim: opts.dedupSim ?? 0.9, stateDir: dir }), dir };
13
+ }
14
+ // --- Phase 0: data-safety invariant ---------------------------------------
15
+ test("Phase 0: every added region retains a compressed-original and deletes nothing", () => {
16
+ const { s, dir } = store();
17
+ s.add({ sessionId: "sess_a", summary: "s1", regionText: "the quick brown fox jumps", tokenEstimate: 5, originalTokenEstimate: 50, timestamp: 1 });
18
+ s.add({ sessionId: "sess_a", summary: "s2", regionText: "a totally different region of work", tokenEstimate: 6, originalTokenEstimate: 60, timestamp: 2 });
19
+ const di = dataInvariantStats(dir);
20
+ assert.equal(di.regionsRetained, 2, "both regions retained");
21
+ assert.ok(di.compressedOriginalBytes > 0, "compressed-original bytes retained");
22
+ assert.equal(di.bytesPermanentlyDeleted, 0, "INVARIANT: nothing permanently deleted");
23
+ });
24
+ test("Phase 0: dedup collapses a duplicate but original is still retained (not deleted)", () => {
25
+ const { s, dir } = store();
26
+ const text = "identical region content that will dedup on the second add";
27
+ s.add({ sessionId: "sess_b", summary: "s", regionText: text, tokenEstimate: 5, originalTokenEstimate: 50, timestamp: 1 });
28
+ const r2 = s.add({ sessionId: "sess_b", summary: "s", regionText: text, tokenEstimate: 5, originalTokenEstimate: 50, timestamp: 2 });
29
+ assert.equal(r2.deduped, true, "second add deduped");
30
+ const di = dataInvariantStats(dir);
31
+ assert.equal(di.bytesPermanentlyDeleted, 0, "INVARIANT: dedup never deletes data");
32
+ assert.ok(di.regionsRetained >= 1, "survivor region still retained");
33
+ });
34
+ // --- Phase 1: per-tier progress callback ----------------------------------
35
+ test("Phase 1: onTier fires L0β†’L1β†’L2β†’stored for a genuinely new region", () => {
36
+ const { s } = store();
37
+ // Seed one checkpoint so L2 has something to scan against.
38
+ s.add({ sessionId: "sess_c", summary: "seed", regionText: "seed region alpha", timestamp: 1 });
39
+ const events = [];
40
+ s.add({
41
+ sessionId: "sess_c",
42
+ summary: "new",
43
+ regionText: "a brand new region beta that is not a duplicate",
44
+ timestamp: 2,
45
+ onTier: (ev) => events.push({ tier: ev.tier, status: ev.status }),
46
+ });
47
+ const tiers = events.map((e) => e.tier);
48
+ assert.ok(tiers.includes("L0"), "L0 fired");
49
+ assert.ok(tiers.includes("L1"), "L1 fired");
50
+ assert.ok(tiers.includes("L2"), "L2 fired");
51
+ assert.equal(events.at(-1)?.tier, "new", "final event is the 'new' tier");
52
+ assert.equal(events.at(-1)?.status, "stored", "final status is stored");
53
+ });
54
+ test("Phase 1: onTier reports a deduped outcome and short-circuits at the matching tier", () => {
55
+ const { s } = store();
56
+ const text = "region that will exact-dedup on the second pass";
57
+ s.add({ sessionId: "sess_d", summary: "s", regionText: text, timestamp: 1 });
58
+ const events = [];
59
+ const r = s.add({
60
+ sessionId: "sess_d",
61
+ summary: "s",
62
+ regionText: text,
63
+ timestamp: 2,
64
+ onTier: (ev) => events.push(ev),
65
+ });
66
+ assert.equal(r.deduped, true);
67
+ const deduped = events.find((e) => e.status === "deduped");
68
+ assert.ok(deduped, "a deduped event fired");
69
+ assert.equal(deduped?.tier, "L0", "exact duplicate short-circuits at L0");
70
+ // Must NOT reach the final stored event.
71
+ assert.ok(!events.some((e) => e.status === "stored"), "no stored event on dedup");
72
+ });
73
+ test("Phase 1: onTier is optional (back-compat) β€” add() works without it", () => {
74
+ const { s } = store();
75
+ const r = s.add({ sessionId: "sess_e", summary: "s", regionText: "no callback here", timestamp: 1 });
76
+ assert.equal(r.deduped, false);
77
+ });
78
+ process.on("exit", () => { try {
79
+ rmSync(baseTmp, { recursive: true, force: true });
80
+ }
81
+ catch { /* ignore */ } });
@@ -495,6 +495,26 @@ export function storeStats(sessionId, stateDir = getStateDir()) {
495
495
  lastSummary,
496
496
  };
497
497
  }
498
+ export function dataInvariantStats(stateDir = getStateDir()) {
499
+ const db = openStore(stateDir);
500
+ const row = db
501
+ .prepare(`SELECT
502
+ COUNT(compressed_original) AS withBlob,
503
+ COALESCE(SUM(LENGTH(compressed_original)),0) AS blobBytes,
504
+ SUM(CASE WHEN compressed_original IS NULL THEN 1 ELSE 0 END) AS noBlob
505
+ FROM context_chunks WHERE dedup_status != 'removed'`)
506
+ .get();
507
+ const removed = db
508
+ .prepare(`SELECT COUNT(*) AS c FROM context_chunks WHERE dedup_status = 'removed'`)
509
+ .get();
510
+ return {
511
+ regionsRetained: row.withBlob,
512
+ compressedOriginalBytes: row.blobBytes,
513
+ regionsWithoutBlob: row.noBlob ?? 0,
514
+ bytesPermanentlyDeleted: 0,
515
+ duplicatesCollapsed: removed.c,
516
+ };
517
+ }
498
518
  export function repoStats(stateDir = getStateDir()) {
499
519
  const db = openStore(stateDir);
500
520
  const row = db
@@ -19,7 +19,7 @@ import { isNearDuplicate } from "./dedup/l1-verify.js";
19
19
  import { mmrRerank } from "./dedup/mmr.js";
20
20
  import { topK } from "./dedup/topk.js";
21
21
  import { openBloom, saveBloom } from "./store/bloom.js";
22
- import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, setDedupStatus, addTokensSaved, getDedupStats, bumpDedupStats, repoStats as repoStatsFromStore, } from "./store/sqlite.js";
22
+ import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, setDedupStatus, addTokensSaved, getDedupStats, bumpDedupStats, repoStats as repoStatsFromStore, dataInvariantStats, } from "./store/sqlite.js";
23
23
  import { migrateJsonToSqlite } from "./store/migrate.js";
24
24
  /** Default L2 semantic-dedup enable flag (trigram embedder is local, zero-network). */
25
25
  export const L2_ENABLED = true;
@@ -85,6 +85,9 @@ export class VectorStore {
85
85
  // we persist (orig βˆ’ stored). Falls back to stored when orig is unknown.
86
86
  const origTokens = input.originalTokenEstimate ?? input.tokenEstimate ?? 0;
87
87
  const cfg = this.cfg;
88
+ // Live per-tier progress hook (Phase 1). Sync + optional; fired at each tier
89
+ // so the UI can paint "L0 βœ“ β†’ L1 βœ“ β†’ L2 0.91 β†’ stored" during a compaction.
90
+ const onTier = input.onTier;
88
91
  // Tracks whether a tier matched while in MARK_ONLY (record-but-don't-collapse),
89
92
  // and which tier.
90
93
  let markOnly = null;
@@ -95,6 +98,7 @@ export class VectorStore {
95
98
  // skips the scan; a hit is only a candidate, confirmed against `all` below.
96
99
  // Gated by L0_ENABLED (Sprint 14). MARK_ONLY_L0 records the decision but
97
100
  // does not collapse β€” the new region is still stored.
101
+ onTier?.({ tier: "L0", status: "scanning" });
98
102
  const digest = computeContentDigest(input.regionText);
99
103
  const bloom = openBloom(this.stateDir);
100
104
  if (cfg.L0_ENABLED && bloom.maybeHas(digest.contentHash)) {
@@ -112,6 +116,7 @@ export class VectorStore {
112
116
  addTokensSaved(origTokens, this.stateDir);
113
117
  const r = { checkpoint: contentMatch, deduped: true, reason: "contentHash" };
114
118
  this.record("L0", "deduped", "contentHash", Date.now() - t0);
119
+ onTier?.({ tier: "L0", status: "deduped", detail: "contentHash" });
115
120
  return r;
116
121
  }
117
122
  }
@@ -129,6 +134,7 @@ export class VectorStore {
129
134
  addTokensSaved(origTokens, this.stateDir);
130
135
  const r = { checkpoint: regionMatch, deduped: true, reason: "regionHash" };
131
136
  this.record("L0", "deduped", "regionHash", Date.now() - t0);
137
+ onTier?.({ tier: "L0", status: "deduped", detail: "regionHash" });
132
138
  return r;
133
139
  }
134
140
  }
@@ -152,14 +158,18 @@ export class VectorStore {
152
158
  addTokensSaved(origTokens, this.stateDir);
153
159
  const r = { checkpoint: summaryMatch, deduped: true, reason: "summaryHash" };
154
160
  this.record("L0", "deduped", "summaryHash", Date.now() - t0);
161
+ onTier?.({ tier: "L0", status: "deduped", detail: "summaryHash" });
155
162
  return r;
156
163
  }
157
164
  }
158
165
  }
166
+ // L0 did not collapse this region.
167
+ onTier?.({ tier: "L0", status: "passed" });
159
168
  // 2b. L1 MinHash/LSH near-duplicate dedup (Sprint 11) β€” catches one-word
160
169
  // edits / rewordings that L0's exact hash misses. Cheap LSH bucket
161
170
  // retrieval β†’ trigram verification (pg_trgm-equivalent) as the final gate.
162
171
  // Gated by L1_ENABLED (Sprint 14); MARK_ONLY_L1 records but doesn't collapse.
172
+ onTier?.({ tier: "L1", status: "scanning" });
163
173
  if (cfg.L1_ENABLED) {
164
174
  const l1 = this.findL1Duplicate(sessionId, input.regionText, all);
165
175
  if (l1 && !cfg.MARK_ONLY_L1) {
@@ -168,11 +178,13 @@ export class VectorStore {
168
178
  bumpDedupStats(true, this.stateDir);
169
179
  const r = { checkpoint: l1, deduped: true, reason: "l1MinHash" };
170
180
  this.record("L1", "deduped", "l1MinHash", Date.now() - t0);
181
+ onTier?.({ tier: "L1", status: "deduped", detail: "l1MinHash" });
171
182
  return r;
172
183
  }
173
184
  if (l1 && cfg.MARK_ONLY_L1)
174
185
  markOnly = "L1";
175
186
  }
187
+ onTier?.({ tier: "L1", status: "passed" });
176
188
  // 3. L2 semantic dedup β€” catches near-identical / semantically-similar regions
177
189
  // via cosine over the embedding. topicSummary is used for summaryHash dedup
178
190
  // (tier 2); the vector index is keyed on the original region for backward-
@@ -183,6 +195,7 @@ export class VectorStore {
183
195
  const SIMILARITY_BUDGET_MS = cfg.SIMILARITY_BUDGET_MS;
184
196
  const simThreshold = this.l2Threshold; // from cfg.L2_COSINE (default 0.85 trigram)
185
197
  const embedding = this.embedder.embed(input.regionText);
198
+ onTier?.({ tier: "L2", status: "scanning" });
186
199
  if (cfg.L2_ENABLED && all.length > 0) {
187
200
  const start = Date.now();
188
201
  let timedOut = false;
@@ -204,10 +217,12 @@ export class VectorStore {
204
217
  addTokensSaved(origTokens, this.stateDir);
205
218
  const r = { checkpoint: nearest.checkpoint, deduped: true, reason: "contentSimilarity" };
206
219
  this.record("L2", "deduped", "contentSimilarity", Date.now() - t0);
220
+ onTier?.({ tier: "L2", status: "deduped", detail: nearest.sim.toFixed(2) });
207
221
  return r;
208
222
  }
209
223
  markOnly = "L2";
210
224
  }
225
+ onTier?.({ tier: "L2", status: "passed", detail: `best ${nearest.sim.toFixed(2)}` });
211
226
  }
212
227
  // 4. Genuinely new β€” create checkpoint
213
228
  const checkpointId = nextCheckpointId(sessionId, this.stateDir);
@@ -265,6 +280,7 @@ export class VectorStore {
265
280
  }
266
281
  // Cumulative store-wide dedup accounting (attempt, not collapsed).
267
282
  bumpDedupStats(false, this.stateDir);
283
+ onTier?.({ tier: "new", status: "stored" });
268
284
  return { checkpoint, deduped: false };
269
285
  }
270
286
  /**
@@ -462,4 +478,8 @@ export class VectorStore {
462
478
  repoStats() {
463
479
  return repoStatsFromStore(this.stateDir);
464
480
  }
481
+ /** Data-safety invariant (Phase 0): regions retained vs bytes permanently deleted. */
482
+ dataInvariant() {
483
+ return dataInvariantStats(this.stateDir);
484
+ }
465
485
  }
@@ -96,6 +96,7 @@ function readSnapshot(snapshotPath: string) {
96
96
  store: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, injectedCount: 0, dedupHitRate: 0, storageDedupRate: 0, dedupCollapsed: 0 },
97
97
  crew: { activeAgents: 0, currentTurn: 0 },
98
98
  repo: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupAttempts: 0, dedupCollapsed: 0, storageDedupRate: 0 },
99
+ integrity: { regionsRetained: 0, compressedOriginalBytes: 0, duplicatesCollapsed: 0, bytesPermanentlyDeleted: 0 },
99
100
  } as Snapshot;
100
101
  }
101
102
  }
@@ -128,6 +129,10 @@ function dashboardHtml(tierName: string): string {
128
129
  h1 .tier { background: #1f6feb; color: #fff; font-size: 11px; font-weight: 700; padding: 2px 8px; border-radius: 10px; text-transform: uppercase; letter-spacing: .5px; }
129
130
  .grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; margin-bottom: 20px; }
130
131
  .card { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 16px; }
132
+ .card.safe { border-color: #238636; }
133
+ .card.safe h2 { color: #3fb950; }
134
+ .safe-note { font-size: 12px; color: #8b949e; margin: 12px 0 0; line-height: 1.5; }
135
+ .value.ok { color: #3fb950; }
131
136
  .card h2 { font-size: 12px; text-transform: uppercase; letter-spacing: .5px; color: #8b949e; margin-bottom: 12px; font-weight: 600; }
132
137
  .meter-track { background: #21262d; border-radius: 4px; height: 20px; overflow: hidden; margin: 8px 0; }
133
138
  .meter-fill { height: 100%; border-radius: 4px; transition: width .6s ease; min-width: 2px; }
@@ -208,6 +213,16 @@ function dashboardHtml(tierName: string): string {
208
213
  <span class="label">Storage Dedup</span><span class="value" id="rp-sdedup">0%</span>
209
214
  </div>
210
215
  </div>
216
+ <div class="card safe">
217
+ <h2>πŸ›‘ Data Safety</h2>
218
+ <div class="stat-grid">
219
+ <span class="label">Regions Retained</span><span class="value" id="ig-retained">0</span>
220
+ <span class="label">Compressed-Original</span><span class="value" id="ig-bytes">0 B</span>
221
+ <span class="label">Dedup Duplicates</span><span class="value" id="ig-dupes">0</span>
222
+ <span class="label">Permanently Deleted</span><span class="value ok" id="ig-deleted">0 B</span>
223
+ </div>
224
+ <p class="safe-note">Every compacted region is kept verbatim (compressed). "Drop" = removed from the live window only. We never delete your data.</p>
225
+ </div>
211
226
  <div class="card">
212
227
  <h2>Configuration</h2>
213
228
  <div class="conf-grid">
@@ -287,6 +302,19 @@ function dashboardHtml(tierName: string): string {
287
302
  var rsdr = repo.storageDedupRate || 0;
288
303
  document.getElementById('rp-sdedup').textContent = (rsdr * 100 >= 10 ? Math.round(rsdr * 100) : (rsdr * 100).toFixed(1)) + '%';
289
304
 
305
+ // Data-safety invariant (Phase 0 β€” trust foundation).
306
+ var ig = d.integrity || { regionsRetained: 0, compressedOriginalBytes: 0, duplicatesCollapsed: 0, bytesPermanentlyDeleted: 0 };
307
+ function fmtBytes(b) {
308
+ b = b || 0;
309
+ if (b >= 1048576) return (b / 1048576).toFixed(1) + ' MiB';
310
+ if (b >= 1024) return (b / 1024).toFixed(1) + ' KiB';
311
+ return b + ' B';
312
+ }
313
+ document.getElementById('ig-retained').textContent = (ig.regionsRetained || 0).toLocaleString();
314
+ document.getElementById('ig-bytes').textContent = fmtBytes(ig.compressedOriginalBytes);
315
+ document.getElementById('ig-dupes').textContent = (ig.duplicatesCollapsed || 0).toLocaleString();
316
+ document.getElementById('ig-deleted').textContent = fmtBytes(ig.bytesPermanentlyDeleted);
317
+
290
318
  // Crew / agents (live sub-agent activity + turn).
291
319
  var crew = d.crew || { activeAgents: 0, currentTurn: 0 };
292
320
  document.getElementById('cr-agents').textContent = crew.activeAgents || 0;
@@ -214,6 +214,13 @@ interface DashboardSnapshot {
214
214
  dedupCollapsed: number; // cumulative deduped collapses (store-wide)
215
215
  storageDedupRate: number; // deduped / attempts, 0..1
216
216
  };
217
+ /** Phase 0 data-safety invariant (trust foundation). */
218
+ integrity: {
219
+ regionsRetained: number; // checkpoints with a recoverable compressed-original
220
+ compressedOriginalBytes: number; // bytes of compressed-original retained (recoverable)
221
+ duplicatesCollapsed: number; // dedup duplicates (original kept on survivor)
222
+ bytesPermanentlyDeleted: number; // ALWAYS 0 β€” the invariant
223
+ };
217
224
  }
218
225
 
219
226
  class Dashboard {
@@ -280,6 +287,7 @@ export default function (pi: ExtensionAPI) {
280
287
  if (ctx) bindRepo(ctx.cwd);
281
288
  const st = store.stats(rt.sessionId);
282
289
  const repo = store.repoStats();
290
+ const di = store.dataInvariant();
283
291
  const armed = lastCtxPercent != null && lastCtxPercent >= config.fastGatePct;
284
292
  const ready = armed && (lastCtxTokens ?? 0) >= config.thresholdTokens;
285
293
  dashboard.snapshot({
@@ -318,6 +326,12 @@ export default function (pi: ExtensionAPI) {
318
326
  dedupCollapsed: repo.dedupCollapsed,
319
327
  storageDedupRate: repo.storageDedupRate,
320
328
  },
329
+ integrity: {
330
+ regionsRetained: di.regionsRetained,
331
+ compressedOriginalBytes: di.compressedOriginalBytes,
332
+ duplicatesCollapsed: di.duplicatesCollapsed,
333
+ bytesPermanentlyDeleted: di.bytesPermanentlyDeleted,
334
+ },
321
335
  });
322
336
 
323
337
  // Live stats widget above the editor
@@ -350,7 +364,13 @@ export default function (pi: ExtensionAPI) {
350
364
  ];
351
365
  // Live "now processing" line β€” teal while fresh (≀4s), then the last-seen
352
366
  // action keeps the widget lively. Cleared on session reset.
353
- if (currentActivity) {
367
+ if (tierTrace) {
368
+ // Per-tier dedup progress is more relevant than the last action while a
369
+ // compaction is mid-flight, so prefer it when fresh.
370
+ const fresh = Date.now() - lastActivityAt < 4000;
371
+ if (fresh) lines.push(` ${tierTrace}`);
372
+ else if (currentActivity) lines.push(` ${C.dim}${currentActivity}${C.reset}`);
373
+ } else if (currentActivity) {
354
374
  const fresh = Date.now() - lastActivityAt < 4000;
355
375
  lines.push(` ${fresh ? C.teal : C.dim}${currentActivity}${C.reset}`);
356
376
  }
@@ -382,6 +402,10 @@ export default function (pi: ExtensionAPI) {
382
402
  // the widget is never blank. Cleared on session reset.
383
403
  let currentActivity: string | undefined;
384
404
  let lastActivityAt = 0;
405
+ // Live per-tier dedup trace (Phase 1): e.g. "L0 βœ“ β†’ L1 βœ“ β†’ L2 0.91 β†’ stored".
406
+ // Built from the store's sync onTier callback during a compaction so the user
407
+ // watches each tier evaluate in real time. Cleared once the outcome settles.
408
+ let tierTrace: string | undefined;
385
409
  // ANSI palette for the toolbar. The pi TUI's Text component preserves ANSI
386
410
  // escape codes (see wrapTextWithAnsi), so raw escapes render as colors. No
387
411
  // chalk dependency needed β€” these are just strings.
@@ -421,6 +445,32 @@ export default function (pi: ExtensionAPI) {
421
445
  currentTurn = 0;
422
446
  currentActivity = undefined;
423
447
  lastActivityAt = 0;
448
+ tierTrace = undefined;
449
+ }
450
+
451
+ /** Build the sync onTier callback that paints the live per-tier trace. */
452
+ function makeTierCallback(ctx: ExtensionContext): (ev: { tier: "L0" | "L1" | "L2" | "new"; status: "scanning" | "deduped" | "passed" | "stored"; detail?: string }) => void {
453
+ const order: Array<"L0" | "L1" | "L2" | "new"> = ["L0", "L1", "L2", "new"];
454
+ const seen = new Map<string, string>();
455
+ const glyph = (status: string) =>
456
+ status === "deduped" ? `${C.green}βœ“${C.reset}` :
457
+ status === "passed" ? `${C.dim}β—‹${C.reset}` :
458
+ status === "scanning" ? `${C.amber}…${C.reset}` :
459
+ `${C.cyan}●${C.reset}`;
460
+ return (ev) => {
461
+ const label =
462
+ ev.tier === "new"
463
+ ? `${C.cyan}stored${C.reset}`
464
+ : `${ev.tier} ${glyph(ev.status)}` +
465
+ (ev.detail ? ` ${C.gray}(${ev.detail})${C.reset}` : "");
466
+ // Show the most recent outcome per tier (collapses re-fires).
467
+ seen.set(ev.tier, label);
468
+ const show: string[] = [];
469
+ for (const t of order) if (seen.has(t)) show.push(seen.get(t)!);
470
+ tierTrace = `${C.teal}βš™${C.reset} ${show.join(` ${C.gray}β†’${C.reset} `)}`;
471
+ lastActivityAt = Date.now();
472
+ try { snapshot(ctx); } catch { /* non-fatal */ }
473
+ };
424
474
  }
425
475
 
426
476
  /** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
@@ -445,6 +495,7 @@ export default function (pi: ExtensionAPI) {
445
495
  keepFrom,
446
496
  summary: opts.summary,
447
497
  timestamp: Date.now(),
498
+ onTier: makeTierCallback(ctx),
448
499
  },
449
500
  store,
450
501
  );
@@ -478,6 +529,9 @@ export default function (pi: ExtensionAPI) {
478
529
  ? `β™» deduped ${fileLabel}`
479
530
  : `πŸ—œ compacted ${result.checkpointId} Β· ${fileLabel}`;
480
531
  lastActivityAt = Date.now();
532
+ // The per-tier trace has settled into the final outcome β€” fold it back into
533
+ // the activity line and stop showing the live trace.
534
+ tierTrace = undefined;
481
535
 
482
536
  // Record session activity + a daily-log entry in the per-repo SQLite store
483
537
  // (foundation for resume-sessions / daily-log features). Best-effort β€” never
@@ -740,7 +794,7 @@ export default function (pi: ExtensionAPI) {
740
794
  });
741
795
 
742
796
  pi.registerCommand("mega-status", {
743
- description: "Show mega-compact config and current context usage.",
797
+ description: "Show mega-compact config, context usage, and the data-safety invariant.",
744
798
  handler: async (_args: string, ctx: ExtensionContext) => {
745
799
  bindRepo(ctx.cwd);
746
800
  const usage = ctx.getContextUsage();
@@ -748,6 +802,10 @@ export default function (pi: ExtensionAPI) {
748
802
  const tokens = usage?.tokens != null ? `${usage.tokens} tok` : "n/a";
749
803
  const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
750
804
  const st = store.stats(sid);
805
+ const di = store.dataInvariant();
806
+ const fmtB = (b: number) =>
807
+ b >= 1_048_576 ? `${(b / 1_048_576).toFixed(1)} MiB` :
808
+ b >= 1024 ? `${(b / 1024).toFixed(1)} KiB` : `${b} B`;
751
809
  ctx.ui.notify(
752
810
  `[mega-compact] pct=${pct} tokens=${tokens} tier=${config.tier} fastGate=${config.fastGatePct}% ` +
753
811
  `threshold=${config.thresholdTokens} auto=${config.auto} autoInline=${config.autoInline}\n` +
@@ -756,6 +814,10 @@ export default function (pi: ExtensionAPI) {
756
814
  `injected=${st.injectedCount} Β· dedup=${(st.dedupHitRate * 100).toFixed(0)}%\n` +
757
815
  `[mega-compact] anchor=${config.anchorUserMessages} preserveRecent=${config.preserveRecent} ` +
758
816
  `autoInlineK=${config.autoInlineK} dedupSim=${config.dedupSim} debug=${config.debug}\n` +
817
+ `[mega-compact] πŸ›‘ data-safe: ${di.regionsRetained} regions retained ` +
818
+ `(${fmtB(di.compressedOriginalBytes)} compressed-original) Β· ` +
819
+ `${di.duplicatesCollapsed} dedup-duplicates collapsed Β· ` +
820
+ `${C.green}0 bytes permanently deleted${C.reset}\n` +
759
821
  `[mega-compact] stateDir=${currentStateDir}`,
760
822
  );
761
823
  },
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-mega-compact",
3
- "version": "0.4.10",
3
+ "version": "0.4.11",
4
4
  "description": "Layered, local, vector-backed context compressor for pi β€” supersede/collapse/cluster compaction with deduped inline recall.",
5
5
  "type": "module",
6
6
  "license": "BSD-2-Clause",
package/src/engine.ts CHANGED
@@ -38,6 +38,22 @@ export interface CompactInput {
38
38
  timestamp?: number;
39
39
  /** When true (default), use extractive summary instead of raw concatenation. */
40
40
  useExtractiveSummary?: boolean;
41
+ /** Sync progress callback fired by the store as each dedup tier is evaluated
42
+ * (L0β†’L1β†’L2β†’new). Lets the UI render a live "L0 βœ“ β†’ L1 βœ“ β†’ L2 0.91 β†’ stored"
43
+ * progress line during compaction. Never awaited; must be side-effect-free-ish
44
+ * and cheap. Optional; back-compat with callers that don't pass it. */
45
+ onTier?: (ev: TierProgress) => void;
46
+ }
47
+
48
+ /** Progress event emitted by the store as each dedup tier is evaluated. */
49
+ export interface TierProgress {
50
+ /** Tier being evaluated: "L0" | "L1" | "L2" | "new". */
51
+ tier: "L0" | "L1" | "L2" | "new";
52
+ /** "scanning" while the tier is being evaluated, "deduped" when it matched,
53
+ * "passed" when no match, "stored" at the final outcome. */
54
+ status: "scanning" | "deduped" | "passed" | "stored";
55
+ /** Optional detail β€” e.g. the L2 cosine sim ("0.91") or the dedup reason. */
56
+ detail?: string;
41
57
  }
42
58
 
43
59
  export interface CompactResult {
@@ -157,6 +173,7 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
157
173
  tokenEstimate: storedTokens,
158
174
  originalTokenEstimate,
159
175
  timestamp: input.timestamp ?? 0,
176
+ onTier: input.onTier,
160
177
  });
161
178
 
162
179
  return {
@@ -0,0 +1,88 @@
1
+ import { test } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { mkdtempSync, rmSync } from "node:fs";
4
+ import { tmpdir } from "node:os";
5
+ import { join } from "node:path";
6
+ import { VectorStore } from "../vectorStore.js";
7
+ import { dataInvariantStats } from "./sqlite.js";
8
+
9
+ const baseTmp = mkdtempSync(join(tmpdir(), "mc-p01-"));
10
+
11
+ let counter = 0;
12
+ function store(opts: { dedupSim?: number } = {}) {
13
+ const dir = join(baseTmp, `run-${counter++}`);
14
+ return { s: new VectorStore({ dedupSim: opts.dedupSim ?? 0.9, stateDir: dir }), dir };
15
+ }
16
+
17
+ // --- Phase 0: data-safety invariant ---------------------------------------
18
+
19
+ test("Phase 0: every added region retains a compressed-original and deletes nothing", () => {
20
+ const { s, dir } = store();
21
+ s.add({ sessionId: "sess_a", summary: "s1", regionText: "the quick brown fox jumps", tokenEstimate: 5, originalTokenEstimate: 50, timestamp: 1 });
22
+ s.add({ sessionId: "sess_a", summary: "s2", regionText: "a totally different region of work", tokenEstimate: 6, originalTokenEstimate: 60, timestamp: 2 });
23
+ const di = dataInvariantStats(dir);
24
+ assert.equal(di.regionsRetained, 2, "both regions retained");
25
+ assert.ok(di.compressedOriginalBytes > 0, "compressed-original bytes retained");
26
+ assert.equal(di.bytesPermanentlyDeleted, 0, "INVARIANT: nothing permanently deleted");
27
+ });
28
+
29
+ test("Phase 0: dedup collapses a duplicate but original is still retained (not deleted)", () => {
30
+ const { s, dir } = store();
31
+ const text = "identical region content that will dedup on the second add";
32
+ s.add({ sessionId: "sess_b", summary: "s", regionText: text, tokenEstimate: 5, originalTokenEstimate: 50, timestamp: 1 });
33
+ const r2 = s.add({ sessionId: "sess_b", summary: "s", regionText: text, tokenEstimate: 5, originalTokenEstimate: 50, timestamp: 2 });
34
+ assert.equal(r2.deduped, true, "second add deduped");
35
+ const di = dataInvariantStats(dir);
36
+ assert.equal(di.bytesPermanentlyDeleted, 0, "INVARIANT: dedup never deletes data");
37
+ assert.ok(di.regionsRetained >= 1, "survivor region still retained");
38
+ });
39
+
40
+ // --- Phase 1: per-tier progress callback ----------------------------------
41
+
42
+ test("Phase 1: onTier fires L0β†’L1β†’L2β†’stored for a genuinely new region", () => {
43
+ const { s } = store();
44
+ // Seed one checkpoint so L2 has something to scan against.
45
+ s.add({ sessionId: "sess_c", summary: "seed", regionText: "seed region alpha", timestamp: 1 });
46
+ const events: Array<{ tier: string; status: string }> = [];
47
+ s.add({
48
+ sessionId: "sess_c",
49
+ summary: "new",
50
+ regionText: "a brand new region beta that is not a duplicate",
51
+ timestamp: 2,
52
+ onTier: (ev) => events.push({ tier: ev.tier, status: ev.status }),
53
+ });
54
+ const tiers = events.map((e) => e.tier);
55
+ assert.ok(tiers.includes("L0"), "L0 fired");
56
+ assert.ok(tiers.includes("L1"), "L1 fired");
57
+ assert.ok(tiers.includes("L2"), "L2 fired");
58
+ assert.equal(events.at(-1)?.tier, "new", "final event is the 'new' tier");
59
+ assert.equal(events.at(-1)?.status, "stored", "final status is stored");
60
+ });
61
+
62
+ test("Phase 1: onTier reports a deduped outcome and short-circuits at the matching tier", () => {
63
+ const { s } = store();
64
+ const text = "region that will exact-dedup on the second pass";
65
+ s.add({ sessionId: "sess_d", summary: "s", regionText: text, timestamp: 1 });
66
+ const events: Array<{ tier: string; status: string; detail?: string }> = [];
67
+ const r = s.add({
68
+ sessionId: "sess_d",
69
+ summary: "s",
70
+ regionText: text,
71
+ timestamp: 2,
72
+ onTier: (ev) => events.push(ev),
73
+ });
74
+ assert.equal(r.deduped, true);
75
+ const deduped = events.find((e) => e.status === "deduped");
76
+ assert.ok(deduped, "a deduped event fired");
77
+ assert.equal(deduped?.tier, "L0", "exact duplicate short-circuits at L0");
78
+ // Must NOT reach the final stored event.
79
+ assert.ok(!events.some((e) => e.status === "stored"), "no stored event on dedup");
80
+ });
81
+
82
+ test("Phase 1: onTier is optional (back-compat) β€” add() works without it", () => {
83
+ const { s } = store();
84
+ const r = s.add({ sessionId: "sess_e", summary: "s", regionText: "no callback here", timestamp: 1 });
85
+ assert.equal(r.deduped, false);
86
+ });
87
+
88
+ process.on("exit", () => { try { rmSync(baseTmp, { recursive: true, force: true }); } catch { /* ignore */ } });
@@ -636,6 +636,49 @@ export interface RepoStats {
636
636
  storageDedupRate: number;
637
637
  }
638
638
 
639
+ /**
640
+ * Data-safety invariant metrics (Phase 0 β€” trust foundation). Proves that every
641
+ * compacted region is still recoverable: we retain a compressed_original blob for
642
+ * each checkpoint and permanently delete nothing. "removed" rows are SemDeDup
643
+ * duplicates whose ORIGINAL is still retained on the surviving checkpoint β€” they
644
+ * are not data loss, so they are reported separately, not as deletions.
645
+ */
646
+ export interface DataInvariantStats {
647
+ /** Checkpoints with a recoverable compressed_original blob. */
648
+ regionsRetained: number;
649
+ /** Total bytes of compressed_original retained (recoverable verbatim). */
650
+ compressedOriginalBytes: number;
651
+ /** Checkpoints missing a compressed_original blob (pre-blob or direct add). */
652
+ regionsWithoutBlob: number;
653
+ /** Bytes permanently deleted by the extension. ALWAYS 0 β€” the invariant. */
654
+ bytesPermanentlyDeleted: number;
655
+ /** Duplicate rows collapsed by dedup (original retained on the survivor). */
656
+ duplicatesCollapsed: number;
657
+ }
658
+
659
+ export function dataInvariantStats(stateDir: string = getStateDir()): DataInvariantStats {
660
+ const db = openStore(stateDir);
661
+ const row = db
662
+ .prepare(
663
+ `SELECT
664
+ COUNT(compressed_original) AS withBlob,
665
+ COALESCE(SUM(LENGTH(compressed_original)),0) AS blobBytes,
666
+ SUM(CASE WHEN compressed_original IS NULL THEN 1 ELSE 0 END) AS noBlob
667
+ FROM context_chunks WHERE dedup_status != 'removed'`,
668
+ )
669
+ .get() as { withBlob: number; blobBytes: number; noBlob: number };
670
+ const removed = db
671
+ .prepare(`SELECT COUNT(*) AS c FROM context_chunks WHERE dedup_status = 'removed'`)
672
+ .get() as { c: number };
673
+ return {
674
+ regionsRetained: row.withBlob,
675
+ compressedOriginalBytes: row.blobBytes,
676
+ regionsWithoutBlob: row.noBlob ?? 0,
677
+ bytesPermanentlyDeleted: 0,
678
+ duplicatesCollapsed: removed.c,
679
+ };
680
+ }
681
+
639
682
  export function repoStats(stateDir: string = getStateDir()): RepoStats {
640
683
  const db = openStore(stateDir);
641
684
  const row = db
@@ -36,6 +36,7 @@ import {
36
36
  getDedupStats,
37
37
  bumpDedupStats,
38
38
  repoStats as repoStatsFromStore,
39
+ dataInvariantStats,
39
40
  } from "./store/sqlite.js";
40
41
  import { migrateJsonToSqlite } from "./store/migrate.js";
41
42
 
@@ -61,6 +62,10 @@ export interface AddInput {
61
62
  /** Raw text of the compacted region β€” used to derive the regionHash + vector. */
62
63
  regionText: string;
63
64
  timestamp: number;
65
+ /** Sync progress callback fired as each dedup tier is evaluated (L0β†’L1β†’L2β†’new).
66
+ * Lets the UI render live per-tier progress during compaction. Never awaited;
67
+ * must be cheap. Optional for back-compat. */
68
+ onTier?: (ev: { tier: "L0" | "L1" | "L2" | "new"; status: "scanning" | "deduped" | "passed" | "stored"; detail?: string }) => void;
64
69
  }
65
70
 
66
71
  /** Default L2 semantic-dedup enable flag (trigram embedder is local, zero-network). */
@@ -150,6 +155,9 @@ export class VectorStore {
150
155
  // we persist (orig βˆ’ stored). Falls back to stored when orig is unknown.
151
156
  const origTokens = input.originalTokenEstimate ?? input.tokenEstimate ?? 0;
152
157
  const cfg = this.cfg;
158
+ // Live per-tier progress hook (Phase 1). Sync + optional; fired at each tier
159
+ // so the UI can paint "L0 βœ“ β†’ L1 βœ“ β†’ L2 0.91 β†’ stored" during a compaction.
160
+ const onTier = input.onTier;
153
161
  // Tracks whether a tier matched while in MARK_ONLY (record-but-don't-collapse),
154
162
  // and which tier.
155
163
  let markOnly: DedupTier | null = null;
@@ -161,6 +169,7 @@ export class VectorStore {
161
169
  // skips the scan; a hit is only a candidate, confirmed against `all` below.
162
170
  // Gated by L0_ENABLED (Sprint 14). MARK_ONLY_L0 records the decision but
163
171
  // does not collapse β€” the new region is still stored.
172
+ onTier?.({ tier: "L0", status: "scanning" });
164
173
  const digest = computeContentDigest(input.regionText);
165
174
  const bloom = openBloom(this.stateDir);
166
175
  if (cfg.L0_ENABLED && bloom.maybeHas(digest.contentHash)) {
@@ -180,6 +189,7 @@ export class VectorStore {
180
189
  addTokensSaved(origTokens, this.stateDir);
181
190
  const r = { checkpoint: contentMatch, deduped: true, reason: "contentHash" };
182
191
  this.record("L0", "deduped", "contentHash", Date.now() - t0);
192
+ onTier?.({ tier: "L0", status: "deduped", detail: "contentHash" });
183
193
  return r;
184
194
  }
185
195
  }
@@ -197,6 +207,7 @@ export class VectorStore {
197
207
  addTokensSaved(origTokens, this.stateDir);
198
208
  const r = { checkpoint: regionMatch, deduped: true, reason: "regionHash" };
199
209
  this.record("L0", "deduped", "regionHash", Date.now() - t0);
210
+ onTier?.({ tier: "L0", status: "deduped", detail: "regionHash" });
200
211
  return r;
201
212
  }
202
213
  }
@@ -220,15 +231,19 @@ export class VectorStore {
220
231
  addTokensSaved(origTokens, this.stateDir);
221
232
  const r = { checkpoint: summaryMatch, deduped: true, reason: "summaryHash" };
222
233
  this.record("L0", "deduped", "summaryHash", Date.now() - t0);
234
+ onTier?.({ tier: "L0", status: "deduped", detail: "summaryHash" });
223
235
  return r;
224
236
  }
225
237
  }
226
238
  }
239
+ // L0 did not collapse this region.
240
+ onTier?.({ tier: "L0", status: "passed" });
227
241
 
228
242
  // 2b. L1 MinHash/LSH near-duplicate dedup (Sprint 11) β€” catches one-word
229
243
  // edits / rewordings that L0's exact hash misses. Cheap LSH bucket
230
244
  // retrieval β†’ trigram verification (pg_trgm-equivalent) as the final gate.
231
245
  // Gated by L1_ENABLED (Sprint 14); MARK_ONLY_L1 records but doesn't collapse.
246
+ onTier?.({ tier: "L1", status: "scanning" });
232
247
  if (cfg.L1_ENABLED) {
233
248
  const l1 = this.findL1Duplicate(sessionId, input.regionText, all);
234
249
  if (l1 && !cfg.MARK_ONLY_L1) {
@@ -237,10 +252,12 @@ export class VectorStore {
237
252
  bumpDedupStats(true, this.stateDir);
238
253
  const r = { checkpoint: l1, deduped: true, reason: "l1MinHash" };
239
254
  this.record("L1", "deduped", "l1MinHash", Date.now() - t0);
255
+ onTier?.({ tier: "L1", status: "deduped", detail: "l1MinHash" });
240
256
  return r;
241
257
  }
242
258
  if (l1 && cfg.MARK_ONLY_L1) markOnly = "L1";
243
259
  }
260
+ onTier?.({ tier: "L1", status: "passed" });
244
261
 
245
262
  // 3. L2 semantic dedup β€” catches near-identical / semantically-similar regions
246
263
  // via cosine over the embedding. topicSummary is used for summaryHash dedup
@@ -252,6 +269,7 @@ export class VectorStore {
252
269
  const SIMILARITY_BUDGET_MS = cfg.SIMILARITY_BUDGET_MS;
253
270
  const simThreshold = this.l2Threshold; // from cfg.L2_COSINE (default 0.85 trigram)
254
271
  const embedding = this.embedder.embed(input.regionText);
272
+ onTier?.({ tier: "L2", status: "scanning" });
255
273
  if (cfg.L2_ENABLED && all.length > 0) {
256
274
  const start = Date.now();
257
275
  let timedOut = false;
@@ -274,10 +292,12 @@ export class VectorStore {
274
292
  addTokensSaved(origTokens, this.stateDir);
275
293
  const r = { checkpoint: nearest.checkpoint, deduped: true, reason: "contentSimilarity" };
276
294
  this.record("L2", "deduped", "contentSimilarity", Date.now() - t0);
295
+ onTier?.({ tier: "L2", status: "deduped", detail: nearest.sim.toFixed(2) });
277
296
  return r;
278
297
  }
279
298
  markOnly = "L2";
280
299
  }
300
+ onTier?.({ tier: "L2", status: "passed", detail: `best ${nearest.sim.toFixed(2)}` });
281
301
  }
282
302
 
283
303
  // 4. Genuinely new β€” create checkpoint
@@ -342,6 +362,7 @@ export class VectorStore {
342
362
  }
343
363
  // Cumulative store-wide dedup accounting (attempt, not collapsed).
344
364
  bumpDedupStats(false, this.stateDir);
365
+ onTier?.({ tier: "new", status: "stored" });
345
366
  return { checkpoint, deduped: false };
346
367
  }
347
368
 
@@ -585,4 +606,8 @@ export class VectorStore {
585
606
  repoStats(): ReturnType<typeof repoStatsFromStore> {
586
607
  return repoStatsFromStore(this.stateDir);
587
608
  }
609
+ /** Data-safety invariant (Phase 0): regions retained vs bytes permanently deleted. */
610
+ dataInvariant(): ReturnType<typeof dataInvariantStats> {
611
+ return dataInvariantStats(this.stateDir);
612
+ }
588
613
  }