pi-mega-compact 0.4.10 β 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/dashboard-server.js +28 -0
- package/dist/extensions/mega-compact.js +61 -2
- package/dist/src/engine.js +1 -0
- package/dist/src/store/phase01.test.js +81 -0
- package/dist/src/store/sqlite.js +20 -0
- package/dist/src/vectorStore.js +21 -1
- package/extensions/dashboard-server.ts +28 -0
- package/extensions/mega-compact.ts +64 -2
- package/package.json +1 -1
- package/src/engine.ts +17 -0
- package/src/store/phase01.test.ts +88 -0
- package/src/store/sqlite.ts +43 -0
- package/src/vectorStore.ts +25 -0
|
@@ -33,6 +33,7 @@ function readSnapshot(snapshotPath) {
|
|
|
33
33
|
store: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, injectedCount: 0, dedupHitRate: 0, storageDedupRate: 0, dedupCollapsed: 0 },
|
|
34
34
|
crew: { activeAgents: 0, currentTurn: 0 },
|
|
35
35
|
repo: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupAttempts: 0, dedupCollapsed: 0, storageDedupRate: 0 },
|
|
36
|
+
integrity: { regionsRetained: 0, compressedOriginalBytes: 0, duplicatesCollapsed: 0, bytesPermanentlyDeleted: 0 },
|
|
36
37
|
};
|
|
37
38
|
}
|
|
38
39
|
}
|
|
@@ -64,6 +65,10 @@ function dashboardHtml(tierName) {
|
|
|
64
65
|
h1 .tier { background: #1f6feb; color: #fff; font-size: 11px; font-weight: 700; padding: 2px 8px; border-radius: 10px; text-transform: uppercase; letter-spacing: .5px; }
|
|
65
66
|
.grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; margin-bottom: 20px; }
|
|
66
67
|
.card { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 16px; }
|
|
68
|
+
.card.safe { border-color: #238636; }
|
|
69
|
+
.card.safe h2 { color: #3fb950; }
|
|
70
|
+
.safe-note { font-size: 12px; color: #8b949e; margin: 12px 0 0; line-height: 1.5; }
|
|
71
|
+
.value.ok { color: #3fb950; }
|
|
67
72
|
.card h2 { font-size: 12px; text-transform: uppercase; letter-spacing: .5px; color: #8b949e; margin-bottom: 12px; font-weight: 600; }
|
|
68
73
|
.meter-track { background: #21262d; border-radius: 4px; height: 20px; overflow: hidden; margin: 8px 0; }
|
|
69
74
|
.meter-fill { height: 100%; border-radius: 4px; transition: width .6s ease; min-width: 2px; }
|
|
@@ -144,6 +149,16 @@ function dashboardHtml(tierName) {
|
|
|
144
149
|
<span class="label">Storage Dedup</span><span class="value" id="rp-sdedup">0%</span>
|
|
145
150
|
</div>
|
|
146
151
|
</div>
|
|
152
|
+
<div class="card safe">
|
|
153
|
+
<h2>π‘ Data Safety</h2>
|
|
154
|
+
<div class="stat-grid">
|
|
155
|
+
<span class="label">Regions Retained</span><span class="value" id="ig-retained">0</span>
|
|
156
|
+
<span class="label">Compressed-Original</span><span class="value" id="ig-bytes">0 B</span>
|
|
157
|
+
<span class="label">Dedup Duplicates</span><span class="value" id="ig-dupes">0</span>
|
|
158
|
+
<span class="label">Permanently Deleted</span><span class="value ok" id="ig-deleted">0 B</span>
|
|
159
|
+
</div>
|
|
160
|
+
<p class="safe-note">Every compacted region is kept verbatim (compressed). "Drop" = removed from the live window only. We never delete your data.</p>
|
|
161
|
+
</div>
|
|
147
162
|
<div class="card">
|
|
148
163
|
<h2>Configuration</h2>
|
|
149
164
|
<div class="conf-grid">
|
|
@@ -223,6 +238,19 @@ function dashboardHtml(tierName) {
|
|
|
223
238
|
var rsdr = repo.storageDedupRate || 0;
|
|
224
239
|
document.getElementById('rp-sdedup').textContent = (rsdr * 100 >= 10 ? Math.round(rsdr * 100) : (rsdr * 100).toFixed(1)) + '%';
|
|
225
240
|
|
|
241
|
+
// Data-safety invariant (Phase 0 β trust foundation).
|
|
242
|
+
var ig = d.integrity || { regionsRetained: 0, compressedOriginalBytes: 0, duplicatesCollapsed: 0, bytesPermanentlyDeleted: 0 };
|
|
243
|
+
function fmtBytes(b) {
|
|
244
|
+
b = b || 0;
|
|
245
|
+
if (b >= 1048576) return (b / 1048576).toFixed(1) + ' MiB';
|
|
246
|
+
if (b >= 1024) return (b / 1024).toFixed(1) + ' KiB';
|
|
247
|
+
return b + ' B';
|
|
248
|
+
}
|
|
249
|
+
document.getElementById('ig-retained').textContent = (ig.regionsRetained || 0).toLocaleString();
|
|
250
|
+
document.getElementById('ig-bytes').textContent = fmtBytes(ig.compressedOriginalBytes);
|
|
251
|
+
document.getElementById('ig-dupes').textContent = (ig.duplicatesCollapsed || 0).toLocaleString();
|
|
252
|
+
document.getElementById('ig-deleted').textContent = fmtBytes(ig.bytesPermanentlyDeleted);
|
|
253
|
+
|
|
226
254
|
// Crew / agents (live sub-agent activity + turn).
|
|
227
255
|
var crew = d.crew || { activeAgents: 0, currentTurn: 0 };
|
|
228
256
|
document.getElementById('cr-agents').textContent = crew.activeAgents || 0;
|
|
@@ -185,6 +185,7 @@ export default function (pi) {
|
|
|
185
185
|
bindRepo(ctx.cwd);
|
|
186
186
|
const st = store.stats(rt.sessionId);
|
|
187
187
|
const repo = store.repoStats();
|
|
188
|
+
const di = store.dataInvariant();
|
|
188
189
|
const armed = lastCtxPercent != null && lastCtxPercent >= config.fastGatePct;
|
|
189
190
|
const ready = armed && (lastCtxTokens ?? 0) >= config.thresholdTokens;
|
|
190
191
|
dashboard.snapshot({
|
|
@@ -223,6 +224,12 @@ export default function (pi) {
|
|
|
223
224
|
dedupCollapsed: repo.dedupCollapsed,
|
|
224
225
|
storageDedupRate: repo.storageDedupRate,
|
|
225
226
|
},
|
|
227
|
+
integrity: {
|
|
228
|
+
regionsRetained: di.regionsRetained,
|
|
229
|
+
compressedOriginalBytes: di.compressedOriginalBytes,
|
|
230
|
+
duplicatesCollapsed: di.duplicatesCollapsed,
|
|
231
|
+
bytesPermanentlyDeleted: di.bytesPermanentlyDeleted,
|
|
232
|
+
},
|
|
226
233
|
});
|
|
227
234
|
// Live stats widget above the editor
|
|
228
235
|
if (ctx) {
|
|
@@ -254,7 +261,16 @@ export default function (pi) {
|
|
|
254
261
|
];
|
|
255
262
|
// Live "now processing" line β teal while fresh (β€4s), then the last-seen
|
|
256
263
|
// action keeps the widget lively. Cleared on session reset.
|
|
257
|
-
if (
|
|
264
|
+
if (tierTrace) {
|
|
265
|
+
// Per-tier dedup progress is more relevant than the last action while a
|
|
266
|
+
// compaction is mid-flight, so prefer it when fresh.
|
|
267
|
+
const fresh = Date.now() - lastActivityAt < 4000;
|
|
268
|
+
if (fresh)
|
|
269
|
+
lines.push(` ${tierTrace}`);
|
|
270
|
+
else if (currentActivity)
|
|
271
|
+
lines.push(` ${C.dim}${currentActivity}${C.reset}`);
|
|
272
|
+
}
|
|
273
|
+
else if (currentActivity) {
|
|
258
274
|
const fresh = Date.now() - lastActivityAt < 4000;
|
|
259
275
|
lines.push(` ${fresh ? C.teal : C.dim}${currentActivity}${C.reset}`);
|
|
260
276
|
}
|
|
@@ -285,6 +301,10 @@ export default function (pi) {
|
|
|
285
301
|
// the widget is never blank. Cleared on session reset.
|
|
286
302
|
let currentActivity;
|
|
287
303
|
let lastActivityAt = 0;
|
|
304
|
+
// Live per-tier dedup trace (Phase 1): e.g. "L0 β β L1 β β L2 0.91 β stored".
|
|
305
|
+
// Built from the store's sync onTier callback during a compaction so the user
|
|
306
|
+
// watches each tier evaluate in real time. Cleared once the outcome settles.
|
|
307
|
+
let tierTrace;
|
|
288
308
|
// ANSI palette for the toolbar. The pi TUI's Text component preserves ANSI
|
|
289
309
|
// escape codes (see wrapTextWithAnsi), so raw escapes render as colors. No
|
|
290
310
|
// chalk dependency needed β these are just strings.
|
|
@@ -323,6 +343,34 @@ export default function (pi) {
|
|
|
323
343
|
currentTurn = 0;
|
|
324
344
|
currentActivity = undefined;
|
|
325
345
|
lastActivityAt = 0;
|
|
346
|
+
tierTrace = undefined;
|
|
347
|
+
}
|
|
348
|
+
/** Build the sync onTier callback that paints the live per-tier trace. */
|
|
349
|
+
function makeTierCallback(ctx) {
|
|
350
|
+
const order = ["L0", "L1", "L2", "new"];
|
|
351
|
+
const seen = new Map();
|
|
352
|
+
const glyph = (status) => status === "deduped" ? `${C.green}β${C.reset}` :
|
|
353
|
+
status === "passed" ? `${C.dim}β${C.reset}` :
|
|
354
|
+
status === "scanning" ? `${C.amber}β¦${C.reset}` :
|
|
355
|
+
`${C.cyan}β${C.reset}`;
|
|
356
|
+
return (ev) => {
|
|
357
|
+
const label = ev.tier === "new"
|
|
358
|
+
? `${C.cyan}stored${C.reset}`
|
|
359
|
+
: `${ev.tier} ${glyph(ev.status)}` +
|
|
360
|
+
(ev.detail ? ` ${C.gray}(${ev.detail})${C.reset}` : "");
|
|
361
|
+
// Show the most recent outcome per tier (collapses re-fires).
|
|
362
|
+
seen.set(ev.tier, label);
|
|
363
|
+
const show = [];
|
|
364
|
+
for (const t of order)
|
|
365
|
+
if (seen.has(t))
|
|
366
|
+
show.push(seen.get(t));
|
|
367
|
+
tierTrace = `${C.teal}β${C.reset} ${show.join(` ${C.gray}β${C.reset} `)}`;
|
|
368
|
+
lastActivityAt = Date.now();
|
|
369
|
+
try {
|
|
370
|
+
snapshot(ctx);
|
|
371
|
+
}
|
|
372
|
+
catch { /* non-fatal */ }
|
|
373
|
+
};
|
|
326
374
|
}
|
|
327
375
|
/** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
|
|
328
376
|
function runCompact(ctx, messages, opts = {}) {
|
|
@@ -340,6 +388,7 @@ export default function (pi) {
|
|
|
340
388
|
keepFrom,
|
|
341
389
|
summary: opts.summary,
|
|
342
390
|
timestamp: Date.now(),
|
|
391
|
+
onTier: makeTierCallback(ctx),
|
|
343
392
|
}, store);
|
|
344
393
|
if (result.skipped)
|
|
345
394
|
return { skipped: true };
|
|
@@ -371,6 +420,9 @@ export default function (pi) {
|
|
|
371
420
|
? `β» deduped ${fileLabel}`
|
|
372
421
|
: `π compacted ${result.checkpointId} Β· ${fileLabel}`;
|
|
373
422
|
lastActivityAt = Date.now();
|
|
423
|
+
// The per-tier trace has settled into the final outcome β fold it back into
|
|
424
|
+
// the activity line and stop showing the live trace.
|
|
425
|
+
tierTrace = undefined;
|
|
374
426
|
// Record session activity + a daily-log entry in the per-repo SQLite store
|
|
375
427
|
// (foundation for resume-sessions / daily-log features). Best-effort β never
|
|
376
428
|
// block a compaction on bookkeeping.
|
|
@@ -608,7 +660,7 @@ export default function (pi) {
|
|
|
608
660
|
},
|
|
609
661
|
});
|
|
610
662
|
pi.registerCommand("mega-status", {
|
|
611
|
-
description: "Show mega-compact config and
|
|
663
|
+
description: "Show mega-compact config, context usage, and the data-safety invariant.",
|
|
612
664
|
handler: async (_args, ctx) => {
|
|
613
665
|
bindRepo(ctx.cwd);
|
|
614
666
|
const usage = ctx.getContextUsage();
|
|
@@ -616,6 +668,9 @@ export default function (pi) {
|
|
|
616
668
|
const tokens = usage?.tokens != null ? `${usage.tokens} tok` : "n/a";
|
|
617
669
|
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
618
670
|
const st = store.stats(sid);
|
|
671
|
+
const di = store.dataInvariant();
|
|
672
|
+
const fmtB = (b) => b >= 1_048_576 ? `${(b / 1_048_576).toFixed(1)} MiB` :
|
|
673
|
+
b >= 1024 ? `${(b / 1024).toFixed(1)} KiB` : `${b} B`;
|
|
619
674
|
ctx.ui.notify(`[mega-compact] pct=${pct} tokens=${tokens} tier=${config.tier} fastGate=${config.fastGatePct}% ` +
|
|
620
675
|
`threshold=${config.thresholdTokens} auto=${config.auto} autoInline=${config.autoInline}\n` +
|
|
621
676
|
`[mega-compact] store: ${st.checkpointCount} chkpt Β· ` +
|
|
@@ -623,6 +678,10 @@ export default function (pi) {
|
|
|
623
678
|
`injected=${st.injectedCount} Β· dedup=${(st.dedupHitRate * 100).toFixed(0)}%\n` +
|
|
624
679
|
`[mega-compact] anchor=${config.anchorUserMessages} preserveRecent=${config.preserveRecent} ` +
|
|
625
680
|
`autoInlineK=${config.autoInlineK} dedupSim=${config.dedupSim} debug=${config.debug}\n` +
|
|
681
|
+
`[mega-compact] π‘ data-safe: ${di.regionsRetained} regions retained ` +
|
|
682
|
+
`(${fmtB(di.compressedOriginalBytes)} compressed-original) Β· ` +
|
|
683
|
+
`${di.duplicatesCollapsed} dedup-duplicates collapsed Β· ` +
|
|
684
|
+
`${C.green}0 bytes permanently deleted${C.reset}\n` +
|
|
626
685
|
`[mega-compact] stateDir=${currentStateDir}`);
|
|
627
686
|
},
|
|
628
687
|
});
|
package/dist/src/engine.js
CHANGED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { VectorStore } from "../vectorStore.js";
|
|
7
|
+
import { dataInvariantStats } from "./sqlite.js";
|
|
8
|
+
const baseTmp = mkdtempSync(join(tmpdir(), "mc-p01-"));
|
|
9
|
+
let counter = 0;
|
|
10
|
+
function store(opts = {}) {
|
|
11
|
+
const dir = join(baseTmp, `run-${counter++}`);
|
|
12
|
+
return { s: new VectorStore({ dedupSim: opts.dedupSim ?? 0.9, stateDir: dir }), dir };
|
|
13
|
+
}
|
|
14
|
+
// --- Phase 0: data-safety invariant ---------------------------------------
|
|
15
|
+
test("Phase 0: every added region retains a compressed-original and deletes nothing", () => {
|
|
16
|
+
const { s, dir } = store();
|
|
17
|
+
s.add({ sessionId: "sess_a", summary: "s1", regionText: "the quick brown fox jumps", tokenEstimate: 5, originalTokenEstimate: 50, timestamp: 1 });
|
|
18
|
+
s.add({ sessionId: "sess_a", summary: "s2", regionText: "a totally different region of work", tokenEstimate: 6, originalTokenEstimate: 60, timestamp: 2 });
|
|
19
|
+
const di = dataInvariantStats(dir);
|
|
20
|
+
assert.equal(di.regionsRetained, 2, "both regions retained");
|
|
21
|
+
assert.ok(di.compressedOriginalBytes > 0, "compressed-original bytes retained");
|
|
22
|
+
assert.equal(di.bytesPermanentlyDeleted, 0, "INVARIANT: nothing permanently deleted");
|
|
23
|
+
});
|
|
24
|
+
test("Phase 0: dedup collapses a duplicate but original is still retained (not deleted)", () => {
|
|
25
|
+
const { s, dir } = store();
|
|
26
|
+
const text = "identical region content that will dedup on the second add";
|
|
27
|
+
s.add({ sessionId: "sess_b", summary: "s", regionText: text, tokenEstimate: 5, originalTokenEstimate: 50, timestamp: 1 });
|
|
28
|
+
const r2 = s.add({ sessionId: "sess_b", summary: "s", regionText: text, tokenEstimate: 5, originalTokenEstimate: 50, timestamp: 2 });
|
|
29
|
+
assert.equal(r2.deduped, true, "second add deduped");
|
|
30
|
+
const di = dataInvariantStats(dir);
|
|
31
|
+
assert.equal(di.bytesPermanentlyDeleted, 0, "INVARIANT: dedup never deletes data");
|
|
32
|
+
assert.ok(di.regionsRetained >= 1, "survivor region still retained");
|
|
33
|
+
});
|
|
34
|
+
// --- Phase 1: per-tier progress callback ----------------------------------
|
|
35
|
+
test("Phase 1: onTier fires L0βL1βL2βstored for a genuinely new region", () => {
|
|
36
|
+
const { s } = store();
|
|
37
|
+
// Seed one checkpoint so L2 has something to scan against.
|
|
38
|
+
s.add({ sessionId: "sess_c", summary: "seed", regionText: "seed region alpha", timestamp: 1 });
|
|
39
|
+
const events = [];
|
|
40
|
+
s.add({
|
|
41
|
+
sessionId: "sess_c",
|
|
42
|
+
summary: "new",
|
|
43
|
+
regionText: "a brand new region beta that is not a duplicate",
|
|
44
|
+
timestamp: 2,
|
|
45
|
+
onTier: (ev) => events.push({ tier: ev.tier, status: ev.status }),
|
|
46
|
+
});
|
|
47
|
+
const tiers = events.map((e) => e.tier);
|
|
48
|
+
assert.ok(tiers.includes("L0"), "L0 fired");
|
|
49
|
+
assert.ok(tiers.includes("L1"), "L1 fired");
|
|
50
|
+
assert.ok(tiers.includes("L2"), "L2 fired");
|
|
51
|
+
assert.equal(events.at(-1)?.tier, "new", "final event is the 'new' tier");
|
|
52
|
+
assert.equal(events.at(-1)?.status, "stored", "final status is stored");
|
|
53
|
+
});
|
|
54
|
+
test("Phase 1: onTier reports a deduped outcome and short-circuits at the matching tier", () => {
|
|
55
|
+
const { s } = store();
|
|
56
|
+
const text = "region that will exact-dedup on the second pass";
|
|
57
|
+
s.add({ sessionId: "sess_d", summary: "s", regionText: text, timestamp: 1 });
|
|
58
|
+
const events = [];
|
|
59
|
+
const r = s.add({
|
|
60
|
+
sessionId: "sess_d",
|
|
61
|
+
summary: "s",
|
|
62
|
+
regionText: text,
|
|
63
|
+
timestamp: 2,
|
|
64
|
+
onTier: (ev) => events.push(ev),
|
|
65
|
+
});
|
|
66
|
+
assert.equal(r.deduped, true);
|
|
67
|
+
const deduped = events.find((e) => e.status === "deduped");
|
|
68
|
+
assert.ok(deduped, "a deduped event fired");
|
|
69
|
+
assert.equal(deduped?.tier, "L0", "exact duplicate short-circuits at L0");
|
|
70
|
+
// Must NOT reach the final stored event.
|
|
71
|
+
assert.ok(!events.some((e) => e.status === "stored"), "no stored event on dedup");
|
|
72
|
+
});
|
|
73
|
+
test("Phase 1: onTier is optional (back-compat) β add() works without it", () => {
|
|
74
|
+
const { s } = store();
|
|
75
|
+
const r = s.add({ sessionId: "sess_e", summary: "s", regionText: "no callback here", timestamp: 1 });
|
|
76
|
+
assert.equal(r.deduped, false);
|
|
77
|
+
});
|
|
78
|
+
process.on("exit", () => { try {
|
|
79
|
+
rmSync(baseTmp, { recursive: true, force: true });
|
|
80
|
+
}
|
|
81
|
+
catch { /* ignore */ } });
|
package/dist/src/store/sqlite.js
CHANGED
|
@@ -495,6 +495,26 @@ export function storeStats(sessionId, stateDir = getStateDir()) {
|
|
|
495
495
|
lastSummary,
|
|
496
496
|
};
|
|
497
497
|
}
|
|
498
|
+
export function dataInvariantStats(stateDir = getStateDir()) {
|
|
499
|
+
const db = openStore(stateDir);
|
|
500
|
+
const row = db
|
|
501
|
+
.prepare(`SELECT
|
|
502
|
+
COUNT(compressed_original) AS withBlob,
|
|
503
|
+
COALESCE(SUM(LENGTH(compressed_original)),0) AS blobBytes,
|
|
504
|
+
SUM(CASE WHEN compressed_original IS NULL THEN 1 ELSE 0 END) AS noBlob
|
|
505
|
+
FROM context_chunks WHERE dedup_status != 'removed'`)
|
|
506
|
+
.get();
|
|
507
|
+
const removed = db
|
|
508
|
+
.prepare(`SELECT COUNT(*) AS c FROM context_chunks WHERE dedup_status = 'removed'`)
|
|
509
|
+
.get();
|
|
510
|
+
return {
|
|
511
|
+
regionsRetained: row.withBlob,
|
|
512
|
+
compressedOriginalBytes: row.blobBytes,
|
|
513
|
+
regionsWithoutBlob: row.noBlob ?? 0,
|
|
514
|
+
bytesPermanentlyDeleted: 0,
|
|
515
|
+
duplicatesCollapsed: removed.c,
|
|
516
|
+
};
|
|
517
|
+
}
|
|
498
518
|
export function repoStats(stateDir = getStateDir()) {
|
|
499
519
|
const db = openStore(stateDir);
|
|
500
520
|
const row = db
|
package/dist/src/vectorStore.js
CHANGED
|
@@ -19,7 +19,7 @@ import { isNearDuplicate } from "./dedup/l1-verify.js";
|
|
|
19
19
|
import { mmrRerank } from "./dedup/mmr.js";
|
|
20
20
|
import { topK } from "./dedup/topk.js";
|
|
21
21
|
import { openBloom, saveBloom } from "./store/bloom.js";
|
|
22
|
-
import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, setDedupStatus, addTokensSaved, getDedupStats, bumpDedupStats, repoStats as repoStatsFromStore, } from "./store/sqlite.js";
|
|
22
|
+
import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, setDedupStatus, addTokensSaved, getDedupStats, bumpDedupStats, repoStats as repoStatsFromStore, dataInvariantStats, } from "./store/sqlite.js";
|
|
23
23
|
import { migrateJsonToSqlite } from "./store/migrate.js";
|
|
24
24
|
/** Default L2 semantic-dedup enable flag (trigram embedder is local, zero-network). */
|
|
25
25
|
export const L2_ENABLED = true;
|
|
@@ -85,6 +85,9 @@ export class VectorStore {
|
|
|
85
85
|
// we persist (orig β stored). Falls back to stored when orig is unknown.
|
|
86
86
|
const origTokens = input.originalTokenEstimate ?? input.tokenEstimate ?? 0;
|
|
87
87
|
const cfg = this.cfg;
|
|
88
|
+
// Live per-tier progress hook (Phase 1). Sync + optional; fired at each tier
|
|
89
|
+
// so the UI can paint "L0 β β L1 β β L2 0.91 β stored" during a compaction.
|
|
90
|
+
const onTier = input.onTier;
|
|
88
91
|
// Tracks whether a tier matched while in MARK_ONLY (record-but-don't-collapse),
|
|
89
92
|
// and which tier.
|
|
90
93
|
let markOnly = null;
|
|
@@ -95,6 +98,7 @@ export class VectorStore {
|
|
|
95
98
|
// skips the scan; a hit is only a candidate, confirmed against `all` below.
|
|
96
99
|
// Gated by L0_ENABLED (Sprint 14). MARK_ONLY_L0 records the decision but
|
|
97
100
|
// does not collapse β the new region is still stored.
|
|
101
|
+
onTier?.({ tier: "L0", status: "scanning" });
|
|
98
102
|
const digest = computeContentDigest(input.regionText);
|
|
99
103
|
const bloom = openBloom(this.stateDir);
|
|
100
104
|
if (cfg.L0_ENABLED && bloom.maybeHas(digest.contentHash)) {
|
|
@@ -112,6 +116,7 @@ export class VectorStore {
|
|
|
112
116
|
addTokensSaved(origTokens, this.stateDir);
|
|
113
117
|
const r = { checkpoint: contentMatch, deduped: true, reason: "contentHash" };
|
|
114
118
|
this.record("L0", "deduped", "contentHash", Date.now() - t0);
|
|
119
|
+
onTier?.({ tier: "L0", status: "deduped", detail: "contentHash" });
|
|
115
120
|
return r;
|
|
116
121
|
}
|
|
117
122
|
}
|
|
@@ -129,6 +134,7 @@ export class VectorStore {
|
|
|
129
134
|
addTokensSaved(origTokens, this.stateDir);
|
|
130
135
|
const r = { checkpoint: regionMatch, deduped: true, reason: "regionHash" };
|
|
131
136
|
this.record("L0", "deduped", "regionHash", Date.now() - t0);
|
|
137
|
+
onTier?.({ tier: "L0", status: "deduped", detail: "regionHash" });
|
|
132
138
|
return r;
|
|
133
139
|
}
|
|
134
140
|
}
|
|
@@ -152,14 +158,18 @@ export class VectorStore {
|
|
|
152
158
|
addTokensSaved(origTokens, this.stateDir);
|
|
153
159
|
const r = { checkpoint: summaryMatch, deduped: true, reason: "summaryHash" };
|
|
154
160
|
this.record("L0", "deduped", "summaryHash", Date.now() - t0);
|
|
161
|
+
onTier?.({ tier: "L0", status: "deduped", detail: "summaryHash" });
|
|
155
162
|
return r;
|
|
156
163
|
}
|
|
157
164
|
}
|
|
158
165
|
}
|
|
166
|
+
// L0 did not collapse this region.
|
|
167
|
+
onTier?.({ tier: "L0", status: "passed" });
|
|
159
168
|
// 2b. L1 MinHash/LSH near-duplicate dedup (Sprint 11) β catches one-word
|
|
160
169
|
// edits / rewordings that L0's exact hash misses. Cheap LSH bucket
|
|
161
170
|
// retrieval β trigram verification (pg_trgm-equivalent) as the final gate.
|
|
162
171
|
// Gated by L1_ENABLED (Sprint 14); MARK_ONLY_L1 records but doesn't collapse.
|
|
172
|
+
onTier?.({ tier: "L1", status: "scanning" });
|
|
163
173
|
if (cfg.L1_ENABLED) {
|
|
164
174
|
const l1 = this.findL1Duplicate(sessionId, input.regionText, all);
|
|
165
175
|
if (l1 && !cfg.MARK_ONLY_L1) {
|
|
@@ -168,11 +178,13 @@ export class VectorStore {
|
|
|
168
178
|
bumpDedupStats(true, this.stateDir);
|
|
169
179
|
const r = { checkpoint: l1, deduped: true, reason: "l1MinHash" };
|
|
170
180
|
this.record("L1", "deduped", "l1MinHash", Date.now() - t0);
|
|
181
|
+
onTier?.({ tier: "L1", status: "deduped", detail: "l1MinHash" });
|
|
171
182
|
return r;
|
|
172
183
|
}
|
|
173
184
|
if (l1 && cfg.MARK_ONLY_L1)
|
|
174
185
|
markOnly = "L1";
|
|
175
186
|
}
|
|
187
|
+
onTier?.({ tier: "L1", status: "passed" });
|
|
176
188
|
// 3. L2 semantic dedup β catches near-identical / semantically-similar regions
|
|
177
189
|
// via cosine over the embedding. topicSummary is used for summaryHash dedup
|
|
178
190
|
// (tier 2); the vector index is keyed on the original region for backward-
|
|
@@ -183,6 +195,7 @@ export class VectorStore {
|
|
|
183
195
|
const SIMILARITY_BUDGET_MS = cfg.SIMILARITY_BUDGET_MS;
|
|
184
196
|
const simThreshold = this.l2Threshold; // from cfg.L2_COSINE (default 0.85 trigram)
|
|
185
197
|
const embedding = this.embedder.embed(input.regionText);
|
|
198
|
+
onTier?.({ tier: "L2", status: "scanning" });
|
|
186
199
|
if (cfg.L2_ENABLED && all.length > 0) {
|
|
187
200
|
const start = Date.now();
|
|
188
201
|
let timedOut = false;
|
|
@@ -204,10 +217,12 @@ export class VectorStore {
|
|
|
204
217
|
addTokensSaved(origTokens, this.stateDir);
|
|
205
218
|
const r = { checkpoint: nearest.checkpoint, deduped: true, reason: "contentSimilarity" };
|
|
206
219
|
this.record("L2", "deduped", "contentSimilarity", Date.now() - t0);
|
|
220
|
+
onTier?.({ tier: "L2", status: "deduped", detail: nearest.sim.toFixed(2) });
|
|
207
221
|
return r;
|
|
208
222
|
}
|
|
209
223
|
markOnly = "L2";
|
|
210
224
|
}
|
|
225
|
+
onTier?.({ tier: "L2", status: "passed", detail: `best ${nearest.sim.toFixed(2)}` });
|
|
211
226
|
}
|
|
212
227
|
// 4. Genuinely new β create checkpoint
|
|
213
228
|
const checkpointId = nextCheckpointId(sessionId, this.stateDir);
|
|
@@ -265,6 +280,7 @@ export class VectorStore {
|
|
|
265
280
|
}
|
|
266
281
|
// Cumulative store-wide dedup accounting (attempt, not collapsed).
|
|
267
282
|
bumpDedupStats(false, this.stateDir);
|
|
283
|
+
onTier?.({ tier: "new", status: "stored" });
|
|
268
284
|
return { checkpoint, deduped: false };
|
|
269
285
|
}
|
|
270
286
|
/**
|
|
@@ -462,4 +478,8 @@ export class VectorStore {
|
|
|
462
478
|
repoStats() {
|
|
463
479
|
return repoStatsFromStore(this.stateDir);
|
|
464
480
|
}
|
|
481
|
+
/** Data-safety invariant (Phase 0): regions retained vs bytes permanently deleted. */
|
|
482
|
+
dataInvariant() {
|
|
483
|
+
return dataInvariantStats(this.stateDir);
|
|
484
|
+
}
|
|
465
485
|
}
|
|
@@ -96,6 +96,7 @@ function readSnapshot(snapshotPath: string) {
|
|
|
96
96
|
store: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, injectedCount: 0, dedupHitRate: 0, storageDedupRate: 0, dedupCollapsed: 0 },
|
|
97
97
|
crew: { activeAgents: 0, currentTurn: 0 },
|
|
98
98
|
repo: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupAttempts: 0, dedupCollapsed: 0, storageDedupRate: 0 },
|
|
99
|
+
integrity: { regionsRetained: 0, compressedOriginalBytes: 0, duplicatesCollapsed: 0, bytesPermanentlyDeleted: 0 },
|
|
99
100
|
} as Snapshot;
|
|
100
101
|
}
|
|
101
102
|
}
|
|
@@ -128,6 +129,10 @@ function dashboardHtml(tierName: string): string {
|
|
|
128
129
|
h1 .tier { background: #1f6feb; color: #fff; font-size: 11px; font-weight: 700; padding: 2px 8px; border-radius: 10px; text-transform: uppercase; letter-spacing: .5px; }
|
|
129
130
|
.grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; margin-bottom: 20px; }
|
|
130
131
|
.card { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 16px; }
|
|
132
|
+
.card.safe { border-color: #238636; }
|
|
133
|
+
.card.safe h2 { color: #3fb950; }
|
|
134
|
+
.safe-note { font-size: 12px; color: #8b949e; margin: 12px 0 0; line-height: 1.5; }
|
|
135
|
+
.value.ok { color: #3fb950; }
|
|
131
136
|
.card h2 { font-size: 12px; text-transform: uppercase; letter-spacing: .5px; color: #8b949e; margin-bottom: 12px; font-weight: 600; }
|
|
132
137
|
.meter-track { background: #21262d; border-radius: 4px; height: 20px; overflow: hidden; margin: 8px 0; }
|
|
133
138
|
.meter-fill { height: 100%; border-radius: 4px; transition: width .6s ease; min-width: 2px; }
|
|
@@ -208,6 +213,16 @@ function dashboardHtml(tierName: string): string {
|
|
|
208
213
|
<span class="label">Storage Dedup</span><span class="value" id="rp-sdedup">0%</span>
|
|
209
214
|
</div>
|
|
210
215
|
</div>
|
|
216
|
+
<div class="card safe">
|
|
217
|
+
<h2>π‘ Data Safety</h2>
|
|
218
|
+
<div class="stat-grid">
|
|
219
|
+
<span class="label">Regions Retained</span><span class="value" id="ig-retained">0</span>
|
|
220
|
+
<span class="label">Compressed-Original</span><span class="value" id="ig-bytes">0 B</span>
|
|
221
|
+
<span class="label">Dedup Duplicates</span><span class="value" id="ig-dupes">0</span>
|
|
222
|
+
<span class="label">Permanently Deleted</span><span class="value ok" id="ig-deleted">0 B</span>
|
|
223
|
+
</div>
|
|
224
|
+
<p class="safe-note">Every compacted region is kept verbatim (compressed). "Drop" = removed from the live window only. We never delete your data.</p>
|
|
225
|
+
</div>
|
|
211
226
|
<div class="card">
|
|
212
227
|
<h2>Configuration</h2>
|
|
213
228
|
<div class="conf-grid">
|
|
@@ -287,6 +302,19 @@ function dashboardHtml(tierName: string): string {
|
|
|
287
302
|
var rsdr = repo.storageDedupRate || 0;
|
|
288
303
|
document.getElementById('rp-sdedup').textContent = (rsdr * 100 >= 10 ? Math.round(rsdr * 100) : (rsdr * 100).toFixed(1)) + '%';
|
|
289
304
|
|
|
305
|
+
// Data-safety invariant (Phase 0 β trust foundation).
|
|
306
|
+
var ig = d.integrity || { regionsRetained: 0, compressedOriginalBytes: 0, duplicatesCollapsed: 0, bytesPermanentlyDeleted: 0 };
|
|
307
|
+
function fmtBytes(b) {
|
|
308
|
+
b = b || 0;
|
|
309
|
+
if (b >= 1048576) return (b / 1048576).toFixed(1) + ' MiB';
|
|
310
|
+
if (b >= 1024) return (b / 1024).toFixed(1) + ' KiB';
|
|
311
|
+
return b + ' B';
|
|
312
|
+
}
|
|
313
|
+
document.getElementById('ig-retained').textContent = (ig.regionsRetained || 0).toLocaleString();
|
|
314
|
+
document.getElementById('ig-bytes').textContent = fmtBytes(ig.compressedOriginalBytes);
|
|
315
|
+
document.getElementById('ig-dupes').textContent = (ig.duplicatesCollapsed || 0).toLocaleString();
|
|
316
|
+
document.getElementById('ig-deleted').textContent = fmtBytes(ig.bytesPermanentlyDeleted);
|
|
317
|
+
|
|
290
318
|
// Crew / agents (live sub-agent activity + turn).
|
|
291
319
|
var crew = d.crew || { activeAgents: 0, currentTurn: 0 };
|
|
292
320
|
document.getElementById('cr-agents').textContent = crew.activeAgents || 0;
|
|
@@ -214,6 +214,13 @@ interface DashboardSnapshot {
|
|
|
214
214
|
dedupCollapsed: number; // cumulative deduped collapses (store-wide)
|
|
215
215
|
storageDedupRate: number; // deduped / attempts, 0..1
|
|
216
216
|
};
|
|
217
|
+
/** Phase 0 data-safety invariant (trust foundation). */
|
|
218
|
+
integrity: {
|
|
219
|
+
regionsRetained: number; // checkpoints with a recoverable compressed-original
|
|
220
|
+
compressedOriginalBytes: number; // bytes of compressed-original retained (recoverable)
|
|
221
|
+
duplicatesCollapsed: number; // dedup duplicates (original kept on survivor)
|
|
222
|
+
bytesPermanentlyDeleted: number; // ALWAYS 0 β the invariant
|
|
223
|
+
};
|
|
217
224
|
}
|
|
218
225
|
|
|
219
226
|
class Dashboard {
|
|
@@ -280,6 +287,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
280
287
|
if (ctx) bindRepo(ctx.cwd);
|
|
281
288
|
const st = store.stats(rt.sessionId);
|
|
282
289
|
const repo = store.repoStats();
|
|
290
|
+
const di = store.dataInvariant();
|
|
283
291
|
const armed = lastCtxPercent != null && lastCtxPercent >= config.fastGatePct;
|
|
284
292
|
const ready = armed && (lastCtxTokens ?? 0) >= config.thresholdTokens;
|
|
285
293
|
dashboard.snapshot({
|
|
@@ -318,6 +326,12 @@ export default function (pi: ExtensionAPI) {
|
|
|
318
326
|
dedupCollapsed: repo.dedupCollapsed,
|
|
319
327
|
storageDedupRate: repo.storageDedupRate,
|
|
320
328
|
},
|
|
329
|
+
integrity: {
|
|
330
|
+
regionsRetained: di.regionsRetained,
|
|
331
|
+
compressedOriginalBytes: di.compressedOriginalBytes,
|
|
332
|
+
duplicatesCollapsed: di.duplicatesCollapsed,
|
|
333
|
+
bytesPermanentlyDeleted: di.bytesPermanentlyDeleted,
|
|
334
|
+
},
|
|
321
335
|
});
|
|
322
336
|
|
|
323
337
|
// Live stats widget above the editor
|
|
@@ -350,7 +364,13 @@ export default function (pi: ExtensionAPI) {
|
|
|
350
364
|
];
|
|
351
365
|
// Live "now processing" line β teal while fresh (β€4s), then the last-seen
|
|
352
366
|
// action keeps the widget lively. Cleared on session reset.
|
|
353
|
-
if (
|
|
367
|
+
if (tierTrace) {
|
|
368
|
+
// Per-tier dedup progress is more relevant than the last action while a
|
|
369
|
+
// compaction is mid-flight, so prefer it when fresh.
|
|
370
|
+
const fresh = Date.now() - lastActivityAt < 4000;
|
|
371
|
+
if (fresh) lines.push(` ${tierTrace}`);
|
|
372
|
+
else if (currentActivity) lines.push(` ${C.dim}${currentActivity}${C.reset}`);
|
|
373
|
+
} else if (currentActivity) {
|
|
354
374
|
const fresh = Date.now() - lastActivityAt < 4000;
|
|
355
375
|
lines.push(` ${fresh ? C.teal : C.dim}${currentActivity}${C.reset}`);
|
|
356
376
|
}
|
|
@@ -382,6 +402,10 @@ export default function (pi: ExtensionAPI) {
|
|
|
382
402
|
// the widget is never blank. Cleared on session reset.
|
|
383
403
|
let currentActivity: string | undefined;
|
|
384
404
|
let lastActivityAt = 0;
|
|
405
|
+
// Live per-tier dedup trace (Phase 1): e.g. "L0 β β L1 β β L2 0.91 β stored".
|
|
406
|
+
// Built from the store's sync onTier callback during a compaction so the user
|
|
407
|
+
// watches each tier evaluate in real time. Cleared once the outcome settles.
|
|
408
|
+
let tierTrace: string | undefined;
|
|
385
409
|
// ANSI palette for the toolbar. The pi TUI's Text component preserves ANSI
|
|
386
410
|
// escape codes (see wrapTextWithAnsi), so raw escapes render as colors. No
|
|
387
411
|
// chalk dependency needed β these are just strings.
|
|
@@ -421,6 +445,32 @@ export default function (pi: ExtensionAPI) {
|
|
|
421
445
|
currentTurn = 0;
|
|
422
446
|
currentActivity = undefined;
|
|
423
447
|
lastActivityAt = 0;
|
|
448
|
+
tierTrace = undefined;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/** Build the sync onTier callback that paints the live per-tier trace. */
|
|
452
|
+
function makeTierCallback(ctx: ExtensionContext): (ev: { tier: "L0" | "L1" | "L2" | "new"; status: "scanning" | "deduped" | "passed" | "stored"; detail?: string }) => void {
|
|
453
|
+
const order: Array<"L0" | "L1" | "L2" | "new"> = ["L0", "L1", "L2", "new"];
|
|
454
|
+
const seen = new Map<string, string>();
|
|
455
|
+
const glyph = (status: string) =>
|
|
456
|
+
status === "deduped" ? `${C.green}β${C.reset}` :
|
|
457
|
+
status === "passed" ? `${C.dim}β${C.reset}` :
|
|
458
|
+
status === "scanning" ? `${C.amber}β¦${C.reset}` :
|
|
459
|
+
`${C.cyan}β${C.reset}`;
|
|
460
|
+
return (ev) => {
|
|
461
|
+
const label =
|
|
462
|
+
ev.tier === "new"
|
|
463
|
+
? `${C.cyan}stored${C.reset}`
|
|
464
|
+
: `${ev.tier} ${glyph(ev.status)}` +
|
|
465
|
+
(ev.detail ? ` ${C.gray}(${ev.detail})${C.reset}` : "");
|
|
466
|
+
// Show the most recent outcome per tier (collapses re-fires).
|
|
467
|
+
seen.set(ev.tier, label);
|
|
468
|
+
const show: string[] = [];
|
|
469
|
+
for (const t of order) if (seen.has(t)) show.push(seen.get(t)!);
|
|
470
|
+
tierTrace = `${C.teal}β${C.reset} ${show.join(` ${C.gray}β${C.reset} `)}`;
|
|
471
|
+
lastActivityAt = Date.now();
|
|
472
|
+
try { snapshot(ctx); } catch { /* non-fatal */ }
|
|
473
|
+
};
|
|
424
474
|
}
|
|
425
475
|
|
|
426
476
|
/** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
|
|
@@ -445,6 +495,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
445
495
|
keepFrom,
|
|
446
496
|
summary: opts.summary,
|
|
447
497
|
timestamp: Date.now(),
|
|
498
|
+
onTier: makeTierCallback(ctx),
|
|
448
499
|
},
|
|
449
500
|
store,
|
|
450
501
|
);
|
|
@@ -478,6 +529,9 @@ export default function (pi: ExtensionAPI) {
|
|
|
478
529
|
? `β» deduped ${fileLabel}`
|
|
479
530
|
: `π compacted ${result.checkpointId} Β· ${fileLabel}`;
|
|
480
531
|
lastActivityAt = Date.now();
|
|
532
|
+
// The per-tier trace has settled into the final outcome β fold it back into
|
|
533
|
+
// the activity line and stop showing the live trace.
|
|
534
|
+
tierTrace = undefined;
|
|
481
535
|
|
|
482
536
|
// Record session activity + a daily-log entry in the per-repo SQLite store
|
|
483
537
|
// (foundation for resume-sessions / daily-log features). Best-effort β never
|
|
@@ -740,7 +794,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
740
794
|
});
|
|
741
795
|
|
|
742
796
|
pi.registerCommand("mega-status", {
|
|
743
|
-
description: "Show mega-compact config and
|
|
797
|
+
description: "Show mega-compact config, context usage, and the data-safety invariant.",
|
|
744
798
|
handler: async (_args: string, ctx: ExtensionContext) => {
|
|
745
799
|
bindRepo(ctx.cwd);
|
|
746
800
|
const usage = ctx.getContextUsage();
|
|
@@ -748,6 +802,10 @@ export default function (pi: ExtensionAPI) {
|
|
|
748
802
|
const tokens = usage?.tokens != null ? `${usage.tokens} tok` : "n/a";
|
|
749
803
|
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
750
804
|
const st = store.stats(sid);
|
|
805
|
+
const di = store.dataInvariant();
|
|
806
|
+
const fmtB = (b: number) =>
|
|
807
|
+
b >= 1_048_576 ? `${(b / 1_048_576).toFixed(1)} MiB` :
|
|
808
|
+
b >= 1024 ? `${(b / 1024).toFixed(1)} KiB` : `${b} B`;
|
|
751
809
|
ctx.ui.notify(
|
|
752
810
|
`[mega-compact] pct=${pct} tokens=${tokens} tier=${config.tier} fastGate=${config.fastGatePct}% ` +
|
|
753
811
|
`threshold=${config.thresholdTokens} auto=${config.auto} autoInline=${config.autoInline}\n` +
|
|
@@ -756,6 +814,10 @@ export default function (pi: ExtensionAPI) {
|
|
|
756
814
|
`injected=${st.injectedCount} Β· dedup=${(st.dedupHitRate * 100).toFixed(0)}%\n` +
|
|
757
815
|
`[mega-compact] anchor=${config.anchorUserMessages} preserveRecent=${config.preserveRecent} ` +
|
|
758
816
|
`autoInlineK=${config.autoInlineK} dedupSim=${config.dedupSim} debug=${config.debug}\n` +
|
|
817
|
+
`[mega-compact] π‘ data-safe: ${di.regionsRetained} regions retained ` +
|
|
818
|
+
`(${fmtB(di.compressedOriginalBytes)} compressed-original) Β· ` +
|
|
819
|
+
`${di.duplicatesCollapsed} dedup-duplicates collapsed Β· ` +
|
|
820
|
+
`${C.green}0 bytes permanently deleted${C.reset}\n` +
|
|
759
821
|
`[mega-compact] stateDir=${currentStateDir}`,
|
|
760
822
|
);
|
|
761
823
|
},
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-mega-compact",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.11",
|
|
4
4
|
"description": "Layered, local, vector-backed context compressor for pi β supersede/collapse/cluster compaction with deduped inline recall.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BSD-2-Clause",
|
package/src/engine.ts
CHANGED
|
@@ -38,6 +38,22 @@ export interface CompactInput {
|
|
|
38
38
|
timestamp?: number;
|
|
39
39
|
/** When true (default), use extractive summary instead of raw concatenation. */
|
|
40
40
|
useExtractiveSummary?: boolean;
|
|
41
|
+
/** Sync progress callback fired by the store as each dedup tier is evaluated
|
|
42
|
+
* (L0βL1βL2βnew). Lets the UI render a live "L0 β β L1 β β L2 0.91 β stored"
|
|
43
|
+
* progress line during compaction. Never awaited; must be side-effect-free-ish
|
|
44
|
+
* and cheap. Optional; back-compat with callers that don't pass it. */
|
|
45
|
+
onTier?: (ev: TierProgress) => void;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Progress event emitted by the store as each dedup tier is evaluated. */
|
|
49
|
+
export interface TierProgress {
|
|
50
|
+
/** Tier being evaluated: "L0" | "L1" | "L2" | "new". */
|
|
51
|
+
tier: "L0" | "L1" | "L2" | "new";
|
|
52
|
+
/** "scanning" while the tier is being evaluated, "deduped" when it matched,
|
|
53
|
+
* "passed" when no match, "stored" at the final outcome. */
|
|
54
|
+
status: "scanning" | "deduped" | "passed" | "stored";
|
|
55
|
+
/** Optional detail β e.g. the L2 cosine sim ("0.91") or the dedup reason. */
|
|
56
|
+
detail?: string;
|
|
41
57
|
}
|
|
42
58
|
|
|
43
59
|
export interface CompactResult {
|
|
@@ -157,6 +173,7 @@ export function compactSession(input: CompactInput, store: VectorStore = getDefa
|
|
|
157
173
|
tokenEstimate: storedTokens,
|
|
158
174
|
originalTokenEstimate,
|
|
159
175
|
timestamp: input.timestamp ?? 0,
|
|
176
|
+
onTier: input.onTier,
|
|
160
177
|
});
|
|
161
178
|
|
|
162
179
|
return {
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { VectorStore } from "../vectorStore.js";
|
|
7
|
+
import { dataInvariantStats } from "./sqlite.js";
|
|
8
|
+
|
|
9
|
+
const baseTmp = mkdtempSync(join(tmpdir(), "mc-p01-"));
|
|
10
|
+
|
|
11
|
+
let counter = 0;
|
|
12
|
+
function store(opts: { dedupSim?: number } = {}) {
|
|
13
|
+
const dir = join(baseTmp, `run-${counter++}`);
|
|
14
|
+
return { s: new VectorStore({ dedupSim: opts.dedupSim ?? 0.9, stateDir: dir }), dir };
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// --- Phase 0: data-safety invariant ---------------------------------------
|
|
18
|
+
|
|
19
|
+
test("Phase 0: every added region retains a compressed-original and deletes nothing", () => {
|
|
20
|
+
const { s, dir } = store();
|
|
21
|
+
s.add({ sessionId: "sess_a", summary: "s1", regionText: "the quick brown fox jumps", tokenEstimate: 5, originalTokenEstimate: 50, timestamp: 1 });
|
|
22
|
+
s.add({ sessionId: "sess_a", summary: "s2", regionText: "a totally different region of work", tokenEstimate: 6, originalTokenEstimate: 60, timestamp: 2 });
|
|
23
|
+
const di = dataInvariantStats(dir);
|
|
24
|
+
assert.equal(di.regionsRetained, 2, "both regions retained");
|
|
25
|
+
assert.ok(di.compressedOriginalBytes > 0, "compressed-original bytes retained");
|
|
26
|
+
assert.equal(di.bytesPermanentlyDeleted, 0, "INVARIANT: nothing permanently deleted");
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
test("Phase 0: dedup collapses a duplicate but original is still retained (not deleted)", () => {
|
|
30
|
+
const { s, dir } = store();
|
|
31
|
+
const text = "identical region content that will dedup on the second add";
|
|
32
|
+
s.add({ sessionId: "sess_b", summary: "s", regionText: text, tokenEstimate: 5, originalTokenEstimate: 50, timestamp: 1 });
|
|
33
|
+
const r2 = s.add({ sessionId: "sess_b", summary: "s", regionText: text, tokenEstimate: 5, originalTokenEstimate: 50, timestamp: 2 });
|
|
34
|
+
assert.equal(r2.deduped, true, "second add deduped");
|
|
35
|
+
const di = dataInvariantStats(dir);
|
|
36
|
+
assert.equal(di.bytesPermanentlyDeleted, 0, "INVARIANT: dedup never deletes data");
|
|
37
|
+
assert.ok(di.regionsRetained >= 1, "survivor region still retained");
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
// --- Phase 1: per-tier progress callback ----------------------------------
|
|
41
|
+
|
|
42
|
+
test("Phase 1: onTier fires L0βL1βL2βstored for a genuinely new region", () => {
|
|
43
|
+
const { s } = store();
|
|
44
|
+
// Seed one checkpoint so L2 has something to scan against.
|
|
45
|
+
s.add({ sessionId: "sess_c", summary: "seed", regionText: "seed region alpha", timestamp: 1 });
|
|
46
|
+
const events: Array<{ tier: string; status: string }> = [];
|
|
47
|
+
s.add({
|
|
48
|
+
sessionId: "sess_c",
|
|
49
|
+
summary: "new",
|
|
50
|
+
regionText: "a brand new region beta that is not a duplicate",
|
|
51
|
+
timestamp: 2,
|
|
52
|
+
onTier: (ev) => events.push({ tier: ev.tier, status: ev.status }),
|
|
53
|
+
});
|
|
54
|
+
const tiers = events.map((e) => e.tier);
|
|
55
|
+
assert.ok(tiers.includes("L0"), "L0 fired");
|
|
56
|
+
assert.ok(tiers.includes("L1"), "L1 fired");
|
|
57
|
+
assert.ok(tiers.includes("L2"), "L2 fired");
|
|
58
|
+
assert.equal(events.at(-1)?.tier, "new", "final event is the 'new' tier");
|
|
59
|
+
assert.equal(events.at(-1)?.status, "stored", "final status is stored");
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test("Phase 1: onTier reports a deduped outcome and short-circuits at the matching tier", () => {
|
|
63
|
+
const { s } = store();
|
|
64
|
+
const text = "region that will exact-dedup on the second pass";
|
|
65
|
+
s.add({ sessionId: "sess_d", summary: "s", regionText: text, timestamp: 1 });
|
|
66
|
+
const events: Array<{ tier: string; status: string; detail?: string }> = [];
|
|
67
|
+
const r = s.add({
|
|
68
|
+
sessionId: "sess_d",
|
|
69
|
+
summary: "s",
|
|
70
|
+
regionText: text,
|
|
71
|
+
timestamp: 2,
|
|
72
|
+
onTier: (ev) => events.push(ev),
|
|
73
|
+
});
|
|
74
|
+
assert.equal(r.deduped, true);
|
|
75
|
+
const deduped = events.find((e) => e.status === "deduped");
|
|
76
|
+
assert.ok(deduped, "a deduped event fired");
|
|
77
|
+
assert.equal(deduped?.tier, "L0", "exact duplicate short-circuits at L0");
|
|
78
|
+
// Must NOT reach the final stored event.
|
|
79
|
+
assert.ok(!events.some((e) => e.status === "stored"), "no stored event on dedup");
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test("Phase 1: onTier is optional (back-compat) β add() works without it", () => {
|
|
83
|
+
const { s } = store();
|
|
84
|
+
const r = s.add({ sessionId: "sess_e", summary: "s", regionText: "no callback here", timestamp: 1 });
|
|
85
|
+
assert.equal(r.deduped, false);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
process.on("exit", () => { try { rmSync(baseTmp, { recursive: true, force: true }); } catch { /* ignore */ } });
|
package/src/store/sqlite.ts
CHANGED
|
@@ -636,6 +636,49 @@ export interface RepoStats {
|
|
|
636
636
|
storageDedupRate: number;
|
|
637
637
|
}
|
|
638
638
|
|
|
639
|
+
/**
|
|
640
|
+
* Data-safety invariant metrics (Phase 0 β trust foundation). Proves that every
|
|
641
|
+
* compacted region is still recoverable: we retain a compressed_original blob for
|
|
642
|
+
* each checkpoint and permanently delete nothing. "removed" rows are SemDeDup
|
|
643
|
+
* duplicates whose ORIGINAL is still retained on the surviving checkpoint β they
|
|
644
|
+
* are not data loss, so they are reported separately, not as deletions.
|
|
645
|
+
*/
|
|
646
|
+
export interface DataInvariantStats {
|
|
647
|
+
/** Checkpoints with a recoverable compressed_original blob. */
|
|
648
|
+
regionsRetained: number;
|
|
649
|
+
/** Total bytes of compressed_original retained (recoverable verbatim). */
|
|
650
|
+
compressedOriginalBytes: number;
|
|
651
|
+
/** Checkpoints missing a compressed_original blob (pre-blob or direct add). */
|
|
652
|
+
regionsWithoutBlob: number;
|
|
653
|
+
/** Bytes permanently deleted by the extension. ALWAYS 0 β the invariant. */
|
|
654
|
+
bytesPermanentlyDeleted: number;
|
|
655
|
+
/** Duplicate rows collapsed by dedup (original retained on the survivor). */
|
|
656
|
+
duplicatesCollapsed: number;
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
export function dataInvariantStats(stateDir: string = getStateDir()): DataInvariantStats {
|
|
660
|
+
const db = openStore(stateDir);
|
|
661
|
+
const row = db
|
|
662
|
+
.prepare(
|
|
663
|
+
`SELECT
|
|
664
|
+
COUNT(compressed_original) AS withBlob,
|
|
665
|
+
COALESCE(SUM(LENGTH(compressed_original)),0) AS blobBytes,
|
|
666
|
+
SUM(CASE WHEN compressed_original IS NULL THEN 1 ELSE 0 END) AS noBlob
|
|
667
|
+
FROM context_chunks WHERE dedup_status != 'removed'`,
|
|
668
|
+
)
|
|
669
|
+
.get() as { withBlob: number; blobBytes: number; noBlob: number };
|
|
670
|
+
const removed = db
|
|
671
|
+
.prepare(`SELECT COUNT(*) AS c FROM context_chunks WHERE dedup_status = 'removed'`)
|
|
672
|
+
.get() as { c: number };
|
|
673
|
+
return {
|
|
674
|
+
regionsRetained: row.withBlob,
|
|
675
|
+
compressedOriginalBytes: row.blobBytes,
|
|
676
|
+
regionsWithoutBlob: row.noBlob ?? 0,
|
|
677
|
+
bytesPermanentlyDeleted: 0,
|
|
678
|
+
duplicatesCollapsed: removed.c,
|
|
679
|
+
};
|
|
680
|
+
}
|
|
681
|
+
|
|
639
682
|
export function repoStats(stateDir: string = getStateDir()): RepoStats {
|
|
640
683
|
const db = openStore(stateDir);
|
|
641
684
|
const row = db
|
package/src/vectorStore.ts
CHANGED
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
getDedupStats,
|
|
37
37
|
bumpDedupStats,
|
|
38
38
|
repoStats as repoStatsFromStore,
|
|
39
|
+
dataInvariantStats,
|
|
39
40
|
} from "./store/sqlite.js";
|
|
40
41
|
import { migrateJsonToSqlite } from "./store/migrate.js";
|
|
41
42
|
|
|
@@ -61,6 +62,10 @@ export interface AddInput {
|
|
|
61
62
|
/** Raw text of the compacted region β used to derive the regionHash + vector. */
|
|
62
63
|
regionText: string;
|
|
63
64
|
timestamp: number;
|
|
65
|
+
/** Sync progress callback fired as each dedup tier is evaluated (L0βL1βL2βnew).
|
|
66
|
+
* Lets the UI render live per-tier progress during compaction. Never awaited;
|
|
67
|
+
* must be cheap. Optional for back-compat. */
|
|
68
|
+
onTier?: (ev: { tier: "L0" | "L1" | "L2" | "new"; status: "scanning" | "deduped" | "passed" | "stored"; detail?: string }) => void;
|
|
64
69
|
}
|
|
65
70
|
|
|
66
71
|
/** Default L2 semantic-dedup enable flag (trigram embedder is local, zero-network). */
|
|
@@ -150,6 +155,9 @@ export class VectorStore {
|
|
|
150
155
|
// we persist (orig β stored). Falls back to stored when orig is unknown.
|
|
151
156
|
const origTokens = input.originalTokenEstimate ?? input.tokenEstimate ?? 0;
|
|
152
157
|
const cfg = this.cfg;
|
|
158
|
+
// Live per-tier progress hook (Phase 1). Sync + optional; fired at each tier
|
|
159
|
+
// so the UI can paint "L0 β β L1 β β L2 0.91 β stored" during a compaction.
|
|
160
|
+
const onTier = input.onTier;
|
|
153
161
|
// Tracks whether a tier matched while in MARK_ONLY (record-but-don't-collapse),
|
|
154
162
|
// and which tier.
|
|
155
163
|
let markOnly: DedupTier | null = null;
|
|
@@ -161,6 +169,7 @@ export class VectorStore {
|
|
|
161
169
|
// skips the scan; a hit is only a candidate, confirmed against `all` below.
|
|
162
170
|
// Gated by L0_ENABLED (Sprint 14). MARK_ONLY_L0 records the decision but
|
|
163
171
|
// does not collapse β the new region is still stored.
|
|
172
|
+
onTier?.({ tier: "L0", status: "scanning" });
|
|
164
173
|
const digest = computeContentDigest(input.regionText);
|
|
165
174
|
const bloom = openBloom(this.stateDir);
|
|
166
175
|
if (cfg.L0_ENABLED && bloom.maybeHas(digest.contentHash)) {
|
|
@@ -180,6 +189,7 @@ export class VectorStore {
|
|
|
180
189
|
addTokensSaved(origTokens, this.stateDir);
|
|
181
190
|
const r = { checkpoint: contentMatch, deduped: true, reason: "contentHash" };
|
|
182
191
|
this.record("L0", "deduped", "contentHash", Date.now() - t0);
|
|
192
|
+
onTier?.({ tier: "L0", status: "deduped", detail: "contentHash" });
|
|
183
193
|
return r;
|
|
184
194
|
}
|
|
185
195
|
}
|
|
@@ -197,6 +207,7 @@ export class VectorStore {
|
|
|
197
207
|
addTokensSaved(origTokens, this.stateDir);
|
|
198
208
|
const r = { checkpoint: regionMatch, deduped: true, reason: "regionHash" };
|
|
199
209
|
this.record("L0", "deduped", "regionHash", Date.now() - t0);
|
|
210
|
+
onTier?.({ tier: "L0", status: "deduped", detail: "regionHash" });
|
|
200
211
|
return r;
|
|
201
212
|
}
|
|
202
213
|
}
|
|
@@ -220,15 +231,19 @@ export class VectorStore {
|
|
|
220
231
|
addTokensSaved(origTokens, this.stateDir);
|
|
221
232
|
const r = { checkpoint: summaryMatch, deduped: true, reason: "summaryHash" };
|
|
222
233
|
this.record("L0", "deduped", "summaryHash", Date.now() - t0);
|
|
234
|
+
onTier?.({ tier: "L0", status: "deduped", detail: "summaryHash" });
|
|
223
235
|
return r;
|
|
224
236
|
}
|
|
225
237
|
}
|
|
226
238
|
}
|
|
239
|
+
// L0 did not collapse this region.
|
|
240
|
+
onTier?.({ tier: "L0", status: "passed" });
|
|
227
241
|
|
|
228
242
|
// 2b. L1 MinHash/LSH near-duplicate dedup (Sprint 11) β catches one-word
|
|
229
243
|
// edits / rewordings that L0's exact hash misses. Cheap LSH bucket
|
|
230
244
|
// retrieval β trigram verification (pg_trgm-equivalent) as the final gate.
|
|
231
245
|
// Gated by L1_ENABLED (Sprint 14); MARK_ONLY_L1 records but doesn't collapse.
|
|
246
|
+
onTier?.({ tier: "L1", status: "scanning" });
|
|
232
247
|
if (cfg.L1_ENABLED) {
|
|
233
248
|
const l1 = this.findL1Duplicate(sessionId, input.regionText, all);
|
|
234
249
|
if (l1 && !cfg.MARK_ONLY_L1) {
|
|
@@ -237,10 +252,12 @@ export class VectorStore {
|
|
|
237
252
|
bumpDedupStats(true, this.stateDir);
|
|
238
253
|
const r = { checkpoint: l1, deduped: true, reason: "l1MinHash" };
|
|
239
254
|
this.record("L1", "deduped", "l1MinHash", Date.now() - t0);
|
|
255
|
+
onTier?.({ tier: "L1", status: "deduped", detail: "l1MinHash" });
|
|
240
256
|
return r;
|
|
241
257
|
}
|
|
242
258
|
if (l1 && cfg.MARK_ONLY_L1) markOnly = "L1";
|
|
243
259
|
}
|
|
260
|
+
onTier?.({ tier: "L1", status: "passed" });
|
|
244
261
|
|
|
245
262
|
// 3. L2 semantic dedup β catches near-identical / semantically-similar regions
|
|
246
263
|
// via cosine over the embedding. topicSummary is used for summaryHash dedup
|
|
@@ -252,6 +269,7 @@ export class VectorStore {
|
|
|
252
269
|
const SIMILARITY_BUDGET_MS = cfg.SIMILARITY_BUDGET_MS;
|
|
253
270
|
const simThreshold = this.l2Threshold; // from cfg.L2_COSINE (default 0.85 trigram)
|
|
254
271
|
const embedding = this.embedder.embed(input.regionText);
|
|
272
|
+
onTier?.({ tier: "L2", status: "scanning" });
|
|
255
273
|
if (cfg.L2_ENABLED && all.length > 0) {
|
|
256
274
|
const start = Date.now();
|
|
257
275
|
let timedOut = false;
|
|
@@ -274,10 +292,12 @@ export class VectorStore {
|
|
|
274
292
|
addTokensSaved(origTokens, this.stateDir);
|
|
275
293
|
const r = { checkpoint: nearest.checkpoint, deduped: true, reason: "contentSimilarity" };
|
|
276
294
|
this.record("L2", "deduped", "contentSimilarity", Date.now() - t0);
|
|
295
|
+
onTier?.({ tier: "L2", status: "deduped", detail: nearest.sim.toFixed(2) });
|
|
277
296
|
return r;
|
|
278
297
|
}
|
|
279
298
|
markOnly = "L2";
|
|
280
299
|
}
|
|
300
|
+
onTier?.({ tier: "L2", status: "passed", detail: `best ${nearest.sim.toFixed(2)}` });
|
|
281
301
|
}
|
|
282
302
|
|
|
283
303
|
// 4. Genuinely new β create checkpoint
|
|
@@ -342,6 +362,7 @@ export class VectorStore {
|
|
|
342
362
|
}
|
|
343
363
|
// Cumulative store-wide dedup accounting (attempt, not collapsed).
|
|
344
364
|
bumpDedupStats(false, this.stateDir);
|
|
365
|
+
onTier?.({ tier: "new", status: "stored" });
|
|
345
366
|
return { checkpoint, deduped: false };
|
|
346
367
|
}
|
|
347
368
|
|
|
@@ -585,4 +606,8 @@ export class VectorStore {
|
|
|
585
606
|
repoStats(): ReturnType<typeof repoStatsFromStore> {
|
|
586
607
|
return repoStatsFromStore(this.stateDir);
|
|
587
608
|
}
|
|
609
|
+
/** Data-safety invariant (Phase 0): regions retained vs bytes permanently deleted. */
|
|
610
|
+
dataInvariant(): ReturnType<typeof dataInvariantStats> {
|
|
611
|
+
return dataInvariantStats(this.stateDir);
|
|
612
|
+
}
|
|
588
613
|
}
|