pi-mega-compact 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -23
- package/dist/extensions/conflict-scan.js +133 -60
- package/dist/extensions/conflict-scan.test.js +115 -0
- package/dist/extensions/dashboard-server.js +101 -35
- package/dist/extensions/dashboard-server.test.js +2 -2
- package/dist/extensions/mega-runtime.js +47 -20
- package/dist/src/store/memoryIndex.js +32 -7
- package/dist/src/store/vectorIndex.js +32 -7
- package/extensions/conflict-scan.test.ts +129 -0
- package/extensions/conflict-scan.ts +243 -158
- package/extensions/dashboard-server.test.ts +2 -2
- package/extensions/dashboard-server.ts +106 -36
- package/extensions/mega-dashboard.ts +16 -0
- package/extensions/mega-runtime.ts +48 -20
- package/package.json +1 -1
- package/src/store/memoryIndex.ts +40 -6
- package/src/store/vectorIndex.ts +40 -6
- package/dist/extensions/openclaw-mega-compact.js +0 -291
- package/dist/src/minilm.js +0 -92
- package/dist/src/wordpiece.js +0 -129
|
@@ -72,6 +72,7 @@ function readIndex() {
|
|
|
72
72
|
const mapped = rows.map((r) => ({
|
|
73
73
|
repoRoot: String(r.repo_root ?? ""),
|
|
74
74
|
displayName: String(r.display_name ?? ""),
|
|
75
|
+
stateDir: String(r.state_dir ?? ""),
|
|
75
76
|
checkpointCount: Number(r.checkpoint_count ?? 0),
|
|
76
77
|
tokensSaved: Number(r.tokens_saved ?? 0),
|
|
77
78
|
compressedOriginalBytes: Number(r.compressed_original_bytes ?? 0),
|
|
@@ -290,27 +291,31 @@ function dashboardHtml(tierName) {
|
|
|
290
291
|
<h2>Vector Store</h2>
|
|
291
292
|
<div class="stat-grid">
|
|
292
293
|
<span class="label" title="A saved summary of a chunk of your conversation that was compacted to free up space.">Checkpoints</span><span class="value" id="st-count">0</span>
|
|
293
|
-
<span class="label" title="
|
|
294
|
-
<span class="label" title="
|
|
295
|
-
<span class="label" title="
|
|
294
|
+
<span class="label" title="Total size of the original conversation text dropped into compaction this session, including redundant regions skipped by dedup. This is the 'in'.">Original (dropped)</span><span class="value" id="st-in">0</span>
|
|
295
|
+
<span class="label" title="Compact summaries we are currently holding as 'memory' for this session (the 'out'). Smaller is better.">Kept (summaries)</span><span class="value" id="st-kept">0</span>
|
|
296
|
+
<span class="label" title="Conversation space freed = dropped − kept (the 'saved').">Freed (dropped − kept)</span><span class="value" id="st-freed">0</span>
|
|
296
297
|
<span class="label" title="How many times old context was automatically brought back into the conversation because it was relevant to what you were doing.">Injected</span><span class="value" id="st-injected">0</span>
|
|
297
298
|
<span class="label" title="Of the times we recalled old context, how often it was actually on-topic.">Recall Relevance</span><span class="value" id="st-dedup">0%</span>
|
|
298
299
|
<span class="label" title="How often new content matched something we already had, so we skipped storing a duplicate copy. Higher = less wasted space.">Storage Dedup</span><span class="value" id="st-sdedup">0%</span>
|
|
299
300
|
<span class="label" title="How many duplicate chunks we collapsed into one instead of storing separately.">Collapsed</span><span class="value" id="st-collapsed">0</span>
|
|
300
301
|
<span class="label" title="The ID of the most recent saved checkpoint.">Last ID</span><span class="value" id="st-lastid">—</span>
|
|
301
302
|
</div>
|
|
303
|
+
<div class="meter-track" style="margin-top:10px"><div class="meter-fill" id="st-compress-bar" style="width:0%"></div></div>
|
|
304
|
+
<div class="meter-sub" id="st-compress-sub">waiting for compaction…</div>
|
|
302
305
|
</div>
|
|
303
306
|
<div class="card">
|
|
304
307
|
<h2>Repo (all sessions)</h2>
|
|
305
308
|
<div class="stat-grid">
|
|
306
309
|
<span class="label">Checkpoints</span><span class="value" id="rp-count">0</span>
|
|
307
|
-
<span class="label">
|
|
308
|
-
<span class="label">
|
|
309
|
-
<span class="label">
|
|
310
|
+
<span class="label">Original (dropped)</span><span class="value" id="rp-in">0</span>
|
|
311
|
+
<span class="label">Kept (summaries)</span><span class="value" id="rp-kept">0</span>
|
|
312
|
+
<span class="label">Freed (dropped − kept)</span><span class="value" id="rp-freed">0</span>
|
|
310
313
|
<span class="label">Sessions</span><span class="value" id="rp-sessions">0</span>
|
|
311
314
|
<span class="label">Collapsed</span><span class="value" id="rp-collapsed">0</span>
|
|
312
315
|
<span class="label">Storage Dedup</span><span class="value" id="rp-sdedup">0%</span>
|
|
313
316
|
</div>
|
|
317
|
+
<div class="meter-track" style="margin-top:10px"><div class="meter-fill" id="rp-compress-bar" style="width:0%"></div></div>
|
|
318
|
+
<div class="meter-sub" id="rp-compress-sub">waiting for compaction…</div>
|
|
314
319
|
</div>
|
|
315
320
|
<div class="card safe">
|
|
316
321
|
<h2>🛡 Data Safety</h2>
|
|
@@ -356,11 +361,11 @@ function dashboardHtml(tierName) {
|
|
|
356
361
|
<div class="card legend">
|
|
357
362
|
<h2>What these numbers mean</h2>
|
|
358
363
|
<ul class="legend-list">
|
|
359
|
-
<li><b>
|
|
360
|
-
<li><b>
|
|
361
|
-
<li><b>
|
|
362
|
-
<li><b>
|
|
363
|
-
<li><b>Storage dedup
|
|
364
|
+
<li><b>Original (dropped)</b> — everything compacted away (including duplicates caught by dedup). The "in."</li>
|
|
365
|
+
<li><b>Kept (summaries)</b> — compact summaries still held as "memory" (the "out").</li>
|
|
366
|
+
<li><b>Freed</b> = dropped − kept — tokens saved so far (higher = better).</li>
|
|
367
|
+
<li><b>Compression %</b> — Freed ÷ Dropped — the headline efficiency number. Higher = more space reclaimed.</li>
|
|
368
|
+
<li><b>Storage dedup %</b> — how often new content matched something already saved, so no duplicate copy was written.</li>
|
|
364
369
|
<li><b>Data safety</b> — every compacted region is kept verbatim (compressed). Nothing is permanently deleted; you can restore any of it.</li>
|
|
365
370
|
</ul>
|
|
366
371
|
<p class="legend-note">Hover any label above for a quick explanation.</p>
|
|
@@ -464,10 +469,20 @@ function dashboardHtml(tierName) {
|
|
|
464
469
|
d.trigger.armed ? 'past fast gate — monitoring token count' : 'idle — below fast gate';
|
|
465
470
|
document.getElementById('tr-state').textContent = state;
|
|
466
471
|
|
|
472
|
+
// ---- Vector Store — reconciled token accounting (same formula as widget) -
|
|
467
473
|
document.getElementById('st-count').textContent = d.store.checkpointCount;
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
474
|
+
// Compression block from the snapshot (Freed = In − Out, single formula).
|
|
475
|
+
var c = d.compression || {};
|
|
476
|
+
var sess = c.session || { tokensIn:0, tokensOut:0, tokensFreed:0, compressionPct:0, dedupPct:0 };
|
|
477
|
+
var cRepo = c.repo || { tokensIn:0, tokensOut:0, tokensFreed:0, compressionPct:0, dedupPct:0 };
|
|
478
|
+
document.getElementById('st-in').textContent = sess.tokensIn.toLocaleString();
|
|
479
|
+
document.getElementById('st-kept').textContent = sess.tokensOut.toLocaleString();
|
|
480
|
+
document.getElementById('st-freed').textContent = sess.tokensFreed.toLocaleString();
|
|
481
|
+
var sp = sess.compressionPct || 0;
|
|
482
|
+
document.getElementById('st-compress-bar').style.width = Math.max(sp * 100, 0.5) + '%';
|
|
483
|
+
document.getElementById('st-compress-bar').className = 'meter-fill ' + (sp >= 0.9 ? 'meter-green' : sp >= 0.6 ? 'meter-yellow' : 'meter-red');
|
|
484
|
+
document.getElementById('st-compress-sub').textContent = (sp * 100 >= 10 ? Math.round(sp * 100) : (sp * 100).toFixed(1)) + '% tokens saved · dedup: ' + (sess.dedupPct * 100 >= 10 ? Math.round(sess.dedupPct * 100) : (sess.dedupPct * 100).toFixed(1)) + '%';
|
|
485
|
+
// ------
|
|
471
486
|
document.getElementById('st-injected').textContent = d.store.injectedCount;
|
|
472
487
|
document.getElementById('st-dedup').textContent = Math.round(d.store.dedupHitRate * 100) + '%';
|
|
473
488
|
var sdr = d.store.storageDedupRate || 0;
|
|
@@ -475,16 +490,19 @@ function dashboardHtml(tierName) {
|
|
|
475
490
|
document.getElementById('st-collapsed').textContent = d.store.dedupCollapsed || 0;
|
|
476
491
|
document.getElementById('st-lastid').textContent = d.session.lastCheckpointId || '—';
|
|
477
492
|
|
|
478
|
-
// Repo
|
|
479
|
-
|
|
480
|
-
document.getElementById('rp-
|
|
481
|
-
document.getElementById('rp-
|
|
482
|
-
document.getElementById('rp-
|
|
483
|
-
document.getElementById('rp-
|
|
484
|
-
document.getElementById('rp-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
493
|
+
// ---- Repo (all sessions) — same compression fields, repo scope ----------
|
|
494
|
+
document.getElementById('rp-count').textContent = (d.repo && d.repo.checkpointCount || 0).toLocaleString();
|
|
495
|
+
document.getElementById('rp-in').textContent = cRepo.tokensIn.toLocaleString();
|
|
496
|
+
document.getElementById('rp-kept').textContent = cRepo.tokensOut.toLocaleString();
|
|
497
|
+
document.getElementById('rp-freed').textContent = cRepo.tokensFreed.toLocaleString();
|
|
498
|
+
document.getElementById('rp-sessions').textContent = (d.repo && d.repo.sessionCount || 0).toLocaleString();
|
|
499
|
+
document.getElementById('rp-collapsed').textContent = (d.repo && d.repo.dedupCollapsed || 0).toLocaleString();
|
|
500
|
+
var rdr = d.repo && d.repo.storageDedupRate || 0;
|
|
501
|
+
document.getElementById('rp-sdedup').textContent = (rdr * 100 >= 10 ? Math.round(rdr * 100) : (rdr * 100).toFixed(1)) + '%';
|
|
502
|
+
var rp = cRepo.compressionPct || 0;
|
|
503
|
+
document.getElementById('rp-compress-bar').style.width = Math.max(rp * 100, 0.5) + '%';
|
|
504
|
+
document.getElementById('rp-compress-bar').className = 'meter-fill ' + (rp >= 0.9 ? 'meter-green' : rp >= 0.6 ? 'meter-yellow' : 'meter-red');
|
|
505
|
+
document.getElementById('rp-compress-sub').textContent = (rp * 100 >= 10 ? Math.round(rp * 100) : (rp * 100).toFixed(1)) + '% tokens saved · dedup: ' + (cRepo.dedupPct * 100 >= 10 ? Math.round(cRepo.dedupPct * 100) : (cRepo.dedupPct * 100).toFixed(1)) + '%';
|
|
488
506
|
|
|
489
507
|
// Data-safety invariant (Phase 0 — trust foundation).
|
|
490
508
|
var ig = d.integrity || { regionsRetained: 0, compressedOriginalBytes: 0, duplicatesCollapsed: 0, bytesPermanentlyDeleted: 0 };
|
|
@@ -524,10 +542,11 @@ function dashboardHtml(tierName) {
|
|
|
524
542
|
document.getElementById('md-provider').textContent = model && model.providerName ? model.providerName : (model && model.provider ? model.provider : '—');
|
|
525
543
|
document.getElementById('md-input').textContent = model && model.inputRate ? '$' + (model.inputRate).toFixed(6) : '—';
|
|
526
544
|
document.getElementById('md-output').textContent = model && model.outputRate ? '$' + (model.outputRate).toFixed(6) : '—';
|
|
527
|
-
|
|
528
|
-
|
|
545
|
+
var repoSaved = cRepo.tokensFreed || 0;
|
|
546
|
+
if (model && model.inputRate && repoSaved > 0) {
|
|
547
|
+
var usd = (repoSaved * model.inputRate);
|
|
529
548
|
var win = d.context.contextWindow || 0;
|
|
530
|
-
var windows = win > 0 ? (
|
|
549
|
+
var windows = win > 0 ? (repoSaved / win).toFixed(1) : '0';
|
|
531
550
|
document.getElementById('cost-usd').textContent = '≈ $' + usd.toFixed(4) + ' saved';
|
|
532
551
|
document.getElementById('cost-windows').textContent = windows + ' context-windows extended';
|
|
533
552
|
} else {
|
|
@@ -757,7 +776,47 @@ export async function launchDashboardServer(stateDir) {
|
|
|
757
776
|
// ── New server ────────────────────────────────────────────────────────────
|
|
758
777
|
mkdirSync(stateDir, { recursive: true });
|
|
759
778
|
let eventOffset = 0;
|
|
779
|
+
// Overlay the live current-repo snapshot (snapshot.json, rewritten every
|
|
780
|
+
// context event) onto its registry row so the All-repos / Summary views stay
|
|
781
|
+
// in sync with the live menu bar + Current-repo card in real time. The
|
|
782
|
+
// registry (index.sqlite) is only written on repo-switch (bindRepo), so
|
|
783
|
+
// without this the current repo's row freezes between switches. Read-only —
|
|
784
|
+
// no extra writes to index.sqlite. Matched by stateDir, which equals the
|
|
785
|
+
// value this server was launched with (runtime.currentStateDir).
|
|
786
|
+
function overlayCurrentRepo(idx) {
|
|
787
|
+
if (!idx || !idx.repos.length)
|
|
788
|
+
return;
|
|
789
|
+
let snap = null;
|
|
790
|
+
try {
|
|
791
|
+
snap = readSnapshot(snapshotPath);
|
|
792
|
+
}
|
|
793
|
+
catch {
|
|
794
|
+
return;
|
|
795
|
+
}
|
|
796
|
+
if (!snap || !snap.repo)
|
|
797
|
+
return;
|
|
798
|
+
const cur = idx.repos.find((r) => r.stateDir === stateDir);
|
|
799
|
+
if (!cur)
|
|
800
|
+
return;
|
|
801
|
+
const prevSaved = cur.tokensSaved;
|
|
802
|
+
const prevCp = cur.checkpointCount;
|
|
803
|
+
const prevBytes = cur.compressedOriginalBytes;
|
|
804
|
+
const comp = snap.compression?.repo;
|
|
805
|
+
const liveSaved = comp ? comp.tokensFreed : (snap.repo.tokensSaved ?? prevSaved);
|
|
806
|
+
const liveCp = snap.repo.checkpointCount ?? prevCp;
|
|
807
|
+
const liveBytes = snap.integrity?.compressedOriginalBytes ?? prevBytes;
|
|
808
|
+
cur.tokensSaved = liveSaved;
|
|
809
|
+
cur.checkpointCount = liveCp;
|
|
810
|
+
cur.compressedOriginalBytes = liveBytes;
|
|
811
|
+
if (idx.summary) {
|
|
812
|
+
idx.summary.totalTokensSaved += liveSaved - prevSaved;
|
|
813
|
+
idx.summary.totalCheckpoints += liveCp - prevCp;
|
|
814
|
+
idx.summary.totalCompressedOriginalBytes += liveBytes - prevBytes;
|
|
815
|
+
}
|
|
816
|
+
idx.updatedAt = snap.updatedAt ?? idx.updatedAt;
|
|
817
|
+
}
|
|
760
818
|
const server = createServer((req, res) => {
|
|
819
|
+
// guardrails-allow PREVENT-PI-004: optional, user-triggered /dashboard localhost server (loopback-only) — CORS open for local browser access
|
|
761
820
|
// CORS for local access
|
|
762
821
|
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
763
822
|
res.setHeader("Access-Control-Allow-Methods", "GET, OPTIONS");
|
|
@@ -790,8 +849,11 @@ export async function launchDashboardServer(stateDir) {
|
|
|
790
849
|
// directly from SQLite (index.sqlite). Lets one dashboard show every repo's
|
|
791
850
|
// checkpoints, tokens saved, and active model. Read-only.
|
|
792
851
|
if (req.url === "/api/index") {
|
|
852
|
+
const idx = readIndex();
|
|
853
|
+
if (idx)
|
|
854
|
+
overlayCurrentRepo(idx);
|
|
793
855
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
794
|
-
res.end(JSON.stringify(
|
|
856
|
+
res.end(JSON.stringify(idx ?? { updatedAt: null, summary: null, repos: [] }));
|
|
795
857
|
return;
|
|
796
858
|
}
|
|
797
859
|
// /api/repos — registry list. Optional `?active=24h` filters to repos
|
|
@@ -800,8 +862,10 @@ export async function launchDashboardServer(stateDir) {
|
|
|
800
862
|
if (req.url?.startsWith("/api/repos")) {
|
|
801
863
|
const url = new URL(req.url, "http://x");
|
|
802
864
|
const activeParam = url.searchParams.get("active");
|
|
803
|
-
const idx = readIndex()
|
|
804
|
-
|
|
865
|
+
const idx = readIndex();
|
|
866
|
+
if (idx)
|
|
867
|
+
overlayCurrentRepo(idx);
|
|
868
|
+
let repos = idx?.repos ?? [];
|
|
805
869
|
if (activeParam) {
|
|
806
870
|
const m = /^(\d+)h$/.exec(activeParam);
|
|
807
871
|
if (m) {
|
|
@@ -810,21 +874,23 @@ export async function launchDashboardServer(stateDir) {
|
|
|
810
874
|
}
|
|
811
875
|
}
|
|
812
876
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
813
|
-
res.end(JSON.stringify({ updatedAt: idx
|
|
877
|
+
res.end(JSON.stringify({ updatedAt: idx?.updatedAt ?? null, repos, count: repos.length }));
|
|
814
878
|
return;
|
|
815
879
|
}
|
|
816
880
|
// /api/summary — header tiles without the full repo list (keeps payload
|
|
817
881
|
// small for embed scenarios). activeRepos mirrors the /api/repos?active=24h
|
|
818
882
|
// count so the dashboard can render the active badge alongside totals.
|
|
819
883
|
if (req.url?.startsWith("/api/summary")) {
|
|
820
|
-
const idx = readIndex()
|
|
821
|
-
|
|
884
|
+
const idx = readIndex();
|
|
885
|
+
if (idx)
|
|
886
|
+
overlayCurrentRepo(idx);
|
|
887
|
+
const repos = idx?.repos ?? [];
|
|
822
888
|
const cutoffSec = Math.floor(Date.now() / 1000) - 24 * 3600;
|
|
823
889
|
const activeRepos = repos.filter((r) => (r.lastSeen ?? 0) >= cutoffSec).length;
|
|
824
890
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
825
891
|
res.end(JSON.stringify({
|
|
826
|
-
updatedAt: idx
|
|
827
|
-
summary: idx
|
|
892
|
+
updatedAt: idx?.updatedAt ?? null,
|
|
893
|
+
summary: idx?.summary ?? null,
|
|
828
894
|
activeRepos,
|
|
829
895
|
totalRepos: repos.length,
|
|
830
896
|
}));
|
|
@@ -123,8 +123,8 @@ describe("multi-repo /api/index (S19)", () => {
|
|
|
123
123
|
process.env.MEGACOMPACT_INDEX_DIR = indexDir;
|
|
124
124
|
process.env.MEGACOMPACT_DASHBOARD_PORT = "19321"; // private base, non-colliding
|
|
125
125
|
const { upsertRepoRegistry } = await import("../src/store/sqlite.js");
|
|
126
|
-
upsertRepoRegistry({ repoRoot: "/home/u/repoA", displayName: "repoA", stateDir: dir, checkpointCount: 3, tokensSaved: 1000, compressedOriginalBytes: 0 }, indexDir);
|
|
127
|
-
upsertRepoRegistry({ repoRoot: "/home/u/repoB", displayName: "repoB", stateDir: dir, checkpointCount: 5, tokensSaved: 2000, compressedOriginalBytes: 0 }, indexDir);
|
|
126
|
+
upsertRepoRegistry({ repoRoot: "/home/u/repoA", displayName: "repoA", stateDir: join(dir, "a"), checkpointCount: 3, tokensSaved: 1000, compressedOriginalBytes: 0 }, indexDir);
|
|
127
|
+
upsertRepoRegistry({ repoRoot: "/home/u/repoB", displayName: "repoB", stateDir: join(dir, "b"), checkpointCount: 5, tokensSaved: 2000, compressedOriginalBytes: 0 }, indexDir);
|
|
128
128
|
const child = spawn(process.execPath, [SERVER_ENTRY, dir], { stdio: "ignore" });
|
|
129
129
|
try {
|
|
130
130
|
await waitFor(async () => {
|
|
@@ -255,6 +255,25 @@ export class MegaRuntime {
|
|
|
255
255
|
trigger: { armed, ready, currentTokens: this.lastCtxTokens, thresholdTokens: this.config.thresholdTokens, fastGatePct: this.config.fastGatePct },
|
|
256
256
|
crew: { activeAgents: this.activeAgents, currentTurn: this.currentTurn },
|
|
257
257
|
store: { checkpointCount: st.checkpointCount, totalTokenEstimate: st.totalTokenEstimate, originalTokens: st.originalTokens, tokensSaved: this.rt.tokensSaved, injectedCount: st.injectedCount, dedupHitRate: st.dedupHitRate, storageDedupRate: st.storageDedupRate, dedupAttempts: st.dedupAttempts, dedupCollapsed: st.dedupCollapsed },
|
|
258
|
+
// Reconciled token accounting (single canonical formula, session + repo).
|
|
259
|
+
// Freed = In − Out; In = Freed + Out. session.Freed = rt.tokensSaved (incl.
|
|
260
|
+
// deduped-away originals); repo.Freed = repo.tokensSaved meta counter.
|
|
261
|
+
compression: {
|
|
262
|
+
session: {
|
|
263
|
+
tokensIn: this.rt.tokensSaved + st.totalTokenEstimate,
|
|
264
|
+
tokensOut: st.totalTokenEstimate,
|
|
265
|
+
tokensFreed: this.rt.tokensSaved,
|
|
266
|
+
compressionPct: (this.rt.tokensSaved + st.totalTokenEstimate) > 0 ? this.rt.tokensSaved / (this.rt.tokensSaved + st.totalTokenEstimate) : 0,
|
|
267
|
+
dedupPct: st.storageDedupRate,
|
|
268
|
+
},
|
|
269
|
+
repo: {
|
|
270
|
+
tokensIn: repo.tokensSaved + repo.totalTokenEstimate,
|
|
271
|
+
tokensOut: repo.totalTokenEstimate,
|
|
272
|
+
tokensFreed: repo.tokensSaved,
|
|
273
|
+
compressionPct: (repo.tokensSaved + repo.totalTokenEstimate) > 0 ? repo.tokensSaved / (repo.tokensSaved + repo.totalTokenEstimate) : 0,
|
|
274
|
+
dedupPct: repo.storageDedupRate,
|
|
275
|
+
},
|
|
276
|
+
},
|
|
258
277
|
repo: {
|
|
259
278
|
checkpointCount: repo.checkpointCount,
|
|
260
279
|
totalTokenEstimate: repo.totalTokenEstimate,
|
|
@@ -291,33 +310,41 @@ export class MegaRuntime {
|
|
|
291
310
|
const dedupStr = storageRate * 100 >= 10
|
|
292
311
|
? `${Math.round(storageRate * 100)}%`
|
|
293
312
|
: `${(storageRate * 100).toFixed(1)}%`;
|
|
294
|
-
//
|
|
295
|
-
//
|
|
296
|
-
//
|
|
297
|
-
//
|
|
298
|
-
//
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
const usedStr = `${C.cyan}${fmt(st.totalTokenEstimate)} sess${C.reset} / ${C.blue}${fmt(repo.totalTokenEstimate)} repo${C.reset}`;
|
|
313
|
+
// Reconciled token accounting — ONE canonical formula for session + repo,
|
|
314
|
+
// matching the dashboard so the two never disagree. unit format: M at/above
|
|
315
|
+
// 1e6, k at/above 1e3, raw below — so 5,472,700 → "5.5M", 24,100 → "24.1k",
|
|
316
|
+
// 142 → "142". Dropped (in) = Freed + Kept; Freed = rt.tokensSaved (session)
|
|
317
|
+
// / repo.tokensSaved meta (repo); Kept = totalTokenEstimate (stored).
|
|
318
|
+
const fmt = (x) => x >= 1_000_000 ? `${(x / 1_000_000).toFixed(1)}M`
|
|
319
|
+
: x >= 1000 ? `${(x / 1000).toFixed(1)}k`
|
|
320
|
+
: `${Math.round(x)}`;
|
|
303
321
|
const agentStr = this.activeAgents > 0 ? ` │ 🤖 ${this.activeAgents} agent${this.activeAgents === 1 ? "" : "s"}` : "";
|
|
304
322
|
const turnStr = this.currentTurn > 0 ? ` │ turn ${this.currentTurn}` : "";
|
|
305
323
|
// Phase 3 — pulsing status glyph while a compaction is in flight.
|
|
306
324
|
const pulse = this.pulsing ? `${C.cyan}${PULSE[Math.floor(Date.now() / 250) % PULSE.length]}${C.reset} ` : "";
|
|
325
|
+
// --- reconciled in/out view (session + repo) ---------------------------
|
|
326
|
+
const sessIn = this.rt.tokensSaved + st.totalTokenEstimate;
|
|
327
|
+
const sessKept = st.totalTokenEstimate;
|
|
328
|
+
const sessFreed = this.rt.tokensSaved;
|
|
329
|
+
const sessPct = sessIn > 0 ? sessFreed / sessIn : 0;
|
|
330
|
+
const repoIn = repo.tokensSaved + repo.totalTokenEstimate;
|
|
331
|
+
const repoKept = repo.totalTokenEstimate;
|
|
332
|
+
const repoFreed = repo.tokensSaved;
|
|
333
|
+
const repoPct = repoIn > 0 ? repoFreed / repoIn : 0;
|
|
307
334
|
const lines = [
|
|
308
335
|
` ${C.amber}⚡ ${tierLabel}${C.reset} v${C.bold}${ownVersion()}${C.reset} │ ${tokStr}/${maxStr} tokens (${C.bold}${pctStr}${C.reset}) │ ${st.checkpointCount} saved${agentStr}${turnStr}`,
|
|
309
|
-
` ${triggerLabel} │ ${C.magenta}repeat-skipped: ${dedupStr}${C.reset}
|
|
336
|
+
` ${triggerLabel} │ ${C.magenta}repeat-skipped: ${dedupStr}${C.reset}`,
|
|
337
|
+
` ${C.gray}dropped ${fmt(sessIn)} → kept ${fmt(sessKept)} sess / ${fmt(repoKept)} repo · freed ${fmt(sessFreed)} sess / ${fmt(repoFreed)} repo${C.reset}`,
|
|
310
338
|
];
|
|
311
|
-
//
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
const
|
|
315
|
-
const filled = Math.
|
|
316
|
-
const
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
lines.push(` ${C.green}saved ${fmt(this.rt.tokensSaved)} ${bar}${C.reset} ${pct}% of ${fmt(goal)} ${C.gray}│${C.reset} ${C.blue}${fmt(this.rt.tokensSaved)}${C.reset}/${C.blue}${fmt(totalHeld)}${C.reset} tok held`);
|
|
339
|
+
// Compression meter — the single headline "% tokens saved" (Freed / In),
|
|
340
|
+
// same formula as the dashboard. Higher = better, so it reads green.
|
|
341
|
+
{
|
|
342
|
+
const w = 10;
|
|
343
|
+
const filled = Math.max(0, Math.min(w, Math.round(sessPct * w)));
|
|
344
|
+
const cbar = C.green + "▓".repeat(filled) + C.dim + "░".repeat(w - filled) + C.reset;
|
|
345
|
+
const sTxt = (sessPct * 100).toFixed(sessPct * 100 >= 10 ? 0 : 1);
|
|
346
|
+
const rTxt = (repoPct * 100).toFixed(repoPct * 100 >= 10 ? 0 : 1);
|
|
347
|
+
lines.push(` ${cbar} ${sTxt}% tokens saved (sess) · ${rTxt}% repo${C.reset}`);
|
|
321
348
|
}
|
|
322
349
|
// Live "now processing" line + why + recent deduped/compacted events,
|
|
323
350
|
// collapsed to ONE rotating line (fresh only). The ticker ring buffer
|
|
@@ -22,17 +22,15 @@
|
|
|
22
22
|
import { homedir } from "node:os";
|
|
23
23
|
import { join } from "node:path";
|
|
24
24
|
import { mkdirSync, rmSync, existsSync } from "node:fs";
|
|
25
|
-
// PGlite + pgvector are script-free WASM (no native build) → survive pi's
|
|
26
|
-
// install-script block. Imported lazily so a missing/broken package degrades
|
|
27
|
-
// gracefully instead of crashing module load.
|
|
28
|
-
import { PGlite } from "@electric-sql/pglite";
|
|
29
|
-
import { vector } from "@electric-sql/pglite-pgvector";
|
|
30
25
|
/** Vector dimension produced by the default TrigramEmbedder (src/embedder.ts). */
|
|
31
26
|
export const MEMORY_INDEX_DIM = 512;
|
|
32
27
|
let db;
|
|
33
28
|
let initPromise;
|
|
34
29
|
let disabled = false;
|
|
35
30
|
let warned = false;
|
|
31
|
+
/** Lazily-loaded PGlite module + pgvector extension (see loadPgLite). */
|
|
32
|
+
let pgliteMod;
|
|
33
|
+
let pgliteLoadFailed = false;
|
|
36
34
|
function indexDir() {
|
|
37
35
|
const override = process.env.MEGACOMPACT_INDEX_DIR;
|
|
38
36
|
if (override && override.trim() !== "")
|
|
@@ -77,17 +75,44 @@ export function initMemoryIndex() {
|
|
|
77
75
|
initPromise = openPgLite(/* retryOnCorrupt */ true);
|
|
78
76
|
return initPromise;
|
|
79
77
|
}
|
|
78
|
+
/**
|
|
79
|
+
* Lazily load the PGlite module + pgvector extension via dynamic import. Caches
|
|
80
|
+
* success and permanent failure. Returns undefined (once, then forever) when the
|
|
81
|
+
* package is missing/broken so callers fall back to the same-repo scan. Never throws.
|
|
82
|
+
*/
|
|
83
|
+
async function loadPgLite() {
|
|
84
|
+
if (pgliteMod)
|
|
85
|
+
return pgliteMod;
|
|
86
|
+
if (pgliteLoadFailed)
|
|
87
|
+
return undefined;
|
|
88
|
+
try {
|
|
89
|
+
const [pglitePkg, pgvectorPkg] = await Promise.all([
|
|
90
|
+
import("@electric-sql/pglite"),
|
|
91
|
+
import("@electric-sql/pglite-pgvector"),
|
|
92
|
+
]);
|
|
93
|
+
pgliteMod = { PGlite: pglitePkg.PGlite, vector: pgvectorPkg.vector };
|
|
94
|
+
return pgliteMod;
|
|
95
|
+
}
|
|
96
|
+
catch (err) {
|
|
97
|
+
pgliteLoadFailed = true;
|
|
98
|
+
logWarn(`package unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
80
102
|
/**
|
|
81
103
|
* Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level abort
|
|
82
104
|
* (typically from a corrupted/torn data dir) triggers a delete + one retry.
|
|
83
105
|
*/
|
|
84
106
|
async function openPgLite(retryOnCorrupt) {
|
|
85
107
|
try {
|
|
108
|
+
const mod = await loadPgLite();
|
|
109
|
+
if (!mod)
|
|
110
|
+
return undefined;
|
|
86
111
|
const dir = indexDir();
|
|
87
112
|
mkdirSync(dir, { recursive: true });
|
|
88
|
-
const pg = await new PGlite({
|
|
113
|
+
const pg = await new mod.PGlite({
|
|
89
114
|
dataDir: dir,
|
|
90
|
-
extensions: { vector },
|
|
115
|
+
extensions: { vector: mod.vector },
|
|
91
116
|
});
|
|
92
117
|
await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
|
|
93
118
|
await pg.exec(`
|
|
@@ -18,17 +18,15 @@
|
|
|
18
18
|
import { homedir } from "node:os";
|
|
19
19
|
import { join } from "node:path";
|
|
20
20
|
import { mkdirSync, rmSync, existsSync } from "node:fs";
|
|
21
|
-
// PGlite + pgvector are script-free WASM (no native build) → survive pi's
|
|
22
|
-
// install-script block. Imported lazily so a missing/broken package degrades
|
|
23
|
-
// gracefully instead of crashing module load.
|
|
24
|
-
import { PGlite } from "@electric-sql/pglite";
|
|
25
|
-
import { vector } from "@electric-sql/pglite-pgvector";
|
|
26
21
|
/** Vector dimension produced by the default TrigramEmbedder (src/embedder.ts). */
|
|
27
22
|
export const EMBEDDING_DIM = 512;
|
|
28
23
|
let db;
|
|
29
24
|
let initPromise;
|
|
30
25
|
let disabled = false;
|
|
31
26
|
let warned = false;
|
|
27
|
+
/** Lazily-loaded PGlite module + pgvector extension (see loadPgLite). */
|
|
28
|
+
let pgliteMod;
|
|
29
|
+
let pgliteLoadFailed = false;
|
|
32
30
|
function indexDir() {
|
|
33
31
|
const override = process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
34
32
|
if (override && override.trim() !== "")
|
|
@@ -73,6 +71,30 @@ export function initVectorIndex() {
|
|
|
73
71
|
initPromise = openPgLite(/* retryOnCorrupt */ true);
|
|
74
72
|
return initPromise;
|
|
75
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Lazily load the PGlite module + pgvector extension via dynamic import. Caches
|
|
76
|
+
* success and permanent failure. Returns undefined (once, then forever) when the
|
|
77
|
+
* package is missing/broken so callers fall back to the sync scan. Never throws.
|
|
78
|
+
*/
|
|
79
|
+
async function loadPgLite() {
|
|
80
|
+
if (pgliteMod)
|
|
81
|
+
return pgliteMod;
|
|
82
|
+
if (pgliteLoadFailed)
|
|
83
|
+
return undefined;
|
|
84
|
+
try {
|
|
85
|
+
const [pglitePkg, pgvectorPkg] = await Promise.all([
|
|
86
|
+
import("@electric-sql/pglite"),
|
|
87
|
+
import("@electric-sql/pglite-pgvector"),
|
|
88
|
+
]);
|
|
89
|
+
pgliteMod = { PGlite: pglitePkg.PGlite, vector: pgvectorPkg.vector };
|
|
90
|
+
return pgliteMod;
|
|
91
|
+
}
|
|
92
|
+
catch (err) {
|
|
93
|
+
pgliteLoadFailed = true;
|
|
94
|
+
logWarn(`package unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
95
|
+
return undefined;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
76
98
|
/**
|
|
77
99
|
* Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level
|
|
78
100
|
* abort (typically from a corrupted/torn data dir) triggers a delete + one
|
|
@@ -80,11 +102,14 @@ export function initVectorIndex() {
|
|
|
80
102
|
*/
|
|
81
103
|
async function openPgLite(retryOnCorrupt) {
|
|
82
104
|
try {
|
|
105
|
+
const mod = await loadPgLite();
|
|
106
|
+
if (!mod)
|
|
107
|
+
return undefined;
|
|
83
108
|
const dir = indexDir();
|
|
84
109
|
mkdirSync(dir, { recursive: true });
|
|
85
|
-
const pg = await new PGlite({
|
|
110
|
+
const pg = await new mod.PGlite({
|
|
86
111
|
dataDir: dir,
|
|
87
|
-
extensions: { vector },
|
|
112
|
+
extensions: { vector: mod.vector },
|
|
88
113
|
});
|
|
89
114
|
await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
|
|
90
115
|
await pg.exec(`
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* conflict-scan.test.ts — unit tests for the extension-conflict scanner.
|
|
3
|
+
*
|
|
4
|
+
* Fixture trees are written under a temp dir and scanned via
|
|
5
|
+
* MEGACOMPACT_EXT_SCAN_DIR (which makes collectScanRoots() return that
|
|
6
|
+
* single root). This covers the S24 follow-up fix:
|
|
7
|
+
*
|
|
8
|
+
* 1. node_modules-style code extensions (package.json + pi.extensions) are
|
|
9
|
+
* still detected by source-marker grep (regression).
|
|
10
|
+
* 2. USER-LEVEL extensions installed outside npm (e.g. pi-hermes-memory)
|
|
11
|
+
* now get scanned too — previously only `node_modules` was walked, so a
|
|
12
|
+
* data-only memory store (MEMORY.md + sessions.db, no package.json)
|
|
13
|
+
* was never flagged (the 5000-char file-buffer error slipped through).
|
|
14
|
+
* 3. The data-only memory-store signature is detected even with no source.
|
|
15
|
+
* 4. pi-mega-compact (selfName) is always skipped.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { test, after } from "node:test";
|
|
19
|
+
import assert from "node:assert/strict";
|
|
20
|
+
import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs";
|
|
21
|
+
import { tmpdir } from "node:os";
|
|
22
|
+
import { join } from "node:path";
|
|
23
|
+
import { detectConflicts, collectScanRoots } from "./conflict-scan.js";
|
|
24
|
+
|
|
25
|
+
const base = mkdtempSync(join(tmpdir(), "mc-scan-"));
|
|
26
|
+
let n = 0;
|
|
27
|
+
|
|
28
|
+
/** Make a fixture root containing one or more fake extensions, return its path. */
|
|
29
|
+
function fixture(build: (root: string) => void): string {
|
|
30
|
+
const root = join(base, `case-${n++}`);
|
|
31
|
+
mkdirSync(root, { recursive: true });
|
|
32
|
+
build(root);
|
|
33
|
+
return root;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
after(() => {
|
|
37
|
+
rmSync(base, { recursive: true, force: true });
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("scans a user-level, data-only memory store (no package.json)", () => {
|
|
41
|
+
const root = fixture((r) => {
|
|
42
|
+
const ext = join(r, "pi-hermes-memory");
|
|
43
|
+
mkdirSync(ext, { recursive: true });
|
|
44
|
+
// No package.json, no source — just pi's memory-store signature.
|
|
45
|
+
writeFileSync(join(ext, "MEMORY.md"), "# memory\n");
|
|
46
|
+
writeFileSync(join(ext, "sessions.db"), "");
|
|
47
|
+
});
|
|
48
|
+
process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
|
|
49
|
+
try {
|
|
50
|
+
const { conflicts } = detectConflicts();
|
|
51
|
+
assert.ok(conflicts.length >= 1, "expected a memory conflict");
|
|
52
|
+
const hit = conflicts.find((c) => c.kind === "memory");
|
|
53
|
+
assert.ok(hit, "expected a memory-kind conflict");
|
|
54
|
+
assert.equal(hit!.severity, "high");
|
|
55
|
+
assert.ok(
|
|
56
|
+
hit!.evidence.includes("MEMORY.md") ||
|
|
57
|
+
hit!.evidence.includes("sessions.db"),
|
|
58
|
+
"evidence should name the on-disk memory signature",
|
|
59
|
+
);
|
|
60
|
+
} finally {
|
|
61
|
+
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("still detects a code extension by source marker (regression)", () => {
|
|
66
|
+
const root = fixture((r) => {
|
|
67
|
+
// A code extension is a DIRECT child of the scan root (mirrors the
|
|
68
|
+
// node_modules layout: packages live one level under the root).
|
|
69
|
+
const ext = join(r, "some-memory-ext");
|
|
70
|
+
mkdirSync(ext, { recursive: true });
|
|
71
|
+
writeFileSync(
|
|
72
|
+
join(ext, "package.json"),
|
|
73
|
+
JSON.stringify({ name: "some-memory-ext", pi: { extensions: ["x.ts"] } }),
|
|
74
|
+
);
|
|
75
|
+
writeFileSync(join(ext, "index.ts"), "export const MEMORY_TOOL = true;");
|
|
76
|
+
});
|
|
77
|
+
process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
|
|
78
|
+
try {
|
|
79
|
+
const { conflicts } = detectConflicts();
|
|
80
|
+
const hit = conflicts.find((c) => c.package === "some-memory-ext");
|
|
81
|
+
assert.ok(hit, "expected some-memory-ext to be flagged");
|
|
82
|
+
assert.equal(hit!.kind, "memory");
|
|
83
|
+
} finally {
|
|
84
|
+
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test("skips pi-mega-compact (selfName) and non-extension dirs", () => {
|
|
89
|
+
const root = fixture((r) => {
|
|
90
|
+
// selfName dir with a memory signature — must be ignored.
|
|
91
|
+
const me = join(r, "node_modules", "pi-mega-compact");
|
|
92
|
+
mkdirSync(me, { recursive: true });
|
|
93
|
+
writeFileSync(join(me, "sessions.db"), "");
|
|
94
|
+
// unrelated dir with no pi.extensions and no memory signature.
|
|
95
|
+
mkdirSync(join(r, "node_modules", "totally-fine"), { recursive: true });
|
|
96
|
+
});
|
|
97
|
+
process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
|
|
98
|
+
try {
|
|
99
|
+
const { scanned, conflicts } = detectConflicts();
|
|
100
|
+
assert.equal(conflicts.length, 0, "no conflicts expected");
|
|
101
|
+
assert.ok(
|
|
102
|
+
!scanned.some((s) => s.includes("pi-mega-compact")),
|
|
103
|
+
"selfName should not appear in scanned",
|
|
104
|
+
);
|
|
105
|
+
} finally {
|
|
106
|
+
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
test("collectScanRoots honors MEGACOMPACT_EXT_SCAN_DIR override", () => {
|
|
111
|
+
const root = fixture(() => {});
|
|
112
|
+
process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
|
|
113
|
+
try {
|
|
114
|
+
const roots = collectScanRoots();
|
|
115
|
+
assert.deepEqual(roots, [root], "override replaces the whole root list");
|
|
116
|
+
} finally {
|
|
117
|
+
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
118
|
+
}
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
test("collectScanRoots falls back to node_modules + user dir when no override", () => {
|
|
122
|
+
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
123
|
+
delete process.env.MEGACOMPACT_EXT_USER_DIR;
|
|
124
|
+
// No override set and this test file lives under extensions/, so node_modules
|
|
125
|
+
// resolution walks up from here; the user dir (~/.pi/agent) may or may
|
|
126
|
+
// not exist in CI. We only assert the call returns a non-throwing array.
|
|
127
|
+
const roots = collectScanRoots();
|
|
128
|
+
assert.ok(Array.isArray(roots), "collectScanRoots must return an array");
|
|
129
|
+
});
|