pi-mega-compact 0.6.2 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -23
- package/dist/extensions/conflict-scan.js +133 -60
- package/dist/extensions/conflict-scan.test.js +115 -0
- package/dist/extensions/dashboard-server.js +101 -35
- package/dist/extensions/dashboard-server.test.js +2 -2
- package/dist/extensions/mega-runtime.js +47 -20
- package/dist/src/store/memoryIndex.js +32 -7
- package/dist/src/store/vectorIndex.js +32 -7
- package/extensions/conflict-scan.test.ts +129 -0
- package/extensions/conflict-scan.ts +243 -158
- package/extensions/dashboard-server.test.ts +2 -2
- package/extensions/dashboard-server.ts +106 -36
- package/extensions/mega-dashboard.ts +16 -0
- package/extensions/mega-runtime.ts +48 -20
- package/package.json +1 -1
- package/src/store/memoryIndex.ts +40 -6
- package/src/store/vectorIndex.ts +40 -6
- package/dist/extensions/openclaw-mega-compact.js +0 -291
- package/dist/src/minilm.js +0 -92
- package/dist/src/wordpiece.js +0 -129
|
@@ -59,6 +59,7 @@ function getIndexDir(): string {
|
|
|
59
59
|
interface IndexRepo {
|
|
60
60
|
repoRoot: string;
|
|
61
61
|
displayName: string;
|
|
62
|
+
stateDir: string;
|
|
62
63
|
checkpointCount: number;
|
|
63
64
|
tokensSaved: number;
|
|
64
65
|
compressedOriginalBytes: number;
|
|
@@ -71,8 +72,17 @@ interface IndexRepo {
|
|
|
71
72
|
lastSeen: number;
|
|
72
73
|
}
|
|
73
74
|
|
|
75
|
+
interface IndexSummary {
|
|
76
|
+
totalRepos: number;
|
|
77
|
+
totalCheckpoints: number;
|
|
78
|
+
totalTokensSaved: number;
|
|
79
|
+
totalCompressedOriginalBytes: number;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
type IndexIndex = { updatedAt: string; summary: IndexSummary | null; repos: IndexRepo[] };
|
|
83
|
+
|
|
74
84
|
/** Read the machine-wide repo registry from SQLite (read-only, single shot). */
|
|
75
|
-
function readIndex():
|
|
85
|
+
function readIndex(): IndexIndex | null {
|
|
76
86
|
const indexPath = join(getIndexDir(), "index.sqlite");
|
|
77
87
|
if (!existsSync(indexPath)) return null;
|
|
78
88
|
let db: DatabaseSync | undefined;
|
|
@@ -86,6 +96,7 @@ function readIndex(): { updatedAt: string; summary: unknown; repos: unknown[] }
|
|
|
86
96
|
const mapped: IndexRepo[] = rows.map((r) => ({
|
|
87
97
|
repoRoot: String(r.repo_root ?? ""),
|
|
88
98
|
displayName: String(r.display_name ?? ""),
|
|
99
|
+
stateDir: String(r.state_dir ?? ""),
|
|
89
100
|
checkpointCount: Number(r.checkpoint_count ?? 0),
|
|
90
101
|
tokensSaved: Number(r.tokens_saved ?? 0),
|
|
91
102
|
compressedOriginalBytes: Number(r.compressed_original_bytes ?? 0),
|
|
@@ -197,6 +208,10 @@ interface Snapshot {
|
|
|
197
208
|
duplicatesCollapsed: number;
|
|
198
209
|
bytesPermanentlyDeleted: number;
|
|
199
210
|
};
|
|
211
|
+
compression: {
|
|
212
|
+
session: { tokensIn: number; tokensOut: number; tokensFreed: number; compressionPct: number; dedupPct: number };
|
|
213
|
+
repo: { tokensIn: number; tokensOut: number; tokensFreed: number; compressionPct: number; dedupPct: number };
|
|
214
|
+
};
|
|
200
215
|
model?: {
|
|
201
216
|
name: string;
|
|
202
217
|
provider: string;
|
|
@@ -374,27 +389,31 @@ function dashboardHtml(tierName: string): string {
|
|
|
374
389
|
<h2>Vector Store</h2>
|
|
375
390
|
<div class="stat-grid">
|
|
376
391
|
<span class="label" title="A saved summary of a chunk of your conversation that was compacted to free up space.">Checkpoints</span><span class="value" id="st-count">0</span>
|
|
377
|
-
<span class="label" title="
|
|
378
|
-
<span class="label" title="
|
|
379
|
-
<span class="label" title="
|
|
392
|
+
<span class="label" title="Total size of the original conversation text dropped into compaction this session, including redundant regions skipped by dedup. This is the 'in'.">Original (dropped)</span><span class="value" id="st-in">0</span>
|
|
393
|
+
<span class="label" title="Compact summaries we are currently holding as 'memory' for this session (the 'out'). Smaller is better.">Kept (summaries)</span><span class="value" id="st-kept">0</span>
|
|
394
|
+
<span class="label" title="Conversation space freed = dropped − kept (the 'saved').">Freed (dropped − kept)</span><span class="value" id="st-freed">0</span>
|
|
380
395
|
<span class="label" title="How many times old context was automatically brought back into the conversation because it was relevant to what you were doing.">Injected</span><span class="value" id="st-injected">0</span>
|
|
381
396
|
<span class="label" title="Of the times we recalled old context, how often it was actually on-topic.">Recall Relevance</span><span class="value" id="st-dedup">0%</span>
|
|
382
397
|
<span class="label" title="How often new content matched something we already had, so we skipped storing a duplicate copy. Higher = less wasted space.">Storage Dedup</span><span class="value" id="st-sdedup">0%</span>
|
|
383
398
|
<span class="label" title="How many duplicate chunks we collapsed into one instead of storing separately.">Collapsed</span><span class="value" id="st-collapsed">0</span>
|
|
384
399
|
<span class="label" title="The ID of the most recent saved checkpoint.">Last ID</span><span class="value" id="st-lastid">—</span>
|
|
385
400
|
</div>
|
|
401
|
+
<div class="meter-track" style="margin-top:10px"><div class="meter-fill" id="st-compress-bar" style="width:0%"></div></div>
|
|
402
|
+
<div class="meter-sub" id="st-compress-sub">waiting for compaction…</div>
|
|
386
403
|
</div>
|
|
387
404
|
<div class="card">
|
|
388
405
|
<h2>Repo (all sessions)</h2>
|
|
389
406
|
<div class="stat-grid">
|
|
390
407
|
<span class="label">Checkpoints</span><span class="value" id="rp-count">0</span>
|
|
391
|
-
<span class="label">
|
|
392
|
-
<span class="label">
|
|
393
|
-
<span class="label">
|
|
408
|
+
<span class="label">Original (dropped)</span><span class="value" id="rp-in">0</span>
|
|
409
|
+
<span class="label">Kept (summaries)</span><span class="value" id="rp-kept">0</span>
|
|
410
|
+
<span class="label">Freed (dropped − kept)</span><span class="value" id="rp-freed">0</span>
|
|
394
411
|
<span class="label">Sessions</span><span class="value" id="rp-sessions">0</span>
|
|
395
412
|
<span class="label">Collapsed</span><span class="value" id="rp-collapsed">0</span>
|
|
396
413
|
<span class="label">Storage Dedup</span><span class="value" id="rp-sdedup">0%</span>
|
|
397
414
|
</div>
|
|
415
|
+
<div class="meter-track" style="margin-top:10px"><div class="meter-fill" id="rp-compress-bar" style="width:0%"></div></div>
|
|
416
|
+
<div class="meter-sub" id="rp-compress-sub">waiting for compaction…</div>
|
|
398
417
|
</div>
|
|
399
418
|
<div class="card safe">
|
|
400
419
|
<h2>🛡 Data Safety</h2>
|
|
@@ -440,11 +459,11 @@ function dashboardHtml(tierName: string): string {
|
|
|
440
459
|
<div class="card legend">
|
|
441
460
|
<h2>What these numbers mean</h2>
|
|
442
461
|
<ul class="legend-list">
|
|
443
|
-
<li><b>
|
|
444
|
-
<li><b>
|
|
445
|
-
<li><b>
|
|
446
|
-
<li><b>
|
|
447
|
-
<li><b>Storage dedup
|
|
462
|
+
<li><b>Original (dropped)</b> — everything compacted away (including duplicates caught by dedup). The "in."</li>
|
|
463
|
+
<li><b>Kept (summaries)</b> — compact summaries still held as "memory" (the "out").</li>
|
|
464
|
+
<li><b>Freed</b> = dropped − kept — tokens saved so far (higher = better).</li>
|
|
465
|
+
<li><b>Compression %</b> — Freed ÷ Dropped — the headline efficiency number. Higher = more space reclaimed.</li>
|
|
466
|
+
<li><b>Storage dedup %</b> — how often new content matched something already saved, so no duplicate copy was written.</li>
|
|
448
467
|
<li><b>Data safety</b> — every compacted region is kept verbatim (compressed). Nothing is permanently deleted; you can restore any of it.</li>
|
|
449
468
|
</ul>
|
|
450
469
|
<p class="legend-note">Hover any label above for a quick explanation.</p>
|
|
@@ -548,10 +567,20 @@ function dashboardHtml(tierName: string): string {
|
|
|
548
567
|
d.trigger.armed ? 'past fast gate — monitoring token count' : 'idle — below fast gate';
|
|
549
568
|
document.getElementById('tr-state').textContent = state;
|
|
550
569
|
|
|
570
|
+
// ---- Vector Store — reconciled token accounting (same formula as widget) -
|
|
551
571
|
document.getElementById('st-count').textContent = d.store.checkpointCount;
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
572
|
+
// Compression block from the snapshot (Freed = In − Out, single formula).
|
|
573
|
+
var c = d.compression || {};
|
|
574
|
+
var sess = c.session || { tokensIn:0, tokensOut:0, tokensFreed:0, compressionPct:0, dedupPct:0 };
|
|
575
|
+
var cRepo = c.repo || { tokensIn:0, tokensOut:0, tokensFreed:0, compressionPct:0, dedupPct:0 };
|
|
576
|
+
document.getElementById('st-in').textContent = sess.tokensIn.toLocaleString();
|
|
577
|
+
document.getElementById('st-kept').textContent = sess.tokensOut.toLocaleString();
|
|
578
|
+
document.getElementById('st-freed').textContent = sess.tokensFreed.toLocaleString();
|
|
579
|
+
var sp = sess.compressionPct || 0;
|
|
580
|
+
document.getElementById('st-compress-bar').style.width = Math.max(sp * 100, 0.5) + '%';
|
|
581
|
+
document.getElementById('st-compress-bar').className = 'meter-fill ' + (sp >= 0.9 ? 'meter-green' : sp >= 0.6 ? 'meter-yellow' : 'meter-red');
|
|
582
|
+
document.getElementById('st-compress-sub').textContent = (sp * 100 >= 10 ? Math.round(sp * 100) : (sp * 100).toFixed(1)) + '% tokens saved · dedup: ' + (sess.dedupPct * 100 >= 10 ? Math.round(sess.dedupPct * 100) : (sess.dedupPct * 100).toFixed(1)) + '%';
|
|
583
|
+
// ------
|
|
555
584
|
document.getElementById('st-injected').textContent = d.store.injectedCount;
|
|
556
585
|
document.getElementById('st-dedup').textContent = Math.round(d.store.dedupHitRate * 100) + '%';
|
|
557
586
|
var sdr = d.store.storageDedupRate || 0;
|
|
@@ -559,16 +588,19 @@ function dashboardHtml(tierName: string): string {
|
|
|
559
588
|
document.getElementById('st-collapsed').textContent = d.store.dedupCollapsed || 0;
|
|
560
589
|
document.getElementById('st-lastid').textContent = d.session.lastCheckpointId || '—';
|
|
561
590
|
|
|
562
|
-
// Repo
|
|
563
|
-
|
|
564
|
-
document.getElementById('rp-
|
|
565
|
-
document.getElementById('rp-
|
|
566
|
-
document.getElementById('rp-
|
|
567
|
-
document.getElementById('rp-
|
|
568
|
-
document.getElementById('rp-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
591
|
+
// ---- Repo (all sessions) — same compression fields, repo scope ----------
|
|
592
|
+
document.getElementById('rp-count').textContent = (d.repo && d.repo.checkpointCount || 0).toLocaleString();
|
|
593
|
+
document.getElementById('rp-in').textContent = cRepo.tokensIn.toLocaleString();
|
|
594
|
+
document.getElementById('rp-kept').textContent = cRepo.tokensOut.toLocaleString();
|
|
595
|
+
document.getElementById('rp-freed').textContent = cRepo.tokensFreed.toLocaleString();
|
|
596
|
+
document.getElementById('rp-sessions').textContent = (d.repo && d.repo.sessionCount || 0).toLocaleString();
|
|
597
|
+
document.getElementById('rp-collapsed').textContent = (d.repo && d.repo.dedupCollapsed || 0).toLocaleString();
|
|
598
|
+
var rdr = d.repo && d.repo.storageDedupRate || 0;
|
|
599
|
+
document.getElementById('rp-sdedup').textContent = (rdr * 100 >= 10 ? Math.round(rdr * 100) : (rdr * 100).toFixed(1)) + '%';
|
|
600
|
+
var rp = cRepo.compressionPct || 0;
|
|
601
|
+
document.getElementById('rp-compress-bar').style.width = Math.max(rp * 100, 0.5) + '%';
|
|
602
|
+
document.getElementById('rp-compress-bar').className = 'meter-fill ' + (rp >= 0.9 ? 'meter-green' : rp >= 0.6 ? 'meter-yellow' : 'meter-red');
|
|
603
|
+
document.getElementById('rp-compress-sub').textContent = (rp * 100 >= 10 ? Math.round(rp * 100) : (rp * 100).toFixed(1)) + '% tokens saved · dedup: ' + (cRepo.dedupPct * 100 >= 10 ? Math.round(cRepo.dedupPct * 100) : (cRepo.dedupPct * 100).toFixed(1)) + '%';
|
|
572
604
|
|
|
573
605
|
// Data-safety invariant (Phase 0 — trust foundation).
|
|
574
606
|
var ig = d.integrity || { regionsRetained: 0, compressedOriginalBytes: 0, duplicatesCollapsed: 0, bytesPermanentlyDeleted: 0 };
|
|
@@ -608,10 +640,11 @@ function dashboardHtml(tierName: string): string {
|
|
|
608
640
|
document.getElementById('md-provider').textContent = model && model.providerName ? model.providerName : (model && model.provider ? model.provider : '—');
|
|
609
641
|
document.getElementById('md-input').textContent = model && model.inputRate ? '$' + (model.inputRate).toFixed(6) : '—';
|
|
610
642
|
document.getElementById('md-output').textContent = model && model.outputRate ? '$' + (model.outputRate).toFixed(6) : '—';
|
|
611
|
-
|
|
612
|
-
|
|
643
|
+
var repoSaved = cRepo.tokensFreed || 0;
|
|
644
|
+
if (model && model.inputRate && repoSaved > 0) {
|
|
645
|
+
var usd = (repoSaved * model.inputRate);
|
|
613
646
|
var win = d.context.contextWindow || 0;
|
|
614
|
-
var windows = win > 0 ? (
|
|
647
|
+
var windows = win > 0 ? (repoSaved / win).toFixed(1) : '0';
|
|
615
648
|
document.getElementById('cost-usd').textContent = '≈ $' + usd.toFixed(4) + ' saved';
|
|
616
649
|
document.getElementById('cost-windows').textContent = windows + ' context-windows extended';
|
|
617
650
|
} else {
|
|
@@ -838,7 +871,40 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
838
871
|
|
|
839
872
|
let eventOffset = 0;
|
|
840
873
|
|
|
874
|
+
// Overlay the live current-repo snapshot (snapshot.json, rewritten every
|
|
875
|
+
// context event) onto its registry row so the All-repos / Summary views stay
|
|
876
|
+
// in sync with the live menu bar + Current-repo card in real time. The
|
|
877
|
+
// registry (index.sqlite) is only written on repo-switch (bindRepo), so
|
|
878
|
+
// without this the current repo's row freezes between switches. Read-only —
|
|
879
|
+
// no extra writes to index.sqlite. Matched by stateDir, which equals the
|
|
880
|
+
// value this server was launched with (runtime.currentStateDir).
|
|
881
|
+
function overlayCurrentRepo(idx: IndexIndex | null): void {
|
|
882
|
+
if (!idx || !idx.repos.length) return;
|
|
883
|
+
let snap: Snapshot | null = null;
|
|
884
|
+
try { snap = readSnapshot(snapshotPath); } catch { return; }
|
|
885
|
+
if (!snap || !snap.repo) return;
|
|
886
|
+
const cur = idx.repos.find((r) => r.stateDir === stateDir);
|
|
887
|
+
if (!cur) return;
|
|
888
|
+
const prevSaved = cur.tokensSaved;
|
|
889
|
+
const prevCp = cur.checkpointCount;
|
|
890
|
+
const prevBytes = cur.compressedOriginalBytes;
|
|
891
|
+
const comp = snap.compression?.repo;
|
|
892
|
+
const liveSaved = comp ? comp.tokensFreed : (snap.repo.tokensSaved ?? prevSaved);
|
|
893
|
+
const liveCp = snap.repo.checkpointCount ?? prevCp;
|
|
894
|
+
const liveBytes = snap.integrity?.compressedOriginalBytes ?? prevBytes;
|
|
895
|
+
cur.tokensSaved = liveSaved;
|
|
896
|
+
cur.checkpointCount = liveCp;
|
|
897
|
+
cur.compressedOriginalBytes = liveBytes;
|
|
898
|
+
if (idx.summary) {
|
|
899
|
+
idx.summary.totalTokensSaved += liveSaved - prevSaved;
|
|
900
|
+
idx.summary.totalCheckpoints += liveCp - prevCp;
|
|
901
|
+
idx.summary.totalCompressedOriginalBytes += liveBytes - prevBytes;
|
|
902
|
+
}
|
|
903
|
+
idx.updatedAt = snap.updatedAt ?? idx.updatedAt;
|
|
904
|
+
}
|
|
905
|
+
|
|
841
906
|
const server = createServer((req: IncomingMessage, res: ServerResponse) => {
|
|
907
|
+
// guardrails-allow PREVENT-PI-004: optional, user-triggered /dashboard localhost server (loopback-only) — CORS open for local browser access
|
|
842
908
|
// CORS for local access
|
|
843
909
|
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
844
910
|
res.setHeader("Access-Control-Allow-Methods", "GET, OPTIONS");
|
|
@@ -876,8 +942,10 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
876
942
|
// directly from SQLite (index.sqlite). Lets one dashboard show every repo's
|
|
877
943
|
// checkpoints, tokens saved, and active model. Read-only.
|
|
878
944
|
if (req.url === "/api/index") {
|
|
945
|
+
const idx = readIndex();
|
|
946
|
+
if (idx) overlayCurrentRepo(idx);
|
|
879
947
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
880
|
-
res.end(JSON.stringify(
|
|
948
|
+
res.end(JSON.stringify(idx ?? { updatedAt: null, summary: null, repos: [] }));
|
|
881
949
|
return;
|
|
882
950
|
}
|
|
883
951
|
|
|
@@ -887,8 +955,9 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
887
955
|
if (req.url?.startsWith("/api/repos")) {
|
|
888
956
|
const url = new URL(req.url, "http://x");
|
|
889
957
|
const activeParam = url.searchParams.get("active");
|
|
890
|
-
const idx = readIndex()
|
|
891
|
-
|
|
958
|
+
const idx = readIndex();
|
|
959
|
+
if (idx) overlayCurrentRepo(idx);
|
|
960
|
+
let repos = idx?.repos ?? [];
|
|
892
961
|
if (activeParam) {
|
|
893
962
|
const m = /^(\d+)h$/.exec(activeParam);
|
|
894
963
|
if (m) {
|
|
@@ -897,7 +966,7 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
897
966
|
}
|
|
898
967
|
}
|
|
899
968
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
900
|
-
res.end(JSON.stringify({ updatedAt: idx
|
|
969
|
+
res.end(JSON.stringify({ updatedAt: idx?.updatedAt ?? null, repos, count: repos.length }));
|
|
901
970
|
return;
|
|
902
971
|
}
|
|
903
972
|
|
|
@@ -905,14 +974,15 @@ export async function launchDashboardServer(stateDir: string): Promise<{ port: n
|
|
|
905
974
|
// small for embed scenarios). activeRepos mirrors the /api/repos?active=24h
|
|
906
975
|
// count so the dashboard can render the active badge alongside totals.
|
|
907
976
|
if (req.url?.startsWith("/api/summary")) {
|
|
908
|
-
const idx = readIndex()
|
|
909
|
-
|
|
977
|
+
const idx = readIndex();
|
|
978
|
+
if (idx) overlayCurrentRepo(idx);
|
|
979
|
+
const repos = idx?.repos ?? [];
|
|
910
980
|
const cutoffSec = Math.floor(Date.now() / 1000) - 24 * 3600;
|
|
911
981
|
const activeRepos = repos.filter((r) => (r.lastSeen ?? 0) >= cutoffSec).length;
|
|
912
982
|
res.writeHead(200, { "Content-Type": "application/json" });
|
|
913
983
|
res.end(JSON.stringify({
|
|
914
|
-
updatedAt: idx
|
|
915
|
-
summary: idx
|
|
984
|
+
updatedAt: idx?.updatedAt ?? null,
|
|
985
|
+
summary: idx?.summary ?? null,
|
|
916
986
|
activeRepos,
|
|
917
987
|
totalRepos: repos.length,
|
|
918
988
|
}));
|
|
@@ -80,6 +80,22 @@ export interface DashboardSnapshot {
|
|
|
80
80
|
dedupCollapsed: number; // cumulative deduped collapses (store-wide)
|
|
81
81
|
storageDedupRate: number; // deduped / attempts, 0..1
|
|
82
82
|
};
|
|
83
|
+
/**
|
|
84
|
+
* Reconciled token accounting — ONE canonical formula for both session + repo
|
|
85
|
+
* so the dashboard and widget tell the same story:
|
|
86
|
+
* tokensIn = original conversation dropped into compaction (incl. the
|
|
87
|
+
* redundant regions skipped by dedup) — the "in".
|
|
88
|
+
* tokensOut = compact summaries currently held (stored) — the "out".
|
|
89
|
+
* tokensFreed= tokensIn − tokensOut (the "saved").
|
|
90
|
+
* compressionPct = tokensFreed / tokensIn (0..1) — the headline "% saved".
|
|
91
|
+
* dedupPct = storageDedupRate (0..1) — share of adds that collapsed.
|
|
92
|
+
* session.Freed = rt.tokensSaved (honest net freed this session, incl. deduped-away);
|
|
93
|
+
* repo.Freed = repo.tokensSaved meta (honest net freed repo-wide).
|
|
94
|
+
*/
|
|
95
|
+
compression: {
|
|
96
|
+
session: { tokensIn: number; tokensOut: number; tokensFreed: number; compressionPct: number; dedupPct: number };
|
|
97
|
+
repo: { tokensIn: number; tokensOut: number; tokensFreed: number; compressionPct: number; dedupPct: number };
|
|
98
|
+
};
|
|
83
99
|
/** Phase 0 data-safety invariant (trust foundation). */
|
|
84
100
|
integrity: {
|
|
85
101
|
regionsRetained: number; // checkpoints with a recoverable compressed-original
|
|
@@ -282,6 +282,25 @@ export class MegaRuntime {
|
|
|
282
282
|
trigger: { armed, ready, currentTokens: this.lastCtxTokens, thresholdTokens: this.config.thresholdTokens, fastGatePct: this.config.fastGatePct },
|
|
283
283
|
crew: { activeAgents: this.activeAgents, currentTurn: this.currentTurn },
|
|
284
284
|
store: { checkpointCount: st.checkpointCount, totalTokenEstimate: st.totalTokenEstimate, originalTokens: st.originalTokens, tokensSaved: this.rt.tokensSaved, injectedCount: st.injectedCount, dedupHitRate: st.dedupHitRate, storageDedupRate: st.storageDedupRate, dedupAttempts: st.dedupAttempts, dedupCollapsed: st.dedupCollapsed },
|
|
285
|
+
// Reconciled token accounting (single canonical formula, session + repo).
|
|
286
|
+
// Freed = In − Out; In = Freed + Out. session.Freed = rt.tokensSaved (incl.
|
|
287
|
+
// deduped-away originals); repo.Freed = repo.tokensSaved meta counter.
|
|
288
|
+
compression: {
|
|
289
|
+
session: {
|
|
290
|
+
tokensIn: this.rt.tokensSaved + st.totalTokenEstimate,
|
|
291
|
+
tokensOut: st.totalTokenEstimate,
|
|
292
|
+
tokensFreed: this.rt.tokensSaved,
|
|
293
|
+
compressionPct: (this.rt.tokensSaved + st.totalTokenEstimate) > 0 ? this.rt.tokensSaved / (this.rt.tokensSaved + st.totalTokenEstimate) : 0,
|
|
294
|
+
dedupPct: st.storageDedupRate,
|
|
295
|
+
},
|
|
296
|
+
repo: {
|
|
297
|
+
tokensIn: repo.tokensSaved + repo.totalTokenEstimate,
|
|
298
|
+
tokensOut: repo.totalTokenEstimate,
|
|
299
|
+
tokensFreed: repo.tokensSaved,
|
|
300
|
+
compressionPct: (repo.tokensSaved + repo.totalTokenEstimate) > 0 ? repo.tokensSaved / (repo.tokensSaved + repo.totalTokenEstimate) : 0,
|
|
301
|
+
dedupPct: repo.storageDedupRate,
|
|
302
|
+
},
|
|
303
|
+
},
|
|
285
304
|
repo: {
|
|
286
305
|
checkpointCount: repo.checkpointCount,
|
|
287
306
|
totalTokenEstimate: repo.totalTokenEstimate,
|
|
@@ -319,33 +338,42 @@ export class MegaRuntime {
|
|
|
319
338
|
const dedupStr = storageRate * 100 >= 10
|
|
320
339
|
? `${Math.round(storageRate * 100)}%`
|
|
321
340
|
: `${(storageRate * 100).toFixed(1)}%`;
|
|
322
|
-
//
|
|
323
|
-
//
|
|
324
|
-
//
|
|
325
|
-
//
|
|
326
|
-
//
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
341
|
+
// Reconciled token accounting — ONE canonical formula for session + repo,
|
|
342
|
+
// matching the dashboard so the two never disagree. unit format: M at/above
|
|
343
|
+
// 1e6, k at/above 1e3, raw below — so 5,472,700 → "5.5M", 24,100 → "24.1k",
|
|
344
|
+
// 142 → "142". Dropped (in) = Freed + Kept; Freed = rt.tokensSaved (session)
|
|
345
|
+
// / repo.tokensSaved meta (repo); Kept = totalTokenEstimate (stored).
|
|
346
|
+
const fmt = (x: number) =>
|
|
347
|
+
x >= 1_000_000 ? `${(x / 1_000_000).toFixed(1)}M`
|
|
348
|
+
: x >= 1000 ? `${(x / 1000).toFixed(1)}k`
|
|
349
|
+
: `${Math.round(x)}`;
|
|
331
350
|
const agentStr = this.activeAgents > 0 ? ` │ 🤖 ${this.activeAgents} agent${this.activeAgents === 1 ? "" : "s"}` : "";
|
|
332
351
|
const turnStr = this.currentTurn > 0 ? ` │ turn ${this.currentTurn}` : "";
|
|
333
352
|
// Phase 3 — pulsing status glyph while a compaction is in flight.
|
|
334
353
|
const pulse = this.pulsing ? `${C.cyan}${PULSE[Math.floor(Date.now() / 250) % PULSE.length]}${C.reset} ` : "";
|
|
354
|
+
// --- reconciled in/out view (session + repo) ---------------------------
|
|
355
|
+
const sessIn = this.rt.tokensSaved + st.totalTokenEstimate;
|
|
356
|
+
const sessKept = st.totalTokenEstimate;
|
|
357
|
+
const sessFreed = this.rt.tokensSaved;
|
|
358
|
+
const sessPct = sessIn > 0 ? sessFreed / sessIn : 0;
|
|
359
|
+
const repoIn = repo.tokensSaved + repo.totalTokenEstimate;
|
|
360
|
+
const repoKept = repo.totalTokenEstimate;
|
|
361
|
+
const repoFreed = repo.tokensSaved;
|
|
362
|
+
const repoPct = repoIn > 0 ? repoFreed / repoIn : 0;
|
|
335
363
|
const lines = [
|
|
336
364
|
` ${C.amber}⚡ ${tierLabel}${C.reset} v${C.bold}${ownVersion()}${C.reset} │ ${tokStr}/${maxStr} tokens (${C.bold}${pctStr}${C.reset}) │ ${st.checkpointCount} saved${agentStr}${turnStr}`,
|
|
337
|
-
` ${triggerLabel} │ ${C.magenta}repeat-skipped: ${dedupStr}${C.reset}
|
|
365
|
+
` ${triggerLabel} │ ${C.magenta}repeat-skipped: ${dedupStr}${C.reset}`,
|
|
366
|
+
` ${C.gray}dropped ${fmt(sessIn)} → kept ${fmt(sessKept)} sess / ${fmt(repoKept)} repo · freed ${fmt(sessFreed)} sess / ${fmt(repoFreed)} repo${C.reset}`,
|
|
338
367
|
];
|
|
339
|
-
//
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
const
|
|
343
|
-
const filled = Math.
|
|
344
|
-
const
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
lines.push(` ${C.green}saved ${fmt(this.rt.tokensSaved)} ${bar}${C.reset} ${pct}% of ${fmt(goal)} ${C.gray}│${C.reset} ${C.blue}${fmt(this.rt.tokensSaved)}${C.reset}/${C.blue}${fmt(totalHeld)}${C.reset} tok held`);
|
|
368
|
+
// Compression meter — the single headline "% tokens saved" (Freed / In),
|
|
369
|
+
// same formula as the dashboard. Higher = better, so it reads green.
|
|
370
|
+
{
|
|
371
|
+
const w = 10;
|
|
372
|
+
const filled = Math.max(0, Math.min(w, Math.round(sessPct * w)));
|
|
373
|
+
const cbar = C.green + "▓".repeat(filled) + C.dim + "░".repeat(w - filled) + C.reset;
|
|
374
|
+
const sTxt = (sessPct * 100).toFixed(sessPct * 100 >= 10 ? 0 : 1);
|
|
375
|
+
const rTxt = (repoPct * 100).toFixed(repoPct * 100 >= 10 ? 0 : 1);
|
|
376
|
+
lines.push(` ${cbar} ${sTxt}% tokens saved (sess) · ${rTxt}% repo${C.reset}`);
|
|
349
377
|
}
|
|
350
378
|
// Live "now processing" line + why + recent deduped/compacted events,
|
|
351
379
|
// collapsed to ONE rotating line (fresh only). The ticker ring buffer
|
package/package.json
CHANGED
package/src/store/memoryIndex.ts
CHANGED
|
@@ -25,10 +25,12 @@ import { join } from "node:path";
|
|
|
25
25
|
import { mkdirSync, rmSync, existsSync } from "node:fs";
|
|
26
26
|
|
|
27
27
|
// PGlite + pgvector are script-free WASM (no native build) → survive pi's
|
|
28
|
-
// install-script block.
|
|
29
|
-
//
|
|
30
|
-
|
|
31
|
-
|
|
28
|
+
// install-script block. The VALUE import is LAZY (dynamic import inside
|
|
29
|
+
// openPgLite) so a missing/broken package degrades to the same-repo scan instead
|
|
30
|
+
// of crashing module load. A static top-level `import { PGlite }` would throw
|
|
31
|
+
// "Cannot find module" at pi startup and take down the whole extension. The
|
|
32
|
+
// `import type` below is erased at compile time and emits NO runtime load.
|
|
33
|
+
import type { PGlite as PGliteInstance, Extension } from "@electric-sql/pglite";
|
|
32
34
|
|
|
33
35
|
/** Vector dimension produced by the default TrigramEmbedder (src/embedder.ts). */
|
|
34
36
|
export const MEMORY_INDEX_DIM = 512;
|
|
@@ -47,6 +49,12 @@ let db: PGliteInstance | undefined;
|
|
|
47
49
|
let initPromise: Promise<PGliteInstance | undefined> | undefined;
|
|
48
50
|
let disabled = false;
|
|
49
51
|
let warned = false;
|
|
52
|
+
/** Lazily-loaded PGlite module + pgvector extension (see loadPgLite). */
|
|
53
|
+
let pgliteMod: {
|
|
54
|
+
PGlite: typeof import("@electric-sql/pglite")["PGlite"];
|
|
55
|
+
vector: Extension;
|
|
56
|
+
} | undefined;
|
|
57
|
+
let pgliteLoadFailed = false;
|
|
50
58
|
|
|
51
59
|
function indexDir(): string {
|
|
52
60
|
const override = process.env.MEGACOMPACT_INDEX_DIR;
|
|
@@ -91,6 +99,30 @@ export function initMemoryIndex(): Promise<PGliteInstance | undefined> {
|
|
|
91
99
|
return initPromise;
|
|
92
100
|
}
|
|
93
101
|
|
|
102
|
+
/**
|
|
103
|
+
* Lazily load the PGlite module + pgvector extension via dynamic import. Caches
|
|
104
|
+
* success and permanent failure. Returns undefined (once, then forever) when the
|
|
105
|
+
* package is missing/broken so callers fall back to the same-repo scan. Never throws.
|
|
106
|
+
*/
|
|
107
|
+
async function loadPgLite(): Promise<
|
|
108
|
+
{ PGlite: typeof import("@electric-sql/pglite")["PGlite"]; vector: Extension } | undefined
|
|
109
|
+
> {
|
|
110
|
+
if (pgliteMod) return pgliteMod;
|
|
111
|
+
if (pgliteLoadFailed) return undefined;
|
|
112
|
+
try {
|
|
113
|
+
const [pglitePkg, pgvectorPkg] = await Promise.all([
|
|
114
|
+
import("@electric-sql/pglite"),
|
|
115
|
+
import("@electric-sql/pglite-pgvector"),
|
|
116
|
+
]);
|
|
117
|
+
pgliteMod = { PGlite: pglitePkg.PGlite, vector: pgvectorPkg.vector };
|
|
118
|
+
return pgliteMod;
|
|
119
|
+
} catch (err) {
|
|
120
|
+
pgliteLoadFailed = true;
|
|
121
|
+
logWarn(`package unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
122
|
+
return undefined;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
94
126
|
/**
|
|
95
127
|
* Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level abort
|
|
96
128
|
* (typically from a corrupted/torn data dir) triggers a delete + one retry.
|
|
@@ -99,11 +131,13 @@ async function openPgLite(
|
|
|
99
131
|
retryOnCorrupt: boolean,
|
|
100
132
|
): Promise<PGliteInstance | undefined> {
|
|
101
133
|
try {
|
|
134
|
+
const mod = await loadPgLite();
|
|
135
|
+
if (!mod) return undefined;
|
|
102
136
|
const dir = indexDir();
|
|
103
137
|
mkdirSync(dir, { recursive: true });
|
|
104
|
-
const pg = await new PGlite({
|
|
138
|
+
const pg = await new mod.PGlite({
|
|
105
139
|
dataDir: dir,
|
|
106
|
-
extensions: { vector },
|
|
140
|
+
extensions: { vector: mod.vector },
|
|
107
141
|
});
|
|
108
142
|
await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
|
|
109
143
|
await pg.exec(`
|
package/src/store/vectorIndex.ts
CHANGED
|
@@ -21,10 +21,12 @@ import { join } from "node:path";
|
|
|
21
21
|
import { mkdirSync, rmSync, existsSync } from "node:fs";
|
|
22
22
|
|
|
23
23
|
// PGlite + pgvector are script-free WASM (no native build) → survive pi's
|
|
24
|
-
// install-script block.
|
|
25
|
-
//
|
|
26
|
-
|
|
27
|
-
|
|
24
|
+
// install-script block. The VALUE import is LAZY (dynamic import inside
|
|
25
|
+
// openPgLite) so a missing/broken package degrades to the sync scan instead of
|
|
26
|
+
// crashing module load. A static top-level `import { PGlite }` would throw
|
|
27
|
+
// "Cannot find module" at pi startup and take down the whole extension. The
|
|
28
|
+
// `import type` below is erased at compile time and emits NO runtime load.
|
|
29
|
+
import type { PGlite as PGliteInstance, Extension } from "@electric-sql/pglite";
|
|
28
30
|
|
|
29
31
|
/** Vector dimension produced by the default TrigramEmbedder (src/embedder.ts). */
|
|
30
32
|
export const EMBEDDING_DIM = 512;
|
|
@@ -42,6 +44,12 @@ let db: PGliteInstance | undefined;
|
|
|
42
44
|
let initPromise: Promise<PGliteInstance | undefined> | undefined;
|
|
43
45
|
let disabled = false;
|
|
44
46
|
let warned = false;
|
|
47
|
+
/** Lazily-loaded PGlite module + pgvector extension (see loadPgLite). */
|
|
48
|
+
let pgliteMod: {
|
|
49
|
+
PGlite: typeof import("@electric-sql/pglite")["PGlite"];
|
|
50
|
+
vector: Extension;
|
|
51
|
+
} | undefined;
|
|
52
|
+
let pgliteLoadFailed = false;
|
|
45
53
|
|
|
46
54
|
function indexDir(): string {
|
|
47
55
|
const override = process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
|
|
@@ -86,6 +94,30 @@ export function initVectorIndex(): Promise<PGliteInstance | undefined> {
|
|
|
86
94
|
return initPromise;
|
|
87
95
|
}
|
|
88
96
|
|
|
97
|
+
/**
|
|
98
|
+
* Lazily load the PGlite module + pgvector extension via dynamic import. Caches
|
|
99
|
+
* success and permanent failure. Returns undefined (once, then forever) when the
|
|
100
|
+
* package is missing/broken so callers fall back to the sync scan. Never throws.
|
|
101
|
+
*/
|
|
102
|
+
async function loadPgLite(): Promise<
|
|
103
|
+
{ PGlite: typeof import("@electric-sql/pglite")["PGlite"]; vector: Extension } | undefined
|
|
104
|
+
> {
|
|
105
|
+
if (pgliteMod) return pgliteMod;
|
|
106
|
+
if (pgliteLoadFailed) return undefined;
|
|
107
|
+
try {
|
|
108
|
+
const [pglitePkg, pgvectorPkg] = await Promise.all([
|
|
109
|
+
import("@electric-sql/pglite"),
|
|
110
|
+
import("@electric-sql/pglite-pgvector"),
|
|
111
|
+
]);
|
|
112
|
+
pgliteMod = { PGlite: pglitePkg.PGlite, vector: pgvectorPkg.vector };
|
|
113
|
+
return pgliteMod;
|
|
114
|
+
} catch (err) {
|
|
115
|
+
pgliteLoadFailed = true;
|
|
116
|
+
logWarn(`package unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
117
|
+
return undefined;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
89
121
|
/**
|
|
90
122
|
* Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level
|
|
91
123
|
* abort (typically from a corrupted/torn data dir) triggers a delete + one
|
|
@@ -95,11 +127,13 @@ async function openPgLite(
|
|
|
95
127
|
retryOnCorrupt: boolean,
|
|
96
128
|
): Promise<PGliteInstance | undefined> {
|
|
97
129
|
try {
|
|
130
|
+
const mod = await loadPgLite();
|
|
131
|
+
if (!mod) return undefined;
|
|
98
132
|
const dir = indexDir();
|
|
99
133
|
mkdirSync(dir, { recursive: true });
|
|
100
|
-
const pg = await new PGlite({
|
|
134
|
+
const pg = await new mod.PGlite({
|
|
101
135
|
dataDir: dir,
|
|
102
|
-
extensions: { vector },
|
|
136
|
+
extensions: { vector: mod.vector },
|
|
103
137
|
});
|
|
104
138
|
await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
|
|
105
139
|
await pg.exec(`
|