pi-mega-compact 0.4.14 → 0.4.16

package/dist/extensions/dashboard-server.js

@@ -69,6 +69,15 @@ function dashboardHtml(tierName) {
   .card.safe h2 { color: #3fb950; }
   .safe-note { font-size: 12px; color: #8b949e; margin: 12px 0 0; line-height: 1.5; }
   .value.ok { color: #3fb950; }
+  .label {
+    cursor: help;
+    border-bottom: 1px dotted #484f58;
+  }
+  .card.legend { grid-column: 1 / -1; }
+  .legend-list { margin: 0; padding-left: 18px; color: #c9d1d9; }
+  .legend-list li { margin-bottom: 8px; line-height: 1.5; }
+  .legend-list b { color: #f0f6fc; }
+  .legend-note { font-size: 12px; color: #8b949e; margin: 12px 0 0; font-style: italic; }
   .card h2 { font-size: 12px; text-transform: uppercase; letter-spacing: .5px; color: #8b949e; margin-bottom: 12px; font-weight: 600; }
   .meter-track { background: #21262d; border-radius: 4px; height: 20px; overflow: hidden; margin: 8px 0; }
   .meter-fill { height: 100%; border-radius: 4px; transition: width .6s ease; min-width: 2px; }
@@ -126,15 +135,15 @@ function dashboardHtml(tierName) {
   <div class="card">
     <h2>Vector Store</h2>
     <div class="stat-grid">
-      <span class="label">Checkpoints</span><span class="value" id="st-count">0</span>
-      <span class="label">Tokens Stored</span><span class="value" id="st-tokens">0</span>
-      <span class="label">Original Tokens</span><span class="value" id="st-orig">0</span>
-      <span class="label">Tokens Saved</span><span class="value" id="st-saved">0</span>
-      <span class="label">Injected</span><span class="value" id="st-injected">0</span>
-      <span class="label">Dedup Rate</span><span class="value" id="st-dedup">0%</span>
-      <span class="label">Storage Dedup</span><span class="value" id="st-sdedup">0%</span>
-      <span class="label">Collapsed</span><span class="value" id="st-collapsed">0</span>
-      <span class="label">Last ID</span><span class="value" id="st-lastid">—</span>
+      <span class="label" title="A saved summary of a chunk of your conversation that was compacted to free up space.">Checkpoints</span><span class="value" id="st-count">0</span>
+      <span class="label" title="How much conversation we are currently holding as compact summaries (the 'memory' this extension keeps). Smaller is better.">Tokens Stored</span><span class="value" id="st-tokens">0</span>
+      <span class="label" title="Total size of the original conversation text before it was compacted.">Original Tokens</span><span class="value" id="st-orig">0</span>
+      <span class="label" title="How much conversation space we have freed up for you (original size minus the compact summary we kept).">Tokens Saved</span><span class="value" id="st-saved">0</span>
+      <span class="label" title="How many times old context was automatically brought back into the conversation because it was relevant to what you were doing.">Injected</span><span class="value" id="st-injected">0</span>
+      <span class="label" title="Of the times we recalled old context, how often it was actually on-topic.">Recall Relevance</span><span class="value" id="st-dedup">0%</span>
+      <span class="label" title="How often new content matched something we already had, so we skipped storing a duplicate copy. Higher = less wasted space.">Storage Dedup</span><span class="value" id="st-sdedup">0%</span>
+      <span class="label" title="How many duplicate chunks we collapsed into one instead of storing separately.">Collapsed</span><span class="value" id="st-collapsed">0</span>
+      <span class="label" title="The ID of the most recent saved checkpoint.">Last ID</span><span class="value" id="st-lastid">—</span>
     </div>
   </div>
   <div class="card">
@@ -177,6 +186,18 @@ function dashboardHtml(tierName) {
       <span class="label">Status</span><span class="value" id="cr-status">idle</span>
     </div>
   </div>
+  <div class="card legend">
+    <h2>What these numbers mean</h2>
+    <ul class="legend-list">
+      <li><b>Tokens saved</b> — conversation space this extension has freed up for you (it compacted old text into short summaries).</li>
+      <li><b>Tokens stored</b> — how much "memory" (compact summaries) the extension is currently holding for this repo.</li>
+      <li><b>Injected</b> — times old context was automatically pasted back in because it was relevant to your current task.</li>
+      <li><b>Recall relevance</b> — of those, how often the recalled context was actually on-topic.</li>
+      <li><b>Storage dedup</b> — how often new content matched something already saved, so a duplicate copy was skipped (saves space).</li>
+      <li><b>Data safety</b> — every compacted region is kept verbatim (compressed). Nothing is permanently deleted; you can restore any of it.</li>
+    </ul>
+    <p class="legend-note">Hover any label above for a quick explanation.</p>
+  </div>
 </div>
 
 <div class="events">

package/dist/extensions/mega-commands.js

@@ -0,0 +1,223 @@
+/**
+ * mega-commands.ts — the data/inspection slash commands.
+ *
+ * Registers the 8 user-facing commands that operate on the local vector store
+ * and live runtime state. The cost estimate in /mega-status now uses the real
+ * captured model rate (model_snapshots in SQLite) instead of a $3/1M stub.
+ */
+import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
+import { normalizeSessionId } from "../src/store.js";
+import { listCheckpoints, latestModelSnapshot } from "../src/store/sqlite.js";
+import { decompressSmart } from "../src/store/compression.js";
+import { loadMetrics, fpRate, p95 } from "../src/monitoring.js";
+import { C, recentUserQuery } from "./mega-runtime.js";
+import { runCompact, doRecall } from "./mega-pipeline.js";
+import { setTier, COMPACT_TIERS } from "./mega-config.js";
+/** Resolve a checkpoint by id (or "recent"/"last") from this session's store. */
+export function findCheckpoint(runtime, sid, ref) {
+    const all = listCheckpoints(sid, runtime.currentStateDir);
+    if (all.length === 0)
+        return undefined;
+    if (!ref || ref === "recent" || ref === "last")
+        return all[all.length - 1];
+    return all.find((c) => c.checkpointId === ref) ?? all.find((c) => c.checkpointId.endsWith(ref));
+}
+/** Register all data/inspection commands. */
+export function registerCommands(pi, runtime, config) {
+    pi.registerCommand("mega-compact", {
+        description: "Compress current session context into the local vector store.",
+        handler: async (args, ctx) => {
+            const sessionEntries = ctx.sessionManager.getEntries();
+            // Project entries (branch-aware) into the message view.
+            const messages = sessionEntries.flatMap((e) => sessionEntryToContextMessages(e));
+            const summaryArg = args.trim();
+            const ran = runCompact(pi, runtime, config, ctx, messages, summaryArg ? { summary: summaryArg } : {});
+            if ("skipped" in ran && ran.skipped) {
+                ctx.ui.notify("[mega-compact] Nothing to compact (session too small).");
+                return;
+            }
+            const r = ran.result;
+            ctx.ui.notify(`[mega-compact] ${r.deduped ? "region already compacted (deduped)" : `persisted ${r.checkpointId}`} · ` +
+                `${r.tokenEstimate} tok · ${runtime.currentStateDir}`);
+        },
+    });
+    pi.registerCommand("mega-recall", {
+        description: "Recall relevant compacted context from the vector store and inline it.",
+        handler: async (args, ctx) => {
+            const query = args.trim() || recentUserQuery(ctx);
+            if (!query) {
+                ctx.ui.notify("[mega-compact] /mega-recall needs a query or a prior user message.");
+                return;
+            }
+            const r = doRecall(runtime, config, ctx, query, "command");
+            if (r.empty) {
+                runtime.logger.info("recall-empty", { query });
+                ctx.ui.notify(`[mega-compact] recall found nothing new for "${query}".`);
+                return;
+            }
+            // Stage the block so the next before_agent_start prepends it (actual
+            // injection). Report what was selected now for immediate feedback.
+            runtime.pendingRecallBlock = r.block;
+            const list = r.report.map((l) => l).join("\n");
+            runtime.logger.info("recall", { query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
+            runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt`);
+            ctx.ui.notify(`[mega-compact] recall staged ${r.toInject.length} checkpoint(s) for "${query}":\n${list}\n` +
+                `(injected at the next turn via system prompt)`);
+        },
+    });
+    pi.registerCommand("mega-status", {
+        description: "Show mega-compact config, context usage, and the data-safety invariant.",
+        handler: async (_args, ctx) => {
+            runtime.bindRepo(ctx.cwd);
+            const usage = ctx.getContextUsage();
+            const pct = usage?.percent != null ? `${usage.percent}%` : "n/a";
+            const tokens = usage?.tokens != null ? `${usage.tokens} tok` : "n/a";
+            const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
+            const st = runtime.store.stats(sid);
+            const repo = runtime.store.repoStats();
+            const di = runtime.store.dataInvariant();
+            const fmtB = (b) => b >= 1_048_576 ? `${(b / 1_048_576).toFixed(1)} MiB` :
+                b >= 1024 ? `${(b / 1024).toFixed(1)} KiB` : `${b} B`;
+            // Real cost: tokens saved × the captured model's input rate (USD/token),
+            // read from the model_snapshots table (Phase 5b schema). Falls back to 0
+            // when no model has been captured yet. contextWindow ÷ savedRate = context
+            // windows extended (how much "extra" conversation the freed space buys).
+            const model = latestModelSnapshot(runtime.currentStateDir);
+            const rate = model?.inputRate ?? 0;
+            const usd = (repo.tokensSaved * rate).toFixed(4);
+            const ctxWindow = usage?.contextWindow ?? 0;
+            const daysExtended = ctxWindow > 0 && repo.tokensSaved > 0
+                ? (repo.tokensSaved / ctxWindow).toFixed(1)
+                : "0";
+            // Identified model/provider (captured on model_select / session_start).
+            // Shows the human model name + provider so the user knows WHICH model's
+            // pricing drives the cost figure. Falls back when none captured yet.
+            const modelStr = model
+                ? `${model.modelName ?? model.modelId} · ${model.providerName ?? model.provider}`
+                : "unknown (no model captured)";
+            const costStr = `≈ $${usd} saved · ${daysExtended} context-windows extended`;
+            // Recall-quality badge (Phase 4): trust score from monitoring metrics.
+            const m = loadMetrics(runtime.currentStateDir);
+            const fp = fpRate(m, "L2");
+            const p95L2 = p95(m.latency.L2 ?? []);
+            const relPct = (st.dedupHitRate * 100).toFixed(0);
+            const qualityStr = `recall ${relPct}% relevant · FP ${(fp * 100).toFixed(1)}% · L2 p95 ${p95L2.toFixed(0)}ms`;
+            ctx.ui.notify(`[mega-compact] pct=${pct} tokens=${tokens} tier=${config.tier} fastGate=${config.fastGatePct}% ` +
+                `threshold=${config.thresholdTokens} auto=${config.auto} autoInline=${config.autoInline}\n` +
+                `[mega-compact] store: ${st.checkpointCount} chkpt · ` +
+                `${st.totalTokenEstimate} tok · last=${st.lastCheckpointId ?? "—"} · ` +
+                `injected=${st.injectedCount} · dedup=${(st.dedupHitRate * 100).toFixed(0)}%\n` +
+                `[mega-compact] anchor=${config.anchorUserMessages} preserveRecent=${config.preserveRecent} ` +
+                `autoInlineK=${config.autoInlineK} dedupSim=${config.dedupSim} debug=${config.debug}\n` +
+                `[mega-compact] 🛡 data-safe: ${di.regionsRetained} regions retained ` +
+                `(${fmtB(di.compressedOriginalBytes)} compressed-original) · ` +
+                `${di.duplicatesCollapsed} dedup-duplicates collapsed · ` +
+                `${C.green}0 bytes permanently deleted${C.reset}\n` +
+                `[mega-compact] 💰 ${costStr}\n` +
+                `[mega-compact] 🤖 model: ${modelStr}\n` +
+                `[mega-compact] 🎯 ${qualityStr}\n` +
+                `[mega-compact] stateDir=${runtime.currentStateDir}`);
+        },
+    });
+    // ---- Phase 4: cheap standout commands (data is already persisted) -------
+    pi.registerCommand("mega-restore", {
+        description: "Re-inject a checkpoint's verbatim original region into context. Usage: /mega-restore <chkpt|recent>",
+        handler: async (args, ctx) => {
+            runtime.bindRepo(ctx.cwd);
+            const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
+            const cp = findCheckpoint(runtime, sid, args.trim());
+            if (!cp) {
+                ctx.ui.notify(`[mega-compact] no checkpoint found${args.trim() ? ` for "${args.trim()}"` : ""} in this session. Try /mega-history.`);
+                return;
+            }
+            if (!cp.compressedOriginal) {
+                ctx.ui.notify(`[mega-compact] ${cp.checkpointId} has no recoverable original (pre-blob or direct add). Cannot restore verbatim.`);
+                return;
+            }
+            const original = decompressSmart(cp.compressedOriginal).toString("utf-8");
+            // Re-inject verbatim via before_agent_start (PREVENT-PI-003) — never
+            // touches live messages, only prepends the restored region to systemPrompt.
+            runtime.pendingRecallBlock = `The following compacted context was RESTORED from checkpoint ${cp.checkpointId} (verbatim original region):\n\n${original}`;
+            const files = cp.filesModified?.length ? cp.filesModified.join(", ") : "(no files captured)";
+            ctx.ui.notify(`[mega-compact] ♻ restored ${cp.checkpointId} — ${original.length} chars re-injected on next turn.\n` +
+                `[mega-compact] files: ${files}`);
+            runtime.dashboard.event("restore", { checkpointId: cp.checkpointId, chars: original.length });
+        },
+    });
+    pi.registerCommand("mega-history", {
+        description: "List this session's checkpoints (id, date, files, tokens). Usage: /mega-history",
+        handler: async (_args, ctx) => {
+            runtime.bindRepo(ctx.cwd);
+            const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
+            const all = listCheckpoints(sid, runtime.currentStateDir);
+            if (all.length === 0) {
+                ctx.ui.notify("[mega-compact] no checkpoints in this session yet.");
+                return;
+            }
+            const rows = all.map((c) => {
+                const when = c.timestamp ? new Date(c.timestamp).toISOString().slice(0, 16).replace("T", " ") : "—";
+                const files = c.filesModified?.length ? c.filesModified.map((f) => f.split("/").pop()).join(", ") : "—";
+                const orig = c.originalTokenEstimate ?? 0;
+                const stored = c.tokenEstimate ?? 0;
+                const saved = Math.max(0, orig - stored);
+                return `  ${c.checkpointId}  ${when}  ${C.cyan}${saved}t saved${C.reset}  ${files}`;
+            });
+            ctx.ui.notify(`[mega-compact] ${all.length} checkpoint(s) in this session:\n` + rows.join("\n") +
+                `\n[mega-compact] /mega-view <chkpt> to see the original region · /mega-restore <chkpt> to re-inject it`);
+        },
+    });
+    pi.registerCommand("mega-view", {
+        description: "Show a checkpoint's verbatim original region. Usage: /mega-view <chkpt|recent>",
+        handler: async (args, ctx) => {
+            runtime.bindRepo(ctx.cwd);
+            const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
+            const cp = findCheckpoint(runtime, sid, args.trim());
+            if (!cp) {
+                ctx.ui.notify(`[mega-compact] no checkpoint found${args.trim() ? ` for "${args.trim()}"` : ""}. Try /mega-history.`);
+                return;
+            }
+            if (!cp.compressedOriginal) {
+                ctx.ui.notify(`[mega-compact] ${cp.checkpointId} summary:\n${cp.summary.slice(0, 500)}${cp.summary.length > 500 ? "…" : ""}\n(no verbatim original stored)`);
+                return;
+            }
+            const original = decompressSmart(cp.compressedOriginal).toString("utf-8");
+            ctx.ui.notify(`[mega-compact] ${cp.checkpointId} — original region (${original.length} chars):\n` +
+                `${original.slice(0, 1500)}${original.length > 1500 ? "\n…(truncated)" : ""}`);
+        },
+    });
+    pi.registerCommand("mega-help", {
+        description: "Plain-language glossary of what mega-compact's stats mean.",
+        handler: async (_args, ctx) => {
+            ctx.ui.notify(`[mega-compact] glossary — what the numbers mean:\n` +
+                `• token — a chunk of text (~4 chars). Context window = how much text fits in memory at once.\n` +
+                `• space freed — how much conversation we've compressed away to make room (the win).\n` +
+                `• memory held — how much compact summary we're currently keeping as your 'notes'.\n` +
+                `• saved checkpoint — a compact summary of an old conversation chunk we stored.\n` +
+                `• repeat-skipped — how often new text matched something we already had, so we didn't store a duplicate.\n` +
+                `• injected — times we pasted an old saved note back into the chat because it was relevant.\n` +
+                `• recall relevance — of those, how often the note was actually on-topic.\n` +
+                `• data safety — every compressed region is kept verbatim; nothing is permanently deleted. /mega-restore brings any of it back.`);
+        },
+    });
+    pi.registerCommand("mega-tier", {
+        description: "Show or change the compaction tier at runtime. Usage: /mega-tier [low|medium|high|ultra|mega]",
+        handler: async (args, ctx) => {
+            const arg = args.trim().toLowerCase();
+            if (!arg) {
+                // Show current tier and available options.
+                ctx.ui.notify(`[mega-compact] current tier: ${config.tier} (${config.thresholdTokens} tok)\n` +
+                    `[mega-compact] available tiers: ${Object.entries(COMPACT_TIERS).map(([k, v]) => `${k}=${v}`).join(", ")}`);
+                return;
+            }
+            if (!(arg in COMPACT_TIERS)) {
+                ctx.ui.notify(`[mega-compact] unknown tier "${arg}". Available: ${Object.keys(COMPACT_TIERS).join(", ")}`);
+                return;
+            }
+            const newTier = arg;
+            setTier(config, newTier);
+            runtime.setStatus(ctx, `mega-compact: tier → ${newTier} (${config.thresholdTokens} tok)`);
+            ctx.ui.notify(`[mega-compact] tier changed to ${newTier} (threshold: ${config.thresholdTokens} tokens)`);
+            runtime.snapshot(ctx);
+        },
+    });
+}