pi-mega-compact 0.13.1 → 0.13.3

package/dist/extensions/mega-commands/helpers.js

@@ -0,0 +1,61 @@
+import { listCheckpoints } from "../../src/store/sqlite.js";
+import { defaultEventsPath } from "../../src/monitoring.js";
+import { existsSync, readFileSync } from "node:fs";
+import path from "node:path";
+/** Resolve a checkpoint by id (or "recent"/"last") from this session's store. */
+export function findCheckpoint(runtime, sid, ref) {
+    const all = listCheckpoints(sid, runtime.currentStateDir);
+    if (all.length === 0)
+        return undefined;
+    if (!ref || ref === "recent" || ref === "last")
+        return all[all.length - 1];
+    return all.find((c) => c.checkpointId === ref) ?? all.find((c) => c.checkpointId.endsWith(ref));
+}
+/**
+ * Try to peek at recent events.log and mega-compact.log for low recall quality signals.
+ * Best-effort, non-fatal — returns true if a low-quality signal was found.
+ */
+export function checkRecallQuality(stateDir) {
+    const eventPaths = [
+        defaultEventsPath(stateDir),
+        path.join(stateDir, "mega-compact.log"),
+    ];
+    for (const fp of eventPaths) {
+        try {
+            if (!existsSync(fp))
+                continue;
+            const buf = readFileSync(fp, { encoding: "utf-8" });
+            // Only scan last 64 KiB of the file
+            const tail = buf.length > 65536 ? buf.slice(buf.length - 65536) : buf;
+            const lines = tail.split("\n").filter(Boolean);
+            for (const line of lines) {
+                try {
+                    // guardrails-allow PREVENT-001: JSON.parse with null check
+                    const parsed = JSON.parse(line);
+                    if (typeof parsed !== "object" || parsed === null)
+                        continue;
+                    // Check for recall_metrics_low_quality event (latest format)
+                    if (parsed.event === "recall_metrics_low_quality" && parsed.score !== undefined) {
+                        if (typeof parsed.score === "number" && parsed.score < 0.4)
+                            return true;
+                    }
+                    // Check for recall_metrics with low relevanceScore
+                    if (parsed.event === "recall_metrics" && parsed.relevanceScore !== undefined) {
+                        if (typeof parsed.relevanceScore === "number" && parsed.relevanceScore < 0.4)
+                            return true;
+                    }
+                    // Check for RecallQualityEvent format from monitoring.ts
+                    if (parsed.score !== undefined && typeof parsed.score === "number" && parsed.score < 0.4)
+                        return true;
+                }
+                catch {
+                    // skip unparseable lines
+                }
+            }
+        }
+        catch {
+            // non-fatal: skip if can't read
+        }
+    }
+    return false;
+}

package/dist/extensions/mega-commands/historyCommands.js

@@ -0,0 +1,97 @@
+import { normalizeSessionId } from "../../src/store.js";
+import { listCheckpoints } from "../../src/store/sqlite.js";
+import { decompressSmart } from "../../src/store/compression.js";
+import { C } from "../mega-runtime.js";
+import { findCheckpoint } from "./helpers.js";
+/** Register the checkpoint history/inspection commands. */
+export function registerHistoryCommands(pi, runtime) {
+    pi.registerCommand("mega-restore", {
+        description: "Re-inject a checkpoint's verbatim original region into context. Usage: /mega-restore <chkpt|recent>",
+        handler: async (args, ctx) => {
+            try {
+                runtime.bindRepo(ctx.cwd);
+                const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
+                const cp = findCheckpoint(runtime, sid, args.trim());
+                if (!cp) {
+                    ctx.ui.notify(`[mega-compact] no checkpoint found${args.trim() ? ` for "${args.trim()}"` : ""} in this session. Try /mega-history.`);
+                    return;
+                }
+                if (!cp.compressedOriginal) {
+                    ctx.ui.notify(`[mega-compact] ${cp.checkpointId} has no recoverable original (pre-blob or direct add). Cannot restore verbatim.`);
+                    return;
+                }
+                const original = decompressSmart(cp.compressedOriginal).toString("utf-8");
+                // Re-inject verbatim via before_agent_start (PREVENT-PI-003) — never
+                // touches live messages, only prepends the restored region to systemPrompt.
+                runtime.pendingRecallBlock = `The following compacted context was RESTORED from checkpoint ${cp.checkpointId} (verbatim original region):\n\n${original}`;
+                const files = cp.filesModified?.length ? cp.filesModified.join(", ") : "(no files captured)";
+                ctx.ui.notify(`[mega-compact] ♻ restored ${cp.checkpointId} — ${original.length} chars re-injected on next turn.\n` +
+                    `[mega-compact] files: ${files}`);
+                runtime.dashboard.event("restore", { checkpointId: cp.checkpointId, chars: original.length });
+            }
+            catch (e) {
+                ctx.ui.notify(`[mega-compact] /mega-restore failed (checkpoint may be corrupt): ${String(e)}`);
+            }
+        },
+    });
+    pi.registerCommand("mega-history", {
+        description: "List this session's checkpoints (id, date, files, tokens). Usage: /mega-history",
+        handler: async (_args, ctx) => {
+            runtime.bindRepo(ctx.cwd);
+            const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
+            const all = listCheckpoints(sid, runtime.currentStateDir);
+            if (all.length === 0) {
+                ctx.ui.notify("[mega-compact] no checkpoints in this session yet.");
+                return;
+            }
+            const rows = all.map((c) => {
+                const when = c.timestamp ? new Date(c.timestamp).toISOString().slice(0, 16).replace("T", " ") : "—";
+                const files = c.filesModified?.length ? c.filesModified.map((f) => f.split("/").pop() ?? f).join(", ") : "—";
+                const orig = c.originalTokenEstimate ?? 0;
+                const stored = c.tokenEstimate ?? 0;
+                const saved = Math.max(0, orig - stored);
+                return `  ${c.checkpointId}  ${when}  ${C.cyan}${saved}t saved${C.reset}  ${files}`;
+            });
+            ctx.ui.notify(`[mega-compact] ${all.length} checkpoint(s) in this session:\n` + rows.join("\n") +
+                `\n[mega-compact] /mega-view <chkpt> to see the original region · /mega-restore <chkpt> to re-inject it`);
+        },
+    });
+    pi.registerCommand("mega-view", {
+        description: "Show a checkpoint's verbatim original region. Usage: /mega-view <chkpt|recent>",
+        handler: async (args, ctx) => {
+            try {
+                runtime.bindRepo(ctx.cwd);
+                const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
+                const cp = findCheckpoint(runtime, sid, args.trim());
+                if (!cp) {
+                    ctx.ui.notify(`[mega-compact] no checkpoint found${args.trim() ? ` for "${args.trim()}"` : ""}. Try /mega-history.`);
+                    return;
+                }
+                if (!cp.compressedOriginal) {
+                    ctx.ui.notify(`[mega-compact] ${cp.checkpointId} summary:\n${cp.summary.slice(0, 500)}${cp.summary.length > 500 ? "…" : ""}\n(no verbatim original stored)`);
+                    return;
+                }
+                const original = decompressSmart(cp.compressedOriginal).toString("utf-8");
+                ctx.ui.notify(`[mega-compact] ${cp.checkpointId} — original region (${original.length} chars):\n` +
+                    `${original.slice(0, 1500)}${original.length > 1500 ? "\n…(truncated)" : ""}`);
+            }
+            catch (e) {
+                ctx.ui.notify(`[mega-compact] /mega-view failed (checkpoint may be corrupt): ${String(e)}`);
+            }
+        },
+    });
+    pi.registerCommand("mega-help", {
+        description: "Plain-language glossary of what mega-compact's stats mean.",
+        handler: async (_args, ctx) => {
+            ctx.ui.notify(`[mega-compact] glossary — what the numbers mean:\n` +
+                `• token — a chunk of text (~4 chars). Context window = how much text fits in memory at once.\n` +
+                `• space freed — how much conversation we've compressed away to make room (the win).\n` +
+                `• memory held — how much compact summary we're currently keeping as your 'notes'.\n` +
+                `• saved checkpoint — a compact summary of an old conversation chunk we stored.\n` +
+                `• repeat-skipped — how often new text matched something we already had, so we didn't store a duplicate.\n` +
+                `• injected — times we pasted an old saved note back into the chat because it was relevant.\n` +
+                `• recall relevance — of those, how often the note was actually on-topic.\n` +
+                `• data safety — every compressed region is kept verbatim; nothing is permanently deleted. /mega-restore brings any of it back.`);
+        },
+    });
+}

package/dist/extensions/mega-commands/setupCommand.js

@@ -0,0 +1,200 @@
+import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: local subprocess detection only (no network)
+import { mkdirSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import { createRequire } from "node:module";
+import { checkRecallQuality } from "./helpers.js";
+/** Register the /mega-setup embedder wizard command. */
+export function registerSetupCommand(pi) {
+    pi.registerCommand("mega-setup", {
+        description: "Local embedding setup wizard — detect available embedders and configure mega-compact.",
+        handler: async (_args, ctx) => {
+            try {
+                const log = (msg) => ctx.ui.notify(msg);
+                // --- 1. Current config ---
+                const embeddingUrl = process.env["MEGACOMPACT_EMBEDDING_URL"];
+                const minilm = process.env["MEGACOMPACT_MINILM"];
+                let currentEmbedder = "TrigramEmbedder (heuristic, default)";
+                if (embeddingUrl)
+                    currentEmbedder = `httpEmbedder → ${embeddingUrl}`;
+                else if (minilm)
+                    currentEmbedder = "MiniLM (experimental)";
+                // --- 2. Detection ---
+                let ollamaDetected = false;
+                let ollamaModels = [];
+                let ollamaRunning = false;
+                let llamaDetected = false;
+                let onnxDetected = false;
+                try {
+                    const ollamaVer = spawnSync("ollama", ["--version"], {
+                        timeout: 5000,
+                        encoding: "utf-8",
+                        stdio: ["ignore", "pipe", "pipe"],
+                    }); // guardrails-allow PREVENT-PI-004: local subprocess detection only
+                    if (ollamaVer.status === 0) {
+                        ollamaDetected = true;
+                        try {
+                            const listResult = spawnSync("ollama", ["list"], {
+                                timeout: 5000,
+                                encoding: "utf-8",
+                                stdio: ["ignore", "pipe", "pipe"],
+                            }); // guardrails-allow PREVENT-PI-004: local subprocess detection only
+                            if (listResult.status === 0) {
+                                ollamaRunning = true;
+                                const lines = listResult.stdout?.split("\n") ?? [];
+                                for (const line of lines) {
+                                    const name = line.split(/\s+/)[0];
+                                    if (name && name !== "NAME")
+                                        ollamaModels.push(name);
+                                }
+                            }
+                        }
+                        catch {
+                            // ollama list fails if server not running — that's fine
+                        }
+                    }
+                }
+                catch {
+                    // ollama not found
+                }
+                try {
+                    const whichLlama = spawnSync("which", ["llama-server"], {
+                        timeout: 3000,
+                        encoding: "utf-8",
+                        stdio: ["ignore", "pipe", "pipe"],
+                    }); // guardrails-allow PREVENT-PI-004: local subprocess detection only
+                    llamaDetected = whichLlama.status === 0;
+                }
+                catch {
+                    // not found
+                }
+                try {
+                    const req = createRequire(import.meta.url);
+                    try {
+                        req.resolve("onnxruntime-node");
+                        onnxDetected = true;
+                    }
+                    catch {
+                        // not in node_modules
+                    }
+                }
+                catch {
+                    // require error
+                }
+                // --- 2.5 Quality check ---
+                // Scan recent events for low recall quality (best-effort, non-fatal)
+                const stateDir = process.env["MEGACOMPACT_STATE_DIR"] ?? path.join(os.homedir(), ".pi", "mega-compact");
+                const lowRecallQuality = checkRecallQuality(stateDir);
+                const choices = [];
+                if (ollamaDetected && ollamaRunning) {
+                    const model = ollamaModels.find((m) => m.includes("nomic-embed") ||
+                        m.includes("embed") ||
+                        m.includes("mxbai"));
+                    const suggestedModel = model ?? "nomic-embed-text";
+                    choices.push({
+                        label: `Use Ollama (${suggestedModel}) at localhost:11434`,
+                        value: `ollama:${suggestedModel}`,
+                    });
+                }
+                else if (ollamaDetected && !ollamaRunning) {
+                    choices.push({
+                        label: "Ollama detected but server is not running — start it first",
+                        value: "ollama:offline",
+                    });
+                }
+                if (llamaDetected) {
+                    choices.push({
+                        label: "Use llama.cpp server at localhost:8080",
+                        value: "llama:default",
+                    });
+                }
+                if (onnxDetected) {
+                    choices.push({
+                        label: "Use ONNX Runtime (experimental, requires config)",
+                        value: "onnx:default",
+                    });
+                }
+                choices.push({
+                    label: "Keep TrigramEmbedder (default, no setup)",
+                    value: "trigram",
+                });
+                // --- 4. Prompt user ---
+                log(`Current embedder: ${currentEmbedder}`);
+                if (lowRecallQuality) {
+                    log("Tap: recent recall quality is low. Consider switching to Ollama or llama.cpp for better results.");
+                }
+                let selected = "trigram";
+                if (choices.length > 1) {
+                    try {
+                        const labels = choices.map((c) => c.label);
+                        const picked = await ctx.ui.select("Select an embedder to configure:", labels);
+                        if (picked === undefined) {
+                            log("Setup cancelled by user.");
+                            return;
+                        }
+                        const match = choices.find((c) => c.label === picked);
+                        selected = match?.value ?? "trigram";
+                    }
+                    catch {
+                        log("Setup cancelled by user.");
+                        return;
+                    }
+                }
+                else {
+                    log("No local embedders detected. Keeping TrigramEmbedder default.");
+                }
+                // --- 5. Apply selection ---
+                if (selected.startsWith("ollama:")) {
+                    const model = selected.split(":")[1];
+                    const url = `http://localhost:11434/api/embeddings`; // guardrails-allow PREVENT-PI-004: localhost-only config string, not a runtime fetch
+                    const lines = [
+                        `# Mega-Compact Embedder Configuration — Ollama`,
+                        `# Generated by /megasetup at ${new Date().toISOString()}`,
+                        `export MEGACOMPACT_EMBEDDING_URL="${url}"`,
+                        `# Optional: uncomment to set cache size (default 1000)`,
+                        `# export MEGACOMPACT_EMBED_CACHE=1000`,
+                    ];
+                    const envContent = lines.join("\n") + "\n";
+                    // Write .mega-compact.env to state dir
+                    try {
+                        mkdirSync(stateDir, { recursive: true });
+                        writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
+                    }
+                    catch {
+                        // non-fatal
+                    }
+                    log(`Configured for Ollama (${model}).\n\nAdd to your shell profile (~/.bashrc, ~/.zshrc, or ~/.config/fish/config.fish):\n\n  source "${path.join(stateDir, ".mega-compact.env")}"\n\nOr export manually:\n\n  export MEGACOMPACT_EMBEDDING_URL="${url}"\n\nThen restart pi to activate the http embedder.`);
+                }
+                else if (selected.startsWith("llama:")) {
+                    const url = "http://localhost:8080/v1/embeddings"; // guardrails-allow PREVENT-PI-004: localhost-only config string, not a runtime fetch
+                    const lines = [
+                        `# Mega-Compact Embedder Configuration — llama.cpp`,
+                        `# Generated by /megasetup at ${new Date().toISOString()}`,
+                        `export MEGACOMPACT_EMBEDDING_URL="${url}"`,
+                    ];
+                    const envContent = lines.join("\n") + "\n";
+                    try {
+                        mkdirSync(stateDir, { recursive: true });
+                        writeFileSync(path.join(stateDir, ".mega-compact.env"), envContent, "utf-8");
+                    }
+                    catch {
+                        // non-fatal
+                    }
+                    log(`Configured for llama.cpp server.\n\nAdd to your shell profile:\n\n  source "${path.join(stateDir, ".mega-compact.env")}"\n\nOr export manually:\n\n  export MEGACOMPACT_EMBEDDING_URL="${url}"\n\nThen restart pi to activate the http embedder.`);
+                }
+                else if (selected.startsWith("onnx:")) {
+                    log("ONNX Runtime configuration is experimental.\n\nSet MEGACOMPACT_EMBEDDING_URL to your local ONNX server URL, or see docs for manual setup.\nKeeping TrigramEmbedder for now.");
+                }
+                else {
+                    const qualityTip = lowRecallQuality
+                        ? " Recent recall quality is low — consider running /mega-setup again to switch to Ollama or llama.cpp for better results."
+                        : "";
+                    log(`Keeping TrigramEmbedder (default). No configuration needed.${qualityTip}`);
+                }
+            }
+            catch (e) {
+                ctx.ui.notify(`[mega-compact] /mega-setup failed: ${String(e)}`);
+            }
+        },
+    });
+}