pi-mega-compact 0.4.24 → 0.4.26

package/dist/extensions/mega-pipeline.js

@@ -15,6 +15,7 @@ import { C, MARKER_TYPE, } from "./mega-runtime.js";
 import { resolveRepoRoot, preserveRecentForPressure } from "./mega-config.js";
 import { runRaptor } from "../src/dedup/raptor/index.js";
 import { loadDedupConfig } from "../src/config/dedup.js";
+import { upsertEmbedding as indexUpsertEmbedding } from "../src/store/vectorIndex.js";
 /** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
 export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
     runtime.bindRepo(ctx.cwd);
@@ -131,6 +132,24 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
             /* non-fatal: tree refresh never blocks a compaction */
         }
     }
+    // Slice 2: best-effort mirror of the new checkpoint into the async global
+    // PGlite/HNSW vector index. Fires once per compaction (not per-add), so the
+    // shared global dir is never hammered by concurrent test workers.
+    // Non-fatal: a WASM init failure degrades to the sync scan silently.
+    if (!result.deduped) {
+        try {
+            const all = runtime.store.list(sid);
+            const latest = all.find((cp) => cp.checkpointId === result.checkpointId);
+            if (latest?.embedding) {
+                void indexUpsertEmbedding(runtime.currentStateDir, sid, latest.checkpointId, latest.embedding).catch(() => {
+                    /* non-fatal: index refresh never blocks a compaction */
+                });
+            }
+        }
+        catch {
+            /* non-fatal: index refresh never blocks a compaction */
+        }
+    }
     runtime.setStatus(ctx, runtime.rt.persistedThisSession
         ? `mega-compact: ${result.checkpointId} · ${saved} tok saved`
         : `mega-compact: ready`);

package/dist/extensions/openclaw-mega-compact.js

@@ -0,0 +1,291 @@
+/**
+ * openclaw-mega-compact — OpenClaw plugin adapter for the pi-mega-compact engine.
+ *
+ * Wires the pi-agnostic Trident engine (src/) into OpenClaw's plugin lifecycle:
+ *  - Registers a CompactionProvider that replaces the built-in summarizeInStages.
+ *  - Exposes `mega_status` and `mega_recall` tools for on-demand inspection.
+ *  - Hooks into `before_compaction` / `after_compaction` for diagnostics.
+ *
+ * Design constraints:
+ *  - NO imports from `@earendil-works/pi-coding-agent` or pi-agent-core.
+ *  - The engine core (src/) is pi-agnostic; this file is the sole OpenClaw boundary.
+ *  - No network at runtime — everything is local (stores + extractive summarizer).
+ */
+import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
+import { compactSession, setDefaultStore, } from "../src/engine.js";
+import { recallAndInline } from "../src/recall.js";
+import { VectorStore } from "../src/vectorStore.js";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const PLUGIN_ID = "mega-compact";
+const PLUGIN_LABEL = "Mega Compact (Trident)";
+/** Default state directory for vector store persistence. */
+const STATE_DIR = process.env.MEGA_COMPACT_STATE_DIR ?? undefined;
+/** Minimum messages before we bother compacting. */
+const MIN_MESSAGES_FOR_COMPACT = 6;
+// ---------------------------------------------------------------------------
+// Message conversion — OpenClaw unknown[] → EngineMessage[]
+// ---------------------------------------------------------------------------
+/**
+ * Best-effort conversion from OpenClaw's opaque message array to our
+ * EngineMessage shape. OpenClaw messages are typed as `unknown[]` so we
+ * handle whatever shape comes through gracefully.
+ */
+function toEngineMessages(messages) {
+    return messages.map((msg) => {
+        if (!msg || typeof msg !== "object") {
+            // Primitive fallback — treat as custom text.
+            return {
+                role: "custom",
+                text: String(msg ?? ""),
+            };
+        }
+        const m = msg;
+        const role = typeof m.role === "string" ? m.role : "custom";
+        // Normalize role to one of our four engine roles.
+        let engineRole;
+        switch (role) {
+            case "user":
+                engineRole = "user";
+                break;
+            case "assistant":
+                engineRole = "assistant";
+                break;
+            case "tool":
+            case "function":
+                engineRole = "tool";
+                break;
+            default:
+                engineRole = "custom";
+                break;
+        }
+        // Extract text content from common message shapes.
+        const text = typeof m.content === "string"
+            ? m.content
+            : typeof m.text === "string"
+                ? m.text
+                : Array.isArray(m.content)
+                    ? m.content
+                        .filter((part) => part.type === "text" && typeof part.text === "string")
+                        .map((part) => part.text)
+                        .join("\n")
+                    : "";
+        // Preserve tool metadata when present.
+        const toolName = typeof m.name === "string"
+            ? m.name
+            : typeof m.toolName === "string"
+                ? m.toolName
+                : undefined;
+        const input = typeof m.input === "string"
+            ? m.input
+            : typeof m.arguments === "string"
+                ? m.arguments
+                : m.arguments !== undefined
+                    ? JSON.stringify(m.arguments)
+                    : undefined;
+        const output = typeof m.output === "string"
+            ? m.output
+            : engineRole === "tool" && typeof m.content === "string"
+                ? m.content
+                : undefined;
+        return { role: engineRole, text, toolName, input, output };
+    });
+}
+// ---------------------------------------------------------------------------
+// Compaction provider
+// ---------------------------------------------------------------------------
+function createCompactionProvider(store) {
+    return {
+        id: PLUGIN_ID,
+        label: PLUGIN_LABEL,
+        async summarize({ messages, signal, compressionRatio, }) {
+            // Abort check — bail early if the caller cancelled.
+            if (signal?.aborted) {
+                throw new DOMException("Aborted", "AbortError");
+            }
+            const engineMessages = toEngineMessages(messages);
+            // Nothing meaningful to compact.
+            if (engineMessages.length < MIN_MESSAGES_FOR_COMPACT) {
+                return "";
+            }
+            // Map compression ratio → keepFrom boundary.
+            // compressionRatio=0.5 means "compact the oldest 50%".
+            // Default to compacting the oldest half if not specified.
+            const ratio = compressionRatio ?? 0.5;
+            const keepFrom = Math.max(MIN_MESSAGES_FOR_COMPACT, Math.floor(engineMessages.length * (1 - ratio)));
+            // Abort check after conversion (conversion is cheap but check anyway).
+            if (signal?.aborted) {
+                throw new DOMException("Aborted", "AbortError");
+            }
+            const sessionId = `openclaw-${Date.now()}`;
+            const input = {
+                sessionId,
+                messages: engineMessages,
+                keepFrom,
+            };
+            const result = compactSession(input, store);
+            if (result.skipped) {
+                return "";
+            }
+            return result.summary;
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Plugin entry
+// ---------------------------------------------------------------------------
+export default definePluginEntry({
+    id: PLUGIN_ID,
+    name: "Mega Compact",
+    description: "Layered, local, vector-backed context compressor (Trident engine) for OpenClaw compaction.",
+    register(api) {
+        const logger = api.logger;
+        // Resolve state directory — prefer plugin config override.
+        const pluginCfg = (api.pluginConfig ?? {});
+        const stateDir = typeof pluginCfg.stateDir === "string" && pluginCfg.stateDir.length > 0
+            ? pluginCfg.stateDir
+            : STATE_DIR;
+        // Initialize vector store.
+        let store;
+        try {
+            store = new VectorStore({ stateDir });
+            setDefaultStore(store);
+            logger.info?.(`${PLUGIN_ID}: vector store initialized (stateDir=${stateDir ?? "default"})`);
+        }
+        catch (err) {
+            logger.error?.(`${PLUGIN_ID}: failed to init vector store:`, err);
+            return; // Hard bail — no point registering if store is broken.
+        }
+        // -----------------------------------------------------------------------
+        // Register compaction provider
+        // -----------------------------------------------------------------------
+        const provider = createCompactionProvider(store);
+        api.registerCompactionProvider(provider);
+        logger.info?.(`${PLUGIN_ID}: registered compaction provider "${provider.id}"`);
+        // -----------------------------------------------------------------------
+        // Hooks — before / after compaction diagnostics
+        // -----------------------------------------------------------------------
+        api.registerHook({
+            event: "before_compaction",
+            handler: async (ctx) => {
+                const msgCount = Array.isArray(ctx?.messages) ? ctx.messages.length : 0;
+                logger.info?.(`${PLUGIN_ID}: before_compaction — ${msgCount} messages in scope`);
+            },
+        });
+        api.registerHook({
+            event: "after_compaction",
+            handler: async (ctx) => {
+                const summaryLen = typeof ctx?.summary === "string" ? ctx.summary.length : 0;
+                logger.info?.(`${PLUGIN_ID}: after_compaction — summary ${summaryLen} chars`);
+            },
+        });
+        // -----------------------------------------------------------------------
+        // Tool: mega_status
+        // -----------------------------------------------------------------------
+        api.registerTool({
+            name: "mega_status",
+            description: "Show the current status of the mega-compact engine: vector store stats, checkpoint count, and recent compaction activity.",
+            parameters: {
+                type: "object",
+                properties: {
+                    sessionId: {
+                        type: "string",
+                        description: "Optional session ID to scope stats to.",
+                    },
+                },
+                additionalProperties: false,
+            },
+            handler: async (args) => {
+                const sessionId = args?.sessionId ?? "global";
+                try {
+                    const stats = store.stats(sessionId);
+                    const parts = [
+                        `**Mega Compact Status**`,
+                        `Session: ${sessionId}`,
+                        `Checkpoints: ${stats.checkpointCount}`,
+                        `Total tokens saved: ${stats.totalTokenEstimate}`,
+                        `Last checkpoint: ${stats.lastCheckpointId ?? "—"}`,
+                        `Injected count: ${stats.injectedCount}`,
+                        `Dedup hit rate: ${(stats.dedupHitRate * 100).toFixed(0)}%`,
+                    ];
+                    if (stats.lastSummary) {
+                        parts.push(`\nLast summary (truncated):\n  ${stats.lastSummary.slice(0, 120).replace(/\n/g, " ")}…`);
+                    }
+                    return { content: [{ type: "text", text: parts.join("\n") }] };
+                }
+                catch (err) {
+                    return {
+                        content: [{ type: "text", text: `Error reading mega-compact status: ${err}` }],
+                        isError: true,
+                    };
+                }
+            },
+        });
+        // -----------------------------------------------------------------------
+        // Tool: mega_recall
+        // -----------------------------------------------------------------------
+        api.registerTool({
+            name: "mega_recall",
+            description: "Recall and inline relevant context from the mega-compact vector store for the current session.",
+            parameters: {
+                type: "object",
+                properties: {
+                    sessionId: {
+                        type: "string",
+                        description: "Session ID to recall context for.",
+                    },
+                    query: {
+                        type: "string",
+                        description: "Natural language query for relevant context.",
+                    },
+                    limit: {
+                        type: "number",
+                        description: "Max checkpoints to recall (default 3).",
+                    },
+                },
+                required: ["sessionId", "query"],
+                additionalProperties: false,
+            },
+            handler: async (args) => {
+                const { sessionId, query, limit } = args;
+                if (!sessionId || !query) {
+                    return {
+                        content: [{ type: "text", text: "Both `sessionId` and `query` are required." }],
+                        isError: true,
+                    };
+                }
+                try {
+                    const result = recallAndInline({ sessionId, query, limit: limit ?? 3, source: "command", skipInjected: false }, store);
+                    if (result.toInject.length === 0) {
+                        return {
+                            content: [{ type: "text", text: "No relevant context found in the mega-compact store." }],
+                        };
+                    }
+                    const parts = [
+                        `**Recalled ${result.toInject.length} checkpoint(s):**`,
+                        ...result.report,
+                        "",
+                        "---",
+                        result.block,
+                    ];
+                    return { content: [{ type: "text", text: parts.join("\n") }] };
+                }
+                catch (err) {
+                    return {
+                        content: [{ type: "text", text: `Error during mega-recall: ${err}` }],
+                        isError: true,
+                    };
+                }
+            },
+        });
+        // -----------------------------------------------------------------------
+        // Cleanup on shutdown
+        // -----------------------------------------------------------------------
+        api.on("shutdown", () => {
+            logger.info?.(`${PLUGIN_ID}: shutting down — clearing default store`);
+            setDefaultStore(undefined);
+        });
+        logger.info?.(`${PLUGIN_ID}: plugin registered (tools: mega_status, mega_recall)`);
+    },
+});

package/dist/src/minilm.js

@@ -0,0 +1,92 @@
+/**
+ * minilm.ts — local MiniLM (all-MiniLM-L6-v2) sentence embedder (Sprint 12).
+ *
+ * Implements the `Embedder` interface so it drops into the existing VectorStore
+ * dedup cascade and search with no call-site changes. Inference is 100% local:
+ * the ONNX model + WordPiece vocab are on-disk artifacts fetched once by
+ * scripts/setup-minilm.mjs. There is NO network call at runtime (PREVENT-PI-004).
+ *
+ * Inputs (dynamic): input_ids, attention_mask, token_type_ids (int64).
+ * Output: last_hidden_state (batch, seq, 384). We mean-pool over non-padded
+ * tokens (attention_mask == 1) and L2-normalize → 384-dim unit vector.
+ *
+ * The ONNX session + tokenizer are loaded LAZILY on first embed() so the default
+ * TrigramEmbedder path (and its zero native-init cost) is untouched unless
+ * MEGACOMPACT_EMBEDDER=minilm is selected.
+ */
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { existsSync } from "node:fs";
+import { l2Normalize, awaitSync } from "./embedder.js";
+import { WordPieceTokenizer } from "./wordpiece.js";
+export const MINILM_DIM = 384;
+export const MINILM_MAX_LEN = 256;
+/** Resolve the model directory: MEGACOMPACT_MINILM_DIR > ./models/minilm > ~/.pi … */
+function resolveModelDir() {
+    if (process.env.MEGACOMPACT_MINILM_DIR)
+        return process.env.MEGACOMPACT_MINILM_DIR;
+    // Repo-local vendored path (gitignored).
+    const local = join(process.cwd(), "models", "minilm");
+    if (existsSync(local))
+        return local;
+    return join(homedir(), ".pi", "agent", "extensions", "mega-compact", "models", "minilm");
+}
+export class MiniLMEmbedder {
+    dim = MINILM_DIM;
+    session = null;
+    tokenizer = null;
+    modelDir;
+    loadPromise = null;
+    constructor(modelDir = resolveModelDir()) {
+        this.modelDir = modelDir;
+    }
+    async ensureLoaded() {
+        if (this.session && this.tokenizer)
+            return;
+        if (this.loadPromise)
+            return this.loadPromise;
+        this.loadPromise = (async () => {
+            const ort = await import("onnxruntime-node");
+            const modelPath = join(this.modelDir, "model_quantized.onnx");
+            const vocabPath = join(this.modelDir, "vocab.txt");
+            if (!existsSync(modelPath) || !existsSync(vocabPath)) {
+                throw new Error(`MiniLM artifacts missing in ${this.modelDir}. Run: node scripts/setup-minilm.mjs`);
+            }
+            // 1 thread is plenty for a single short-region embed and bounds CPU.
+            this.session = await ort.InferenceSession.create(modelPath, {
+                executionProviders: ["cpu"],
+                graphOptimizationLevel: "all",
+            });
+            this.tokenizer = WordPieceTokenizer.fromVocabFile(vocabPath);
+        })();
+        return this.loadPromise;
+    }
+    embed(text) {
+        awaitSync(this.ensureLoaded());
+        const enc = this.tokenizer.encode(text, MINILM_MAX_LEN);
+        const n = enc.inputIds.length;
+        const BigInt64 = (arr) => arr.map((x) => BigInt(x));
+        const ort = awaitSync(import("onnxruntime-node"));
+        const tensors = {
+            input_ids: new ort.Tensor("int64", BigInt64(enc.inputIds), [1, n]),
+            attention_mask: new ort.Tensor("int64", BigInt64(enc.attentionMask), [1, n]),
+            token_type_ids: new ort.Tensor("int64", BigInt64(enc.tokenTypeIds), [1, n]),
+        };
+        const out = awaitSync(this.session.run(tensors));
+        const hidden = out.last_hidden_state.data;
+        // hidden shape: [1, n, 384]. Mean-pool over non-padded positions.
+        const pooled = new Array(MINILM_DIM).fill(0);
+        let count = 0;
+        for (let i = 0; i < n; i++) {
+            if (enc.attentionMask[i] === 0)
+                continue;
+            const base = i * MINILM_DIM;
+            for (let d = 0; d < MINILM_DIM; d++)
+                pooled[d] += hidden[base + d];
+            count++;
+        }
+        if (count === 0)
+            return l2Normalize(new Array(MINILM_DIM).fill(0));
+        return l2Normalize(pooled.map((x) => x / count));
+    }
+}

package/dist/src/recall.js

@@ -84,3 +84,58 @@ export function recallAndInline(opts, store) {
         empty: toInject.length === 0,
     };
 }
+/**
+ * Slice 2 async cross-repo recall. Same dedup/bound/inline contract as
+ * `recallAndInline`, but backed by `VectorStore.searchAsync` so it can recall
+ * across repos (HNSW NN over the global PGlite index) when `opts.crossRepo` is
+ * set. The synchronous `recallAndInline` is unchanged and remains the default
+ * per-session path. Inline-window dedupe + token cap (Fix C) apply here too.
+ *
+ * `store` must provide `searchAsync` (the live VectorStore does). Errors fall
+ * back to an empty result — recall is a bonus, never a hard dependency.
+ */
+export async function recallAndInlineAsync(opts, store) {
+    const limit = opts.limit ?? 3;
+    const skip = opts.skipInjected ?? true;
+    const maxTokens = opts.recallMaxTokens ?? 0;
+    const doWindowDedupe = opts.windowDedupe ?? false;
+    const dedupSim = opts.dedupSim ?? 0.9;
+    let hits = [];
+    try {
+        hits = await store.searchAsync(opts.sessionId, opts.query, limit, {
+            crossRepo: opts.crossRepo,
+            repoId: opts.repoId,
+        });
+    }
+    catch {
+        hits = [];
+    }
+    let liveEmbeddings = [];
+    if (doWindowDedupe && opts.liveWindow && opts.liveWindow.length > 0) {
+        const embedder = defaultEmbedder();
+        liveEmbeddings = opts.liveWindow.map((m) => embedder.embed(m));
+    }
+    const toInject = [];
+    const parts = [];
+    let blockTokens = 0;
+    for (const h of hits) {
+        if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId))
+            continue;
+        if (doWindowDedupe && liveEmbeddings.length > 0) {
+            const hitVec = defaultEmbedder().embed(h.checkpoint.summary);
+            if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim))
+                continue;
+        }
+        const part = formatRecallBlock([h]);
+        const partTokens = estimateBlockTokens(part);
+        if (maxTokens > 0 && blockTokens + partTokens > maxTokens)
+            break;
+        parts.push(part);
+        toInject.push(h);
+        blockTokens += partTokens;
+        store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
+    }
+    const block = parts.join("\n");
+    const report = toInject.map((h) => `  • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`);
+    return { toInject, report, block, empty: toInject.length === 0 };
+}

package/dist/src/store/sqlite.js

@@ -705,6 +705,14 @@ export function hasCheckpoint(sessionId, checkpointId, stateDir = getStateDir())
         .get(normalizeSessionId(sessionId), checkpointId);
     return row !== undefined;
 }
+/** Fetch a single checkpoint by (session, id), or undefined if absent. */
+export function getCheckpoint(sessionId, checkpointId, stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const row = db
+        .prepare("SELECT * FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
+        .get(normalizeSessionId(sessionId), checkpointId);
+    return row ? rowToCheckpoint(row) : undefined;
+}
 /** Mark a checkpoint's dedup_status (e.g. 'removed' by SemDeDup). */
 export function setDedupStatus(checkpointId, sessionId, status, stateDir = getStateDir()) {
     const db = openStore(stateDir);