pi-mega-compact 0.9.2 → 0.11.0

package/dist/extensions/openclaw-mega-compact.js

@@ -1,291 +0,0 @@
-/**
- * openclaw-mega-compact — OpenClaw plugin adapter for the pi-mega-compact engine.
- *
- * Wires the pi-agnostic Trident engine (src/) into OpenClaw's plugin lifecycle:
- *  - Registers a CompactionProvider that replaces the built-in summarizeInStages.
- *  - Exposes `mega_status` and `mega_recall` tools for on-demand inspection.
- *  - Hooks into `before_compaction` / `after_compaction` for diagnostics.
- *
- * Design constraints:
- *  - NO imports from `@earendil-works/pi-coding-agent` or pi-agent-core.
- *  - The engine core (src/) is pi-agnostic; this file is the sole OpenClaw boundary.
- *  - No network at runtime — everything is local (stores + extractive summarizer).
- */
-import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
-import { compactSession, setDefaultStore, } from "../src/engine.js";
-import { recallAndInline } from "../src/recall.js";
-import { VectorStore } from "../src/vectorStore.js";
-// ---------------------------------------------------------------------------
-// Constants
-// ---------------------------------------------------------------------------
-const PLUGIN_ID = "mega-compact";
-const PLUGIN_LABEL = "Mega Compact (Trident)";
-/** Default state directory for vector store persistence. */
-const STATE_DIR = process.env.MEGA_COMPACT_STATE_DIR ?? undefined;
-/** Minimum messages before we bother compacting. */
-const MIN_MESSAGES_FOR_COMPACT = 6;
-// ---------------------------------------------------------------------------
-// Message conversion — OpenClaw unknown[] → EngineMessage[]
-// ---------------------------------------------------------------------------
-/**
- * Best-effort conversion from OpenClaw's opaque message array to our
- * EngineMessage shape. OpenClaw messages are typed as `unknown[]` so we
- * handle whatever shape comes through gracefully.
- */
-function toEngineMessages(messages) {
-    return messages.map((msg) => {
-        if (!msg || typeof msg !== "object") {
-            // Primitive fallback — treat as custom text.
-            return {
-                role: "custom",
-                text: String(msg ?? ""),
-            };
-        }
-        const m = msg;
-        const role = typeof m.role === "string" ? m.role : "custom";
-        // Normalize role to one of our four engine roles.
-        let engineRole;
-        switch (role) {
-            case "user":
-                engineRole = "user";
-                break;
-            case "assistant":
-                engineRole = "assistant";
-                break;
-            case "tool":
-            case "function":
-                engineRole = "tool";
-                break;
-            default:
-                engineRole = "custom";
-                break;
-        }
-        // Extract text content from common message shapes.
-        const text = typeof m.content === "string"
-            ? m.content
-            : typeof m.text === "string"
-                ? m.text
-                : Array.isArray(m.content)
-                    ? m.content
-                        .filter((part) => part.type === "text" && typeof part.text === "string")
-                        .map((part) => part.text)
-                        .join("\n")
-                    : "";
-        // Preserve tool metadata when present.
-        const toolName = typeof m.name === "string"
-            ? m.name
-            : typeof m.toolName === "string"
-                ? m.toolName
-                : undefined;
-        const input = typeof m.input === "string"
-            ? m.input
-            : typeof m.arguments === "string"
-                ? m.arguments
-                : m.arguments !== undefined
-                    ? JSON.stringify(m.arguments)
-                    : undefined;
-        const output = typeof m.output === "string"
-            ? m.output
-            : engineRole === "tool" && typeof m.content === "string"
-                ? m.content
-                : undefined;
-        return { role: engineRole, text, toolName, input, output };
-    });
-}
-// ---------------------------------------------------------------------------
-// Compaction provider
-// ---------------------------------------------------------------------------
-function createCompactionProvider(store) {
-    return {
-        id: PLUGIN_ID,
-        label: PLUGIN_LABEL,
-        async summarize({ messages, signal, compressionRatio, }) {
-            // Abort check — bail early if the caller cancelled.
-            if (signal?.aborted) {
-                throw new DOMException("Aborted", "AbortError");
-            }
-            const engineMessages = toEngineMessages(messages);
-            // Nothing meaningful to compact.
-            if (engineMessages.length < MIN_MESSAGES_FOR_COMPACT) {
-                return "";
-            }
-            // Map compression ratio → keepFrom boundary.
-            // compressionRatio=0.5 means "compact the oldest 50%".
-            // Default to compacting the oldest half if not specified.
-            const ratio = compressionRatio ?? 0.5;
-            const keepFrom = Math.max(MIN_MESSAGES_FOR_COMPACT, Math.floor(engineMessages.length * (1 - ratio)));
-            // Abort check after conversion (conversion is cheap but check anyway).
-            if (signal?.aborted) {
-                throw new DOMException("Aborted", "AbortError");
-            }
-            const sessionId = `openclaw-${Date.now()}`;
-            const input = {
-                sessionId,
-                messages: engineMessages,
-                keepFrom,
-            };
-            const result = compactSession(input, store);
-            if (result.skipped) {
-                return "";
-            }
-            return result.summary;
-        },
-    };
-}
-// ---------------------------------------------------------------------------
-// Plugin entry
-// ---------------------------------------------------------------------------
-export default definePluginEntry({
-    id: PLUGIN_ID,
-    name: "Mega Compact",
-    description: "Layered, local, vector-backed context compressor (Trident engine) for OpenClaw compaction.",
-    register(api) {
-        const logger = api.logger;
-        // Resolve state directory — prefer plugin config override.
-        const pluginCfg = (api.pluginConfig ?? {});
-        const stateDir = typeof pluginCfg.stateDir === "string" && pluginCfg.stateDir.length > 0
-            ? pluginCfg.stateDir
-            : STATE_DIR;
-        // Initialize vector store.
-        let store;
-        try {
-            store = new VectorStore({ stateDir });
-            setDefaultStore(store);
-            logger.info?.(`${PLUGIN_ID}: vector store initialized (stateDir=${stateDir ?? "default"})`);
-        }
-        catch (err) {
-            logger.error?.(`${PLUGIN_ID}: failed to init vector store:`, err);
-            return; // Hard bail — no point registering if store is broken.
-        }
-        // -----------------------------------------------------------------------
-        // Register compaction provider
-        // -----------------------------------------------------------------------
-        const provider = createCompactionProvider(store);
-        api.registerCompactionProvider(provider);
-        logger.info?.(`${PLUGIN_ID}: registered compaction provider "${provider.id}"`);
-        // -----------------------------------------------------------------------
-        // Hooks — before / after compaction diagnostics
-        // -----------------------------------------------------------------------
-        api.registerHook({
-            event: "before_compaction",
-            handler: async (ctx) => {
-                const msgCount = Array.isArray(ctx?.messages) ? ctx.messages.length : 0;
-                logger.info?.(`${PLUGIN_ID}: before_compaction — ${msgCount} messages in scope`);
-            },
-        });
-        api.registerHook({
-            event: "after_compaction",
-            handler: async (ctx) => {
-                const summaryLen = typeof ctx?.summary === "string" ? ctx.summary.length : 0;
-                logger.info?.(`${PLUGIN_ID}: after_compaction — summary ${summaryLen} chars`);
-            },
-        });
-        // -----------------------------------------------------------------------
-        // Tool: mega_status
-        // -----------------------------------------------------------------------
-        api.registerTool({
-            name: "mega_status",
-            description: "Show the current status of the mega-compact engine: vector store stats, checkpoint count, and recent compaction activity.",
-            parameters: {
-                type: "object",
-                properties: {
-                    sessionId: {
-                        type: "string",
-                        description: "Optional session ID to scope stats to.",
-                    },
-                },
-                additionalProperties: false,
-            },
-            handler: async (args) => {
-                const sessionId = args?.sessionId ?? "global";
-                try {
-                    const stats = store.stats(sessionId);
-                    const parts = [
-                        `**Mega Compact Status**`,
-                        `Session: ${sessionId}`,
-                        `Checkpoints: ${stats.checkpointCount}`,
-                        `Total tokens saved: ${stats.totalTokenEstimate}`,
-                        `Last checkpoint: ${stats.lastCheckpointId ?? "—"}`,
-                        `Injected count: ${stats.injectedCount}`,
-                        `Dedup hit rate: ${(stats.dedupHitRate * 100).toFixed(0)}%`,
-                    ];
-                    if (stats.lastSummary) {
-                        parts.push(`\nLast summary (truncated):\n  ${stats.lastSummary.slice(0, 120).replace(/\n/g, " ")}…`);
-                    }
-                    return { content: [{ type: "text", text: parts.join("\n") }] };
-                }
-                catch (err) {
-                    return {
-                        content: [{ type: "text", text: `Error reading mega-compact status: ${err}` }],
-                        isError: true,
-                    };
-                }
-            },
-        });
-        // -----------------------------------------------------------------------
-        // Tool: mega_recall
-        // -----------------------------------------------------------------------
-        api.registerTool({
-            name: "mega_recall",
-            description: "Recall and inline relevant context from the mega-compact vector store for the current session.",
-            parameters: {
-                type: "object",
-                properties: {
-                    sessionId: {
-                        type: "string",
-                        description: "Session ID to recall context for.",
-                    },
-                    query: {
-                        type: "string",
-                        description: "Natural language query for relevant context.",
-                    },
-                    limit: {
-                        type: "number",
-                        description: "Max checkpoints to recall (default 3).",
-                    },
-                },
-                required: ["sessionId", "query"],
-                additionalProperties: false,
-            },
-            handler: async (args) => {
-                const { sessionId, query, limit } = args;
-                if (!sessionId || !query) {
-                    return {
-                        content: [{ type: "text", text: "Both `sessionId` and `query` are required." }],
-                        isError: true,
-                    };
-                }
-                try {
-                    const result = recallAndInline({ sessionId, query, limit: limit ?? 3, source: "command", skipInjected: false }, store);
-                    if (result.toInject.length === 0) {
-                        return {
-                            content: [{ type: "text", text: "No relevant context found in the mega-compact store." }],
-                        };
-                    }
-                    const parts = [
-                        `**Recalled ${result.toInject.length} checkpoint(s):**`,
-                        ...result.report,
-                        "",
-                        "---",
-                        result.block,
-                    ];
-                    return { content: [{ type: "text", text: parts.join("\n") }] };
-                }
-                catch (err) {
-                    return {
-                        content: [{ type: "text", text: `Error during mega-recall: ${err}` }],
-                        isError: true,
-                    };
-                }
-            },
-        });
-        // -----------------------------------------------------------------------
-        // Cleanup on shutdown
-        // -----------------------------------------------------------------------
-        api.on("shutdown", () => {
-            logger.info?.(`${PLUGIN_ID}: shutting down — clearing default store`);
-            setDefaultStore(undefined);
-        });
-        logger.info?.(`${PLUGIN_ID}: plugin registered (tools: mega_status, mega_recall)`);
-    },
-});

package/dist/src/minilm.js

@@ -1,92 +0,0 @@
-/**
- * minilm.ts — local MiniLM (all-MiniLM-L6-v2) sentence embedder (Sprint 12).
- *
- * Implements the `Embedder` interface so it drops into the existing VectorStore
- * dedup cascade and search with no call-site changes. Inference is 100% local:
- * the ONNX model + WordPiece vocab are on-disk artifacts fetched once by
- * scripts/setup-minilm.mjs. There is NO network call at runtime (PREVENT-PI-004).
- *
- * Inputs (dynamic): input_ids, attention_mask, token_type_ids (int64).
- * Output: last_hidden_state (batch, seq, 384). We mean-pool over non-padded
- * tokens (attention_mask == 1) and L2-normalize → 384-dim unit vector.
- *
- * The ONNX session + tokenizer are loaded LAZILY on first embed() so the default
- * TrigramEmbedder path (and its zero native-init cost) is untouched unless
- * MEGACOMPACT_EMBEDDER=minilm is selected.
- */
-import { join } from "node:path";
-import { homedir } from "node:os";
-import { existsSync } from "node:fs";
-import { l2Normalize, awaitSync } from "./embedder.js";
-import { WordPieceTokenizer } from "./wordpiece.js";
-export const MINILM_DIM = 384;
-export const MINILM_MAX_LEN = 256;
-/** Resolve the model directory: MEGACOMPACT_MINILM_DIR > ./models/minilm > ~/.pi … */
-function resolveModelDir() {
-    if (process.env.MEGACOMPACT_MINILM_DIR)
-        return process.env.MEGACOMPACT_MINILM_DIR;
-    // Repo-local vendored path (gitignored).
-    const local = join(process.cwd(), "models", "minilm");
-    if (existsSync(local))
-        return local;
-    return join(homedir(), ".pi", "agent", "extensions", "mega-compact", "models", "minilm");
-}
-export class MiniLMEmbedder {
-    dim = MINILM_DIM;
-    session = null;
-    tokenizer = null;
-    modelDir;
-    loadPromise = null;
-    constructor(modelDir = resolveModelDir()) {
-        this.modelDir = modelDir;
-    }
-    async ensureLoaded() {
-        if (this.session && this.tokenizer)
-            return;
-        if (this.loadPromise)
-            return this.loadPromise;
-        this.loadPromise = (async () => {
-            const ort = await import("onnxruntime-node");
-            const modelPath = join(this.modelDir, "model_quantized.onnx");
-            const vocabPath = join(this.modelDir, "vocab.txt");
-            if (!existsSync(modelPath) || !existsSync(vocabPath)) {
-                throw new Error(`MiniLM artifacts missing in ${this.modelDir}. Run: node scripts/setup-minilm.mjs`);
-            }
-            // 1 thread is plenty for a single short-region embed and bounds CPU.
-            this.session = await ort.InferenceSession.create(modelPath, {
-                executionProviders: ["cpu"],
-                graphOptimizationLevel: "all",
-            });
-            this.tokenizer = WordPieceTokenizer.fromVocabFile(vocabPath);
-        })();
-        return this.loadPromise;
-    }
-    embed(text) {
-        awaitSync(this.ensureLoaded());
-        const enc = this.tokenizer.encode(text, MINILM_MAX_LEN);
-        const n = enc.inputIds.length;
-        const BigInt64 = (arr) => arr.map((x) => BigInt(x));
-        const ort = awaitSync(import("onnxruntime-node"));
-        const tensors = {
-            input_ids: new ort.Tensor("int64", BigInt64(enc.inputIds), [1, n]),
-            attention_mask: new ort.Tensor("int64", BigInt64(enc.attentionMask), [1, n]),
-            token_type_ids: new ort.Tensor("int64", BigInt64(enc.tokenTypeIds), [1, n]),
-        };
-        const out = awaitSync(this.session.run(tensors));
-        const hidden = out.last_hidden_state.data;
-        // hidden shape: [1, n, 384]. Mean-pool over non-padded positions.
-        const pooled = new Array(MINILM_DIM).fill(0);
-        let count = 0;
-        for (let i = 0; i < n; i++) {
-            if (enc.attentionMask[i] === 0)
-                continue;
-            const base = i * MINILM_DIM;
-            for (let d = 0; d < MINILM_DIM; d++)
-                pooled[d] += hidden[base + d];
-            count++;
-        }
-        if (count === 0)
-            return l2Normalize(new Array(MINILM_DIM).fill(0));
-        return l2Normalize(pooled.map((x) => x / count));
-    }
-}

package/dist/src/wordpiece.js

@@ -1,129 +0,0 @@
-/**
- * wordpiece.ts — a self-contained WordPiece tokenizer for BERT/MiniLM.
- *
- * Loads the canonical `vocab.txt` (bert-base-uncased, ~30K tokens) from disk and
- * implements the standard uncased BERT preprocessing + greedy longest-match
- * WordPiece segmentation. No native dependency, no network — the vocab file is a
- * local artifact fetched once by scripts/setup-minilm.mjs (PREVENT-PI-004).
- *
- * This mirrors HuggingFace `BertTokenizer` closely enough for sentence-embedding
- * use: lowercase, strip accents, split on whitespace + punctuation, then
- * WordPiece each token with the `##` continuation convention. Special tokens
- * [CLS]/[SEP] are added by the caller's encode().
- */
-import { readFileSync, existsSync } from "node:fs";
-const UNK = "[UNK]";
-const CLS = "[CLS]";
-const SEP = "[SEP]";
-const PAD = "[PAD]";
-const MAX_INPUT_CHARS_PER_WORD = 200;
-export class WordPieceTokenizer {
-    vocab;
-    clsId;
-    sepId;
-    padId;
-    unkId;
-    constructor(vocab) {
-        this.vocab = vocab;
-        this.clsId = vocab.get(CLS) ?? 101;
-        this.sepId = vocab.get(SEP) ?? 102;
-        this.padId = vocab.get(PAD) ?? 0;
-        this.unkId = vocab.get(UNK) ?? 100;
-    }
-    /** Build a tokenizer from a vocab.txt file (one token per line, index = line). */
-    static fromVocabFile(path) {
-        if (!existsSync(path)) {
-            throw new Error(`WordPiece vocab not found at ${path}. Run: node scripts/setup-minilm.mjs`);
-        }
-        const lines = readFileSync(path, "utf-8").split("\n");
-        const vocab = new Map();
-        for (let i = 0; i < lines.length; i++) {
-            const tok = lines[i].replace(/\r$/, "");
-            if (tok.length > 0 || i < lines.length - 1)
-                vocab.set(tok, i);
-        }
-        return new WordPieceTokenizer(vocab);
-    }
-    /** Uncased BERT basic tokenization: lowercase, strip accents, split on ws+punct. */
-    basicTokenize(text) {
-        // NFD + strip combining marks (accent removal), then lowercase.
-        const cleaned = text
-            .normalize("NFD")
-            .replace(/[̀-ͯ]/g, "")
-            .toLowerCase();
-        const tokens = [];
-        let buf = "";
-        const flush = () => {
-            if (buf.length > 0) {
-                tokens.push(buf);
-                buf = "";
-            }
-        };
-        for (const ch of cleaned) {
-            if (/\s/.test(ch)) {
-                flush();
-            }
-            else if (/[!-/:-@[-`{-~¡-¿]/.test(ch)) {
-                // Punctuation becomes its own token.
-                flush();
-                tokens.push(ch);
-            }
-            else {
-                buf += ch;
-            }
-        }
-        flush();
-        return tokens;
-    }
-    /** Greedy longest-match WordPiece for a single word. */
-    wordpiece(word) {
-        if (word.length > MAX_INPUT_CHARS_PER_WORD)
-            return [UNK];
-        const pieces = [];
-        let start = 0;
-        while (start < word.length) {
-            let end = word.length;
-            let cur = null;
-            while (start < end) {
-                let sub = word.slice(start, end);
-                if (start > 0)
-                    sub = "##" + sub;
-                if (this.vocab.has(sub)) {
-                    cur = sub;
-                    break;
-                }
-                end--;
-            }
-            if (cur === null)
-                return [UNK]; // any unmatchable piece → whole word is UNK
-            pieces.push(cur);
-            start = end;
-        }
-        return pieces;
-    }
-    /** Tokenize text into WordPiece token strings (no special tokens). */
-    tokenize(text) {
-        const out = [];
-        for (const word of this.basicTokenize(text)) {
-            for (const piece of this.wordpiece(word))
-                out.push(piece);
-        }
-        return out;
-    }
-    /**
-     * Encode text into model inputs with [CLS]…[SEP], truncated to `maxLen`.
-     * attention_mask is all 1s (no padding for single-sequence inference).
-     */
-    encode(text, maxLen = 256) {
-        const pieces = this.tokenize(text).slice(0, Math.max(0, maxLen - 2));
-        const inputIds = [this.clsId];
-        for (const p of pieces)
-            inputIds.push(this.vocab.get(p) ?? this.unkId);
-        inputIds.push(this.sepId);
-        return {
-            inputIds,
-            attentionMask: inputIds.map(() => 1),
-            tokenTypeIds: inputIds.map(() => 0),
-        };
-    }
-}