pi-mega-compact 0.4.0

package/extensions/openclaw-mega-compact.ts

@@ -0,0 +1,370 @@
+/**
+ * openclaw-mega-compact — OpenClaw plugin adapter for the pi-mega-compact engine.
+ *
+ * Wires the pi-agnostic Trident engine (src/) into OpenClaw's plugin lifecycle:
+ *  - Registers a CompactionProvider that replaces the built-in summarizeInStages.
+ *  - Exposes `mega_status` and `mega_recall` tools for on-demand inspection.
+ *  - Hooks into `before_compaction` / `after_compaction` for diagnostics.
+ *
+ * Design constraints:
+ *  - NO imports from `@earendil-works/pi-coding-agent` or pi-agent-core.
+ *  - The engine core (src/) is pi-agnostic; this file is the sole OpenClaw boundary.
+ *  - No network at runtime — everything is local (stores + extractive summarizer).
+ */
+
+import { definePluginEntry, type OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
+import type { CompactionProvider } from "openclaw/plugin-sdk/compaction-provider";
+
+import {
+  compactSession,
+  setDefaultStore,
+  type CompactInput,
+  type CompactResult,
+} from "../src/engine.js";
+import { recallAndInline, type RecallInjectResult } from "../src/recall.js";
+import { VectorStore } from "../src/vectorStore.js";
+import type { EngineMessage } from "../src/types.js";
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const PLUGIN_ID = "mega-compact";
+const PLUGIN_LABEL = "Mega Compact (Trident)";
+
+/** Default state directory for vector store persistence. */
+const STATE_DIR = process.env.MEGA_COMPACT_STATE_DIR ?? undefined;
+
+/** Minimum messages before we bother compacting. */
+const MIN_MESSAGES_FOR_COMPACT = 6;
+
+// ---------------------------------------------------------------------------
+// Message conversion — OpenClaw unknown[] → EngineMessage[]
+// ---------------------------------------------------------------------------
+
+/**
+ * Best-effort conversion from OpenClaw's opaque message array to our
+ * EngineMessage shape. OpenClaw messages are typed as `unknown[]` so we
+ * handle whatever shape comes through gracefully.
+ */
+function toEngineMessages(messages: unknown[]): EngineMessage[] {
+  return messages.map((msg) => {
+    if (!msg || typeof msg !== "object") {
+      // Primitive fallback — treat as custom text.
+      return {
+        role: "custom" as const,
+        text: String(msg ?? ""),
+      };
+    }
+
+    const m = msg as Record<string, unknown>;
+    const role = typeof m.role === "string" ? m.role : "custom";
+
+    // Normalize role to one of our four engine roles.
+    let engineRole: EngineMessage["role"];
+    switch (role) {
+      case "user":
+        engineRole = "user";
+        break;
+      case "assistant":
+        engineRole = "assistant";
+        break;
+      case "tool":
+      case "function":
+        engineRole = "tool";
+        break;
+      default:
+        engineRole = "custom";
+        break;
+    }
+
+    // Extract text content from common message shapes.
+    const text =
+      typeof m.content === "string"
+        ? m.content
+        : typeof m.text === "string"
+          ? m.text
+          : Array.isArray(m.content)
+            ? (m.content as Array<{ type?: string; text?: string }>)
+                .filter((part) => part.type === "text" && typeof part.text === "string")
+                .map((part) => part.text)
+                .join("\n")
+            : "";
+
+    // Preserve tool metadata when present.
+    const toolName =
+      typeof m.name === "string"
+        ? m.name
+        : typeof m.toolName === "string"
+          ? m.toolName
+          : undefined;
+
+    const input =
+      typeof m.input === "string"
+        ? m.input
+        : typeof m.arguments === "string"
+          ? m.arguments
+          : m.arguments !== undefined
+            ? JSON.stringify(m.arguments)
+            : undefined;
+
+    const output =
+      typeof m.output === "string"
+        ? m.output
+        : engineRole === "tool" && typeof m.content === "string"
+          ? m.content
+          : undefined;
+
+    return { role: engineRole, text, toolName, input, output };
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Compaction provider
+// ---------------------------------------------------------------------------
+
+function createCompactionProvider(store: VectorStore): CompactionProvider {
+  return {
+    id: PLUGIN_ID,
+    label: PLUGIN_LABEL,
+
+    async summarize({
+      messages,
+      signal,
+      compressionRatio,
+    }): Promise<string> {
+      // Abort check — bail early if the caller cancelled.
+      if (signal?.aborted) {
+        throw new DOMException("Aborted", "AbortError");
+      }
+
+      const engineMessages = toEngineMessages(messages);
+
+      // Nothing meaningful to compact.
+      if (engineMessages.length < MIN_MESSAGES_FOR_COMPACT) {
+        return "";
+      }
+
+      // Map compression ratio → keepFrom boundary.
+      // compressionRatio=0.5 means "compact the oldest 50%".
+      // Default to compacting the oldest half if not specified.
+      const ratio = compressionRatio ?? 0.5;
+      const keepFrom = Math.max(
+        MIN_MESSAGES_FOR_COMPACT,
+        Math.floor(engineMessages.length * (1 - ratio)),
+      );
+
+      // Abort check after conversion (conversion is cheap but check anyway).
+      if (signal?.aborted) {
+        throw new DOMException("Aborted", "AbortError");
+      }
+
+      const sessionId = `openclaw-${Date.now()}`;
+
+      const input: CompactInput = {
+        sessionId,
+        messages: engineMessages,
+        keepFrom,
+      };
+
+      const result: CompactResult = compactSession(input, store);
+
+      if (result.skipped) {
+        return "";
+      }
+
+      return result.summary;
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Plugin entry
+// ---------------------------------------------------------------------------
+
+export default definePluginEntry({
+  id: PLUGIN_ID,
+  name: "Mega Compact",
+  description:
+    "Layered, local, vector-backed context compressor (Trident engine) for OpenClaw compaction.",
+
+  register(api: OpenClawPluginApi) {
+    const logger = api.logger;
+
+    // Resolve state directory — prefer plugin config override.
+    const pluginCfg = (api.pluginConfig ?? {}) as Record<string, unknown>;
+    const stateDir =
+      typeof pluginCfg.stateDir === "string" && pluginCfg.stateDir.length > 0
+        ? pluginCfg.stateDir
+        : STATE_DIR;
+
+    // Initialize vector store.
+    let store: VectorStore;
+    try {
+      store = new VectorStore({ stateDir });
+      setDefaultStore(store);
+      logger.info?.(`${PLUGIN_ID}: vector store initialized (stateDir=${stateDir ?? "default"})`);
+    } catch (err) {
+      logger.error?.(`${PLUGIN_ID}: failed to init vector store:`, err);
+      return; // Hard bail — no point registering if store is broken.
+    }
+
+    // -----------------------------------------------------------------------
+    // Register compaction provider
+    // -----------------------------------------------------------------------
+    const provider = createCompactionProvider(store);
+
+    api.registerCompactionProvider(provider);
+    logger.info?.(`${PLUGIN_ID}: registered compaction provider "${provider.id}"`);
+
+    // -----------------------------------------------------------------------
+    // Hooks — before / after compaction diagnostics
+    // -----------------------------------------------------------------------
+    api.registerHook({
+      event: "before_compaction",
+      handler: async (ctx) => {
+        const msgCount = Array.isArray(ctx?.messages) ? ctx.messages.length : 0;
+        logger.info?.(`${PLUGIN_ID}: before_compaction — ${msgCount} messages in scope`);
+      },
+    });
+
+    api.registerHook({
+      event: "after_compaction",
+      handler: async (ctx) => {
+        const summaryLen =
+          typeof ctx?.summary === "string" ? ctx.summary.length : 0;
+        logger.info?.(
+          `${PLUGIN_ID}: after_compaction — summary ${summaryLen} chars`,
+        );
+      },
+    });
+
+    // -----------------------------------------------------------------------
+    // Tool: mega_status
+    // -----------------------------------------------------------------------
+    api.registerTool({
+      name: "mega_status",
+      description:
+        "Show the current status of the mega-compact engine: vector store stats, checkpoint count, and recent compaction activity.",
+      parameters: {
+        type: "object",
+        properties: {
+          sessionId: {
+            type: "string",
+            description: "Optional session ID to scope stats to.",
+          },
+        },
+        additionalProperties: false,
+      },
+      handler: async (args) => {
+        const sessionId = (args as Record<string, string>)?.sessionId ?? "global";
+
+        try {
+          const stats = store.stats(sessionId);
+          const parts: string[] = [
+            `**Mega Compact Status**`,
+            `Session: ${sessionId}`,
+            `Checkpoints: ${stats.checkpointCount}`,
+            `Total tokens saved: ${stats.totalTokenEstimate}`,
+            `Last checkpoint: ${stats.lastCheckpointId ?? "—"}`,
+            `Injected count: ${stats.injectedCount}`,
+            `Dedup hit rate: ${(stats.dedupHitRate * 100).toFixed(0)}%`,
+          ];
+
+          if (stats.lastSummary) {
+            parts.push(
+              `\nLast summary (truncated):\n  ${stats.lastSummary.slice(0, 120).replace(/\n/g, " ")}…`,
+            );
+          }
+
+          return { content: [{ type: "text", text: parts.join("\n") }] };
+        } catch (err) {
+          return {
+            content: [{ type: "text", text: `Error reading mega-compact status: ${err}` }],
+            isError: true,
+          };
+        }
+      },
+    });
+
+    // -----------------------------------------------------------------------
+    // Tool: mega_recall
+    // -----------------------------------------------------------------------
+    api.registerTool({
+      name: "mega_recall",
+      description:
+        "Recall and inline relevant context from the mega-compact vector store for the current session.",
+      parameters: {
+        type: "object",
+        properties: {
+          sessionId: {
+            type: "string",
+            description: "Session ID to recall context for.",
+          },
+          query: {
+            type: "string",
+            description: "Natural language query for relevant context.",
+          },
+          limit: {
+            type: "number",
+            description: "Max checkpoints to recall (default 3).",
+          },
+        },
+        required: ["sessionId", "query"],
+        additionalProperties: false,
+      },
+      handler: async (args) => {
+        const { sessionId, query, limit } = args as {
+          sessionId: string;
+          query: string;
+          limit?: number;
+        };
+
+        if (!sessionId || !query) {
+          return {
+            content: [{ type: "text", text: "Both `sessionId` and `query` are required." }],
+            isError: true,
+          };
+        }
+
+        try {
+          const result: RecallInjectResult = recallAndInline(
+            { sessionId, query, limit: limit ?? 3, source: "command", skipInjected: false },
+            store,
+          );
+
+          if (result.toInject.length === 0) {
+            return {
+              content: [{ type: "text", text: "No relevant context found in the mega-compact store." }],
+            };
+          }
+
+          const parts: string[] = [
+            `**Recalled ${result.toInject.length} checkpoint(s):**`,
+            ...result.report,
+            "",
+            "---",
+            result.block,
+          ];
+
+          return { content: [{ type: "text", text: parts.join("\n") }] };
+        } catch (err) {
+          return {
+            content: [{ type: "text", text: `Error during mega-recall: ${err}` }],
+            isError: true,
+          };
+        }
+      },
+    });
+
+    // -----------------------------------------------------------------------
+    // Cleanup on shutdown
+    // -----------------------------------------------------------------------
+    api.on("shutdown", () => {
+      logger.info?.(`${PLUGIN_ID}: shutting down — clearing default store`);
+      setDefaultStore(undefined);
+    });
+
+    logger.info?.(`${PLUGIN_ID}: plugin registered (tools: mega_status, mega_recall)`);
+  },
+});

package/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "pi-mega-compact",
+  "version": "0.4.0",
+  "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
+  "type": "module",
+  "license": "BSD-2-Clause",
+  "author": "TheArchitectit",
+  "homepage": "https://github.com/TheArchitectit/pi-mega-compact",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/TheArchitectit/pi-mega-compact.git"
+  },
+  "keywords": [
+    "pi-extension",
+    "pi-coding-agent",
+    "openclaw-plugin",
+    "context-compaction",
+    "auto-compact",
+    "vector-store",
+    "trident"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "files": [
+    "extensions",
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
+  "pi": {
+    "extensions": [
+      "./extensions/mega-compact.ts"
+    ]
+  },
+  "openclaw": {
+    "plugin": "./extensions/openclaw-mega-compact.ts",
+    "manifest": "./openclaw.plugin.json",
+    "compactionProvider": "mega-compact"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "lint": "tsc --noEmit && node scripts/guardrails-scan.mjs",
+    "test": "npm run build && node --test \"dist/src/**/*.test.js\" \"dist/extensions/**/*.test.js\"",
+    "guardrails": "python3 scripts/regression_check.py --all || node scripts/guardrails-scan.mjs",
+    "precommit": "bash .claude/hooks/pre-commit.sh"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "openclaw": ">=0.1.0"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^20.0.0",
+    "typescript": "^5.4.0"
+  },
+  "dependencies": {
+    "@mongodb-js/zstd": "^7.0.0",
+    "better-sqlite3": "^12.11.1"
+  }
+}

package/src/adapt.ts

@@ -0,0 +1,120 @@
+/**
+ * adapt.ts — the adapter between pi's runtime message types and the engine's
+ * pi-agnostic `EngineMessage` shape.
+ *
+ * The engine (src/compact.ts, supersede.ts, boundary.ts, vectorStore.ts) only
+ * ever reasons about EngineMessage, so it stays unit-testable without a pi
+ * runtime. This module is the single conversion boundary. The conversion is
+ * 1:1 and index-aligned: every output EngineMessage corresponds to exactly one
+ * input AgentMessage at the same index, which lets the extension apply
+ * drop-range indices computed on the engine view straight back onto the real
+ * message array.
+ */
+
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import type { SessionEntry } from "@earendil-works/pi-coding-agent";
+import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
+import type { EngineMessage } from "./types.js";
+import { computeDropRange } from "./boundary.js";
+
+/** Coarse role the engine cares about. */
+export type EngineRole = "user" | "assistant" | "tool" | "custom";
+
+/** Map a pi AgentMessage role to the engine's coarse role. */
+export function messageRole(m: AgentMessage): EngineRole {
+  if (m.role === "toolResult") return "tool";
+  if (m.role === "user" || m.role === "assistant") return m.role;
+  // custom / bashExecution / branchSummary / compactionSummary — all non-LLM
+  // bookkeeping as far as the engine is concerned.
+  return "custom";
+}
+
+/** Extract a tool name from a message, if it carries a tool call/result. */
+export function messageToolName(m: AgentMessage): string | undefined {
+  if (m.role === "toolResult") return m.toolName;
+  if (m.role === "assistant") {
+    const tc = (m.content as Array<{ type: string; name?: string }>).find(
+      (c) => c.type === "toolCall",
+    );
+    return tc?.name;
+  }
+  return undefined;
+}
+
+/** Pull the text out of a string-or-blocks content field. */
+function contentText(content: string | Array<{ type: string; text?: string }>): string {
+  if (typeof content === "string") return content;
+  return content
+    .filter((c) => c.type === "text" && typeof c.text === "string")
+    .map((c) => c.text as string)
+    .join("\n");
+}
+
+/** Project any AgentMessage into a single text blob the engine can reason on. */
+function messageText(m: AgentMessage): string {
+  switch (m.role) {
+    case "toolResult":
+    case "user":
+    case "assistant":
+    case "custom":
+      return contentText((m as { content: string | Array<{ type: string; text?: string }> }).content);
+    case "bashExecution":
+      return `${(m as { command: string }).command}\n${(m as { output: string }).output}`;
+    case "branchSummary":
+    case "compactionSummary":
+      return (m as { summary: string }).summary;
+  }
+}
+
+/**
+ * Convert a pi message array into the engine's EngineMessage view, keeping
+ * index alignment (output[i] corresponds to input[i]).
+ */
+export function toEngineMessages(messages: AgentMessage[]): EngineMessage[] {
+  return messages.map((m) => {
+    const role = messageRole(m);
+    const toolName = messageToolName(m);
+    const text = messageText(m);
+    if (m.role === "toolResult") {
+      return { role, text, toolName, output: text } satisfies EngineMessage;
+    }
+    if (m.role === "assistant") {
+      const blocks = m.content as Array<{ type: string; text?: string; arguments?: unknown }>;
+      const callBlock = blocks.find((c) => c.type === "toolCall");
+      const input = callBlock
+        ? typeof callBlock.arguments === "string"
+          ? callBlock.arguments
+          : JSON.stringify(callBlock.arguments ?? {})
+        : undefined;
+      return { role, text, toolName, input } satisfies EngineMessage;
+    }
+    return { role, text, toolName } satisfies EngineMessage;
+  });
+}
+
+/**
+ * Project session entries into the engine view. Reuses pi's own
+ * sessionEntryToContextMessages so branching/compaction entries are resolved the
+ * same way the runtime would.
+ */
+export function toEngineFromEntries(entries: SessionEntry[]): EngineMessage[] {
+  return entries.flatMap((e) => toEngineMessages(sessionEntryToContextMessages(e)));
+}
+
+/**
+ * Compute the safe drop range over a pi message array using the engine's
+ * boundary guards (anchor floor + tool-pair), then return the surviving
+ * messages. Reuses the tested `computeDropRange` on an engine view and maps the
+ * indices back onto the original array (index alignment guarantees correctness).
+ */
+export function dropCompactedRange(
+  messages: AgentMessage[],
+  keepFrom: number,
+  anchorUserMessages: number,
+): AgentMessage[] {
+  if (messages.length === 0) return messages;
+  const view = toEngineMessages(messages);
+  const [, dropEnd] = computeDropRange(view, keepFrom, anchorUserMessages);
+  if (dropEnd <= 0) return messages;
+  return messages.slice(dropEnd);
+}

package/src/boundary.test.ts

@@ -0,0 +1,61 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import type { EngineMessage } from "./types.js";
+import { computeDropRange, isBoundarySafe, dropBefore } from "./boundary.js";
+
+function user(t: string): EngineMessage { return { role: "user", text: t }; }
+function assistant(t: string): EngineMessage { return { role: "assistant", text: t }; }
+function toolUse(n: string, i = "{}"): EngineMessage { return { role: "assistant", text: "", toolName: n, input: i }; }
+function toolResult(n: string, o = "ok"): EngineMessage { return { role: "tool", text: "", toolName: n, output: o }; }
+
+test("walks back so first preserved message is not an orphaned tool result", () => {
+  const messages = [
+    user("Search for files"),
+    toolUse("search"),
+    toolResult("search", "found 5 files"),
+    assistant("Done."),
+  ];
+  // keepFrom=2 would start the preserved run on the tool result at index 2,
+  // orphaning it. The guard walks back to include the assistant tool-call.
+  const [start, end] = computeDropRange(messages, 2, 0);
+  assert.equal(start, 0);
+  assert.equal(end, 1);
+  const kept = messages.slice(end);
+  assert.notEqual(kept[0].role, "tool");
+  assert.equal(kept[0].toolName, "search"); // assistant tool-call preserved
+});
+
+test("isBoundarySafe: tool result at boundary with preceding tool use is safe", () => {
+  const messages = [user("a"), toolUse("search"), toolResult("search")];
+  assert.equal(isBoundarySafe(messages, 2), true);
+});
+
+test("isBoundarySafe: orphaned tool result without preceding tool use is unsafe", () => {
+  const messages = [user("a"), toolResult("search", "orphan")];
+  assert.equal(isBoundarySafe(messages, 1), false);
+});
+
+test("anchor floor preserves the last N user messages", () => {
+  const messages = [
+    user("u1"), user("u2"), user("u3"),
+    assistant("a1"), assistant("a2"), assistant("a3"), assistant("a4"), assistant("a5"),
+  ];
+  // Caller wants to keep from index 2 (would drop u2). Anchor=2 forces keeping
+  // from u2 (index 1) onward.
+  const out = dropBefore(messages, 2, 2);
+  assert.ok(out.some((m) => m.text === "u2"));
+  assert.ok(out.some((m) => m.text === "u3"));
+});
+
+test("anchor floor is a no-op when fewer users than anchor", () => {
+  const messages = [user("u1"), assistant("a1"), assistant("a2"), assistant("a3")];
+  const out = dropBefore(messages, 1, 2);
+  // only 1 user, anchor=2 → no floor; keep from index 1 (drop the user)
+  assert.ok(!out.some((m) => m.text === "u1"));
+  assert.equal(out.length, 3);
+});
+
+test("dropBefore returns original when range is empty", () => {
+  const messages = [user("a"), assistant("b")];
+  assert.equal(dropBefore(messages, 0, 1), messages);
+});