pi-mega-compact 0.4.5 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/extensions/dashboard-server.js +450 -0
  2. package/dist/extensions/dashboard-server.test.js +111 -0
  3. package/dist/extensions/error-patterns.js +115 -0
  4. package/dist/extensions/mega-compact.js +821 -0
  5. package/dist/extensions/mega-compact.test.js +328 -0
  6. package/dist/extensions/openclaw-mega-compact.js +291 -0
  7. package/dist/src/adapt.js +106 -0
  8. package/dist/src/boundary.js +88 -0
  9. package/dist/src/boundary.test.js +53 -0
  10. package/dist/src/canary.js +118 -0
  11. package/dist/src/compact.js +250 -0
  12. package/dist/src/compact.test.js +78 -0
  13. package/dist/src/config/dedup.js +81 -0
  14. package/dist/src/config.js +12 -0
  15. package/dist/src/dedup/dedup.test.js +41 -0
  16. package/dist/src/dedup/digest.js +30 -0
  17. package/dist/src/dedup/l1-lsh.js +52 -0
  18. package/dist/src/dedup/l1-minhash.js +91 -0
  19. package/dist/src/dedup/l1-verify.js +54 -0
  20. package/dist/src/dedup/l1.test.js +50 -0
  21. package/dist/src/dedup/mmr.js +45 -0
  22. package/dist/src/dedup/normalize.js +39 -0
  23. package/dist/src/dedup/raptor/guardrails.js +83 -0
  24. package/dist/src/dedup/raptor/index.js +94 -0
  25. package/dist/src/dedup/raptor/kmeans.js +152 -0
  26. package/dist/src/dedup/raptor/raptor.test.js +205 -0
  27. package/dist/src/dedup/raptor/retrieval.js +81 -0
  28. package/dist/src/dedup/raptor/summarizer.js +85 -0
  29. package/dist/src/dedup/raptor/tree.js +177 -0
  30. package/dist/src/dedup/sprint12.test.js +219 -0
  31. package/dist/src/dedup/topk.js +60 -0
  32. package/dist/src/dedup-engine.test.js +447 -0
  33. package/dist/src/e2e.test.js +698 -0
  34. package/dist/src/embedder.js +102 -0
  35. package/dist/src/engine.js +139 -0
  36. package/dist/src/engine.test.js +111 -0
  37. package/dist/src/extractive.js +209 -0
  38. package/dist/src/extractive.test.js +130 -0
  39. package/dist/src/httpEmbedder.js +143 -0
  40. package/dist/src/log.js +47 -0
  41. package/dist/src/log.test.js +42 -0
  42. package/dist/src/minilm.js +92 -0
  43. package/dist/src/monitoring.js +131 -0
  44. package/dist/src/ratio.bench.test.js +897 -0
  45. package/dist/src/recall.integration.test.js +77 -0
  46. package/dist/src/recall.js +60 -0
  47. package/dist/src/recall.test.js +50 -0
  48. package/dist/src/sprint14.test.js +219 -0
  49. package/dist/src/store/backfill.js +189 -0
  50. package/dist/src/store/bloom.js +114 -0
  51. package/dist/src/store/compression.js +177 -0
  52. package/dist/src/store/compression.test.js +67 -0
  53. package/dist/src/store/integrity.js +44 -0
  54. package/dist/src/store/migrate.js +79 -0
  55. package/dist/src/store/migrate.test.js +139 -0
  56. package/dist/src/store/sprint10.test.js +186 -0
  57. package/dist/src/store/sqlite.js +574 -0
  58. package/dist/src/store.js +115 -0
  59. package/dist/src/store.test.js +142 -0
  60. package/dist/src/supersede.js +68 -0
  61. package/dist/src/supersede.test.js +36 -0
  62. package/dist/src/tokens.js +31 -0
  63. package/dist/src/types.js +8 -0
  64. package/dist/src/types.test.js +9 -0
  65. package/dist/src/vectorStore.js +465 -0
  66. package/dist/src/vectorStore.test.js +479 -0
  67. package/dist/src/wordpiece.js +129 -0
  68. package/extensions/mega-compact.ts +47 -11
  69. package/package.json +4 -2
  70. package/src/engine.ts +5 -0
@@ -0,0 +1,328 @@
1
+ /**
2
+ * mega-compact.extension.test.ts — end-to-end drive of the REAL extension
3
+ * entry (extensions/mega-compact.ts) through a faithful mock pi.
4
+ *
5
+ * This is the closest we get to "a live pi session" without a model: it
6
+ * loads the compiled extension, captures its event/command handlers, and
7
+ * fires them with mock ctx objects — proving the three compact layers
8
+ * (auto-trigger -> compactSession) AND the three recall entries all
9
+ * route through the real code, not just the unit-tested src/ modules.
10
+ *
11
+ * Uses a per-test isolated state dir (process.env.MEGACOMPACT_STATE_DIR)
12
+ * so concurrent node --test runs do not collide on disk.
13
+ */
14
+ import { test } from "node:test";
15
+ import assert from "node:assert/strict";
16
+ import { mkdtempSync, rmSync } from "node:fs";
17
+ import { tmpdir } from "node:os";
18
+ import { join } from "node:path";
19
+ import { createRequire } from "node:module";
20
+ const require = createRequire(import.meta.url);
21
+ const baseTmp = mkdtempSync(join(tmpdir(), "mc-ext-"));
22
+ let counter = 0;
23
+ /** Build a mock pi + ctx and load the extension into them. */
24
+ function harness(opts = {}) {
25
+ const stateDir = join(baseTmp, `run-${counter++}`);
26
+ process.env.MEGACOMPACT_STATE_DIR = stateDir;
27
+ process.env.MEGACOMPACT_DEBUG = "true";
28
+ // Low threshold so the auto-trigger gate trips on our small mock context.
29
+ // Tier tests opt out (keepTier/keepThreshold) so they can drive the real
30
+ // tier resolution instead of the forced 50-token threshold.
31
+ if (!opts.keepThreshold)
32
+ process.env.MEGACOMPACT_THRESHOLD_TOKENS = "50";
33
+ if (!opts.keepTier)
34
+ delete process.env.MEGACOMPACT_TIER;
35
+ process.env.MEGACOMPACT_FAST_GATE_PCT = "1";
36
+ const handlers = {};
37
+ const commands = {};
38
+ const appended = [];
39
+ let statusKey;
40
+ let statusText;
41
+ const notifies = [];
42
+ // Minimal AgentMessage factory for the session we project into the extension.
43
+ function msg(role, text, toolName) {
44
+ if (role === "assistant" && toolName) {
45
+ return { role: "assistant", content: [{ type: "toolCall", name: toolName, id: "c1", arguments: {} }], api: "anthropic-messages", provider: "anthropic", model: "m", usage: { inputTokens: 1, outputTokens: 1, cacheReadTokens: 0, cacheWriteTokens: 0 }, stopReason: "tool_use", timestamp: 0 };
46
+ }
47
+ if (role === "toolResult" && toolName) {
48
+ return { role: "toolResult", toolCallId: "c1", toolName, content: [{ type: "text", text }], isError: false, timestamp: 0 };
49
+ }
50
+ return { role: "user", content: text, timestamp: 0 };
51
+ }
52
+ const session = [
53
+ msg("user", "read src/vec.ts and understand the index"),
54
+ msg("assistant", "ok", "Read"),
55
+ msg("user", "edit src/vec.ts to add a cosine helper"),
56
+ msg("assistant", "ok", "Edit"),
57
+ msg("user", "now fix the dedupe bug in store.ts"),
58
+ msg("assistant", "ok", "Edit"),
59
+ msg("user", "actually we should add recall sorting too"),
60
+ msg("assistant", "ok", "Edit"),
61
+ ];
62
+ // Mirror the REAL SessionManager: getEntries() returns SessionEntry objects,
63
+ // which the extension projects to messages via the SDK's
64
+ // sessionEntryToContextMessages(entry). The harness must use the same shape
65
+ // (type:"message" with a .message) or recentUserQuery() silently queries "".
66
+ const toEntry = (m, i) => ({
67
+ type: "message",
68
+ id: `e${i}`,
69
+ parentId: null,
70
+ timestamp: String(i),
71
+ message: m,
72
+ });
73
+ const sessionManager = {
74
+ getSessionId: () => "sess_ext_001",
75
+ getEntries: () => session.map(toEntry),
76
+ };
77
+ function makeCtx(over = {}) {
78
+ return {
79
+ ui: {
80
+ setStatus: (k, t) => { statusKey = k; statusText = t; },
81
+ notify: (s) => notifies.push(s),
82
+ select: () => { },
83
+ confirm: async () => true,
84
+ input: async () => "",
85
+ setWidget: () => { },
86
+ },
87
+ mode: "tui",
88
+ hasUI: true,
89
+ cwd: stateDir,
90
+ sessionManager,
91
+ modelRegistry: {},
92
+ model: undefined,
93
+ isIdle: () => true,
94
+ isProjectTrusted: () => true,
95
+ signal: undefined,
96
+ abort: () => { },
97
+ hasPendingMessages: () => false,
98
+ shutdown: () => { },
99
+ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }),
100
+ compact: () => { },
101
+ getSystemPrompt: () => "system base",
102
+ ...over,
103
+ };
104
+ }
105
+ const pi = {
106
+ on: (ev, h) => { handlers[ev] = h; },
107
+ registerCommand: (name, opts) => { commands[name] = opts; },
108
+ registerTool: () => { },
109
+ registerShortcut: () => { },
110
+ registerFlag: () => { },
111
+ getFlag: () => undefined,
112
+ registerMessageRenderer: () => { },
113
+ registerEntryRenderer: () => { },
114
+ sendMessage: (_m) => { },
115
+ sendUserMessage: () => { },
116
+ appendEntry: (t, d) => appended.push({ t, d }),
117
+ setSessionName: () => { },
118
+ getSessionName: () => undefined,
119
+ setLabel: () => { },
120
+ exec: async () => ({ stdout: "", stderr: "", code: 0 }),
121
+ getActiveTools: () => [],
122
+ getAllTools: () => [],
123
+ setActiveTools: () => { },
124
+ getCommands: () => [],
125
+ setModel: async () => false,
126
+ getThinkingLevel: () => "off",
127
+ setThinkingLevel: () => { },
128
+ };
129
+ // Import the compiled extension (same dist/extensions dir as this test).
130
+ const mod = require("./mega-compact.js");
131
+ mod.default(pi);
132
+ return {
133
+ stateDir, handlers, commands, appended, get status() { return { statusKey, statusText }; }, notifies,
134
+ fire: (ev, event, ctx) => handlers[ev](event, ctx),
135
+ ctx: makeCtx,
136
+ session,
137
+ };
138
+ }
139
+ test("auto-trigger: past threshold persists a chkpt and drops context", async () => {
140
+ const h = harness();
141
+ const messages = h.session;
142
+ const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
143
+ const res = await h.fire("context", { type: "context", messages }, ctx);
144
+ // L1->L4 ran: a checkpoint was persisted to the SQLite store + a marker entry written.
145
+ const { listCheckpoints } = await import("../src/store/sqlite.js");
146
+ assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "checkpoint persisted to local vector db");
147
+ assert.equal(h.appended.some((a) => a.t === "mega-compact-marker"), true, "marker sentinel appended");
148
+ // Context dropped (the compacted range was trimmed).
149
+ assert.ok(res && Array.isArray(res.messages), "context handler returns filtered messages");
150
+ assert.ok(res.messages.length < messages.length, "outgoing context shrank");
151
+ });
152
+ test("session_before_compact cancels once we've persisted", async () => {
153
+ const h = harness();
154
+ const ctx = h.ctx();
155
+ // First fire the auto-trigger so a checkpoint is persisted this session.
156
+ await h.fire("context", { type: "context", messages: h.session }, h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) }));
157
+ // Now pi tries to compact natively — we must cancel (no double-compact).
158
+ const res = await h.fire("session_before_compact", { type: "session_before_compact", reason: "overflow", willRetry: true, preparation: {}, signal: undefined }, ctx);
159
+ assert.deepEqual(res, { cancel: true });
160
+ });
161
+ test("session_before_compact does NOT cancel when nothing persisted", async () => {
162
+ const h = harness();
163
+ const ctx = h.ctx();
164
+ // Do NOT fire context first; this session has no checkpoint.
165
+ const res = await h.fire("session_before_compact", { type: "session_before_compact", reason: "threshold", willRetry: false, preparation: {}, signal: undefined }, ctx);
166
+ assert.deepEqual(res, {});
167
+ });
168
+ test("resume auto-inline stages recall into the system prompt", async () => {
169
+ const h = harness();
170
+ // Seed a checkpoint first (simulate a prior session that compacted).
171
+ await h.fire("context", { type: "context", messages: h.session }, h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) }));
172
+ // Fresh resume: session_start with reason "resume".
173
+ const ctx = h.ctx();
174
+ await h.fire("session_start", { type: "session_start", reason: "resume", previousSessionFile: undefined }, ctx);
175
+ // The next before_agent_start must prepend the recalled block.
176
+ const res = await h.fire("before_agent_start", { type: "before_agent_start", prompt: "base system", images: undefined, systemPrompt: "base system", systemPromptOptions: {} }, ctx);
177
+ assert.ok(res && typeof res.systemPrompt === "string", "before_agent_start returns a systemPrompt");
178
+ assert.ok(res.systemPrompt.includes("Recalled context"), "recalled block injected into system prompt");
179
+ });
180
+ test("/recall-context reports and stages the top checkpoint", async () => {
181
+ const h = harness();
182
+ await h.fire("context", { type: "context", messages: h.session }, h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) }));
183
+ const ctx = h.ctx();
184
+ await h.commands["mega-recall"].handler("dedupe bug store.ts", ctx);
185
+ assert.ok(h.notifies.some((n) => n.includes("recall staged")), "command reports staged checkpoints");
186
+ assert.ok(h.notifies.some((n) => n.includes("chkpt_")), "command names the checkpoint");
187
+ });
188
+ test("/megacompact-status reports live store stats", async () => {
189
+ const h = harness();
190
+ await h.fire("context", { type: "context", messages: h.session }, h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) }));
191
+ const ctx = h.ctx({ getContextUsage: () => ({ tokens: 50000, contextWindow: 200000, percent: 25 }) });
192
+ await h.commands["mega-status"].handler("", ctx);
193
+ assert.ok(h.notifies.some((n) => n.includes("store:") && n.includes("chkpt")), "status shows checkpoint count");
194
+ });
195
+ // ---- Named compaction tiers -------------------------------------------------
196
+ // low=50k, medium=100k, high=200k, ultra=1M, mega=10M. Driven through the REAL
197
+ // loadConfig()/status path by setting MEGACOMPACT_TIER before loading the ext.
198
+ const TIER_CASES = [
199
+ ["low", 50_000],
200
+ ["medium", 100_000],
201
+ ["high", 200_000],
202
+ ["ultra", 1_000_000],
203
+ ["mega", 10_000_000],
204
+ ];
205
+ for (const [tier, threshold] of TIER_CASES) {
206
+ test(`tier "${tier}" resolves to a ${threshold}-token threshold`, async () => {
207
+ // Keep tier + keep threshold UNSET so the tier (not an explicit number)
208
+ // drives the threshold. harness() would otherwise reset the threshold.
209
+ delete process.env.MEGACOMPACT_THRESHOLD_TOKENS;
210
+ process.env.MEGACOMPACT_TIER = tier;
211
+ const h = harness({ keepTier: true, keepThreshold: true });
212
+ const ctx = h.ctx({ getContextUsage: () => ({ tokens: 1, contextWindow: 2_000_000, percent: 0.01 }) });
213
+ await h.commands["mega-status"].handler("", ctx);
214
+ delete process.env.MEGACOMPACT_TIER;
215
+ assert.ok(h.notifies.some((n) => n.includes(`tier=${tier}`) && n.includes(`threshold=${threshold}`)), `status should report tier=${tier} threshold=${threshold}`);
216
+ });
217
+ }
218
+ test("explicit MEGACOMPACT_THRESHOLD_TOKENS overrides the tier", async () => {
219
+ process.env.MEGACOMPACT_TIER = "mega";
220
+ process.env.MEGACOMPACT_THRESHOLD_TOKENS = "777";
221
+ const h = harness({ keepTier: true, keepThreshold: true });
222
+ const ctx = h.ctx({ getContextUsage: () => ({ tokens: 1, contextWindow: 2_000_000, percent: 0.01 }) });
223
+ await h.commands["mega-status"].handler("", ctx);
224
+ delete process.env.MEGACOMPACT_TIER;
225
+ assert.ok(h.notifies.some((n) => n.includes("tier=custom") && n.includes("threshold=777")), "explicit threshold wins over tier (tier=custom)");
226
+ });
227
+ // ---- /dashboard commands ----------------------------------------------------
228
+ test("/dashboard-status reports no server when pid file missing", async () => {
229
+ const h = harness();
230
+ const ctx = h.ctx();
231
+ await h.commands["mega-dashboard-status"].handler("", ctx);
232
+ assert.ok(h.notifies.some((n) => n.includes("not running")), "reports no server running");
233
+ });
234
+ test("/dashboard-stop reports no server when pid file missing", async () => {
235
+ const h = harness();
236
+ const ctx = h.ctx();
237
+ await h.commands["mega-dashboard-stop"].handler("", ctx);
238
+ assert.ok(h.notifies.some((n) => n.includes("no dashboard server running")), "reports no server");
239
+ });
240
+ test("/dashboard skips server spawn when already running", async () => {
241
+ const h = harness();
242
+ const confirms = [];
243
+ // Set up a fake HTTP server at a random port
244
+ const { createServer } = await import("node:http");
245
+ const server = createServer((_req, res) => {
246
+ res.writeHead(200, { "Content-Type": "application/json" });
247
+ res.end(JSON.stringify({ updatedAt: new Date().toISOString(), tier: "test", version: 1, config: {}, session: {}, context: {}, trigger: {}, store: {} }));
248
+ });
249
+ await new Promise((r) => server.listen(0, "127.0.0.1", r));
250
+ const addr = server.address();
251
+ const { join: j } = await import("node:path");
252
+ const { writeFileSync: wf } = await import("node:fs");
253
+ wf(j(h.stateDir, "port.pid"), JSON.stringify({ port: addr.port, pid: process.pid }));
254
+ const ctx = h.ctx({
255
+ ui: {
256
+ setStatus: () => { },
257
+ notify: (s) => { h.notifies.push(s); },
258
+ select: () => { },
259
+ confirm: async () => { confirms.push(true); return true; },
260
+ input: async () => "",
261
+ },
262
+ });
263
+ await h.commands["mega-dashboard"].handler("", ctx);
264
+ assert.ok(h.notifies.some((n) => n.includes("already running")), "reports already running");
265
+ assert.ok(confirms.length > 0, "confirm dialog was shown");
266
+ await new Promise((r) => server.close(() => r()));
267
+ });
268
+ test("/dashboard-status reports running after dashboard start", async () => {
269
+ const h = harness();
270
+ // Write a fake port.pid with a real port (use a server we control)
271
+ const { createServer } = await import("node:http");
272
+ const { join: j } = await import("node:path");
273
+ const { writeFileSync: wf } = await import("node:fs");
274
+ const server = createServer((_req, res) => {
275
+ res.writeHead(200, { "Content-Type": "application/json" });
276
+ res.end(JSON.stringify({ updatedAt: new Date().toISOString(), tier: "test" }));
277
+ });
278
+ await new Promise((r) => server.listen(0, "127.0.0.1", r));
279
+ const addr = server.address();
280
+ wf(j(h.stateDir, "port.pid"), JSON.stringify({ port: addr.port, pid: process.pid }));
281
+ const ctx = h.ctx();
282
+ await h.commands["mega-dashboard-status"].handler("", ctx);
283
+ assert.ok(h.notifies.some((n) => n.includes("running") && n.includes(String(addr.port))), "reports running with port");
284
+ await new Promise((r) => server.close(() => r()));
285
+ });
286
+ test("state snapshot writes dashboard.json after compaction", async () => {
287
+ const h = harness();
288
+ const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
289
+ // Fire auto-trigger compaction (context event above 80% threshold)
290
+ await h.fire("context", { type: "context", messages: h.session }, ctx);
291
+ const { existsSync: ex, readFileSync: rf } = await import("node:fs");
292
+ const { join: j } = await import("node:path");
293
+ const snapPath = j(h.stateDir, "dashboard.json");
294
+ assert.ok(ex(snapPath), "dashboard.json written after compaction");
295
+ const snap = JSON.parse(rf(snapPath, "utf-8"));
296
+ // Item B: the honest token model is wired — the original dropped region was
297
+ // captured (originalTokens > 0), and the saved amount never exceeds the
298
+ // original (saved = max(0, original − stored) ≤ original). For this tiny
299
+ // harness session the summary can be ≥ the region, so saved may be 0; the
300
+ // positive "saved > 0" case with a large region is covered by the
301
+ // vectorStore unit tests.
302
+ assert.ok(snap.store.originalTokens > 0, "snapshot.store.originalTokens captured after compaction");
303
+ assert.ok(snap.store.originalTokens >= snap.store.tokensSaved, "model invariant: original region >= tokens saved");
304
+ // Item A: crew (live agent) block is present in the dashboard snapshot.
305
+ assert.ok(snap.crew && typeof snap.crew.activeAgents === "number", "snapshot.crew.activeAgents present");
306
+ });
307
+ test("events.log receives compaction events", async () => {
308
+ const h = harness();
309
+ const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
310
+ // Fire auto-trigger compaction twice (first fires compaction, second also fires)
311
+ await h.fire("context", { type: "context", messages: h.session }, ctx);
312
+ const { readFileSync: rf, existsSync: ex } = await import("node:fs");
313
+ const { join: j } = await import("node:path");
314
+ const logPath = j(h.stateDir, "events.log");
315
+ if (ex(logPath)) {
316
+ const content = rf(logPath, "utf-8").trim();
317
+ // At minimum, we expect at least one event logged
318
+ assert.ok(content.length > 0, "events.log is non-empty after compaction");
319
+ }
320
+ else {
321
+ // events.log may not exist if the DashboardEmitter path differs from stateDir;
322
+ // verify dashboard.json was written (proves the post-compact path executed)
323
+ assert.ok(ex(j(h.stateDir, "dashboard.json")), "dashboard.json proves post-compact ran");
324
+ }
325
+ });
326
+ test("cleanup", () => {
327
+ rmSync(baseTmp, { recursive: true, force: true });
328
+ });
@@ -0,0 +1,291 @@
1
+ /**
2
+ * openclaw-mega-compact — OpenClaw plugin adapter for the pi-mega-compact engine.
3
+ *
4
+ * Wires the pi-agnostic Trident engine (src/) into OpenClaw's plugin lifecycle:
5
+ * - Registers a CompactionProvider that replaces the built-in summarizeInStages.
6
+ * - Exposes `mega_status` and `mega_recall` tools for on-demand inspection.
7
+ * - Hooks into `before_compaction` / `after_compaction` for diagnostics.
8
+ *
9
+ * Design constraints:
10
+ * - NO imports from `@earendil-works/pi-coding-agent` or pi-agent-core.
11
+ * - The engine core (src/) is pi-agnostic; this file is the sole OpenClaw boundary.
12
+ * - No network at runtime — everything is local (stores + extractive summarizer).
13
+ */
14
+ import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
15
+ import { compactSession, setDefaultStore, } from "../src/engine.js";
16
+ import { recallAndInline } from "../src/recall.js";
17
+ import { VectorStore } from "../src/vectorStore.js";
18
+ // ---------------------------------------------------------------------------
19
+ // Constants
20
+ // ---------------------------------------------------------------------------
21
+ const PLUGIN_ID = "mega-compact";
22
+ const PLUGIN_LABEL = "Mega Compact (Trident)";
23
+ /** Default state directory for vector store persistence. */
24
+ const STATE_DIR = process.env.MEGA_COMPACT_STATE_DIR ?? undefined;
25
+ /** Minimum messages before we bother compacting. */
26
+ const MIN_MESSAGES_FOR_COMPACT = 6;
27
+ // ---------------------------------------------------------------------------
28
+ // Message conversion — OpenClaw unknown[] → EngineMessage[]
29
+ // ---------------------------------------------------------------------------
30
+ /**
31
+ * Best-effort conversion from OpenClaw's opaque message array to our
32
+ * EngineMessage shape. OpenClaw messages are typed as `unknown[]` so we
33
+ * handle whatever shape comes through gracefully.
34
+ */
35
+ function toEngineMessages(messages) {
36
+ return messages.map((msg) => {
37
+ if (!msg || typeof msg !== "object") {
38
+ // Primitive fallback — treat as custom text.
39
+ return {
40
+ role: "custom",
41
+ text: String(msg ?? ""),
42
+ };
43
+ }
44
+ const m = msg;
45
+ const role = typeof m.role === "string" ? m.role : "custom";
46
+ // Normalize role to one of our four engine roles.
47
+ let engineRole;
48
+ switch (role) {
49
+ case "user":
50
+ engineRole = "user";
51
+ break;
52
+ case "assistant":
53
+ engineRole = "assistant";
54
+ break;
55
+ case "tool":
56
+ case "function":
57
+ engineRole = "tool";
58
+ break;
59
+ default:
60
+ engineRole = "custom";
61
+ break;
62
+ }
63
+ // Extract text content from common message shapes.
64
+ const text = typeof m.content === "string"
65
+ ? m.content
66
+ : typeof m.text === "string"
67
+ ? m.text
68
+ : Array.isArray(m.content)
69
+ ? m.content
70
+ .filter((part) => part.type === "text" && typeof part.text === "string")
71
+ .map((part) => part.text)
72
+ .join("\n")
73
+ : "";
74
+ // Preserve tool metadata when present.
75
+ const toolName = typeof m.name === "string"
76
+ ? m.name
77
+ : typeof m.toolName === "string"
78
+ ? m.toolName
79
+ : undefined;
80
+ const input = typeof m.input === "string"
81
+ ? m.input
82
+ : typeof m.arguments === "string"
83
+ ? m.arguments
84
+ : m.arguments !== undefined
85
+ ? JSON.stringify(m.arguments)
86
+ : undefined;
87
+ const output = typeof m.output === "string"
88
+ ? m.output
89
+ : engineRole === "tool" && typeof m.content === "string"
90
+ ? m.content
91
+ : undefined;
92
+ return { role: engineRole, text, toolName, input, output };
93
+ });
94
+ }
95
+ // ---------------------------------------------------------------------------
96
+ // Compaction provider
97
+ // ---------------------------------------------------------------------------
98
+ function createCompactionProvider(store) {
99
+ return {
100
+ id: PLUGIN_ID,
101
+ label: PLUGIN_LABEL,
102
+ async summarize({ messages, signal, compressionRatio, }) {
103
+ // Abort check — bail early if the caller cancelled.
104
+ if (signal?.aborted) {
105
+ throw new DOMException("Aborted", "AbortError");
106
+ }
107
+ const engineMessages = toEngineMessages(messages);
108
+ // Nothing meaningful to compact.
109
+ if (engineMessages.length < MIN_MESSAGES_FOR_COMPACT) {
110
+ return "";
111
+ }
112
+ // Map compression ratio → keepFrom boundary.
113
+ // compressionRatio=0.5 means "compact the oldest 50%".
114
+ // Default to compacting the oldest half if not specified.
115
+ const ratio = compressionRatio ?? 0.5;
116
+ const keepFrom = Math.max(MIN_MESSAGES_FOR_COMPACT, Math.floor(engineMessages.length * (1 - ratio)));
117
+ // Abort check after conversion (conversion is cheap but check anyway).
118
+ if (signal?.aborted) {
119
+ throw new DOMException("Aborted", "AbortError");
120
+ }
121
+ const sessionId = `openclaw-${Date.now()}`;
122
+ const input = {
123
+ sessionId,
124
+ messages: engineMessages,
125
+ keepFrom,
126
+ };
127
+ const result = compactSession(input, store);
128
+ if (result.skipped) {
129
+ return "";
130
+ }
131
+ return result.summary;
132
+ },
133
+ };
134
+ }
135
+ // ---------------------------------------------------------------------------
136
+ // Plugin entry
137
+ // ---------------------------------------------------------------------------
138
+ export default definePluginEntry({
139
+ id: PLUGIN_ID,
140
+ name: "Mega Compact",
141
+ description: "Layered, local, vector-backed context compressor (Trident engine) for OpenClaw compaction.",
142
+ register(api) {
143
+ const logger = api.logger;
144
+ // Resolve state directory — prefer plugin config override.
145
+ const pluginCfg = (api.pluginConfig ?? {});
146
+ const stateDir = typeof pluginCfg.stateDir === "string" && pluginCfg.stateDir.length > 0
147
+ ? pluginCfg.stateDir
148
+ : STATE_DIR;
149
+ // Initialize vector store.
150
+ let store;
151
+ try {
152
+ store = new VectorStore({ stateDir });
153
+ setDefaultStore(store);
154
+ logger.info?.(`${PLUGIN_ID}: vector store initialized (stateDir=${stateDir ?? "default"})`);
155
+ }
156
+ catch (err) {
157
+ logger.error?.(`${PLUGIN_ID}: failed to init vector store:`, err);
158
+ return; // Hard bail — no point registering if store is broken.
159
+ }
160
+ // -----------------------------------------------------------------------
161
+ // Register compaction provider
162
+ // -----------------------------------------------------------------------
163
+ const provider = createCompactionProvider(store);
164
+ api.registerCompactionProvider(provider);
165
+ logger.info?.(`${PLUGIN_ID}: registered compaction provider "${provider.id}"`);
166
+ // -----------------------------------------------------------------------
167
+ // Hooks — before / after compaction diagnostics
168
+ // -----------------------------------------------------------------------
169
+ api.registerHook({
170
+ event: "before_compaction",
171
+ handler: async (ctx) => {
172
+ const msgCount = Array.isArray(ctx?.messages) ? ctx.messages.length : 0;
173
+ logger.info?.(`${PLUGIN_ID}: before_compaction — ${msgCount} messages in scope`);
174
+ },
175
+ });
176
+ api.registerHook({
177
+ event: "after_compaction",
178
+ handler: async (ctx) => {
179
+ const summaryLen = typeof ctx?.summary === "string" ? ctx.summary.length : 0;
180
+ logger.info?.(`${PLUGIN_ID}: after_compaction — summary ${summaryLen} chars`);
181
+ },
182
+ });
183
+ // -----------------------------------------------------------------------
184
+ // Tool: mega_status
185
+ // -----------------------------------------------------------------------
186
+ api.registerTool({
187
+ name: "mega_status",
188
+ description: "Show the current status of the mega-compact engine: vector store stats, checkpoint count, and recent compaction activity.",
189
+ parameters: {
190
+ type: "object",
191
+ properties: {
192
+ sessionId: {
193
+ type: "string",
194
+ description: "Optional session ID to scope stats to.",
195
+ },
196
+ },
197
+ additionalProperties: false,
198
+ },
199
+ handler: async (args) => {
200
+ const sessionId = args?.sessionId ?? "global";
201
+ try {
202
+ const stats = store.stats(sessionId);
203
+ const parts = [
204
+ `**Mega Compact Status**`,
205
+ `Session: ${sessionId}`,
206
+ `Checkpoints: ${stats.checkpointCount}`,
207
+ `Total tokens saved: ${stats.totalTokenEstimate}`,
208
+ `Last checkpoint: ${stats.lastCheckpointId ?? "—"}`,
209
+ `Injected count: ${stats.injectedCount}`,
210
+ `Dedup hit rate: ${(stats.dedupHitRate * 100).toFixed(0)}%`,
211
+ ];
212
+ if (stats.lastSummary) {
213
+ parts.push(`\nLast summary (truncated):\n ${stats.lastSummary.slice(0, 120).replace(/\n/g, " ")}…`);
214
+ }
215
+ return { content: [{ type: "text", text: parts.join("\n") }] };
216
+ }
217
+ catch (err) {
218
+ return {
219
+ content: [{ type: "text", text: `Error reading mega-compact status: ${err}` }],
220
+ isError: true,
221
+ };
222
+ }
223
+ },
224
+ });
225
+ // -----------------------------------------------------------------------
226
+ // Tool: mega_recall
227
+ // -----------------------------------------------------------------------
228
+ api.registerTool({
229
+ name: "mega_recall",
230
+ description: "Recall and inline relevant context from the mega-compact vector store for the current session.",
231
+ parameters: {
232
+ type: "object",
233
+ properties: {
234
+ sessionId: {
235
+ type: "string",
236
+ description: "Session ID to recall context for.",
237
+ },
238
+ query: {
239
+ type: "string",
240
+ description: "Natural language query for relevant context.",
241
+ },
242
+ limit: {
243
+ type: "number",
244
+ description: "Max checkpoints to recall (default 3).",
245
+ },
246
+ },
247
+ required: ["sessionId", "query"],
248
+ additionalProperties: false,
249
+ },
250
+ handler: async (args) => {
251
+ const { sessionId, query, limit } = args;
252
+ if (!sessionId || !query) {
253
+ return {
254
+ content: [{ type: "text", text: "Both `sessionId` and `query` are required." }],
255
+ isError: true,
256
+ };
257
+ }
258
+ try {
259
+ const result = recallAndInline({ sessionId, query, limit: limit ?? 3, source: "command", skipInjected: false }, store);
260
+ if (result.toInject.length === 0) {
261
+ return {
262
+ content: [{ type: "text", text: "No relevant context found in the mega-compact store." }],
263
+ };
264
+ }
265
+ const parts = [
266
+ `**Recalled ${result.toInject.length} checkpoint(s):**`,
267
+ ...result.report,
268
+ "",
269
+ "---",
270
+ result.block,
271
+ ];
272
+ return { content: [{ type: "text", text: parts.join("\n") }] };
273
+ }
274
+ catch (err) {
275
+ return {
276
+ content: [{ type: "text", text: `Error during mega-recall: ${err}` }],
277
+ isError: true,
278
+ };
279
+ }
280
+ },
281
+ });
282
+ // -----------------------------------------------------------------------
283
+ // Cleanup on shutdown
284
+ // -----------------------------------------------------------------------
285
+ api.on("shutdown", () => {
286
+ logger.info?.(`${PLUGIN_ID}: shutting down — clearing default store`);
287
+ setDefaultStore(undefined);
288
+ });
289
+ logger.info?.(`${PLUGIN_ID}: plugin registered (tools: mega_status, mega_recall)`);
290
+ },
291
+ });