pi-mega-compact 0.4.5 → 0.4.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/dist/extensions/dashboard-server.js +450 -0
  2. package/dist/extensions/dashboard-server.test.js +111 -0
  3. package/dist/extensions/error-patterns.js +115 -0
  4. package/dist/extensions/mega-compact.js +782 -0
  5. package/dist/extensions/mega-compact.test.js +328 -0
  6. package/dist/extensions/openclaw-mega-compact.js +291 -0
  7. package/dist/src/adapt.js +106 -0
  8. package/dist/src/boundary.js +88 -0
  9. package/dist/src/boundary.test.js +53 -0
  10. package/dist/src/canary.js +118 -0
  11. package/dist/src/compact.js +250 -0
  12. package/dist/src/compact.test.js +78 -0
  13. package/dist/src/config/dedup.js +81 -0
  14. package/dist/src/config.js +12 -0
  15. package/dist/src/dedup/dedup.test.js +41 -0
  16. package/dist/src/dedup/digest.js +30 -0
  17. package/dist/src/dedup/l1-lsh.js +52 -0
  18. package/dist/src/dedup/l1-minhash.js +91 -0
  19. package/dist/src/dedup/l1-verify.js +54 -0
  20. package/dist/src/dedup/l1.test.js +50 -0
  21. package/dist/src/dedup/mmr.js +45 -0
  22. package/dist/src/dedup/normalize.js +39 -0
  23. package/dist/src/dedup/raptor/guardrails.js +83 -0
  24. package/dist/src/dedup/raptor/index.js +94 -0
  25. package/dist/src/dedup/raptor/kmeans.js +152 -0
  26. package/dist/src/dedup/raptor/raptor.test.js +205 -0
  27. package/dist/src/dedup/raptor/retrieval.js +81 -0
  28. package/dist/src/dedup/raptor/summarizer.js +85 -0
  29. package/dist/src/dedup/raptor/tree.js +177 -0
  30. package/dist/src/dedup/sprint12.test.js +219 -0
  31. package/dist/src/dedup/topk.js +60 -0
  32. package/dist/src/dedup-engine.test.js +447 -0
  33. package/dist/src/e2e.test.js +698 -0
  34. package/dist/src/embedder.js +102 -0
  35. package/dist/src/engine.js +137 -0
  36. package/dist/src/engine.test.js +111 -0
  37. package/dist/src/extractive.js +209 -0
  38. package/dist/src/extractive.test.js +130 -0
  39. package/dist/src/httpEmbedder.js +143 -0
  40. package/dist/src/log.js +47 -0
  41. package/dist/src/log.test.js +42 -0
  42. package/dist/src/minilm.js +92 -0
  43. package/dist/src/monitoring.js +131 -0
  44. package/dist/src/ratio.bench.test.js +897 -0
  45. package/dist/src/recall.integration.test.js +77 -0
  46. package/dist/src/recall.js +60 -0
  47. package/dist/src/recall.test.js +50 -0
  48. package/dist/src/sprint14.test.js +219 -0
  49. package/dist/src/store/backfill.js +189 -0
  50. package/dist/src/store/bloom.js +114 -0
  51. package/dist/src/store/compression.js +177 -0
  52. package/dist/src/store/compression.test.js +67 -0
  53. package/dist/src/store/integrity.js +44 -0
  54. package/dist/src/store/migrate.js +79 -0
  55. package/dist/src/store/migrate.test.js +139 -0
  56. package/dist/src/store/sprint10.test.js +186 -0
  57. package/dist/src/store/sqlite.js +574 -0
  58. package/dist/src/store.js +115 -0
  59. package/dist/src/store.test.js +142 -0
  60. package/dist/src/supersede.js +68 -0
  61. package/dist/src/supersede.test.js +36 -0
  62. package/dist/src/tokens.js +31 -0
  63. package/dist/src/types.js +8 -0
  64. package/dist/src/types.test.js +9 -0
  65. package/dist/src/vectorStore.js +465 -0
  66. package/dist/src/vectorStore.test.js +479 -0
  67. package/dist/src/wordpiece.js +129 -0
  68. package/package.json +4 -2
@@ -0,0 +1,782 @@
1
+ /**
2
+ * pi-mega-compact — layered, local, vector-backed context compressor.
3
+ *
4
+ * Extension entry. Wires the pi-agnostic Trident engine (src/) into pi's
5
+ * extension lifecycle.
6
+ *
7
+ * Design constraints (from RESEARCH.md):
8
+ * - No network at runtime (PREVENT-PI-004).
9
+ * - pi Message has no system-role entry (PREVENT-PI-003); inject compacted
10
+ * context via `before_agent_start` systemPrompt (Sprint 4), or a
11
+ * `compactionSummary` message so it renders like native compaction.
12
+ * - Message drops must preserve an anchor floor (PREVENT-PI-001) and never
13
+ * split a toolCall/toolResult pair (PREVENT-PI-002).
14
+ *
15
+ * Sprint 3 wires: config, session state reset, the auto-trigger pipeline
16
+ * (fast-gate → auto_compact_check → Trident+persist → context drop),
17
+ * session_before_compact cancellation, the compact-marker sentinel, and the
18
+ * /megacompact + /megacompact-status commands.
19
+ *
20
+ * Sprint 4 wires the unified recall layer (Layer 5): recallAndInline() is the
21
+ * ONLY code path that injects compacted context. It serves three entry points —
22
+ * auto-inline on resume/branch (before_agent_start), on-demand /recall-context,
23
+ * and the dedup sentinel — all through one dedup engine, injected via the
24
+ * before_agent_start systemPrompt prepend (PREVENT-PI-003).
25
+ */
26
+ import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
27
+ import { join, dirname } from "node:path";
28
+ import { fileURLToPath } from "node:url";
29
+ import { STATE_DIR_DEFAULT } from "../src/config.js";
30
+ import { VectorStore } from "../src/vectorStore.js";
31
+ import { toEngineMessages, dropCompactedRange } from "../src/adapt.js";
32
+ import { compactSession } from "../src/engine.js";
33
+ import { recallAndInline } from "../src/recall.js";
34
+ import { autoCompactCheck } from "../src/compact.js";
35
+ import { estimateSessionTokens } from "../src/tokens.js";
36
+ import { normalizeSessionId } from "../src/store.js";
37
+ import { touchSession, logDaily } from "../src/store/sqlite.js";
38
+ import { Logger } from "../src/log.js";
39
+ import { writeFileSync, appendFileSync, readFileSync } from "node:fs";
40
+ import { existsSync, mkdirSync, unlinkSync } from "node:fs";
41
+ import { spawn } from "node:child_process"; // guardrails-allow PREVENT-PI-004: spawns the optional, user-triggered localhost dashboard server only
42
+ import { execSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: read-only `git rev-parse` to scope the store per-repo
43
+ const STATUS_KEY = "mega-compact";
44
+ const WIDGET_KEY = "mega-compact-stats";
45
+ const MARKER_TYPE = "mega-compact-marker";
46
+ function envFlag(name, fallback) {
47
+ const v = process.env[name];
48
+ if (v == null || v === "")
49
+ return fallback;
50
+ const n = Number(v);
51
+ return Number.isFinite(n) ? n : fallback;
52
+ }
53
+ function envBool(name, fallback) {
54
+ const v = process.env[name];
55
+ if (v == null || v === "")
56
+ return fallback;
57
+ return v === "true" || v === "1";
58
+ }
59
+ /**
60
+ * Named compaction tiers. A tier sets the token threshold at which the
61
+ * auto-trigger persists a checkpoint; pick by how aggressively you want the
62
+ * session trimmed. Explicit MEGACOMPACT_THRESHOLD_TOKENS always wins.
63
+ */
64
+ const COMPACT_TIERS = {
65
+ low: 50_000,
66
+ medium: 100_000,
67
+ high: 200_000,
68
+ ultra: 1_000_000,
69
+ mega: 10_000_000,
70
+ };
71
+ /** Resolve the effective token threshold from TIER (or explicit) env vars. */
72
+ function resolveThreshold() {
73
+ const explicit = process.env.MEGACOMPACT_THRESHOLD_TOKENS;
74
+ if (explicit != null && explicit !== "") {
75
+ const n = Number(explicit);
76
+ if (Number.isFinite(n))
77
+ return { tier: "custom", thresholdTokens: n };
78
+ }
79
+ const raw = (process.env.MEGACOMPACT_TIER ?? "low").toLowerCase();
80
+ const tier = (raw in COMPACT_TIERS ? raw : "low");
81
+ return { tier, thresholdTokens: COMPACT_TIERS[tier] };
82
+ }
83
+ function loadConfig() {
84
+ const { tier, thresholdTokens } = resolveThreshold();
85
+ return {
86
+ tier,
87
+ // Global default; the live store/dashboard are rebound per-repo at runtime
88
+ // via bindRepo() so each git repo gets its own isolated state dir.
89
+ stateDir: process.env.MEGACOMPACT_STATE_DIR ?? STATE_DIR_DEFAULT,
90
+ fastGatePct: envFlag("MEGACOMPACT_FAST_GATE_PCT", 70),
91
+ thresholdTokens,
92
+ anchorUserMessages: envFlag("MEGACOMPACT_ANCHOR_USER_MESSAGES", 3),
93
+ preserveRecent: envFlag("MEGACOMPACT_PRESERVE_RECENT", 4),
94
+ auto: envBool("MEGACOMPACT_AUTO", true),
95
+ autoInline: envBool("MEGACOMPACT_AUTO_INLINE", true),
96
+ autoInlineK: envFlag("MEGACOMPACT_AUTO_INLINE_K", 3),
97
+ dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
98
+ debug: envBool("MEGACOMPACT_DEBUG", false),
99
+ };
100
+ }
101
+ /**
102
+ * Resolve the current repo's git root from a cwd. Returns undefined for a
103
+ * non-git directory (caller falls back to a global state dir).
104
+ */
105
+ function resolveRepoRoot(cwd) {
106
+ try {
107
+ const out = execSync("git rev-parse --show-toplevel", {
108
+ cwd,
109
+ encoding: "utf-8",
110
+ stdio: ["ignore", "pipe", "ignore"],
111
+ }).trim();
112
+ return out || undefined;
113
+ }
114
+ catch {
115
+ return undefined;
116
+ }
117
+ }
118
+ /**
119
+ * Per-repo state dir: <repo>/.pi/mega-compact (tracked, so it travels with the
120
+ * repo across devices — not gitignored). Falls back to `fallback` for non-git
121
+ * cwds (the explicit MEGACOMPACT_STATE_DIR override, if set).
122
+ */
123
+ function repoStateDir(cwd, fallback) {
124
+ const root = resolveRepoRoot(cwd);
125
+ if (!root)
126
+ return fallback;
127
+ return join(root, ".pi", "mega-compact");
128
+ }
129
+ class Dashboard {
130
+ snapshotPath;
131
+ eventsPath;
132
+ constructor(stateDir) {
133
+ if (!existsSync(stateDir))
134
+ mkdirSync(stateDir, { recursive: true });
135
+ this.snapshotPath = join(stateDir, "dashboard.json");
136
+ this.eventsPath = join(stateDir, "events.log");
137
+ }
138
+ /** Write a full state snapshot (atomically replaces previous). */
139
+ snapshot(data) {
140
+ writeFileSync(this.snapshotPath, JSON.stringify(data, null, 2) + "\n");
141
+ }
142
+ /** Append a timestamped JSONL event line. */
143
+ event(type, data) {
144
+ const line = JSON.stringify({ ts: new Date().toISOString(), type, ...data });
145
+ appendFileSync(this.eventsPath, line + "\n");
146
+ }
147
+ }
148
+ /** Convert the messages pi hands us in the `context` event into the engine view. */
149
+ function engineView(messages) {
150
+ return toEngineMessages(messages);
151
+ }
152
+ export default function (pi) {
153
+ const config = loadConfig();
154
+ // Store/dashboard/logger are rebound per-repo by bindRepo() (below) so each
155
+ // git repo gets its own isolated state dir. They start bound to the global
156
+ // default until the first handler resolves a cwd.
157
+ let store = new VectorStore({ dedupSim: config.dedupSim, stateDir: config.stateDir });
158
+ let logger = new Logger({ enabled: config.debug, path: join(config.stateDir, "mega-compact.log") });
159
+ let dashboard = new Dashboard(config.stateDir);
160
+ let activeRepoRoot = null;
161
+ let currentStateDir = config.stateDir;
162
+ /**
163
+ * Point store/dashboard/logger at the current repo's state dir. Rebuilds the
164
+ * instances only when the repo root changes, so cross-repo dedup stats, db,
165
+ * and events are fully isolated. Falls back to the global default outside git.
166
+ */
167
+ function bindRepo(cwd) {
168
+ const dir = cwd ? repoStateDir(cwd, config.stateDir) : config.stateDir;
169
+ const key = cwd ? resolveRepoRoot(cwd) ?? dir : dir;
170
+ if (key === activeRepoRoot)
171
+ return dir;
172
+ activeRepoRoot = key;
173
+ currentStateDir = dir;
174
+ store = new VectorStore({ dedupSim: config.dedupSim, stateDir: dir });
175
+ logger = new Logger({ enabled: config.debug, path: join(dir, "mega-compact.log") });
176
+ dashboard = new Dashboard(dir);
177
+ return dir;
178
+ }
179
+ // --- snapshot() helper: collect live state and write it to disk ---
180
+ let lastCtxTokens = null;
181
+ let lastCtxPercent = null;
182
+ let lastCtxWindow = 0;
183
+ function snapshot(ctx) {
184
+ if (ctx)
185
+ bindRepo(ctx.cwd);
186
+ const st = store.stats(rt.sessionId);
187
+ const repo = store.repoStats();
188
+ const armed = lastCtxPercent != null && lastCtxPercent >= config.fastGatePct;
189
+ const ready = armed && (lastCtxTokens ?? 0) >= config.thresholdTokens;
190
+ dashboard.snapshot({
191
+ version: 1,
192
+ updatedAt: new Date().toISOString(),
193
+ tier: config.tier,
194
+ config: {
195
+ fastGatePct: config.fastGatePct,
196
+ thresholdTokens: config.thresholdTokens,
197
+ anchorUserMessages: config.anchorUserMessages,
198
+ preserveRecent: config.preserveRecent,
199
+ auto: config.auto,
200
+ autoInline: config.autoInline,
201
+ },
202
+ session: {
203
+ id: rt.sessionId,
204
+ state: statusKey ?? "idle",
205
+ persistedThisSession: rt.persistedThisSession,
206
+ lastCheckpointId: rt.lastCheckpointId ?? null,
207
+ lastCompactedFrom: rt.lastCompactedFrom,
208
+ lastCompactedTokens: rt.lastCompactedTokens,
209
+ dedupSkips: rt.dedupSkips,
210
+ dedupAttempts: rt.dedupAttempts,
211
+ },
212
+ context: { tokens: lastCtxTokens, percent: lastCtxPercent, contextWindow: lastCtxWindow },
213
+ trigger: { armed, ready, currentTokens: lastCtxTokens, thresholdTokens: config.thresholdTokens, fastGatePct: config.fastGatePct },
214
+ crew: { activeAgents, currentTurn },
215
+ store: { checkpointCount: st.checkpointCount, totalTokenEstimate: st.totalTokenEstimate, originalTokens: st.originalTokens, tokensSaved: rt.tokensSaved, injectedCount: st.injectedCount, dedupHitRate: st.dedupHitRate, storageDedupRate: st.storageDedupRate, dedupAttempts: st.dedupAttempts, dedupCollapsed: st.dedupCollapsed },
216
+ repo: {
217
+ checkpointCount: repo.checkpointCount,
218
+ totalTokenEstimate: repo.totalTokenEstimate,
219
+ originalTokens: repo.originalTokens,
220
+ tokensSaved: repo.tokensSaved,
221
+ sessionCount: repo.sessionCount,
222
+ dedupAttempts: repo.dedupAttempts,
223
+ dedupCollapsed: repo.dedupCollapsed,
224
+ storageDedupRate: repo.storageDedupRate,
225
+ },
226
+ });
227
+ // Live stats widget above the editor
228
+ if (ctx) {
229
+ const tokStr = lastCtxTokens != null ? `${Math.round(lastCtxTokens / 1000)}k` : "?";
230
+ const maxStr = lastCtxWindow > 0 ? `${Math.round(lastCtxWindow / 1000)}k` : "?";
231
+ const pctStr = lastCtxPercent != null ? `${Math.round(lastCtxPercent * 10) / 10}%` : "?%";
232
+ const triggerLabel = ready ? "● ready" : armed ? "◐ armed" : "○ idle";
233
+ // Storage dedup rate is cumulative (store-wide, per-repo) and survives
234
+ // session resets. Always show a number: 0% before any compaction, a
235
+ // decimal for sub-10% rates so small-but-real dedup isn't rounded away.
236
+ const storageRate = st.storageDedupRate; // 0..1
237
+ const dedupStr = storageRate * 100 >= 10
238
+ ? `${Math.round(storageRate * 100)}%`
239
+ : `${(storageRate * 100).toFixed(1)}%`;
240
+ // saved = tokens removed from context (cumulative original − stored).
241
+ // Show BOTH this-session (rt.tokensSaved) and repo-wide-total
242
+ // (repo.tokensSaved) so the user sees per-session progress vs the running
243
+ // repo total. "used" = stored checkpoint tokens (repo.totalTokenEstimate
244
+ // vs st.totalTokenEstimate). Use "k" only at/above 1000 so small-but-real
245
+ // numbers stay visible (previously Math.round(x/1000) zeroed <1000).
246
+ const fmt = (x) => (x >= 1000 ? `${(x / 1000).toFixed(1)}k` : `${x}`);
247
+ const savedStr = `${fmt(rt.tokensSaved)} sess / ${fmt(repo.tokensSaved)} repo`;
248
+ const usedStr = `${fmt(st.totalTokenEstimate)} sess / ${fmt(repo.totalTokenEstimate)} repo`;
249
+ const agentStr = activeAgents > 0 ? ` │ 🤖 ${activeAgents} agent${activeAgents === 1 ? "" : "s"}` : "";
250
+ const turnStr = currentTurn > 0 ? ` │ turn ${currentTurn}` : "";
251
+ ctx.ui.setWidget(WIDGET_KEY, [
252
+ ` ⚡ ${config.tier} │ ${tokStr}/${maxStr} tokens (${pctStr}) │ ${st.checkpointCount} chkpt${st.checkpointCount === 1 ? "" : "s"}${agentStr}${turnStr}`,
253
+ ` ${triggerLabel} │ dedup: ${dedupStr} │ used: ${usedStr} │ saved: ${savedStr}`,
254
+ ], { placement: "aboveEditor" });
255
+ }
256
+ }
257
+ // The only mutable per-session state. Reset on session_start / session_tree.
258
+ let rt = {
259
+ sessionId: normalizeSessionId(undefined),
260
+ persistedThisSession: false,
261
+ lastCheckpointId: undefined,
262
+ lastCompactedFrom: 0,
263
+ lastCompactedTokens: 0,
264
+ dedupSkips: 0,
265
+ dedupAttempts: 0,
266
+ tokensSaved: 0,
267
+ };
268
+ let debounceUntil = 0;
269
+ // Agent tracking for real-time widget updates
270
+ let activeAgents = 0;
271
+ let currentTurn = 0;
272
+ // Recall block produced by auto-inline (resume/branch) that the next
273
+ // before_agent_start should prepend to the system prompt. Unset after use.
274
+ let pendingRecallBlock;
275
+ let statusKey; // current status text for dashboard
276
+ function setStatus(ctx, text) {
277
+ statusKey = text;
278
+ ctx.ui.setStatus(STATUS_KEY, text);
279
+ }
280
+ function resetRuntime(sessionId) {
281
+ const sid = normalizeSessionId(sessionId);
282
+ if (rt.sessionId === sid && rt.persistedThisSession)
283
+ return; // same session, keep checkpoint memory
284
+ rt = {
285
+ sessionId: sid,
286
+ persistedThisSession: false,
287
+ lastCheckpointId: undefined,
288
+ lastCompactedFrom: 0,
289
+ lastCompactedTokens: 0,
290
+ dedupSkips: 0,
291
+ dedupAttempts: 0,
292
+ tokensSaved: 0,
293
+ };
294
+ statusKey = undefined;
295
+ activeAgents = 0;
296
+ currentTurn = 0;
297
+ }
298
+ /** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
299
+ function runCompact(ctx, messages, opts = {}) {
300
+ bindRepo(ctx.cwd);
301
+ const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
302
+ resetRuntime(sid);
303
+ rt.sessionId = sid;
304
+ const view = engineView(messages);
305
+ const keepFrom = opts.keepFrom ?? Math.max(0, view.length - config.preserveRecent);
306
+ if (keepFrom <= 0)
307
+ return { skipped: true };
308
+ const result = compactSession({
309
+ sessionId: sid,
310
+ messages: view,
311
+ keepFrom,
312
+ summary: opts.summary,
313
+ timestamp: Date.now(),
314
+ }, store);
315
+ if (result.skipped)
316
+ return { skipped: true };
317
+ if (!result.deduped) {
318
+ rt.persistedThisSession = true;
319
+ rt.lastCheckpointId = result.checkpointId;
320
+ }
321
+ rt.lastCompactedFrom = result.compactedFrom;
322
+ rt.lastCompactedTokens = result.tokenEstimate;
323
+ rt.dedupAttempts++;
324
+ // Honest "tokens saved" for this session-instance only:
325
+ // new checkpoint → original − stored
326
+ // deduped onto existing → whole original region (nothing new stored)
327
+ // Resets to 0 on session_start (rt is rebuilt) — so a fresh session shows 0
328
+ // while the repo's cumulative saved (SQLite meta) keeps the running total.
329
+ const saved = result.deduped
330
+ ? result.originalTokenEstimate
331
+ : Math.max(0, result.originalTokenEstimate - result.tokenEstimate);
332
+ rt.tokensSaved += saved;
333
+ if (result.deduped)
334
+ rt.dedupSkips++;
335
+ // Record session activity + a daily-log entry in the per-repo SQLite store
336
+ // (foundation for resume-sessions / daily-log features). Best-effort — never
337
+ // block a compaction on bookkeeping.
338
+ try {
339
+ const repo = resolveRepoRoot(ctx.cwd);
340
+ touchSession(sid, repo, currentStateDir);
341
+ logDaily(sid, "compact", result.checkpointId, saved, currentStateDir);
342
+ }
343
+ catch {
344
+ /* non-fatal: stats bookkeeping only */
345
+ }
346
+ // Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
347
+ // skip re-vectorizing an already-compacted region (zero token cost).
348
+ pi.appendEntry(MARKER_TYPE, {
349
+ checkpointId: result.checkpointId,
350
+ regionHash: result.regionHash,
351
+ tokenEstimate: result.tokenEstimate,
352
+ deduped: result.deduped,
353
+ });
354
+ setStatus(ctx, rt.persistedThisSession
355
+ ? `mega-compact: ${result.checkpointId} · ${saved} tok saved`
356
+ : `mega-compact: ready`);
357
+ logger.info("compact", {
358
+ sessionId: sid,
359
+ checkpointId: result.checkpointId ?? "(deduped)",
360
+ deduped: result.deduped,
361
+ tokenEstimate: saved,
362
+ compactedFrom: result.compactedFrom,
363
+ });
364
+ dashboard.event("compact", {
365
+ sessionId: sid,
366
+ checkpointId: result.checkpointId ?? "(deduped)",
367
+ deduped: result.deduped,
368
+ tokenEstimate: saved,
369
+ compactedFrom: result.compactedFrom,
370
+ });
371
+ snapshot(ctx);
372
+ return { skipped: false, result, keepFrom, saved };
373
+ }
374
+ /**
375
+ * Unified recall (Layer 5). The ONE path that injects. Returns the recall
376
+ * result; callers decide whether to stage it for before_agent_start (resume)
377
+ * or report it (command).
378
+ */
379
+ function doRecall(ctx, query, source) {
380
+ bindRepo(ctx.cwd);
381
+ const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
382
+ const result = recallAndInline({ sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true }, store);
383
+ dashboard.event("recall", { source, query: query.slice(0, 120), injected: result.toInject.length, empty: result.empty });
384
+ return result;
385
+ }
386
+ // ---- Session lifecycle (state reset points) -------------------------------
387
+ pi.on("session_start", async (event, ctx) => {
388
+ resetRuntime(ctx.sessionManager.getSessionId());
389
+ setStatus(ctx, config.auto ? "mega-compact: ready" : "mega-compact: manual only");
390
+ // Auto-inline on resume/fork/continue: stage the most relevant checkpoints
391
+ // so the next before_agent_start prepends them to the system prompt.
392
+ // Triggered whenever this session already has persisted checkpoints AND a
393
+ // usable query — that covers reason "resume"/"fork" (explicit) and
394
+ // reason "startup" (e.g. `pi --continue`s an existing session, which still
395
+ // emits "startup" but with a populated message window). A brand-new empty
396
+ // session has no checkpoints, so it's naturally excluded.
397
+ if (config.autoInline) {
398
+ const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
399
+ const query = recentUserQuery(ctx);
400
+ if (query && store.stats(sid).checkpointCount > 0) {
401
+ const r = doRecall(ctx, query, "resume");
402
+ if (!r.empty) {
403
+ pendingRecallBlock = r.block;
404
+ setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt`);
405
+ logger.info("auto-inline", { reason: event.reason, query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
406
+ }
407
+ }
408
+ }
409
+ dashboard.event("session_start", { reason: event.reason, sessionId: rt.sessionId });
410
+ snapshot(ctx);
411
+ });
412
+ pi.on("session_tree", async (_event, ctx) => {
413
+ // Branch navigation invalidates region indexes — reset checkpoint memory but
414
+ // keep the on-disk store (markers replayed from entries below if needed).
415
+ resetRuntime(ctx.sessionManager.getSessionId());
416
+ setStatus(ctx, "mega-compact: ready (branch)");
417
+ if (config.autoInline) {
418
+ const query = recentUserQuery(ctx);
419
+ if (query) {
420
+ const r = doRecall(ctx, query, "resume");
421
+ if (!r.empty) {
422
+ pendingRecallBlock = r.block;
423
+ logger.info("auto-inline", { reason: "session_tree", query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
424
+ }
425
+ }
426
+ }
427
+ dashboard.event("session_tree", { sessionId: rt.sessionId });
428
+ snapshot(ctx);
429
+ });
430
+ // ---- Auto-inline injection point: prepend staged recall to systemPrompt ----
431
+ pi.on("before_agent_start", async (event, _ctx) => {
432
+ if (!pendingRecallBlock)
433
+ return;
434
+ const block = pendingRecallBlock;
435
+ pendingRecallBlock = undefined; // one-shot: consume so we never double-inject
436
+ return { systemPrompt: `${event.systemPrompt}\n\n${block}` };
437
+ });
438
+ pi.on("session_shutdown", async (_event, ctx) => {
439
+ setStatus(ctx, undefined);
440
+ activeAgents = 0;
441
+ currentTurn = 0;
442
+ ctx.ui.setWidget(WIDGET_KEY, [], { placement: "aboveEditor" });
443
+ });
444
+ // ---- Agent tracking for real-time widget + status-line updates ---------
445
+ pi.on("agent_start", async (_event, ctx) => {
446
+ activeAgents++;
447
+ dashboard.event("agent_start", { activeAgents });
448
+ // Surface live agent activity on the status line (toolbar), not just the
449
+ // above-editor widget — otherwise concurrent agents look frozen.
450
+ setStatus(ctx, `mega-compact: ▶ ${activeAgents} agent${activeAgents === 1 ? "" : "s"}`);
451
+ snapshot(ctx);
452
+ });
453
+ pi.on("agent_end", async (_event, ctx) => {
454
+ activeAgents = Math.max(0, activeAgents - 1);
455
+ dashboard.event("agent_end", { activeAgents });
456
+ if (activeAgents > 0) {
457
+ setStatus(ctx, `mega-compact: ▶ ${activeAgents} agent${activeAgents === 1 ? "" : "s"}`);
458
+ }
459
+ else {
460
+ setStatus(ctx, config.auto ? "mega-compact: ready" : "mega-compact: manual only");
461
+ }
462
+ snapshot(ctx);
463
+ });
464
+ pi.on("turn_start", async (event, ctx) => {
465
+ currentTurn = event.turnIndex;
466
+ dashboard.event("turn_start", { turnIndex: event.turnIndex });
467
+ snapshot(ctx);
468
+ });
469
+ pi.on("turn_end", async (event, ctx) => {
470
+ dashboard.event("turn_end", { turnIndex: event.turnIndex });
471
+ snapshot(ctx);
472
+ });
473
+ // ---- Auto-trigger: fast-gate → confirm → Trident+persist → drop --------
474
+ pi.on("context", async (event, ctx) => {
475
+ if (!config.auto)
476
+ return;
477
+ const usage = ctx.getContextUsage();
478
+ const pct = usage?.percent;
479
+ // Always track context for the dashboard, even if we return early below.
480
+ lastCtxTokens = usage?.tokens ?? null;
481
+ lastCtxPercent = pct ?? null;
482
+ lastCtxWindow = usage?.contextWindow ?? 0;
483
+ snapshot(ctx);
484
+ if (pct == null)
485
+ return;
486
+ const messages = event.messages;
487
+ const view = engineView(messages);
488
+ // Prefer the runtime's real token estimate; fall back to our heuristic
489
+ // (and to a percent-of-window proxy when tokens is unknown).
490
+ const currentTokens = usage?.tokens ?? estimateSessionTokens(view) ??
491
+ Math.round((pct / 100) * (usage?.contextWindow ?? 0));
492
+ // FAST GATE: token-based (tier threshold), not percentage-based.
493
+ // A 20% gate on a 2M window = 400k, which is way above the 50k low-tier
494
+ // threshold. Gate on the actual token count instead.
495
+ if (currentTokens < config.thresholdTokens)
496
+ return;
497
+ const check = autoCompactCheck(currentTokens, config.thresholdTokens); // SERVER-STYLE CONFIRM (local)
498
+ if (!check.shouldCompact)
499
+ return;
500
+ // Debounce so we don't fire on every context event past threshold.
501
+ const now = Date.now();
502
+ if (now < debounceUntil)
503
+ return;
504
+ debounceUntil = now + 2000;
505
+ const ran = runCompact(ctx, messages);
506
+ if (ran.skipped)
507
+ return;
508
+ // DROP the compacted range from the outgoing context, honoring the anchor
509
+ // floor + tool-pair boundary guards (PREVENT-PI-001/002).
510
+ const kept = dropCompactedRange(messages, ran.keepFrom, config.anchorUserMessages);
511
+ if (kept.length < messages.length) {
512
+ return { messages: kept };
513
+ }
514
+ });
515
+ // ---- Cancel native compaction once we've persisted our own -------------
516
+ pi.on("session_before_compact", async (_event, ctx) => {
517
+ resetRuntime(ctx.sessionManager.getSessionId());
518
+ if (rt.persistedThisSession) {
519
+ // We already persisted a checkpoint for this session (via the context
520
+ // hook drop) — cancel pi's own compaction to avoid double-compacting.
521
+ // Our context-hook drop already trimmed the window.
522
+ return { cancel: true };
523
+ }
524
+ // We haven't persisted yet this session: let pi run its native compaction.
525
+ // (Our auto-trigger only fires again past the threshold, and will then
526
+ // capture a checkpoint next time around.)
527
+ return {};
528
+ });
529
+ // ---- Commands ----------------------------------------------------------
530
+ pi.registerCommand("mega-compact", {
531
+ description: "Compress current session context into the local vector store.",
532
+ handler: async (args, ctx) => {
533
+ const sessionEntries = ctx.sessionManager.getEntries();
534
+ // Project entries (branch-aware) into the message view.
535
+ const messages = sessionEntries.flatMap((e) => sessionEntryToContextMessages(e));
536
+ const summaryArg = args.trim();
537
+ const ran = runCompact(ctx, messages, summaryArg ? { summary: summaryArg } : {});
538
+ if ("skipped" in ran && ran.skipped) {
539
+ ctx.ui.notify("[mega-compact] Nothing to compact (session too small).");
540
+ return;
541
+ }
542
+ const r = ran.result;
543
+ ctx.ui.notify(`[mega-compact] ${r.deduped ? "region already compacted (deduped)" : `persisted ${r.checkpointId}`} · ` +
544
+ `${r.tokenEstimate} tok · ${currentStateDir}`);
545
+ },
546
+ });
547
+ pi.registerCommand("mega-recall", {
548
+ description: "Recall relevant compacted context from the vector store and inline it.",
549
+ handler: async (args, ctx) => {
550
+ const query = args.trim() || recentUserQuery(ctx);
551
+ if (!query) {
552
+ ctx.ui.notify("[mega-compact] /mega-recall needs a query or a prior user message.");
553
+ return;
554
+ }
555
+ const r = doRecall(ctx, query, "command");
556
+ if (r.empty) {
557
+ logger.info("recall-empty", { query });
558
+ ctx.ui.notify(`[mega-compact] recall found nothing new for "${query}".`);
559
+ return;
560
+ }
561
+ // Stage the block so the next before_agent_start prepends it (actual
562
+ // injection). Report what was selected now for immediate feedback.
563
+ pendingRecallBlock = r.block;
564
+ const list = r.report.map((l) => l).join("\n");
565
+ logger.info("recall", { query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
566
+ setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt`);
567
+ ctx.ui.notify(`[mega-compact] recall staged ${r.toInject.length} checkpoint(s) for "${query}":\n${list}\n` +
568
+ `(injected at the next turn via system prompt)`);
569
+ },
570
+ });
571
+ pi.registerCommand("mega-status", {
572
+ description: "Show mega-compact config and current context usage.",
573
+ handler: async (_args, ctx) => {
574
+ bindRepo(ctx.cwd);
575
+ const usage = ctx.getContextUsage();
576
+ const pct = usage?.percent != null ? `${usage.percent}%` : "n/a";
577
+ const tokens = usage?.tokens != null ? `${usage.tokens} tok` : "n/a";
578
+ const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
579
+ const st = store.stats(sid);
580
+ ctx.ui.notify(`[mega-compact] pct=${pct} tokens=${tokens} tier=${config.tier} fastGate=${config.fastGatePct}% ` +
581
+ `threshold=${config.thresholdTokens} auto=${config.auto} autoInline=${config.autoInline}\n` +
582
+ `[mega-compact] store: ${st.checkpointCount} chkpt · ` +
583
+ `${st.totalTokenEstimate} tok · last=${st.lastCheckpointId ?? "—"} · ` +
584
+ `injected=${st.injectedCount} · dedup=${(st.dedupHitRate * 100).toFixed(0)}%\n` +
585
+ `[mega-compact] anchor=${config.anchorUserMessages} preserveRecent=${config.preserveRecent} ` +
586
+ `autoInlineK=${config.autoInlineK} dedupSim=${config.dedupSim} debug=${config.debug}\n` +
587
+ `[mega-compact] stateDir=${currentStateDir}`);
588
+ },
589
+ });
590
+ pi.registerCommand("mega-tier", {
591
+ description: "Show or change the compaction tier at runtime. Usage: /mega-tier [low|medium|high|ultra|mega]",
592
+ handler: async (args, ctx) => {
593
+ const arg = args.trim().toLowerCase();
594
+ if (!arg) {
595
+ // Show current tier and available options.
596
+ ctx.ui.notify(`[mega-compact] current tier: ${config.tier} (${config.thresholdTokens} tok)\n` +
597
+ `[mega-compact] available tiers: ${Object.entries(COMPACT_TIERS).map(([k, v]) => `${k}=${v}`).join(", ")}`);
598
+ return;
599
+ }
600
+ if (!(arg in COMPACT_TIERS)) {
601
+ ctx.ui.notify(`[mega-compact] unknown tier "${arg}". Available: ${Object.keys(COMPACT_TIERS).join(", ")}`);
602
+ return;
603
+ }
604
+ const newTier = arg;
605
+ config.tier = newTier;
606
+ config.thresholdTokens = COMPACT_TIERS[newTier];
607
+ setStatus(ctx, `mega-compact: tier → ${newTier} (${config.thresholdTokens} tok)`);
608
+ ctx.ui.notify(`[mega-compact] tier changed to ${newTier} (threshold: ${config.thresholdTokens} tokens)`);
609
+ snapshot(ctx);
610
+ },
611
+ });
612
+ // ---- Dashboard server commands ----------------------------------------
613
+ const portFile = join(currentStateDir, "port.pid");
614
+ const runnerFile = join(currentStateDir, "_dashboard-runner.mjs");
615
+ /** Try to reach a running dashboard server. Returns { port, url } or null. */
616
+ async function isServerRunning() {
617
+ if (!existsSync(portFile))
618
+ return null;
619
+ try {
620
+ const info = JSON.parse(readFileSync(portFile, "utf-8"));
621
+ if (!info?.port)
622
+ return null;
623
+ const url = `http://localhost:${info.port}`; // guardrails-allow PREVENT-PI-004: localhost URL of the dashboard server this extension spawned
624
+ // Quick liveness probe
625
+ const res = await fetch(`${url}/api/snapshot`, { signal: AbortSignal.timeout(1500) }); // guardrails-allow PREVENT-PI-004: localhost probe to the dashboard server this extension spawned
626
+ if (res.ok)
627
+ return { port: info.port, url };
628
+ }
629
+ catch {
630
+ // stale or unreachable — clean up
631
+ try {
632
+ unlinkSync(portFile);
633
+ }
634
+ catch { /* ignore */ }
635
+ }
636
+ return null;
637
+ }
638
+ /** Write a small ESM runner script that imports and launches the dashboard server. */
639
+ function writeRunnerScript() {
640
+ // The npm package ships SOURCE only (no dist/), so the launchable entry is
641
+ // the .ts file — not dashboard-server.js (which only exists after a local
642
+ // build). dashboard-server.ts imports only Node built-ins, so it loads under
643
+ // node's --experimental-strip-types without any other compiled code. The
644
+ // child is spawned with that flag (see the spawn below) so the import below
645
+ // resolves from the source install path.
646
+ const sourceServer = join(dirname(fileURLToPath(import.meta.url)), "dashboard-server.ts");
647
+ const script = [
648
+ `import { launchDashboardServer } from ${JSON.stringify(sourceServer)};`,
649
+ `launchDashboardServer(${JSON.stringify(currentStateDir)}).catch(err => {`,
650
+ ` console.error("[mega-compact] dashboard failed:", err);`,
651
+ ` process.exit(1);`,
652
+ `});`,
653
+ ].join("\n");
654
+ writeFileSync(runnerFile, script);
655
+ }
656
+ /** Open a URL in the default browser. Platform-aware. Uses spawn (not exec) to avoid shell injection. */
657
+ function openBrowser(url) {
658
+ const cmd = process.platform === "darwin" ? "open" :
659
+ process.platform === "win32" ? "start" :
660
+ "xdg-open";
661
+ try {
662
+ spawn(cmd, [url], { detached: true, stdio: "ignore" }).unref();
663
+ }
664
+ catch {
665
+ /* non-fatal — user can open manually */
666
+ }
667
+ }
668
+ pi.registerCommand("mega-dashboard", {
669
+ description: "Start the local web dashboard and optionally open it in the default browser.",
670
+ handler: async (_args, ctx) => {
671
+ bindRepo(ctx.cwd);
672
+ let info = await isServerRunning();
673
+ if (info) {
674
+ ctx.ui.notify(`[mega-compact] dashboard already running at ${info.url}`);
675
+ const open = await ctx.ui.confirm("mega-compact dashboard", `Open ${info.url} in browser?`);
676
+ if (open)
677
+ openBrowser(info.url);
678
+ return;
679
+ }
680
+ // Start the server
681
+ ctx.ui.notify("[mega-compact] starting dashboard server…");
682
+ writeRunnerScript();
683
+ const child = spawn(process.execPath, ["--experimental-strip-types", runnerFile], {
684
+ detached: true,
685
+ stdio: "ignore",
686
+ });
687
+ child.unref();
688
+ // Poll for port.pid (up to 5 seconds)
689
+ const deadline = Date.now() + 5_000;
690
+ let port;
691
+ while (Date.now() < deadline) {
692
+ await new Promise((r) => setTimeout(r, 300));
693
+ if (existsSync(portFile)) {
694
+ try {
695
+ const raw = JSON.parse(readFileSync(portFile, "utf-8"));
696
+ if (raw?.port) {
697
+ port = raw.port;
698
+ break;
699
+ }
700
+ }
701
+ catch { /* keep polling */ }
702
+ }
703
+ }
704
+ if (!port) {
705
+ ctx.ui.notify("[mega-compact] dashboard server failed to start — check logs.");
706
+ return;
707
+ }
708
+ const url = `http://localhost:${port}`; // guardrails-allow PREVENT-PI-004: localhost URL of the dashboard server this extension spawned
709
+ ctx.ui.notify(`[mega-compact] dashboard running at ${url}`);
710
+ const open = await ctx.ui.confirm("mega-compact dashboard", `Open ${url} in browser?`);
711
+ if (open)
712
+ openBrowser(url);
713
+ },
714
+ });
715
+ pi.registerCommand("mega-dashboard-stop", {
716
+ description: "Stop the local dashboard server.",
717
+ handler: async (_args, ctx) => {
718
+ if (!existsSync(portFile)) {
719
+ ctx.ui.notify("[mega-compact] no dashboard server running.");
720
+ return;
721
+ }
722
+ try {
723
+ const info = JSON.parse(readFileSync(portFile, "utf-8"));
724
+ // Verify the server is actually ours by probing the port before killing
725
+ try {
726
+ await fetch(`http://localhost:${info.port}/api/snapshot`, { signal: AbortSignal.timeout(1000) }); // guardrails-allow PREVENT-PI-004: localhost probe to verify the dashboard server is ours before stopping it
727
+ }
728
+ catch {
729
+ // Not responding — just clean up stale pid file
730
+ try {
731
+ unlinkSync(portFile);
732
+ }
733
+ catch { /* ok */ }
734
+ ctx.ui.notify("[mega-compact] dashboard was not running (stale pid file cleaned up).");
735
+ return;
736
+ }
737
+ if (info?.pid)
738
+ process.kill(info.pid, "SIGTERM");
739
+ }
740
+ catch { /* already dead */ }
741
+ try {
742
+ unlinkSync(portFile);
743
+ }
744
+ catch { /* ok */ }
745
+ ctx.ui.notify("[mega-compact] dashboard stopped.");
746
+ },
747
+ });
748
+ pi.registerCommand("mega-dashboard-status", {
749
+ description: "Check if the dashboard server is running.",
750
+ handler: async (_args, ctx) => {
751
+ const info = await isServerRunning();
752
+ if (info) {
753
+ ctx.ui.notify(`[mega-compact] dashboard running at ${info.url}`);
754
+ }
755
+ else {
756
+ ctx.ui.notify("[mega-compact] dashboard is not running. Use /dashboard to start it.");
757
+ }
758
+ },
759
+ });
760
+ }
761
+ /** Latest user message text — used as the auto-inline recall query. */
762
+ function recentUserQuery(ctx) {
763
+ try {
764
+ const entries = ctx.sessionManager.getEntries();
765
+ for (let i = entries.length - 1; i >= 0; i--) {
766
+ const msgs = sessionEntryToContextMessages(entries[i]);
767
+ for (let j = msgs.length - 1; j >= 0; j--) {
768
+ if (msgs[j].role === "user") {
769
+ const c = msgs[j].content;
770
+ if (typeof c === "string")
771
+ return c;
772
+ if (Array.isArray(c))
773
+ return c.map((b) => b.text).join(" ");
774
+ }
775
+ }
776
+ }
777
+ }
778
+ catch {
779
+ /* best-effort */
780
+ }
781
+ return "";
782
+ }