pi-mega-compact 0.4.27 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +47 -2
  2. package/dist/extensions/dashboard-server.js +58 -2
  3. package/dist/extensions/dashboard-server.test.js +95 -3
  4. package/dist/extensions/mega-commands.js +25 -9
  5. package/dist/extensions/mega-compact.test.js +161 -29
  6. package/dist/extensions/mega-config.js +5 -0
  7. package/dist/extensions/mega-conflict-cmds.js +79 -0
  8. package/dist/extensions/mega-dashboard-cmds.js +6 -4
  9. package/dist/extensions/mega-events.js +145 -20
  10. package/dist/extensions/mega-pipeline.js +179 -1
  11. package/dist/extensions/mega-runtime.js +14 -0
  12. package/dist/extensions/mega-trim.js +48 -0
  13. package/dist/extensions/mega-trim.test.js +58 -0
  14. package/dist/src/config/dedup.js +1 -0
  15. package/dist/src/driftDetection.js +103 -0
  16. package/dist/src/driftDetection.test.js +87 -0
  17. package/dist/src/memory.js +147 -0
  18. package/dist/src/memory.test.js +41 -0
  19. package/dist/src/memoryConsolidate.test.js +38 -0
  20. package/dist/src/memoryOps.js +58 -0
  21. package/dist/src/memoryOps.test.js +41 -0
  22. package/dist/src/memoryRecall.js +60 -0
  23. package/dist/src/memoryRecall.test.js +92 -0
  24. package/dist/src/recall.js +70 -1
  25. package/dist/src/recall.test.js +69 -1
  26. package/dist/src/store/sqlite.js +127 -11
  27. package/dist/src/vectorStore.js +6 -1
  28. package/extensions/dashboard-server.test.ts +115 -3
  29. package/extensions/dashboard-server.ts +63 -2
  30. package/extensions/mega-commands.ts +24 -9
  31. package/extensions/mega-compact.test.ts +162 -29
  32. package/extensions/mega-config.ts +22 -0
  33. package/extensions/mega-conflict-cmds.ts +81 -0
  34. package/extensions/mega-dashboard-cmds.ts +6 -4
  35. package/extensions/mega-events.ts +139 -20
  36. package/extensions/mega-pipeline.ts +179 -1
  37. package/extensions/mega-runtime.ts +15 -0
  38. package/extensions/mega-trim.test.ts +64 -0
  39. package/extensions/mega-trim.ts +75 -0
  40. package/extensions/openclaw-mega-compact.ts +24 -9
  41. package/package.json +2 -2
  42. package/src/config/dedup.ts +2 -0
  43. package/src/driftDetection.test.ts +100 -0
  44. package/src/driftDetection.ts +136 -0
  45. package/src/memory.test.ts +46 -0
  46. package/src/memory.ts +164 -0
  47. package/src/memoryConsolidate.test.ts +47 -0
  48. package/src/memoryOps.test.ts +53 -0
  49. package/src/memoryOps.ts +75 -0
  50. package/src/memoryRecall.test.ts +100 -0
  51. package/src/memoryRecall.ts +83 -0
  52. package/src/recall.test.ts +77 -1
  53. package/src/recall.ts +94 -1
  54. package/src/store/sqlite.ts +188 -11
  55. package/src/store.ts +3 -0
  56. package/src/vectorStore.ts +10 -1
  57. package/dist/extensions/openclaw-mega-compact.js +0 -291
  58. package/dist/src/minilm.js +0 -92
  59. package/dist/src/wordpiece.js +0 -129
@@ -70,6 +70,11 @@ export function loadConfig() {
70
70
  autoInlineK: envFlag("MEGACOMPACT_AUTO_INLINE_K", 3),
71
71
  dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
72
72
  raptorEnabled: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
73
+ legacyDurableTrim: envBool("MEGACOMPACT_LEGACY_DURABLE_TRIM", false),
74
+ crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
75
+ crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
76
+ memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),
77
+ memoryReviewInterval: envFlag("MEGACOMPACT_MEMORY_REVIEW_INTERVAL", 10),
73
78
  recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
74
79
  windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
75
80
  debug: envBool("MEGACOMPACT_DEBUG", false),
@@ -118,4 +118,83 @@ export function registerConflictCommands(pi, runtime) {
118
118
  ctx.ui.notify(memoryLine(m));
119
119
  },
120
120
  });
121
+ // Shortform aliases — `m save "..."`, `m status`, `m list`, `m search <q>`,
122
+ // `m recall <id>`. Delegates to the same SQLite store so there's one source
123
+ // of truth. The /mega-memory command remains the canonical form.
124
+ pi.registerCommand("m", {
125
+ description: "Shortform alias for /mega-memory. Usage: /m save <text> | list | search <q> | recall <id> | status",
126
+ handler: async (args, ctx) => {
127
+ const repo = resolveRepoRoot(ctx.cwd) ?? runtime.currentStateDir;
128
+ const parts = args.trim().split(/\s+/);
129
+ const sub = parts[0]?.toLowerCase() ?? "list";
130
+ if (sub === "save") {
131
+ // Strip leading "save" so /m save "#foo bar" works the same as the
132
+ // canonical form. Then strip a balanced outer quote pair if the user
133
+ // wrote /m save "..." — common when the text contains spaces.
134
+ let text = args.trim().slice(4).trim();
135
+ const mq = text.match(/^["“](.*)["”]$/s);
136
+ if (mq)
137
+ text = mq[1].trim();
138
+ if (!text) {
139
+ ctx.ui.notify('[/m] usage: /m save "<text>" or /m save <text>');
140
+ return;
141
+ }
142
+ const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
143
+ const content = text.replace(/#[\w-]+/g, "").trim();
144
+ const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
145
+ ctx.ui.notify(`[/m] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
146
+ return;
147
+ }
148
+ if (sub === "search") {
149
+ const q = parts.slice(1).join(" ").trim();
150
+ if (!q) {
151
+ ctx.ui.notify("[/m] usage: /m search <query>");
152
+ return;
153
+ }
154
+ const hits = searchMemories(q, repo, 50, runtime.currentStateDir);
155
+ if (!hits.length) {
156
+ ctx.ui.notify("[/m] no memories match.");
157
+ return;
158
+ }
159
+ for (const mem of hits)
160
+ ctx.ui.notify(memoryLine(mem));
161
+ return;
162
+ }
163
+ if (sub === "recall") {
164
+ const id = Number(parts[1]);
165
+ if (!Number.isFinite(id) || parts[1] === undefined) {
166
+ ctx.ui.notify("[/m] usage: /m recall <id>");
167
+ return;
168
+ }
169
+ if (recallMemory(id, runtime.currentStateDir)) {
170
+ const found = listMemories(repo, 1000, runtime.currentStateDir).find((mem) => mem.id === id);
171
+ ctx.ui.notify(found ? `[/m] ${memoryLine(found)}` : `[/m] recalled #${id}`);
172
+ }
173
+ else {
174
+ ctx.ui.notify(`[/m] #${id} not found.`);
175
+ }
176
+ return;
177
+ }
178
+ if (sub === "status") {
179
+ const all = listMemories(repo, 1000, runtime.currentStateDir);
180
+ const byKind = all.reduce((acc, m) => {
181
+ acc[m.kind] = (acc[m.kind] ?? 0) + 1;
182
+ return acc;
183
+ }, {});
184
+ const kinds = Object.entries(byKind).map(([k, n]) => `${k}=${n}`).join(", ");
185
+ const head = `[/m] ${all.length} memory record(s) in ${repo.split(/[\\/]/).pop() ?? repo}`;
186
+ ctx.ui.notify(kinds ? `${head} (${kinds})` : head);
187
+ return;
188
+ }
189
+ // default: list
190
+ const all = listMemories(repo, 50, runtime.currentStateDir);
191
+ if (!all.length) {
192
+ ctx.ui.notify("[/m] no saved memories yet. Use /m save <text>.");
193
+ return;
194
+ }
195
+ ctx.ui.notify(`[/m] ${all.length} memory record(s):`);
196
+ for (const mem of all)
197
+ ctx.ui.notify(memoryLine(mem));
198
+ },
199
+ });
121
200
  }
@@ -18,11 +18,13 @@ export function registerDashboardCommands(pi, runtime) {
18
18
  // when we fall back to the .ts source outside node_modules; false when using
19
19
  // the shipped compiled dist/extensions/dashboard-server.js).
20
20
  let dashboardNeedsStrip = false;
21
- // The dashboard server binds 9320–9329 (TARGET_PORT..TARGET_PORT+PORT_RANGE-1
22
- // in dashboard-server.js). Probe each for a live /api/snapshot so we can detect
23
- // readiness even when port.pid landed in a different state dir than we poll.
21
+ // The dashboard server binds a 10-port range starting at MEGACOMPACT_DASHBOARD_PORT
22
+ // (default 9320) — see TARGET_PORT/PORT_RANGE in dashboard-server.js. Probe each for
23
+ // a live /api/snapshot so we detect readiness even when port.pid landed in a different
24
+ // state dir than we poll. Configurable so tests can use a private, non-colliding range.
25
+ const DASH_BASE = Number(process.env.MEGACOMPACT_DASHBOARD_PORT ?? "9320");
24
26
  async function findLivePort() {
25
- for (let port = 9320; port <= 9329; port++) {
27
+ for (let port = DASH_BASE; port <= DASH_BASE + 9; port++) {
26
28
  try {
27
29
  const res = await fetch(`http://localhost:${port}/api/snapshot`, { signal: AbortSignal.timeout(800) }); // guardrails-allow PREVENT-PI-004: localhost liveness probe of the dashboard server this extension spawned
28
30
  if (res.ok)
@@ -10,8 +10,10 @@ import { normalizeSessionId } from "../src/store.js";
10
10
  import { autoCompactCheck } from "../src/compact.js";
11
11
  import { estimateSessionTokens } from "../src/tokens.js";
12
12
  import { recentUserQuery, WIDGET_KEY } from "./mega-runtime.js";
13
- import { runCompact, doRecall } from "./mega-pipeline.js";
13
+ import { runCompact, doRecall, doRecallAsync, piCompactWouldNoop } from "./mega-pipeline.js";
14
+ import { recallMemoriesAndInline } from "../src/recall.js";
14
15
  import { driveNativeCompaction } from "./mega-compact-driver.js";
16
+ import { computeLiveTrimCut, liveTrimSummaryMessage } from "./mega-trim.js";
15
17
  import { pressureFromPct } from "./mega-config.js";
16
18
  /** Register all pi lifecycle event handlers. */
17
19
  export function registerEventHandlers(pi, runtime, config) {
@@ -25,6 +27,8 @@ export function registerEventHandlers(pi, runtime, config) {
25
27
  runtime.resetRuntime(ctx.sessionManager.getSessionId());
26
28
  runtime.captureModel(ctx); // best-effort: ctx.model may be set by session start
27
29
  runtime.setStatus(ctx, config.auto ? "mega-compact: ready" : "mega-compact: manual only");
30
+ // S21: clear any stale memory block from a prior session.
31
+ runtime.pendingMemoryRecallBlock = undefined;
28
32
  // Auto-inline on resume/fork/continue: stage the most relevant checkpoints
29
33
  // so the next before_agent_start prepends them to the system prompt.
30
34
  // Triggered whenever this session already has persisted checkpoints AND a
@@ -36,13 +40,29 @@ export function registerEventHandlers(pi, runtime, config) {
36
40
  const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
37
41
  const query = recentUserQuery(ctx);
38
42
  if (query && runtime.store.stats(sid).checkpointCount > 0) {
39
- const r = doRecall(runtime, config, ctx, query, "resume");
43
+ // S17: use the async variant on resume so cross-repo HNSW recall can
44
+ // augment when this repo's store is thin. session_start is an async-safe
45
+ // point (unlike the mid-turn context handler, which stays sync).
46
+ const r = await doRecallAsync(runtime, config, ctx, query, "resume", { crossRepo: config.crossRepoEnabled });
40
47
  if (!r.empty) {
41
48
  runtime.pendingRecallBlock = r.block;
42
- runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt`);
43
- runtime.logger.info("auto-inline", { reason: event.reason, query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
49
+ const crossLabel = r.toInject.some((h) => h.repoId) ? " (cross-repo)" : "";
50
+ runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt${crossLabel}`);
51
+ runtime.logger.info("auto-inline", { reason: event.reason, query, injected: r.toInject.map((h) => h.checkpoint.checkpointId), crossRepo: r.toInject.some((h) => h.repoId) });
44
52
  }
45
53
  }
54
+ // S21: parallel memory recall. Same async context so we can await without
55
+ // breaking the handler contract. Best-effort — never throws.
56
+ try {
57
+ const mr = await recallMemoriesAndInline({
58
+ query, stateDir: runtime.getStateDir(), limit: 5,
59
+ });
60
+ if (!mr.empty)
61
+ runtime.pendingMemoryRecallBlock = mr.block;
62
+ }
63
+ catch (err) {
64
+ runtime.logger.warn("memory-recall skipped", { err: String(err) });
65
+ }
46
66
  }
47
67
  runtime.dashboard.event("session_start", { reason: event.reason, sessionId: runtime.rt.sessionId });
48
68
  runtime.snapshot(ctx);
@@ -60,6 +80,15 @@ export function registerEventHandlers(pi, runtime, config) {
60
80
  runtime.pendingRecallBlock = r.block;
61
81
  runtime.logger.info("auto-inline", { reason: "session_tree", query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
62
82
  }
83
+ // S21: parallel memory recall. Trigram embedder is sub-ms; await is fine.
84
+ try {
85
+ const mr = await recallMemoriesAndInline({ query, stateDir: runtime.getStateDir(), limit: 5 });
86
+ if (!mr.empty)
87
+ runtime.pendingMemoryRecallBlock = mr.block;
88
+ }
89
+ catch (err) {
90
+ runtime.logger.warn("memory-recall skipped", { err: String(err) });
91
+ }
63
92
  }
64
93
  }
65
94
  runtime.dashboard.event("session_tree", { sessionId: runtime.rt.sessionId });
@@ -68,11 +97,14 @@ export function registerEventHandlers(pi, runtime, config) {
68
97
  // ---- Auto-inline injection point: prepend staged recall to systemPrompt ----
69
98
  pi.on("before_agent_start", async (event, ctx) => {
70
99
  runtime.captureModel(ctx); // most reliable point ctx.model is populated
71
- if (!runtime.pendingRecallBlock)
100
+ const cpBlock = runtime.pendingRecallBlock;
101
+ const memBlock = runtime.pendingMemoryRecallBlock;
102
+ if (!cpBlock && !memBlock)
72
103
  return;
73
- const block = runtime.pendingRecallBlock;
74
- runtime.pendingRecallBlock = undefined; // one-shot: consume so we never double-inject
75
- return { systemPrompt: `${event.systemPrompt}\n\n${block}` };
104
+ runtime.pendingRecallBlock = undefined;
105
+ runtime.pendingMemoryRecallBlock = undefined;
106
+ const composed = [cpBlock, memBlock].filter(Boolean).join("\n\n");
107
+ return { systemPrompt: `${event.systemPrompt}\n\n${composed}` };
76
108
  });
77
109
  pi.on("session_shutdown", async (_event, ctx) => {
78
110
  runtime.setStatus(ctx, undefined);
@@ -98,6 +130,24 @@ export function registerEventHandlers(pi, runtime, config) {
98
130
  else {
99
131
  runtime.setStatus(ctx, config.auto ? "mega-compact: ready" : "mega-compact: manual only");
100
132
  }
133
+ // S16 continuation fallback: if the turn settled idle right after a live-trim
134
+ // compaction AND there is queued work AND we haven't nudged recently, nudge
135
+ // once so the agent continues (the live trim should make this rare). Guarded
136
+ // to never busy-loop: one nudge per 30s, only when truly idle + queued.
137
+ if (config.auto && runtime.activeAgents === 0) {
138
+ try {
139
+ const idle = ctx.isIdle?.() ?? true;
140
+ const queued = ctx.hasPendingMessages?.() ?? false;
141
+ const now = Date.now();
142
+ if (idle && queued && now >= runtime.resumeNudgeUntil) {
143
+ runtime.resumeNudgeUntil = now + 30_000;
144
+ pi.sendUserMessage("[mega-compact] continue from the compacted context above.");
145
+ }
146
+ }
147
+ catch {
148
+ /* non-fatal: a failed nudge never blocks */
149
+ }
150
+ }
101
151
  runtime.snapshot(ctx);
102
152
  });
103
153
  pi.on("turn_start", async (event, ctx) => {
@@ -108,16 +158,43 @@ export function registerEventHandlers(pi, runtime, config) {
108
158
  pi.on("turn_end", async (event, ctx) => {
109
159
  runtime.dashboard.event("turn_end", { turnIndex: event.turnIndex });
110
160
  runtime.snapshot(ctx);
161
+ // S20: auto-review the conversation every N turns and persist durable
162
+ // memories. Best-effort + non-fatal: a review failure must never break the
163
+ // agent loop. Debounced by memoryReviewInterval turns.
164
+ if (config.memoryAutoReview && runtime.currentTurn > 0 && runtime.currentTurn % config.memoryReviewInterval === 0) {
165
+ try {
166
+ const { reviewConversation } = await import("../src/memory.js");
167
+ const { applyMemoryOps } = await import("../src/memoryOps.js");
168
+ const entries = ctx.sessionManager.getEntries();
169
+ const view = runtime.engineView(entries.flatMap((e) => (e.message ? [e.message] : [])));
170
+ const ops = reviewConversation(view, []);
171
+ if (ops.length) {
172
+ await applyMemoryOps(ops, runtime.currentStateDir);
173
+ // S21.2: a memory op landed in this turn window. The pipeline reads
174
+ // this counter after a successful compaction and fires
175
+ // `consolidateMemories` only when it's > 0.
176
+ runtime.memoriesTouchedThisCompaction += ops.length;
177
+ }
178
+ }
179
+ catch {
180
+ /* non-fatal — auto-review must not break the turn loop */
181
+ }
182
+ }
111
183
  });
112
- // ---- Auto-trigger: own the decision, pi owns the durable write ----------
113
- // OUR auto-trigger (over threshold + debounce): persist our Trident checkpoint,
114
- // then start pi's compaction flow via ctx.compact(). That fires
115
- // `session_before_compact`, where OUR handler returns our summary +
116
- // firstKeptEntryId, and pi durably writes the trim to disk (appendCompaction).
117
- // Result: auto-compact AND a durable trim — resume reloads the trimmed window,
118
- // no full-reload + additive recall inflation (Fix B kills the token-growth bug).
119
- // We do NOT drop messages here (that would be ephemeral; the read-only session
120
- // manager can't trim disk, so the trim has to come through pi).
184
+ // ---- Auto-trigger: live trim (compact and continue) + native durable ----
185
+ // S16 redesign: we NO LONGER call ctx.compact() from the auto-trigger by
186
+ // default. That mapped to pi's MANUAL compaction path, which abort()s the
187
+ // in-flight turn (agent-session.js:1345) and stops the agent. Instead:
188
+ // - LIVE: return { messages: trimmedView } from the context event. This
189
+ // feeds pi's transformContext (sdk.js:226 agent-loop.js:180) so the
190
+ // model sees a compacted window EVERY LLM call, with no abort. The turn
191
+ // continues. We persist our recall checkpoint (the durable value) first.
192
+ // - DURABLE: pi's NATIVE auto-compaction fires at agent-end
193
+ // (agent-session.js:1565), continues (return hasQueuedMessages()), and
194
+ // emits session_before_compact — where OUR driveNativeCompaction supplies
195
+ // the summary and pi truncates the transcript on disk. No ctx.compact().
196
+ // Legacy: MEGACOMPACT_LEGACY_DURABLE_TRIM=true restores the v0.4.28 ctx.compact
197
+ // path (kept one release as rollback).
121
198
  pi.on("context", async (event, ctx) => {
122
199
  if (!config.auto)
123
200
  return;
@@ -151,9 +228,57 @@ export function registerEventHandlers(pi, runtime, config) {
151
228
  const ran = runCompact(pi, runtime, config, ctx, messages, { compressionPressure: pressure });
152
229
  if (ran.skipped)
153
230
  return;
154
- // Start pi's compaction flow so our session_before_compact handler can
155
- // supply the durable trim (pi writes it to disk). We never use pi's summary.
156
- ctx.compact({ customInstructions: undefined });
231
+ // LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
232
+ // manual compact path aborts the in-flight turn only used behind the flag.
233
+ // Read live from env (in addition to the load-time config) so the flag can be
234
+ // toggled per-test without reloading the module; config.legacyDurableTrim is
235
+ // the cached default. (Mirrors how piCompactWouldNoop re-reads its floor.)
236
+ const legacy = config.legacyDurableTrim || process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "true" || process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "1";
237
+ if (legacy) {
238
+ if (piCompactWouldNoop(ctx))
239
+ return;
240
+ ctx.compact({ customInstructions: undefined });
241
+ return;
242
+ }
243
+ // S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
244
+ // Non-destructive: pi keeps the real transcript; only this LLM call sees the
245
+ // trimmed window. We compute the cut on the engine view (pure, tested) then
246
+ // slice the ORIGINAL pi AgentMessage[] from that index (lossless alignment,
247
+ // mirroring dropCompactedRange) and prepend a user-role summary message.
248
+ // A build failure or unsafe cut returns nothing (no trim this call — the
249
+ // next context event retries). The anchor floor is read live from env (the
250
+ // config value is the cached default) so it can be tuned per-test / per-run
251
+ // without reloading the module.
252
+ try {
253
+ const anchorEnv = process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES;
254
+ const anchorUserMessages = (anchorEnv != null && anchorEnv !== "" && Number.isFinite(Number(anchorEnv)))
255
+ ? Number(anchorEnv)
256
+ : config.anchorUserMessages;
257
+ const cut = computeLiveTrimCut(view, {
258
+ compactedFrom: ran.result.compactedFrom,
259
+ summary: ran.result.summary,
260
+ anchorUserMessages,
261
+ });
262
+ if (cut === null)
263
+ return; // unsafe / below anchor floor — no trim this call
264
+ const summaryMsg = liveTrimSummaryMessage({
265
+ compactedFrom: ran.result.compactedFrom,
266
+ summary: ran.result.summary,
267
+ anchorUserMessages: config.anchorUserMessages,
268
+ });
269
+ // Synthesize a user-role AgentMessage carrying the compacted summary.
270
+ const summaryAgentMsg = {
271
+ role: "user",
272
+ content: summaryMsg.text,
273
+ timestamp: Date.now(),
274
+ };
275
+ const recent = messages.slice(cut); // guardrails-allow PREVENT-PI-002: `cut` is the pre-sanitized `compactedFrom` produced by src/boundary.ts computeDropRange, so the preserved run begins on a toolPair-safe index.
276
+ runtime.snapshot(ctx);
277
+ return { messages: [summaryAgentMsg, ...recent] };
278
+ }
279
+ catch {
280
+ return; // non-fatal: no trim this call; the next context event retries
281
+ }
157
282
  });
158
283
  // ---- Supply a DURABLE trim to pi's native compaction (Fix B) ----------
159
284
  // We run the Trident pipeline to produce a compressed summary, then return
@@ -8,9 +8,11 @@
8
8
  */
9
9
  import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
10
10
  import { compactSession } from "../src/engine.js";
11
- import { recallAndInline } from "../src/recall.js";
11
+ import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "../src/recall.js";
12
12
  import { normalizeSessionId } from "../src/store.js";
13
+ import { estimateBlockTokens } from "../src/tokens.js";
13
14
  import { touchSession, logDaily } from "../src/store/sqlite.js";
15
+ import { consolidateMemories } from "../src/memory.js";
14
16
  import { C, MARKER_TYPE, } from "./mega-runtime.js";
15
17
  import { resolveRepoRoot, preserveRecentForPressure } from "./mega-config.js";
16
18
  import { runRaptor } from "../src/dedup/raptor/index.js";
@@ -41,6 +43,10 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
41
43
  }
42
44
  function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
43
45
  runtime.pulsing = true; // animate the status line while the (sync) pipeline runs
46
+ // S21.2: reset the per-compaction memory-op counter so the post-compact
47
+ // consolidate pass only fires when memory rows actually changed during the
48
+ // compaction window (turn_end → auto-review may have written some).
49
+ runtime.memoriesTouchedThisCompaction = 0;
44
50
  const result = compactSession({
45
51
  sessionId: sid,
46
52
  messages: view,
@@ -107,6 +113,26 @@ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
107
113
  catch {
108
114
  /* non-fatal: stats bookkeeping only */
109
115
  }
116
+ // S21.2: best-effort consolidation of near-duplicate memories for this repo.
117
+ // Runs after the per-repo stats touch so `consolidateMemories` can use the
118
+ // same stateDir. Non-fatal — a failed consolidate never blocks a compaction.
119
+ // Only runs when new memory ops landed in this pass (otherwise the prior
120
+ // compaction's consolidate already had its shot — re-running would just
121
+ // touch every row again with no merges).
122
+ if (!result.deduped && runtime.memoriesTouchedThisCompaction > 0) {
123
+ try {
124
+ const root = resolveRepoRoot(ctx.cwd);
125
+ void consolidateMemories(runtime.currentStateDir, root).then((n) => {
126
+ if (n > 0)
127
+ runtime.pushTicker(`${C.green}∫${C.reset} consolidated ${n} memory dup${n === 1 ? "" : "s"}`);
128
+ }, () => {
129
+ /* swallow: consolidate failures must never surface to the user */
130
+ });
131
+ }
132
+ catch {
133
+ /* non-fatal */
134
+ }
135
+ }
110
136
  // Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
111
137
  // skip re-vectorizing an already-compacted region (zero token cost).
112
138
  pi.appendEntry(MARKER_TYPE, {
@@ -181,6 +207,100 @@ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
181
207
  runtime.snapshot(ctx);
182
208
  return { skipped: false, result, keepFrom, saved };
183
209
  }
210
+ /**
211
+ * Predict whether pi's `ctx.compact()` would throw a no-op error — "Already
212
+ * compacted" or "Nothing to compact (session too small)" — so the auto-trigger
213
+ * can SKIP the call instead of surfacing a hard, user-facing error.
214
+ *
215
+ * Why we can't intercept or suppress it: pi's public `compact()` computes
216
+ * `prepareCompaction()` and throws *before* it emits `session_before_compact`,
217
+ * so our handler there never runs on the no-op path. And `ctx.compact()`'s
218
+ * `onError` callback fires only AFTER pi has already emitted a `compaction_end`
219
+ * event carrying the error message (which the interactive UI renders) — so
220
+ * `onError` cannot mute it either. The only robust fix is to not call
221
+ * `ctx.compact()` when pi would no-op. (pi's own `_runAutoCompaction` path is
222
+ * silent on this same condition; the public path we're forced through is the
223
+ * one that throws.)
224
+ *
225
+ * Skipping is correct, not a compromise: by the time this runs, `runCompact()`
226
+ * has already persisted the recall checkpoint (Path A). The durable on-disk
227
+ * trim is only useful when pi can actually summarize a region; a transcript
228
+ * under pi's `keepRecentTokens` budget is small enough that reloading it on
229
+ * resume isn't a token-growth problem, so the durable trim is unnecessary
230
+ * there anyway.
231
+ *
232
+ * Mirrors pi's `prepareCompaction()` return-undefined conditions (compaction.js):
233
+ * (1) last entry is a compaction → "Already compacted"
234
+ * (2) <2 cut-point messages since the last compaction → nothing to summarize
235
+ * (a cut point = any non-toolResult message — user/assistant/bash/custom/
236
+ * branchSummary/compactionSummary — matching pi's isCutPointMessage)
237
+ * (3) transcript tokens since the last compaction < keepRecentTokens → pi
238
+ * keeps everything → nothing to summarize
239
+ * `keepRecentTokens` isn't readable from the extension API, so (3) uses the pi
240
+ * default (20000) as a conservative floor; raise it via
241
+ * `MEGACOMPACT_DURABLE_TRIM_FLOOR` if you raise pi's `compact.keepRecentTokens`.
242
+ *
243
+ * Best-effort: on any read error returns true (skip) — skipping a durable trim
244
+ * is always safe; calling `ctx.compact()` on a no-op throws to the user.
245
+ */
246
+ export function piCompactWouldNoop(ctx) {
247
+ try {
248
+ const branch = ctx.sessionManager.getBranch();
249
+ if (branch.length === 0)
250
+ return true;
251
+ // (1) already compacted — pi throws "Already compacted"
252
+ if (branch[branch.length - 1].type === "compaction")
253
+ return true;
254
+ // boundaryStart = index just after the most recent compaction entry (or 0)
255
+ let boundaryStart = 0;
256
+ for (let i = branch.length - 1; i >= 0; i--) {
257
+ if (branch[i].type === "compaction") {
258
+ boundaryStart = i + 1;
259
+ break;
260
+ }
261
+ }
262
+ let cutPoints = 0;
263
+ let tokens = 0;
264
+ for (let i = boundaryStart; i < branch.length; i++) {
265
+ const e = branch[i];
266
+ if (e.type === "compaction")
267
+ continue;
268
+ let isCut = false;
269
+ for (const m of sessionEntryToContextMessages(e)) {
270
+ // pi's isCutPointMessage: every role except toolResult
271
+ if (m.role !== "toolResult")
272
+ isCut = true;
273
+ const c = m.content;
274
+ const text = typeof c === "string" ? c
275
+ : Array.isArray(c)
276
+ ? c.map((b) => b?.text ?? "").join(" ")
277
+ : "";
278
+ if (text)
279
+ tokens += estimateBlockTokens(text);
280
+ }
281
+ if (isCut)
282
+ cutPoints++;
283
+ }
284
+ // (2) need >=2 cut points so the kept cut isn't the first message
285
+ if (cutPoints < 2)
286
+ return true;
287
+ // (3) transcript under pi's keepRecentTokens budget → pi keeps everything
288
+ if (tokens < durableTrimFloorTokens())
289
+ return true;
290
+ return false;
291
+ }
292
+ catch {
293
+ return true; // safe: skip the durable trim rather than risk a user-facing throw
294
+ }
295
+ }
296
+ /** pi's default keepRecentTokens (compaction settings). Override with
297
+ * MEGACOMPACT_DURABLE_TRIM_FLOOR if you raise pi's compact.keepRecentTokens. */
298
+ function durableTrimFloorTokens() {
299
+ const raw = process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
300
+ if (raw !== undefined && Number.isFinite(Number(raw)))
301
+ return Number(raw);
302
+ return 20_000;
303
+ }
184
304
  /**
185
305
  * Unified recall (Layer 5). The ONE path that injects. Returns the recall
186
306
  * result; callers decide whether to stage it for before_agent_start (resume)
@@ -215,6 +335,64 @@ export function doRecall(runtime, config, ctx, query, source) {
215
335
  }
216
336
  return result;
217
337
  }
338
+ /**
339
+ * S17: async recall with optional cross-repo augmentation. Used on resume
340
+ * (session_start) and /mega-recall --cross-repo — NEVER from the mid-turn
341
+ * context handler (that stays sync). Runs the sync same-repo scan first; if it
342
+ * returns < config.autoInlineK hits AND crossRepo is enabled, awaits the PGlite
343
+ * HNSW cross-repo path and merges (source-labeled, deduped by checkpointId). The
344
+ * recallMaxTokens cap + windowDedupe apply to the merged set so cross-repo can
345
+ * never net-inflate the window. Cross-repo uses a stricter cosine floor
346
+ * (config.crossRepoCosine) than same-repo. Non-fatal: any async failure returns
347
+ * the same-repo result unchanged.
348
+ */
349
+ export async function doRecallAsync(runtime, config, ctx, query, source, opts = {}) {
350
+ runtime.bindRepo(ctx.cwd);
351
+ const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
352
+ const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
353
+ // Sync same-repo first (fast, never blocks).
354
+ const sameRepo = recallAndInline({
355
+ sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
356
+ recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
357
+ liveWindow, dedupSim: config.dedupSim,
358
+ }, runtime.store);
359
+ if (!config.crossRepoEnabled || !opts.crossRepo)
360
+ return sameRepo;
361
+ if (sameRepo.toInject.length >= config.autoInlineK)
362
+ return sameRepo; // same-repo satisfied
363
+ // Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
364
+ try {
365
+ const x = await recallAndInlineAsync({
366
+ sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
367
+ recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
368
+ liveWindow, dedupSim: config.crossRepoCosine, crossRepo: true,
369
+ globalIndexDir: process.env.MEGACOMPACT_INDEX_DIR,
370
+ }, runtime.store);
371
+ runtime.dashboard.event("recall-crossrepo", {
372
+ source, query: query.slice(0, 120), injected: x.toInject.length,
373
+ sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
374
+ });
375
+ // Merge, dedup by checkpointId, respect the same token cap by reformatting.
376
+ const seen = new Set(sameRepo.toInject.map((h) => h.checkpoint.checkpointId));
377
+ const merged = [...sameRepo.toInject];
378
+ for (const h of x.toInject) {
379
+ if (!seen.has(h.checkpoint.checkpointId)) {
380
+ merged.push(h);
381
+ seen.add(h.checkpoint.checkpointId);
382
+ }
383
+ }
384
+ const block = merged.length ? formatRecallBlock(merged) : "";
385
+ return {
386
+ toInject: merged,
387
+ report: merged.map((h) => ` • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`),
388
+ block,
389
+ empty: merged.length === 0,
390
+ };
391
+ }
392
+ catch {
393
+ return sameRepo; // cross-repo failure → same-repo only (non-fatal)
394
+ }
395
+ }
218
396
  /**
219
397
  * Extract the live-window message texts from the session manager (Fix C),
220
398
  * for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
@@ -57,12 +57,17 @@ export class MegaRuntime {
57
57
  tokensSaved: 0,
58
58
  };
59
59
  debounceUntil = 0;
60
+ // S16: debounce for the agent_end resume nudge (avoid busy-loops).
61
+ resumeNudgeUntil = 0;
60
62
  // Agent tracking for real-time widget updates
61
63
  activeAgents = 0;
62
64
  currentTurn = 0;
63
65
  // Recall block produced by auto-inline (resume/branch) that the next
64
66
  // before_agent_start should prepend to the system prompt. Unset after use.
65
67
  pendingRecallBlock;
68
+ // S21: memory recall block, parallel to pendingRecallBlock. Same one-shot
69
+ // semantics; composed with the checkpoint block in before_agent_start.
70
+ pendingMemoryRecallBlock;
66
71
  statusKey; // current status text for dashboard
67
72
  // Active model/provider (for real cost estimation). Captured from ctx.model
68
73
  // on model_select + session_start; persisted to SQL so cost + the dashboard
@@ -81,6 +86,11 @@ export class MegaRuntime {
81
86
  TICKER_MAX = 5;
82
87
  // Pulsing status: set true while a compaction is in flight, cleared on result.
83
88
  pulsing = false;
89
+ // S21.2: set by `applyMemoryOps` when a memory add/replace/remove lands in
90
+ // the current compaction. The pipeline reads this after a successful compact
91
+ // to decide whether to fire `consolidateMemories` (skip the work entirely
92
+ // when no memory rows changed).
93
+ memoriesTouchedThisCompaction = 0;
84
94
  // Rolling "saved" goal for the progress bar — grows as we save more, so the
85
95
  // bar always has a meaningful denominator (never sits at 100% forever).
86
96
  savedGoal = 50_000;
@@ -343,6 +353,10 @@ export class MegaRuntime {
343
353
  }
344
354
  catch { /* non-fatal: cost estimation degrades to model-in-memory only */ }
345
355
  }
356
+ /** S21: state dir of the currently bound repo (where memories live). */
357
+ getStateDir() {
358
+ return this.currentStateDir;
359
+ }
346
360
  /** Build the sync onTier callback that paints the live per-tier trace. */
347
361
  makeTierCallback(ctx) {
348
362
  const order = ["L0", "L1", "L2", "new"];
@@ -0,0 +1,48 @@
1
+ import { isBoundarySafe } from "../src/boundary.js";
2
+ import { formatCompactSummary } from "../src/compact.js";
3
+ /**
4
+ * Compute the safe cut index for the live trim. Snaps `compactedFrom` back to a
5
+ * boundary-safe index (PREVENT-PI-002: never start the preserved run on an
6
+ * orphaned tool result), and enforces the anchor floor (PREVENT-PI-001: keep at
7
+ * least `anchorUserMessages` user-role messages). Returns `null` when no trim is
8
+ * safe this call (empty summary, unsafe boundary, or below the anchor floor) so
9
+ * the caller keeps the original view and retries on the next context event.
10
+ *
11
+ * Exposed separately from `buildLiveTrimmedView` so the context handler can map
12
+ * the cut back onto the original pi `AgentMessage[]` (lossless index alignment,
13
+ * mirroring `dropCompactedRange` in src/adapt.ts).
14
+ */
15
+ export function computeLiveTrimCut(view, opts) {
16
+ if (!opts.summary || !opts.summary.trim())
17
+ return null;
18
+ let cut = opts.compactedFrom;
19
+ while (cut > 0 && !isBoundarySafe(view, cut))
20
+ cut--;
21
+ if (cut <= 0)
22
+ return null; // nothing safe to cut — keep everything this call
23
+ const recent = view.slice(cut);
24
+ const userCount = recent.filter((m) => m.role === "user").length;
25
+ if (userCount < opts.anchorUserMessages)
26
+ return null;
27
+ return cut;
28
+ }
29
+ /** The formatted compacted-region summary as a user-role engine message. */
30
+ export function liveTrimSummaryMessage(opts) {
31
+ return {
32
+ role: "user",
33
+ text: formatCompactSummary(opts.summary),
34
+ toolName: undefined,
35
+ input: undefined,
36
+ output: undefined,
37
+ };
38
+ }
39
+ /** Build the live trimmed view. Returns the original view if summary is empty
40
+ * or the boundary is unsafe (no trim this call — try next). Pure + tested. */
41
+ export function buildLiveTrimmedView(view, opts) {
42
+ const cut = computeLiveTrimCut(view, opts);
43
+ if (cut === null)
44
+ return view;
45
+ const recent = view.slice(cut);
46
+ const summaryMsg = liveTrimSummaryMessage(opts);
47
+ return [summaryMsg, ...recent];
48
+ }