pi-mega-compact 0.4.28 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +47 -2
  2. package/dist/extensions/dashboard-server.js +58 -2
  3. package/dist/extensions/dashboard-server.test.js +95 -3
  4. package/dist/extensions/mega-commands.js +25 -9
  5. package/dist/extensions/mega-compact.test.js +133 -31
  6. package/dist/extensions/mega-config.js +5 -0
  7. package/dist/extensions/mega-conflict-cmds.js +79 -0
  8. package/dist/extensions/mega-dashboard-cmds.js +6 -4
  9. package/dist/extensions/mega-events.js +144 -27
  10. package/dist/extensions/mega-pipeline.js +84 -1
  11. package/dist/extensions/mega-runtime.js +14 -0
  12. package/dist/extensions/mega-trim.js +48 -0
  13. package/dist/extensions/mega-trim.test.js +58 -0
  14. package/dist/src/config/dedup.js +1 -0
  15. package/dist/src/driftDetection.js +103 -0
  16. package/dist/src/driftDetection.test.js +87 -0
  17. package/dist/src/memory.js +147 -0
  18. package/dist/src/memory.test.js +41 -0
  19. package/dist/src/memoryConsolidate.test.js +38 -0
  20. package/dist/src/memoryOps.js +58 -0
  21. package/dist/src/memoryOps.test.js +41 -0
  22. package/dist/src/memoryRecall.js +60 -0
  23. package/dist/src/memoryRecall.test.js +92 -0
  24. package/dist/src/recall.js +70 -1
  25. package/dist/src/recall.test.js +69 -1
  26. package/dist/src/store/sqlite.js +127 -11
  27. package/dist/src/vectorStore.js +6 -1
  28. package/extensions/dashboard-server.test.ts +115 -3
  29. package/extensions/dashboard-server.ts +63 -2
  30. package/extensions/mega-commands.ts +24 -9
  31. package/extensions/mega-compact.test.ts +134 -31
  32. package/extensions/mega-config.ts +22 -0
  33. package/extensions/mega-conflict-cmds.ts +81 -0
  34. package/extensions/mega-dashboard-cmds.ts +6 -4
  35. package/extensions/mega-events.ts +139 -28
  36. package/extensions/mega-pipeline.ts +94 -1
  37. package/extensions/mega-runtime.ts +15 -0
  38. package/extensions/mega-trim.test.ts +64 -0
  39. package/extensions/mega-trim.ts +75 -0
  40. package/extensions/openclaw-mega-compact.ts +24 -9
  41. package/package.json +2 -2
  42. package/src/config/dedup.ts +2 -0
  43. package/src/driftDetection.test.ts +100 -0
  44. package/src/driftDetection.ts +136 -0
  45. package/src/memory.test.ts +46 -0
  46. package/src/memory.ts +164 -0
  47. package/src/memoryConsolidate.test.ts +47 -0
  48. package/src/memoryOps.test.ts +53 -0
  49. package/src/memoryOps.ts +75 -0
  50. package/src/memoryRecall.test.ts +100 -0
  51. package/src/memoryRecall.ts +83 -0
  52. package/src/recall.test.ts +77 -1
  53. package/src/recall.ts +94 -1
  54. package/src/store/sqlite.ts +188 -11
  55. package/src/store.ts +3 -0
  56. package/src/vectorStore.ts +10 -1
  57. package/dist/extensions/openclaw-mega-compact.js +0 -291
  58. package/dist/src/minilm.js +0 -92
  59. package/dist/src/wordpiece.js +0 -129
@@ -118,4 +118,83 @@ export function registerConflictCommands(pi, runtime) {
118
118
  ctx.ui.notify(memoryLine(m));
119
119
  },
120
120
  });
121
+ // Shortform aliases — `m save "..."`, `m status`, `m list`, `m search <q>`,
122
+ // `m recall <id>`. Delegates to the same SQLite store so there's one source
123
+ // of truth. The /mega-memory command remains the canonical form.
124
+ pi.registerCommand("m", {
125
+ description: "Shortform alias for /mega-memory. Usage: /m save <text> | list | search <q> | recall <id> | status",
126
+ handler: async (args, ctx) => {
127
+ const repo = resolveRepoRoot(ctx.cwd) ?? runtime.currentStateDir;
128
+ const parts = args.trim().split(/\s+/);
129
+ const sub = parts[0]?.toLowerCase() ?? "list";
130
+ if (sub === "save") {
131
+ // Strip leading "save" so /m save "#foo bar" works the same as the
132
+ // canonical form. Then strip a balanced outer quote pair if the user
133
+ // wrote /m save "..." — common when the text contains spaces.
134
+ let text = args.trim().slice(4).trim();
135
+ const mq = text.match(/^["“](.*)["”]$/s);
136
+ if (mq)
137
+ text = mq[1].trim();
138
+ if (!text) {
139
+ ctx.ui.notify('[/m] usage: /m save "<text>" or /m save <text>');
140
+ return;
141
+ }
142
+ const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
143
+ const content = text.replace(/#[\w-]+/g, "").trim();
144
+ const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
145
+ ctx.ui.notify(`[/m] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
146
+ return;
147
+ }
148
+ if (sub === "search") {
149
+ const q = parts.slice(1).join(" ").trim();
150
+ if (!q) {
151
+ ctx.ui.notify("[/m] usage: /m search <query>");
152
+ return;
153
+ }
154
+ const hits = searchMemories(q, repo, 50, runtime.currentStateDir);
155
+ if (!hits.length) {
156
+ ctx.ui.notify("[/m] no memories match.");
157
+ return;
158
+ }
159
+ for (const mem of hits)
160
+ ctx.ui.notify(memoryLine(mem));
161
+ return;
162
+ }
163
+ if (sub === "recall") {
164
+ const id = Number(parts[1]);
165
+ if (!Number.isFinite(id) || parts[1] === undefined) {
166
+ ctx.ui.notify("[/m] usage: /m recall <id>");
167
+ return;
168
+ }
169
+ if (recallMemory(id, runtime.currentStateDir)) {
170
+ const found = listMemories(repo, 1000, runtime.currentStateDir).find((mem) => mem.id === id);
171
+ ctx.ui.notify(found ? `[/m] ${memoryLine(found)}` : `[/m] recalled #${id}`);
172
+ }
173
+ else {
174
+ ctx.ui.notify(`[/m] #${id} not found.`);
175
+ }
176
+ return;
177
+ }
178
+ if (sub === "status") {
179
+ const all = listMemories(repo, 1000, runtime.currentStateDir);
180
+ const byKind = all.reduce((acc, m) => {
181
+ acc[m.kind] = (acc[m.kind] ?? 0) + 1;
182
+ return acc;
183
+ }, {});
184
+ const kinds = Object.entries(byKind).map(([k, n]) => `${k}=${n}`).join(", ");
185
+ const head = `[/m] ${all.length} memory record(s) in ${repo.split(/[\\/]/).pop() ?? repo}`;
186
+ ctx.ui.notify(kinds ? `${head} (${kinds})` : head);
187
+ return;
188
+ }
189
+ // default: list
190
+ const all = listMemories(repo, 50, runtime.currentStateDir);
191
+ if (!all.length) {
192
+ ctx.ui.notify("[/m] no saved memories yet. Use /m save <text>.");
193
+ return;
194
+ }
195
+ ctx.ui.notify(`[/m] ${all.length} memory record(s):`);
196
+ for (const mem of all)
197
+ ctx.ui.notify(memoryLine(mem));
198
+ },
199
+ });
121
200
  }
@@ -18,11 +18,13 @@ export function registerDashboardCommands(pi, runtime) {
18
18
  // when we fall back to the .ts source outside node_modules; false when using
19
19
  // the shipped compiled dist/extensions/dashboard-server.js).
20
20
  let dashboardNeedsStrip = false;
21
- // The dashboard server binds 9320–9329 (TARGET_PORT..TARGET_PORT+PORT_RANGE-1
22
- // in dashboard-server.js). Probe each for a live /api/snapshot so we can detect
23
- // readiness even when port.pid landed in a different state dir than we poll.
21
+ // The dashboard server binds a 10-port range starting at MEGACOMPACT_DASHBOARD_PORT
22
+ // (default 9320) — see TARGET_PORT/PORT_RANGE in dashboard-server.js. Probe each for
23
+ // a live /api/snapshot so we detect readiness even when port.pid landed in a different
24
+ // state dir than we poll. Configurable so tests can use a private, non-colliding range.
25
+ const DASH_BASE = Number(process.env.MEGACOMPACT_DASHBOARD_PORT ?? "9320");
24
26
  async function findLivePort() {
25
- for (let port = 9320; port <= 9329; port++) {
27
+ for (let port = DASH_BASE; port <= DASH_BASE + 9; port++) {
26
28
  try {
27
29
  const res = await fetch(`http://localhost:${port}/api/snapshot`, { signal: AbortSignal.timeout(800) }); // guardrails-allow PREVENT-PI-004: localhost liveness probe of the dashboard server this extension spawned
28
30
  if (res.ok)
@@ -10,8 +10,10 @@ import { normalizeSessionId } from "../src/store.js";
10
10
  import { autoCompactCheck } from "../src/compact.js";
11
11
  import { estimateSessionTokens } from "../src/tokens.js";
12
12
  import { recentUserQuery, WIDGET_KEY } from "./mega-runtime.js";
13
- import { runCompact, doRecall, piCompactWouldNoop } from "./mega-pipeline.js";
13
+ import { runCompact, doRecall, doRecallAsync, piCompactWouldNoop } from "./mega-pipeline.js";
14
+ import { recallMemoriesAndInline } from "../src/recall.js";
14
15
  import { driveNativeCompaction } from "./mega-compact-driver.js";
16
+ import { computeLiveTrimCut, liveTrimSummaryMessage } from "./mega-trim.js";
15
17
  import { pressureFromPct } from "./mega-config.js";
16
18
  /** Register all pi lifecycle event handlers. */
17
19
  export function registerEventHandlers(pi, runtime, config) {
@@ -25,6 +27,8 @@ export function registerEventHandlers(pi, runtime, config) {
25
27
  runtime.resetRuntime(ctx.sessionManager.getSessionId());
26
28
  runtime.captureModel(ctx); // best-effort: ctx.model may be set by session start
27
29
  runtime.setStatus(ctx, config.auto ? "mega-compact: ready" : "mega-compact: manual only");
30
+ // S21: clear any stale memory block from a prior session.
31
+ runtime.pendingMemoryRecallBlock = undefined;
28
32
  // Auto-inline on resume/fork/continue: stage the most relevant checkpoints
29
33
  // so the next before_agent_start prepends them to the system prompt.
30
34
  // Triggered whenever this session already has persisted checkpoints AND a
@@ -36,13 +40,29 @@ export function registerEventHandlers(pi, runtime, config) {
36
40
  const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
37
41
  const query = recentUserQuery(ctx);
38
42
  if (query && runtime.store.stats(sid).checkpointCount > 0) {
39
- const r = doRecall(runtime, config, ctx, query, "resume");
43
+ // S17: use the async variant on resume so cross-repo HNSW recall can
44
+ // augment when this repo's store is thin. session_start is an async-safe
45
+ // point (unlike the mid-turn context handler, which stays sync).
46
+ const r = await doRecallAsync(runtime, config, ctx, query, "resume", { crossRepo: config.crossRepoEnabled });
40
47
  if (!r.empty) {
41
48
  runtime.pendingRecallBlock = r.block;
42
- runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt`);
43
- runtime.logger.info("auto-inline", { reason: event.reason, query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
49
+ const crossLabel = r.toInject.some((h) => h.repoId) ? " (cross-repo)" : "";
50
+ runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt${crossLabel}`);
51
+ runtime.logger.info("auto-inline", { reason: event.reason, query, injected: r.toInject.map((h) => h.checkpoint.checkpointId), crossRepo: r.toInject.some((h) => h.repoId) });
44
52
  }
45
53
  }
54
+ // S21: parallel memory recall. Same async context so we can await without
55
+ // breaking the handler contract. Best-effort — never throws.
56
+ try {
57
+ const mr = await recallMemoriesAndInline({
58
+ query, stateDir: runtime.getStateDir(), limit: 5,
59
+ });
60
+ if (!mr.empty)
61
+ runtime.pendingMemoryRecallBlock = mr.block;
62
+ }
63
+ catch (err) {
64
+ runtime.logger.warn("memory-recall skipped", { err: String(err) });
65
+ }
46
66
  }
47
67
  runtime.dashboard.event("session_start", { reason: event.reason, sessionId: runtime.rt.sessionId });
48
68
  runtime.snapshot(ctx);
@@ -60,6 +80,15 @@ export function registerEventHandlers(pi, runtime, config) {
60
80
  runtime.pendingRecallBlock = r.block;
61
81
  runtime.logger.info("auto-inline", { reason: "session_tree", query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
62
82
  }
83
+ // S21: parallel memory recall. Trigram embedder is sub-ms; await is fine.
84
+ try {
85
+ const mr = await recallMemoriesAndInline({ query, stateDir: runtime.getStateDir(), limit: 5 });
86
+ if (!mr.empty)
87
+ runtime.pendingMemoryRecallBlock = mr.block;
88
+ }
89
+ catch (err) {
90
+ runtime.logger.warn("memory-recall skipped", { err: String(err) });
91
+ }
63
92
  }
64
93
  }
65
94
  runtime.dashboard.event("session_tree", { sessionId: runtime.rt.sessionId });
@@ -68,11 +97,14 @@ export function registerEventHandlers(pi, runtime, config) {
68
97
  // ---- Auto-inline injection point: prepend staged recall to systemPrompt ----
69
98
  pi.on("before_agent_start", async (event, ctx) => {
70
99
  runtime.captureModel(ctx); // most reliable point ctx.model is populated
71
- if (!runtime.pendingRecallBlock)
100
+ const cpBlock = runtime.pendingRecallBlock;
101
+ const memBlock = runtime.pendingMemoryRecallBlock;
102
+ if (!cpBlock && !memBlock)
72
103
  return;
73
- const block = runtime.pendingRecallBlock;
74
- runtime.pendingRecallBlock = undefined; // one-shot: consume so we never double-inject
75
- return { systemPrompt: `${event.systemPrompt}\n\n${block}` };
104
+ runtime.pendingRecallBlock = undefined;
105
+ runtime.pendingMemoryRecallBlock = undefined;
106
+ const composed = [cpBlock, memBlock].filter(Boolean).join("\n\n");
107
+ return { systemPrompt: `${event.systemPrompt}\n\n${composed}` };
76
108
  });
77
109
  pi.on("session_shutdown", async (_event, ctx) => {
78
110
  runtime.setStatus(ctx, undefined);
@@ -98,6 +130,24 @@ export function registerEventHandlers(pi, runtime, config) {
98
130
  else {
99
131
  runtime.setStatus(ctx, config.auto ? "mega-compact: ready" : "mega-compact: manual only");
100
132
  }
133
+ // S16 continuation fallback: if the turn settled idle right after a live-trim
134
+ // compaction AND there is queued work AND we haven't nudged recently, nudge
135
+ // once so the agent continues (the live trim should make this rare). Guarded
136
+ // to never busy-loop: one nudge per 30s, only when truly idle + queued.
137
+ if (config.auto && runtime.activeAgents === 0) {
138
+ try {
139
+ const idle = ctx.isIdle?.() ?? true;
140
+ const queued = ctx.hasPendingMessages?.() ?? false;
141
+ const now = Date.now();
142
+ if (idle && queued && now >= runtime.resumeNudgeUntil) {
143
+ runtime.resumeNudgeUntil = now + 30_000;
144
+ pi.sendUserMessage("[mega-compact] continue from the compacted context above.");
145
+ }
146
+ }
147
+ catch {
148
+ /* non-fatal: a failed nudge never blocks */
149
+ }
150
+ }
101
151
  runtime.snapshot(ctx);
102
152
  });
103
153
  pi.on("turn_start", async (event, ctx) => {
@@ -108,16 +158,43 @@ export function registerEventHandlers(pi, runtime, config) {
108
158
  pi.on("turn_end", async (event, ctx) => {
109
159
  runtime.dashboard.event("turn_end", { turnIndex: event.turnIndex });
110
160
  runtime.snapshot(ctx);
161
+ // S20: auto-review the conversation every N turns and persist durable
162
+ // memories. Best-effort + non-fatal: a review failure must never break the
163
+ // agent loop. Debounced by memoryReviewInterval turns.
164
+ if (config.memoryAutoReview && runtime.currentTurn > 0 && runtime.currentTurn % config.memoryReviewInterval === 0) {
165
+ try {
166
+ const { reviewConversation } = await import("../src/memory.js");
167
+ const { applyMemoryOps } = await import("../src/memoryOps.js");
168
+ const entries = ctx.sessionManager.getEntries();
169
+ const view = runtime.engineView(entries.flatMap((e) => (e.message ? [e.message] : [])));
170
+ const ops = reviewConversation(view, []);
171
+ if (ops.length) {
172
+ await applyMemoryOps(ops, runtime.currentStateDir);
173
+ // S21.2: a memory op landed in this turn window. The pipeline reads
174
+ // this counter after a successful compaction and fires
175
+ // `consolidateMemories` only when it's > 0.
176
+ runtime.memoriesTouchedThisCompaction += ops.length;
177
+ }
178
+ }
179
+ catch {
180
+ /* non-fatal — auto-review must not break the turn loop */
181
+ }
182
+ }
111
183
  });
112
- // ---- Auto-trigger: own the decision, pi owns the durable write ----------
113
- // OUR auto-trigger (over threshold + debounce): persist our Trident checkpoint,
114
- // then start pi's compaction flow via ctx.compact(). That fires
115
- // `session_before_compact`, where OUR handler returns our summary +
116
- // firstKeptEntryId, and pi durably writes the trim to disk (appendCompaction).
117
- // Result: auto-compact AND a durable trim — resume reloads the trimmed window,
118
- // no full-reload + additive recall inflation (Fix B kills the token-growth bug).
119
- // We do NOT drop messages here (that would be ephemeral; the read-only session
120
- // manager can't trim disk, so the trim has to come through pi).
184
+ // ---- Auto-trigger: live trim (compact and continue) + native durable ----
185
+ // S16 redesign: we NO LONGER call ctx.compact() from the auto-trigger by
186
+ // default. That mapped to pi's MANUAL compaction path, which abort()s the
187
+ // in-flight turn (agent-session.js:1345) and stops the agent. Instead:
188
+ // - LIVE: return { messages: trimmedView } from the context event. This
189
+ // feeds pi's transformContext (sdk.js:226 agent-loop.js:180) so the
190
+ // model sees a compacted window EVERY LLM call, with no abort. The turn
191
+ // continues. We persist our recall checkpoint (the durable value) first.
192
+ // - DURABLE: pi's NATIVE auto-compaction fires at agent-end
193
+ // (agent-session.js:1565), continues (return hasQueuedMessages()), and
194
+ // emits session_before_compact — where OUR driveNativeCompaction supplies
195
+ // the summary and pi truncates the transcript on disk. No ctx.compact().
196
+ // Legacy: MEGACOMPACT_LEGACY_DURABLE_TRIM=true restores the v0.4.28 ctx.compact
197
+ // path (kept one release as rollback).
121
198
  pi.on("context", async (event, ctx) => {
122
199
  if (!config.auto)
123
200
  return;
@@ -151,17 +228,57 @@ export function registerEventHandlers(pi, runtime, config) {
151
228
  const ran = runCompact(pi, runtime, config, ctx, messages, { compressionPressure: pressure });
152
229
  if (ran.skipped)
153
230
  return;
154
- // Skip pi's durable-trim flow when pi would no-op ("Nothing to compact /
155
- // Already compacted"). pi throws that error *before* session_before_compact
156
- // fires and renders it via a compaction_end event we can't mute (onError is
157
- // too late), so the only fix is to not call ctx.compact() on a no-op. Our
158
- // recall checkpoint above is already persisted; the durable trim is
159
- // unnecessary for a transcript pi would keep in full anyway.
160
- if (piCompactWouldNoop(ctx))
231
+ // LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
232
+ // manual compact path aborts the in-flight turn — only used behind the flag.
233
+ // Read live from env (in addition to the load-time config) so the flag can be
234
+ // toggled per-test without reloading the module; config.legacyDurableTrim is
235
+ // the cached default. (Mirrors how piCompactWouldNoop re-reads its floor.)
236
+ const legacy = config.legacyDurableTrim || process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "true" || process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "1";
237
+ if (legacy) {
238
+ if (piCompactWouldNoop(ctx))
239
+ return;
240
+ ctx.compact({ customInstructions: undefined });
161
241
  return;
162
- // Start pi's compaction flow so our session_before_compact handler can
163
- // supply the durable trim (pi writes it to disk). We never use pi's summary.
164
- ctx.compact({ customInstructions: undefined });
242
+ }
243
+ // S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
244
+ // Non-destructive: pi keeps the real transcript; only this LLM call sees the
245
+ // trimmed window. We compute the cut on the engine view (pure, tested) then
246
+ // slice the ORIGINAL pi AgentMessage[] from that index (lossless alignment,
247
+ // mirroring dropCompactedRange) and prepend a user-role summary message.
248
+ // A build failure or unsafe cut returns nothing (no trim this call — the
249
+ // next context event retries). The anchor floor is read live from env (the
250
+ // config value is the cached default) so it can be tuned per-test / per-run
251
+ // without reloading the module.
252
+ try {
253
+ const anchorEnv = process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES;
254
+ const anchorUserMessages = (anchorEnv != null && anchorEnv !== "" && Number.isFinite(Number(anchorEnv)))
255
+ ? Number(anchorEnv)
256
+ : config.anchorUserMessages;
257
+ const cut = computeLiveTrimCut(view, {
258
+ compactedFrom: ran.result.compactedFrom,
259
+ summary: ran.result.summary,
260
+ anchorUserMessages,
261
+ });
262
+ if (cut === null)
263
+ return; // unsafe / below anchor floor — no trim this call
264
+ const summaryMsg = liveTrimSummaryMessage({
265
+ compactedFrom: ran.result.compactedFrom,
266
+ summary: ran.result.summary,
267
+ anchorUserMessages: config.anchorUserMessages,
268
+ });
269
+ // Synthesize a user-role AgentMessage carrying the compacted summary.
270
+ const summaryAgentMsg = {
271
+ role: "user",
272
+ content: summaryMsg.text,
273
+ timestamp: Date.now(),
274
+ };
275
+ const recent = messages.slice(cut); // guardrails-allow PREVENT-PI-002: `cut` is the pre-sanitized `compactedFrom` produced by src/boundary.ts computeDropRange, so the preserved run begins on a toolPair-safe index.
276
+ runtime.snapshot(ctx);
277
+ return { messages: [summaryAgentMsg, ...recent] };
278
+ }
279
+ catch {
280
+ return; // non-fatal: no trim this call; the next context event retries
281
+ }
165
282
  });
166
283
  // ---- Supply a DURABLE trim to pi's native compaction (Fix B) ----------
167
284
  // We run the Trident pipeline to produce a compressed summary, then return
@@ -8,10 +8,11 @@
8
8
  */
9
9
  import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
10
10
  import { compactSession } from "../src/engine.js";
11
- import { recallAndInline } from "../src/recall.js";
11
+ import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "../src/recall.js";
12
12
  import { normalizeSessionId } from "../src/store.js";
13
13
  import { estimateBlockTokens } from "../src/tokens.js";
14
14
  import { touchSession, logDaily } from "../src/store/sqlite.js";
15
+ import { consolidateMemories } from "../src/memory.js";
15
16
  import { C, MARKER_TYPE, } from "./mega-runtime.js";
16
17
  import { resolveRepoRoot, preserveRecentForPressure } from "./mega-config.js";
17
18
  import { runRaptor } from "../src/dedup/raptor/index.js";
@@ -42,6 +43,10 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
42
43
  }
43
44
  function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
44
45
  runtime.pulsing = true; // animate the status line while the (sync) pipeline runs
46
+ // S21.2: reset the per-compaction memory-op counter so the post-compact
47
+ // consolidate pass only fires when memory rows actually changed during the
48
+ // compaction window (turn_end → auto-review may have written some).
49
+ runtime.memoriesTouchedThisCompaction = 0;
45
50
  const result = compactSession({
46
51
  sessionId: sid,
47
52
  messages: view,
@@ -108,6 +113,26 @@ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
108
113
  catch {
109
114
  /* non-fatal: stats bookkeeping only */
110
115
  }
116
+ // S21.2: best-effort consolidation of near-duplicate memories for this repo.
117
+ // Runs after the per-repo stats touch so `consolidateMemories` can use the
118
+ // same stateDir. Non-fatal — a failed consolidate never blocks a compaction.
119
+ // Only runs when new memory ops landed in this pass (otherwise the prior
120
+ // compaction's consolidate already had its shot — re-running would just
121
+ // touch every row again with no merges).
122
+ if (!result.deduped && runtime.memoriesTouchedThisCompaction > 0) {
123
+ try {
124
+ const root = resolveRepoRoot(ctx.cwd);
125
+ void consolidateMemories(runtime.currentStateDir, root).then((n) => {
126
+ if (n > 0)
127
+ runtime.pushTicker(`${C.green}∫${C.reset} consolidated ${n} memory dup${n === 1 ? "" : "s"}`);
128
+ }, () => {
129
+ /* swallow: consolidate failures must never surface to the user */
130
+ });
131
+ }
132
+ catch {
133
+ /* non-fatal */
134
+ }
135
+ }
111
136
  // Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
112
137
  // skip re-vectorizing an already-compacted region (zero token cost).
113
138
  pi.appendEntry(MARKER_TYPE, {
@@ -310,6 +335,64 @@ export function doRecall(runtime, config, ctx, query, source) {
310
335
  }
311
336
  return result;
312
337
  }
338
+ /**
339
+ * S17: async recall with optional cross-repo augmentation. Used on resume
340
+ * (session_start) and /mega-recall --cross-repo — NEVER from the mid-turn
341
+ * context handler (that stays sync). Runs the sync same-repo scan first; if it
342
+ * returns < config.autoInlineK hits AND crossRepo is enabled, awaits the PGlite
343
+ * HNSW cross-repo path and merges (source-labeled, deduped by checkpointId). The
344
+ * recallMaxTokens cap + windowDedupe apply to the merged set so cross-repo can
345
+ * never net-inflate the window. Cross-repo uses a stricter cosine floor
346
+ * (config.crossRepoCosine) than same-repo. Non-fatal: any async failure returns
347
+ * the same-repo result unchanged.
348
+ */
349
+ export async function doRecallAsync(runtime, config, ctx, query, source, opts = {}) {
350
+ runtime.bindRepo(ctx.cwd);
351
+ const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
352
+ const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
353
+ // Sync same-repo first (fast, never blocks).
354
+ const sameRepo = recallAndInline({
355
+ sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
356
+ recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
357
+ liveWindow, dedupSim: config.dedupSim,
358
+ }, runtime.store);
359
+ if (!config.crossRepoEnabled || !opts.crossRepo)
360
+ return sameRepo;
361
+ if (sameRepo.toInject.length >= config.autoInlineK)
362
+ return sameRepo; // same-repo satisfied
363
+ // Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
364
+ try {
365
+ const x = await recallAndInlineAsync({
366
+ sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
367
+ recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
368
+ liveWindow, dedupSim: config.crossRepoCosine, crossRepo: true,
369
+ globalIndexDir: process.env.MEGACOMPACT_INDEX_DIR,
370
+ }, runtime.store);
371
+ runtime.dashboard.event("recall-crossrepo", {
372
+ source, query: query.slice(0, 120), injected: x.toInject.length,
373
+ sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
374
+ });
375
+ // Merge, dedup by checkpointId, respect the same token cap by reformatting.
376
+ const seen = new Set(sameRepo.toInject.map((h) => h.checkpoint.checkpointId));
377
+ const merged = [...sameRepo.toInject];
378
+ for (const h of x.toInject) {
379
+ if (!seen.has(h.checkpoint.checkpointId)) {
380
+ merged.push(h);
381
+ seen.add(h.checkpoint.checkpointId);
382
+ }
383
+ }
384
+ const block = merged.length ? formatRecallBlock(merged) : "";
385
+ return {
386
+ toInject: merged,
387
+ report: merged.map((h) => ` • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`),
388
+ block,
389
+ empty: merged.length === 0,
390
+ };
391
+ }
392
+ catch {
393
+ return sameRepo; // cross-repo failure → same-repo only (non-fatal)
394
+ }
395
+ }
313
396
  /**
314
397
  * Extract the live-window message texts from the session manager (Fix C),
315
398
  * for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
@@ -57,12 +57,17 @@ export class MegaRuntime {
57
57
  tokensSaved: 0,
58
58
  };
59
59
  debounceUntil = 0;
60
+ // S16: debounce for the agent_end resume nudge (avoid busy-loops).
61
+ resumeNudgeUntil = 0;
60
62
  // Agent tracking for real-time widget updates
61
63
  activeAgents = 0;
62
64
  currentTurn = 0;
63
65
  // Recall block produced by auto-inline (resume/branch) that the next
64
66
  // before_agent_start should prepend to the system prompt. Unset after use.
65
67
  pendingRecallBlock;
68
+ // S21: memory recall block, parallel to pendingRecallBlock. Same one-shot
69
+ // semantics; composed with the checkpoint block in before_agent_start.
70
+ pendingMemoryRecallBlock;
66
71
  statusKey; // current status text for dashboard
67
72
  // Active model/provider (for real cost estimation). Captured from ctx.model
68
73
  // on model_select + session_start; persisted to SQL so cost + the dashboard
@@ -81,6 +86,11 @@ export class MegaRuntime {
81
86
  TICKER_MAX = 5;
82
87
  // Pulsing status: set true while a compaction is in flight, cleared on result.
83
88
  pulsing = false;
89
+ // S21.2: set by `applyMemoryOps` when a memory add/replace/remove lands in
90
+ // the current compaction. The pipeline reads this after a successful compact
91
+ // to decide whether to fire `consolidateMemories` (skip the work entirely
92
+ // when no memory rows changed).
93
+ memoriesTouchedThisCompaction = 0;
84
94
  // Rolling "saved" goal for the progress bar — grows as we save more, so the
85
95
  // bar always has a meaningful denominator (never sits at 100% forever).
86
96
  savedGoal = 50_000;
@@ -343,6 +353,10 @@ export class MegaRuntime {
343
353
  }
344
354
  catch { /* non-fatal: cost estimation degrades to model-in-memory only */ }
345
355
  }
356
+ /** S21: state dir of the currently bound repo (where memories live). */
357
+ getStateDir() {
358
+ return this.currentStateDir;
359
+ }
346
360
  /** Build the sync onTier callback that paints the live per-tier trace. */
347
361
  makeTierCallback(ctx) {
348
362
  const order = ["L0", "L1", "L2", "new"];
@@ -0,0 +1,48 @@
1
+ import { isBoundarySafe } from "../src/boundary.js";
2
+ import { formatCompactSummary } from "../src/compact.js";
3
+ /**
4
+ * Compute the safe cut index for the live trim. Snaps `compactedFrom` back to a
5
+ * boundary-safe index (PREVENT-PI-002: never start the preserved run on an
6
+ * orphaned tool result), and enforces the anchor floor (PREVENT-PI-001: keep at
7
+ * least `anchorUserMessages` user-role messages). Returns `null` when no trim is
8
+ * safe this call (empty summary, unsafe boundary, or below the anchor floor) so
9
+ * the caller keeps the original view and retries on the next context event.
10
+ *
11
+ * Exposed separately from `buildLiveTrimmedView` so the context handler can map
12
+ * the cut back onto the original pi `AgentMessage[]` (lossless index alignment,
13
+ * mirroring `dropCompactedRange` in src/adapt.ts).
14
+ */
15
+ export function computeLiveTrimCut(view, opts) {
16
+ if (!opts.summary || !opts.summary.trim())
17
+ return null;
18
+ let cut = opts.compactedFrom;
19
+ while (cut > 0 && !isBoundarySafe(view, cut))
20
+ cut--;
21
+ if (cut <= 0)
22
+ return null; // nothing safe to cut — keep everything this call
23
+ const recent = view.slice(cut);
24
+ const userCount = recent.filter((m) => m.role === "user").length;
25
+ if (userCount < opts.anchorUserMessages)
26
+ return null;
27
+ return cut;
28
+ }
29
+ /** The formatted compacted-region summary as a user-role engine message. */
30
+ export function liveTrimSummaryMessage(opts) {
31
+ return {
32
+ role: "user",
33
+ text: formatCompactSummary(opts.summary),
34
+ toolName: undefined,
35
+ input: undefined,
36
+ output: undefined,
37
+ };
38
+ }
39
+ /** Build the live trimmed view. Returns the original view if summary is empty
40
+ * or the boundary is unsafe (no trim this call — try next). Pure + tested. */
41
+ export function buildLiveTrimmedView(view, opts) {
42
+ const cut = computeLiveTrimCut(view, opts);
43
+ if (cut === null)
44
+ return view;
45
+ const recent = view.slice(cut);
46
+ const summaryMsg = liveTrimSummaryMessage(opts);
47
+ return [summaryMsg, ...recent];
48
+ }
@@ -0,0 +1,58 @@
1
+ /**
2
+ * mega-trim.test.ts — tests for the live compaction view builder (S16).
3
+ */
4
+ import { test } from "node:test";
5
+ import assert from "node:assert/strict";
6
+ import { buildLiveTrimmedView } from "./mega-trim.js";
7
+ function m(role, text, extra = {}) {
8
+ return { role, text, toolName: undefined, input: undefined, output: undefined, ...extra };
9
+ }
10
+ test("buildLiveTrimmedView: prepends a compacted summary and keeps the recent anchor", () => {
11
+ const view = [
12
+ m("user", "old request one"), m("assistant", "old answer one"),
13
+ m("user", "old request two"), m("assistant", "old answer two"),
14
+ m("user", "recent keep me"), m("assistant", "recent keep me too"),
15
+ ];
16
+ // Compacted region = first 4; recent anchor = last 2.
17
+ const result = buildLiveTrimmedView(view, {
18
+ compactedFrom: 4, // index where the compacted region ends
19
+ summary: "<summary>earlier work on old requests</summary>",
20
+ anchorUserMessages: 1,
21
+ });
22
+ // First element is the injected compacted summary as a user-role message.
23
+ assert.equal(result[0].role, "user");
24
+ assert.ok(String(result[0].text).includes("earlier work on old requests"));
25
+ // Recent anchor preserved in order, no older messages leak through.
26
+ assert.equal(result.length, 1 + 2, "summary + 2 recent");
27
+ assert.ok(result.slice(1).some((x) => String(x.text).includes("recent keep me")));
28
+ });
29
+ test("buildLiveTrimmedView: empty summary returns the original view unchanged", () => {
30
+ const view = [m("user", "x"), m("assistant", "y")];
31
+ const result = buildLiveTrimmedView(view, { compactedFrom: 0, summary: "", anchorUserMessages: 1 });
32
+ assert.deepEqual(result, view);
33
+ });
34
+ test("buildLiveTrimmedView: never splits a toolCall/toolResult pair (PREVENT-PI-002)", () => {
35
+ const view = [
36
+ m("user", "q"), m("assistant", "calls tool", { toolName: "read" }), m("tool", "result"),
37
+ m("user", "keep"), m("assistant", "ok"),
38
+ ];
39
+ // cut=3 would start the preserved run on the orphaned tool result at index 2 —
40
+ // the builder must snap back so the toolCall/toolResult pair is not split.
41
+ const result = buildLiveTrimmedView(view, { compactedFrom: 3, summary: "<summary>s</summary>", anchorUserMessages: 1 });
42
+ // The tool result must never appear preserved WITHOUT its preceding toolCall.
43
+ const preserved = result.slice(1);
44
+ const hasToolResult = preserved.some((x) => x.role === "tool");
45
+ const hasToolCall = preserved.some((x) => x.role === "assistant" && x.toolName);
46
+ // Either the tool pair is kept together, or the tool result is dropped into
47
+ // the compacted region — it is never left orphaned.
48
+ assert.ok(!(hasToolResult && !hasToolCall), "no orphaned tool result in the preserved run");
49
+ });
50
+ test("buildLiveTrimmedView: honors the anchor floor (PREVENT-PI-001)", () => {
51
+ // cut would leave zero user messages in the anchor — must skip the trim.
52
+ const view = [
53
+ m("user", "old q"), m("assistant", "old a"),
54
+ m("assistant", "only assistant kept"),
55
+ ];
56
+ const result = buildLiveTrimmedView(view, { compactedFrom: 2, summary: "<summary>s</summary>", anchorUserMessages: 1 });
57
+ assert.deepEqual(result, view, "below anchor floor → no trim this call");
58
+ });
@@ -46,6 +46,7 @@ export function loadDedupConfig() {
46
46
  DEDUP_SIM: envNum("MEGACOMPACT_DEDUP_SIM", 0.9),
47
47
  MMR_LAMBDA: envNum("MEGACOMPACT_MMR_LAMBDA", 0.5),
48
48
  SEMDEDUP_COSINE: envNum("MEGACOMPACT_SEMDEDUP_COSINE", 0.95),
49
+ CONSOLIDATE_COSINE: envNum("MEGACOMPACT_CONSOLIDATE_COSINE", 0.7),
49
50
  SIMILARITY_BUDGET_MS: envNum("MEGACOMPACT_SIMILARITY_BUDGET_MS", 50),
50
51
  L1_VERIFY_BUDGET_MS: envNum("MEGACOMPACT_L1_VERIFY_BUDGET_MS", 20),
51
52
  L1_CANDIDATE_CAP: envNum("MEGACOMPACT_L1_CANDIDATE_CAP", 100),