pi-mega-compact 0.4.28 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +47 -2
  2. package/dist/extensions/dashboard-server.js +66 -3
  3. package/dist/extensions/dashboard-server.test.js +95 -3
  4. package/dist/extensions/mega-commands.js +25 -9
  5. package/dist/extensions/mega-compact.test.js +133 -31
  6. package/dist/extensions/mega-config.js +5 -0
  7. package/dist/extensions/mega-conflict-cmds.js +79 -0
  8. package/dist/extensions/mega-dashboard-cmds.js +6 -4
  9. package/dist/extensions/mega-events.js +144 -27
  10. package/dist/extensions/mega-pipeline.js +84 -1
  11. package/dist/extensions/mega-runtime.js +35 -2
  12. package/dist/extensions/mega-trim.js +48 -0
  13. package/dist/extensions/mega-trim.test.js +58 -0
  14. package/dist/src/config/dedup.js +1 -0
  15. package/dist/src/driftDetection.js +103 -0
  16. package/dist/src/driftDetection.test.js +87 -0
  17. package/dist/src/memory.js +147 -0
  18. package/dist/src/memory.test.js +41 -0
  19. package/dist/src/memoryConsolidate.test.js +38 -0
  20. package/dist/src/memoryOps.js +58 -0
  21. package/dist/src/memoryOps.test.js +41 -0
  22. package/dist/src/memoryRecall.js +60 -0
  23. package/dist/src/memoryRecall.test.js +92 -0
  24. package/dist/src/recall.js +70 -1
  25. package/dist/src/recall.test.js +69 -1
  26. package/dist/src/store/sqlite.js +127 -11
  27. package/dist/src/vectorStore.js +6 -1
  28. package/extensions/dashboard-server.test.ts +115 -3
  29. package/extensions/dashboard-server.ts +69 -4
  30. package/extensions/mega-commands.ts +24 -9
  31. package/extensions/mega-compact.test.ts +134 -31
  32. package/extensions/mega-config.ts +22 -0
  33. package/extensions/mega-conflict-cmds.ts +81 -0
  34. package/extensions/mega-dashboard-cmds.ts +6 -4
  35. package/extensions/mega-events.ts +139 -28
  36. package/extensions/mega-pipeline.ts +94 -1
  37. package/extensions/mega-runtime.ts +35 -2
  38. package/extensions/mega-trim.test.ts +64 -0
  39. package/extensions/mega-trim.ts +75 -0
  40. package/extensions/openclaw-mega-compact.ts +24 -9
  41. package/package.json +2 -2
  42. package/src/config/dedup.ts +2 -0
  43. package/src/driftDetection.test.ts +100 -0
  44. package/src/driftDetection.ts +136 -0
  45. package/src/memory.test.ts +46 -0
  46. package/src/memory.ts +164 -0
  47. package/src/memoryConsolidate.test.ts +47 -0
  48. package/src/memoryOps.test.ts +53 -0
  49. package/src/memoryOps.ts +75 -0
  50. package/src/memoryRecall.test.ts +100 -0
  51. package/src/memoryRecall.ts +83 -0
  52. package/src/recall.test.ts +77 -1
  53. package/src/recall.ts +94 -1
  54. package/src/store/sqlite.ts +188 -11
  55. package/src/store.ts +3 -0
  56. package/src/vectorStore.ts +10 -1
@@ -18,6 +18,7 @@ import { mkdtempSync, rmSync } from "node:fs";
18
18
  import { tmpdir } from "node:os";
19
19
  import { join } from "node:path";
20
20
  import { createRequire } from "node:module";
21
+ import { closeVectorIndex } from "../src/store/vectorIndex.js";
21
22
  import type { AgentMessage } from "@earendil-works/pi-agent-core";
22
23
 
23
24
  const require = createRequire(import.meta.url);
@@ -177,14 +178,18 @@ function harness(opts: { keepTier?: boolean; keepThreshold?: boolean } = {}) {
177
178
  };
178
179
  }
179
180
 
180
- test("auto-trigger: past threshold persists a chkpt and starts a durable trim", async () => {
181
+ test("auto-trigger (legacy): past threshold persists a chkpt and starts a durable trim via ctx.compact", async () => {
181
182
  const h = harness();
182
183
  const messages = h.session;
183
184
  // The mock session is tiny (~100 tokens). piCompactWouldNoop() would skip
184
185
  // ctx.compact() for a transcript under pi's keepRecentTokens budget — so
185
186
  // lower the floor to 0 to simulate a transcript large enough that pi WOULD
186
187
  // compact (the positive path this test exercises).
188
+ // S16: this is the LEGACY path — the default no longer calls ctx.compact()
189
+ // (it returns a live-trimmed view instead). Set the legacy flag to exercise
190
+ // the v0.4.28 ctx.compact durable-trim flow this test asserts.
187
191
  process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR = "0";
192
+ process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM = "true";
188
193
  try {
189
194
  const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
190
195
  const res = await h.fire("context", { type: "context", messages }, ctx);
@@ -192,34 +197,115 @@ test("auto-trigger: past threshold persists a chkpt and starts a durable trim",
192
197
  const { listCheckpoints } = await import("../src/store/sqlite.js");
193
198
  assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "checkpoint persisted to local vector db");
194
199
  assert.equal(h.appended.some((a) => a.t === "mega-compact-marker"), true, "marker sentinel appended");
195
- // The context handler no longer drops messages itself (that was ephemeral —
196
- // the read-path token-growth bug). It triggers pi's compaction flow, which
197
- // calls our session_before_compact handler to supply the DURABLE trim.
198
- assert.equal(res, undefined, "context handler returns nothing (no local drop)");
199
- assert.equal(h.compactCalls.length, 1, "ctx.compact() called to start durable trim");
200
+ // The legacy context handler triggers pi's compaction flow (ctx.compact),
201
+ // which calls our session_before_compact handler to supply the DURABLE trim.
202
+ assert.equal(res, undefined, "legacy context handler returns nothing (no local drop)");
203
+ assert.equal(h.compactCalls.length, 1, "ctx.compact() called to start durable trim (legacy path)");
200
204
  // The durable trim was supplied (summary + firstKeptEntryId from pi's prep).
201
205
  assert.ok(h.compactCalls[0] !== undefined, "compaction flow executed");
202
206
  } finally {
203
207
  delete process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
208
+ delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
204
209
  }
205
210
  });
206
211
 
207
- test("auto-trigger: skips ctx.compact() when pi would no-op (session too small)", async () => {
212
+ test("auto-trigger: skips ctx.compact() when pi would no-op (session too small, legacy path)", async () => {
208
213
  const h = harness();
209
214
  const messages = h.session;
210
215
  // Default floor (20000): the tiny mock transcript is below pi's
211
216
  // keepRecentTokens budget, so piCompactWouldNoop() must skip ctx.compact()
212
217
  // rather than surface pi's "Nothing to compact (session too small)" throw.
218
+ // S16: exercised under the legacy flag (the default path never calls ctx.compact).
213
219
  delete process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
220
+ process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM = "true";
221
+ try {
222
+ const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
223
+ const res = await h.fire("context", { type: "context", messages }, ctx);
224
+ assert.equal(res, undefined, "legacy context handler returns nothing (no local drop)");
225
+ assert.equal(h.compactCalls.length, 0, "ctx.compact() NOT called — pi would no-op");
226
+ // Our recall checkpoint still persisted (Path A) — the durable trim is the
227
+ // only thing skipped; recall is independent of it.
228
+ const { listCheckpoints } = await import("../src/store/sqlite.js");
229
+ assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "recall checkpoint still persisted");
230
+ assert.equal(h.appended.some((a) => a.t === "mega-compact-marker"), true, "marker sentinel still appended");
231
+ } finally {
232
+ delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
233
+ }
234
+ });
235
+
236
+ test("auto-trigger (S16): trims the live view and does NOT call ctx.compact()", async () => {
237
+ const h = harness();
238
+ const messages = h.session;
239
+ // S16 default: live context-event trim. No legacy flag. Lower the anchor floor
240
+ // so the trimmed recent window (4 messages, 2 user) clears the anchor check
241
+ // and the live trim actually fires — mirrors how the legacy test lowers the
242
+ // durable floor to exercise its positive path.
243
+ delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
244
+ delete process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
245
+ process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES = "1";
246
+ try {
247
+ const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
248
+ const res = await h.fire("context", { type: "context", messages }, ctx);
249
+ // S16: context handler returns a TRIMMED messages array (live trim), not undefined.
250
+ assert.ok(res && typeof res === "object", "context handler returns a result object (live trim)");
251
+ assert.ok(Array.isArray((res as any).messages), "result has a trimmed messages array");
252
+ // The trimmed view starts with the compacted summary (user-role) + is shorter.
253
+ assert.ok((res as any).messages.length < messages.length, "trimmed view is shorter than the full session");
254
+ // S16: ctx.compact() is NEVER called (it would stop the agent).
255
+ assert.equal(h.compactCalls.length, 0, "ctx.compact() NOT called — compact-and-continue");
256
+ // The recall checkpoint is still persisted (the durable value).
257
+ const { listCheckpoints } = await import("../src/store/sqlite.js");
258
+ assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "recall checkpoint persisted under live trim");
259
+ } finally {
260
+ delete process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES;
261
+ }
262
+ });
263
+
264
+ test("auto-trigger (S16): does not trim when below the anchor floor (returns undefined, no ctx.compact)", async () => {
265
+ const h = harness();
266
+ // A session so short that buildLiveTrimmedView's anchor floor can't hold — the
267
+ // live trim skips this call (returns undefined, the next context event retries).
268
+ delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
269
+ delete process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
270
+ const shortSession = [h.session[0], h.session[1]]; // one user + one assistant
214
271
  const ctx = h.ctx({ getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }) });
215
- const res = await h.fire("context", { type: "context", messages }, ctx);
216
- assert.equal(res, undefined, "context handler returns nothing (no local drop)");
217
- assert.equal(h.compactCalls.length, 0, "ctx.compact() NOT called pi would no-op");
218
- // Our recall checkpoint still persisted (Path A) — the durable trim is the
219
- // only thing skipped; recall is independent of it.
220
- const { listCheckpoints } = await import("../src/store/sqlite.js");
221
- assert.ok(listCheckpoints("sess_ext_001", h.stateDir).length > 0, "recall checkpoint still persisted");
222
- assert.equal(h.appended.some((a) => a.t === "mega-compact-marker"), true, "marker sentinel still appended");
272
+ const res = await h.fire("context", { type: "context", messages: shortSession }, ctx);
273
+ // Either it skipped (undefined) or trimmed safely but it must never call ctx.compact.
274
+ assert.equal(h.compactCalls.length, 0, "ctx.compact() NOT called under live trim (short session)");
275
+ if (res === undefined) {
276
+ // skipped path is fine
277
+ assert.ok(true, "below anchor floor no trim this call (retries next event)");
278
+ }
279
+ });
280
+
281
+ test("auto-trigger (S16): sendUserMessage resume nudge fires only when idle + queued + not already nudged", async () => {
282
+ const h = harness();
283
+ // No queued messages → the nudge must NOT fire (the guard prevents busy-loops).
284
+ // We assert the extension did not throw and did not push a spurious resume.
285
+ const ctx = h.ctx({ isIdle: () => true, hasPendingMessages: () => false });
286
+ await h.fire("agent_end", { type: "agent_end", messages: [] }, ctx);
287
+ // No throw + no spurious nudge side-effect is the contract; appended stays
288
+ // free of any auto "continue" marker when there is no queued work.
289
+ assert.equal(h.appended.some((a) => a.t && /continue/i.test(String(a.d ?? ""))), false, "no spurious continue when no queued work");
290
+ });
291
+
292
+ test("auto-trigger (S16): durable trim still happens via pi native auto-compaction (session_before_compact)", async () => {
293
+ const h = harness();
294
+ // pi's native auto-compaction fires at agent-end with reason "threshold" (the
295
+ // CONTINUING path). Our session_before_compact handler must still supply the
296
+ // durable trim summary — independent of the live context-event trim.
297
+ const prep = {
298
+ firstKeptEntryId: "e2",
299
+ messagesToSummarize: h.session.slice(0, 4),
300
+ tokensBefore: 500,
301
+ };
302
+ const res = await h.fire("session_before_compact", {
303
+ type: "session_before_compact", reason: "threshold", willRetry: false,
304
+ signal: undefined, preparation: prep,
305
+ } as any, h.ctx());
306
+ assert.ok(res?.compaction, "we supply a durable compaction result to pi's native path");
307
+ assert.ok(res.compaction.firstKeptEntryId === "e2", "reuses pi's boundary (PREVENT-PI-002)");
308
+ assert.ok(res.compaction.summary.length > 0, "summary is non-empty");
223
309
  });
224
310
 
225
311
  test("session_before_compact supplies our durable trim (not pi's summary)", async () => {
@@ -361,10 +447,17 @@ test("explicit MEGACOMPACT_THRESHOLD_TOKENS overrides the tier", async () => {
361
447
 
362
448
  // ---- /dashboard commands ----------------------------------------------------
363
449
  test("/dashboard-status reports no server when pid file missing", async () => {
364
- const h = harness();
365
- const ctx = h.ctx();
366
- await h.commands["mega-dashboard-status"].handler("", ctx);
367
- assert.ok(h.notifies.some((n) => n.includes("not running")), "reports no server running");
450
+ // Private base so this asserts "no server" on a range nothing else uses,
451
+ // not the machine-global 9320 family (which may hold a leftover/production server).
452
+ process.env.MEGACOMPACT_DASHBOARD_PORT = "49320";
453
+ try {
454
+ const h = harness();
455
+ const ctx = h.ctx();
456
+ await h.commands["mega-dashboard-status"].handler("", ctx);
457
+ assert.ok(h.notifies.some((n) => n.includes("not running")), "reports no server running");
458
+ } finally {
459
+ delete process.env.MEGACOMPACT_DASHBOARD_PORT;
460
+ }
368
461
  });
369
462
 
370
463
  test("/dashboard-stop reports no server when pid file missing", async () => {
@@ -375,20 +468,23 @@ test("/dashboard-stop reports no server when pid file missing", async () => {
375
468
  });
376
469
 
377
470
  test("/dashboard skips server spawn when already running", async () => {
471
+ // Use a private dashboard port base for THIS test's harness + fake server so
472
+ // it never races the (parallel, hard-coded-9320) dashboard-server.test.js or
473
+ // a leftover production server. Set BEFORE harness() so registerDashboardCommands
474
+ // reads our base for findLivePort().
475
+ process.env.MEGACOMPACT_DASHBOARD_PORT = "29320";
378
476
  const h = harness();
379
477
  const confirms: boolean[] = [];
380
- // Set up a fake HTTP server on a port inside the dashboard's scan range
381
- // (9320–9329) — isServerRunning() probes those ports, not the port.pid value.
478
+ const livPort = 29320; // inside the harness's private scan range (29320–29329)
382
479
  const { createServer } = await import("node:http");
383
480
  const server = createServer((_req, res) => {
384
481
  res.writeHead(200, { "Content-Type": "application/json" });
385
482
  res.end(JSON.stringify({ updatedAt: new Date().toISOString(), tier: "test", version: 1, config: {}, session: {}, context: {}, trigger: {}, store: {} }));
386
483
  });
387
- await new Promise<void>((r) => server.listen(9320, "127.0.0.1", r));
388
- const addr = server.address() as any;
484
+ await new Promise<void>((r) => server.listen(livPort, "127.0.0.1", r));
389
485
  const { join: j } = await import("node:path");
390
486
  const { writeFileSync: wf } = await import("node:fs");
391
- wf(j(h.stateDir, "port.pid"), JSON.stringify({ port: addr.port, pid: process.pid }));
487
+ wf(j(h.stateDir, "port.pid"), JSON.stringify({ port: livPort, pid: process.pid }));
392
488
 
393
489
  const ctx = h.ctx({
394
490
  ui: {
@@ -405,12 +501,15 @@ test("/dashboard skips server spawn when already running", async () => {
405
501
  assert.ok(confirms.length > 0, "confirm dialog was shown");
406
502
 
407
503
  await new Promise<void>((r) => server.close(() => r()));
504
+ delete process.env.MEGACOMPACT_DASHBOARD_PORT;
408
505
  });
409
506
 
410
507
  test("/dashboard-status reports running after dashboard start", async () => {
508
+ // Private dashboard port base for this harness — never collides with the
509
+ // parallel dashboard-server.test.js (9320 family) or a leftover server.
510
+ process.env.MEGACOMPACT_DASHBOARD_PORT = "39320";
411
511
  const h = harness();
412
- // Write a fake port.pid; the server must listen inside the scan range
413
- // (9320–9329) or isServerRunning() won't detect it.
512
+ const livPort = 39320;
414
513
  const { createServer } = await import("node:http");
415
514
  const { join: j } = await import("node:path");
416
515
  const { writeFileSync: wf } = await import("node:fs");
@@ -418,15 +517,15 @@ test("/dashboard-status reports running after dashboard start", async () => {
418
517
  res.writeHead(200, { "Content-Type": "application/json" });
419
518
  res.end(JSON.stringify({ updatedAt: new Date().toISOString(), tier: "test" }));
420
519
  });
421
- await new Promise<void>((r) => server.listen(9321, "127.0.0.1", r));
422
- const addr = server.address() as any;
423
- wf(j(h.stateDir, "port.pid"), JSON.stringify({ port: addr.port, pid: process.pid }));
520
+ await new Promise<void>((r) => server.listen(livPort, "127.0.0.1", r));
521
+ wf(j(h.stateDir, "port.pid"), JSON.stringify({ port: livPort, pid: process.pid }));
424
522
 
425
523
  const ctx = h.ctx();
426
524
  await h.commands["mega-dashboard-status"].handler("", ctx);
427
- assert.ok(h.notifies.some((n) => n.includes("running") && n.includes(String(addr.port))), "reports running with port");
525
+ assert.ok(h.notifies.some((n) => n.includes("running") && n.includes(String(livPort))), "reports running with port");
428
526
 
429
527
  await new Promise<void>((r) => server.close(() => r()));
528
+ delete process.env.MEGACOMPACT_DASHBOARD_PORT;
430
529
  });
431
530
 
432
531
  test("state snapshot writes dashboard.json after compaction", async () => {
@@ -473,6 +572,10 @@ test("events.log receives compaction events", async () => {
473
572
  }
474
573
  });
475
574
 
476
- test("cleanup", () => {
575
+ test("cleanup", async () => {
576
+ // Terminate the global PGlite cross-repo index (WASM worker thread) so the
577
+ // test process can exit. Without this, node --test never returns even though
578
+ // every test passed — the leaked worker keeps the event loop alive.
579
+ await closeVectorIndex();
477
580
  rmSync(baseTmp, { recursive: true, force: true });
478
581
  });
@@ -44,6 +44,23 @@ export interface MegaConfig {
44
44
  /** RAPTOR hierarchical recall enabled (Fix D). Drives both live recall and
45
45
  * the durable-trim summary source (root summary). */
46
46
  raptorEnabled: boolean;
47
+ /** Legacy v0.4.28 behavior: auto-trigger calls ctx.compact() (which STOPS
48
+ * the agent). Default false — the S16 redesign uses the live context-event
49
+ * trim + pi native auto-compaction instead (compact and continue). Kept for
50
+ * one release as rollback. */
51
+ legacyDurableTrim: boolean;
52
+ /** Cross-repo recall enabled (S17). Resume + /mega-recall --cross-repo can
53
+ * pull checkpoints from OTHER repos via the PGlite HNSW index. Default true. */
54
+ crossRepoEnabled: boolean;
55
+ /** Stricter cosine floor for cross-repo hits (S17). Default 0.90 (trigram) /
56
+ * tighter than same-repo so only genuinely-relevant cross-repo context is
57
+ * injected. */
58
+ crossRepoCosine: number;
59
+ /** Memory-RAG auto-review enabled (S20). Every memoryReviewInterval turns the
60
+ * conversation is auto-reviewed into durable add/replace/remove memories. */
61
+ memoryAutoReview: boolean;
62
+ /** Turn cadence for the auto-review scan (S20). Default 10. */
63
+ memoryReviewInterval: number;
47
64
  /** Token ceiling for the re-injected recall block (Fix C). Recall stops
48
65
  * adding checkpoints once the block would exceed this — bounds read-path
49
66
  * token cost so it can never net-inflate the window. */
@@ -104,6 +121,11 @@ export function loadConfig(): MegaConfig {
104
121
  autoInlineK: envFlag("MEGACOMPACT_AUTO_INLINE_K", 3),
105
122
  dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
106
123
  raptorEnabled: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
124
+ legacyDurableTrim: envBool("MEGACOMPACT_LEGACY_DURABLE_TRIM", false),
125
+ crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
126
+ crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
127
+ memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),
128
+ memoryReviewInterval: envFlag("MEGACOMPACT_MEMORY_REVIEW_INTERVAL", 10),
107
129
  recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
108
130
  windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
109
131
  debug: envBool("MEGACOMPACT_DEBUG", false),
@@ -126,4 +126,85 @@ export function registerConflictCommands(pi: ExtensionAPI, runtime: MegaRuntime)
126
126
  for (const m of all) ctx.ui.notify(memoryLine(m));
127
127
  },
128
128
  });
129
+
130
+ // Shortform aliases — `m save "..."`, `m status`, `m list`, `m search <q>`,
131
+ // `m recall <id>`. Delegates to the same SQLite store so there's one source
132
+ // of truth. The /mega-memory command remains the canonical form.
133
+ pi.registerCommand("m", {
134
+ description: "Shortform alias for /mega-memory. Usage: /m save <text> | list | search <q> | recall <id> | status",
135
+ handler: async (args: string, ctx: ExtensionContext) => {
136
+ const repo = resolveRepoRoot(ctx.cwd) ?? runtime.currentStateDir;
137
+ const parts = args.trim().split(/\s+/);
138
+ const sub = parts[0]?.toLowerCase() ?? "list";
139
+
140
+ if (sub === "save") {
141
+ // Strip leading "save" so /m save "#foo bar" works the same as the
142
+ // canonical form. Then strip a balanced outer quote pair if the user
143
+ // wrote /m save "..." — common when the text contains spaces.
144
+ let text = args.trim().slice(4).trim();
145
+ const mq = text.match(/^["“](.*)["”]$/s);
146
+ if (mq) text = mq[1].trim();
147
+ if (!text) {
148
+ ctx.ui.notify('[/m] usage: /m save "<text>" or /m save <text>');
149
+ return;
150
+ }
151
+ const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
152
+ const content = text.replace(/#[\w-]+/g, "").trim();
153
+ const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
154
+ ctx.ui.notify(`[/m] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
155
+ return;
156
+ }
157
+
158
+ if (sub === "search") {
159
+ const q = parts.slice(1).join(" ").trim();
160
+ if (!q) {
161
+ ctx.ui.notify("[/m] usage: /m search <query>");
162
+ return;
163
+ }
164
+ const hits = searchMemories(q, repo, 50, runtime.currentStateDir);
165
+ if (!hits.length) {
166
+ ctx.ui.notify("[/m] no memories match.");
167
+ return;
168
+ }
169
+ for (const mem of hits) ctx.ui.notify(memoryLine(mem));
170
+ return;
171
+ }
172
+
173
+ if (sub === "recall") {
174
+ const id = Number(parts[1]);
175
+ if (!Number.isFinite(id) || parts[1] === undefined) {
176
+ ctx.ui.notify("[/m] usage: /m recall <id>");
177
+ return;
178
+ }
179
+ if (recallMemory(id, runtime.currentStateDir)) {
180
+ const found = listMemories(repo, 1000, runtime.currentStateDir).find((mem) => mem.id === id);
181
+ ctx.ui.notify(found ? `[/m] ${memoryLine(found)}` : `[/m] recalled #${id}`);
182
+ } else {
183
+ ctx.ui.notify(`[/m] #${id} not found.`);
184
+ }
185
+ return;
186
+ }
187
+
188
+ if (sub === "status") {
189
+ const all = listMemories(repo, 1000, runtime.currentStateDir);
190
+ const byKind = all.reduce<Record<string, number>>((acc, m) => {
191
+ acc[m.kind] = (acc[m.kind] ?? 0) + 1;
192
+ return acc;
193
+ }, {});
194
+ const kinds = Object.entries(byKind).map(([k, n]) => `${k}=${n}`).join(", ");
195
+ const head = `[/m] ${all.length} memory record(s) in ${repo.split(/[\\/]/).pop() ?? repo}`;
196
+ ctx.ui.notify(kinds ? `${head} (${kinds})` : head);
197
+ return;
198
+ }
199
+
200
+ // default: list
201
+ const all = listMemories(repo, 50, runtime.currentStateDir);
202
+ if (!all.length) {
203
+ ctx.ui.notify("[/m] no saved memories yet. Use /m save <text>.");
204
+ return;
205
+ }
206
+ ctx.ui.notify(`[/m] ${all.length} memory record(s):`);
207
+ for (const mem of all) ctx.ui.notify(memoryLine(mem));
208
+ },
209
+ });
129
210
  }
@@ -23,11 +23,13 @@ export function registerDashboardCommands(pi: ExtensionAPI, runtime: MegaRuntime
23
23
  // the shipped compiled dist/extensions/dashboard-server.js).
24
24
  let dashboardNeedsStrip = false;
25
25
 
26
- // The dashboard server binds 9320–9329 (TARGET_PORT..TARGET_PORT+PORT_RANGE-1
27
- // in dashboard-server.js). Probe each for a live /api/snapshot so we can detect
28
- // readiness even when port.pid landed in a different state dir than we poll.
26
+ // The dashboard server binds a 10-port range starting at MEGACOMPACT_DASHBOARD_PORT
27
+ // (default 9320) — see TARGET_PORT/PORT_RANGE in dashboard-server.js. Probe each for
28
+ // a live /api/snapshot so we detect readiness even when port.pid landed in a different
29
+ // state dir than we poll. Configurable so tests can use a private, non-colliding range.
30
+ const DASH_BASE = Number(process.env.MEGACOMPACT_DASHBOARD_PORT ?? "9320");
29
31
  async function findLivePort(): Promise<number | null> {
30
- for (let port = 9320; port <= 9329; port++) {
32
+ for (let port = DASH_BASE; port <= DASH_BASE + 9; port++) {
31
33
  try {
32
34
  const res = await fetch(`http://localhost:${port}/api/snapshot`, { signal: AbortSignal.timeout(800) }); // guardrails-allow PREVENT-PI-004: localhost liveness probe of the dashboard server this extension spawned
33
35
  if (res.ok) return port;
@@ -8,12 +8,15 @@
8
8
  */
9
9
 
10
10
  import type { ExtensionAPI, ExtensionContext, ContextEvent, SessionBeforeCompactEvent } from "@earendil-works/pi-coding-agent";
11
+ import type { AgentMessage } from "@earendil-works/pi-agent-core";
11
12
  import { normalizeSessionId } from "../src/store.js";
12
13
  import { autoCompactCheck } from "../src/compact.js";
13
14
  import { estimateSessionTokens } from "../src/tokens.js";
14
15
  import { MegaRuntime, recentUserQuery, WIDGET_KEY } from "./mega-runtime.js";
15
- import { runCompact, doRecall, piCompactWouldNoop } from "./mega-pipeline.js";
16
+ import { runCompact, doRecall, doRecallAsync, piCompactWouldNoop } from "./mega-pipeline.js";
17
+ import { recallMemoriesAndInline } from "../src/recall.js";
16
18
  import { driveNativeCompaction } from "./mega-compact-driver.js";
19
+ import { computeLiveTrimCut, liveTrimSummaryMessage } from "./mega-trim.js";
17
20
  import { pressureFromPct, type MegaConfig } from "./mega-config.js";
18
21
 
19
22
  /** Register all pi lifecycle event handlers. */
@@ -29,6 +32,8 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
29
32
  runtime.resetRuntime(ctx.sessionManager.getSessionId());
30
33
  runtime.captureModel(ctx); // best-effort: ctx.model may be set by session start
31
34
  runtime.setStatus(ctx, config.auto ? "mega-compact: ready" : "mega-compact: manual only");
35
+ // S21: clear any stale memory block from a prior session.
36
+ runtime.pendingMemoryRecallBlock = undefined;
32
37
  // Auto-inline on resume/fork/continue: stage the most relevant checkpoints
33
38
  // so the next before_agent_start prepends them to the system prompt.
34
39
  // Triggered whenever this session already has persisted checkpoints AND a
@@ -40,13 +45,27 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
40
45
  const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
41
46
  const query = recentUserQuery(ctx);
42
47
  if (query && runtime.store.stats(sid).checkpointCount > 0) {
43
- const r = doRecall(runtime, config, ctx, query, "resume");
48
+ // S17: use the async variant on resume so cross-repo HNSW recall can
49
+ // augment when this repo's store is thin. session_start is an async-safe
50
+ // point (unlike the mid-turn context handler, which stays sync).
51
+ const r = await doRecallAsync(runtime, config, ctx, query, "resume", { crossRepo: config.crossRepoEnabled });
44
52
  if (!r.empty) {
45
53
  runtime.pendingRecallBlock = r.block;
46
- runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt`);
47
- runtime.logger.info("auto-inline", { reason: event.reason, query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
54
+ const crossLabel = r.toInject.some((h) => h.repoId) ? " (cross-repo)" : "";
55
+ runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt${crossLabel}`);
56
+ runtime.logger.info("auto-inline", { reason: event.reason, query, injected: r.toInject.map((h) => h.checkpoint.checkpointId), crossRepo: r.toInject.some((h) => h.repoId) });
48
57
  }
49
58
  }
59
+ // S21: parallel memory recall. Same async context so we can await without
60
+ // breaking the handler contract. Best-effort — never throws.
61
+ try {
62
+ const mr = await recallMemoriesAndInline({
63
+ query, stateDir: runtime.getStateDir(), limit: 5,
64
+ });
65
+ if (!mr.empty) runtime.pendingMemoryRecallBlock = mr.block;
66
+ } catch (err) {
67
+ runtime.logger.warn("memory-recall skipped", { err: String(err) });
68
+ }
50
69
  }
51
70
  runtime.dashboard.event("session_start", { reason: event.reason, sessionId: runtime.rt.sessionId });
52
71
  runtime.snapshot(ctx);
@@ -65,6 +84,13 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
65
84
  runtime.pendingRecallBlock = r.block;
66
85
  runtime.logger.info("auto-inline", { reason: "session_tree", query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
67
86
  }
87
+ // S21: parallel memory recall. Trigram embedder is sub-ms; await is fine.
88
+ try {
89
+ const mr = await recallMemoriesAndInline({ query, stateDir: runtime.getStateDir(), limit: 5 });
90
+ if (!mr.empty) runtime.pendingMemoryRecallBlock = mr.block;
91
+ } catch (err) {
92
+ runtime.logger.warn("memory-recall skipped", { err: String(err) });
93
+ }
68
94
  }
69
95
  }
70
96
  runtime.dashboard.event("session_tree", { sessionId: runtime.rt.sessionId });
@@ -74,10 +100,13 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
74
100
  // ---- Auto-inline injection point: prepend staged recall to systemPrompt ----
75
101
  pi.on("before_agent_start", async (event, ctx) => {
76
102
  runtime.captureModel(ctx); // most reliable point ctx.model is populated
77
- if (!runtime.pendingRecallBlock) return;
78
- const block = runtime.pendingRecallBlock;
79
- runtime.pendingRecallBlock = undefined; // one-shot: consume so we never double-inject
80
- return { systemPrompt: `${event.systemPrompt}\n\n${block}` };
103
+ const cpBlock = runtime.pendingRecallBlock;
104
+ const memBlock = runtime.pendingMemoryRecallBlock;
105
+ if (!cpBlock && !memBlock) return;
106
+ runtime.pendingRecallBlock = undefined;
107
+ runtime.pendingMemoryRecallBlock = undefined;
108
+ const composed = [cpBlock, memBlock].filter(Boolean).join("\n\n");
109
+ return { systemPrompt: `${event.systemPrompt}\n\n${composed}` };
81
110
  });
82
111
 
83
112
  pi.on("session_shutdown", async (_event, ctx) => {
@@ -105,6 +134,23 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
105
134
  } else {
106
135
  runtime.setStatus(ctx, config.auto ? "mega-compact: ready" : "mega-compact: manual only");
107
136
  }
137
+ // S16 continuation fallback: if the turn settled idle right after a live-trim
138
+ // compaction AND there is queued work AND we haven't nudged recently, nudge
139
+ // once so the agent continues (the live trim should make this rare). Guarded
140
+ // to never busy-loop: one nudge per 30s, only when truly idle + queued.
141
+ if (config.auto && runtime.activeAgents === 0) {
142
+ try {
143
+ const idle = ctx.isIdle?.() ?? true;
144
+ const queued = ctx.hasPendingMessages?.() ?? false;
145
+ const now = Date.now();
146
+ if (idle && queued && now >= runtime.resumeNudgeUntil) {
147
+ runtime.resumeNudgeUntil = now + 30_000;
148
+ pi.sendUserMessage("[mega-compact] continue from the compacted context above.");
149
+ }
150
+ } catch {
151
+ /* non-fatal: a failed nudge never blocks */
152
+ }
153
+ }
108
154
  runtime.snapshot(ctx);
109
155
  });
110
156
 
@@ -117,17 +163,44 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
117
163
  pi.on("turn_end", async (event, ctx) => {
118
164
  runtime.dashboard.event("turn_end", { turnIndex: event.turnIndex });
119
165
  runtime.snapshot(ctx);
166
+
167
+ // S20: auto-review the conversation every N turns and persist durable
168
+ // memories. Best-effort + non-fatal: a review failure must never break the
169
+ // agent loop. Debounced by memoryReviewInterval turns.
170
+ if (config.memoryAutoReview && runtime.currentTurn > 0 && runtime.currentTurn % config.memoryReviewInterval === 0) {
171
+ try {
172
+ const { reviewConversation } = await import("../src/memory.js");
173
+ const { applyMemoryOps } = await import("../src/memoryOps.js");
174
+ const entries = ctx.sessionManager.getEntries();
175
+ const view = runtime.engineView(entries.flatMap((e: any) => (e.message ? [e.message] : [])));
176
+ const ops = reviewConversation(view, []);
177
+ if (ops.length) {
178
+ await applyMemoryOps(ops, runtime.currentStateDir);
179
+ // S21.2: a memory op landed in this turn window. The pipeline reads
180
+ // this counter after a successful compaction and fires
181
+ // `consolidateMemories` only when it's > 0.
182
+ runtime.memoriesTouchedThisCompaction += ops.length;
183
+ }
184
+ } catch {
185
+ /* non-fatal — auto-review must not break the turn loop */
186
+ }
187
+ }
120
188
  });
121
189
 
122
- // ---- Auto-trigger: own the decision, pi owns the durable write ----------
123
- // OUR auto-trigger (over threshold + debounce): persist our Trident checkpoint,
124
- // then start pi's compaction flow via ctx.compact(). That fires
125
- // `session_before_compact`, where OUR handler returns our summary +
126
- // firstKeptEntryId, and pi durably writes the trim to disk (appendCompaction).
127
- // Result: auto-compact AND a durable trim — resume reloads the trimmed window,
128
- // no full-reload + additive recall inflation (Fix B kills the token-growth bug).
129
- // We do NOT drop messages here (that would be ephemeral; the read-only session
130
- // manager can't trim disk, so the trim has to come through pi).
190
+ // ---- Auto-trigger: live trim (compact and continue) + native durable ----
191
+ // S16 redesign: we NO LONGER call ctx.compact() from the auto-trigger by
192
+ // default. That mapped to pi's MANUAL compaction path, which abort()s the
193
+ // in-flight turn (agent-session.js:1345) and stops the agent. Instead:
194
+ // - LIVE: return { messages: trimmedView } from the context event. This
195
+ // feeds pi's transformContext (sdk.js:226 agent-loop.js:180) so the
196
+ // model sees a compacted window EVERY LLM call, with no abort. The turn
197
+ // continues. We persist our recall checkpoint (the durable value) first.
198
+ // - DURABLE: pi's NATIVE auto-compaction fires at agent-end
199
+ // (agent-session.js:1565), continues (return hasQueuedMessages()), and
200
+ // emits session_before_compact — where OUR driveNativeCompaction supplies
201
+ // the summary and pi truncates the transcript on disk. No ctx.compact().
202
+ // Legacy: MEGACOMPACT_LEGACY_DURABLE_TRIM=true restores the v0.4.28 ctx.compact
203
+ // path (kept one release as rollback).
131
204
  pi.on("context", async (event: ContextEvent, ctx: ExtensionContext) => {
132
205
  if (!config.auto) return;
133
206
  const usage = ctx.getContextUsage();
@@ -162,17 +235,55 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
162
235
  const ran = runCompact(pi, runtime, config, ctx, messages, { compressionPressure: pressure });
163
236
  if (ran.skipped) return;
164
237
 
165
- // Skip pi's durable-trim flow when pi would no-op ("Nothing to compact /
166
- // Already compacted"). pi throws that error *before* session_before_compact
167
- // fires and renders it via a compaction_end event we can't mute (onError is
168
- // too late), so the only fix is to not call ctx.compact() on a no-op. Our
169
- // recall checkpoint above is already persisted; the durable trim is
170
- // unnecessary for a transcript pi would keep in full anyway.
171
- if (piCompactWouldNoop(ctx)) return;
172
-
173
- // Start pi's compaction flow so our session_before_compact handler can
174
- // supply the durable trim (pi writes it to disk). We never use pi's summary.
175
- ctx.compact({ customInstructions: undefined });
238
+ // LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
239
+ // manual compact path aborts the in-flight turn — only used behind the flag.
240
+ // Read live from env (in addition to the load-time config) so the flag can be
241
+ // toggled per-test without reloading the module; config.legacyDurableTrim is
242
+ // the cached default. (Mirrors how piCompactWouldNoop re-reads its floor.)
243
+ const legacy = config.legacyDurableTrim || process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "true" || process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "1";
244
+ if (legacy) {
245
+ if (piCompactWouldNoop(ctx)) return;
246
+ ctx.compact({ customInstructions: undefined });
247
+ return;
248
+ }
249
+
250
+ // S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
251
+ // Non-destructive: pi keeps the real transcript; only this LLM call sees the
252
+ // trimmed window. We compute the cut on the engine view (pure, tested) then
253
+ // slice the ORIGINAL pi AgentMessage[] from that index (lossless alignment,
254
+ // mirroring dropCompactedRange) and prepend a user-role summary message.
255
+ // A build failure or unsafe cut returns nothing (no trim this call — the
256
+ // next context event retries). The anchor floor is read live from env (the
257
+ // config value is the cached default) so it can be tuned per-test / per-run
258
+ // without reloading the module.
259
+ try {
260
+ const anchorEnv = process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES;
261
+ const anchorUserMessages = (anchorEnv != null && anchorEnv !== "" && Number.isFinite(Number(anchorEnv)))
262
+ ? Number(anchorEnv)
263
+ : config.anchorUserMessages;
264
+ const cut = computeLiveTrimCut(view, {
265
+ compactedFrom: ran.result.compactedFrom,
266
+ summary: ran.result.summary,
267
+ anchorUserMessages,
268
+ });
269
+ if (cut === null) return; // unsafe / below anchor floor — no trim this call
270
+ const summaryMsg = liveTrimSummaryMessage({
271
+ compactedFrom: ran.result.compactedFrom,
272
+ summary: ran.result.summary,
273
+ anchorUserMessages: config.anchorUserMessages,
274
+ });
275
+ // Synthesize a user-role AgentMessage carrying the compacted summary.
276
+ const summaryAgentMsg = {
277
+ role: "user" as const,
278
+ content: summaryMsg.text,
279
+ timestamp: Date.now(),
280
+ } as unknown as AgentMessage;
281
+ const recent = messages.slice(cut); // guardrails-allow PREVENT-PI-002: `cut` is the pre-sanitized `compactedFrom` produced by src/boundary.ts computeDropRange, so the preserved run begins on a toolPair-safe index.
282
+ runtime.snapshot(ctx);
283
+ return { messages: [summaryAgentMsg, ...recent] };
284
+ } catch {
285
+ return; // non-fatal: no trim this call; the next context event retries
286
+ }
176
287
  });
177
288
 
178
289
  // ---- Supply a DURABLE trim to pi's native compaction (Fix B) ----------