pi-mega-compact 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/mega-conflict-cmds.js +17 -0
- package/dist/extensions/mega-events.js +102 -6
- package/dist/extensions/mega-runtime.js +20 -0
- package/dist/extensions/mega-teamrun.test.js +143 -0
- package/dist/extensions/mega-trim.js +33 -2
- package/dist/src/memoryOps.js +42 -2
- package/dist/src/memoryRecall.js +48 -0
- package/dist/src/memoryRecall.test.js +52 -0
- package/dist/src/recall.js +34 -8
- package/dist/src/store/memoryIndex.js +205 -0
- package/dist/src/store/memoryIndex.test.js +51 -0
- package/extensions/mega-conflict-cmds.ts +15 -0
- package/extensions/mega-events.ts +96 -6
- package/extensions/mega-runtime.ts +21 -0
- package/extensions/mega-teamrun.test.ts +164 -0
- package/extensions/mega-trim.ts +28 -1
- package/package.json +1 -1
- package/src/memoryOps.ts +42 -2
- package/src/memoryRecall.test.ts +58 -0
- package/src/memoryRecall.ts +52 -0
- package/src/recall.ts +35 -10
- package/src/store/memoryIndex.test.ts +61 -0
- package/src/store/memoryIndex.ts +235 -0
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
import { detectConflicts } from "./conflict-scan.js";
|
|
10
10
|
import { addMemory, listMemories, searchMemories, recallMemory } from "../src/store/sqlite.js";
|
|
11
11
|
import { resolveRepoRoot } from "./mega-config.js";
|
|
12
|
+
import { defaultEmbedder } from "../src/embedder.js";
|
|
13
|
+
import { upsertMemoryEmbedding } from "../src/store/memoryIndex.js";
|
|
12
14
|
/** Run the conflict scan and format a human-readable report. */
|
|
13
15
|
export function validateExtensions() {
|
|
14
16
|
const report = detectConflicts();
|
|
@@ -74,6 +76,14 @@ export function registerConflictCommands(pi, runtime) {
|
|
|
74
76
|
const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
|
|
75
77
|
const content = text.replace(/#[\w-]+/g, "").trim();
|
|
76
78
|
const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
|
|
79
|
+
// S24: mirror into the cross-repo memory index (fire-and-forget).
|
|
80
|
+
try {
|
|
81
|
+
const vec = defaultEmbedder().embed(content);
|
|
82
|
+
void upsertMemoryEmbedding(repo, id, content, vec);
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
/* non-fatal */
|
|
86
|
+
}
|
|
77
87
|
ctx.ui.notify(`[mega-memory] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
|
|
78
88
|
return;
|
|
79
89
|
}
|
|
@@ -142,6 +152,13 @@ export function registerConflictCommands(pi, runtime) {
|
|
|
142
152
|
const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
|
|
143
153
|
const content = text.replace(/#[\w-]+/g, "").trim();
|
|
144
154
|
const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
|
|
155
|
+
try {
|
|
156
|
+
const vec = defaultEmbedder().embed(content);
|
|
157
|
+
void upsertMemoryEmbedding(repo, id, content, vec);
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
/* non-fatal */
|
|
161
|
+
}
|
|
145
162
|
ctx.ui.notify(`[/m] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
|
|
146
163
|
return;
|
|
147
164
|
}
|
|
@@ -15,8 +15,17 @@ import { recallMemoriesAndInline } from "../src/recall.js";
|
|
|
15
15
|
import { driveNativeCompaction } from "./mega-compact-driver.js";
|
|
16
16
|
import { computeLiveTrimCut, liveTrimSummaryMessage } from "./mega-trim.js";
|
|
17
17
|
import { pressureFromPct, memoryReviewCadence } from "./mega-config.js";
|
|
18
|
+
/**
|
|
19
|
+
* DIAG accessor for the headless test harness: the most recently constructed
|
|
20
|
+
* MegaRuntime, so a test that loads the compiled extension via its default
|
|
21
|
+
* export can read diag counters (diagLiveTrimFires / diagBeforeCompactFires /
|
|
22
|
+
* diagBeforeCompactSupplied / diagAgentEndIdle) after firing synthetic events.
|
|
23
|
+
* No-op in production — nothing reads this outside tests.
|
|
24
|
+
*/
|
|
25
|
+
export let lastRuntime;
|
|
18
26
|
/** Register all pi lifecycle event handlers. */
|
|
19
27
|
export function registerEventHandlers(pi, runtime, config) {
|
|
28
|
+
lastRuntime = runtime;
|
|
20
29
|
// ---- Session lifecycle (state reset points) -------------------------------
|
|
21
30
|
// Capture model/provider whenever it changes (drives real cost estimation).
|
|
22
31
|
pi.on("model_select", async (_event, ctx) => {
|
|
@@ -56,6 +65,8 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
56
65
|
try {
|
|
57
66
|
const mr = await recallMemoriesAndInline({
|
|
58
67
|
query, stateDir: runtime.getStateDir(), limit: 5,
|
|
68
|
+
crossRepo: config.crossRepoEnabled,
|
|
69
|
+
crossRepoCosine: config.crossRepoCosine,
|
|
59
70
|
});
|
|
60
71
|
if (!mr.empty)
|
|
61
72
|
runtime.pendingMemoryRecallBlock = mr.block;
|
|
@@ -82,7 +93,7 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
82
93
|
}
|
|
83
94
|
// S21: parallel memory recall. Trigram embedder is sub-ms; await is fine.
|
|
84
95
|
try {
|
|
85
|
-
const mr = await recallMemoriesAndInline({ query, stateDir: runtime.getStateDir(), limit: 5 });
|
|
96
|
+
const mr = await recallMemoriesAndInline({ query, stateDir: runtime.getStateDir(), limit: 5, crossRepo: config.crossRepoEnabled, crossRepoCosine: config.crossRepoCosine });
|
|
86
97
|
if (!mr.empty)
|
|
87
98
|
runtime.pendingMemoryRecallBlock = mr.block;
|
|
88
99
|
}
|
|
@@ -139,6 +150,46 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
139
150
|
const idle = ctx.isIdle?.() ?? true;
|
|
140
151
|
const queued = ctx.hasPendingMessages?.() ?? false;
|
|
141
152
|
const now = Date.now();
|
|
153
|
+
// DIAG (team-run relief): surface whether the agent is idle + over
|
|
154
|
+
// threshold at agent_end so we can see if a mid-run durable-trim trigger
|
|
155
|
+
// *should* have fired but didn't.
|
|
156
|
+
const overThreshold = (runtime.lastCtxTokens ?? 0) >= config.thresholdTokens;
|
|
157
|
+
runtime.diagAgentEndIdle++;
|
|
158
|
+
runtime.logger.info("agent-end-idle", {
|
|
159
|
+
sessionId: runtime.rt.sessionId,
|
|
160
|
+
idle,
|
|
161
|
+
queued,
|
|
162
|
+
overThreshold,
|
|
163
|
+
ctxPct: runtime.lastCtxPercent,
|
|
164
|
+
ctxTokens: runtime.lastCtxTokens,
|
|
165
|
+
thresholdTokens: config.thresholdTokens,
|
|
166
|
+
wouldNudge: idle && queued && now >= runtime.resumeNudgeUntil,
|
|
167
|
+
});
|
|
168
|
+
// S16+S24: MID-RUN DURABLE TRIM. During a long team run (sub-agents),
|
|
169
|
+
// pi's native durable compaction only fires from _checkCompaction at
|
|
170
|
+
// PARENT settle (agent-session.js:760/844), so the on-disk transcript +
|
|
171
|
+
// context meter balloon to ~150k and never relieve until the very end
|
|
172
|
+
// ("compacts but doesn't resume"). agent_end with activeAgents===0 is a
|
|
173
|
+
// SAFE, settled point: calling ctx.compact() here does NOT abort an
|
|
174
|
+
// in-flight turn (the S16 danger is only mid-turn). ctx.compact() runs
|
|
175
|
+
// pi's flow, which fires our session_before_compact handler to supply
|
|
176
|
+
// the durable trim (truncates the transcript from firstKeptEntryId).
|
|
177
|
+
// Guarded three ways: only when truly idle + over threshold, only when
|
|
178
|
+
// pi would actually compact (piCompactWouldNoop skips the user-facing
|
|
179
|
+
// no-op throw), and debounced (one durable trim per 2s) to avoid
|
|
180
|
+
// thrashing the transcript while sub-agents keep settling.
|
|
181
|
+
if (idle && overThreshold && now >= runtime.debounceUntil) {
|
|
182
|
+
if (!piCompactWouldNoop(ctx)) {
|
|
183
|
+
runtime.debounceUntil = now + 2000;
|
|
184
|
+
runtime.diagAgentEndDurable++;
|
|
185
|
+
runtime.logger.info("agent-end-durable-trigger", {
|
|
186
|
+
sessionId: runtime.rt.sessionId,
|
|
187
|
+
ctxTokens: runtime.lastCtxTokens,
|
|
188
|
+
thresholdTokens: config.thresholdTokens,
|
|
189
|
+
});
|
|
190
|
+
ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort
|
|
191
|
+
}
|
|
192
|
+
}
|
|
142
193
|
if (idle && queued && now >= runtime.resumeNudgeUntil) {
|
|
143
194
|
runtime.resumeNudgeUntil = now + 30_000;
|
|
144
195
|
pi.sendUserMessage("[mega-compact] continue from the compacted context above.");
|
|
@@ -207,22 +258,30 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
207
258
|
const currentTokens = usage?.tokens ?? estimateSessionTokens(view) ??
|
|
208
259
|
Math.round((pct / 100) * (usage?.contextWindow ?? 0));
|
|
209
260
|
// FAST GATE: token-based (tier threshold), not percentage-based.
|
|
210
|
-
if (currentTokens < config.thresholdTokens)
|
|
261
|
+
if (currentTokens < config.thresholdTokens) {
|
|
262
|
+
runtime.diagCtxFastGate++;
|
|
211
263
|
return;
|
|
264
|
+
}
|
|
212
265
|
const check = autoCompactCheck(currentTokens, config.thresholdTokens); // SERVER-STYLE CONFIRM (local)
|
|
213
|
-
if (!check.shouldCompact)
|
|
266
|
+
if (!check.shouldCompact) {
|
|
267
|
+
runtime.diagCtxNoCompact++;
|
|
214
268
|
return;
|
|
269
|
+
}
|
|
215
270
|
// Debounce so we don't fire on every context event past threshold.
|
|
216
271
|
const now = Date.now();
|
|
217
|
-
if (now < runtime.debounceUntil)
|
|
272
|
+
if (now < runtime.debounceUntil) {
|
|
273
|
+
runtime.diagCtxDebounce++;
|
|
218
274
|
return;
|
|
275
|
+
}
|
|
219
276
|
runtime.debounceUntil = now + 2000;
|
|
220
277
|
// Adaptive compression (Fix E): scale compression strength + keepFrom depth
|
|
221
278
|
// with how close we are to the model context limit.
|
|
222
279
|
const pressure = pressureFromPct(pct);
|
|
223
280
|
const ran = runCompact(pi, runtime, config, ctx, messages, { compressionPressure: pressure });
|
|
224
|
-
if (ran.skipped)
|
|
281
|
+
if (ran.skipped) {
|
|
282
|
+
runtime.diagCtxRunSkipped++;
|
|
225
283
|
return;
|
|
284
|
+
}
|
|
226
285
|
// LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
|
|
227
286
|
// manual compact path aborts the in-flight turn — only used behind the flag.
|
|
228
287
|
// Read live from env (in addition to the load-time config) so the flag can be
|
|
@@ -254,8 +313,16 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
254
313
|
summary: ran.result.summary,
|
|
255
314
|
anchorUserMessages,
|
|
256
315
|
});
|
|
257
|
-
if (cut === null)
|
|
316
|
+
if (cut === null) {
|
|
317
|
+
runtime.diagCtxCutNull++;
|
|
318
|
+
runtime.logger.info("live-trim-skip", {
|
|
319
|
+
sessionId: runtime.rt.sessionId,
|
|
320
|
+
compactedFrom: ran.result.compactedFrom,
|
|
321
|
+
viewLen: view.length,
|
|
322
|
+
anchorUserMessages,
|
|
323
|
+
});
|
|
258
324
|
return; // unsafe / below anchor floor — no trim this call
|
|
325
|
+
}
|
|
259
326
|
const summaryMsg = liveTrimSummaryMessage({
|
|
260
327
|
compactedFrom: ran.result.compactedFrom,
|
|
261
328
|
summary: ran.result.summary,
|
|
@@ -269,9 +336,23 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
269
336
|
};
|
|
270
337
|
const recent = messages.slice(cut); // guardrails-allow PREVENT-PI-002: `cut` is the pre-sanitized `compactedFrom` produced by src/boundary.ts computeDropRange, so the preserved run begins on a toolPair-safe index.
|
|
271
338
|
runtime.snapshot(ctx);
|
|
339
|
+
// DIAG (team-run relief): confirm the live trim actually fires + how big
|
|
340
|
+
// the window still is. The return is non-durable (per-LLM-call only), so
|
|
341
|
+
// this is the signal that the model is being fed a compacted view while
|
|
342
|
+
// the on-disk transcript + context meter keep growing.
|
|
343
|
+
runtime.diagLiveTrimFires++;
|
|
344
|
+
runtime.logger.info("live-trim", {
|
|
345
|
+
sessionId: runtime.rt.sessionId,
|
|
346
|
+
inputMsgs: messages.length,
|
|
347
|
+
outputMsgs: recent.length + 1,
|
|
348
|
+
compactedFrom: cut,
|
|
349
|
+
ctxPct: pct,
|
|
350
|
+
ctxTokens: usage?.tokens ?? null,
|
|
351
|
+
});
|
|
272
352
|
return { messages: [summaryAgentMsg, ...recent] };
|
|
273
353
|
}
|
|
274
354
|
catch {
|
|
355
|
+
runtime.diagCtxThrown++;
|
|
275
356
|
return; // non-fatal: no trim this call; the next context event retries
|
|
276
357
|
}
|
|
277
358
|
});
|
|
@@ -283,11 +364,26 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
283
364
|
// there is no full-reload + additive recall inflation.
|
|
284
365
|
pi.on("session_before_compact", async (event, ctx) => {
|
|
285
366
|
runtime.resetRuntime(ctx.sessionManager.getSessionId());
|
|
367
|
+
// DIAG (team-run relief): this is the ONLY durable-trim entry point. Log
|
|
368
|
+
// every fire + whether we supplied a compaction (truncates transcript) or
|
|
369
|
+
// fell through to {} (pi runs its own). If this is sparse during a team
|
|
370
|
+
// run, the durable trim is firing too late (only at parent settle).
|
|
371
|
+
const prep = event.preparation;
|
|
372
|
+
runtime.diagBeforeCompactFires++;
|
|
373
|
+
runtime.logger.info("before-compact-entry", {
|
|
374
|
+
sessionId: runtime.rt.sessionId,
|
|
375
|
+
reason: event.reason,
|
|
376
|
+
hasPrep: !!prep,
|
|
377
|
+
msgsToSummarize: prep?.messagesToSummarize?.length ?? 0,
|
|
378
|
+
firstKeptEntryId: prep?.firstKeptEntryId ?? null,
|
|
379
|
+
activeAgents: runtime.activeAgents,
|
|
380
|
+
});
|
|
286
381
|
if (!config.auto)
|
|
287
382
|
return {}; // let pi run its own native compaction
|
|
288
383
|
try {
|
|
289
384
|
const result = driveNativeCompaction(event, runtime, config);
|
|
290
385
|
if (result) {
|
|
386
|
+
runtime.diagBeforeCompactSupplied++;
|
|
291
387
|
runtime.logger.info("native-compact", {
|
|
292
388
|
sessionId: runtime.rt.sessionId,
|
|
293
389
|
firstKeptEntryId: result.compaction.firstKeptEntryId,
|
|
@@ -120,6 +120,26 @@ export class MegaRuntime {
|
|
|
120
120
|
lastCtxTokens = null;
|
|
121
121
|
lastCtxPercent = null;
|
|
122
122
|
lastCtxWindow = 0;
|
|
123
|
+
/**
|
|
124
|
+
* DIAG counters for the "team run doesn't relieve context" investigation.
|
|
125
|
+
* Plain integers, incremented at the three compaction decision points. They
|
|
126
|
+
* let a headless test drive the real event handlers and assert the firing
|
|
127
|
+
* cadence without scraping log files. Inert in production (the live-trim and
|
|
128
|
+
* before-compact probes also emit logger.info, but these counters are always
|
|
129
|
+
* updated and cost nothing).
|
|
130
|
+
*/
|
|
131
|
+
diagLiveTrimFires = 0; // context handler returned a trimmed view
|
|
132
|
+
diagBeforeCompactFires = 0; // session_before_compact handler entered
|
|
133
|
+
diagBeforeCompactSupplied = 0; // session_before_compact supplied our trim
|
|
134
|
+
diagAgentEndIdle = 0; // agent_end with activeAgents===0
|
|
135
|
+
diagAgentEndDurable = 0; // agent_end fired ctx.compact() (mid-run durable trim)
|
|
136
|
+
// Per-skip-path counters for the team-run diagnosis.
|
|
137
|
+
diagCtxFastGate = 0; // returned at token fast-gate (below threshold)
|
|
138
|
+
diagCtxNoCompact = 0; // autoCompactCheck().shouldCompact === false
|
|
139
|
+
diagCtxDebounce = 0; // debounceUntil not yet elapsed
|
|
140
|
+
diagCtxRunSkipped = 0; // runCompact() returned skipped
|
|
141
|
+
diagCtxCutNull = 0; // computeLiveTrimCut returned null (anchor/boundary)
|
|
142
|
+
diagCtxThrown = 0; // live-trim try threw (caught)
|
|
123
143
|
/**
|
|
124
144
|
* Live 0–1 pressure: how full the context window is relative to the compaction
|
|
125
145
|
* threshold. Computed from the most recent context event the runtime already
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* mega-teamrun.test.ts — regression test for the "auto-compact runs but context
|
|
3
|
+
* never relieves during a team run (sub-agents)" bug.
|
|
4
|
+
*
|
|
5
|
+
* Loads the REAL compiled extension (extensions/mega-compact.js) through a
|
|
6
|
+
* faithful mock pi (mirrors mega-compact.test.ts's harness) and drives the
|
|
7
|
+
* exact event sequence a long team run produces:
|
|
8
|
+
*
|
|
9
|
+
* agent_start -> context (over threshold) xN -> agent_end (repeat x3)
|
|
10
|
+
*
|
|
11
|
+
* Asserts the TWO fixes:
|
|
12
|
+
* 1. live trim FIRES per-call (computeLiveTrimCut no longer returns null on
|
|
13
|
+
* the anchor floor — was `cutNull`, liveTrimFires===0 before the fix).
|
|
14
|
+
* 2. the DURABLE trim fires at agent_end while idle + over threshold
|
|
15
|
+
* (mid-run durable trigger), not only at parent settle.
|
|
16
|
+
*
|
|
17
|
+
* The mock ctx.compact() drives session_before_compact so we observe the
|
|
18
|
+
* durable truncation. Counters come from MegaRuntime.diag* (set behind the
|
|
19
|
+
* real handler code, inert in production).
|
|
20
|
+
*
|
|
21
|
+
* MEGACOMPACT_PGLITE_DISABLED keeps the run fast (no WASM index init).
|
|
22
|
+
*/
|
|
23
|
+
import { test } from "node:test";
|
|
24
|
+
import assert from "node:assert/strict";
|
|
25
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
26
|
+
import { tmpdir } from "node:os";
|
|
27
|
+
import { join } from "node:path";
|
|
28
|
+
import { createRequire } from "node:module";
|
|
29
|
+
import { closeVectorIndex } from "../src/store/vectorIndex.js";
|
|
30
|
+
const require = createRequire(import.meta.url);
|
|
31
|
+
const baseTmp = mkdtempSync(join(tmpdir(), "mc-team-"));
|
|
32
|
+
process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "index");
|
|
33
|
+
process.env.MEGACOMPACT_PGLITE_DISABLED = "true"; // fast: skip WASM index
|
|
34
|
+
let counter = 0;
|
|
35
|
+
function harness() {
|
|
36
|
+
const stateDir = join(baseTmp, `run-${counter++}`);
|
|
37
|
+
process.env.MEGACOMPACT_STATE_DIR = stateDir;
|
|
38
|
+
process.env.MEGACOMPACT_DEBUG = "true";
|
|
39
|
+
process.env.MEGACOMPACT_THRESHOLD_TOKENS = "50";
|
|
40
|
+
process.env.MEGACOMPACT_FAST_GATE_PCT = "1";
|
|
41
|
+
process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES = "1";
|
|
42
|
+
process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR = "0"; // piCompactWouldNoop must not skip
|
|
43
|
+
process.env.MEGACOMPACT_MEMORY_AUTO_REVIEW = "false";
|
|
44
|
+
process.env.MEGACOMPACT_RAPTOR_ENABLED = "false";
|
|
45
|
+
delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
|
|
46
|
+
const handlers = {};
|
|
47
|
+
const compactCalls = [];
|
|
48
|
+
function msg(role, text, toolName) {
|
|
49
|
+
if (role === "assistant" && toolName) {
|
|
50
|
+
return { role: "assistant", content: [{ type: "toolCall", name: toolName, id: "c1", arguments: {} }], api: "anthropic-messages", provider: "anthropic", model: "m", usage: { inputTokens: 1, outputTokens: 1, cacheReadTokens: 0, cacheWriteTokens: 0 }, stopReason: "tool_use", timestamp: 0 };
|
|
51
|
+
}
|
|
52
|
+
if (role === "toolResult" && toolName) {
|
|
53
|
+
return { role: "toolResult", content: [{ type: "text", text }], toolCallId: "c1", toolName, isError: false, timestamp: 0 };
|
|
54
|
+
}
|
|
55
|
+
return { role: "user", content: text, timestamp: 0 };
|
|
56
|
+
}
|
|
57
|
+
const session = [];
|
|
58
|
+
for (let i = 0; i < 14; i++) {
|
|
59
|
+
session.push(msg("user", `actually we decided to use approach ${i} for module ${i}`));
|
|
60
|
+
session.push(msg("assistant", `edited module ${i}`, "Edit"));
|
|
61
|
+
session.push(msg("toolResult", `edited module ${i}`, "Edit"));
|
|
62
|
+
}
|
|
63
|
+
const toEntry = (m, i) => ({ type: "message", id: `e${i}`, parentId: null, timestamp: String(i), message: m });
|
|
64
|
+
const sessionManager = {
|
|
65
|
+
getSessionId: () => "sess_team_001",
|
|
66
|
+
getEntries: () => session.map(toEntry),
|
|
67
|
+
getBranch: () => session.map(toEntry),
|
|
68
|
+
};
|
|
69
|
+
function makeCtx(over = {}) {
|
|
70
|
+
return {
|
|
71
|
+
ui: { setStatus: () => { }, notify: () => { }, select: () => { }, confirm: async () => true, input: async () => "", setWidget: () => { } },
|
|
72
|
+
mode: "tui", hasUI: true, cwd: stateDir, sessionManager,
|
|
73
|
+
modelRegistry: {}, model: undefined, isIdle: () => true, isProjectTrusted: () => true,
|
|
74
|
+
signal: undefined, abort: () => { }, hasPendingMessages: () => false, shutdown: () => { },
|
|
75
|
+
getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }),
|
|
76
|
+
// Mock ctx.compact() runs pi's flow and fires session_before_compact.
|
|
77
|
+
compact: (opts) => {
|
|
78
|
+
compactCalls.push(opts);
|
|
79
|
+
if (handlers["session_before_compact"]) {
|
|
80
|
+
return handlers["session_before_compact"]({ type: "session_before_compact", reason: "threshold", willRetry: false, signal: undefined, preparation: { firstKeptEntryId: "e2", messagesToSummarize: session.slice(0, 2), tokensBefore: 500 } }, makeCtx());
|
|
81
|
+
}
|
|
82
|
+
return undefined;
|
|
83
|
+
},
|
|
84
|
+
getSystemPrompt: () => "system base",
|
|
85
|
+
...over,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
const pi = {
|
|
89
|
+
on: (ev, h) => { handlers[ev] = h; },
|
|
90
|
+
registerCommand: () => { }, registerTool: () => { }, registerShortcut: () => { },
|
|
91
|
+
registerFlag: () => { }, getFlag: () => undefined, registerMessageRenderer: () => { },
|
|
92
|
+
registerEntryRenderer: () => { }, sendMessage: () => { }, sendUserMessage: () => { },
|
|
93
|
+
appendEntry: () => { }, setSessionName: () => { }, getSessionName: () => undefined,
|
|
94
|
+
setLabel: () => { }, exec: async () => ({ stdout: "", stderr: "", code: 0 }),
|
|
95
|
+
getActiveTools: () => [], getAllTools: () => [], setActiveTools: () => { },
|
|
96
|
+
getCommands: () => [], setModel: async () => false, getThinkingLevel: () => "off",
|
|
97
|
+
setThinkingLevel: () => { },
|
|
98
|
+
};
|
|
99
|
+
const mod = require("./mega-compact.js");
|
|
100
|
+
mod.default(pi);
|
|
101
|
+
const { lastRuntime } = require("./mega-events.js");
|
|
102
|
+
const fire = (ev, event, ctx) => handlers[ev](event, ctx);
|
|
103
|
+
return {
|
|
104
|
+
stateDir, handlers, compactCalls, fire, ctx: makeCtx, session,
|
|
105
|
+
runtime: lastRuntime, // MegaRuntime with diag* counters
|
|
106
|
+
// Advance the debounce so agent_end (same instant) can trigger durable trim.
|
|
107
|
+
clearDebounce: () => { if (lastRuntime)
|
|
108
|
+
lastRuntime.debounceUntil = 0; },
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
test("team run: live trim fires AND durable trim fires per sub-agent (relieves context)", async () => {
|
|
112
|
+
const h = harness();
|
|
113
|
+
const ctx = h.ctx();
|
|
114
|
+
for (let a = 0; a < 3; a++) {
|
|
115
|
+
await h.fire("agent_start", { type: "agent_start", messages: [] }, ctx);
|
|
116
|
+
for (let i = 0; i < 4; i++) {
|
|
117
|
+
await h.fire("context", { type: "context", messages: h.session }, ctx);
|
|
118
|
+
}
|
|
119
|
+
// Real team runs settle seconds after the last context event; mimic that
|
|
120
|
+
// so the 2s debounce has elapsed and the durable trigger can fire.
|
|
121
|
+
await new Promise((r) => setTimeout(r, 2100));
|
|
122
|
+
h.clearDebounce();
|
|
123
|
+
await h.fire("agent_end", { type: "agent_end", messages: [] }, ctx);
|
|
124
|
+
}
|
|
125
|
+
const rt = h.runtime;
|
|
126
|
+
// FIX 1: live trim must actually fire (was 0 — computeLiveTrimCut returned null).
|
|
127
|
+
assert.ok(rt.diagLiveTrimFires > 0, "live trim fires during the team run (anchor-floor fix)");
|
|
128
|
+
assert.equal(rt.diagCtxCutNull, 0, "no live-trim cut skipped on anchor floor");
|
|
129
|
+
// FIX 2: durable trim must fire at each agent_end (was 0 — only at parent settle).
|
|
130
|
+
assert.equal(rt.diagAgentEndDurable, 3, "mid-run durable trigger fired at each agent_end");
|
|
131
|
+
assert.equal(rt.diagBeforeCompactSupplied, 3, "our durable trim supplied 3x (context relieved)");
|
|
132
|
+
assert.ok(h.compactCalls.length >= 3, "ctx.compact() invoked for durable trim between sub-agents");
|
|
133
|
+
});
|
|
134
|
+
test("control: session_before_compact supplies a durable compaction (parent settles)", async () => {
|
|
135
|
+
const h = harness();
|
|
136
|
+
const res = await h.fire("session_before_compact", { type: "session_before_compact", reason: "threshold", willRetry: false, signal: undefined, preparation: { firstKeptEntryId: "e2", messagesToSummarize: h.session.slice(0, 4), tokensBefore: 500 } }, h.ctx());
|
|
137
|
+
assert.ok(res?.compaction, "compaction result returned to pi");
|
|
138
|
+
assert.equal(res.compaction.firstKeptEntryId, "e2", "reuses pi's boundary (PREVENT-PI-002)");
|
|
139
|
+
});
|
|
140
|
+
test("cleanup", async () => {
|
|
141
|
+
await closeVectorIndex();
|
|
142
|
+
rmSync(baseTmp, { recursive: true, force: true });
|
|
143
|
+
});
|
|
@@ -22,8 +22,39 @@ export function computeLiveTrimCut(view, opts) {
|
|
|
22
22
|
return null; // nothing safe to cut — keep everything this call
|
|
23
23
|
const recent = view.slice(cut);
|
|
24
24
|
const userCount = recent.filter((m) => m.role === "user").length;
|
|
25
|
-
|
|
26
|
-
|
|
25
|
+
// ANCHOR FLOOR (PREVENT-PI-001): the recent window must keep at least
|
|
26
|
+
// `anchorUserMessages` user messages. The original compactedFrom can land on a
|
|
27
|
+
// run that starts with fewer than that (e.g. the preserved region begins on a
|
|
28
|
+
// tool pair, or the session's tail is tool-heavy). Instead of bailing out and
|
|
29
|
+
// skipping the live trim entirely this call (which left the model fed a
|
|
30
|
+
// 150k-context window during long team runs), walk `cut` backward until the
|
|
31
|
+
// preserved run contains enough user messages — bounded by the boundary-safe
|
|
32
|
+
// constraint so we never split a tool pair. Falls back to null only when the
|
|
33
|
+
// whole view can't satisfy the floor (tiny sessions) — the next context event
|
|
34
|
+
// retries.
|
|
35
|
+
if (userCount < opts.anchorUserMessages) {
|
|
36
|
+
let c = cut;
|
|
37
|
+
while (c > 1) {
|
|
38
|
+
c--;
|
|
39
|
+
if (!isBoundarySafe(view, c))
|
|
40
|
+
continue;
|
|
41
|
+
const recentNow = view.slice(c);
|
|
42
|
+
const usersNow = recentNow.filter((m) => m.role === "user").length;
|
|
43
|
+
if (usersNow >= opts.anchorUserMessages) {
|
|
44
|
+
cut = c;
|
|
45
|
+
break;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (cut > 1) {
|
|
49
|
+
const finalRecent = view.slice(cut);
|
|
50
|
+
if (finalRecent.filter((m) => m.role === "user").length < opts.anchorUserMessages) {
|
|
51
|
+
return null; // cannot satisfy the floor without dropping too much — retry next call
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
27
58
|
return cut;
|
|
28
59
|
}
|
|
29
60
|
/** The formatted compacted-region summary as a user-role engine message. */
|
package/dist/src/memoryOps.js
CHANGED
|
@@ -1,9 +1,44 @@
|
|
|
1
1
|
import { addMemory, listMemories, replaceMemory, removeMemory, } from "./store/sqlite.js";
|
|
2
|
+
import { defaultEmbedder } from "./embedder.js";
|
|
3
|
+
import { upsertMemoryEmbedding } from "./store/memoryIndex.js";
|
|
4
|
+
import { execSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: read-only `git rev-parse` to scope the memory index per-repo
|
|
5
|
+
/** Resolve the current repo's git root (mirrors extensions/mega-config.ts but
|
|
6
|
+
* kept local so src/ stays pi-agnostic — no extension-layer import). */
|
|
7
|
+
function resolveRepoRootLocal(cwd) {
|
|
8
|
+
try {
|
|
9
|
+
const out = execSync("git rev-parse --show-toplevel", {
|
|
10
|
+
cwd,
|
|
11
|
+
encoding: "utf-8",
|
|
12
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
13
|
+
}).trim();
|
|
14
|
+
return out || undefined;
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return undefined;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
2
20
|
/** Find a memory row whose content exactly matches (case-insensitive). */
|
|
3
21
|
function findByContent(memories, content) {
|
|
4
22
|
const norm = content.trim().toLowerCase();
|
|
5
23
|
return memories.find((m) => m.content.trim().toLowerCase() === norm);
|
|
6
24
|
}
|
|
25
|
+
/**
|
|
26
|
+
* Fire-and-forget mirror of a memory write into the cross-repo PGlite index
|
|
27
|
+
* (S24 optional memory-RAG mirror). Best-effort + non-fatal: never blocks the
|
|
28
|
+
* SQLite write and degrades to the same-repo scan if the index is disabled or
|
|
29
|
+
* fails. `repoId` is the resolved git root so the memory is findable from other
|
|
30
|
+
* repos; falls back to the state dir when outside git.
|
|
31
|
+
*/
|
|
32
|
+
function indexMemoryWrite(stateDir, memoryId, content) {
|
|
33
|
+
const repoId = resolveRepoRootLocal(stateDir) ?? stateDir;
|
|
34
|
+
try {
|
|
35
|
+
const vec = defaultEmbedder().embed(content);
|
|
36
|
+
void upsertMemoryEmbedding(repoId, memoryId, content, vec);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
/* non-fatal — embedding/index failure must never break the SQLite write */
|
|
40
|
+
}
|
|
41
|
+
}
|
|
7
42
|
/**
|
|
8
43
|
* Apply add/replace/remove ops to the memories table. Replaces are matched by
|
|
9
44
|
* existing content; removes by content. Idempotent: an add that already exists
|
|
@@ -19,7 +54,7 @@ export async function applyMemoryOps(ops, stateDir) {
|
|
|
19
54
|
// Skip if an identical memory already exists.
|
|
20
55
|
if (findByContent(existing, op.memory.content))
|
|
21
56
|
continue;
|
|
22
|
-
addMemory({
|
|
57
|
+
const id = addMemory({
|
|
23
58
|
kind: op.memory.category,
|
|
24
59
|
content: op.memory.content,
|
|
25
60
|
tags: [],
|
|
@@ -27,6 +62,8 @@ export async function applyMemoryOps(ops, stateDir) {
|
|
|
27
62
|
target: op.memory.target,
|
|
28
63
|
sourceTurn: op.memory.sourceTurn,
|
|
29
64
|
}, repo, stateDir);
|
|
65
|
+
// S24: mirror into the cross-repo index (fire-and-forget; non-fatal).
|
|
66
|
+
indexMemoryWrite(stateDir, id, op.memory.content);
|
|
30
67
|
}
|
|
31
68
|
else if (op.op === "replace") {
|
|
32
69
|
const match = findByContent(existing, op.targetContent);
|
|
@@ -37,16 +74,19 @@ export async function applyMemoryOps(ops, stateDir) {
|
|
|
37
74
|
category: op.memory.category,
|
|
38
75
|
sourceTurn: op.memory.sourceTurn,
|
|
39
76
|
}, stateDir);
|
|
77
|
+
// S24: re-mirror under the same memory id (fire-and-forget; non-fatal).
|
|
78
|
+
indexMemoryWrite(stateDir, match.id, op.memory.content);
|
|
40
79
|
}
|
|
41
80
|
else {
|
|
42
81
|
// Target missing (e.g. earlier in-conversation contradiction) → add.
|
|
43
|
-
addMemory({
|
|
82
|
+
const id = addMemory({
|
|
44
83
|
kind: op.memory.category,
|
|
45
84
|
content: op.memory.content,
|
|
46
85
|
tags: [],
|
|
47
86
|
category: op.memory.category,
|
|
48
87
|
sourceTurn: op.memory.sourceTurn,
|
|
49
88
|
}, repo, stateDir);
|
|
89
|
+
indexMemoryWrite(stateDir, id, op.memory.content);
|
|
50
90
|
}
|
|
51
91
|
}
|
|
52
92
|
else {
|
package/dist/src/memoryRecall.js
CHANGED
|
@@ -58,3 +58,51 @@ export async function recallMemories(query, stateDir, opts = {}) {
|
|
|
58
58
|
}
|
|
59
59
|
return top;
|
|
60
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* Cross-repo memory recall (S24): augments the same-repo `recallMemories` with
|
|
63
|
+
* HNSW NN over the global PGlite `memory_index` (other repos' memories). Content
|
|
64
|
+
* is read inline from the index hit (the recall process can't open other repos'
|
|
65
|
+
* SQLite dirs), so no other-repo db access is required. Returns hits sorted by
|
|
66
|
+
* descending cosine, above `crossRepoCosine`. De-duped by content against
|
|
67
|
+
* `sameRepoContent` so we never surface a memory the same-repo scan already has.
|
|
68
|
+
* Non-fatal: any index failure returns []. Best-effort + PREVENT-PI-004 (local
|
|
69
|
+
* WASM only).
|
|
70
|
+
*/
|
|
71
|
+
export async function recallMemoriesCrossRepo(query, stateDir, opts = {}) {
|
|
72
|
+
const embedder = opts.embedder ?? defaultEmbedder();
|
|
73
|
+
const queryVec = embedder.embed(query);
|
|
74
|
+
const { searchMemoriesAsync } = await import("./store/memoryIndex.js");
|
|
75
|
+
const k = opts.limit ?? 5;
|
|
76
|
+
const floor = opts.crossRepoCosine ?? 0.3;
|
|
77
|
+
const hits = await searchMemoriesAsync(queryVec, { k });
|
|
78
|
+
if (!hits.length)
|
|
79
|
+
return [];
|
|
80
|
+
// Mark same-repo content as already-covered so we don't duplicate it.
|
|
81
|
+
const sameRepo = new Set(listMemories(opts.repo ?? null, 1000, stateDir).map((m) => m.content.trim().toLowerCase()));
|
|
82
|
+
const out = [];
|
|
83
|
+
for (const h of hits) {
|
|
84
|
+
if (h.score < floor)
|
|
85
|
+
continue;
|
|
86
|
+
if (sameRepo.has(h.content.trim().toLowerCase()))
|
|
87
|
+
continue;
|
|
88
|
+
out.push({
|
|
89
|
+
memory: {
|
|
90
|
+
id: h.memoryId,
|
|
91
|
+
repo: h.repoId,
|
|
92
|
+
kind: "note",
|
|
93
|
+
content: h.content,
|
|
94
|
+
tags: [],
|
|
95
|
+
createdAt: 0,
|
|
96
|
+
lastRecalledAt: null,
|
|
97
|
+
category: null,
|
|
98
|
+
target: null,
|
|
99
|
+
lastReferenced: null,
|
|
100
|
+
sourceTurn: null,
|
|
101
|
+
},
|
|
102
|
+
score: h.score,
|
|
103
|
+
repoId: h.repoId,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
out.sort((a, b) => b.score - a.score);
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
@@ -90,3 +90,55 @@ test("recallMemories: fresher reference beats older at equal similarity", async
|
|
|
90
90
|
test("cleanup memrec", () => {
|
|
91
91
|
rmSync(baseTmp, { recursive: true, force: true });
|
|
92
92
|
});
|
|
93
|
+
// ---- S24: cross-repo memory recall (PGlite mirror) ---------------------------
|
|
94
|
+
test("recallMemoriesAndInline: surfaces a memory saved in ANOTHER repo via cross-repo index", async () => {
|
|
95
|
+
// Isolate the global PGlite index to a temp dir shared by both "repos".
|
|
96
|
+
process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "xrepo-index");
|
|
97
|
+
const repoA = join(baseTmp, "repo-a");
|
|
98
|
+
const repoB = join(baseTmp, "repo-b");
|
|
99
|
+
try {
|
|
100
|
+
// repoA owns a decision about the store backend.
|
|
101
|
+
const { applyMemoryOps } = await import("./memoryOps.js");
|
|
102
|
+
await applyMemoryOps([{ op: "add", memory: { content: "we standardized on node:sqlite for the store backend", category: "decision", sourceTurn: 0 } }], repoA);
|
|
103
|
+
// repoB is a fresh session with NO local memory about the store backend.
|
|
104
|
+
const { recallMemoriesAndInline } = await import("./recall.js");
|
|
105
|
+
const res = await recallMemoriesAndInline({
|
|
106
|
+
query: "what store backend do we use?",
|
|
107
|
+
stateDir: repoB,
|
|
108
|
+
limit: 5,
|
|
109
|
+
crossRepo: true,
|
|
110
|
+
crossRepoCosine: 0.3,
|
|
111
|
+
});
|
|
112
|
+
assert.ok(!res.empty, "cross-repo recall found the other repo's memory");
|
|
113
|
+
assert.ok(/node:sqlite/.test(res.block), "the node:sqlite decision was recalled from repo A");
|
|
114
|
+
assert.ok(res.report.some((r) => /from /.test(r)), "report labels the memory as cross-repo");
|
|
115
|
+
}
|
|
116
|
+
finally {
|
|
117
|
+
const { closeMemoryIndex } = await import("./store/memoryIndex.js");
|
|
118
|
+
await closeMemoryIndex();
|
|
119
|
+
delete process.env.MEGACOMPACT_INDEX_DIR;
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
test("recallMemoriesAndInline: cross-repo disabled when MEGACOMPACT_PGLITE_DISABLED", async () => {
|
|
123
|
+
process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "xrepo-index-off");
|
|
124
|
+
process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
|
|
125
|
+
const repoA = join(baseTmp, "repo-a2");
|
|
126
|
+
const repoB = join(baseTmp, "repo-b2");
|
|
127
|
+
try {
|
|
128
|
+
const { applyMemoryOps } = await import("./memoryOps.js");
|
|
129
|
+
await applyMemoryOps([{ op: "add", memory: { content: "we standardized on node:sqlite for the store backend", category: "decision", sourceTurn: 0 } }], repoA);
|
|
130
|
+
const { recallMemoriesAndInline } = await import("./recall.js");
|
|
131
|
+
const res = await recallMemoriesAndInline({
|
|
132
|
+
query: "what store backend do we use?",
|
|
133
|
+
stateDir: repoB,
|
|
134
|
+
limit: 5,
|
|
135
|
+
crossRepo: true,
|
|
136
|
+
});
|
|
137
|
+
// Index disabled → no cross-repo hit; repoB has no local memory → empty.
|
|
138
|
+
assert.equal(res.empty, true, "cross-repo recall degrades to empty when disabled");
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
delete process.env.MEGACOMPACT_PGLITE_DISABLED;
|
|
142
|
+
delete process.env.MEGACOMPACT_INDEX_DIR;
|
|
143
|
+
}
|
|
144
|
+
});
|