pi-mega-compact 0.6.1 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -23
- package/dist/extensions/conflict-scan.test.js +115 -0
- package/dist/extensions/mega-conflict-cmds.js +17 -0
- package/dist/extensions/mega-events.js +102 -6
- package/dist/extensions/mega-runtime.js +20 -0
- package/dist/extensions/mega-teamrun.test.js +143 -0
- package/dist/extensions/mega-trim.js +33 -2
- package/dist/src/memoryOps.js +42 -2
- package/dist/src/memoryRecall.js +48 -0
- package/dist/src/memoryRecall.test.js +52 -0
- package/dist/src/recall.js +34 -8
- package/dist/src/store/memoryIndex.js +230 -0
- package/dist/src/store/memoryIndex.test.js +51 -0
- package/dist/src/store/vectorIndex.js +32 -7
- package/extensions/mega-conflict-cmds.ts +15 -0
- package/extensions/mega-events.ts +96 -6
- package/extensions/mega-runtime.ts +21 -0
- package/extensions/mega-teamrun.test.ts +164 -0
- package/extensions/mega-trim.ts +28 -1
- package/package.json +1 -1
- package/src/memoryOps.ts +42 -2
- package/src/memoryRecall.test.ts +58 -0
- package/src/memoryRecall.ts +52 -0
- package/src/recall.ts +35 -10
- package/src/store/memoryIndex.test.ts +61 -0
- package/src/store/memoryIndex.ts +269 -0
- package/src/store/vectorIndex.ts +40 -6
- package/dist/extensions/openclaw-mega-compact.js +0 -291
- package/dist/src/minilm.js +0 -92
- package/dist/src/wordpiece.js +0 -129
package/README.md
CHANGED
|
@@ -6,28 +6,39 @@ sessions into a **local SQLite store** and offers **deduped inline recall** —
|
|
|
6
6
|
running **locally inside the extension**, with **no remote MCP server** and
|
|
7
7
|
**zero network calls at runtime** (PREVENT-PI-004).
|
|
8
8
|
|
|
9
|
-
> **Current version:** `v0.6.
|
|
9
|
+
> **Current version:** `v0.6.2` — storage backend is **`node:sqlite`**
|
|
10
10
|
> (`DatabaseSync`, a Node ≥22.13 built-in), replacing the old `better-sqlite3`
|
|
11
11
|
> native addon and the per-session gzipped JSON checkpoint files. **Zero native
|
|
12
12
|
> build step, fully local, zero network at runtime.** Legacy
|
|
13
13
|
> `.checkpoints.json.gz` snapshots are retained as disaster-recovery fallbacks
|
|
14
|
-
> and auto-imported on first run.
|
|
15
|
-
>
|
|
14
|
+
> and auto-imported on first run. The S24 line ties auto-compact, the tier
|
|
15
|
+
> label, trim depth, and durable-memory review to one **unified pressure
|
|
16
|
+
> signal**, adds a **cross-repo memory-RAG index**, and relieves context
|
|
17
|
+
> **during team runs** (not just at the end).
|
|
16
18
|
|
|
17
19
|
---
|
|
18
20
|
|
|
19
21
|
## What this is (the 30-second version)
|
|
20
22
|
|
|
21
|
-
pi's context window is finite. When a session gets long
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
23
|
+
pi's context window is finite. When a session gets long — especially a team run
|
|
24
|
+
with sub-agents — pi-mega-compact keeps it going without overflowing:
|
|
25
|
+
|
|
26
|
+
1. **Watches one signal.** A single live `pressure = currentTokens / thresholdTokens`
|
|
27
|
+
drives everything — the tier label, how aggressively the live trim drops
|
|
28
|
+
context, and how often durable memory is reviewed. As context fills, the whole
|
|
29
|
+
system reacts together; as it's relieved, it backs off.
|
|
30
|
+
2. **Compacts in two layers.** On every LLM call it returns a **live, compacted
|
|
31
|
+
view** (the model sees a summary + recent anchor, non-destructively). And it
|
|
32
|
+
persists a durable **checkpoint** — and, at each agent settle during a team
|
|
33
|
+
run, fires pi's **native durable trim** so the on-disk transcript is actually
|
|
34
|
+
truncated (context relieves mid-run, and resume reloads the trimmed transcript
|
|
35
|
+
instead of a 150k window).
|
|
36
|
+
3. **Stores** each checkpoint in a **local vector database** (SQLite) with an
|
|
37
|
+
embedding, so similar regions are found later and **duplicate work is never
|
|
38
|
+
stored twice**.
|
|
39
|
+
4. **Recalls** the right context automatically — same-repo checkpoints on resume,
|
|
40
|
+
plus **cross-repo memory-RAG**: decisions you saved in one repo are inlined as
|
|
41
|
+
context when you start a session in another.
|
|
31
42
|
|
|
32
43
|
Everything lives on **your disk**. No telemetry, no API, no MCP server, no cloud.
|
|
33
44
|
The only optional network surface is a **user-triggered localhost dashboard** you
|
|
@@ -35,11 +46,13 @@ open yourself.
|
|
|
35
46
|
|
|
36
47
|
### Why "mega"?
|
|
37
48
|
|
|
38
|
-
The compaction pipeline is a **Trident** — three deterministic stages
|
|
39
|
-
|
|
40
|
-
is small (a summary + key decisions +
|
|
41
|
-
|
|
42
|
-
tokens.
|
|
49
|
+
The compaction pipeline is a **Trident** — three deterministic stages
|
|
50
|
+
(supersede → collapse → cluster) that run over your conversation before anything
|
|
51
|
+
is persisted. The checkpoint it produces is small (a summary + key decisions +
|
|
52
|
+
next steps + files touched), so the same session that would otherwise overflow
|
|
53
|
+
its window keeps going on a fraction of the tokens. On top of the Trident, a
|
|
54
|
+
single pressure signal orchestrates the live trim, the durable trim, and memory
|
|
55
|
+
review as one coherent system rather than four independent triggers.
|
|
43
56
|
|
|
44
57
|
---
|
|
45
58
|
|
|
@@ -52,9 +65,12 @@ Layer 3 Cluster (vectorize) local vector index → semantic dedup + recall
|
|
|
52
65
|
Layer 2 Collapse (summarize) summarizeMessages() heuristic + agent summary on /mega-compact
|
|
53
66
|
Layer 1 Supersede (prune) drop obsolete file-reads / superseded turns (zero cost)
|
|
54
67
|
─────────────────────────────────────────────────────────────────────────
|
|
55
|
-
Trigger context
|
|
68
|
+
Trigger context → token fast-gate → autoCompactCheck → live trim (per call)
|
|
69
|
+
Durable agent_end (idle + over threshold) → ctx.compact() → session_before_compact
|
|
70
|
+
supplies the summary; pi truncates the transcript (relieves context)
|
|
71
|
+
Live context handler returns { messages:[summary, …recent] } — model sees a
|
|
72
|
+
compacted window every LLM call; the on-disk transcript is untouched
|
|
56
73
|
Marker insert compact-marker; dedupe so repeated triggers cost ~0 tokens
|
|
57
|
-
Cancel session_before_compact → { cancel:true } once persisted (no double-compact)
|
|
58
74
|
```
|
|
59
75
|
|
|
60
76
|
**One store, three ways to read it back — one dedup engine:**
|
|
@@ -168,7 +184,7 @@ building.
|
|
|
168
184
|
pi-mega-compact uses a dual local backend — **zero network, no native build step**:
|
|
169
185
|
|
|
170
186
|
- **`node:sqlite`** (`DatabaseSync`, Node ≥22.13 built-in) — the synchronous source of truth for checkpoints, session state, and the dedup index. No dependency, no install script, survives pi's `install-scripts` block.
|
|
171
|
-
- **PGlite + `@electric-sql/pglite-pgvector`** (WASM Postgres + HNSW `vector_cosine_ops`) — an optional, best-effort async vector index for **cross-repo recall** at `~/.pi/mega-compact-vector`. The sync store stays authoritative; the index degrades to the sync per-session scan on any failure.
|
|
187
|
+
- **PGlite + `@electric-sql/pglite-pgvector`** (WASM Postgres + HNSW `vector_cosine_ops`) — an optional, best-effort async vector index for **cross-repo recall** at `~/.pi/mega-compact-vector`. It holds both checkpoint embeddings and durable-memory embeddings, so decisions saved in one repo are findable from another. The sync store stays authoritative; the index degrades to the sync per-session scan on any failure.
|
|
172
188
|
|
|
173
189
|
Kill-switch: `MEGACOMPACT_PGLITE_DISABLED=1` fully disables the PGlite index (falls back to sync scan). Requires Node ≥22.13 (`engines.node`).
|
|
174
190
|
|
|
@@ -178,7 +194,7 @@ On resume, recall augments from other repos' checkpoints when this repo's store
|
|
|
178
194
|
|
|
179
195
|
### Memory
|
|
180
196
|
|
|
181
|
-
pi-mega-compact auto-reviews the conversation every 10 turns and writes durable `decision`/`fact`/`preference` memories to SQLite (local, hallucination-guarded). Relevant memories are injected as RAG context on recall (capped, deduped). Manual: `/mega-memory save|list|forget
|
|
197
|
+
pi-mega-compact auto-reviews the conversation every 10 turns (the cadence shortens as pressure climbs) and writes durable `decision`/`fact`/`preference` memories to SQLite (local, hallucination-guarded). Relevant memories are injected as RAG context on recall (capped, deduped). **Cross-repo memory-RAG (S24):** every memory write is mirrored into the PGlite/HNSW index, so when same-repo recall is thin the system augments with the nearest memories from *other* repos (stricter `MEGACOMPACT_CROSSREPO_COSINE` floor, deduped against what's already in view). Manual: `/mega-memory save|list|forget` (or `/m`).
|
|
182
198
|
|
|
183
199
|
### Uninstall
|
|
184
200
|
|
|
@@ -352,7 +368,7 @@ The extension entry adapts between the engine and pi's runtime types.
|
|
|
352
368
|
|
|
353
369
|
```bash
|
|
354
370
|
npm run build # tsc
|
|
355
|
-
npm test # build + node --test on dist/**/*.test.js (
|
|
371
|
+
npm test # build + node --test on dist/**/*.test.js (353 tests)
|
|
356
372
|
npm run lint # tsc --noEmit + guardrails-scan
|
|
357
373
|
npm run guardrails # regression_check + guardrails-scan
|
|
358
374
|
```
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* conflict-scan.test.ts — unit tests for the extension-conflict scanner.
|
|
3
|
+
*
|
|
4
|
+
* Fixture trees are written under a temp dir and scanned via
|
|
5
|
+
* MEGACOMPACT_EXT_SCAN_DIR (which makes collectScanRoots() return that
|
|
6
|
+
* single root). This covers the S24 follow-up fix:
|
|
7
|
+
*
|
|
8
|
+
* 1. node_modules-style code extensions (package.json + pi.extensions) are
|
|
9
|
+
* still detected by source-marker grep (regression).
|
|
10
|
+
* 2. USER-LEVEL extensions installed outside npm (e.g. pi-hermes-memory)
|
|
11
|
+
* now get scanned too — previously only `node_modules` was walked, so a
|
|
12
|
+
* data-only memory store (MEMORY.md + sessions.db, no package.json)
|
|
13
|
+
* was never flagged (the 5000-char file-buffer error slipped through).
|
|
14
|
+
* 3. The data-only memory-store signature is detected even with no source.
|
|
15
|
+
* 4. pi-mega-compact (selfName) is always skipped.
|
|
16
|
+
*/
|
|
17
|
+
import { test, after } from "node:test";
|
|
18
|
+
import assert from "node:assert/strict";
|
|
19
|
+
import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs";
|
|
20
|
+
import { tmpdir } from "node:os";
|
|
21
|
+
import { join } from "node:path";
|
|
22
|
+
import { detectConflicts, collectScanRoots } from "./conflict-scan.js";
|
|
23
|
+
const base = mkdtempSync(join(tmpdir(), "mc-scan-"));
|
|
24
|
+
let n = 0;
|
|
25
|
+
/** Make a fixture root containing one or more fake extensions, return its path. */
|
|
26
|
+
function fixture(build) {
|
|
27
|
+
const root = join(base, `case-${n++}`);
|
|
28
|
+
mkdirSync(root, { recursive: true });
|
|
29
|
+
build(root);
|
|
30
|
+
return root;
|
|
31
|
+
}
|
|
32
|
+
after(() => {
|
|
33
|
+
rmSync(base, { recursive: true, force: true });
|
|
34
|
+
});
|
|
35
|
+
test("scans a user-level, data-only memory store (no package.json)", () => {
|
|
36
|
+
const root = fixture((r) => {
|
|
37
|
+
const ext = join(r, "pi-hermes-memory");
|
|
38
|
+
mkdirSync(ext, { recursive: true });
|
|
39
|
+
// No package.json, no source — just pi's memory-store signature.
|
|
40
|
+
writeFileSync(join(ext, "MEMORY.md"), "# memory\n");
|
|
41
|
+
writeFileSync(join(ext, "sessions.db"), "");
|
|
42
|
+
});
|
|
43
|
+
process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
|
|
44
|
+
try {
|
|
45
|
+
const { conflicts } = detectConflicts();
|
|
46
|
+
assert.ok(conflicts.length >= 1, "expected a memory conflict");
|
|
47
|
+
const hit = conflicts.find((c) => c.kind === "memory");
|
|
48
|
+
assert.ok(hit, "expected a memory-kind conflict");
|
|
49
|
+
assert.equal(hit.severity, "high");
|
|
50
|
+
assert.ok(hit.evidence.includes("MEMORY.md") ||
|
|
51
|
+
hit.evidence.includes("sessions.db"), "evidence should name the on-disk memory signature");
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
test("still detects a code extension by source marker (regression)", () => {
|
|
58
|
+
const root = fixture((r) => {
|
|
59
|
+
// A code extension is a DIRECT child of the scan root (mirrors the
|
|
60
|
+
// node_modules layout: packages live one level under the root).
|
|
61
|
+
const ext = join(r, "some-memory-ext");
|
|
62
|
+
mkdirSync(ext, { recursive: true });
|
|
63
|
+
writeFileSync(join(ext, "package.json"), JSON.stringify({ name: "some-memory-ext", pi: { extensions: ["x.ts"] } }));
|
|
64
|
+
writeFileSync(join(ext, "index.ts"), "export const MEMORY_TOOL = true;");
|
|
65
|
+
});
|
|
66
|
+
process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
|
|
67
|
+
try {
|
|
68
|
+
const { conflicts } = detectConflicts();
|
|
69
|
+
const hit = conflicts.find((c) => c.package === "some-memory-ext");
|
|
70
|
+
assert.ok(hit, "expected some-memory-ext to be flagged");
|
|
71
|
+
assert.equal(hit.kind, "memory");
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
test("skips pi-mega-compact (selfName) and non-extension dirs", () => {
|
|
78
|
+
const root = fixture((r) => {
|
|
79
|
+
// selfName dir with a memory signature — must be ignored.
|
|
80
|
+
const me = join(r, "node_modules", "pi-mega-compact");
|
|
81
|
+
mkdirSync(me, { recursive: true });
|
|
82
|
+
writeFileSync(join(me, "sessions.db"), "");
|
|
83
|
+
// unrelated dir with no pi.extensions and no memory signature.
|
|
84
|
+
mkdirSync(join(r, "node_modules", "totally-fine"), { recursive: true });
|
|
85
|
+
});
|
|
86
|
+
process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
|
|
87
|
+
try {
|
|
88
|
+
const { scanned, conflicts } = detectConflicts();
|
|
89
|
+
assert.equal(conflicts.length, 0, "no conflicts expected");
|
|
90
|
+
assert.ok(!scanned.some((s) => s.includes("pi-mega-compact")), "selfName should not appear in scanned");
|
|
91
|
+
}
|
|
92
|
+
finally {
|
|
93
|
+
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
test("collectScanRoots honors MEGACOMPACT_EXT_SCAN_DIR override", () => {
|
|
97
|
+
const root = fixture(() => { });
|
|
98
|
+
process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
|
|
99
|
+
try {
|
|
100
|
+
const roots = collectScanRoots();
|
|
101
|
+
assert.deepEqual(roots, [root], "override replaces the whole root list");
|
|
102
|
+
}
|
|
103
|
+
finally {
|
|
104
|
+
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
test("collectScanRoots falls back to node_modules + user dir when no override", () => {
|
|
108
|
+
delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
|
|
109
|
+
delete process.env.MEGACOMPACT_EXT_USER_DIR;
|
|
110
|
+
// No override set and this test file lives under extensions/, so node_modules
|
|
111
|
+
// resolution walks up from here; the user dir (~/.pi/agent) may or may
|
|
112
|
+
// not exist in CI. We only assert the call returns a non-throwing array.
|
|
113
|
+
const roots = collectScanRoots();
|
|
114
|
+
assert.ok(Array.isArray(roots), "collectScanRoots must return an array");
|
|
115
|
+
});
|
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
import { detectConflicts } from "./conflict-scan.js";
|
|
10
10
|
import { addMemory, listMemories, searchMemories, recallMemory } from "../src/store/sqlite.js";
|
|
11
11
|
import { resolveRepoRoot } from "./mega-config.js";
|
|
12
|
+
import { defaultEmbedder } from "../src/embedder.js";
|
|
13
|
+
import { upsertMemoryEmbedding } from "../src/store/memoryIndex.js";
|
|
12
14
|
/** Run the conflict scan and format a human-readable report. */
|
|
13
15
|
export function validateExtensions() {
|
|
14
16
|
const report = detectConflicts();
|
|
@@ -74,6 +76,14 @@ export function registerConflictCommands(pi, runtime) {
|
|
|
74
76
|
const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
|
|
75
77
|
const content = text.replace(/#[\w-]+/g, "").trim();
|
|
76
78
|
const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
|
|
79
|
+
// S24: mirror into the cross-repo memory index (fire-and-forget).
|
|
80
|
+
try {
|
|
81
|
+
const vec = defaultEmbedder().embed(content);
|
|
82
|
+
void upsertMemoryEmbedding(repo, id, content, vec);
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
/* non-fatal */
|
|
86
|
+
}
|
|
77
87
|
ctx.ui.notify(`[mega-memory] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
|
|
78
88
|
return;
|
|
79
89
|
}
|
|
@@ -142,6 +152,13 @@ export function registerConflictCommands(pi, runtime) {
|
|
|
142
152
|
const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
|
|
143
153
|
const content = text.replace(/#[\w-]+/g, "").trim();
|
|
144
154
|
const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
|
|
155
|
+
try {
|
|
156
|
+
const vec = defaultEmbedder().embed(content);
|
|
157
|
+
void upsertMemoryEmbedding(repo, id, content, vec);
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
/* non-fatal */
|
|
161
|
+
}
|
|
145
162
|
ctx.ui.notify(`[/m] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
|
|
146
163
|
return;
|
|
147
164
|
}
|
|
@@ -15,8 +15,17 @@ import { recallMemoriesAndInline } from "../src/recall.js";
|
|
|
15
15
|
import { driveNativeCompaction } from "./mega-compact-driver.js";
|
|
16
16
|
import { computeLiveTrimCut, liveTrimSummaryMessage } from "./mega-trim.js";
|
|
17
17
|
import { pressureFromPct, memoryReviewCadence } from "./mega-config.js";
|
|
18
|
+
/**
|
|
19
|
+
* DIAG accessor for the headless test harness: the most recently constructed
|
|
20
|
+
* MegaRuntime, so a test that loads the compiled extension via its default
|
|
21
|
+
* export can read diag counters (diagLiveTrimFires / diagBeforeCompactFires /
|
|
22
|
+
* diagBeforeCompactSupplied / diagAgentEndIdle) after firing synthetic events.
|
|
23
|
+
* No-op in production — nothing reads this outside tests.
|
|
24
|
+
*/
|
|
25
|
+
export let lastRuntime;
|
|
18
26
|
/** Register all pi lifecycle event handlers. */
|
|
19
27
|
export function registerEventHandlers(pi, runtime, config) {
|
|
28
|
+
lastRuntime = runtime;
|
|
20
29
|
// ---- Session lifecycle (state reset points) -------------------------------
|
|
21
30
|
// Capture model/provider whenever it changes (drives real cost estimation).
|
|
22
31
|
pi.on("model_select", async (_event, ctx) => {
|
|
@@ -56,6 +65,8 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
56
65
|
try {
|
|
57
66
|
const mr = await recallMemoriesAndInline({
|
|
58
67
|
query, stateDir: runtime.getStateDir(), limit: 5,
|
|
68
|
+
crossRepo: config.crossRepoEnabled,
|
|
69
|
+
crossRepoCosine: config.crossRepoCosine,
|
|
59
70
|
});
|
|
60
71
|
if (!mr.empty)
|
|
61
72
|
runtime.pendingMemoryRecallBlock = mr.block;
|
|
@@ -82,7 +93,7 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
82
93
|
}
|
|
83
94
|
// S21: parallel memory recall. Trigram embedder is sub-ms; await is fine.
|
|
84
95
|
try {
|
|
85
|
-
const mr = await recallMemoriesAndInline({ query, stateDir: runtime.getStateDir(), limit: 5 });
|
|
96
|
+
const mr = await recallMemoriesAndInline({ query, stateDir: runtime.getStateDir(), limit: 5, crossRepo: config.crossRepoEnabled, crossRepoCosine: config.crossRepoCosine });
|
|
86
97
|
if (!mr.empty)
|
|
87
98
|
runtime.pendingMemoryRecallBlock = mr.block;
|
|
88
99
|
}
|
|
@@ -139,6 +150,46 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
139
150
|
const idle = ctx.isIdle?.() ?? true;
|
|
140
151
|
const queued = ctx.hasPendingMessages?.() ?? false;
|
|
141
152
|
const now = Date.now();
|
|
153
|
+
// DIAG (team-run relief): surface whether the agent is idle + over
|
|
154
|
+
// threshold at agent_end so we can see if a mid-run durable-trim trigger
|
|
155
|
+
// *should* have fired but didn't.
|
|
156
|
+
const overThreshold = (runtime.lastCtxTokens ?? 0) >= config.thresholdTokens;
|
|
157
|
+
runtime.diagAgentEndIdle++;
|
|
158
|
+
runtime.logger.info("agent-end-idle", {
|
|
159
|
+
sessionId: runtime.rt.sessionId,
|
|
160
|
+
idle,
|
|
161
|
+
queued,
|
|
162
|
+
overThreshold,
|
|
163
|
+
ctxPct: runtime.lastCtxPercent,
|
|
164
|
+
ctxTokens: runtime.lastCtxTokens,
|
|
165
|
+
thresholdTokens: config.thresholdTokens,
|
|
166
|
+
wouldNudge: idle && queued && now >= runtime.resumeNudgeUntil,
|
|
167
|
+
});
|
|
168
|
+
// S16+S24: MID-RUN DURABLE TRIM. During a long team run (sub-agents),
|
|
169
|
+
// pi's native durable compaction only fires from _checkCompaction at
|
|
170
|
+
// PARENT settle (agent-session.js:760/844), so the on-disk transcript +
|
|
171
|
+
// context meter balloon to ~150k and never relieve until the very end
|
|
172
|
+
// ("compacts but doesn't resume"). agent_end with activeAgents===0 is a
|
|
173
|
+
// SAFE, settled point: calling ctx.compact() here does NOT abort an
|
|
174
|
+
// in-flight turn (the S16 danger is only mid-turn). ctx.compact() runs
|
|
175
|
+
// pi's flow, which fires our session_before_compact handler to supply
|
|
176
|
+
// the durable trim (truncates the transcript from firstKeptEntryId).
|
|
177
|
+
// Guarded three ways: only when truly idle + over threshold, only when
|
|
178
|
+
// pi would actually compact (piCompactWouldNoop skips the user-facing
|
|
179
|
+
// no-op throw), and debounced (one durable trim per 2s) to avoid
|
|
180
|
+
// thrashing the transcript while sub-agents keep settling.
|
|
181
|
+
if (idle && overThreshold && now >= runtime.debounceUntil) {
|
|
182
|
+
if (!piCompactWouldNoop(ctx)) {
|
|
183
|
+
runtime.debounceUntil = now + 2000;
|
|
184
|
+
runtime.diagAgentEndDurable++;
|
|
185
|
+
runtime.logger.info("agent-end-durable-trigger", {
|
|
186
|
+
sessionId: runtime.rt.sessionId,
|
|
187
|
+
ctxTokens: runtime.lastCtxTokens,
|
|
188
|
+
thresholdTokens: config.thresholdTokens,
|
|
189
|
+
});
|
|
190
|
+
ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort
|
|
191
|
+
}
|
|
192
|
+
}
|
|
142
193
|
if (idle && queued && now >= runtime.resumeNudgeUntil) {
|
|
143
194
|
runtime.resumeNudgeUntil = now + 30_000;
|
|
144
195
|
pi.sendUserMessage("[mega-compact] continue from the compacted context above.");
|
|
@@ -207,22 +258,30 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
207
258
|
const currentTokens = usage?.tokens ?? estimateSessionTokens(view) ??
|
|
208
259
|
Math.round((pct / 100) * (usage?.contextWindow ?? 0));
|
|
209
260
|
// FAST GATE: token-based (tier threshold), not percentage-based.
|
|
210
|
-
if (currentTokens < config.thresholdTokens)
|
|
261
|
+
if (currentTokens < config.thresholdTokens) {
|
|
262
|
+
runtime.diagCtxFastGate++;
|
|
211
263
|
return;
|
|
264
|
+
}
|
|
212
265
|
const check = autoCompactCheck(currentTokens, config.thresholdTokens); // SERVER-STYLE CONFIRM (local)
|
|
213
|
-
if (!check.shouldCompact)
|
|
266
|
+
if (!check.shouldCompact) {
|
|
267
|
+
runtime.diagCtxNoCompact++;
|
|
214
268
|
return;
|
|
269
|
+
}
|
|
215
270
|
// Debounce so we don't fire on every context event past threshold.
|
|
216
271
|
const now = Date.now();
|
|
217
|
-
if (now < runtime.debounceUntil)
|
|
272
|
+
if (now < runtime.debounceUntil) {
|
|
273
|
+
runtime.diagCtxDebounce++;
|
|
218
274
|
return;
|
|
275
|
+
}
|
|
219
276
|
runtime.debounceUntil = now + 2000;
|
|
220
277
|
// Adaptive compression (Fix E): scale compression strength + keepFrom depth
|
|
221
278
|
// with how close we are to the model context limit.
|
|
222
279
|
const pressure = pressureFromPct(pct);
|
|
223
280
|
const ran = runCompact(pi, runtime, config, ctx, messages, { compressionPressure: pressure });
|
|
224
|
-
if (ran.skipped)
|
|
281
|
+
if (ran.skipped) {
|
|
282
|
+
runtime.diagCtxRunSkipped++;
|
|
225
283
|
return;
|
|
284
|
+
}
|
|
226
285
|
// LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
|
|
227
286
|
// manual compact path aborts the in-flight turn — only used behind the flag.
|
|
228
287
|
// Read live from env (in addition to the load-time config) so the flag can be
|
|
@@ -254,8 +313,16 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
254
313
|
summary: ran.result.summary,
|
|
255
314
|
anchorUserMessages,
|
|
256
315
|
});
|
|
257
|
-
if (cut === null)
|
|
316
|
+
if (cut === null) {
|
|
317
|
+
runtime.diagCtxCutNull++;
|
|
318
|
+
runtime.logger.info("live-trim-skip", {
|
|
319
|
+
sessionId: runtime.rt.sessionId,
|
|
320
|
+
compactedFrom: ran.result.compactedFrom,
|
|
321
|
+
viewLen: view.length,
|
|
322
|
+
anchorUserMessages,
|
|
323
|
+
});
|
|
258
324
|
return; // unsafe / below anchor floor — no trim this call
|
|
325
|
+
}
|
|
259
326
|
const summaryMsg = liveTrimSummaryMessage({
|
|
260
327
|
compactedFrom: ran.result.compactedFrom,
|
|
261
328
|
summary: ran.result.summary,
|
|
@@ -269,9 +336,23 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
269
336
|
};
|
|
270
337
|
const recent = messages.slice(cut); // guardrails-allow PREVENT-PI-002: `cut` is the pre-sanitized `compactedFrom` produced by src/boundary.ts computeDropRange, so the preserved run begins on a toolPair-safe index.
|
|
271
338
|
runtime.snapshot(ctx);
|
|
339
|
+
// DIAG (team-run relief): confirm the live trim actually fires + how big
|
|
340
|
+
// the window still is. The return is non-durable (per-LLM-call only), so
|
|
341
|
+
// this is the signal that the model is being fed a compacted view while
|
|
342
|
+
// the on-disk transcript + context meter keep growing.
|
|
343
|
+
runtime.diagLiveTrimFires++;
|
|
344
|
+
runtime.logger.info("live-trim", {
|
|
345
|
+
sessionId: runtime.rt.sessionId,
|
|
346
|
+
inputMsgs: messages.length,
|
|
347
|
+
outputMsgs: recent.length + 1,
|
|
348
|
+
compactedFrom: cut,
|
|
349
|
+
ctxPct: pct,
|
|
350
|
+
ctxTokens: usage?.tokens ?? null,
|
|
351
|
+
});
|
|
272
352
|
return { messages: [summaryAgentMsg, ...recent] };
|
|
273
353
|
}
|
|
274
354
|
catch {
|
|
355
|
+
runtime.diagCtxThrown++;
|
|
275
356
|
return; // non-fatal: no trim this call; the next context event retries
|
|
276
357
|
}
|
|
277
358
|
});
|
|
@@ -283,11 +364,26 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
283
364
|
// there is no full-reload + additive recall inflation.
|
|
284
365
|
pi.on("session_before_compact", async (event, ctx) => {
|
|
285
366
|
runtime.resetRuntime(ctx.sessionManager.getSessionId());
|
|
367
|
+
// DIAG (team-run relief): this is the ONLY durable-trim entry point. Log
|
|
368
|
+
// every fire + whether we supplied a compaction (truncates transcript) or
|
|
369
|
+
// fell through to {} (pi runs its own). If this is sparse during a team
|
|
370
|
+
// run, the durable trim is firing too late (only at parent settle).
|
|
371
|
+
const prep = event.preparation;
|
|
372
|
+
runtime.diagBeforeCompactFires++;
|
|
373
|
+
runtime.logger.info("before-compact-entry", {
|
|
374
|
+
sessionId: runtime.rt.sessionId,
|
|
375
|
+
reason: event.reason,
|
|
376
|
+
hasPrep: !!prep,
|
|
377
|
+
msgsToSummarize: prep?.messagesToSummarize?.length ?? 0,
|
|
378
|
+
firstKeptEntryId: prep?.firstKeptEntryId ?? null,
|
|
379
|
+
activeAgents: runtime.activeAgents,
|
|
380
|
+
});
|
|
286
381
|
if (!config.auto)
|
|
287
382
|
return {}; // let pi run its own native compaction
|
|
288
383
|
try {
|
|
289
384
|
const result = driveNativeCompaction(event, runtime, config);
|
|
290
385
|
if (result) {
|
|
386
|
+
runtime.diagBeforeCompactSupplied++;
|
|
291
387
|
runtime.logger.info("native-compact", {
|
|
292
388
|
sessionId: runtime.rt.sessionId,
|
|
293
389
|
firstKeptEntryId: result.compaction.firstKeptEntryId,
|
|
@@ -120,6 +120,26 @@ export class MegaRuntime {
|
|
|
120
120
|
lastCtxTokens = null;
|
|
121
121
|
lastCtxPercent = null;
|
|
122
122
|
lastCtxWindow = 0;
|
|
123
|
+
/**
|
|
124
|
+
* DIAG counters for the "team run doesn't relieve context" investigation.
|
|
125
|
+
* Plain integers, incremented at the three compaction decision points. They
|
|
126
|
+
* let a headless test drive the real event handlers and assert the firing
|
|
127
|
+
* cadence without scraping log files. Inert in production (the live-trim and
|
|
128
|
+
* before-compact probes also emit logger.info, but these counters are always
|
|
129
|
+
* updated and cost nothing).
|
|
130
|
+
*/
|
|
131
|
+
diagLiveTrimFires = 0; // context handler returned a trimmed view
|
|
132
|
+
diagBeforeCompactFires = 0; // session_before_compact handler entered
|
|
133
|
+
diagBeforeCompactSupplied = 0; // session_before_compact supplied our trim
|
|
134
|
+
diagAgentEndIdle = 0; // agent_end with activeAgents===0
|
|
135
|
+
diagAgentEndDurable = 0; // agent_end fired ctx.compact() (mid-run durable trim)
|
|
136
|
+
// Per-skip-path counters for the team-run diagnosis.
|
|
137
|
+
diagCtxFastGate = 0; // returned at token fast-gate (below threshold)
|
|
138
|
+
diagCtxNoCompact = 0; // autoCompactCheck().shouldCompact === false
|
|
139
|
+
diagCtxDebounce = 0; // debounceUntil not yet elapsed
|
|
140
|
+
diagCtxRunSkipped = 0; // runCompact() returned skipped
|
|
141
|
+
diagCtxCutNull = 0; // computeLiveTrimCut returned null (anchor/boundary)
|
|
142
|
+
diagCtxThrown = 0; // live-trim try threw (caught)
|
|
123
143
|
/**
|
|
124
144
|
* Live 0–1 pressure: how full the context window is relative to the compaction
|
|
125
145
|
* threshold. Computed from the most recent context event the runtime already
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* mega-teamrun.test.ts — regression test for the "auto-compact runs but context
|
|
3
|
+
* never relieves during a team run (sub-agents)" bug.
|
|
4
|
+
*
|
|
5
|
+
* Loads the REAL compiled extension (extensions/mega-compact.js) through a
|
|
6
|
+
* faithful mock pi (mirrors mega-compact.test.ts's harness) and drives the
|
|
7
|
+
* exact event sequence a long team run produces:
|
|
8
|
+
*
|
|
9
|
+
* agent_start -> context (over threshold) xN -> agent_end (repeat x3)
|
|
10
|
+
*
|
|
11
|
+
* Asserts the TWO fixes:
|
|
12
|
+
* 1. live trim FIRES per-call (computeLiveTrimCut no longer returns null on
|
|
13
|
+
* the anchor floor — was `cutNull`, liveTrimFires===0 before the fix).
|
|
14
|
+
* 2. the DURABLE trim fires at agent_end while idle + over threshold
|
|
15
|
+
* (mid-run durable trigger), not only at parent settle.
|
|
16
|
+
*
|
|
17
|
+
* The mock ctx.compact() drives session_before_compact so we observe the
|
|
18
|
+
* durable truncation. Counters come from MegaRuntime.diag* (set behind the
|
|
19
|
+
* real handler code, inert in production).
|
|
20
|
+
*
|
|
21
|
+
* MEGACOMPACT_PGLITE_DISABLED keeps the run fast (no WASM index init).
|
|
22
|
+
*/
|
|
23
|
+
import { test } from "node:test";
|
|
24
|
+
import assert from "node:assert/strict";
|
|
25
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
26
|
+
import { tmpdir } from "node:os";
|
|
27
|
+
import { join } from "node:path";
|
|
28
|
+
import { createRequire } from "node:module";
|
|
29
|
+
import { closeVectorIndex } from "../src/store/vectorIndex.js";
|
|
30
|
+
const require = createRequire(import.meta.url);
|
|
31
|
+
const baseTmp = mkdtempSync(join(tmpdir(), "mc-team-"));
|
|
32
|
+
process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "index");
|
|
33
|
+
process.env.MEGACOMPACT_PGLITE_DISABLED = "true"; // fast: skip WASM index
|
|
34
|
+
let counter = 0;
|
|
35
|
+
function harness() {
|
|
36
|
+
const stateDir = join(baseTmp, `run-${counter++}`);
|
|
37
|
+
process.env.MEGACOMPACT_STATE_DIR = stateDir;
|
|
38
|
+
process.env.MEGACOMPACT_DEBUG = "true";
|
|
39
|
+
process.env.MEGACOMPACT_THRESHOLD_TOKENS = "50";
|
|
40
|
+
process.env.MEGACOMPACT_FAST_GATE_PCT = "1";
|
|
41
|
+
process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES = "1";
|
|
42
|
+
process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR = "0"; // piCompactWouldNoop must not skip
|
|
43
|
+
process.env.MEGACOMPACT_MEMORY_AUTO_REVIEW = "false";
|
|
44
|
+
process.env.MEGACOMPACT_RAPTOR_ENABLED = "false";
|
|
45
|
+
delete process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM;
|
|
46
|
+
const handlers = {};
|
|
47
|
+
const compactCalls = [];
|
|
48
|
+
function msg(role, text, toolName) {
|
|
49
|
+
if (role === "assistant" && toolName) {
|
|
50
|
+
return { role: "assistant", content: [{ type: "toolCall", name: toolName, id: "c1", arguments: {} }], api: "anthropic-messages", provider: "anthropic", model: "m", usage: { inputTokens: 1, outputTokens: 1, cacheReadTokens: 0, cacheWriteTokens: 0 }, stopReason: "tool_use", timestamp: 0 };
|
|
51
|
+
}
|
|
52
|
+
if (role === "toolResult" && toolName) {
|
|
53
|
+
return { role: "toolResult", content: [{ type: "text", text }], toolCallId: "c1", toolName, isError: false, timestamp: 0 };
|
|
54
|
+
}
|
|
55
|
+
return { role: "user", content: text, timestamp: 0 };
|
|
56
|
+
}
|
|
57
|
+
const session = [];
|
|
58
|
+
for (let i = 0; i < 14; i++) {
|
|
59
|
+
session.push(msg("user", `actually we decided to use approach ${i} for module ${i}`));
|
|
60
|
+
session.push(msg("assistant", `edited module ${i}`, "Edit"));
|
|
61
|
+
session.push(msg("toolResult", `edited module ${i}`, "Edit"));
|
|
62
|
+
}
|
|
63
|
+
const toEntry = (m, i) => ({ type: "message", id: `e${i}`, parentId: null, timestamp: String(i), message: m });
|
|
64
|
+
const sessionManager = {
|
|
65
|
+
getSessionId: () => "sess_team_001",
|
|
66
|
+
getEntries: () => session.map(toEntry),
|
|
67
|
+
getBranch: () => session.map(toEntry),
|
|
68
|
+
};
|
|
69
|
+
function makeCtx(over = {}) {
|
|
70
|
+
return {
|
|
71
|
+
ui: { setStatus: () => { }, notify: () => { }, select: () => { }, confirm: async () => true, input: async () => "", setWidget: () => { } },
|
|
72
|
+
mode: "tui", hasUI: true, cwd: stateDir, sessionManager,
|
|
73
|
+
modelRegistry: {}, model: undefined, isIdle: () => true, isProjectTrusted: () => true,
|
|
74
|
+
signal: undefined, abort: () => { }, hasPendingMessages: () => false, shutdown: () => { },
|
|
75
|
+
getContextUsage: () => ({ tokens: 200000, contextWindow: 200000, percent: 100 }),
|
|
76
|
+
// Mock ctx.compact() runs pi's flow and fires session_before_compact.
|
|
77
|
+
compact: (opts) => {
|
|
78
|
+
compactCalls.push(opts);
|
|
79
|
+
if (handlers["session_before_compact"]) {
|
|
80
|
+
return handlers["session_before_compact"]({ type: "session_before_compact", reason: "threshold", willRetry: false, signal: undefined, preparation: { firstKeptEntryId: "e2", messagesToSummarize: session.slice(0, 2), tokensBefore: 500 } }, makeCtx());
|
|
81
|
+
}
|
|
82
|
+
return undefined;
|
|
83
|
+
},
|
|
84
|
+
getSystemPrompt: () => "system base",
|
|
85
|
+
...over,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
const pi = {
|
|
89
|
+
on: (ev, h) => { handlers[ev] = h; },
|
|
90
|
+
registerCommand: () => { }, registerTool: () => { }, registerShortcut: () => { },
|
|
91
|
+
registerFlag: () => { }, getFlag: () => undefined, registerMessageRenderer: () => { },
|
|
92
|
+
registerEntryRenderer: () => { }, sendMessage: () => { }, sendUserMessage: () => { },
|
|
93
|
+
appendEntry: () => { }, setSessionName: () => { }, getSessionName: () => undefined,
|
|
94
|
+
setLabel: () => { }, exec: async () => ({ stdout: "", stderr: "", code: 0 }),
|
|
95
|
+
getActiveTools: () => [], getAllTools: () => [], setActiveTools: () => { },
|
|
96
|
+
getCommands: () => [], setModel: async () => false, getThinkingLevel: () => "off",
|
|
97
|
+
setThinkingLevel: () => { },
|
|
98
|
+
};
|
|
99
|
+
const mod = require("./mega-compact.js");
|
|
100
|
+
mod.default(pi);
|
|
101
|
+
const { lastRuntime } = require("./mega-events.js");
|
|
102
|
+
const fire = (ev, event, ctx) => handlers[ev](event, ctx);
|
|
103
|
+
return {
|
|
104
|
+
stateDir, handlers, compactCalls, fire, ctx: makeCtx, session,
|
|
105
|
+
runtime: lastRuntime, // MegaRuntime with diag* counters
|
|
106
|
+
// Advance the debounce so agent_end (same instant) can trigger durable trim.
|
|
107
|
+
clearDebounce: () => { if (lastRuntime)
|
|
108
|
+
lastRuntime.debounceUntil = 0; },
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
test("team run: live trim fires AND durable trim fires per sub-agent (relieves context)", async () => {
|
|
112
|
+
const h = harness();
|
|
113
|
+
const ctx = h.ctx();
|
|
114
|
+
for (let a = 0; a < 3; a++) {
|
|
115
|
+
await h.fire("agent_start", { type: "agent_start", messages: [] }, ctx);
|
|
116
|
+
for (let i = 0; i < 4; i++) {
|
|
117
|
+
await h.fire("context", { type: "context", messages: h.session }, ctx);
|
|
118
|
+
}
|
|
119
|
+
// Real team runs settle seconds after the last context event; mimic that
|
|
120
|
+
// so the 2s debounce has elapsed and the durable trigger can fire.
|
|
121
|
+
await new Promise((r) => setTimeout(r, 2100));
|
|
122
|
+
h.clearDebounce();
|
|
123
|
+
await h.fire("agent_end", { type: "agent_end", messages: [] }, ctx);
|
|
124
|
+
}
|
|
125
|
+
const rt = h.runtime;
|
|
126
|
+
// FIX 1: live trim must actually fire (was 0 — computeLiveTrimCut returned null).
|
|
127
|
+
assert.ok(rt.diagLiveTrimFires > 0, "live trim fires during the team run (anchor-floor fix)");
|
|
128
|
+
assert.equal(rt.diagCtxCutNull, 0, "no live-trim cut skipped on anchor floor");
|
|
129
|
+
// FIX 2: durable trim must fire at each agent_end (was 0 — only at parent settle).
|
|
130
|
+
assert.equal(rt.diagAgentEndDurable, 3, "mid-run durable trigger fired at each agent_end");
|
|
131
|
+
assert.equal(rt.diagBeforeCompactSupplied, 3, "our durable trim supplied 3x (context relieved)");
|
|
132
|
+
assert.ok(h.compactCalls.length >= 3, "ctx.compact() invoked for durable trim between sub-agents");
|
|
133
|
+
});
|
|
134
|
+
test("control: session_before_compact supplies a durable compaction (parent settles)", async () => {
|
|
135
|
+
const h = harness();
|
|
136
|
+
const res = await h.fire("session_before_compact", { type: "session_before_compact", reason: "threshold", willRetry: false, signal: undefined, preparation: { firstKeptEntryId: "e2", messagesToSummarize: h.session.slice(0, 4), tokensBefore: 500 } }, h.ctx());
|
|
137
|
+
assert.ok(res?.compaction, "compaction result returned to pi");
|
|
138
|
+
assert.equal(res.compaction.firstKeptEntryId, "e2", "reuses pi's boundary (PREVENT-PI-002)");
|
|
139
|
+
});
|
|
140
|
+
test("cleanup", async () => {
|
|
141
|
+
await closeVectorIndex();
|
|
142
|
+
rmSync(baseTmp, { recursive: true, force: true });
|
|
143
|
+
});
|