pi-mega-compact 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/mega-conflict-cmds.js +17 -0
- package/dist/extensions/mega-events.js +102 -6
- package/dist/extensions/mega-runtime.js +20 -0
- package/dist/extensions/mega-teamrun.test.js +143 -0
- package/dist/extensions/mega-trim.js +33 -2
- package/dist/src/memoryOps.js +42 -2
- package/dist/src/memoryRecall.js +48 -0
- package/dist/src/memoryRecall.test.js +52 -0
- package/dist/src/recall.js +34 -8
- package/dist/src/store/memoryIndex.js +205 -0
- package/dist/src/store/memoryIndex.test.js +51 -0
- package/extensions/mega-conflict-cmds.ts +15 -0
- package/extensions/mega-events.ts +96 -6
- package/extensions/mega-runtime.ts +21 -0
- package/extensions/mega-teamrun.test.ts +164 -0
- package/extensions/mega-trim.ts +28 -1
- package/package.json +1 -1
- package/src/memoryOps.ts +42 -2
- package/src/memoryRecall.test.ts +58 -0
- package/src/memoryRecall.ts +52 -0
- package/src/recall.ts +35 -10
- package/src/store/memoryIndex.test.ts +61 -0
- package/src/store/memoryIndex.ts +235 -0
package/dist/src/recall.js
CHANGED
|
@@ -105,27 +105,53 @@ export function formatMemoryRecallBlock(hits) {
|
|
|
105
105
|
export async function recallMemoriesAndInline(opts) {
|
|
106
106
|
const limit = opts.limit ?? 5;
|
|
107
107
|
const maxTokens = opts.recallMaxTokens ?? 0;
|
|
108
|
-
const { recallMemories } = await import("./memoryRecall.js");
|
|
108
|
+
const { recallMemories, recallMemoriesCrossRepo } = await import("./memoryRecall.js");
|
|
109
109
|
const hits = await recallMemories(opts.query, opts.stateDir, {
|
|
110
110
|
topK: limit,
|
|
111
111
|
minSimilarity: opts.minSimilarity ?? 0.2,
|
|
112
112
|
});
|
|
113
|
-
if
|
|
113
|
+
// S24 cross-repo augmentation: if same-repo recall is thin, pull additional
|
|
114
|
+
// memories from OTHER repos via the PGlite HNSW index. Non-fatal: a failure
|
|
115
|
+
// degrades to the same-repo hits only.
|
|
116
|
+
const crossHits = [];
|
|
117
|
+
if (opts.crossRepo && hits.length < limit) {
|
|
118
|
+
try {
|
|
119
|
+
const x = await recallMemoriesCrossRepo(opts.query, opts.stateDir, {
|
|
120
|
+
repo: null,
|
|
121
|
+
limit: limit - hits.length,
|
|
122
|
+
crossRepoCosine: opts.crossRepoCosine ?? 0.3,
|
|
123
|
+
});
|
|
124
|
+
for (const h of x)
|
|
125
|
+
crossHits.push(h);
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
/* non-fatal — cross-repo failure → same-repo only */
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
if (hits.length === 0 && crossHits.length === 0)
|
|
114
132
|
return { empty: true, block: "", report: [] };
|
|
115
133
|
// Same incremental token cap pattern as checkpoint recall.
|
|
116
134
|
const parts = [];
|
|
117
135
|
const report = [];
|
|
118
136
|
let blockTokens = 0;
|
|
119
|
-
|
|
120
|
-
const part = formatMemoryRecallBlock([
|
|
121
|
-
{ content: h.memory.content, category: h.memory.category, score: h.score },
|
|
122
|
-
]);
|
|
137
|
+
const pushHit = (content, category, score, label) => {
|
|
138
|
+
const part = formatMemoryRecallBlock([{ content, category, score }]);
|
|
123
139
|
const partTokens = estimateBlockTokens(part);
|
|
124
140
|
if (maxTokens > 0 && blockTokens + partTokens > maxTokens)
|
|
125
|
-
|
|
141
|
+
return false;
|
|
126
142
|
parts.push(part);
|
|
127
|
-
report.push(` •
|
|
143
|
+
report.push(` • ${label} (${(score * 100).toFixed(0)}%): ${content.slice(0, 60).replace(/\n/g, " ")}…`);
|
|
128
144
|
blockTokens += partTokens;
|
|
145
|
+
return true;
|
|
146
|
+
};
|
|
147
|
+
for (const h of hits) {
|
|
148
|
+
if (!pushHit(h.memory.content, h.memory.category, h.score, `memory#${h.memory.id}`))
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
151
|
+
for (const h of crossHits) {
|
|
152
|
+
const repoLabel = h.repoId.split(/[\\/]/).filter(Boolean).pop() ?? h.repoId;
|
|
153
|
+
if (!pushHit(h.memory.content, h.memory.category, h.score, `memory#${h.memory.id} (from ${repoLabel})`))
|
|
154
|
+
break;
|
|
129
155
|
}
|
|
130
156
|
return { empty: parts.length === 0, block: parts.join("\n"), report };
|
|
131
157
|
}
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* memoryIndex.ts — cross-repo async vector index for durable memories (S24).
|
|
3
|
+
*
|
|
4
|
+
* A REDUNDANT, additive, ASYNC index layered over the authoritative node:sqlite
|
|
5
|
+
* `memories` table. The same-repo linear cosine scan over the in-repo memories
|
|
6
|
+
* (src/memoryRecall.ts) stays the DEFAULT recall path; this global PGlite index
|
|
7
|
+
* exists only to provide real cross-repo nearest-neighbor memory recall — so a
|
|
8
|
+
* decision you saved in repo A can be inlined as RAG context when you start a
|
|
9
|
+
* session in repo B. It is best-effort and non-fatal: any init/write failure
|
|
10
|
+
* degrades to the same-repo scan and must NEVER break memory write, recall, or
|
|
11
|
+
* extension load.
|
|
12
|
+
*
|
|
13
|
+
* PREVENT-PI-004: PGlite is WASM Postgres — fully local, zero network. Memory
|
|
14
|
+
* remains AUTHORITATIVE in SQLite; this index only holds (repo_id, memory_id,
|
|
15
|
+
* content, embedding) for NN lookup and is rebuilt from SQLite at any time.
|
|
16
|
+
*
|
|
17
|
+
* Topology mirrors vectorIndex.ts (Slice 2): ONE global PGlite DB, `repo_id` is
|
|
18
|
+
* a first-class column. `searchMemoriesAsync(q, k, {repoId?})` → omit repoId for
|
|
19
|
+
* cross-repo NN, pass repoId to scope to a single repo. Hit content is stored
|
|
20
|
+
* inline because the recall process cannot open every other repo's SQLite dir.
|
|
21
|
+
*/
|
|
22
|
+
import { homedir } from "node:os";
|
|
23
|
+
import { join } from "node:path";
|
|
24
|
+
import { mkdirSync, rmSync, existsSync } from "node:fs";
|
|
25
|
+
// PGlite + pgvector are script-free WASM (no native build) → survive pi's
|
|
26
|
+
// install-script block. Imported lazily so a missing/broken package degrades
|
|
27
|
+
// gracefully instead of crashing module load.
|
|
28
|
+
import { PGlite } from "@electric-sql/pglite";
|
|
29
|
+
import { vector } from "@electric-sql/pglite-pgvector";
|
|
30
|
+
/** Vector dimension produced by the default TrigramEmbedder (src/embedder.ts). */
|
|
31
|
+
export const MEMORY_INDEX_DIM = 512;
|
|
32
|
+
let db;
|
|
33
|
+
let initPromise;
|
|
34
|
+
let disabled = false;
|
|
35
|
+
let warned = false;
|
|
36
|
+
function indexDir() {
|
|
37
|
+
const override = process.env.MEGACOMPACT_INDEX_DIR;
|
|
38
|
+
if (override && override.trim() !== "")
|
|
39
|
+
return join(override, "memory");
|
|
40
|
+
try {
|
|
41
|
+
return join(homedir(), ".pi", "mega-compact-vector", "memory");
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
return join("/tmp", ".mega-compact-vector", "memory");
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
function logWarn(msg) {
|
|
48
|
+
// Never throw — degradation is the whole point. One warning per process.
|
|
49
|
+
if (warned)
|
|
50
|
+
return;
|
|
51
|
+
warned = true;
|
|
52
|
+
try {
|
|
53
|
+
console.warn(`[mega-compact:memoryIndex] ${msg} (falling back to same-repo scan)`);
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
/* ignore */
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/** Honor the emergency kill-switch (shared with the checkpoint index). */
|
|
60
|
+
export function isMemoryIndexDisabled() {
|
|
61
|
+
return (disabled ||
|
|
62
|
+
process.env.MEGACOMPACT_PGLITE_DISABLED === "true" ||
|
|
63
|
+
process.env.MEGACOMPACT_PGLITE_DISABLED === "1");
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Lazily open + schema-init the global PGlite DB. Idempotent and safe to call
|
|
67
|
+
* from many places. Returns undefined when disabled/unavailable so callers can
|
|
68
|
+
* fall back to the synchronous scan. Never throws.
|
|
69
|
+
*/
|
|
70
|
+
export function initMemoryIndex() {
|
|
71
|
+
if (isMemoryIndexDisabled())
|
|
72
|
+
return Promise.resolve(undefined);
|
|
73
|
+
if (db)
|
|
74
|
+
return Promise.resolve(db);
|
|
75
|
+
if (initPromise)
|
|
76
|
+
return initPromise;
|
|
77
|
+
initPromise = openPgLite(/* retryOnCorrupt */ true);
|
|
78
|
+
return initPromise;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level abort
|
|
82
|
+
* (typically from a corrupted/torn data dir) triggers a delete + one retry.
|
|
83
|
+
*/
|
|
84
|
+
async function openPgLite(retryOnCorrupt) {
|
|
85
|
+
try {
|
|
86
|
+
const dir = indexDir();
|
|
87
|
+
mkdirSync(dir, { recursive: true });
|
|
88
|
+
const pg = await new PGlite({
|
|
89
|
+
dataDir: dir,
|
|
90
|
+
extensions: { vector },
|
|
91
|
+
});
|
|
92
|
+
await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
|
|
93
|
+
await pg.exec(`
|
|
94
|
+
CREATE TABLE IF NOT EXISTS memory_index (
|
|
95
|
+
repo_id TEXT NOT NULL,
|
|
96
|
+
memory_id INTEGER NOT NULL,
|
|
97
|
+
content TEXT NOT NULL,
|
|
98
|
+
embedding vector(${MEMORY_INDEX_DIM}) NOT NULL,
|
|
99
|
+
PRIMARY KEY (repo_id, memory_id)
|
|
100
|
+
);
|
|
101
|
+
`);
|
|
102
|
+
await pg.exec("CREATE INDEX IF NOT EXISTS memory_index_hnsw ON memory_index USING hnsw (embedding vector_cosine_ops);");
|
|
103
|
+
db = pg;
|
|
104
|
+
return pg;
|
|
105
|
+
}
|
|
106
|
+
catch (err) {
|
|
107
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
108
|
+
if (retryOnCorrupt && (msg.includes("Aborted") || msg.includes("RuntimeError"))) {
|
|
109
|
+
try {
|
|
110
|
+
const dir = indexDir();
|
|
111
|
+
if (existsSync(dir))
|
|
112
|
+
rmSync(dir, { recursive: true, force: true });
|
|
113
|
+
initPromise = undefined;
|
|
114
|
+
return openPgLite(/* retryOnCorrupt */ false);
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
/* self-heal failed — fall through to disable */
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
disabled = true;
|
|
121
|
+
logWarn(`init failed: ${msg}`);
|
|
122
|
+
return undefined;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
function toVectorLiteral(v) {
|
|
126
|
+
const parts = v.map((x) => (Number.isFinite(x) ? x : 0));
|
|
127
|
+
return `[${parts.join(",")}]`;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Best-effort upsert of one memory embedding into the global index.
|
|
131
|
+
* Dimension-mismatched vectors (e.g. a BYO embedder with dim ≠ 512) are skipped.
|
|
132
|
+
* Fire-and-forget: callers must NOT await this on the sync write path. Never
|
|
133
|
+
* throws. `content` is stored inline so cross-repo recall can read it directly.
|
|
134
|
+
*/
|
|
135
|
+
export async function upsertMemoryEmbedding(repoId, memoryId, content, embedding) {
|
|
136
|
+
if (isMemoryIndexDisabled())
|
|
137
|
+
return;
|
|
138
|
+
if (!embedding || embedding.length !== MEMORY_INDEX_DIM)
|
|
139
|
+
return;
|
|
140
|
+
try {
|
|
141
|
+
const pg = await initMemoryIndex();
|
|
142
|
+
if (!pg)
|
|
143
|
+
return;
|
|
144
|
+
const lit = toVectorLiteral(embedding);
|
|
145
|
+
await pg.query(`INSERT INTO memory_index (repo_id, memory_id, content, embedding)
|
|
146
|
+
VALUES ($1, $2, $3, $4::vector)
|
|
147
|
+
ON CONFLICT (repo_id, memory_id)
|
|
148
|
+
DO UPDATE SET content = EXCLUDED.content, embedding = EXCLUDED.embedding;`, [repoId, memoryId, content, lit]);
|
|
149
|
+
}
|
|
150
|
+
catch (err) {
|
|
151
|
+
disabled = true;
|
|
152
|
+
logWarn(`upsert failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Cross-repo (or single-repo) HNSW nearest-neighbor memory search. Returns hits
|
|
157
|
+
* sorted by descending similarity. Never throws — on any failure returns [].
|
|
158
|
+
*/
|
|
159
|
+
export async function searchMemoriesAsync(query, opts = {}) {
|
|
160
|
+
if (isMemoryIndexDisabled() || !query || query.length !== MEMORY_INDEX_DIM)
|
|
161
|
+
return [];
|
|
162
|
+
const k = opts.k ?? 5;
|
|
163
|
+
const repoId = opts.repoId;
|
|
164
|
+
try {
|
|
165
|
+
const pg = await initMemoryIndex();
|
|
166
|
+
if (!pg)
|
|
167
|
+
return [];
|
|
168
|
+
const lit = toVectorLiteral(query);
|
|
169
|
+
const params = [lit, k];
|
|
170
|
+
let sql = "SELECT repo_id, memory_id, content, 1 - (embedding <=> $1::vector) AS score " +
|
|
171
|
+
"FROM memory_index";
|
|
172
|
+
if (repoId) {
|
|
173
|
+
sql += " WHERE repo_id = $3";
|
|
174
|
+
params.push(repoId);
|
|
175
|
+
}
|
|
176
|
+
sql += " ORDER BY embedding <=> $1::vector LIMIT $2";
|
|
177
|
+
const res = await pg.query(sql, params);
|
|
178
|
+
return res.rows.map((r) => ({
|
|
179
|
+
repoId: r.repo_id,
|
|
180
|
+
memoryId: Number(r.memory_id),
|
|
181
|
+
content: r.content,
|
|
182
|
+
score: r.score,
|
|
183
|
+
}));
|
|
184
|
+
}
|
|
185
|
+
catch (err) {
|
|
186
|
+
disabled = true;
|
|
187
|
+
logWarn(`search failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
188
|
+
return [];
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
/** Close the index (test teardown / shutdown). Safe to call when unopened. */
|
|
192
|
+
export async function closeMemoryIndex() {
|
|
193
|
+
if (db) {
|
|
194
|
+
try {
|
|
195
|
+
await db.close();
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
/* ignore */
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
db = undefined;
|
|
202
|
+
initPromise = undefined;
|
|
203
|
+
disabled = false;
|
|
204
|
+
warned = false;
|
|
205
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { defaultEmbedder } from "../embedder.js";
|
|
7
|
+
import { upsertMemoryEmbedding, searchMemoriesAsync, initMemoryIndex, closeMemoryIndex, isMemoryIndexDisabled, } from "./memoryIndex.js";
|
|
8
|
+
const baseTmp = mkdtempSync(join(tmpdir(), "mc-memidx-"));
|
|
9
|
+
test("memoryIndex: disabled when MEGACOMPACT_PGLITE_DISABLED", async () => {
|
|
10
|
+
process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
|
|
11
|
+
try {
|
|
12
|
+
assert.equal(isMemoryIndexDisabled(), true, "kill-switch honored");
|
|
13
|
+
const hits = await searchMemoriesAsync(defaultEmbedder().embed("anything"), { k: 3 });
|
|
14
|
+
assert.deepEqual(hits, [], "search returns [] when disabled");
|
|
15
|
+
}
|
|
16
|
+
finally {
|
|
17
|
+
delete process.env.MEGACOMPACT_PGLITE_DISABLED;
|
|
18
|
+
}
|
|
19
|
+
});
|
|
20
|
+
test("memoryIndex: cross-repo upsert + NN search returns the right repo's memory", async () => {
|
|
21
|
+
// Isolate the global PGlite dir so concurrent test runs don't collide.
|
|
22
|
+
process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "index");
|
|
23
|
+
const repoA = "/tmp/repo-a";
|
|
24
|
+
const repoB = "/tmp/repo-b";
|
|
25
|
+
try {
|
|
26
|
+
await initMemoryIndex();
|
|
27
|
+
// Two memories in different repos, with clearly distinct content so their
|
|
28
|
+
// trigram embeddings separate.
|
|
29
|
+
const vecA = defaultEmbedder().embed("We standardized on node:sqlite for the store backend");
|
|
30
|
+
const vecB = defaultEmbedder().embed("The deployment target is a raspberry pi in the closet");
|
|
31
|
+
await upsertMemoryEmbedding(repoA, 1, "We standardized on node:sqlite for the store backend", vecA);
|
|
32
|
+
await upsertMemoryEmbedding(repoB, 7, "The deployment target is a raspberry pi in the closet", vecB);
|
|
33
|
+
// Query close to A's content → top hit should be A's memory, not B's.
|
|
34
|
+
const q = defaultEmbedder().embed("standardized node:sqlite store backend choice");
|
|
35
|
+
const hits = await searchMemoriesAsync(q, { k: 3 });
|
|
36
|
+
assert.ok(hits.length >= 1, "at least one hit");
|
|
37
|
+
assert.equal(hits[0].repoId, repoA, "nearest neighbor is repo A");
|
|
38
|
+
assert.equal(hits[0].memoryId, 1, "correct memory id");
|
|
39
|
+
assert.ok(hits[0].score > 0.5, "high cosine for the matching memory");
|
|
40
|
+
// Scope to repoB only → A must not appear.
|
|
41
|
+
const scoped = await searchMemoriesAsync(q, { k: 3, repoId: repoB });
|
|
42
|
+
assert.ok(scoped.every((h) => h.repoId === repoB), "scoped search stays within repoB");
|
|
43
|
+
}
|
|
44
|
+
finally {
|
|
45
|
+
await closeMemoryIndex();
|
|
46
|
+
delete process.env.MEGACOMPACT_INDEX_DIR;
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
test("memoryIndex: cleanup", () => {
|
|
50
|
+
rmSync(baseTmp, { recursive: true, force: true });
|
|
51
|
+
});
|
|
@@ -11,6 +11,8 @@ import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-a
|
|
|
11
11
|
import { detectConflicts, type ConflictReport } from "./conflict-scan.js";
|
|
12
12
|
import { addMemory, listMemories, searchMemories, recallMemory, type MemoryRecord } from "../src/store/sqlite.js";
|
|
13
13
|
import { resolveRepoRoot } from "./mega-config.js";
|
|
14
|
+
import { defaultEmbedder } from "../src/embedder.js";
|
|
15
|
+
import { upsertMemoryEmbedding } from "../src/store/memoryIndex.js";
|
|
14
16
|
import { MegaRuntime } from "./mega-runtime.js";
|
|
15
17
|
|
|
16
18
|
/** Run the conflict scan and format a human-readable report. */
|
|
@@ -82,6 +84,13 @@ export function registerConflictCommands(pi: ExtensionAPI, runtime: MegaRuntime)
|
|
|
82
84
|
const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
|
|
83
85
|
const content = text.replace(/#[\w-]+/g, "").trim();
|
|
84
86
|
const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
|
|
87
|
+
// S24: mirror into the cross-repo memory index (fire-and-forget).
|
|
88
|
+
try {
|
|
89
|
+
const vec = defaultEmbedder().embed(content);
|
|
90
|
+
void upsertMemoryEmbedding(repo, id, content, vec);
|
|
91
|
+
} catch {
|
|
92
|
+
/* non-fatal */
|
|
93
|
+
}
|
|
85
94
|
ctx.ui.notify(`[mega-memory] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
|
|
86
95
|
return;
|
|
87
96
|
}
|
|
@@ -151,6 +160,12 @@ export function registerConflictCommands(pi: ExtensionAPI, runtime: MegaRuntime)
|
|
|
151
160
|
const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
|
|
152
161
|
const content = text.replace(/#[\w-]+/g, "").trim();
|
|
153
162
|
const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
|
|
163
|
+
try {
|
|
164
|
+
const vec = defaultEmbedder().embed(content);
|
|
165
|
+
void upsertMemoryEmbedding(repo, id, content, vec);
|
|
166
|
+
} catch {
|
|
167
|
+
/* non-fatal */
|
|
168
|
+
}
|
|
154
169
|
ctx.ui.notify(`[/m] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
|
|
155
170
|
return;
|
|
156
171
|
}
|
|
@@ -19,8 +19,18 @@ import { driveNativeCompaction } from "./mega-compact-driver.js";
|
|
|
19
19
|
import { computeLiveTrimCut, liveTrimSummaryMessage } from "./mega-trim.js";
|
|
20
20
|
import { pressureFromPct, memoryReviewCadence, type MegaConfig } from "./mega-config.js";
|
|
21
21
|
|
|
22
|
+
/**
|
|
23
|
+
* DIAG accessor for the headless test harness: the most recently constructed
|
|
24
|
+
* MegaRuntime, so a test that loads the compiled extension via its default
|
|
25
|
+
* export can read diag counters (diagLiveTrimFires / diagBeforeCompactFires /
|
|
26
|
+
* diagBeforeCompactSupplied / diagAgentEndIdle) after firing synthetic events.
|
|
27
|
+
* No-op in production — nothing reads this outside tests.
|
|
28
|
+
*/
|
|
29
|
+
export let lastRuntime: MegaRuntime | undefined;
|
|
30
|
+
|
|
22
31
|
/** Register all pi lifecycle event handlers. */
|
|
23
32
|
export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, config: MegaConfig): void {
|
|
33
|
+
lastRuntime = runtime;
|
|
24
34
|
// ---- Session lifecycle (state reset points) -------------------------------
|
|
25
35
|
// Capture model/provider whenever it changes (drives real cost estimation).
|
|
26
36
|
pi.on("model_select", async (_event, ctx) => {
|
|
@@ -61,6 +71,8 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
|
|
|
61
71
|
try {
|
|
62
72
|
const mr = await recallMemoriesAndInline({
|
|
63
73
|
query, stateDir: runtime.getStateDir(), limit: 5,
|
|
74
|
+
crossRepo: config.crossRepoEnabled,
|
|
75
|
+
crossRepoCosine: config.crossRepoCosine,
|
|
64
76
|
});
|
|
65
77
|
if (!mr.empty) runtime.pendingMemoryRecallBlock = mr.block;
|
|
66
78
|
} catch (err) {
|
|
@@ -86,7 +98,7 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
|
|
|
86
98
|
}
|
|
87
99
|
// S21: parallel memory recall. Trigram embedder is sub-ms; await is fine.
|
|
88
100
|
try {
|
|
89
|
-
const mr = await recallMemoriesAndInline({ query, stateDir: runtime.getStateDir(), limit: 5 });
|
|
101
|
+
const mr = await recallMemoriesAndInline({ query, stateDir: runtime.getStateDir(), limit: 5, crossRepo: config.crossRepoEnabled, crossRepoCosine: config.crossRepoCosine });
|
|
90
102
|
if (!mr.empty) runtime.pendingMemoryRecallBlock = mr.block;
|
|
91
103
|
} catch (err) {
|
|
92
104
|
runtime.logger.warn("memory-recall skipped", { err: String(err) });
|
|
@@ -143,6 +155,46 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
|
|
|
143
155
|
const idle = ctx.isIdle?.() ?? true;
|
|
144
156
|
const queued = ctx.hasPendingMessages?.() ?? false;
|
|
145
157
|
const now = Date.now();
|
|
158
|
+
// DIAG (team-run relief): surface whether the agent is idle + over
|
|
159
|
+
// threshold at agent_end so we can see if a mid-run durable-trim trigger
|
|
160
|
+
// *should* have fired but didn't.
|
|
161
|
+
const overThreshold = (runtime.lastCtxTokens ?? 0) >= config.thresholdTokens;
|
|
162
|
+
runtime.diagAgentEndIdle++;
|
|
163
|
+
runtime.logger.info("agent-end-idle", {
|
|
164
|
+
sessionId: runtime.rt.sessionId,
|
|
165
|
+
idle,
|
|
166
|
+
queued,
|
|
167
|
+
overThreshold,
|
|
168
|
+
ctxPct: runtime.lastCtxPercent,
|
|
169
|
+
ctxTokens: runtime.lastCtxTokens,
|
|
170
|
+
thresholdTokens: config.thresholdTokens,
|
|
171
|
+
wouldNudge: idle && queued && now >= runtime.resumeNudgeUntil,
|
|
172
|
+
});
|
|
173
|
+
// S16+S24: MID-RUN DURABLE TRIM. During a long team run (sub-agents),
|
|
174
|
+
// pi's native durable compaction only fires from _checkCompaction at
|
|
175
|
+
// PARENT settle (agent-session.js:760/844), so the on-disk transcript +
|
|
176
|
+
// context meter balloon to ~150k and never relieve until the very end
|
|
177
|
+
// ("compacts but doesn't resume"). agent_end with activeAgents===0 is a
|
|
178
|
+
// SAFE, settled point: calling ctx.compact() here does NOT abort an
|
|
179
|
+
// in-flight turn (the S16 danger is only mid-turn). ctx.compact() runs
|
|
180
|
+
// pi's flow, which fires our session_before_compact handler to supply
|
|
181
|
+
// the durable trim (truncates the transcript from firstKeptEntryId).
|
|
182
|
+
// Guarded three ways: only when truly idle + over threshold, only when
|
|
183
|
+
// pi would actually compact (piCompactWouldNoop skips the user-facing
|
|
184
|
+
// no-op throw), and debounced (one durable trim per 2s) to avoid
|
|
185
|
+
// thrashing the transcript while sub-agents keep settling.
|
|
186
|
+
if (idle && overThreshold && now >= runtime.debounceUntil) {
|
|
187
|
+
if (!piCompactWouldNoop(ctx)) {
|
|
188
|
+
runtime.debounceUntil = now + 2000;
|
|
189
|
+
runtime.diagAgentEndDurable++;
|
|
190
|
+
runtime.logger.info("agent-end-durable-trigger", {
|
|
191
|
+
sessionId: runtime.rt.sessionId,
|
|
192
|
+
ctxTokens: runtime.lastCtxTokens,
|
|
193
|
+
thresholdTokens: config.thresholdTokens,
|
|
194
|
+
});
|
|
195
|
+
ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort
|
|
196
|
+
}
|
|
197
|
+
}
|
|
146
198
|
if (idle && queued && now >= runtime.resumeNudgeUntil) {
|
|
147
199
|
runtime.resumeNudgeUntil = now + 30_000;
|
|
148
200
|
pi.sendUserMessage("[mega-compact] continue from the compacted context above.");
|
|
@@ -215,21 +267,21 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
|
|
|
215
267
|
Math.round((pct / 100) * (usage?.contextWindow ?? 0));
|
|
216
268
|
|
|
217
269
|
// FAST GATE: token-based (tier threshold), not percentage-based.
|
|
218
|
-
if (currentTokens < config.thresholdTokens) return;
|
|
270
|
+
if (currentTokens < config.thresholdTokens) { runtime.diagCtxFastGate++; return; }
|
|
219
271
|
|
|
220
272
|
const check = autoCompactCheck(currentTokens, config.thresholdTokens); // SERVER-STYLE CONFIRM (local)
|
|
221
|
-
if (!check.shouldCompact) return;
|
|
273
|
+
if (!check.shouldCompact) { runtime.diagCtxNoCompact++; return; }
|
|
222
274
|
|
|
223
275
|
// Debounce so we don't fire on every context event past threshold.
|
|
224
276
|
const now = Date.now();
|
|
225
|
-
if (now < runtime.debounceUntil) return;
|
|
277
|
+
if (now < runtime.debounceUntil) { runtime.diagCtxDebounce++; return; }
|
|
226
278
|
runtime.debounceUntil = now + 2000;
|
|
227
279
|
|
|
228
280
|
// Adaptive compression (Fix E): scale compression strength + keepFrom depth
|
|
229
281
|
// with how close we are to the model context limit.
|
|
230
282
|
const pressure = pressureFromPct(pct);
|
|
231
283
|
const ran = runCompact(pi, runtime, config, ctx, messages, { compressionPressure: pressure });
|
|
232
|
-
if (ran.skipped) return;
|
|
284
|
+
if (ran.skipped) { runtime.diagCtxRunSkipped++; return; }
|
|
233
285
|
|
|
234
286
|
// LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
|
|
235
287
|
// manual compact path aborts the in-flight turn — only used behind the flag.
|
|
@@ -262,7 +314,16 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
|
|
|
262
314
|
summary: ran.result.summary,
|
|
263
315
|
anchorUserMessages,
|
|
264
316
|
});
|
|
265
|
-
if (cut === null)
|
|
317
|
+
if (cut === null) {
|
|
318
|
+
runtime.diagCtxCutNull++;
|
|
319
|
+
runtime.logger.info("live-trim-skip", {
|
|
320
|
+
sessionId: runtime.rt.sessionId,
|
|
321
|
+
compactedFrom: ran.result.compactedFrom,
|
|
322
|
+
viewLen: view.length,
|
|
323
|
+
anchorUserMessages,
|
|
324
|
+
});
|
|
325
|
+
return; // unsafe / below anchor floor — no trim this call
|
|
326
|
+
}
|
|
266
327
|
const summaryMsg = liveTrimSummaryMessage({
|
|
267
328
|
compactedFrom: ran.result.compactedFrom,
|
|
268
329
|
summary: ran.result.summary,
|
|
@@ -276,8 +337,22 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
|
|
|
276
337
|
} as unknown as AgentMessage;
|
|
277
338
|
const recent = messages.slice(cut); // guardrails-allow PREVENT-PI-002: `cut` is the pre-sanitized `compactedFrom` produced by src/boundary.ts computeDropRange, so the preserved run begins on a toolPair-safe index.
|
|
278
339
|
runtime.snapshot(ctx);
|
|
340
|
+
// DIAG (team-run relief): confirm the live trim actually fires + how big
|
|
341
|
+
// the window still is. The return is non-durable (per-LLM-call only), so
|
|
342
|
+
// this is the signal that the model is being fed a compacted view while
|
|
343
|
+
// the on-disk transcript + context meter keep growing.
|
|
344
|
+
runtime.diagLiveTrimFires++;
|
|
345
|
+
runtime.logger.info("live-trim", {
|
|
346
|
+
sessionId: runtime.rt.sessionId,
|
|
347
|
+
inputMsgs: messages.length,
|
|
348
|
+
outputMsgs: recent.length + 1,
|
|
349
|
+
compactedFrom: cut,
|
|
350
|
+
ctxPct: pct,
|
|
351
|
+
ctxTokens: usage?.tokens ?? null,
|
|
352
|
+
});
|
|
279
353
|
return { messages: [summaryAgentMsg, ...recent] };
|
|
280
354
|
} catch {
|
|
355
|
+
runtime.diagCtxThrown++;
|
|
281
356
|
return; // non-fatal: no trim this call; the next context event retries
|
|
282
357
|
}
|
|
283
358
|
});
|
|
@@ -290,10 +365,25 @@ export function registerEventHandlers(pi: ExtensionAPI, runtime: MegaRuntime, co
|
|
|
290
365
|
// there is no full-reload + additive recall inflation.
|
|
291
366
|
pi.on("session_before_compact", async (event: SessionBeforeCompactEvent, ctx: ExtensionContext) => {
|
|
292
367
|
runtime.resetRuntime(ctx.sessionManager.getSessionId());
|
|
368
|
+
// DIAG (team-run relief): this is the ONLY durable-trim entry point. Log
|
|
369
|
+
// every fire + whether we supplied a compaction (truncates transcript) or
|
|
370
|
+
// fell through to {} (pi runs its own). If this is sparse during a team
|
|
371
|
+
// run, the durable trim is firing too late (only at parent settle).
|
|
372
|
+
const prep = event.preparation;
|
|
373
|
+
runtime.diagBeforeCompactFires++;
|
|
374
|
+
runtime.logger.info("before-compact-entry", {
|
|
375
|
+
sessionId: runtime.rt.sessionId,
|
|
376
|
+
reason: event.reason,
|
|
377
|
+
hasPrep: !!prep,
|
|
378
|
+
msgsToSummarize: prep?.messagesToSummarize?.length ?? 0,
|
|
379
|
+
firstKeptEntryId: prep?.firstKeptEntryId ?? null,
|
|
380
|
+
activeAgents: runtime.activeAgents,
|
|
381
|
+
});
|
|
293
382
|
if (!config.auto) return {}; // let pi run its own native compaction
|
|
294
383
|
try {
|
|
295
384
|
const result = driveNativeCompaction(event, runtime, config);
|
|
296
385
|
if (result) {
|
|
386
|
+
runtime.diagBeforeCompactSupplied++;
|
|
297
387
|
runtime.logger.info("native-compact", {
|
|
298
388
|
sessionId: runtime.rt.sessionId,
|
|
299
389
|
firstKeptEntryId: result.compaction.firstKeptEntryId,
|
|
@@ -143,6 +143,27 @@ export class MegaRuntime {
|
|
|
143
143
|
lastCtxPercent: number | null = null;
|
|
144
144
|
lastCtxWindow = 0;
|
|
145
145
|
|
|
146
|
+
/**
|
|
147
|
+
* DIAG counters for the "team run doesn't relieve context" investigation.
|
|
148
|
+
* Plain integers, incremented at the three compaction decision points. They
|
|
149
|
+
* let a headless test drive the real event handlers and assert the firing
|
|
150
|
+
* cadence without scraping log files. Inert in production (the live-trim and
|
|
151
|
+
* before-compact probes also emit logger.info, but these counters are always
|
|
152
|
+
* updated and cost nothing).
|
|
153
|
+
*/
|
|
154
|
+
diagLiveTrimFires = 0; // context handler returned a trimmed view
|
|
155
|
+
diagBeforeCompactFires = 0; // session_before_compact handler entered
|
|
156
|
+
diagBeforeCompactSupplied = 0; // session_before_compact supplied our trim
|
|
157
|
+
diagAgentEndIdle = 0; // agent_end with activeAgents===0
|
|
158
|
+
diagAgentEndDurable = 0; // agent_end fired ctx.compact() (mid-run durable trim)
|
|
159
|
+
// Per-skip-path counters for the team-run diagnosis.
|
|
160
|
+
diagCtxFastGate = 0; // returned at token fast-gate (below threshold)
|
|
161
|
+
diagCtxNoCompact = 0; // autoCompactCheck().shouldCompact === false
|
|
162
|
+
diagCtxDebounce = 0; // debounceUntil not yet elapsed
|
|
163
|
+
diagCtxRunSkipped = 0; // runCompact() returned skipped
|
|
164
|
+
diagCtxCutNull = 0; // computeLiveTrimCut returned null (anchor/boundary)
|
|
165
|
+
diagCtxThrown = 0; // live-trim try threw (caught)
|
|
166
|
+
|
|
146
167
|
/**
|
|
147
168
|
* Live 0–1 pressure: how full the context window is relative to the compaction
|
|
148
169
|
* threshold. Computed from the most recent context event the runtime already
|