pi-mega-compact 0.4.27 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -2
- package/dist/extensions/dashboard-server.js +58 -2
- package/dist/extensions/dashboard-server.test.js +95 -3
- package/dist/extensions/mega-commands.js +25 -9
- package/dist/extensions/mega-compact.test.js +161 -29
- package/dist/extensions/mega-config.js +5 -0
- package/dist/extensions/mega-conflict-cmds.js +79 -0
- package/dist/extensions/mega-dashboard-cmds.js +6 -4
- package/dist/extensions/mega-events.js +145 -20
- package/dist/extensions/mega-pipeline.js +179 -1
- package/dist/extensions/mega-runtime.js +14 -0
- package/dist/extensions/mega-trim.js +48 -0
- package/dist/extensions/mega-trim.test.js +58 -0
- package/dist/src/config/dedup.js +1 -0
- package/dist/src/driftDetection.js +103 -0
- package/dist/src/driftDetection.test.js +87 -0
- package/dist/src/memory.js +147 -0
- package/dist/src/memory.test.js +41 -0
- package/dist/src/memoryConsolidate.test.js +38 -0
- package/dist/src/memoryOps.js +58 -0
- package/dist/src/memoryOps.test.js +41 -0
- package/dist/src/memoryRecall.js +60 -0
- package/dist/src/memoryRecall.test.js +92 -0
- package/dist/src/recall.js +70 -1
- package/dist/src/recall.test.js +69 -1
- package/dist/src/store/sqlite.js +127 -11
- package/dist/src/vectorStore.js +6 -1
- package/extensions/dashboard-server.test.ts +115 -3
- package/extensions/dashboard-server.ts +63 -2
- package/extensions/mega-commands.ts +24 -9
- package/extensions/mega-compact.test.ts +162 -29
- package/extensions/mega-config.ts +22 -0
- package/extensions/mega-conflict-cmds.ts +81 -0
- package/extensions/mega-dashboard-cmds.ts +6 -4
- package/extensions/mega-events.ts +139 -20
- package/extensions/mega-pipeline.ts +179 -1
- package/extensions/mega-runtime.ts +15 -0
- package/extensions/mega-trim.test.ts +64 -0
- package/extensions/mega-trim.ts +75 -0
- package/extensions/openclaw-mega-compact.ts +24 -9
- package/package.json +2 -2
- package/src/config/dedup.ts +2 -0
- package/src/driftDetection.test.ts +100 -0
- package/src/driftDetection.ts +136 -0
- package/src/memory.test.ts +46 -0
- package/src/memory.ts +164 -0
- package/src/memoryConsolidate.test.ts +47 -0
- package/src/memoryOps.test.ts +53 -0
- package/src/memoryOps.ts +75 -0
- package/src/memoryRecall.test.ts +100 -0
- package/src/memoryRecall.ts +83 -0
- package/src/recall.test.ts +77 -1
- package/src/recall.ts +94 -1
- package/src/store/sqlite.ts +188 -11
- package/src/store.ts +3 -0
- package/src/vectorStore.ts +10 -1
- package/dist/extensions/openclaw-mega-compact.js +0 -291
- package/dist/src/minilm.js +0 -92
- package/dist/src/wordpiece.js +0 -129
|
@@ -70,6 +70,11 @@ export function loadConfig() {
|
|
|
70
70
|
autoInlineK: envFlag("MEGACOMPACT_AUTO_INLINE_K", 3),
|
|
71
71
|
dedupSim: Number(process.env.MEGACOMPACT_DEDUP_SIM ?? "0.9"),
|
|
72
72
|
raptorEnabled: envBool("MEGACOMPACT_RAPTOR_ENABLED", true),
|
|
73
|
+
legacyDurableTrim: envBool("MEGACOMPACT_LEGACY_DURABLE_TRIM", false),
|
|
74
|
+
crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
|
|
75
|
+
crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
|
|
76
|
+
memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),
|
|
77
|
+
memoryReviewInterval: envFlag("MEGACOMPACT_MEMORY_REVIEW_INTERVAL", 10),
|
|
73
78
|
recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
|
|
74
79
|
windowDedupe: envBool("MEGACOMPACT_WINDOW_DEDUPE", true),
|
|
75
80
|
debug: envBool("MEGACOMPACT_DEBUG", false),
|
|
@@ -118,4 +118,83 @@ export function registerConflictCommands(pi, runtime) {
|
|
|
118
118
|
ctx.ui.notify(memoryLine(m));
|
|
119
119
|
},
|
|
120
120
|
});
|
|
121
|
+
// Shortform aliases — `m save "..."`, `m status`, `m list`, `m search <q>`,
|
|
122
|
+
// `m recall <id>`. Delegates to the same SQLite store so there's one source
|
|
123
|
+
// of truth. The /mega-memory command remains the canonical form.
|
|
124
|
+
pi.registerCommand("m", {
|
|
125
|
+
description: "Shortform alias for /mega-memory. Usage: /m save <text> | list | search <q> | recall <id> | status",
|
|
126
|
+
handler: async (args, ctx) => {
|
|
127
|
+
const repo = resolveRepoRoot(ctx.cwd) ?? runtime.currentStateDir;
|
|
128
|
+
const parts = args.trim().split(/\s+/);
|
|
129
|
+
const sub = parts[0]?.toLowerCase() ?? "list";
|
|
130
|
+
if (sub === "save") {
|
|
131
|
+
// Strip leading "save" so /m save "#foo bar" works the same as the
|
|
132
|
+
// canonical form. Then strip a balanced outer quote pair if the user
|
|
133
|
+
// wrote /m save "..." — common when the text contains spaces.
|
|
134
|
+
let text = args.trim().slice(4).trim();
|
|
135
|
+
const mq = text.match(/^["“](.*)["”]$/s);
|
|
136
|
+
if (mq)
|
|
137
|
+
text = mq[1].trim();
|
|
138
|
+
if (!text) {
|
|
139
|
+
ctx.ui.notify('[/m] usage: /m save "<text>" or /m save <text>');
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
|
|
143
|
+
const content = text.replace(/#[\w-]+/g, "").trim();
|
|
144
|
+
const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
|
|
145
|
+
ctx.ui.notify(`[/m] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
if (sub === "search") {
|
|
149
|
+
const q = parts.slice(1).join(" ").trim();
|
|
150
|
+
if (!q) {
|
|
151
|
+
ctx.ui.notify("[/m] usage: /m search <query>");
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
const hits = searchMemories(q, repo, 50, runtime.currentStateDir);
|
|
155
|
+
if (!hits.length) {
|
|
156
|
+
ctx.ui.notify("[/m] no memories match.");
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
for (const mem of hits)
|
|
160
|
+
ctx.ui.notify(memoryLine(mem));
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
if (sub === "recall") {
|
|
164
|
+
const id = Number(parts[1]);
|
|
165
|
+
if (!Number.isFinite(id) || parts[1] === undefined) {
|
|
166
|
+
ctx.ui.notify("[/m] usage: /m recall <id>");
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
if (recallMemory(id, runtime.currentStateDir)) {
|
|
170
|
+
const found = listMemories(repo, 1000, runtime.currentStateDir).find((mem) => mem.id === id);
|
|
171
|
+
ctx.ui.notify(found ? `[/m] ${memoryLine(found)}` : `[/m] recalled #${id}`);
|
|
172
|
+
}
|
|
173
|
+
else {
|
|
174
|
+
ctx.ui.notify(`[/m] #${id} not found.`);
|
|
175
|
+
}
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
if (sub === "status") {
|
|
179
|
+
const all = listMemories(repo, 1000, runtime.currentStateDir);
|
|
180
|
+
const byKind = all.reduce((acc, m) => {
|
|
181
|
+
acc[m.kind] = (acc[m.kind] ?? 0) + 1;
|
|
182
|
+
return acc;
|
|
183
|
+
}, {});
|
|
184
|
+
const kinds = Object.entries(byKind).map(([k, n]) => `${k}=${n}`).join(", ");
|
|
185
|
+
const head = `[/m] ${all.length} memory record(s) in ${repo.split(/[\\/]/).pop() ?? repo}`;
|
|
186
|
+
ctx.ui.notify(kinds ? `${head} (${kinds})` : head);
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
// default: list
|
|
190
|
+
const all = listMemories(repo, 50, runtime.currentStateDir);
|
|
191
|
+
if (!all.length) {
|
|
192
|
+
ctx.ui.notify("[/m] no saved memories yet. Use /m save <text>.");
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
ctx.ui.notify(`[/m] ${all.length} memory record(s):`);
|
|
196
|
+
for (const mem of all)
|
|
197
|
+
ctx.ui.notify(memoryLine(mem));
|
|
198
|
+
},
|
|
199
|
+
});
|
|
121
200
|
}
|
|
@@ -18,11 +18,13 @@ export function registerDashboardCommands(pi, runtime) {
|
|
|
18
18
|
// when we fall back to the .ts source outside node_modules; false when using
|
|
19
19
|
// the shipped compiled dist/extensions/dashboard-server.js).
|
|
20
20
|
let dashboardNeedsStrip = false;
|
|
21
|
-
// The dashboard server binds
|
|
22
|
-
// in dashboard-server.js
|
|
23
|
-
// readiness even when port.pid landed in a different
|
|
21
|
+
// The dashboard server binds a 10-port range starting at MEGACOMPACT_DASHBOARD_PORT
|
|
22
|
+
// (default 9320) — see TARGET_PORT/PORT_RANGE in dashboard-server.js. Probe each for
|
|
23
|
+
// a live /api/snapshot so we detect readiness even when port.pid landed in a different
|
|
24
|
+
// state dir than we poll. Configurable so tests can use a private, non-colliding range.
|
|
25
|
+
const DASH_BASE = Number(process.env.MEGACOMPACT_DASHBOARD_PORT ?? "9320");
|
|
24
26
|
async function findLivePort() {
|
|
25
|
-
for (let port =
|
|
27
|
+
for (let port = DASH_BASE; port <= DASH_BASE + 9; port++) {
|
|
26
28
|
try {
|
|
27
29
|
const res = await fetch(`http://localhost:${port}/api/snapshot`, { signal: AbortSignal.timeout(800) }); // guardrails-allow PREVENT-PI-004: localhost liveness probe of the dashboard server this extension spawned
|
|
28
30
|
if (res.ok)
|
|
@@ -10,8 +10,10 @@ import { normalizeSessionId } from "../src/store.js";
|
|
|
10
10
|
import { autoCompactCheck } from "../src/compact.js";
|
|
11
11
|
import { estimateSessionTokens } from "../src/tokens.js";
|
|
12
12
|
import { recentUserQuery, WIDGET_KEY } from "./mega-runtime.js";
|
|
13
|
-
import { runCompact, doRecall } from "./mega-pipeline.js";
|
|
13
|
+
import { runCompact, doRecall, doRecallAsync, piCompactWouldNoop } from "./mega-pipeline.js";
|
|
14
|
+
import { recallMemoriesAndInline } from "../src/recall.js";
|
|
14
15
|
import { driveNativeCompaction } from "./mega-compact-driver.js";
|
|
16
|
+
import { computeLiveTrimCut, liveTrimSummaryMessage } from "./mega-trim.js";
|
|
15
17
|
import { pressureFromPct } from "./mega-config.js";
|
|
16
18
|
/** Register all pi lifecycle event handlers. */
|
|
17
19
|
export function registerEventHandlers(pi, runtime, config) {
|
|
@@ -25,6 +27,8 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
25
27
|
runtime.resetRuntime(ctx.sessionManager.getSessionId());
|
|
26
28
|
runtime.captureModel(ctx); // best-effort: ctx.model may be set by session start
|
|
27
29
|
runtime.setStatus(ctx, config.auto ? "mega-compact: ready" : "mega-compact: manual only");
|
|
30
|
+
// S21: clear any stale memory block from a prior session.
|
|
31
|
+
runtime.pendingMemoryRecallBlock = undefined;
|
|
28
32
|
// Auto-inline on resume/fork/continue: stage the most relevant checkpoints
|
|
29
33
|
// so the next before_agent_start prepends them to the system prompt.
|
|
30
34
|
// Triggered whenever this session already has persisted checkpoints AND a
|
|
@@ -36,13 +40,29 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
36
40
|
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
37
41
|
const query = recentUserQuery(ctx);
|
|
38
42
|
if (query && runtime.store.stats(sid).checkpointCount > 0) {
|
|
39
|
-
|
|
43
|
+
// S17: use the async variant on resume so cross-repo HNSW recall can
|
|
44
|
+
// augment when this repo's store is thin. session_start is an async-safe
|
|
45
|
+
// point (unlike the mid-turn context handler, which stays sync).
|
|
46
|
+
const r = await doRecallAsync(runtime, config, ctx, query, "resume", { crossRepo: config.crossRepoEnabled });
|
|
40
47
|
if (!r.empty) {
|
|
41
48
|
runtime.pendingRecallBlock = r.block;
|
|
42
|
-
|
|
43
|
-
runtime.
|
|
49
|
+
const crossLabel = r.toInject.some((h) => h.repoId) ? " (cross-repo)" : "";
|
|
50
|
+
runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt${crossLabel}`);
|
|
51
|
+
runtime.logger.info("auto-inline", { reason: event.reason, query, injected: r.toInject.map((h) => h.checkpoint.checkpointId), crossRepo: r.toInject.some((h) => h.repoId) });
|
|
44
52
|
}
|
|
45
53
|
}
|
|
54
|
+
// S21: parallel memory recall. Same async context so we can await without
|
|
55
|
+
// breaking the handler contract. Best-effort — never throws.
|
|
56
|
+
try {
|
|
57
|
+
const mr = await recallMemoriesAndInline({
|
|
58
|
+
query, stateDir: runtime.getStateDir(), limit: 5,
|
|
59
|
+
});
|
|
60
|
+
if (!mr.empty)
|
|
61
|
+
runtime.pendingMemoryRecallBlock = mr.block;
|
|
62
|
+
}
|
|
63
|
+
catch (err) {
|
|
64
|
+
runtime.logger.warn("memory-recall skipped", { err: String(err) });
|
|
65
|
+
}
|
|
46
66
|
}
|
|
47
67
|
runtime.dashboard.event("session_start", { reason: event.reason, sessionId: runtime.rt.sessionId });
|
|
48
68
|
runtime.snapshot(ctx);
|
|
@@ -60,6 +80,15 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
60
80
|
runtime.pendingRecallBlock = r.block;
|
|
61
81
|
runtime.logger.info("auto-inline", { reason: "session_tree", query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
|
|
62
82
|
}
|
|
83
|
+
// S21: parallel memory recall. Trigram embedder is sub-ms; await is fine.
|
|
84
|
+
try {
|
|
85
|
+
const mr = await recallMemoriesAndInline({ query, stateDir: runtime.getStateDir(), limit: 5 });
|
|
86
|
+
if (!mr.empty)
|
|
87
|
+
runtime.pendingMemoryRecallBlock = mr.block;
|
|
88
|
+
}
|
|
89
|
+
catch (err) {
|
|
90
|
+
runtime.logger.warn("memory-recall skipped", { err: String(err) });
|
|
91
|
+
}
|
|
63
92
|
}
|
|
64
93
|
}
|
|
65
94
|
runtime.dashboard.event("session_tree", { sessionId: runtime.rt.sessionId });
|
|
@@ -68,11 +97,14 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
68
97
|
// ---- Auto-inline injection point: prepend staged recall to systemPrompt ----
|
|
69
98
|
pi.on("before_agent_start", async (event, ctx) => {
|
|
70
99
|
runtime.captureModel(ctx); // most reliable point ctx.model is populated
|
|
71
|
-
|
|
100
|
+
const cpBlock = runtime.pendingRecallBlock;
|
|
101
|
+
const memBlock = runtime.pendingMemoryRecallBlock;
|
|
102
|
+
if (!cpBlock && !memBlock)
|
|
72
103
|
return;
|
|
73
|
-
|
|
74
|
-
runtime.
|
|
75
|
-
|
|
104
|
+
runtime.pendingRecallBlock = undefined;
|
|
105
|
+
runtime.pendingMemoryRecallBlock = undefined;
|
|
106
|
+
const composed = [cpBlock, memBlock].filter(Boolean).join("\n\n");
|
|
107
|
+
return { systemPrompt: `${event.systemPrompt}\n\n${composed}` };
|
|
76
108
|
});
|
|
77
109
|
pi.on("session_shutdown", async (_event, ctx) => {
|
|
78
110
|
runtime.setStatus(ctx, undefined);
|
|
@@ -98,6 +130,24 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
98
130
|
else {
|
|
99
131
|
runtime.setStatus(ctx, config.auto ? "mega-compact: ready" : "mega-compact: manual only");
|
|
100
132
|
}
|
|
133
|
+
// S16 continuation fallback: if the turn settled idle right after a live-trim
|
|
134
|
+
// compaction AND there is queued work AND we haven't nudged recently, nudge
|
|
135
|
+
// once so the agent continues (the live trim should make this rare). Guarded
|
|
136
|
+
// to never busy-loop: one nudge per 30s, only when truly idle + queued.
|
|
137
|
+
if (config.auto && runtime.activeAgents === 0) {
|
|
138
|
+
try {
|
|
139
|
+
const idle = ctx.isIdle?.() ?? true;
|
|
140
|
+
const queued = ctx.hasPendingMessages?.() ?? false;
|
|
141
|
+
const now = Date.now();
|
|
142
|
+
if (idle && queued && now >= runtime.resumeNudgeUntil) {
|
|
143
|
+
runtime.resumeNudgeUntil = now + 30_000;
|
|
144
|
+
pi.sendUserMessage("[mega-compact] continue from the compacted context above.");
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
/* non-fatal: a failed nudge never blocks */
|
|
149
|
+
}
|
|
150
|
+
}
|
|
101
151
|
runtime.snapshot(ctx);
|
|
102
152
|
});
|
|
103
153
|
pi.on("turn_start", async (event, ctx) => {
|
|
@@ -108,16 +158,43 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
108
158
|
pi.on("turn_end", async (event, ctx) => {
|
|
109
159
|
runtime.dashboard.event("turn_end", { turnIndex: event.turnIndex });
|
|
110
160
|
runtime.snapshot(ctx);
|
|
161
|
+
// S20: auto-review the conversation every N turns and persist durable
|
|
162
|
+
// memories. Best-effort + non-fatal: a review failure must never break the
|
|
163
|
+
// agent loop. Debounced by memoryReviewInterval turns.
|
|
164
|
+
if (config.memoryAutoReview && runtime.currentTurn > 0 && runtime.currentTurn % config.memoryReviewInterval === 0) {
|
|
165
|
+
try {
|
|
166
|
+
const { reviewConversation } = await import("../src/memory.js");
|
|
167
|
+
const { applyMemoryOps } = await import("../src/memoryOps.js");
|
|
168
|
+
const entries = ctx.sessionManager.getEntries();
|
|
169
|
+
const view = runtime.engineView(entries.flatMap((e) => (e.message ? [e.message] : [])));
|
|
170
|
+
const ops = reviewConversation(view, []);
|
|
171
|
+
if (ops.length) {
|
|
172
|
+
await applyMemoryOps(ops, runtime.currentStateDir);
|
|
173
|
+
// S21.2: a memory op landed in this turn window. The pipeline reads
|
|
174
|
+
// this counter after a successful compaction and fires
|
|
175
|
+
// `consolidateMemories` only when it's > 0.
|
|
176
|
+
runtime.memoriesTouchedThisCompaction += ops.length;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
/* non-fatal — auto-review must not break the turn loop */
|
|
181
|
+
}
|
|
182
|
+
}
|
|
111
183
|
});
|
|
112
|
-
// ---- Auto-trigger:
|
|
113
|
-
//
|
|
114
|
-
//
|
|
115
|
-
//
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
// We
|
|
120
|
-
//
|
|
184
|
+
// ---- Auto-trigger: live trim (compact and continue) + native durable ----
|
|
185
|
+
// S16 redesign: we NO LONGER call ctx.compact() from the auto-trigger by
|
|
186
|
+
// default. That mapped to pi's MANUAL compaction path, which abort()s the
|
|
187
|
+
// in-flight turn (agent-session.js:1345) and stops the agent. Instead:
|
|
188
|
+
// - LIVE: return { messages: trimmedView } from the context event. This
|
|
189
|
+
// feeds pi's transformContext (sdk.js:226 → agent-loop.js:180) so the
|
|
190
|
+
// model sees a compacted window EVERY LLM call, with no abort. The turn
|
|
191
|
+
// continues. We persist our recall checkpoint (the durable value) first.
|
|
192
|
+
// - DURABLE: pi's NATIVE auto-compaction fires at agent-end
|
|
193
|
+
// (agent-session.js:1565), continues (return hasQueuedMessages()), and
|
|
194
|
+
// emits session_before_compact — where OUR driveNativeCompaction supplies
|
|
195
|
+
// the summary and pi truncates the transcript on disk. No ctx.compact().
|
|
196
|
+
// Legacy: MEGACOMPACT_LEGACY_DURABLE_TRIM=true restores the v0.4.28 ctx.compact
|
|
197
|
+
// path (kept one release as rollback).
|
|
121
198
|
pi.on("context", async (event, ctx) => {
|
|
122
199
|
if (!config.auto)
|
|
123
200
|
return;
|
|
@@ -151,9 +228,57 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
151
228
|
const ran = runCompact(pi, runtime, config, ctx, messages, { compressionPressure: pressure });
|
|
152
229
|
if (ran.skipped)
|
|
153
230
|
return;
|
|
154
|
-
//
|
|
155
|
-
//
|
|
156
|
-
|
|
231
|
+
// LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
|
|
232
|
+
// manual compact path aborts the in-flight turn — only used behind the flag.
|
|
233
|
+
// Read live from env (in addition to the load-time config) so the flag can be
|
|
234
|
+
// toggled per-test without reloading the module; config.legacyDurableTrim is
|
|
235
|
+
// the cached default. (Mirrors how piCompactWouldNoop re-reads its floor.)
|
|
236
|
+
const legacy = config.legacyDurableTrim || process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "true" || process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "1";
|
|
237
|
+
if (legacy) {
|
|
238
|
+
if (piCompactWouldNoop(ctx))
|
|
239
|
+
return;
|
|
240
|
+
ctx.compact({ customInstructions: undefined });
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
// S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
|
|
244
|
+
// Non-destructive: pi keeps the real transcript; only this LLM call sees the
|
|
245
|
+
// trimmed window. We compute the cut on the engine view (pure, tested) then
|
|
246
|
+
// slice the ORIGINAL pi AgentMessage[] from that index (lossless alignment,
|
|
247
|
+
// mirroring dropCompactedRange) and prepend a user-role summary message.
|
|
248
|
+
// A build failure or unsafe cut returns nothing (no trim this call — the
|
|
249
|
+
// next context event retries). The anchor floor is read live from env (the
|
|
250
|
+
// config value is the cached default) so it can be tuned per-test / per-run
|
|
251
|
+
// without reloading the module.
|
|
252
|
+
try {
|
|
253
|
+
const anchorEnv = process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES;
|
|
254
|
+
const anchorUserMessages = (anchorEnv != null && anchorEnv !== "" && Number.isFinite(Number(anchorEnv)))
|
|
255
|
+
? Number(anchorEnv)
|
|
256
|
+
: config.anchorUserMessages;
|
|
257
|
+
const cut = computeLiveTrimCut(view, {
|
|
258
|
+
compactedFrom: ran.result.compactedFrom,
|
|
259
|
+
summary: ran.result.summary,
|
|
260
|
+
anchorUserMessages,
|
|
261
|
+
});
|
|
262
|
+
if (cut === null)
|
|
263
|
+
return; // unsafe / below anchor floor — no trim this call
|
|
264
|
+
const summaryMsg = liveTrimSummaryMessage({
|
|
265
|
+
compactedFrom: ran.result.compactedFrom,
|
|
266
|
+
summary: ran.result.summary,
|
|
267
|
+
anchorUserMessages: config.anchorUserMessages,
|
|
268
|
+
});
|
|
269
|
+
// Synthesize a user-role AgentMessage carrying the compacted summary.
|
|
270
|
+
const summaryAgentMsg = {
|
|
271
|
+
role: "user",
|
|
272
|
+
content: summaryMsg.text,
|
|
273
|
+
timestamp: Date.now(),
|
|
274
|
+
};
|
|
275
|
+
const recent = messages.slice(cut); // guardrails-allow PREVENT-PI-002: `cut` is the pre-sanitized `compactedFrom` produced by src/boundary.ts computeDropRange, so the preserved run begins on a toolPair-safe index.
|
|
276
|
+
runtime.snapshot(ctx);
|
|
277
|
+
return { messages: [summaryAgentMsg, ...recent] };
|
|
278
|
+
}
|
|
279
|
+
catch {
|
|
280
|
+
return; // non-fatal: no trim this call; the next context event retries
|
|
281
|
+
}
|
|
157
282
|
});
|
|
158
283
|
// ---- Supply a DURABLE trim to pi's native compaction (Fix B) ----------
|
|
159
284
|
// We run the Trident pipeline to produce a compressed summary, then return
|
|
@@ -8,9 +8,11 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
10
10
|
import { compactSession } from "../src/engine.js";
|
|
11
|
-
import { recallAndInline } from "../src/recall.js";
|
|
11
|
+
import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "../src/recall.js";
|
|
12
12
|
import { normalizeSessionId } from "../src/store.js";
|
|
13
|
+
import { estimateBlockTokens } from "../src/tokens.js";
|
|
13
14
|
import { touchSession, logDaily } from "../src/store/sqlite.js";
|
|
15
|
+
import { consolidateMemories } from "../src/memory.js";
|
|
14
16
|
import { C, MARKER_TYPE, } from "./mega-runtime.js";
|
|
15
17
|
import { resolveRepoRoot, preserveRecentForPressure } from "./mega-config.js";
|
|
16
18
|
import { runRaptor } from "../src/dedup/raptor/index.js";
|
|
@@ -41,6 +43,10 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
|
|
|
41
43
|
}
|
|
42
44
|
function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
|
|
43
45
|
runtime.pulsing = true; // animate the status line while the (sync) pipeline runs
|
|
46
|
+
// S21.2: reset the per-compaction memory-op counter so the post-compact
|
|
47
|
+
// consolidate pass only fires when memory rows actually changed during the
|
|
48
|
+
// compaction window (turn_end → auto-review may have written some).
|
|
49
|
+
runtime.memoriesTouchedThisCompaction = 0;
|
|
44
50
|
const result = compactSession({
|
|
45
51
|
sessionId: sid,
|
|
46
52
|
messages: view,
|
|
@@ -107,6 +113,26 @@ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
|
|
|
107
113
|
catch {
|
|
108
114
|
/* non-fatal: stats bookkeeping only */
|
|
109
115
|
}
|
|
116
|
+
// S21.2: best-effort consolidation of near-duplicate memories for this repo.
|
|
117
|
+
// Runs after the per-repo stats touch so `consolidateMemories` can use the
|
|
118
|
+
// same stateDir. Non-fatal — a failed consolidate never blocks a compaction.
|
|
119
|
+
// Only runs when new memory ops landed in this pass (otherwise the prior
|
|
120
|
+
// compaction's consolidate already had its shot — re-running would just
|
|
121
|
+
// touch every row again with no merges).
|
|
122
|
+
if (!result.deduped && runtime.memoriesTouchedThisCompaction > 0) {
|
|
123
|
+
try {
|
|
124
|
+
const root = resolveRepoRoot(ctx.cwd);
|
|
125
|
+
void consolidateMemories(runtime.currentStateDir, root).then((n) => {
|
|
126
|
+
if (n > 0)
|
|
127
|
+
runtime.pushTicker(`${C.green}∫${C.reset} consolidated ${n} memory dup${n === 1 ? "" : "s"}`);
|
|
128
|
+
}, () => {
|
|
129
|
+
/* swallow: consolidate failures must never surface to the user */
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
/* non-fatal */
|
|
134
|
+
}
|
|
135
|
+
}
|
|
110
136
|
// Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
|
|
111
137
|
// skip re-vectorizing an already-compacted region (zero token cost).
|
|
112
138
|
pi.appendEntry(MARKER_TYPE, {
|
|
@@ -181,6 +207,100 @@ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
|
|
|
181
207
|
runtime.snapshot(ctx);
|
|
182
208
|
return { skipped: false, result, keepFrom, saved };
|
|
183
209
|
}
|
|
210
|
+
/**
|
|
211
|
+
* Predict whether pi's `ctx.compact()` would throw a no-op error — "Already
|
|
212
|
+
* compacted" or "Nothing to compact (session too small)" — so the auto-trigger
|
|
213
|
+
* can SKIP the call instead of surfacing a hard, user-facing error.
|
|
214
|
+
*
|
|
215
|
+
* Why we can't intercept or suppress it: pi's public `compact()` computes
|
|
216
|
+
* `prepareCompaction()` and throws *before* it emits `session_before_compact`,
|
|
217
|
+
* so our handler there never runs on the no-op path. And `ctx.compact()`'s
|
|
218
|
+
* `onError` callback fires only AFTER pi has already emitted a `compaction_end`
|
|
219
|
+
* event carrying the error message (which the interactive UI renders) — so
|
|
220
|
+
* `onError` cannot mute it either. The only robust fix is to not call
|
|
221
|
+
* `ctx.compact()` when pi would no-op. (pi's own `_runAutoCompaction` path is
|
|
222
|
+
* silent on this same condition; the public path we're forced through is the
|
|
223
|
+
* one that throws.)
|
|
224
|
+
*
|
|
225
|
+
* Skipping is correct, not a compromise: by the time this runs, `runCompact()`
|
|
226
|
+
* has already persisted the recall checkpoint (Path A). The durable on-disk
|
|
227
|
+
* trim is only useful when pi can actually summarize a region; a transcript
|
|
228
|
+
* under pi's `keepRecentTokens` budget is small enough that reloading it on
|
|
229
|
+
* resume isn't a token-growth problem, so the durable trim is unnecessary
|
|
230
|
+
* there anyway.
|
|
231
|
+
*
|
|
232
|
+
* Mirrors pi's `prepareCompaction()` return-undefined conditions (compaction.js):
|
|
233
|
+
* (1) last entry is a compaction → "Already compacted"
|
|
234
|
+
* (2) <2 cut-point messages since the last compaction → nothing to summarize
|
|
235
|
+
* (a cut point = any non-toolResult message — user/assistant/bash/custom/
|
|
236
|
+
* branchSummary/compactionSummary — matching pi's isCutPointMessage)
|
|
237
|
+
* (3) transcript tokens since the last compaction < keepRecentTokens → pi
|
|
238
|
+
* keeps everything → nothing to summarize
|
|
239
|
+
* `keepRecentTokens` isn't readable from the extension API, so (3) uses the pi
|
|
240
|
+
* default (20000) as a conservative floor; raise it via
|
|
241
|
+
* `MEGACOMPACT_DURABLE_TRIM_FLOOR` if you raise pi's `compact.keepRecentTokens`.
|
|
242
|
+
*
|
|
243
|
+
* Best-effort: on any read error returns true (skip) — skipping a durable trim
|
|
244
|
+
* is always safe; calling `ctx.compact()` on a no-op throws to the user.
|
|
245
|
+
*/
|
|
246
|
+
export function piCompactWouldNoop(ctx) {
|
|
247
|
+
try {
|
|
248
|
+
const branch = ctx.sessionManager.getBranch();
|
|
249
|
+
if (branch.length === 0)
|
|
250
|
+
return true;
|
|
251
|
+
// (1) already compacted — pi throws "Already compacted"
|
|
252
|
+
if (branch[branch.length - 1].type === "compaction")
|
|
253
|
+
return true;
|
|
254
|
+
// boundaryStart = index just after the most recent compaction entry (or 0)
|
|
255
|
+
let boundaryStart = 0;
|
|
256
|
+
for (let i = branch.length - 1; i >= 0; i--) {
|
|
257
|
+
if (branch[i].type === "compaction") {
|
|
258
|
+
boundaryStart = i + 1;
|
|
259
|
+
break;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
let cutPoints = 0;
|
|
263
|
+
let tokens = 0;
|
|
264
|
+
for (let i = boundaryStart; i < branch.length; i++) {
|
|
265
|
+
const e = branch[i];
|
|
266
|
+
if (e.type === "compaction")
|
|
267
|
+
continue;
|
|
268
|
+
let isCut = false;
|
|
269
|
+
for (const m of sessionEntryToContextMessages(e)) {
|
|
270
|
+
// pi's isCutPointMessage: every role except toolResult
|
|
271
|
+
if (m.role !== "toolResult")
|
|
272
|
+
isCut = true;
|
|
273
|
+
const c = m.content;
|
|
274
|
+
const text = typeof c === "string" ? c
|
|
275
|
+
: Array.isArray(c)
|
|
276
|
+
? c.map((b) => b?.text ?? "").join(" ")
|
|
277
|
+
: "";
|
|
278
|
+
if (text)
|
|
279
|
+
tokens += estimateBlockTokens(text);
|
|
280
|
+
}
|
|
281
|
+
if (isCut)
|
|
282
|
+
cutPoints++;
|
|
283
|
+
}
|
|
284
|
+
// (2) need >=2 cut points so the kept cut isn't the first message
|
|
285
|
+
if (cutPoints < 2)
|
|
286
|
+
return true;
|
|
287
|
+
// (3) transcript under pi's keepRecentTokens budget → pi keeps everything
|
|
288
|
+
if (tokens < durableTrimFloorTokens())
|
|
289
|
+
return true;
|
|
290
|
+
return false;
|
|
291
|
+
}
|
|
292
|
+
catch {
|
|
293
|
+
return true; // safe: skip the durable trim rather than risk a user-facing throw
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
/** pi's default keepRecentTokens (compaction settings). Override with
|
|
297
|
+
* MEGACOMPACT_DURABLE_TRIM_FLOOR if you raise pi's compact.keepRecentTokens. */
|
|
298
|
+
function durableTrimFloorTokens() {
|
|
299
|
+
const raw = process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
|
|
300
|
+
if (raw !== undefined && Number.isFinite(Number(raw)))
|
|
301
|
+
return Number(raw);
|
|
302
|
+
return 20_000;
|
|
303
|
+
}
|
|
184
304
|
/**
|
|
185
305
|
* Unified recall (Layer 5). The ONE path that injects. Returns the recall
|
|
186
306
|
* result; callers decide whether to stage it for before_agent_start (resume)
|
|
@@ -215,6 +335,64 @@ export function doRecall(runtime, config, ctx, query, source) {
|
|
|
215
335
|
}
|
|
216
336
|
return result;
|
|
217
337
|
}
|
|
338
|
+
/**
|
|
339
|
+
* S17: async recall with optional cross-repo augmentation. Used on resume
|
|
340
|
+
* (session_start) and /mega-recall --cross-repo — NEVER from the mid-turn
|
|
341
|
+
* context handler (that stays sync). Runs the sync same-repo scan first; if it
|
|
342
|
+
* returns < config.autoInlineK hits AND crossRepo is enabled, awaits the PGlite
|
|
343
|
+
* HNSW cross-repo path and merges (source-labeled, deduped by checkpointId). The
|
|
344
|
+
* recallMaxTokens cap + windowDedupe apply to the merged set so cross-repo can
|
|
345
|
+
* never net-inflate the window. Cross-repo uses a stricter cosine floor
|
|
346
|
+
* (config.crossRepoCosine) than same-repo. Non-fatal: any async failure returns
|
|
347
|
+
* the same-repo result unchanged.
|
|
348
|
+
*/
|
|
349
|
+
export async function doRecallAsync(runtime, config, ctx, query, source, opts = {}) {
|
|
350
|
+
runtime.bindRepo(ctx.cwd);
|
|
351
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
352
|
+
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
353
|
+
// Sync same-repo first (fast, never blocks).
|
|
354
|
+
const sameRepo = recallAndInline({
|
|
355
|
+
sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
|
|
356
|
+
recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
|
|
357
|
+
liveWindow, dedupSim: config.dedupSim,
|
|
358
|
+
}, runtime.store);
|
|
359
|
+
if (!config.crossRepoEnabled || !opts.crossRepo)
|
|
360
|
+
return sameRepo;
|
|
361
|
+
if (sameRepo.toInject.length >= config.autoInlineK)
|
|
362
|
+
return sameRepo; // same-repo satisfied
|
|
363
|
+
// Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
|
|
364
|
+
try {
|
|
365
|
+
const x = await recallAndInlineAsync({
|
|
366
|
+
sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
|
|
367
|
+
recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
|
|
368
|
+
liveWindow, dedupSim: config.crossRepoCosine, crossRepo: true,
|
|
369
|
+
globalIndexDir: process.env.MEGACOMPACT_INDEX_DIR,
|
|
370
|
+
}, runtime.store);
|
|
371
|
+
runtime.dashboard.event("recall-crossrepo", {
|
|
372
|
+
source, query: query.slice(0, 120), injected: x.toInject.length,
|
|
373
|
+
sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
|
|
374
|
+
});
|
|
375
|
+
// Merge, dedup by checkpointId, respect the same token cap by reformatting.
|
|
376
|
+
const seen = new Set(sameRepo.toInject.map((h) => h.checkpoint.checkpointId));
|
|
377
|
+
const merged = [...sameRepo.toInject];
|
|
378
|
+
for (const h of x.toInject) {
|
|
379
|
+
if (!seen.has(h.checkpoint.checkpointId)) {
|
|
380
|
+
merged.push(h);
|
|
381
|
+
seen.add(h.checkpoint.checkpointId);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
const block = merged.length ? formatRecallBlock(merged) : "";
|
|
385
|
+
return {
|
|
386
|
+
toInject: merged,
|
|
387
|
+
report: merged.map((h) => ` • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`),
|
|
388
|
+
block,
|
|
389
|
+
empty: merged.length === 0,
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
catch {
|
|
393
|
+
return sameRepo; // cross-repo failure → same-repo only (non-fatal)
|
|
394
|
+
}
|
|
395
|
+
}
|
|
218
396
|
/**
|
|
219
397
|
* Extract the live-window message texts from the session manager (Fix C),
|
|
220
398
|
* for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
|
|
@@ -57,12 +57,17 @@ export class MegaRuntime {
|
|
|
57
57
|
tokensSaved: 0,
|
|
58
58
|
};
|
|
59
59
|
debounceUntil = 0;
|
|
60
|
+
// S16: debounce for the agent_end resume nudge (avoid busy-loops).
|
|
61
|
+
resumeNudgeUntil = 0;
|
|
60
62
|
// Agent tracking for real-time widget updates
|
|
61
63
|
activeAgents = 0;
|
|
62
64
|
currentTurn = 0;
|
|
63
65
|
// Recall block produced by auto-inline (resume/branch) that the next
|
|
64
66
|
// before_agent_start should prepend to the system prompt. Unset after use.
|
|
65
67
|
pendingRecallBlock;
|
|
68
|
+
// S21: memory recall block, parallel to pendingRecallBlock. Same one-shot
|
|
69
|
+
// semantics; composed with the checkpoint block in before_agent_start.
|
|
70
|
+
pendingMemoryRecallBlock;
|
|
66
71
|
statusKey; // current status text for dashboard
|
|
67
72
|
// Active model/provider (for real cost estimation). Captured from ctx.model
|
|
68
73
|
// on model_select + session_start; persisted to SQL so cost + the dashboard
|
|
@@ -81,6 +86,11 @@ export class MegaRuntime {
|
|
|
81
86
|
TICKER_MAX = 5;
|
|
82
87
|
// Pulsing status: set true while a compaction is in flight, cleared on result.
|
|
83
88
|
pulsing = false;
|
|
89
|
+
// S21.2: set by `applyMemoryOps` when a memory add/replace/remove lands in
|
|
90
|
+
// the current compaction. The pipeline reads this after a successful compact
|
|
91
|
+
// to decide whether to fire `consolidateMemories` (skip the work entirely
|
|
92
|
+
// when no memory rows changed).
|
|
93
|
+
memoriesTouchedThisCompaction = 0;
|
|
84
94
|
// Rolling "saved" goal for the progress bar — grows as we save more, so the
|
|
85
95
|
// bar always has a meaningful denominator (never sits at 100% forever).
|
|
86
96
|
savedGoal = 50_000;
|
|
@@ -343,6 +353,10 @@ export class MegaRuntime {
|
|
|
343
353
|
}
|
|
344
354
|
catch { /* non-fatal: cost estimation degrades to model-in-memory only */ }
|
|
345
355
|
}
|
|
356
|
+
/** S21: state dir of the currently bound repo (where memories live). */
|
|
357
|
+
getStateDir() {
|
|
358
|
+
return this.currentStateDir;
|
|
359
|
+
}
|
|
346
360
|
/** Build the sync onTier callback that paints the live per-tier trace. */
|
|
347
361
|
makeTierCallback(ctx) {
|
|
348
362
|
const order = ["L0", "L1", "L2", "new"];
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { isBoundarySafe } from "../src/boundary.js";
|
|
2
|
+
import { formatCompactSummary } from "../src/compact.js";
|
|
3
|
+
/**
|
|
4
|
+
* Compute the safe cut index for the live trim. Snaps `compactedFrom` back to a
|
|
5
|
+
* boundary-safe index (PREVENT-PI-002: never start the preserved run on an
|
|
6
|
+
* orphaned tool result), and enforces the anchor floor (PREVENT-PI-001: keep at
|
|
7
|
+
* least `anchorUserMessages` user-role messages). Returns `null` when no trim is
|
|
8
|
+
* safe this call (empty summary, unsafe boundary, or below the anchor floor) so
|
|
9
|
+
* the caller keeps the original view and retries on the next context event.
|
|
10
|
+
*
|
|
11
|
+
* Exposed separately from `buildLiveTrimmedView` so the context handler can map
|
|
12
|
+
* the cut back onto the original pi `AgentMessage[]` (lossless index alignment,
|
|
13
|
+
* mirroring `dropCompactedRange` in src/adapt.ts).
|
|
14
|
+
*/
|
|
15
|
+
export function computeLiveTrimCut(view, opts) {
|
|
16
|
+
if (!opts.summary || !opts.summary.trim())
|
|
17
|
+
return null;
|
|
18
|
+
let cut = opts.compactedFrom;
|
|
19
|
+
while (cut > 0 && !isBoundarySafe(view, cut))
|
|
20
|
+
cut--;
|
|
21
|
+
if (cut <= 0)
|
|
22
|
+
return null; // nothing safe to cut — keep everything this call
|
|
23
|
+
const recent = view.slice(cut);
|
|
24
|
+
const userCount = recent.filter((m) => m.role === "user").length;
|
|
25
|
+
if (userCount < opts.anchorUserMessages)
|
|
26
|
+
return null;
|
|
27
|
+
return cut;
|
|
28
|
+
}
|
|
29
|
+
/** The formatted compacted-region summary as a user-role engine message. */
|
|
30
|
+
export function liveTrimSummaryMessage(opts) {
|
|
31
|
+
return {
|
|
32
|
+
role: "user",
|
|
33
|
+
text: formatCompactSummary(opts.summary),
|
|
34
|
+
toolName: undefined,
|
|
35
|
+
input: undefined,
|
|
36
|
+
output: undefined,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
/** Build the live trimmed view. Returns the original view if summary is empty
|
|
40
|
+
* or the boundary is unsafe (no trim this call — try next). Pure + tested. */
|
|
41
|
+
export function buildLiveTrimmedView(view, opts) {
|
|
42
|
+
const cut = computeLiveTrimCut(view, opts);
|
|
43
|
+
if (cut === null)
|
|
44
|
+
return view;
|
|
45
|
+
const recent = view.slice(cut);
|
|
46
|
+
const summaryMsg = liveTrimSummaryMessage(opts);
|
|
47
|
+
return [summaryMsg, ...recent];
|
|
48
|
+
}
|