pi-mega-compact 0.4.28 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -2
- package/dist/extensions/dashboard-server.js +58 -2
- package/dist/extensions/dashboard-server.test.js +95 -3
- package/dist/extensions/mega-commands.js +25 -9
- package/dist/extensions/mega-compact.test.js +133 -31
- package/dist/extensions/mega-config.js +5 -0
- package/dist/extensions/mega-conflict-cmds.js +79 -0
- package/dist/extensions/mega-dashboard-cmds.js +6 -4
- package/dist/extensions/mega-events.js +144 -27
- package/dist/extensions/mega-pipeline.js +84 -1
- package/dist/extensions/mega-runtime.js +14 -0
- package/dist/extensions/mega-trim.js +48 -0
- package/dist/extensions/mega-trim.test.js +58 -0
- package/dist/src/config/dedup.js +1 -0
- package/dist/src/driftDetection.js +103 -0
- package/dist/src/driftDetection.test.js +87 -0
- package/dist/src/memory.js +147 -0
- package/dist/src/memory.test.js +41 -0
- package/dist/src/memoryConsolidate.test.js +38 -0
- package/dist/src/memoryOps.js +58 -0
- package/dist/src/memoryOps.test.js +41 -0
- package/dist/src/memoryRecall.js +60 -0
- package/dist/src/memoryRecall.test.js +92 -0
- package/dist/src/recall.js +70 -1
- package/dist/src/recall.test.js +69 -1
- package/dist/src/store/sqlite.js +127 -11
- package/dist/src/vectorStore.js +6 -1
- package/extensions/dashboard-server.test.ts +115 -3
- package/extensions/dashboard-server.ts +63 -2
- package/extensions/mega-commands.ts +24 -9
- package/extensions/mega-compact.test.ts +134 -31
- package/extensions/mega-config.ts +22 -0
- package/extensions/mega-conflict-cmds.ts +81 -0
- package/extensions/mega-dashboard-cmds.ts +6 -4
- package/extensions/mega-events.ts +139 -28
- package/extensions/mega-pipeline.ts +94 -1
- package/extensions/mega-runtime.ts +15 -0
- package/extensions/mega-trim.test.ts +64 -0
- package/extensions/mega-trim.ts +75 -0
- package/extensions/openclaw-mega-compact.ts +24 -9
- package/package.json +2 -2
- package/src/config/dedup.ts +2 -0
- package/src/driftDetection.test.ts +100 -0
- package/src/driftDetection.ts +136 -0
- package/src/memory.test.ts +46 -0
- package/src/memory.ts +164 -0
- package/src/memoryConsolidate.test.ts +47 -0
- package/src/memoryOps.test.ts +53 -0
- package/src/memoryOps.ts +75 -0
- package/src/memoryRecall.test.ts +100 -0
- package/src/memoryRecall.ts +83 -0
- package/src/recall.test.ts +77 -1
- package/src/recall.ts +94 -1
- package/src/store/sqlite.ts +188 -11
- package/src/store.ts +3 -0
- package/src/vectorStore.ts +10 -1
- package/dist/extensions/openclaw-mega-compact.js +0 -291
- package/dist/src/minilm.js +0 -92
- package/dist/src/wordpiece.js +0 -129
|
@@ -118,4 +118,83 @@ export function registerConflictCommands(pi, runtime) {
|
|
|
118
118
|
ctx.ui.notify(memoryLine(m));
|
|
119
119
|
},
|
|
120
120
|
});
|
|
121
|
+
// Shortform aliases — `m save "..."`, `m status`, `m list`, `m search <q>`,
|
|
122
|
+
// `m recall <id>`. Delegates to the same SQLite store so there's one source
|
|
123
|
+
// of truth. The /mega-memory command remains the canonical form.
|
|
124
|
+
pi.registerCommand("m", {
|
|
125
|
+
description: "Shortform alias for /mega-memory. Usage: /m save <text> | list | search <q> | recall <id> | status",
|
|
126
|
+
handler: async (args, ctx) => {
|
|
127
|
+
const repo = resolveRepoRoot(ctx.cwd) ?? runtime.currentStateDir;
|
|
128
|
+
const parts = args.trim().split(/\s+/);
|
|
129
|
+
const sub = parts[0]?.toLowerCase() ?? "list";
|
|
130
|
+
if (sub === "save") {
|
|
131
|
+
// Strip leading "save" so /m save "#foo bar" works the same as the
|
|
132
|
+
// canonical form. Then strip a balanced outer quote pair if the user
|
|
133
|
+
// wrote /m save "..." — common when the text contains spaces.
|
|
134
|
+
let text = args.trim().slice(4).trim();
|
|
135
|
+
const mq = text.match(/^["“](.*)["”]$/s);
|
|
136
|
+
if (mq)
|
|
137
|
+
text = mq[1].trim();
|
|
138
|
+
if (!text) {
|
|
139
|
+
ctx.ui.notify('[/m] usage: /m save "<text>" or /m save <text>');
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
const tagMatches = [...text.matchAll(/#([\w-]+)/g)].map((m) => m[1]);
|
|
143
|
+
const content = text.replace(/#[\w-]+/g, "").trim();
|
|
144
|
+
const id = addMemory({ content, tags: tagMatches }, repo, runtime.currentStateDir);
|
|
145
|
+
ctx.ui.notify(`[/m] saved #${id} to ${repo.split(/[\\/]/).pop()}`);
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
if (sub === "search") {
|
|
149
|
+
const q = parts.slice(1).join(" ").trim();
|
|
150
|
+
if (!q) {
|
|
151
|
+
ctx.ui.notify("[/m] usage: /m search <query>");
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
const hits = searchMemories(q, repo, 50, runtime.currentStateDir);
|
|
155
|
+
if (!hits.length) {
|
|
156
|
+
ctx.ui.notify("[/m] no memories match.");
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
for (const mem of hits)
|
|
160
|
+
ctx.ui.notify(memoryLine(mem));
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
if (sub === "recall") {
|
|
164
|
+
const id = Number(parts[1]);
|
|
165
|
+
if (!Number.isFinite(id) || parts[1] === undefined) {
|
|
166
|
+
ctx.ui.notify("[/m] usage: /m recall <id>");
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
if (recallMemory(id, runtime.currentStateDir)) {
|
|
170
|
+
const found = listMemories(repo, 1000, runtime.currentStateDir).find((mem) => mem.id === id);
|
|
171
|
+
ctx.ui.notify(found ? `[/m] ${memoryLine(found)}` : `[/m] recalled #${id}`);
|
|
172
|
+
}
|
|
173
|
+
else {
|
|
174
|
+
ctx.ui.notify(`[/m] #${id} not found.`);
|
|
175
|
+
}
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
if (sub === "status") {
|
|
179
|
+
const all = listMemories(repo, 1000, runtime.currentStateDir);
|
|
180
|
+
const byKind = all.reduce((acc, m) => {
|
|
181
|
+
acc[m.kind] = (acc[m.kind] ?? 0) + 1;
|
|
182
|
+
return acc;
|
|
183
|
+
}, {});
|
|
184
|
+
const kinds = Object.entries(byKind).map(([k, n]) => `${k}=${n}`).join(", ");
|
|
185
|
+
const head = `[/m] ${all.length} memory record(s) in ${repo.split(/[\\/]/).pop() ?? repo}`;
|
|
186
|
+
ctx.ui.notify(kinds ? `${head} (${kinds})` : head);
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
// default: list
|
|
190
|
+
const all = listMemories(repo, 50, runtime.currentStateDir);
|
|
191
|
+
if (!all.length) {
|
|
192
|
+
ctx.ui.notify("[/m] no saved memories yet. Use /m save <text>.");
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
ctx.ui.notify(`[/m] ${all.length} memory record(s):`);
|
|
196
|
+
for (const mem of all)
|
|
197
|
+
ctx.ui.notify(memoryLine(mem));
|
|
198
|
+
},
|
|
199
|
+
});
|
|
121
200
|
}
|
|
@@ -18,11 +18,13 @@ export function registerDashboardCommands(pi, runtime) {
|
|
|
18
18
|
// when we fall back to the .ts source outside node_modules; false when using
|
|
19
19
|
// the shipped compiled dist/extensions/dashboard-server.js).
|
|
20
20
|
let dashboardNeedsStrip = false;
|
|
21
|
-
// The dashboard server binds
|
|
22
|
-
// in dashboard-server.js
|
|
23
|
-
// readiness even when port.pid landed in a different
|
|
21
|
+
// The dashboard server binds a 10-port range starting at MEGACOMPACT_DASHBOARD_PORT
|
|
22
|
+
// (default 9320) — see TARGET_PORT/PORT_RANGE in dashboard-server.js. Probe each for
|
|
23
|
+
// a live /api/snapshot so we detect readiness even when port.pid landed in a different
|
|
24
|
+
// state dir than we poll. Configurable so tests can use a private, non-colliding range.
|
|
25
|
+
const DASH_BASE = Number(process.env.MEGACOMPACT_DASHBOARD_PORT ?? "9320");
|
|
24
26
|
async function findLivePort() {
|
|
25
|
-
for (let port =
|
|
27
|
+
for (let port = DASH_BASE; port <= DASH_BASE + 9; port++) {
|
|
26
28
|
try {
|
|
27
29
|
const res = await fetch(`http://localhost:${port}/api/snapshot`, { signal: AbortSignal.timeout(800) }); // guardrails-allow PREVENT-PI-004: localhost liveness probe of the dashboard server this extension spawned
|
|
28
30
|
if (res.ok)
|
|
@@ -10,8 +10,10 @@ import { normalizeSessionId } from "../src/store.js";
|
|
|
10
10
|
import { autoCompactCheck } from "../src/compact.js";
|
|
11
11
|
import { estimateSessionTokens } from "../src/tokens.js";
|
|
12
12
|
import { recentUserQuery, WIDGET_KEY } from "./mega-runtime.js";
|
|
13
|
-
import { runCompact, doRecall, piCompactWouldNoop } from "./mega-pipeline.js";
|
|
13
|
+
import { runCompact, doRecall, doRecallAsync, piCompactWouldNoop } from "./mega-pipeline.js";
|
|
14
|
+
import { recallMemoriesAndInline } from "../src/recall.js";
|
|
14
15
|
import { driveNativeCompaction } from "./mega-compact-driver.js";
|
|
16
|
+
import { computeLiveTrimCut, liveTrimSummaryMessage } from "./mega-trim.js";
|
|
15
17
|
import { pressureFromPct } from "./mega-config.js";
|
|
16
18
|
/** Register all pi lifecycle event handlers. */
|
|
17
19
|
export function registerEventHandlers(pi, runtime, config) {
|
|
@@ -25,6 +27,8 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
25
27
|
runtime.resetRuntime(ctx.sessionManager.getSessionId());
|
|
26
28
|
runtime.captureModel(ctx); // best-effort: ctx.model may be set by session start
|
|
27
29
|
runtime.setStatus(ctx, config.auto ? "mega-compact: ready" : "mega-compact: manual only");
|
|
30
|
+
// S21: clear any stale memory block from a prior session.
|
|
31
|
+
runtime.pendingMemoryRecallBlock = undefined;
|
|
28
32
|
// Auto-inline on resume/fork/continue: stage the most relevant checkpoints
|
|
29
33
|
// so the next before_agent_start prepends them to the system prompt.
|
|
30
34
|
// Triggered whenever this session already has persisted checkpoints AND a
|
|
@@ -36,13 +40,29 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
36
40
|
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
37
41
|
const query = recentUserQuery(ctx);
|
|
38
42
|
if (query && runtime.store.stats(sid).checkpointCount > 0) {
|
|
39
|
-
|
|
43
|
+
// S17: use the async variant on resume so cross-repo HNSW recall can
|
|
44
|
+
// augment when this repo's store is thin. session_start is an async-safe
|
|
45
|
+
// point (unlike the mid-turn context handler, which stays sync).
|
|
46
|
+
const r = await doRecallAsync(runtime, config, ctx, query, "resume", { crossRepo: config.crossRepoEnabled });
|
|
40
47
|
if (!r.empty) {
|
|
41
48
|
runtime.pendingRecallBlock = r.block;
|
|
42
|
-
|
|
43
|
-
runtime.
|
|
49
|
+
const crossLabel = r.toInject.some((h) => h.repoId) ? " (cross-repo)" : "";
|
|
50
|
+
runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt${crossLabel}`);
|
|
51
|
+
runtime.logger.info("auto-inline", { reason: event.reason, query, injected: r.toInject.map((h) => h.checkpoint.checkpointId), crossRepo: r.toInject.some((h) => h.repoId) });
|
|
44
52
|
}
|
|
45
53
|
}
|
|
54
|
+
// S21: parallel memory recall. Same async context so we can await without
|
|
55
|
+
// breaking the handler contract. Best-effort — never throws.
|
|
56
|
+
try {
|
|
57
|
+
const mr = await recallMemoriesAndInline({
|
|
58
|
+
query, stateDir: runtime.getStateDir(), limit: 5,
|
|
59
|
+
});
|
|
60
|
+
if (!mr.empty)
|
|
61
|
+
runtime.pendingMemoryRecallBlock = mr.block;
|
|
62
|
+
}
|
|
63
|
+
catch (err) {
|
|
64
|
+
runtime.logger.warn("memory-recall skipped", { err: String(err) });
|
|
65
|
+
}
|
|
46
66
|
}
|
|
47
67
|
runtime.dashboard.event("session_start", { reason: event.reason, sessionId: runtime.rt.sessionId });
|
|
48
68
|
runtime.snapshot(ctx);
|
|
@@ -60,6 +80,15 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
60
80
|
runtime.pendingRecallBlock = r.block;
|
|
61
81
|
runtime.logger.info("auto-inline", { reason: "session_tree", query, injected: r.toInject.map((h) => h.checkpoint.checkpointId) });
|
|
62
82
|
}
|
|
83
|
+
// S21: parallel memory recall. Trigram embedder is sub-ms; await is fine.
|
|
84
|
+
try {
|
|
85
|
+
const mr = await recallMemoriesAndInline({ query, stateDir: runtime.getStateDir(), limit: 5 });
|
|
86
|
+
if (!mr.empty)
|
|
87
|
+
runtime.pendingMemoryRecallBlock = mr.block;
|
|
88
|
+
}
|
|
89
|
+
catch (err) {
|
|
90
|
+
runtime.logger.warn("memory-recall skipped", { err: String(err) });
|
|
91
|
+
}
|
|
63
92
|
}
|
|
64
93
|
}
|
|
65
94
|
runtime.dashboard.event("session_tree", { sessionId: runtime.rt.sessionId });
|
|
@@ -68,11 +97,14 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
68
97
|
// ---- Auto-inline injection point: prepend staged recall to systemPrompt ----
|
|
69
98
|
pi.on("before_agent_start", async (event, ctx) => {
|
|
70
99
|
runtime.captureModel(ctx); // most reliable point ctx.model is populated
|
|
71
|
-
|
|
100
|
+
const cpBlock = runtime.pendingRecallBlock;
|
|
101
|
+
const memBlock = runtime.pendingMemoryRecallBlock;
|
|
102
|
+
if (!cpBlock && !memBlock)
|
|
72
103
|
return;
|
|
73
|
-
|
|
74
|
-
runtime.
|
|
75
|
-
|
|
104
|
+
runtime.pendingRecallBlock = undefined;
|
|
105
|
+
runtime.pendingMemoryRecallBlock = undefined;
|
|
106
|
+
const composed = [cpBlock, memBlock].filter(Boolean).join("\n\n");
|
|
107
|
+
return { systemPrompt: `${event.systemPrompt}\n\n${composed}` };
|
|
76
108
|
});
|
|
77
109
|
pi.on("session_shutdown", async (_event, ctx) => {
|
|
78
110
|
runtime.setStatus(ctx, undefined);
|
|
@@ -98,6 +130,24 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
98
130
|
else {
|
|
99
131
|
runtime.setStatus(ctx, config.auto ? "mega-compact: ready" : "mega-compact: manual only");
|
|
100
132
|
}
|
|
133
|
+
// S16 continuation fallback: if the turn settled idle right after a live-trim
|
|
134
|
+
// compaction AND there is queued work AND we haven't nudged recently, nudge
|
|
135
|
+
// once so the agent continues (the live trim should make this rare). Guarded
|
|
136
|
+
// to never busy-loop: one nudge per 30s, only when truly idle + queued.
|
|
137
|
+
if (config.auto && runtime.activeAgents === 0) {
|
|
138
|
+
try {
|
|
139
|
+
const idle = ctx.isIdle?.() ?? true;
|
|
140
|
+
const queued = ctx.hasPendingMessages?.() ?? false;
|
|
141
|
+
const now = Date.now();
|
|
142
|
+
if (idle && queued && now >= runtime.resumeNudgeUntil) {
|
|
143
|
+
runtime.resumeNudgeUntil = now + 30_000;
|
|
144
|
+
pi.sendUserMessage("[mega-compact] continue from the compacted context above.");
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
/* non-fatal: a failed nudge never blocks */
|
|
149
|
+
}
|
|
150
|
+
}
|
|
101
151
|
runtime.snapshot(ctx);
|
|
102
152
|
});
|
|
103
153
|
pi.on("turn_start", async (event, ctx) => {
|
|
@@ -108,16 +158,43 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
108
158
|
pi.on("turn_end", async (event, ctx) => {
|
|
109
159
|
runtime.dashboard.event("turn_end", { turnIndex: event.turnIndex });
|
|
110
160
|
runtime.snapshot(ctx);
|
|
161
|
+
// S20: auto-review the conversation every N turns and persist durable
|
|
162
|
+
// memories. Best-effort + non-fatal: a review failure must never break the
|
|
163
|
+
// agent loop. Debounced by memoryReviewInterval turns.
|
|
164
|
+
if (config.memoryAutoReview && runtime.currentTurn > 0 && runtime.currentTurn % config.memoryReviewInterval === 0) {
|
|
165
|
+
try {
|
|
166
|
+
const { reviewConversation } = await import("../src/memory.js");
|
|
167
|
+
const { applyMemoryOps } = await import("../src/memoryOps.js");
|
|
168
|
+
const entries = ctx.sessionManager.getEntries();
|
|
169
|
+
const view = runtime.engineView(entries.flatMap((e) => (e.message ? [e.message] : [])));
|
|
170
|
+
const ops = reviewConversation(view, []);
|
|
171
|
+
if (ops.length) {
|
|
172
|
+
await applyMemoryOps(ops, runtime.currentStateDir);
|
|
173
|
+
// S21.2: a memory op landed in this turn window. The pipeline reads
|
|
174
|
+
// this counter after a successful compaction and fires
|
|
175
|
+
// `consolidateMemories` only when it's > 0.
|
|
176
|
+
runtime.memoriesTouchedThisCompaction += ops.length;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
/* non-fatal — auto-review must not break the turn loop */
|
|
181
|
+
}
|
|
182
|
+
}
|
|
111
183
|
});
|
|
112
|
-
// ---- Auto-trigger:
|
|
113
|
-
//
|
|
114
|
-
//
|
|
115
|
-
//
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
// We
|
|
120
|
-
//
|
|
184
|
+
// ---- Auto-trigger: live trim (compact and continue) + native durable ----
|
|
185
|
+
// S16 redesign: we NO LONGER call ctx.compact() from the auto-trigger by
|
|
186
|
+
// default. That mapped to pi's MANUAL compaction path, which abort()s the
|
|
187
|
+
// in-flight turn (agent-session.js:1345) and stops the agent. Instead:
|
|
188
|
+
// - LIVE: return { messages: trimmedView } from the context event. This
|
|
189
|
+
// feeds pi's transformContext (sdk.js:226 → agent-loop.js:180) so the
|
|
190
|
+
// model sees a compacted window EVERY LLM call, with no abort. The turn
|
|
191
|
+
// continues. We persist our recall checkpoint (the durable value) first.
|
|
192
|
+
// - DURABLE: pi's NATIVE auto-compaction fires at agent-end
|
|
193
|
+
// (agent-session.js:1565), continues (return hasQueuedMessages()), and
|
|
194
|
+
// emits session_before_compact — where OUR driveNativeCompaction supplies
|
|
195
|
+
// the summary and pi truncates the transcript on disk. No ctx.compact().
|
|
196
|
+
// Legacy: MEGACOMPACT_LEGACY_DURABLE_TRIM=true restores the v0.4.28 ctx.compact
|
|
197
|
+
// path (kept one release as rollback).
|
|
121
198
|
pi.on("context", async (event, ctx) => {
|
|
122
199
|
if (!config.auto)
|
|
123
200
|
return;
|
|
@@ -151,17 +228,57 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
151
228
|
const ran = runCompact(pi, runtime, config, ctx, messages, { compressionPressure: pressure });
|
|
152
229
|
if (ran.skipped)
|
|
153
230
|
return;
|
|
154
|
-
//
|
|
155
|
-
//
|
|
156
|
-
//
|
|
157
|
-
//
|
|
158
|
-
//
|
|
159
|
-
|
|
160
|
-
if (
|
|
231
|
+
// LEGACY path (rollback): v0.4.28 ctx.compact() + the no-op gate. The
|
|
232
|
+
// manual compact path aborts the in-flight turn — only used behind the flag.
|
|
233
|
+
// Read live from env (in addition to the load-time config) so the flag can be
|
|
234
|
+
// toggled per-test without reloading the module; config.legacyDurableTrim is
|
|
235
|
+
// the cached default. (Mirrors how piCompactWouldNoop re-reads its floor.)
|
|
236
|
+
const legacy = config.legacyDurableTrim || process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "true" || process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "1";
|
|
237
|
+
if (legacy) {
|
|
238
|
+
if (piCompactWouldNoop(ctx))
|
|
239
|
+
return;
|
|
240
|
+
ctx.compact({ customInstructions: undefined });
|
|
161
241
|
return;
|
|
162
|
-
|
|
163
|
-
//
|
|
164
|
-
|
|
242
|
+
}
|
|
243
|
+
// S16 LIVE trim: collapse the compacted region to a summary + recent anchor.
|
|
244
|
+
// Non-destructive: pi keeps the real transcript; only this LLM call sees the
|
|
245
|
+
// trimmed window. We compute the cut on the engine view (pure, tested) then
|
|
246
|
+
// slice the ORIGINAL pi AgentMessage[] from that index (lossless alignment,
|
|
247
|
+
// mirroring dropCompactedRange) and prepend a user-role summary message.
|
|
248
|
+
// A build failure or unsafe cut returns nothing (no trim this call — the
|
|
249
|
+
// next context event retries). The anchor floor is read live from env (the
|
|
250
|
+
// config value is the cached default) so it can be tuned per-test / per-run
|
|
251
|
+
// without reloading the module.
|
|
252
|
+
try {
|
|
253
|
+
const anchorEnv = process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES;
|
|
254
|
+
const anchorUserMessages = (anchorEnv != null && anchorEnv !== "" && Number.isFinite(Number(anchorEnv)))
|
|
255
|
+
? Number(anchorEnv)
|
|
256
|
+
: config.anchorUserMessages;
|
|
257
|
+
const cut = computeLiveTrimCut(view, {
|
|
258
|
+
compactedFrom: ran.result.compactedFrom,
|
|
259
|
+
summary: ran.result.summary,
|
|
260
|
+
anchorUserMessages,
|
|
261
|
+
});
|
|
262
|
+
if (cut === null)
|
|
263
|
+
return; // unsafe / below anchor floor — no trim this call
|
|
264
|
+
const summaryMsg = liveTrimSummaryMessage({
|
|
265
|
+
compactedFrom: ran.result.compactedFrom,
|
|
266
|
+
summary: ran.result.summary,
|
|
267
|
+
anchorUserMessages: config.anchorUserMessages,
|
|
268
|
+
});
|
|
269
|
+
// Synthesize a user-role AgentMessage carrying the compacted summary.
|
|
270
|
+
const summaryAgentMsg = {
|
|
271
|
+
role: "user",
|
|
272
|
+
content: summaryMsg.text,
|
|
273
|
+
timestamp: Date.now(),
|
|
274
|
+
};
|
|
275
|
+
const recent = messages.slice(cut); // guardrails-allow PREVENT-PI-002: `cut` is the pre-sanitized `compactedFrom` produced by src/boundary.ts computeDropRange, so the preserved run begins on a toolPair-safe index.
|
|
276
|
+
runtime.snapshot(ctx);
|
|
277
|
+
return { messages: [summaryAgentMsg, ...recent] };
|
|
278
|
+
}
|
|
279
|
+
catch {
|
|
280
|
+
return; // non-fatal: no trim this call; the next context event retries
|
|
281
|
+
}
|
|
165
282
|
});
|
|
166
283
|
// ---- Supply a DURABLE trim to pi's native compaction (Fix B) ----------
|
|
167
284
|
// We run the Trident pipeline to produce a compressed summary, then return
|
|
@@ -8,10 +8,11 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
10
10
|
import { compactSession } from "../src/engine.js";
|
|
11
|
-
import { recallAndInline } from "../src/recall.js";
|
|
11
|
+
import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "../src/recall.js";
|
|
12
12
|
import { normalizeSessionId } from "../src/store.js";
|
|
13
13
|
import { estimateBlockTokens } from "../src/tokens.js";
|
|
14
14
|
import { touchSession, logDaily } from "../src/store/sqlite.js";
|
|
15
|
+
import { consolidateMemories } from "../src/memory.js";
|
|
15
16
|
import { C, MARKER_TYPE, } from "./mega-runtime.js";
|
|
16
17
|
import { resolveRepoRoot, preserveRecentForPressure } from "./mega-config.js";
|
|
17
18
|
import { runRaptor } from "../src/dedup/raptor/index.js";
|
|
@@ -42,6 +43,10 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
|
|
|
42
43
|
}
|
|
43
44
|
function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
|
|
44
45
|
runtime.pulsing = true; // animate the status line while the (sync) pipeline runs
|
|
46
|
+
// S21.2: reset the per-compaction memory-op counter so the post-compact
|
|
47
|
+
// consolidate pass only fires when memory rows actually changed during the
|
|
48
|
+
// compaction window (turn_end → auto-review may have written some).
|
|
49
|
+
runtime.memoriesTouchedThisCompaction = 0;
|
|
45
50
|
const result = compactSession({
|
|
46
51
|
sessionId: sid,
|
|
47
52
|
messages: view,
|
|
@@ -108,6 +113,26 @@ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
|
|
|
108
113
|
catch {
|
|
109
114
|
/* non-fatal: stats bookkeeping only */
|
|
110
115
|
}
|
|
116
|
+
// S21.2: best-effort consolidation of near-duplicate memories for this repo.
|
|
117
|
+
// Runs after the per-repo stats touch so `consolidateMemories` can use the
|
|
118
|
+
// same stateDir. Non-fatal — a failed consolidate never blocks a compaction.
|
|
119
|
+
// Only runs when new memory ops landed in this pass (otherwise the prior
|
|
120
|
+
// compaction's consolidate already had its shot — re-running would just
|
|
121
|
+
// touch every row again with no merges).
|
|
122
|
+
if (!result.deduped && runtime.memoriesTouchedThisCompaction > 0) {
|
|
123
|
+
try {
|
|
124
|
+
const root = resolveRepoRoot(ctx.cwd);
|
|
125
|
+
void consolidateMemories(runtime.currentStateDir, root).then((n) => {
|
|
126
|
+
if (n > 0)
|
|
127
|
+
runtime.pushTicker(`${C.green}∫${C.reset} consolidated ${n} memory dup${n === 1 ? "" : "s"}`);
|
|
128
|
+
}, () => {
|
|
129
|
+
/* swallow: consolidate failures must never surface to the user */
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
/* non-fatal */
|
|
134
|
+
}
|
|
135
|
+
}
|
|
111
136
|
// Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
|
|
112
137
|
// skip re-vectorizing an already-compacted region (zero token cost).
|
|
113
138
|
pi.appendEntry(MARKER_TYPE, {
|
|
@@ -310,6 +335,64 @@ export function doRecall(runtime, config, ctx, query, source) {
|
|
|
310
335
|
}
|
|
311
336
|
return result;
|
|
312
337
|
}
|
|
338
|
+
/**
|
|
339
|
+
* S17: async recall with optional cross-repo augmentation. Used on resume
|
|
340
|
+
* (session_start) and /mega-recall --cross-repo — NEVER from the mid-turn
|
|
341
|
+
* context handler (that stays sync). Runs the sync same-repo scan first; if it
|
|
342
|
+
* returns < config.autoInlineK hits AND crossRepo is enabled, awaits the PGlite
|
|
343
|
+
* HNSW cross-repo path and merges (source-labeled, deduped by checkpointId). The
|
|
344
|
+
* recallMaxTokens cap + windowDedupe apply to the merged set so cross-repo can
|
|
345
|
+
* never net-inflate the window. Cross-repo uses a stricter cosine floor
|
|
346
|
+
* (config.crossRepoCosine) than same-repo. Non-fatal: any async failure returns
|
|
347
|
+
* the same-repo result unchanged.
|
|
348
|
+
*/
|
|
349
|
+
export async function doRecallAsync(runtime, config, ctx, query, source, opts = {}) {
|
|
350
|
+
runtime.bindRepo(ctx.cwd);
|
|
351
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
352
|
+
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
353
|
+
// Sync same-repo first (fast, never blocks).
|
|
354
|
+
const sameRepo = recallAndInline({
|
|
355
|
+
sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
|
|
356
|
+
recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
|
|
357
|
+
liveWindow, dedupSim: config.dedupSim,
|
|
358
|
+
}, runtime.store);
|
|
359
|
+
if (!config.crossRepoEnabled || !opts.crossRepo)
|
|
360
|
+
return sameRepo;
|
|
361
|
+
if (sameRepo.toInject.length >= config.autoInlineK)
|
|
362
|
+
return sameRepo; // same-repo satisfied
|
|
363
|
+
// Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
|
|
364
|
+
try {
|
|
365
|
+
const x = await recallAndInlineAsync({
|
|
366
|
+
sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
|
|
367
|
+
recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
|
|
368
|
+
liveWindow, dedupSim: config.crossRepoCosine, crossRepo: true,
|
|
369
|
+
globalIndexDir: process.env.MEGACOMPACT_INDEX_DIR,
|
|
370
|
+
}, runtime.store);
|
|
371
|
+
runtime.dashboard.event("recall-crossrepo", {
|
|
372
|
+
source, query: query.slice(0, 120), injected: x.toInject.length,
|
|
373
|
+
sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
|
|
374
|
+
});
|
|
375
|
+
// Merge, dedup by checkpointId, respect the same token cap by reformatting.
|
|
376
|
+
const seen = new Set(sameRepo.toInject.map((h) => h.checkpoint.checkpointId));
|
|
377
|
+
const merged = [...sameRepo.toInject];
|
|
378
|
+
for (const h of x.toInject) {
|
|
379
|
+
if (!seen.has(h.checkpoint.checkpointId)) {
|
|
380
|
+
merged.push(h);
|
|
381
|
+
seen.add(h.checkpoint.checkpointId);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
const block = merged.length ? formatRecallBlock(merged) : "";
|
|
385
|
+
return {
|
|
386
|
+
toInject: merged,
|
|
387
|
+
report: merged.map((h) => ` • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`),
|
|
388
|
+
block,
|
|
389
|
+
empty: merged.length === 0,
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
catch {
|
|
393
|
+
return sameRepo; // cross-repo failure → same-repo only (non-fatal)
|
|
394
|
+
}
|
|
395
|
+
}
|
|
313
396
|
/**
|
|
314
397
|
* Extract the live-window message texts from the session manager (Fix C),
|
|
315
398
|
* for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
|
|
@@ -57,12 +57,17 @@ export class MegaRuntime {
|
|
|
57
57
|
tokensSaved: 0,
|
|
58
58
|
};
|
|
59
59
|
debounceUntil = 0;
|
|
60
|
+
// S16: debounce for the agent_end resume nudge (avoid busy-loops).
|
|
61
|
+
resumeNudgeUntil = 0;
|
|
60
62
|
// Agent tracking for real-time widget updates
|
|
61
63
|
activeAgents = 0;
|
|
62
64
|
currentTurn = 0;
|
|
63
65
|
// Recall block produced by auto-inline (resume/branch) that the next
|
|
64
66
|
// before_agent_start should prepend to the system prompt. Unset after use.
|
|
65
67
|
pendingRecallBlock;
|
|
68
|
+
// S21: memory recall block, parallel to pendingRecallBlock. Same one-shot
|
|
69
|
+
// semantics; composed with the checkpoint block in before_agent_start.
|
|
70
|
+
pendingMemoryRecallBlock;
|
|
66
71
|
statusKey; // current status text for dashboard
|
|
67
72
|
// Active model/provider (for real cost estimation). Captured from ctx.model
|
|
68
73
|
// on model_select + session_start; persisted to SQL so cost + the dashboard
|
|
@@ -81,6 +86,11 @@ export class MegaRuntime {
|
|
|
81
86
|
TICKER_MAX = 5;
|
|
82
87
|
// Pulsing status: set true while a compaction is in flight, cleared on result.
|
|
83
88
|
pulsing = false;
|
|
89
|
+
// S21.2: set by `applyMemoryOps` when a memory add/replace/remove lands in
|
|
90
|
+
// the current compaction. The pipeline reads this after a successful compact
|
|
91
|
+
// to decide whether to fire `consolidateMemories` (skip the work entirely
|
|
92
|
+
// when no memory rows changed).
|
|
93
|
+
memoriesTouchedThisCompaction = 0;
|
|
84
94
|
// Rolling "saved" goal for the progress bar — grows as we save more, so the
|
|
85
95
|
// bar always has a meaningful denominator (never sits at 100% forever).
|
|
86
96
|
savedGoal = 50_000;
|
|
@@ -343,6 +353,10 @@ export class MegaRuntime {
|
|
|
343
353
|
}
|
|
344
354
|
catch { /* non-fatal: cost estimation degrades to model-in-memory only */ }
|
|
345
355
|
}
|
|
356
|
+
/** S21: state dir of the currently bound repo (where memories live). */
|
|
357
|
+
getStateDir() {
|
|
358
|
+
return this.currentStateDir;
|
|
359
|
+
}
|
|
346
360
|
/** Build the sync onTier callback that paints the live per-tier trace. */
|
|
347
361
|
makeTierCallback(ctx) {
|
|
348
362
|
const order = ["L0", "L1", "L2", "new"];
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { isBoundarySafe } from "../src/boundary.js";
|
|
2
|
+
import { formatCompactSummary } from "../src/compact.js";
|
|
3
|
+
/**
|
|
4
|
+
* Compute the safe cut index for the live trim. Snaps `compactedFrom` back to a
|
|
5
|
+
* boundary-safe index (PREVENT-PI-002: never start the preserved run on an
|
|
6
|
+
* orphaned tool result), and enforces the anchor floor (PREVENT-PI-001: keep at
|
|
7
|
+
* least `anchorUserMessages` user-role messages). Returns `null` when no trim is
|
|
8
|
+
* safe this call (empty summary, unsafe boundary, or below the anchor floor) so
|
|
9
|
+
* the caller keeps the original view and retries on the next context event.
|
|
10
|
+
*
|
|
11
|
+
* Exposed separately from `buildLiveTrimmedView` so the context handler can map
|
|
12
|
+
* the cut back onto the original pi `AgentMessage[]` (lossless index alignment,
|
|
13
|
+
* mirroring `dropCompactedRange` in src/adapt.ts).
|
|
14
|
+
*/
|
|
15
|
+
export function computeLiveTrimCut(view, opts) {
|
|
16
|
+
if (!opts.summary || !opts.summary.trim())
|
|
17
|
+
return null;
|
|
18
|
+
let cut = opts.compactedFrom;
|
|
19
|
+
while (cut > 0 && !isBoundarySafe(view, cut))
|
|
20
|
+
cut--;
|
|
21
|
+
if (cut <= 0)
|
|
22
|
+
return null; // nothing safe to cut — keep everything this call
|
|
23
|
+
const recent = view.slice(cut);
|
|
24
|
+
const userCount = recent.filter((m) => m.role === "user").length;
|
|
25
|
+
if (userCount < opts.anchorUserMessages)
|
|
26
|
+
return null;
|
|
27
|
+
return cut;
|
|
28
|
+
}
|
|
29
|
+
/** The formatted compacted-region summary as a user-role engine message. */
|
|
30
|
+
export function liveTrimSummaryMessage(opts) {
|
|
31
|
+
return {
|
|
32
|
+
role: "user",
|
|
33
|
+
text: formatCompactSummary(opts.summary),
|
|
34
|
+
toolName: undefined,
|
|
35
|
+
input: undefined,
|
|
36
|
+
output: undefined,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
/** Build the live trimmed view. Returns the original view if summary is empty
|
|
40
|
+
* or the boundary is unsafe (no trim this call — try next). Pure + tested. */
|
|
41
|
+
export function buildLiveTrimmedView(view, opts) {
|
|
42
|
+
const cut = computeLiveTrimCut(view, opts);
|
|
43
|
+
if (cut === null)
|
|
44
|
+
return view;
|
|
45
|
+
const recent = view.slice(cut);
|
|
46
|
+
const summaryMsg = liveTrimSummaryMessage(opts);
|
|
47
|
+
return [summaryMsg, ...recent];
|
|
48
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* mega-trim.test.ts — tests for the live compaction view builder (S16).
|
|
3
|
+
*/
|
|
4
|
+
import { test } from "node:test";
|
|
5
|
+
import assert from "node:assert/strict";
|
|
6
|
+
import { buildLiveTrimmedView } from "./mega-trim.js";
|
|
7
|
+
function m(role, text, extra = {}) {
|
|
8
|
+
return { role, text, toolName: undefined, input: undefined, output: undefined, ...extra };
|
|
9
|
+
}
|
|
10
|
+
test("buildLiveTrimmedView: prepends a compacted summary and keeps the recent anchor", () => {
|
|
11
|
+
const view = [
|
|
12
|
+
m("user", "old request one"), m("assistant", "old answer one"),
|
|
13
|
+
m("user", "old request two"), m("assistant", "old answer two"),
|
|
14
|
+
m("user", "recent keep me"), m("assistant", "recent keep me too"),
|
|
15
|
+
];
|
|
16
|
+
// Compacted region = first 4; recent anchor = last 2.
|
|
17
|
+
const result = buildLiveTrimmedView(view, {
|
|
18
|
+
compactedFrom: 4, // index where the compacted region ends
|
|
19
|
+
summary: "<summary>earlier work on old requests</summary>",
|
|
20
|
+
anchorUserMessages: 1,
|
|
21
|
+
});
|
|
22
|
+
// First element is the injected compacted summary as a user-role message.
|
|
23
|
+
assert.equal(result[0].role, "user");
|
|
24
|
+
assert.ok(String(result[0].text).includes("earlier work on old requests"));
|
|
25
|
+
// Recent anchor preserved in order, no older messages leak through.
|
|
26
|
+
assert.equal(result.length, 1 + 2, "summary + 2 recent");
|
|
27
|
+
assert.ok(result.slice(1).some((x) => String(x.text).includes("recent keep me")));
|
|
28
|
+
});
|
|
29
|
+
test("buildLiveTrimmedView: empty summary returns the original view unchanged", () => {
|
|
30
|
+
const view = [m("user", "x"), m("assistant", "y")];
|
|
31
|
+
const result = buildLiveTrimmedView(view, { compactedFrom: 0, summary: "", anchorUserMessages: 1 });
|
|
32
|
+
assert.deepEqual(result, view);
|
|
33
|
+
});
|
|
34
|
+
test("buildLiveTrimmedView: never splits a toolCall/toolResult pair (PREVENT-PI-002)", () => {
|
|
35
|
+
const view = [
|
|
36
|
+
m("user", "q"), m("assistant", "calls tool", { toolName: "read" }), m("tool", "result"),
|
|
37
|
+
m("user", "keep"), m("assistant", "ok"),
|
|
38
|
+
];
|
|
39
|
+
// cut=3 would start the preserved run on the orphaned tool result at index 2 —
|
|
40
|
+
// the builder must snap back so the toolCall/toolResult pair is not split.
|
|
41
|
+
const result = buildLiveTrimmedView(view, { compactedFrom: 3, summary: "<summary>s</summary>", anchorUserMessages: 1 });
|
|
42
|
+
// The tool result must never appear preserved WITHOUT its preceding toolCall.
|
|
43
|
+
const preserved = result.slice(1);
|
|
44
|
+
const hasToolResult = preserved.some((x) => x.role === "tool");
|
|
45
|
+
const hasToolCall = preserved.some((x) => x.role === "assistant" && x.toolName);
|
|
46
|
+
// Either the tool pair is kept together, or the tool result is dropped into
|
|
47
|
+
// the compacted region — it is never left orphaned.
|
|
48
|
+
assert.ok(!(hasToolResult && !hasToolCall), "no orphaned tool result in the preserved run");
|
|
49
|
+
});
|
|
50
|
+
test("buildLiveTrimmedView: honors the anchor floor (PREVENT-PI-001)", () => {
|
|
51
|
+
// cut would leave zero user messages in the anchor — must skip the trim.
|
|
52
|
+
const view = [
|
|
53
|
+
m("user", "old q"), m("assistant", "old a"),
|
|
54
|
+
m("assistant", "only assistant kept"),
|
|
55
|
+
];
|
|
56
|
+
const result = buildLiveTrimmedView(view, { compactedFrom: 2, summary: "<summary>s</summary>", anchorUserMessages: 1 });
|
|
57
|
+
assert.deepEqual(result, view, "below anchor floor → no trim this call");
|
|
58
|
+
});
|
package/dist/src/config/dedup.js
CHANGED
|
@@ -46,6 +46,7 @@ export function loadDedupConfig() {
|
|
|
46
46
|
DEDUP_SIM: envNum("MEGACOMPACT_DEDUP_SIM", 0.9),
|
|
47
47
|
MMR_LAMBDA: envNum("MEGACOMPACT_MMR_LAMBDA", 0.5),
|
|
48
48
|
SEMDEDUP_COSINE: envNum("MEGACOMPACT_SEMDEDUP_COSINE", 0.95),
|
|
49
|
+
CONSOLIDATE_COSINE: envNum("MEGACOMPACT_CONSOLIDATE_COSINE", 0.7),
|
|
49
50
|
SIMILARITY_BUDGET_MS: envNum("MEGACOMPACT_SIMILARITY_BUDGET_MS", 50),
|
|
50
51
|
L1_VERIFY_BUDGET_MS: envNum("MEGACOMPACT_L1_VERIFY_BUDGET_MS", 20),
|
|
51
52
|
L1_CANDIDATE_CAP: envNum("MEGACOMPACT_L1_CANDIDATE_CAP", 100),
|