pi-mega-compact 0.6.9 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -8
- package/dist/extensions/dashboard-server.js +17 -6
- package/dist/extensions/mega-commands.js +12 -1
- package/dist/extensions/mega-compact-driver.js +1 -0
- package/dist/extensions/mega-compact.test.js +286 -51
- package/dist/extensions/mega-config.js +67 -5
- package/dist/extensions/mega-events.js +151 -27
- package/dist/extensions/mega-pipeline.js +53 -17
- package/dist/extensions/mega-runtime.js +443 -115
- package/dist/extensions/openclaw-mega-compact.js +291 -0
- package/dist/src/dedup-engine.test.js +63 -38
- package/dist/src/minilm.js +92 -0
- package/dist/src/wordpiece.js +129 -0
- package/extensions/dashboard-server.ts +17 -6
- package/extensions/mega-commands.ts +12 -1
- package/extensions/mega-compact-driver.ts +56 -55
- package/extensions/mega-compact.test.ts +947 -516
- package/extensions/mega-config.ts +84 -6
- package/extensions/mega-dashboard.ts +11 -0
- package/extensions/mega-events.ts +558 -360
- package/extensions/mega-pipeline.ts +481 -393
- package/extensions/mega-runtime.ts +957 -502
- package/package.json +1 -1
- package/src/dedup-engine.test.ts +103 -42
|
@@ -8,13 +8,13 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { normalizeSessionId } from "../src/store.js";
|
|
10
10
|
import { autoCompactCheck } from "../src/compact.js";
|
|
11
|
-
import { estimateSessionTokens } from "../src/tokens.js";
|
|
12
|
-
import { recentUserQuery, WIDGET_KEY } from "./mega-runtime.js";
|
|
13
|
-
import { runCompact, doRecall, doRecallAsync, piCompactWouldNoop, runMemoryReview } from "./mega-pipeline.js";
|
|
11
|
+
import { estimateSessionTokens, estimateBlockTokens } from "../src/tokens.js";
|
|
12
|
+
import { recentUserQuery, WIDGET_KEY, } from "./mega-runtime.js";
|
|
13
|
+
import { runCompact, doRecall, doRecallAsync, piCompactWouldNoop, runMemoryReview, } from "./mega-pipeline.js";
|
|
14
14
|
import { recallMemoriesAndInline } from "../src/recall.js";
|
|
15
|
-
import { driveNativeCompaction } from "./mega-compact-driver.js";
|
|
15
|
+
import { driveNativeCompaction, } from "./mega-compact-driver.js";
|
|
16
16
|
import { computeLiveTrimCut, liveTrimSummaryMessage } from "./mega-trim.js";
|
|
17
|
-
import { pressureFromPct, memoryReviewCadence } from "./mega-config.js";
|
|
17
|
+
import { pressureFromPct, memoryReviewCadence, } from "./mega-config.js";
|
|
18
18
|
/**
|
|
19
19
|
* DIAG accessor for the headless test harness: the most recently constructed
|
|
20
20
|
* MegaRuntime, so a test that loads the compiled extension via its default
|
|
@@ -52,19 +52,30 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
52
52
|
// S17: use the async variant on resume so cross-repo HNSW recall can
|
|
53
53
|
// augment when this repo's store is thin. session_start is an async-safe
|
|
54
54
|
// point (unlike the mid-turn context handler, which stays sync).
|
|
55
|
-
const r = await doRecallAsync(runtime, config, ctx, query, "resume", {
|
|
55
|
+
const r = await doRecallAsync(runtime, config, ctx, query, "resume", {
|
|
56
|
+
crossRepo: config.crossRepoEnabled,
|
|
57
|
+
});
|
|
56
58
|
if (!r.empty) {
|
|
57
59
|
runtime.pendingRecallBlock = r.block;
|
|
58
|
-
const crossLabel = r.toInject.some((h) => h.repoId)
|
|
60
|
+
const crossLabel = r.toInject.some((h) => h.repoId)
|
|
61
|
+
? " (cross-repo)"
|
|
62
|
+
: "";
|
|
59
63
|
runtime.setStatus(ctx, `mega-compact: recalled ${r.toInject.length} chkpt${crossLabel}`);
|
|
60
|
-
runtime.logger.info("auto-inline", {
|
|
64
|
+
runtime.logger.info("auto-inline", {
|
|
65
|
+
reason: event.reason,
|
|
66
|
+
query,
|
|
67
|
+
injected: r.toInject.map((h) => h.checkpoint.checkpointId),
|
|
68
|
+
crossRepo: r.toInject.some((h) => h.repoId),
|
|
69
|
+
});
|
|
61
70
|
}
|
|
62
71
|
}
|
|
63
72
|
// S21: parallel memory recall. Same async context so we can await without
|
|
64
73
|
// breaking the handler contract. Best-effort — never throws.
|
|
65
74
|
try {
|
|
66
75
|
const mr = await recallMemoriesAndInline({
|
|
67
|
-
query,
|
|
76
|
+
query,
|
|
77
|
+
stateDir: runtime.getStateDir(),
|
|
78
|
+
limit: 5,
|
|
68
79
|
crossRepo: config.crossRepoEnabled,
|
|
69
80
|
crossRepoCosine: config.crossRepoCosine,
|
|
70
81
|
});
|
|
@@ -75,7 +86,10 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
75
86
|
runtime.logger.warn("memory-recall skipped", { err: String(err) });
|
|
76
87
|
}
|
|
77
88
|
}
|
|
78
|
-
runtime.dashboard.event("session_start", {
|
|
89
|
+
runtime.dashboard.event("session_start", {
|
|
90
|
+
reason: event.reason,
|
|
91
|
+
sessionId: runtime.rt.sessionId,
|
|
92
|
+
});
|
|
79
93
|
runtime.snapshot(ctx);
|
|
80
94
|
});
|
|
81
95
|
pi.on("session_tree", async (_event, ctx) => {
|
|
@@ -89,11 +103,21 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
89
103
|
const r = doRecall(runtime, config, ctx, query, "resume");
|
|
90
104
|
if (!r.empty) {
|
|
91
105
|
runtime.pendingRecallBlock = r.block;
|
|
92
|
-
runtime.logger.info("auto-inline", {
|
|
106
|
+
runtime.logger.info("auto-inline", {
|
|
107
|
+
reason: "session_tree",
|
|
108
|
+
query,
|
|
109
|
+
injected: r.toInject.map((h) => h.checkpoint.checkpointId),
|
|
110
|
+
});
|
|
93
111
|
}
|
|
94
112
|
// S21: parallel memory recall. Trigram embedder is sub-ms; await is fine.
|
|
95
113
|
try {
|
|
96
|
-
const mr = await recallMemoriesAndInline({
|
|
114
|
+
const mr = await recallMemoriesAndInline({
|
|
115
|
+
query,
|
|
116
|
+
stateDir: runtime.getStateDir(),
|
|
117
|
+
limit: 5,
|
|
118
|
+
crossRepo: config.crossRepoEnabled,
|
|
119
|
+
crossRepoCosine: config.crossRepoCosine,
|
|
120
|
+
});
|
|
97
121
|
if (!mr.empty)
|
|
98
122
|
runtime.pendingMemoryRecallBlock = mr.block;
|
|
99
123
|
}
|
|
@@ -102,7 +126,9 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
102
126
|
}
|
|
103
127
|
}
|
|
104
128
|
}
|
|
105
|
-
runtime.dashboard.event("session_tree", {
|
|
129
|
+
runtime.dashboard.event("session_tree", {
|
|
130
|
+
sessionId: runtime.rt.sessionId,
|
|
131
|
+
});
|
|
106
132
|
runtime.snapshot(ctx);
|
|
107
133
|
});
|
|
108
134
|
// ---- Auto-inline injection point: prepend staged recall to systemPrompt ----
|
|
@@ -126,7 +152,9 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
126
152
|
// ---- Agent tracking for real-time widget + status-line updates ---------
|
|
127
153
|
pi.on("agent_start", async (_event, ctx) => {
|
|
128
154
|
runtime.activeAgents++;
|
|
129
|
-
runtime.dashboard.event("agent_start", {
|
|
155
|
+
runtime.dashboard.event("agent_start", {
|
|
156
|
+
activeAgents: runtime.activeAgents,
|
|
157
|
+
});
|
|
130
158
|
// Surface live agent activity on the status line (toolbar), not just the
|
|
131
159
|
// above-editor widget — otherwise concurrent agents look frozen.
|
|
132
160
|
runtime.setStatus(ctx, `mega-compact: ▶ ${runtime.activeAgents} agent${runtime.activeAgents === 1 ? "" : "s"}`);
|
|
@@ -134,7 +162,9 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
134
162
|
});
|
|
135
163
|
pi.on("agent_end", async (_event, ctx) => {
|
|
136
164
|
runtime.activeAgents = Math.max(0, runtime.activeAgents - 1);
|
|
137
|
-
runtime.dashboard.event("agent_end", {
|
|
165
|
+
runtime.dashboard.event("agent_end", {
|
|
166
|
+
activeAgents: runtime.activeAgents,
|
|
167
|
+
});
|
|
138
168
|
if (runtime.activeAgents > 0) {
|
|
139
169
|
runtime.setStatus(ctx, `mega-compact: ▶ ${runtime.activeAgents} agent${runtime.activeAgents === 1 ? "" : "s"}`);
|
|
140
170
|
}
|
|
@@ -153,7 +183,7 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
153
183
|
// DIAG (team-run relief): surface whether the agent is idle + over
|
|
154
184
|
// threshold at agent_end so we can see if a mid-run durable-trim trigger
|
|
155
185
|
// *should* have fired but didn't.
|
|
156
|
-
const overThreshold = (runtime.lastCtxTokens ?? 0) >=
|
|
186
|
+
const overThreshold = (runtime.lastCtxTokens ?? 0) >= runtime.effectiveThreshold;
|
|
157
187
|
runtime.diagAgentEndIdle++;
|
|
158
188
|
runtime.logger.info("agent-end-idle", {
|
|
159
189
|
sessionId: runtime.rt.sessionId,
|
|
@@ -163,7 +193,9 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
163
193
|
ctxPct: runtime.lastCtxPercent,
|
|
164
194
|
ctxTokens: runtime.lastCtxTokens,
|
|
165
195
|
thresholdTokens: config.thresholdTokens,
|
|
166
|
-
wouldNudge: idle &&
|
|
196
|
+
wouldNudge: idle &&
|
|
197
|
+
(queued || overThreshold) &&
|
|
198
|
+
now >= runtime.resumeNudgeUntil,
|
|
167
199
|
});
|
|
168
200
|
// S16+S24: MID-RUN DURABLE TRIM. During a long team run (sub-agents),
|
|
169
201
|
// pi's native durable compaction only fires from _checkCompaction at
|
|
@@ -178,6 +210,18 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
178
210
|
// pi would actually compact (piCompactWouldNoop skips the user-facing
|
|
179
211
|
// no-op throw), and debounced (one durable trim per 2s) to avoid
|
|
180
212
|
// thrashing the transcript while sub-agents keep settling.
|
|
213
|
+
//
|
|
214
|
+
// FIX "compacts but doesn't resume": the manual ctx.compact() path
|
|
215
|
+
// STOPS the agent loop (agent-session.js:1345). The old resume-nudge
|
|
216
|
+
// was gated on `queued`, so when a sub-agent settled with no
|
|
217
|
+
// *immediately* queued message, the trim fired but the nudge did not,
|
|
218
|
+
// and the (stopped) session hung. The trim still fires on
|
|
219
|
+
// `idle && overThreshold` — we intentionally do NOT add a `!queued`
|
|
220
|
+
// guard, because that would suppress mid-run relief exactly during
|
|
221
|
+
// team-run waves where queued is usually true and relief is needed
|
|
222
|
+
// most. Instead we DECOUPLE the nudge from `queued`: after a durable
|
|
223
|
+
// trim we ALWAYS nudge so the agent reliably restarts. Debounced 30s.
|
|
224
|
+
let didDurableTrim = false;
|
|
181
225
|
if (idle && overThreshold && now >= runtime.debounceUntil) {
|
|
182
226
|
if (!piCompactWouldNoop(ctx)) {
|
|
183
227
|
runtime.debounceUntil = now + 2000;
|
|
@@ -186,11 +230,18 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
186
230
|
sessionId: runtime.rt.sessionId,
|
|
187
231
|
ctxTokens: runtime.lastCtxTokens,
|
|
188
232
|
thresholdTokens: config.thresholdTokens,
|
|
233
|
+
queued,
|
|
189
234
|
});
|
|
190
235
|
ctx.compact({ customInstructions: undefined }); // guardrails-allow PREVENT-PI-004: local ctx.compact() — no network; agent settled so no in-flight abort
|
|
236
|
+
didDurableTrim = true;
|
|
191
237
|
}
|
|
192
238
|
}
|
|
193
|
-
|
|
239
|
+
// Restart the agent after a mid-run durable trim (which stopped it), or
|
|
240
|
+
// when it settled idle with queued work. Decoupled from `queued` for the
|
|
241
|
+
// durable-trim case — see FIX note above. Debounced 30s; never blocks.
|
|
242
|
+
if (idle &&
|
|
243
|
+
now >= runtime.resumeNudgeUntil &&
|
|
244
|
+
(didDurableTrim || queued)) {
|
|
194
245
|
runtime.resumeNudgeUntil = now + 30_000;
|
|
195
246
|
pi.sendUserMessage("[mega-compact] continue from the compacted context above.");
|
|
196
247
|
}
|
|
@@ -255,14 +306,15 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
255
306
|
return;
|
|
256
307
|
const messages = event.messages;
|
|
257
308
|
const view = runtime.engineView(messages);
|
|
258
|
-
const currentTokens = usage?.tokens ??
|
|
309
|
+
const currentTokens = usage?.tokens ??
|
|
310
|
+
estimateSessionTokens(view) ??
|
|
259
311
|
Math.round((pct / 100) * (usage?.contextWindow ?? 0));
|
|
260
|
-
// FAST GATE: token-based (tier
|
|
261
|
-
if (currentTokens <
|
|
312
|
+
// FAST GATE: token-based (tier% of the window), not a static amount.
|
|
313
|
+
if (currentTokens < runtime.effectiveThreshold) {
|
|
262
314
|
runtime.diagCtxFastGate++;
|
|
263
315
|
return;
|
|
264
316
|
}
|
|
265
|
-
const check = autoCompactCheck(currentTokens,
|
|
317
|
+
const check = autoCompactCheck(currentTokens, runtime.effectiveThreshold); // SERVER-STYLE CONFIRM (local)
|
|
266
318
|
if (!check.shouldCompact) {
|
|
267
319
|
runtime.diagCtxNoCompact++;
|
|
268
320
|
return;
|
|
@@ -277,7 +329,9 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
277
329
|
// Adaptive compression (Fix E): scale compression strength + keepFrom depth
|
|
278
330
|
// with how close we are to the model context limit.
|
|
279
331
|
const pressure = pressureFromPct(pct);
|
|
280
|
-
const ran = runCompact(pi, runtime, config, ctx, messages, {
|
|
332
|
+
const ran = runCompact(pi, runtime, config, ctx, messages, {
|
|
333
|
+
compressionPressure: pressure,
|
|
334
|
+
});
|
|
281
335
|
if (ran.skipped) {
|
|
282
336
|
runtime.diagCtxRunSkipped++;
|
|
283
337
|
return;
|
|
@@ -287,7 +341,9 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
287
341
|
// Read live from env (in addition to the load-time config) so the flag can be
|
|
288
342
|
// toggled per-test without reloading the module; config.legacyDurableTrim is
|
|
289
343
|
// the cached default. (Mirrors how piCompactWouldNoop re-reads its floor.)
|
|
290
|
-
const legacy = config.legacyDurableTrim ||
|
|
344
|
+
const legacy = config.legacyDurableTrim ||
|
|
345
|
+
process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "true" ||
|
|
346
|
+
process.env.MEGACOMPACT_LEGACY_DURABLE_TRIM === "1";
|
|
291
347
|
if (legacy) {
|
|
292
348
|
if (piCompactWouldNoop(ctx))
|
|
293
349
|
return;
|
|
@@ -305,7 +361,9 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
305
361
|
// without reloading the module.
|
|
306
362
|
try {
|
|
307
363
|
const anchorEnv = process.env.MEGACOMPACT_ANCHOR_USER_MESSAGES;
|
|
308
|
-
const anchorUserMessages =
|
|
364
|
+
const anchorUserMessages = anchorEnv != null &&
|
|
365
|
+
anchorEnv !== "" &&
|
|
366
|
+
Number.isFinite(Number(anchorEnv))
|
|
309
367
|
? Number(anchorEnv)
|
|
310
368
|
: config.anchorUserMessages;
|
|
311
369
|
const cut = computeLiveTrimCut(view, {
|
|
@@ -382,7 +440,7 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
382
440
|
return {}; // let pi run its own native compaction
|
|
383
441
|
try {
|
|
384
442
|
const result = driveNativeCompaction(event, runtime, config);
|
|
385
|
-
if (result) {
|
|
443
|
+
if (result && result.compaction.summary?.trim()) {
|
|
386
444
|
runtime.diagBeforeCompactSupplied++;
|
|
387
445
|
runtime.logger.info("native-compact", {
|
|
388
446
|
sessionId: runtime.rt.sessionId,
|
|
@@ -390,8 +448,30 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
390
448
|
tokensBefore: result.compaction.tokensBefore,
|
|
391
449
|
summaryTokens: result.compaction.estimatedTokensAfter,
|
|
392
450
|
});
|
|
451
|
+
nudgeResume(pi, runtime);
|
|
393
452
|
return { compaction: result.compaction };
|
|
394
453
|
}
|
|
454
|
+
// FIX "compacts but doesn't resume" + "Nothing to compact" regression:
|
|
455
|
+
// when we have nothing to summarize (anchor floor protects everything →
|
|
456
|
+
// messagesToSummarize empty) or our Trident/RAPTOR summary came back
|
|
457
|
+
// EMPTY, pi's OWN compact() throws "Nothing to compact (session too
|
|
458
|
+
// small)" and leaves the session stuck with no resume context. Instead
|
|
459
|
+
// of returning {} (which makes pi run its throwing compact()), supply a
|
|
460
|
+
// fallback compaction from prep.firstKeptEntryId with a minimal resume
|
|
461
|
+
// summary. This ALWAYS injects a compact summary so the session
|
|
462
|
+
// resumes, and never surfaces the "Nothing to compact" error to the user.
|
|
463
|
+
const fb = fallbackCompaction(event);
|
|
464
|
+
if (fb) {
|
|
465
|
+
runtime.diagBeforeCompactSupplied++;
|
|
466
|
+
runtime.logger.info("native-compact-fallback", {
|
|
467
|
+
sessionId: runtime.rt.sessionId,
|
|
468
|
+
firstKeptEntryId: fb.compaction.firstKeptEntryId,
|
|
469
|
+
tokensBefore: fb.compaction.tokensBefore,
|
|
470
|
+
reason: event.reason,
|
|
471
|
+
});
|
|
472
|
+
nudgeResume(pi, runtime);
|
|
473
|
+
return { compaction: fb.compaction };
|
|
474
|
+
}
|
|
395
475
|
}
|
|
396
476
|
catch (err) {
|
|
397
477
|
runtime.logger.error("native-compact-failed", {
|
|
@@ -399,7 +479,51 @@ export function registerEventHandlers(pi, runtime, config) {
|
|
|
399
479
|
error: String(err instanceof Error ? err.message : err),
|
|
400
480
|
});
|
|
401
481
|
}
|
|
402
|
-
//
|
|
482
|
+
// Absolute last resort: let pi run its own (may throw "Nothing to compact").
|
|
403
483
|
return {};
|
|
404
484
|
});
|
|
485
|
+
/**
|
|
486
|
+
* Build a minimal fallback compaction so pi never runs its throwing compact().
|
|
487
|
+
*
|
|
488
|
+
* Used when our Trident/RAPTOR summary is empty or there is nothing to
|
|
489
|
+
* summarize (the anchor floor protects every message). We still record a
|
|
490
|
+
* resume summary + truncate from prep.firstKeptEntryId so the session always
|
|
491
|
+
* gets a compact summary and resumes. Returns undefined only if pi handed us
|
|
492
|
+
* no preparation cut point at all.
|
|
493
|
+
*/
|
|
494
|
+
function fallbackCompaction(event) {
|
|
495
|
+
const prep = event.preparation;
|
|
496
|
+
if (!prep?.firstKeptEntryId)
|
|
497
|
+
return undefined;
|
|
498
|
+
// When messagesToSummarize is empty the anchor floor protects everything,
|
|
499
|
+
// so firstKeptEntryId == current first entry and the trim is a no-op — but
|
|
500
|
+
// we still record a resume summary so the session has context after compaction.
|
|
501
|
+
const tokensBefore = prep.tokensBefore ?? 0;
|
|
502
|
+
const summary = `[mega-compact] context compacted at ${tokensBefore.toLocaleString()} tokens ` +
|
|
503
|
+
`(anchor floor active). Continue from the most recent messages above.`;
|
|
504
|
+
return {
|
|
505
|
+
compaction: {
|
|
506
|
+
summary,
|
|
507
|
+
firstKeptEntryId: prep.firstKeptEntryId,
|
|
508
|
+
tokensBefore,
|
|
509
|
+
estimatedTokensAfter: estimateBlockTokens(summary),
|
|
510
|
+
},
|
|
511
|
+
};
|
|
512
|
+
}
|
|
513
|
+
/**
|
|
514
|
+
* Debounced resume-nudge: restart the agent loop after a compaction (which
|
|
515
|
+
* may have stopped it). Idempotent — one nudge per 30s, never blocks.
|
|
516
|
+
*/
|
|
517
|
+
function nudgeResume(pi, runtime) {
|
|
518
|
+
try {
|
|
519
|
+
const now = Date.now();
|
|
520
|
+
if (now >= runtime.resumeNudgeUntil) {
|
|
521
|
+
runtime.resumeNudgeUntil = now + 30_000;
|
|
522
|
+
pi.sendUserMessage("[mega-compact] continue from the compacted context above.");
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
catch {
|
|
526
|
+
/* non-fatal: a failed nudge never blocks */
|
|
527
|
+
}
|
|
528
|
+
}
|
|
405
529
|
}
|
|
@@ -8,13 +8,13 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
10
10
|
import { compactSession } from "../src/engine.js";
|
|
11
|
-
import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "../src/recall.js";
|
|
11
|
+
import { recallAndInline, recallAndInlineAsync, formatRecallBlock, } from "../src/recall.js";
|
|
12
12
|
import { normalizeSessionId } from "../src/store.js";
|
|
13
13
|
import { estimateBlockTokens } from "../src/tokens.js";
|
|
14
14
|
import { touchSession, logDaily } from "../src/store/sqlite.js";
|
|
15
15
|
import { consolidateMemories } from "../src/memory.js";
|
|
16
|
-
import { C, MARKER_TYPE
|
|
17
|
-
import { resolveRepoRoot, preserveRecentForPressure } from "./mega-config.js";
|
|
16
|
+
import { C, MARKER_TYPE } from "./mega-runtime.js";
|
|
17
|
+
import { resolveRepoRoot, preserveRecentForPressure, } from "./mega-config.js";
|
|
18
18
|
import { runRaptor } from "../src/dedup/raptor/index.js";
|
|
19
19
|
import { loadDedupConfig } from "../src/config/dedup.js";
|
|
20
20
|
import { upsertEmbedding as indexUpsertEmbedding } from "../src/store/vectorIndex.js";
|
|
@@ -104,18 +104,23 @@ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
|
|
|
104
104
|
? result.originalTokenEstimate
|
|
105
105
|
: Math.max(0, result.originalTokenEstimate - result.tokenEstimate);
|
|
106
106
|
runtime.rt.tokensSaved += saved;
|
|
107
|
+
runtime.rt.lastCompactAt = Date.now();
|
|
107
108
|
if (result.deduped)
|
|
108
109
|
runtime.rt.dedupSkips++;
|
|
109
110
|
// Grow the rolling "saved" goal so the progress bar always has a fresh
|
|
110
111
|
// denominator (we don't want it pinned at 100% once we pass an old target).
|
|
111
112
|
if (runtime.rt.tokensSaved > runtime.savedGoal)
|
|
112
|
-
runtime.savedGoal =
|
|
113
|
+
runtime.savedGoal =
|
|
114
|
+
Math.ceil((runtime.rt.tokensSaved * 1.25) / 10_000) * 10_000;
|
|
113
115
|
// Live toolbar activity: what file/region just got compacted or deduped.
|
|
114
116
|
// Rendered via the rotating ticker line (see snapshot); the ring buffer is
|
|
115
117
|
// cycled one-per-repaint so the single line scrolls through recent files.
|
|
116
118
|
const files = result.filesModified ?? [];
|
|
117
119
|
const fileLabel = files.length
|
|
118
|
-
? files
|
|
120
|
+
? files
|
|
121
|
+
.map((f) => f.split("/").pop() ?? f)
|
|
122
|
+
.slice(0, 2)
|
|
123
|
+
.join(", ")
|
|
119
124
|
: result.regionHash.slice(0, 8);
|
|
120
125
|
runtime.lastActivityAt = Date.now();
|
|
121
126
|
// Explain-why line: surfaced while fresh. Pulls the dedup reason (which for
|
|
@@ -167,7 +172,10 @@ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
|
|
|
167
172
|
// waiting for the next turn-cadence tick. Uses the shared runMemoryReview
|
|
168
173
|
// helper (fire-and-forget; doCompact is sync). Best-effort + non-fatal. Only
|
|
169
174
|
// fires above the `high` band so low-pressure compactions don't pay the cost.
|
|
170
|
-
if (!result.deduped &&
|
|
175
|
+
if (!result.deduped &&
|
|
176
|
+
config.memoryAutoReview &&
|
|
177
|
+
runtime.pressureBand !== "low" &&
|
|
178
|
+
runtime.pressureBand !== "medium") {
|
|
171
179
|
void runMemoryReview(runtime, view, "pressure");
|
|
172
180
|
}
|
|
173
181
|
// Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
|
|
@@ -195,7 +203,9 @@ function doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime) {
|
|
|
195
203
|
// S25: stamp the tree with the newest checkpoint epoch so the
|
|
196
204
|
// freshness guard in raptorSearchHits can reject stale trees after a
|
|
197
205
|
// later compaction adds newer checkpoints.
|
|
198
|
-
const builtAt = all.length > 0
|
|
206
|
+
const builtAt = all.length > 0
|
|
207
|
+
? Math.max(...all.map((c) => c.timestamp))
|
|
208
|
+
: Date.now();
|
|
199
209
|
runRaptor(leaves, {
|
|
200
210
|
stateDir: runtime.currentStateDir,
|
|
201
211
|
sessionId: sid,
|
|
@@ -313,7 +323,8 @@ export function piCompactWouldNoop(ctx) {
|
|
|
313
323
|
if (m.role !== "toolResult")
|
|
314
324
|
isCut = true;
|
|
315
325
|
const c = m.content;
|
|
316
|
-
const text = typeof c === "string"
|
|
326
|
+
const text = typeof c === "string"
|
|
327
|
+
? c
|
|
317
328
|
: Array.isArray(c)
|
|
318
329
|
? c.map((b) => b?.text ?? "").join(" ")
|
|
319
330
|
: "";
|
|
@@ -366,12 +377,22 @@ export function doRecall(runtime, config, ctx, query, source) {
|
|
|
366
377
|
liveWindow,
|
|
367
378
|
dedupSim: config.dedupSim,
|
|
368
379
|
}, runtime.store);
|
|
369
|
-
runtime.dashboard.event("recall", {
|
|
380
|
+
runtime.dashboard.event("recall", {
|
|
381
|
+
source,
|
|
382
|
+
query: query.slice(0, 120),
|
|
383
|
+
injected: result.toInject.length,
|
|
384
|
+
empty: result.empty,
|
|
385
|
+
});
|
|
370
386
|
if (!result.empty && result.toInject.length > 0) {
|
|
371
387
|
const top = result.toInject[0];
|
|
372
388
|
const scorePct = Math.round((top.score ?? 0) * 100);
|
|
373
389
|
const files = top.checkpoint.filesModified ?? [];
|
|
374
|
-
const label = files.length
|
|
390
|
+
const label = files.length
|
|
391
|
+
? files
|
|
392
|
+
.map((f) => f.split("/").pop() ?? f)
|
|
393
|
+
.slice(0, 2)
|
|
394
|
+
.join(", ")
|
|
395
|
+
: top.checkpoint.checkpointId;
|
|
375
396
|
runtime.pushTicker(`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`);
|
|
376
397
|
runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
|
|
377
398
|
}
|
|
@@ -394,9 +415,15 @@ export async function doRecallAsync(runtime, config, ctx, query, source, opts =
|
|
|
394
415
|
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
395
416
|
// Sync same-repo first (fast, never blocks).
|
|
396
417
|
const sameRepo = recallAndInline({
|
|
397
|
-
sessionId: sid,
|
|
398
|
-
|
|
399
|
-
|
|
418
|
+
sessionId: sid,
|
|
419
|
+
query,
|
|
420
|
+
limit: config.autoInlineK,
|
|
421
|
+
source,
|
|
422
|
+
skipInjected: true,
|
|
423
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
424
|
+
windowDedupe: config.windowDedupe,
|
|
425
|
+
liveWindow,
|
|
426
|
+
dedupSim: config.dedupSim,
|
|
400
427
|
}, runtime.store);
|
|
401
428
|
if (!config.crossRepoEnabled || !opts.crossRepo)
|
|
402
429
|
return sameRepo;
|
|
@@ -405,13 +432,22 @@ export async function doRecallAsync(runtime, config, ctx, query, source, opts =
|
|
|
405
432
|
// Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
|
|
406
433
|
try {
|
|
407
434
|
const x = await recallAndInlineAsync({
|
|
408
|
-
sessionId: sid,
|
|
409
|
-
|
|
410
|
-
|
|
435
|
+
sessionId: sid,
|
|
436
|
+
query,
|
|
437
|
+
limit: config.autoInlineK,
|
|
438
|
+
source,
|
|
439
|
+
skipInjected: true,
|
|
440
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
441
|
+
windowDedupe: config.windowDedupe,
|
|
442
|
+
liveWindow,
|
|
443
|
+
dedupSim: config.crossRepoCosine,
|
|
444
|
+
crossRepo: true,
|
|
411
445
|
globalIndexDir: process.env.MEGACOMPACT_INDEX_DIR,
|
|
412
446
|
}, runtime.store);
|
|
413
447
|
runtime.dashboard.event("recall-crossrepo", {
|
|
414
|
-
source,
|
|
448
|
+
source,
|
|
449
|
+
query: query.slice(0, 120),
|
|
450
|
+
injected: x.toInject.length,
|
|
415
451
|
sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
|
|
416
452
|
});
|
|
417
453
|
// Merge, dedup by checkpointId, respect the same token cap by reformatting.
|