pi-mega-compact 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -7,42 +7,29 @@
|
|
|
7
7
|
* driven by the event + command handlers in mega-events.ts / mega-commands.ts.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
import type {
|
|
11
|
-
ExtensionAPI,
|
|
12
|
-
ExtensionContext,
|
|
13
|
-
} from "@earendil-works/pi-coding-agent";
|
|
10
|
+
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
14
11
|
import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
15
12
|
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
16
13
|
import { compactSession } from "../src/engine.js";
|
|
17
14
|
import type { EngineMessage } from "../src/types.js";
|
|
18
|
-
import {
|
|
19
|
-
recallAndInline,
|
|
20
|
-
recallAndInlineAsync,
|
|
21
|
-
formatRecallBlock,
|
|
22
|
-
type RecallInjectResult,
|
|
23
|
-
} from "../src/recall.js";
|
|
15
|
+
import { recallAndInline, recallAndInlineAsync, formatRecallBlock, type RecallInjectResult } from "../src/recall.js";
|
|
24
16
|
import { normalizeSessionId } from "../src/store.js";
|
|
25
17
|
import { estimateBlockTokens } from "../src/tokens.js";
|
|
26
18
|
import { touchSession, logDaily } from "../src/store/sqlite.js";
|
|
27
19
|
import { consolidateMemories } from "../src/memory.js";
|
|
28
|
-
import { type MegaRuntime, C, MARKER_TYPE } from "./mega-runtime.js";
|
|
29
20
|
import {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
} from "./mega-
|
|
21
|
+
type MegaRuntime,
|
|
22
|
+
C,
|
|
23
|
+
MARKER_TYPE,
|
|
24
|
+
} from "./mega-runtime.js";
|
|
25
|
+
import { resolveRepoRoot, preserveRecentForPressure, type MegaConfig } from "./mega-config.js";
|
|
34
26
|
import { runRaptor } from "../src/dedup/raptor/index.js";
|
|
35
27
|
import { loadDedupConfig } from "../src/config/dedup.js";
|
|
36
28
|
import { upsertEmbedding as indexUpsertEmbedding } from "../src/store/vectorIndex.js";
|
|
37
29
|
|
|
38
30
|
export type RunCompactResult =
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
skipped: false;
|
|
42
|
-
result: ReturnType<typeof compactSession>;
|
|
43
|
-
keepFrom: number;
|
|
44
|
-
saved: number;
|
|
45
|
-
};
|
|
31
|
+
| { skipped: true }
|
|
32
|
+
| { skipped: false; result: ReturnType<typeof compactSession>; keepFrom: number; saved: number };
|
|
46
33
|
|
|
47
34
|
/**
|
|
48
35
|
* Review the live conversation and persist durable memories (S20+S24). Shared by
|
|
@@ -55,306 +42,272 @@ export type RunCompactResult =
|
|
|
55
42
|
* @param label a short source tag for the ticker line (e.g. "pressure" / "turn")
|
|
56
43
|
*/
|
|
57
44
|
export async function runMemoryReview(
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
45
|
+
runtime: MegaRuntime,
|
|
46
|
+
view: ReturnType<MegaRuntime["engineView"]>,
|
|
47
|
+
label: string,
|
|
61
48
|
): Promise<number> {
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
return 0;
|
|
79
|
-
}
|
|
49
|
+
try {
|
|
50
|
+
const { reviewConversation } = await import("../src/memory.js");
|
|
51
|
+
const { applyMemoryOps } = await import("../src/memoryOps.js");
|
|
52
|
+
const ops = reviewConversation(view, []);
|
|
53
|
+
if (ops.length) {
|
|
54
|
+
await applyMemoryOps(ops, runtime.currentStateDir);
|
|
55
|
+
// S21.2: ops landed — the compaction path reads this counter and fires
|
|
56
|
+
// `consolidateMemories` only when > 0.
|
|
57
|
+
runtime.memoriesTouchedThisCompaction += ops.length;
|
|
58
|
+
runtime.pushTicker(`${C.green}🧠${C.reset} reviewed ${ops.length} memory op${ops.length === 1 ? "" : "s"} (${label})`);
|
|
59
|
+
}
|
|
60
|
+
return ops.length;
|
|
61
|
+
} catch {
|
|
62
|
+
/* non-fatal — auto-review must never break the turn loop / compaction */
|
|
63
|
+
return 0;
|
|
64
|
+
}
|
|
80
65
|
}
|
|
81
66
|
|
|
82
67
|
/** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
|
|
83
68
|
export function runCompact(
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
keepFrom?: number;
|
|
91
|
-
summary?: string;
|
|
92
|
-
compressionPressure?: number;
|
|
93
|
-
} = {},
|
|
69
|
+
pi: ExtensionAPI,
|
|
70
|
+
runtime: MegaRuntime,
|
|
71
|
+
config: MegaConfig,
|
|
72
|
+
ctx: ExtensionContext,
|
|
73
|
+
messages: AgentMessage[],
|
|
74
|
+
opts: { keepFrom?: number; summary?: string; compressionPressure?: number } = {},
|
|
94
75
|
): RunCompactResult {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
76
|
+
runtime.bindRepo(ctx.cwd);
|
|
77
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
78
|
+
runtime.resetRuntime(sid);
|
|
79
|
+
runtime.rt.sessionId = sid;
|
|
99
80
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
fallbackKeepFrom,
|
|
119
|
-
opts,
|
|
120
|
-
sid,
|
|
121
|
-
config,
|
|
122
|
-
pi,
|
|
123
|
-
ctx,
|
|
124
|
-
runtime,
|
|
125
|
-
);
|
|
126
|
-
}
|
|
81
|
+
const view = runtime.engineView(messages);
|
|
82
|
+
// keepFrom deepens with context pressure (Fix E): under high pressure we
|
|
83
|
+
// compact more of the session, down to the preserveRecentMin floor.
|
|
84
|
+
const preserve = preserveRecentForPressure(
|
|
85
|
+
opts.compressionPressure ?? 0,
|
|
86
|
+
config.preserveRecent,
|
|
87
|
+
config.preserveRecentMin,
|
|
88
|
+
);
|
|
89
|
+
const keepFrom = opts.keepFrom ?? Math.max(0, view.length - preserve);
|
|
90
|
+
// For very small sessions (fewer messages than preserveRecent), allow
|
|
91
|
+
// compacting everything except the last message — the user explicitly
|
|
92
|
+
// requested compaction, so don't refuse it just because the session is short.
|
|
93
|
+
if (keepFrom <= 0) {
|
|
94
|
+
if (view.length <= 1) return { skipped: true };
|
|
95
|
+
// Use the fallback: compact everything except the last message
|
|
96
|
+
const fallbackKeepFrom = view.length - 1;
|
|
97
|
+
return doCompact(view, fallbackKeepFrom, opts, sid, config, pi, ctx, runtime);
|
|
98
|
+
}
|
|
127
99
|
|
|
128
|
-
|
|
100
|
+
return doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime);
|
|
129
101
|
}
|
|
130
102
|
|
|
131
103
|
function doCompact(
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
104
|
+
view: EngineMessage[],
|
|
105
|
+
keepFrom: number,
|
|
106
|
+
opts: { keepFrom?: number; summary?: string; compressionPressure?: number },
|
|
107
|
+
sid: string,
|
|
108
|
+
config: MegaConfig,
|
|
109
|
+
pi: ExtensionAPI,
|
|
110
|
+
ctx: ExtensionContext,
|
|
111
|
+
runtime: MegaRuntime,
|
|
140
112
|
): RunCompactResult {
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
113
|
+
runtime.pulsing = true; // animate the status line while the (sync) pipeline runs
|
|
114
|
+
// S21.2: reset the per-compaction memory-op counter so the post-compact
|
|
115
|
+
// consolidate pass only fires when memory rows actually changed during the
|
|
116
|
+
// compaction window (turn_end → auto-review may have written some).
|
|
117
|
+
runtime.memoriesTouchedThisCompaction = 0;
|
|
118
|
+
const result = compactSession(
|
|
119
|
+
{
|
|
120
|
+
sessionId: sid,
|
|
121
|
+
messages: view,
|
|
122
|
+
keepFrom,
|
|
123
|
+
summary: opts.summary,
|
|
124
|
+
timestamp: Date.now(),
|
|
125
|
+
onTier: runtime.makeTierCallback(ctx),
|
|
126
|
+
compressionPressure: opts.compressionPressure,
|
|
127
|
+
},
|
|
128
|
+
runtime.store,
|
|
129
|
+
);
|
|
130
|
+
runtime.pulsing = false;
|
|
159
131
|
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
runtime.savedGoal =
|
|
183
|
-
Math.ceil((runtime.rt.tokensSaved * 1.25) / 10_000) * 10_000;
|
|
132
|
+
if (result.skipped) return { skipped: true };
|
|
133
|
+
if (!result.deduped) {
|
|
134
|
+
runtime.rt.persistedThisSession = true;
|
|
135
|
+
runtime.rt.lastCheckpointId = result.checkpointId;
|
|
136
|
+
}
|
|
137
|
+
runtime.rt.lastCompactedFrom = result.compactedFrom;
|
|
138
|
+
runtime.rt.lastCompactedTokens = result.tokenEstimate;
|
|
139
|
+
runtime.rt.dedupAttempts++;
|
|
140
|
+
// Honest "tokens saved" for this session-instance only:
|
|
141
|
+
// new checkpoint → original − stored
|
|
142
|
+
// deduped onto existing → whole original region (nothing new stored)
|
|
143
|
+
// Resets to 0 on session_start (rt is rebuilt) — so a fresh session shows 0
|
|
144
|
+
// while the repo's cumulative saved (SQLite meta) keeps the running total.
|
|
145
|
+
const saved = result.deduped
|
|
146
|
+
? result.originalTokenEstimate
|
|
147
|
+
: Math.max(0, result.originalTokenEstimate - result.tokenEstimate);
|
|
148
|
+
runtime.rt.tokensSaved += saved;
|
|
149
|
+
runtime.rt.lastCompactAt = Date.now();
|
|
150
|
+
if (result.deduped) runtime.rt.dedupSkips++;
|
|
151
|
+
// Grow the rolling "saved" goal so the progress bar always has a fresh
|
|
152
|
+
// denominator (we don't want it pinned at 100% once we pass an old target).
|
|
153
|
+
if (runtime.rt.tokensSaved > runtime.savedGoal) runtime.savedGoal = Math.ceil((runtime.rt.tokensSaved * 1.25) / 10_000) * 10_000;
|
|
184
154
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
// The per-tier trace has settled into the final outcome — fold it back into
|
|
209
|
-
// the activity line and stop showing the live trace.
|
|
210
|
-
runtime.tierTrace = undefined;
|
|
155
|
+
// Live toolbar activity: what file/region just got compacted or deduped.
|
|
156
|
+
// Rendered via the rotating ticker line (see snapshot); the ring buffer is
|
|
157
|
+
// cycled one-per-repaint so the single line scrolls through recent files.
|
|
158
|
+
const files = result.filesModified ?? [];
|
|
159
|
+
const fileLabel = files.length
|
|
160
|
+
? files.map((f) => f.split("/").pop() ?? f).slice(0, 2).join(", ")
|
|
161
|
+
: result.regionHash.slice(0, 8);
|
|
162
|
+
runtime.lastActivityAt = Date.now();
|
|
163
|
+
// Explain-why line: surfaced while fresh. Pulls the dedup reason (which for
|
|
164
|
+
// L2 includes the cosine sim) so the user sees WHY a region was kept/dropped.
|
|
165
|
+
runtime.lastWhy = result.deduped
|
|
166
|
+
? `why: deduped@${result.dedupReason ?? "tier"}`
|
|
167
|
+
: `why: compacted → ${result.checkpointId}`;
|
|
168
|
+
// Recall/activity ticker: record this event in the ring buffer.
|
|
169
|
+
const savedK = (saved / 1000).toFixed(1);
|
|
170
|
+
runtime.pushTicker(
|
|
171
|
+
result.deduped
|
|
172
|
+
? `${C.green}♻${C.reset} deduped ${fileLabel} · ${savedK}k saved`
|
|
173
|
+
: `${C.cyan}🗜${C.reset} ${result.checkpointId} · +${savedK}k · ${fileLabel}`,
|
|
174
|
+
);
|
|
175
|
+
// The per-tier trace has settled into the final outcome — fold it back into
|
|
176
|
+
// the activity line and stop showing the live trace.
|
|
177
|
+
runtime.tierTrace = undefined;
|
|
211
178
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
saved,
|
|
223
|
-
runtime.currentStateDir,
|
|
224
|
-
);
|
|
225
|
-
} catch {
|
|
226
|
-
/* non-fatal: stats bookkeeping only */
|
|
227
|
-
}
|
|
179
|
+
// Record session activity + a daily-log entry in the per-repo SQLite store
|
|
180
|
+
// (foundation for resume-sessions / daily-log features). Best-effort — never
|
|
181
|
+
// block a compaction on bookkeeping.
|
|
182
|
+
try {
|
|
183
|
+
const root = resolveRepoRoot(ctx.cwd);
|
|
184
|
+
touchSession(sid, root, runtime.currentStateDir);
|
|
185
|
+
logDaily(sid, "compact", result.checkpointId, saved, runtime.currentStateDir);
|
|
186
|
+
} catch {
|
|
187
|
+
/* non-fatal: stats bookkeeping only */
|
|
188
|
+
}
|
|
228
189
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
/* non-fatal */
|
|
251
|
-
}
|
|
252
|
-
}
|
|
190
|
+
// S21.2: best-effort consolidation of near-duplicate memories for this repo.
|
|
191
|
+
// Runs after the per-repo stats touch so `consolidateMemories` can use the
|
|
192
|
+
// same stateDir. Non-fatal — a failed consolidate never blocks a compaction.
|
|
193
|
+
// Only runs when new memory ops landed in this pass (otherwise the prior
|
|
194
|
+
// compaction's consolidate already had its shot — re-running would just
|
|
195
|
+
// touch every row again with no merges).
|
|
196
|
+
if (!result.deduped && runtime.memoriesTouchedThisCompaction > 0) {
|
|
197
|
+
try {
|
|
198
|
+
const root = resolveRepoRoot(ctx.cwd);
|
|
199
|
+
void consolidateMemories(runtime.currentStateDir, root).then(
|
|
200
|
+
(n) => {
|
|
201
|
+
if (n > 0) runtime.pushTicker(`${C.green}∫${C.reset} consolidated ${n} memory dup${n === 1 ? "" : "s"}`);
|
|
202
|
+
},
|
|
203
|
+
() => {
|
|
204
|
+
/* swallow: consolidate failures must never surface to the user */
|
|
205
|
+
},
|
|
206
|
+
);
|
|
207
|
+
} catch {
|
|
208
|
+
/* non-fatal */
|
|
209
|
+
}
|
|
210
|
+
}
|
|
253
211
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
runtime.pressureBand !== "low" &&
|
|
263
|
-
runtime.pressureBand !== "medium"
|
|
264
|
-
) {
|
|
265
|
-
void runMemoryReview(runtime, view, "pressure");
|
|
266
|
-
}
|
|
212
|
+
// S24 review-on-compact: when pressure is high, the just-compacted region is
|
|
213
|
+
// exactly the context worth remembering, so review it immediately rather than
|
|
214
|
+
// waiting for the next turn-cadence tick. Uses the shared runMemoryReview
|
|
215
|
+
// helper (fire-and-forget; doCompact is sync). Best-effort + non-fatal. Only
|
|
216
|
+
// fires above the `high` band so low-pressure compactions don't pay the cost.
|
|
217
|
+
if (!result.deduped && config.memoryAutoReview && runtime.pressureBand !== "low" && runtime.pressureBand !== "medium") {
|
|
218
|
+
void runMemoryReview(runtime, view, "pressure");
|
|
219
|
+
}
|
|
267
220
|
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
221
|
+
// Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
|
|
222
|
+
// skip re-vectorizing an already-compacted region (zero token cost).
|
|
223
|
+
pi.appendEntry(MARKER_TYPE, {
|
|
224
|
+
checkpointId: result.checkpointId,
|
|
225
|
+
regionHash: result.regionHash,
|
|
226
|
+
tokenEstimate: result.tokenEstimate,
|
|
227
|
+
deduped: result.deduped,
|
|
228
|
+
});
|
|
276
229
|
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
230
|
+
// Fix D: refresh the RAPTOR tree for this session so live recall (search) can
|
|
231
|
+
// serve high-level summaries. Best-effort + non-fatal: never block compaction.
|
|
232
|
+
// Budget-guarded (RAPTOR_BUDGET_MS) so it can't hang a large session.
|
|
233
|
+
if (config.raptorEnabled && !result.deduped) {
|
|
234
|
+
try {
|
|
235
|
+
const dd = loadDedupConfig();
|
|
236
|
+
const all = runtime.store.list(sid);
|
|
237
|
+
const leaves = all.map((cp) => ({
|
|
238
|
+
id: cp.checkpointId,
|
|
239
|
+
messages: [],
|
|
240
|
+
sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
|
|
241
|
+
embedding: cp.embedding,
|
|
242
|
+
}));
|
|
243
|
+
if (leaves.length >= 2) {
|
|
244
|
+
// S25: stamp the tree with the newest checkpoint epoch so the
|
|
245
|
+
// freshness guard in raptorSearchHits can reject stale trees after a
|
|
246
|
+
// later compaction adds newer checkpoints.
|
|
247
|
+
const builtAt = all.length > 0 ? Math.max(...all.map((c) => c.timestamp)) : Date.now();
|
|
248
|
+
runRaptor(
|
|
249
|
+
leaves,
|
|
250
|
+
{
|
|
251
|
+
stateDir: runtime.currentStateDir,
|
|
252
|
+
sessionId: sid,
|
|
253
|
+
budgetMs: dd.RAPTOR_BUDGET_MS,
|
|
254
|
+
clustersPerLevel: dd.RAPTOR_CLUSTERS_PER_LEVEL,
|
|
255
|
+
consistencyThreshold: dd.RAPTOR_CONSISTENCY,
|
|
256
|
+
logger: runtime.logger,
|
|
257
|
+
builtAt: Number.isFinite(builtAt) ? builtAt : Date.now(),
|
|
258
|
+
},
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
} catch {
|
|
262
|
+
/* non-fatal: tree refresh never blocks a compaction */
|
|
263
|
+
}
|
|
264
|
+
}
|
|
312
265
|
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
266
|
+
// Slice 2: best-effort mirror of the new checkpoint into the async global
|
|
267
|
+
// PGlite/HNSW vector index. Fires once per compaction (not per-add), so the
|
|
268
|
+
// shared global dir is never hammered by concurrent test workers.
|
|
269
|
+
// Non-fatal: a WASM init failure degrades to the sync scan silently.
|
|
270
|
+
if (!result.deduped) {
|
|
271
|
+
try {
|
|
272
|
+
const all = runtime.store.list(sid);
|
|
273
|
+
const latest = all.find((cp) => cp.checkpointId === result.checkpointId);
|
|
274
|
+
if (latest?.embedding) {
|
|
275
|
+
void indexUpsertEmbedding(
|
|
276
|
+
runtime.currentStateDir,
|
|
277
|
+
sid,
|
|
278
|
+
latest.checkpointId,
|
|
279
|
+
latest.embedding,
|
|
280
|
+
).catch(() => {
|
|
281
|
+
/* non-fatal: index refresh never blocks a compaction */
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
} catch {
|
|
285
|
+
/* non-fatal: index refresh never blocks a compaction */
|
|
286
|
+
}
|
|
287
|
+
}
|
|
335
288
|
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
289
|
+
runtime.setStatus(
|
|
290
|
+
ctx,
|
|
291
|
+
runtime.rt.persistedThisSession
|
|
292
|
+
? `mega-compact: ${result.checkpointId} · ${saved} tok saved`
|
|
293
|
+
: `mega-compact: ready`,
|
|
294
|
+
);
|
|
295
|
+
runtime.logger.info("compact", {
|
|
296
|
+
sessionId: sid,
|
|
297
|
+
checkpointId: result.checkpointId ?? "(deduped)",
|
|
298
|
+
deduped: result.deduped,
|
|
299
|
+
tokenEstimate: saved,
|
|
300
|
+
compactedFrom: result.compactedFrom,
|
|
301
|
+
});
|
|
302
|
+
runtime.dashboard.event("compact", {
|
|
303
|
+
sessionId: sid,
|
|
304
|
+
checkpointId: result.checkpointId ?? "(deduped)",
|
|
305
|
+
deduped: result.deduped,
|
|
306
|
+
tokenEstimate: saved,
|
|
307
|
+
compactedFrom: result.compactedFrom,
|
|
308
|
+
});
|
|
309
|
+
runtime.snapshot(ctx);
|
|
310
|
+
return { skipped: false, result, keepFrom, saved };
|
|
358
311
|
}
|
|
359
312
|
|
|
360
313
|
/**
|
|
@@ -394,55 +347,51 @@ function doCompact(
|
|
|
394
347
|
* is always safe; calling `ctx.compact()` on a no-op throws to the user.
|
|
395
348
|
*/
|
|
396
349
|
export function piCompactWouldNoop(ctx: ExtensionContext): boolean {
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
return false;
|
|
435
|
-
} catch {
|
|
436
|
-
return true; // safe: skip the durable trim rather than risk a user-facing throw
|
|
437
|
-
}
|
|
350
|
+
try {
|
|
351
|
+
const branch = ctx.sessionManager.getBranch();
|
|
352
|
+
if (branch.length === 0) return true;
|
|
353
|
+
// (1) already compacted — pi throws "Already compacted"
|
|
354
|
+
if (branch[branch.length - 1].type === "compaction") return true;
|
|
355
|
+
// boundaryStart = index just after the most recent compaction entry (or 0)
|
|
356
|
+
let boundaryStart = 0;
|
|
357
|
+
for (let i = branch.length - 1; i >= 0; i--) {
|
|
358
|
+
if (branch[i].type === "compaction") { boundaryStart = i + 1; break; }
|
|
359
|
+
}
|
|
360
|
+
let cutPoints = 0;
|
|
361
|
+
let tokens = 0;
|
|
362
|
+
for (let i = boundaryStart; i < branch.length; i++) {
|
|
363
|
+
const e = branch[i];
|
|
364
|
+
if (e.type === "compaction") continue;
|
|
365
|
+
let isCut = false;
|
|
366
|
+
for (const m of sessionEntryToContextMessages(e)) {
|
|
367
|
+
// pi's isCutPointMessage: every role except toolResult
|
|
368
|
+
if ((m as { role?: string }).role !== "toolResult") isCut = true;
|
|
369
|
+
const c = (m as { content?: unknown }).content;
|
|
370
|
+
const text =
|
|
371
|
+
typeof c === "string" ? c
|
|
372
|
+
: Array.isArray(c)
|
|
373
|
+
? (c as { text?: string }[]).map((b) => b?.text ?? "").join(" ")
|
|
374
|
+
: "";
|
|
375
|
+
if (text) tokens += estimateBlockTokens(text);
|
|
376
|
+
}
|
|
377
|
+
if (isCut) cutPoints++;
|
|
378
|
+
}
|
|
379
|
+
// (2) need >=2 cut points so the kept cut isn't the first message
|
|
380
|
+
if (cutPoints < 2) return true;
|
|
381
|
+
// (3) transcript under pi's keepRecentTokens budget → pi keeps everything
|
|
382
|
+
if (tokens < durableTrimFloorTokens()) return true;
|
|
383
|
+
return false;
|
|
384
|
+
} catch {
|
|
385
|
+
return true; // safe: skip the durable trim rather than risk a user-facing throw
|
|
386
|
+
}
|
|
438
387
|
}
|
|
439
388
|
|
|
440
389
|
/** pi's default keepRecentTokens (compaction settings). Override with
|
|
441
390
|
* MEGACOMPACT_DURABLE_TRIM_FLOOR if you raise pi's compact.keepRecentTokens. */
|
|
442
391
|
function durableTrimFloorTokens(): number {
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
392
|
+
const raw = process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
|
|
393
|
+
if (raw !== undefined && Number.isFinite(Number(raw))) return Number(raw);
|
|
394
|
+
return 20_000;
|
|
446
395
|
}
|
|
447
396
|
|
|
448
397
|
/**
|
|
@@ -451,54 +400,42 @@ function durableTrimFloorTokens(): number {
|
|
|
451
400
|
* or report it (command).
|
|
452
401
|
*/
|
|
453
402
|
export function doRecall(
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
403
|
+
runtime: MegaRuntime,
|
|
404
|
+
config: MegaConfig,
|
|
405
|
+
ctx: ExtensionContext,
|
|
406
|
+
query: string,
|
|
407
|
+
source: "resume" | "command",
|
|
459
408
|
) {
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
const label = files.length
|
|
491
|
-
? files
|
|
492
|
-
.map((f) => f.split("/").pop() ?? f)
|
|
493
|
-
.slice(0, 2)
|
|
494
|
-
.join(", ")
|
|
495
|
-
: top.checkpoint.checkpointId;
|
|
496
|
-
runtime.pushTicker(
|
|
497
|
-
`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`,
|
|
498
|
-
);
|
|
499
|
-
runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
|
|
500
|
-
}
|
|
501
|
-
return result;
|
|
409
|
+
runtime.bindRepo(ctx.cwd);
|
|
410
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
411
|
+
// Live window text for inline dedupe (Fix C): drop recalled checkpoints that
|
|
412
|
+
// are already resident in the session, so recall never re-injects context the
|
|
413
|
+
// model can already see. Best-effort — an empty window just skips dedupe.
|
|
414
|
+
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
415
|
+
const result = recallAndInline(
|
|
416
|
+
{
|
|
417
|
+
sessionId: sid,
|
|
418
|
+
query,
|
|
419
|
+
limit: config.autoInlineK,
|
|
420
|
+
source,
|
|
421
|
+
skipInjected: true,
|
|
422
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
423
|
+
windowDedupe: config.windowDedupe,
|
|
424
|
+
liveWindow,
|
|
425
|
+
dedupSim: config.dedupSim,
|
|
426
|
+
},
|
|
427
|
+
runtime.store,
|
|
428
|
+
);
|
|
429
|
+
runtime.dashboard.event("recall", { source, query: query.slice(0, 120), injected: result.toInject.length, empty: result.empty });
|
|
430
|
+
if (!result.empty && result.toInject.length > 0) {
|
|
431
|
+
const top = result.toInject[0];
|
|
432
|
+
const scorePct = Math.round((top.score ?? 0) * 100);
|
|
433
|
+
const files = top.checkpoint.filesModified ?? [];
|
|
434
|
+
const label = files.length ? files.map((f) => f.split("/").pop() ?? f).slice(0, 2).join(", ") : top.checkpoint.checkpointId;
|
|
435
|
+
runtime.pushTicker(`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`);
|
|
436
|
+
runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
|
|
437
|
+
}
|
|
438
|
+
return result;
|
|
502
439
|
}
|
|
503
440
|
|
|
504
441
|
/**
|
|
@@ -513,81 +450,58 @@ export function doRecall(
|
|
|
513
450
|
* the same-repo result unchanged.
|
|
514
451
|
*/
|
|
515
452
|
export async function doRecallAsync(
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
453
|
+
runtime: MegaRuntime,
|
|
454
|
+
config: MegaConfig,
|
|
455
|
+
ctx: ExtensionContext,
|
|
456
|
+
query: string,
|
|
457
|
+
source: "resume" | "command",
|
|
458
|
+
opts: { crossRepo?: boolean } = {},
|
|
522
459
|
): Promise<RecallInjectResult> {
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
const seen = new Set(
|
|
569
|
-
sameRepo.toInject.map((h) => h.checkpoint.checkpointId),
|
|
570
|
-
);
|
|
571
|
-
const merged = [...sameRepo.toInject];
|
|
572
|
-
for (const h of x.toInject) {
|
|
573
|
-
if (!seen.has(h.checkpoint.checkpointId)) {
|
|
574
|
-
merged.push(h);
|
|
575
|
-
seen.add(h.checkpoint.checkpointId);
|
|
576
|
-
}
|
|
577
|
-
}
|
|
578
|
-
const block = merged.length ? formatRecallBlock(merged) : "";
|
|
579
|
-
return {
|
|
580
|
-
toInject: merged,
|
|
581
|
-
report: merged.map(
|
|
582
|
-
(h) =>
|
|
583
|
-
` • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`,
|
|
584
|
-
),
|
|
585
|
-
block,
|
|
586
|
-
empty: merged.length === 0,
|
|
587
|
-
};
|
|
588
|
-
} catch {
|
|
589
|
-
return sameRepo; // cross-repo failure → same-repo only (non-fatal)
|
|
590
|
-
}
|
|
460
|
+
runtime.bindRepo(ctx.cwd);
|
|
461
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
462
|
+
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
463
|
+
// Sync same-repo first (fast, never blocks).
|
|
464
|
+
const sameRepo = recallAndInline(
|
|
465
|
+
{
|
|
466
|
+
sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
|
|
467
|
+
recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
|
|
468
|
+
liveWindow, dedupSim: config.dedupSim,
|
|
469
|
+
},
|
|
470
|
+
runtime.store,
|
|
471
|
+
);
|
|
472
|
+
if (!config.crossRepoEnabled || !opts.crossRepo) return sameRepo;
|
|
473
|
+
if (sameRepo.toInject.length >= config.autoInlineK) return sameRepo; // same-repo satisfied
|
|
474
|
+
// Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
|
|
475
|
+
try {
|
|
476
|
+
const x = await recallAndInlineAsync(
|
|
477
|
+
{
|
|
478
|
+
sessionId: sid, query, limit: config.autoInlineK, source, skipInjected: true,
|
|
479
|
+
recallMaxTokens: config.recallMaxTokens, windowDedupe: config.windowDedupe,
|
|
480
|
+
liveWindow, dedupSim: config.crossRepoCosine, crossRepo: true,
|
|
481
|
+
globalIndexDir: process.env.MEGACOMPACT_INDEX_DIR,
|
|
482
|
+
},
|
|
483
|
+
runtime.store,
|
|
484
|
+
);
|
|
485
|
+
runtime.dashboard.event("recall-crossrepo", {
|
|
486
|
+
source, query: query.slice(0, 120), injected: x.toInject.length,
|
|
487
|
+
sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
|
|
488
|
+
});
|
|
489
|
+
// Merge, dedup by checkpointId, respect the same token cap by reformatting.
|
|
490
|
+
const seen = new Set(sameRepo.toInject.map((h) => h.checkpoint.checkpointId));
|
|
491
|
+
const merged = [...sameRepo.toInject];
|
|
492
|
+
for (const h of x.toInject) {
|
|
493
|
+
if (!seen.has(h.checkpoint.checkpointId)) { merged.push(h); seen.add(h.checkpoint.checkpointId); }
|
|
494
|
+
}
|
|
495
|
+
const block = merged.length ? formatRecallBlock(merged) : "";
|
|
496
|
+
return {
|
|
497
|
+
toInject: merged,
|
|
498
|
+
report: merged.map((h) => ` • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`),
|
|
499
|
+
block,
|
|
500
|
+
empty: merged.length === 0,
|
|
501
|
+
};
|
|
502
|
+
} catch {
|
|
503
|
+
return sameRepo; // cross-repo failure → same-repo only (non-fatal)
|
|
504
|
+
}
|
|
591
505
|
}
|
|
592
506
|
|
|
593
507
|
/**
|
|
@@ -597,19 +511,18 @@ export async function doRecallAsync(
|
|
|
597
511
|
* Mirrors recentUserQuery's use of sessionEntryToContextMessages.
|
|
598
512
|
*/
|
|
599
513
|
function extractLiveWindow(ctx: ExtensionContext): string[] {
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
}
|
|
514
|
+
try {
|
|
515
|
+
const entries = ctx.sessionManager.getEntries();
|
|
516
|
+
const texts: string[] = [];
|
|
517
|
+
for (const e of entries) {
|
|
518
|
+
for (const m of sessionEntryToContextMessages(e)) {
|
|
519
|
+
const c = (m as { content?: unknown }).content;
|
|
520
|
+
if (typeof c === "string") texts.push(c);
|
|
521
|
+
else if (Array.isArray(c)) texts.push(c.map((b: any) => b.text).join(" "));
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
return texts;
|
|
525
|
+
} catch {
|
|
526
|
+
return [];
|
|
527
|
+
}
|
|
615
528
|
}
|