pi-mega-compact 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/mega-compact-driver.js +1 -0
- package/dist/extensions/mega-pipeline.js +53 -17
- package/dist/extensions/mega-runtime.js +319 -122
- package/extensions/mega-compact-driver.ts +56 -55
- package/extensions/mega-pipeline.ts +481 -393
- package/extensions/mega-runtime.ts +934 -615
- package/package.json +1 -1
|
@@ -7,29 +7,42 @@
|
|
|
7
7
|
* driven by the event + command handlers in mega-events.ts / mega-commands.ts.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
import type {
|
|
10
|
+
import type {
|
|
11
|
+
ExtensionAPI,
|
|
12
|
+
ExtensionContext,
|
|
13
|
+
} from "@earendil-works/pi-coding-agent";
|
|
11
14
|
import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
12
15
|
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
|
13
16
|
import { compactSession } from "../src/engine.js";
|
|
14
17
|
import type { EngineMessage } from "../src/types.js";
|
|
15
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
recallAndInline,
|
|
20
|
+
recallAndInlineAsync,
|
|
21
|
+
formatRecallBlock,
|
|
22
|
+
type RecallInjectResult,
|
|
23
|
+
} from "../src/recall.js";
|
|
16
24
|
import { normalizeSessionId } from "../src/store.js";
|
|
17
25
|
import { estimateBlockTokens } from "../src/tokens.js";
|
|
18
26
|
import { touchSession, logDaily } from "../src/store/sqlite.js";
|
|
19
27
|
import { consolidateMemories } from "../src/memory.js";
|
|
28
|
+
import { type MegaRuntime, C, MARKER_TYPE } from "./mega-runtime.js";
|
|
20
29
|
import {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
} from "./mega-
|
|
25
|
-
import { resolveRepoRoot, preserveRecentForPressure, type MegaConfig } from "./mega-config.js";
|
|
30
|
+
resolveRepoRoot,
|
|
31
|
+
preserveRecentForPressure,
|
|
32
|
+
type MegaConfig,
|
|
33
|
+
} from "./mega-config.js";
|
|
26
34
|
import { runRaptor } from "../src/dedup/raptor/index.js";
|
|
27
35
|
import { loadDedupConfig } from "../src/config/dedup.js";
|
|
28
36
|
import { upsertEmbedding as indexUpsertEmbedding } from "../src/store/vectorIndex.js";
|
|
29
37
|
|
|
30
38
|
export type RunCompactResult =
|
|
31
|
-
|
|
32
|
-
|
|
39
|
+
| { skipped: true }
|
|
40
|
+
| {
|
|
41
|
+
skipped: false;
|
|
42
|
+
result: ReturnType<typeof compactSession>;
|
|
43
|
+
keepFrom: number;
|
|
44
|
+
saved: number;
|
|
45
|
+
};
|
|
33
46
|
|
|
34
47
|
/**
|
|
35
48
|
* Review the live conversation and persist durable memories (S20+S24). Shared by
|
|
@@ -42,271 +55,306 @@ export type RunCompactResult =
|
|
|
42
55
|
* @param label a short source tag for the ticker line (e.g. "pressure" / "turn")
|
|
43
56
|
*/
|
|
44
57
|
export async function runMemoryReview(
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
58
|
+
runtime: MegaRuntime,
|
|
59
|
+
view: ReturnType<MegaRuntime["engineView"]>,
|
|
60
|
+
label: string,
|
|
48
61
|
): Promise<number> {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
try {
|
|
63
|
+
const { reviewConversation } = await import("../src/memory.js");
|
|
64
|
+
const { applyMemoryOps } = await import("../src/memoryOps.js");
|
|
65
|
+
const ops = reviewConversation(view, []);
|
|
66
|
+
if (ops.length) {
|
|
67
|
+
await applyMemoryOps(ops, runtime.currentStateDir);
|
|
68
|
+
// S21.2: ops landed — the compaction path reads this counter and fires
|
|
69
|
+
// `consolidateMemories` only when > 0.
|
|
70
|
+
runtime.memoriesTouchedThisCompaction += ops.length;
|
|
71
|
+
runtime.pushTicker(
|
|
72
|
+
`${C.green}🧠${C.reset} reviewed ${ops.length} memory op${ops.length === 1 ? "" : "s"} (${label})`,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
return ops.length;
|
|
76
|
+
} catch {
|
|
77
|
+
/* non-fatal — auto-review must never break the turn loop / compaction */
|
|
78
|
+
return 0;
|
|
79
|
+
}
|
|
65
80
|
}
|
|
66
81
|
|
|
67
82
|
/** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
|
|
68
83
|
export function runCompact(
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
84
|
+
pi: ExtensionAPI,
|
|
85
|
+
runtime: MegaRuntime,
|
|
86
|
+
config: MegaConfig,
|
|
87
|
+
ctx: ExtensionContext,
|
|
88
|
+
messages: AgentMessage[],
|
|
89
|
+
opts: {
|
|
90
|
+
keepFrom?: number;
|
|
91
|
+
summary?: string;
|
|
92
|
+
compressionPressure?: number;
|
|
93
|
+
} = {},
|
|
75
94
|
): RunCompactResult {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
95
|
+
runtime.bindRepo(ctx.cwd);
|
|
96
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
97
|
+
runtime.resetRuntime(sid);
|
|
98
|
+
runtime.rt.sessionId = sid;
|
|
80
99
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
100
|
+
const view = runtime.engineView(messages);
|
|
101
|
+
// keepFrom deepens with context pressure (Fix E): under high pressure we
|
|
102
|
+
// compact more of the session, down to the preserveRecentMin floor.
|
|
103
|
+
const preserve = preserveRecentForPressure(
|
|
104
|
+
opts.compressionPressure ?? 0,
|
|
105
|
+
config.preserveRecent,
|
|
106
|
+
config.preserveRecentMin,
|
|
107
|
+
);
|
|
108
|
+
const keepFrom = opts.keepFrom ?? Math.max(0, view.length - preserve);
|
|
109
|
+
// For very small sessions (fewer messages than preserveRecent), allow
|
|
110
|
+
// compacting everything except the last message — the user explicitly
|
|
111
|
+
// requested compaction, so don't refuse it just because the session is short.
|
|
112
|
+
if (keepFrom <= 0) {
|
|
113
|
+
if (view.length <= 1) return { skipped: true };
|
|
114
|
+
// Use the fallback: compact everything except the last message
|
|
115
|
+
const fallbackKeepFrom = view.length - 1;
|
|
116
|
+
return doCompact(
|
|
117
|
+
view,
|
|
118
|
+
fallbackKeepFrom,
|
|
119
|
+
opts,
|
|
120
|
+
sid,
|
|
121
|
+
config,
|
|
122
|
+
pi,
|
|
123
|
+
ctx,
|
|
124
|
+
runtime,
|
|
125
|
+
);
|
|
126
|
+
}
|
|
99
127
|
|
|
100
|
-
|
|
128
|
+
return doCompact(view, keepFrom, opts, sid, config, pi, ctx, runtime);
|
|
101
129
|
}
|
|
102
130
|
|
|
103
131
|
function doCompact(
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
132
|
+
view: EngineMessage[],
|
|
133
|
+
keepFrom: number,
|
|
134
|
+
opts: { keepFrom?: number; summary?: string; compressionPressure?: number },
|
|
135
|
+
sid: string,
|
|
136
|
+
config: MegaConfig,
|
|
137
|
+
pi: ExtensionAPI,
|
|
138
|
+
ctx: ExtensionContext,
|
|
139
|
+
runtime: MegaRuntime,
|
|
112
140
|
): RunCompactResult {
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
141
|
+
runtime.pulsing = true; // animate the status line while the (sync) pipeline runs
|
|
142
|
+
// S21.2: reset the per-compaction memory-op counter so the post-compact
|
|
143
|
+
// consolidate pass only fires when memory rows actually changed during the
|
|
144
|
+
// compaction window (turn_end → auto-review may have written some).
|
|
145
|
+
runtime.memoriesTouchedThisCompaction = 0;
|
|
146
|
+
const result = compactSession(
|
|
147
|
+
{
|
|
148
|
+
sessionId: sid,
|
|
149
|
+
messages: view,
|
|
150
|
+
keepFrom,
|
|
151
|
+
summary: opts.summary,
|
|
152
|
+
timestamp: Date.now(),
|
|
153
|
+
onTier: runtime.makeTierCallback(ctx),
|
|
154
|
+
compressionPressure: opts.compressionPressure,
|
|
155
|
+
},
|
|
156
|
+
runtime.store,
|
|
157
|
+
);
|
|
158
|
+
runtime.pulsing = false;
|
|
131
159
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
160
|
+
if (result.skipped) return { skipped: true };
|
|
161
|
+
if (!result.deduped) {
|
|
162
|
+
runtime.rt.persistedThisSession = true;
|
|
163
|
+
runtime.rt.lastCheckpointId = result.checkpointId;
|
|
164
|
+
}
|
|
165
|
+
runtime.rt.lastCompactedFrom = result.compactedFrom;
|
|
166
|
+
runtime.rt.lastCompactedTokens = result.tokenEstimate;
|
|
167
|
+
runtime.rt.dedupAttempts++;
|
|
168
|
+
// Honest "tokens saved" for this session-instance only:
|
|
169
|
+
// new checkpoint → original − stored
|
|
170
|
+
// deduped onto existing → whole original region (nothing new stored)
|
|
171
|
+
// Resets to 0 on session_start (rt is rebuilt) — so a fresh session shows 0
|
|
172
|
+
// while the repo's cumulative saved (SQLite meta) keeps the running total.
|
|
173
|
+
const saved = result.deduped
|
|
174
|
+
? result.originalTokenEstimate
|
|
175
|
+
: Math.max(0, result.originalTokenEstimate - result.tokenEstimate);
|
|
176
|
+
runtime.rt.tokensSaved += saved;
|
|
177
|
+
runtime.rt.lastCompactAt = Date.now();
|
|
178
|
+
if (result.deduped) runtime.rt.dedupSkips++;
|
|
179
|
+
// Grow the rolling "saved" goal so the progress bar always has a fresh
|
|
180
|
+
// denominator (we don't want it pinned at 100% once we pass an old target).
|
|
181
|
+
if (runtime.rt.tokensSaved > runtime.savedGoal)
|
|
182
|
+
runtime.savedGoal =
|
|
183
|
+
Math.ceil((runtime.rt.tokensSaved * 1.25) / 10_000) * 10_000;
|
|
153
184
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
185
|
+
// Live toolbar activity: what file/region just got compacted or deduped.
|
|
186
|
+
// Rendered via the rotating ticker line (see snapshot); the ring buffer is
|
|
187
|
+
// cycled one-per-repaint so the single line scrolls through recent files.
|
|
188
|
+
const files = result.filesModified ?? [];
|
|
189
|
+
const fileLabel = files.length
|
|
190
|
+
? files
|
|
191
|
+
.map((f) => f.split("/").pop() ?? f)
|
|
192
|
+
.slice(0, 2)
|
|
193
|
+
.join(", ")
|
|
194
|
+
: result.regionHash.slice(0, 8);
|
|
195
|
+
runtime.lastActivityAt = Date.now();
|
|
196
|
+
// Explain-why line: surfaced while fresh. Pulls the dedup reason (which for
|
|
197
|
+
// L2 includes the cosine sim) so the user sees WHY a region was kept/dropped.
|
|
198
|
+
runtime.lastWhy = result.deduped
|
|
199
|
+
? `why: deduped@${result.dedupReason ?? "tier"}`
|
|
200
|
+
: `why: compacted → ${result.checkpointId}`;
|
|
201
|
+
// Recall/activity ticker: record this event in the ring buffer.
|
|
202
|
+
const savedK = (saved / 1000).toFixed(1);
|
|
203
|
+
runtime.pushTicker(
|
|
204
|
+
result.deduped
|
|
205
|
+
? `${C.green}♻${C.reset} deduped ${fileLabel} · ${savedK}k saved`
|
|
206
|
+
: `${C.cyan}🗜${C.reset} ${result.checkpointId} · +${savedK}k · ${fileLabel}`,
|
|
207
|
+
);
|
|
208
|
+
// The per-tier trace has settled into the final outcome — fold it back into
|
|
209
|
+
// the activity line and stop showing the live trace.
|
|
210
|
+
runtime.tierTrace = undefined;
|
|
177
211
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
212
|
+
// Record session activity + a daily-log entry in the per-repo SQLite store
|
|
213
|
+
// (foundation for resume-sessions / daily-log features). Best-effort — never
|
|
214
|
+
// block a compaction on bookkeeping.
|
|
215
|
+
try {
|
|
216
|
+
const root = resolveRepoRoot(ctx.cwd);
|
|
217
|
+
touchSession(sid, root, runtime.currentStateDir);
|
|
218
|
+
logDaily(
|
|
219
|
+
sid,
|
|
220
|
+
"compact",
|
|
221
|
+
result.checkpointId,
|
|
222
|
+
saved,
|
|
223
|
+
runtime.currentStateDir,
|
|
224
|
+
);
|
|
225
|
+
} catch {
|
|
226
|
+
/* non-fatal: stats bookkeeping only */
|
|
227
|
+
}
|
|
188
228
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
229
|
+
// S21.2: best-effort consolidation of near-duplicate memories for this repo.
|
|
230
|
+
// Runs after the per-repo stats touch so `consolidateMemories` can use the
|
|
231
|
+
// same stateDir. Non-fatal — a failed consolidate never blocks a compaction.
|
|
232
|
+
// Only runs when new memory ops landed in this pass (otherwise the prior
|
|
233
|
+
// compaction's consolidate already had its shot — re-running would just
|
|
234
|
+
// touch every row again with no merges).
|
|
235
|
+
if (!result.deduped && runtime.memoriesTouchedThisCompaction > 0) {
|
|
236
|
+
try {
|
|
237
|
+
const root = resolveRepoRoot(ctx.cwd);
|
|
238
|
+
void consolidateMemories(runtime.currentStateDir, root).then(
|
|
239
|
+
(n) => {
|
|
240
|
+
if (n > 0)
|
|
241
|
+
runtime.pushTicker(
|
|
242
|
+
`${C.green}∫${C.reset} consolidated ${n} memory dup${n === 1 ? "" : "s"}`,
|
|
243
|
+
);
|
|
244
|
+
},
|
|
245
|
+
() => {
|
|
246
|
+
/* swallow: consolidate failures must never surface to the user */
|
|
247
|
+
},
|
|
248
|
+
);
|
|
249
|
+
} catch {
|
|
250
|
+
/* non-fatal */
|
|
251
|
+
}
|
|
252
|
+
}
|
|
210
253
|
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
254
|
+
// S24 review-on-compact: when pressure is high, the just-compacted region is
|
|
255
|
+
// exactly the context worth remembering, so review it immediately rather than
|
|
256
|
+
// waiting for the next turn-cadence tick. Uses the shared runMemoryReview
|
|
257
|
+
// helper (fire-and-forget; doCompact is sync). Best-effort + non-fatal. Only
|
|
258
|
+
// fires above the `high` band so low-pressure compactions don't pay the cost.
|
|
259
|
+
if (
|
|
260
|
+
!result.deduped &&
|
|
261
|
+
config.memoryAutoReview &&
|
|
262
|
+
runtime.pressureBand !== "low" &&
|
|
263
|
+
runtime.pressureBand !== "medium"
|
|
264
|
+
) {
|
|
265
|
+
void runMemoryReview(runtime, view, "pressure");
|
|
266
|
+
}
|
|
219
267
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
268
|
+
// Sentinel marker: a non-LLM bookkeeping entry so subsequent triggers can
|
|
269
|
+
// skip re-vectorizing an already-compacted region (zero token cost).
|
|
270
|
+
pi.appendEntry(MARKER_TYPE, {
|
|
271
|
+
checkpointId: result.checkpointId,
|
|
272
|
+
regionHash: result.regionHash,
|
|
273
|
+
tokenEstimate: result.tokenEstimate,
|
|
274
|
+
deduped: result.deduped,
|
|
275
|
+
});
|
|
228
276
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
277
|
+
// Fix D: refresh the RAPTOR tree for this session so live recall (search) can
|
|
278
|
+
// serve high-level summaries. Best-effort + non-fatal: never block compaction.
|
|
279
|
+
// Budget-guarded (RAPTOR_BUDGET_MS) so it can't hang a large session.
|
|
280
|
+
if (config.raptorEnabled && !result.deduped) {
|
|
281
|
+
try {
|
|
282
|
+
const dd = loadDedupConfig();
|
|
283
|
+
const all = runtime.store.list(sid);
|
|
284
|
+
const leaves = all.map((cp) => ({
|
|
285
|
+
id: cp.checkpointId,
|
|
286
|
+
messages: [],
|
|
287
|
+
sourceText: cp.normalizedText ?? cp.summary ?? cp.regionHash,
|
|
288
|
+
embedding: cp.embedding,
|
|
289
|
+
}));
|
|
290
|
+
if (leaves.length >= 2) {
|
|
291
|
+
// S25: stamp the tree with the newest checkpoint epoch so the
|
|
292
|
+
// freshness guard in raptorSearchHits can reject stale trees after a
|
|
293
|
+
// later compaction adds newer checkpoints.
|
|
294
|
+
const builtAt =
|
|
295
|
+
all.length > 0
|
|
296
|
+
? Math.max(...all.map((c) => c.timestamp))
|
|
297
|
+
: Date.now();
|
|
298
|
+
runRaptor(leaves, {
|
|
299
|
+
stateDir: runtime.currentStateDir,
|
|
300
|
+
sessionId: sid,
|
|
301
|
+
budgetMs: dd.RAPTOR_BUDGET_MS,
|
|
302
|
+
clustersPerLevel: dd.RAPTOR_CLUSTERS_PER_LEVEL,
|
|
303
|
+
consistencyThreshold: dd.RAPTOR_CONSISTENCY,
|
|
304
|
+
logger: runtime.logger,
|
|
305
|
+
builtAt: Number.isFinite(builtAt) ? builtAt : Date.now(),
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
} catch {
|
|
309
|
+
/* non-fatal: tree refresh never blocks a compaction */
|
|
310
|
+
}
|
|
311
|
+
}
|
|
264
312
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
313
|
+
// Slice 2: best-effort mirror of the new checkpoint into the async global
|
|
314
|
+
// PGlite/HNSW vector index. Fires once per compaction (not per-add), so the
|
|
315
|
+
// shared global dir is never hammered by concurrent test workers.
|
|
316
|
+
// Non-fatal: a WASM init failure degrades to the sync scan silently.
|
|
317
|
+
if (!result.deduped) {
|
|
318
|
+
try {
|
|
319
|
+
const all = runtime.store.list(sid);
|
|
320
|
+
const latest = all.find((cp) => cp.checkpointId === result.checkpointId);
|
|
321
|
+
if (latest?.embedding) {
|
|
322
|
+
void indexUpsertEmbedding(
|
|
323
|
+
runtime.currentStateDir,
|
|
324
|
+
sid,
|
|
325
|
+
latest.checkpointId,
|
|
326
|
+
latest.embedding,
|
|
327
|
+
).catch(() => {
|
|
328
|
+
/* non-fatal: index refresh never blocks a compaction */
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
} catch {
|
|
332
|
+
/* non-fatal: index refresh never blocks a compaction */
|
|
333
|
+
}
|
|
334
|
+
}
|
|
287
335
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
336
|
+
runtime.setStatus(
|
|
337
|
+
ctx,
|
|
338
|
+
runtime.rt.persistedThisSession
|
|
339
|
+
? `mega-compact: ${result.checkpointId} · ${saved} tok saved`
|
|
340
|
+
: `mega-compact: ready`,
|
|
341
|
+
);
|
|
342
|
+
runtime.logger.info("compact", {
|
|
343
|
+
sessionId: sid,
|
|
344
|
+
checkpointId: result.checkpointId ?? "(deduped)",
|
|
345
|
+
deduped: result.deduped,
|
|
346
|
+
tokenEstimate: saved,
|
|
347
|
+
compactedFrom: result.compactedFrom,
|
|
348
|
+
});
|
|
349
|
+
runtime.dashboard.event("compact", {
|
|
350
|
+
sessionId: sid,
|
|
351
|
+
checkpointId: result.checkpointId ?? "(deduped)",
|
|
352
|
+
deduped: result.deduped,
|
|
353
|
+
tokenEstimate: saved,
|
|
354
|
+
compactedFrom: result.compactedFrom,
|
|
355
|
+
});
|
|
356
|
+
runtime.snapshot(ctx);
|
|
357
|
+
return { skipped: false, result, keepFrom, saved };
|
|
310
358
|
}
|
|
311
359
|
|
|
312
360
|
/**
|
|
@@ -346,51 +394,55 @@ function doCompact(
|
|
|
346
394
|
* is always safe; calling `ctx.compact()` on a no-op throws to the user.
|
|
347
395
|
*/
|
|
348
396
|
export function piCompactWouldNoop(ctx: ExtensionContext): boolean {
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
397
|
+
try {
|
|
398
|
+
const branch = ctx.sessionManager.getBranch();
|
|
399
|
+
if (branch.length === 0) return true;
|
|
400
|
+
// (1) already compacted — pi throws "Already compacted"
|
|
401
|
+
if (branch[branch.length - 1].type === "compaction") return true;
|
|
402
|
+
// boundaryStart = index just after the most recent compaction entry (or 0)
|
|
403
|
+
let boundaryStart = 0;
|
|
404
|
+
for (let i = branch.length - 1; i >= 0; i--) {
|
|
405
|
+
if (branch[i].type === "compaction") {
|
|
406
|
+
boundaryStart = i + 1;
|
|
407
|
+
break;
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
let cutPoints = 0;
|
|
411
|
+
let tokens = 0;
|
|
412
|
+
for (let i = boundaryStart; i < branch.length; i++) {
|
|
413
|
+
const e = branch[i];
|
|
414
|
+
if (e.type === "compaction") continue;
|
|
415
|
+
let isCut = false;
|
|
416
|
+
for (const m of sessionEntryToContextMessages(e)) {
|
|
417
|
+
// pi's isCutPointMessage: every role except toolResult
|
|
418
|
+
if ((m as { role?: string }).role !== "toolResult") isCut = true;
|
|
419
|
+
const c = (m as { content?: unknown }).content;
|
|
420
|
+
const text =
|
|
421
|
+
typeof c === "string"
|
|
422
|
+
? c
|
|
423
|
+
: Array.isArray(c)
|
|
424
|
+
? (c as { text?: string }[]).map((b) => b?.text ?? "").join(" ")
|
|
425
|
+
: "";
|
|
426
|
+
if (text) tokens += estimateBlockTokens(text);
|
|
427
|
+
}
|
|
428
|
+
if (isCut) cutPoints++;
|
|
429
|
+
}
|
|
430
|
+
// (2) need >=2 cut points so the kept cut isn't the first message
|
|
431
|
+
if (cutPoints < 2) return true;
|
|
432
|
+
// (3) transcript under pi's keepRecentTokens budget → pi keeps everything
|
|
433
|
+
if (tokens < durableTrimFloorTokens()) return true;
|
|
434
|
+
return false;
|
|
435
|
+
} catch {
|
|
436
|
+
return true; // safe: skip the durable trim rather than risk a user-facing throw
|
|
437
|
+
}
|
|
386
438
|
}
|
|
387
439
|
|
|
388
440
|
/** pi's default keepRecentTokens (compaction settings). Override with
|
|
389
441
|
* MEGACOMPACT_DURABLE_TRIM_FLOOR if you raise pi's compact.keepRecentTokens. */
|
|
390
442
|
function durableTrimFloorTokens(): number {
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
443
|
+
const raw = process.env.MEGACOMPACT_DURABLE_TRIM_FLOOR;
|
|
444
|
+
if (raw !== undefined && Number.isFinite(Number(raw))) return Number(raw);
|
|
445
|
+
return 20_000;
|
|
394
446
|
}
|
|
395
447
|
|
|
396
448
|
/**
|
|
@@ -399,42 +451,54 @@ function durableTrimFloorTokens(): number {
|
|
|
399
451
|
* or report it (command).
|
|
400
452
|
*/
|
|
401
453
|
export function doRecall(
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
454
|
+
runtime: MegaRuntime,
|
|
455
|
+
config: MegaConfig,
|
|
456
|
+
ctx: ExtensionContext,
|
|
457
|
+
query: string,
|
|
458
|
+
source: "resume" | "command",
|
|
407
459
|
) {
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
460
|
+
runtime.bindRepo(ctx.cwd);
|
|
461
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
462
|
+
// Live window text for inline dedupe (Fix C): drop recalled checkpoints that
|
|
463
|
+
// are already resident in the session, so recall never re-injects context the
|
|
464
|
+
// model can already see. Best-effort — an empty window just skips dedupe.
|
|
465
|
+
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
466
|
+
const result = recallAndInline(
|
|
467
|
+
{
|
|
468
|
+
sessionId: sid,
|
|
469
|
+
query,
|
|
470
|
+
limit: config.autoInlineK,
|
|
471
|
+
source,
|
|
472
|
+
skipInjected: true,
|
|
473
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
474
|
+
windowDedupe: config.windowDedupe,
|
|
475
|
+
liveWindow,
|
|
476
|
+
dedupSim: config.dedupSim,
|
|
477
|
+
},
|
|
478
|
+
runtime.store,
|
|
479
|
+
);
|
|
480
|
+
runtime.dashboard.event("recall", {
|
|
481
|
+
source,
|
|
482
|
+
query: query.slice(0, 120),
|
|
483
|
+
injected: result.toInject.length,
|
|
484
|
+
empty: result.empty,
|
|
485
|
+
});
|
|
486
|
+
if (!result.empty && result.toInject.length > 0) {
|
|
487
|
+
const top = result.toInject[0];
|
|
488
|
+
const scorePct = Math.round((top.score ?? 0) * 100);
|
|
489
|
+
const files = top.checkpoint.filesModified ?? [];
|
|
490
|
+
const label = files.length
|
|
491
|
+
? files
|
|
492
|
+
.map((f) => f.split("/").pop() ?? f)
|
|
493
|
+
.slice(0, 2)
|
|
494
|
+
.join(", ")
|
|
495
|
+
: top.checkpoint.checkpointId;
|
|
496
|
+
runtime.pushTicker(
|
|
497
|
+
`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`,
|
|
498
|
+
);
|
|
499
|
+
runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
|
|
500
|
+
}
|
|
501
|
+
return result;
|
|
438
502
|
}
|
|
439
503
|
|
|
440
504
|
/**
|
|
@@ -449,58 +513,81 @@ export function doRecall(
|
|
|
449
513
|
* the same-repo result unchanged.
|
|
450
514
|
*/
|
|
451
515
|
export async function doRecallAsync(
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
516
|
+
runtime: MegaRuntime,
|
|
517
|
+
config: MegaConfig,
|
|
518
|
+
ctx: ExtensionContext,
|
|
519
|
+
query: string,
|
|
520
|
+
source: "resume" | "command",
|
|
521
|
+
opts: { crossRepo?: boolean } = {},
|
|
458
522
|
): Promise<RecallInjectResult> {
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
523
|
+
runtime.bindRepo(ctx.cwd);
|
|
524
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
525
|
+
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
526
|
+
// Sync same-repo first (fast, never blocks).
|
|
527
|
+
const sameRepo = recallAndInline(
|
|
528
|
+
{
|
|
529
|
+
sessionId: sid,
|
|
530
|
+
query,
|
|
531
|
+
limit: config.autoInlineK,
|
|
532
|
+
source,
|
|
533
|
+
skipInjected: true,
|
|
534
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
535
|
+
windowDedupe: config.windowDedupe,
|
|
536
|
+
liveWindow,
|
|
537
|
+
dedupSim: config.dedupSim,
|
|
538
|
+
},
|
|
539
|
+
runtime.store,
|
|
540
|
+
);
|
|
541
|
+
if (!config.crossRepoEnabled || !opts.crossRepo) return sameRepo;
|
|
542
|
+
if (sameRepo.toInject.length >= config.autoInlineK) return sameRepo; // same-repo satisfied
|
|
543
|
+
// Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
|
|
544
|
+
try {
|
|
545
|
+
const x = await recallAndInlineAsync(
|
|
546
|
+
{
|
|
547
|
+
sessionId: sid,
|
|
548
|
+
query,
|
|
549
|
+
limit: config.autoInlineK,
|
|
550
|
+
source,
|
|
551
|
+
skipInjected: true,
|
|
552
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
553
|
+
windowDedupe: config.windowDedupe,
|
|
554
|
+
liveWindow,
|
|
555
|
+
dedupSim: config.crossRepoCosine,
|
|
556
|
+
crossRepo: true,
|
|
557
|
+
globalIndexDir: process.env.MEGACOMPACT_INDEX_DIR,
|
|
558
|
+
},
|
|
559
|
+
runtime.store,
|
|
560
|
+
);
|
|
561
|
+
runtime.dashboard.event("recall-crossrepo", {
|
|
562
|
+
source,
|
|
563
|
+
query: query.slice(0, 120),
|
|
564
|
+
injected: x.toInject.length,
|
|
565
|
+
sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
|
|
566
|
+
});
|
|
567
|
+
// Merge, dedup by checkpointId, respect the same token cap by reformatting.
|
|
568
|
+
const seen = new Set(
|
|
569
|
+
sameRepo.toInject.map((h) => h.checkpoint.checkpointId),
|
|
570
|
+
);
|
|
571
|
+
const merged = [...sameRepo.toInject];
|
|
572
|
+
for (const h of x.toInject) {
|
|
573
|
+
if (!seen.has(h.checkpoint.checkpointId)) {
|
|
574
|
+
merged.push(h);
|
|
575
|
+
seen.add(h.checkpoint.checkpointId);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
const block = merged.length ? formatRecallBlock(merged) : "";
|
|
579
|
+
return {
|
|
580
|
+
toInject: merged,
|
|
581
|
+
report: merged.map(
|
|
582
|
+
(h) =>
|
|
583
|
+
` • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`,
|
|
584
|
+
),
|
|
585
|
+
block,
|
|
586
|
+
empty: merged.length === 0,
|
|
587
|
+
};
|
|
588
|
+
} catch {
|
|
589
|
+
return sameRepo; // cross-repo failure → same-repo only (non-fatal)
|
|
590
|
+
}
|
|
504
591
|
}
|
|
505
592
|
|
|
506
593
|
/**
|
|
@@ -510,18 +597,19 @@ export async function doRecallAsync(
|
|
|
510
597
|
* Mirrors recentUserQuery's use of sessionEntryToContextMessages.
|
|
511
598
|
*/
|
|
512
599
|
function extractLiveWindow(ctx: ExtensionContext): string[] {
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
600
|
+
try {
|
|
601
|
+
const entries = ctx.sessionManager.getEntries();
|
|
602
|
+
const texts: string[] = [];
|
|
603
|
+
for (const e of entries) {
|
|
604
|
+
for (const m of sessionEntryToContextMessages(e)) {
|
|
605
|
+
const c = (m as { content?: unknown }).content;
|
|
606
|
+
if (typeof c === "string") texts.push(c);
|
|
607
|
+
else if (Array.isArray(c))
|
|
608
|
+
texts.push(c.map((b: any) => b.text).join(" "));
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
return texts;
|
|
612
|
+
} catch {
|
|
613
|
+
return [];
|
|
614
|
+
}
|
|
527
615
|
}
|