pi-mega-compact 0.20.85 → 0.20.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.js +9 -0
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +2 -0
- package/dist/extensions/mega-config.js +12 -0
- package/dist/extensions/mega-events/context-handler/gateCheck.js +27 -0
- package/dist/extensions/mega-events/context-handler/thrashGuard.js +186 -0
- package/dist/extensions/mega-events/context-handler.js +33 -1
- package/dist/extensions/mega-pipeline/compact/noop.js +104 -0
- package/dist/extensions/mega-pipeline/compact/run.js +268 -0
- package/dist/extensions/mega-pipeline/compact/vote.js +72 -0
- package/dist/extensions/mega-pipeline/compact.js +12 -343
- package/dist/extensions/mega-pipeline/recall/impl.js +258 -0
- package/dist/extensions/mega-pipeline/recall.js +6 -253
- package/dist/src/config.js +9 -0
- package/dist/src/failback/compact.js +109 -0
- package/dist/src/recall/readonly.js +39 -0
- package/dist/src/recall/recall3wf.fixture.js +67 -0
- package/dist/src/recall/validator.js +113 -0
- package/dist/src/recall/vote.js +217 -0
- package/dist/src/store/sqlite/fts5-search.js +26 -0
- package/dist/src/store/sqlite/meta.js +32 -0
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +9 -0
- package/extensions/mega-config-types.ts +13 -0
- package/extensions/mega-config.ts +12 -0
- package/extensions/mega-events/context-handler/gateCheck.ts +30 -0
- package/extensions/mega-events/context-handler/thrashGuard.ts +228 -0
- package/extensions/mega-events/context-handler.ts +36 -1
- package/extensions/mega-pipeline/compact/noop.ts +96 -0
- package/extensions/mega-pipeline/compact/run.ts +322 -0
- package/extensions/mega-pipeline/compact/vote.ts +85 -0
- package/extensions/mega-pipeline/compact.ts +12 -385
- package/extensions/mega-pipeline/recall/impl.ts +312 -0
- package/extensions/mega-pipeline/recall.ts +10 -306
- package/package.json +1 -1
- package/src/config.ts +12 -0
- package/src/failback/compact.ts +122 -0
- package/src/failback/types.ts +72 -0
- package/src/recall/readonly.ts +57 -0
- package/src/recall/recall3wf.fixture.ts +87 -0
- package/src/recall/validator.ts +150 -0
- package/src/recall/vote.ts +240 -0
- package/src/store/sqlite/fts5-search.ts +40 -0
- package/src/store/sqlite/meta.ts +36 -0
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* recall/impl.ts — unified Layer-5 recall pipeline implementation (3WF-3 split).
|
|
3
|
+
*
|
|
4
|
+
* Behavior is UNCHANGED from the pre-split recall.ts. `doRecall` is the ONE path
|
|
5
|
+
* that injects (sync). `doRecallAsync` augments with optional cross-repo HNSW
|
|
6
|
+
* on resume / /mega-recall --cross-repo. Both mutate the shared MegaRuntime
|
|
7
|
+
* (token accounting, ticker, dashboard events). The shell recall.ts re-exports
|
|
8
|
+
* these names so `export * from "./mega-pipeline/recall.js"` stays byte-stable.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
12
|
+
import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
13
|
+
import {
|
|
14
|
+
recallAndInline,
|
|
15
|
+
recallAndInlineAsync,
|
|
16
|
+
formatRecallBlock,
|
|
17
|
+
type RecallInjectResult,
|
|
18
|
+
} from "../../../src/recall.js";
|
|
19
|
+
import { normalizeSessionId } from "../../../src/store.js";
|
|
20
|
+
import {
|
|
21
|
+
incRecallInjected,
|
|
22
|
+
incCacheHitTokens,
|
|
23
|
+
getIndexDir,
|
|
24
|
+
} from "../../../src/store/sqlite.js";
|
|
25
|
+
import {
|
|
26
|
+
ensureConversationIdFor,
|
|
27
|
+
recordTurnWrite,
|
|
28
|
+
recordRecallWrite,
|
|
29
|
+
} from "../../mega-turn-store.js";
|
|
30
|
+
import { type MegaRuntime, C } from "../../mega-runtime.js";
|
|
31
|
+
import { recordRecallLatency } from "../../mega-runtime/vc-observer.js";
|
|
32
|
+
import type { MegaConfig } from "../../mega-config.js";
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Unified recall (Layer 5). The ONE path that injects. Returns the recall
|
|
36
|
+
* result; callers decide whether to stage it for before_agent_start (resume)
|
|
37
|
+
* or report it (command).
|
|
38
|
+
*/
|
|
39
|
+
export function doRecall(
|
|
40
|
+
runtime: MegaRuntime,
|
|
41
|
+
config: MegaConfig,
|
|
42
|
+
ctx: ExtensionContext,
|
|
43
|
+
query: string,
|
|
44
|
+
source: "resume" | "command",
|
|
45
|
+
) {
|
|
46
|
+
runtime.bindRepo(ctx.cwd);
|
|
47
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
48
|
+
// Live window text for inline dedupe (Fix C): drop recalled checkpoints that
|
|
49
|
+
// are already resident in the session, so recall never re-injects context the
|
|
50
|
+
// model can already see. Best-effort — an empty window just skips dedupe.
|
|
51
|
+
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
52
|
+
const recallStartMs = Date.now();
|
|
53
|
+
const result = recallAndInline(
|
|
54
|
+
{
|
|
55
|
+
sessionId: sid,
|
|
56
|
+
query,
|
|
57
|
+
limit: config.autoInlineK,
|
|
58
|
+
source,
|
|
59
|
+
skipInjected: true,
|
|
60
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
61
|
+
windowDedupe: config.windowDedupe,
|
|
62
|
+
liveWindow,
|
|
63
|
+
dedupSim: config.dedupSim,
|
|
64
|
+
},
|
|
65
|
+
runtime.store,
|
|
66
|
+
);
|
|
67
|
+
runtime.dashboard.event("recall", {
|
|
68
|
+
source,
|
|
69
|
+
query: query.slice(0, 120),
|
|
70
|
+
injected: result.toInject.length,
|
|
71
|
+
empty: result.empty,
|
|
72
|
+
});
|
|
73
|
+
if (config.ragRecallMetrics && result.hydeInfo) {
|
|
74
|
+
runtime.dashboard.event("hyde_executed", {
|
|
75
|
+
sessionId: sid,
|
|
76
|
+
ran: result.hydeInfo.ran,
|
|
77
|
+
skipped: result.hydeInfo.skipped,
|
|
78
|
+
reason: result.hydeInfo.reason,
|
|
79
|
+
hypotheticalDoc: result.hydeInfo.hypotheticalDoc.slice(0, 400),
|
|
80
|
+
generationMs: result.hydeInfo.generationMs,
|
|
81
|
+
rawHitCount: result.hydeInfo.rawHitCount,
|
|
82
|
+
hydeHitCount: result.hydeInfo.hydeHitCount,
|
|
83
|
+
fusedHitCount: result.hydeInfo.fusedHitCount,
|
|
84
|
+
lift: result.hydeInfo.lift,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
if (config.ragRecallMetrics && result.recallMetrics) {
|
|
88
|
+
runtime.dashboard.event("recall_metrics", {
|
|
89
|
+
sessionId: sid,
|
|
90
|
+
hitCount: result.recallMetrics.hitCount,
|
|
91
|
+
score: result.recallMetrics.score,
|
|
92
|
+
pass: result.recallMetrics.pass,
|
|
93
|
+
relevance: result.recallMetrics.relevance,
|
|
94
|
+
coverage: result.recallMetrics.coverage,
|
|
95
|
+
diversity: result.recallMetrics.diversity,
|
|
96
|
+
specificity: result.recallMetrics.specificity,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
if (!result.empty && result.toInject.length > 0) {
|
|
100
|
+
const top = result.toInject[0];
|
|
101
|
+
const scorePct = Math.round((top.score ?? 0) * 100);
|
|
102
|
+
const files = top.checkpoint.filesModified ?? [];
|
|
103
|
+
const label = files.length
|
|
104
|
+
? files
|
|
105
|
+
.map((f) => f.split("/").pop() ?? f)
|
|
106
|
+
.slice(0, 2)
|
|
107
|
+
.join(", ")
|
|
108
|
+
: top.checkpoint.checkpointId;
|
|
109
|
+
runtime.pushTicker(
|
|
110
|
+
`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`,
|
|
111
|
+
);
|
|
112
|
+
runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
|
|
113
|
+
}
|
|
114
|
+
let sumTokens = 0;
|
|
115
|
+
for (const h of result.toInject) sumTokens += h.checkpoint.tokenEstimate;
|
|
116
|
+
if (result.toInject.length > 0) {
|
|
117
|
+
runtime.rt.recallInjections += result.toInject.length;
|
|
118
|
+
runtime.rt.cacheHitTokens += sumTokens;
|
|
119
|
+
incRecallInjected(result.toInject.length, runtime.currentStateDir);
|
|
120
|
+
incCacheHitTokens(sumTokens, runtime.currentStateDir);
|
|
121
|
+
}
|
|
122
|
+
// S43: record recall provenance — which checkpoints/summaries served this
|
|
123
|
+
// turn, their score + source path. Linked to the turn row written at
|
|
124
|
+
// turn_end via the conversation+turnIndex. Best-effort + non-fatal.
|
|
125
|
+
// Persists telemetry (HyDE + recall metrics) even when recall returned
|
|
126
|
+
// no hits, so empty-recall HyDE invocations are still visible in the
|
|
127
|
+
// dashboard Turns/Metrics tabs.
|
|
128
|
+
const hasTelemetry =
|
|
129
|
+
result.hydeInfo != null || result.recallMetrics != null;
|
|
130
|
+
if (result.toInject.length > 0 || hasTelemetry) {
|
|
131
|
+
try {
|
|
132
|
+
const convId = ensureConversationIdFor(
|
|
133
|
+
config,
|
|
134
|
+
sid,
|
|
135
|
+
runtime.currentStateDir,
|
|
136
|
+
);
|
|
137
|
+
const turnId = recordTurnWrite(
|
|
138
|
+
config,
|
|
139
|
+
{
|
|
140
|
+
conversationId: convId,
|
|
141
|
+
sessionId: sid,
|
|
142
|
+
turnIndex: runtime.currentTurn,
|
|
143
|
+
role: "assistant",
|
|
144
|
+
startedAt: Date.now(),
|
|
145
|
+
hyde: result.hydeInfo ?? undefined,
|
|
146
|
+
recallMetrics: result.recallMetrics ?? undefined,
|
|
147
|
+
},
|
|
148
|
+
runtime.currentStateDir,
|
|
149
|
+
);
|
|
150
|
+
if (result.toInject.length > 0) {
|
|
151
|
+
recordRecallWrite(
|
|
152
|
+
config,
|
|
153
|
+
turnId,
|
|
154
|
+
result.toInject.map((h) => ({
|
|
155
|
+
checkpointId: h.checkpoint.checkpointId,
|
|
156
|
+
score: h.score,
|
|
157
|
+
source:
|
|
158
|
+
h.raptorLevel !== undefined
|
|
159
|
+
? "raptor"
|
|
160
|
+
: h.repoId
|
|
161
|
+
? "cross-repo"
|
|
162
|
+
: "flat",
|
|
163
|
+
raptorLevel: h.raptorLevel,
|
|
164
|
+
})),
|
|
165
|
+
runtime.currentStateDir,
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
} catch {
|
|
169
|
+
/* non-fatal: recall provenance never breaks the recall path */
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
// VC0A: record recall latency on the eval observer (mode A) so the dashboard
|
|
173
|
+
// histogram reflects real data. No-op when the observer is absent (flag off /
|
|
174
|
+
// construction failure).
|
|
175
|
+
try {
|
|
176
|
+
recordRecallLatency(runtime, Date.now() - recallStartMs, sid, 0);
|
|
177
|
+
} catch {
|
|
178
|
+
/* non-fatal: latency recording never breaks recall */
|
|
179
|
+
}
|
|
180
|
+
return result;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* S17: async recall with optional cross-repo augmentation. Used on resume
|
|
185
|
+
* (session_start) and /mega-recall --cross-repo — NEVER from the mid-turn
|
|
186
|
+
* context handler (that stays sync). Runs the sync same-repo scan first; if it
|
|
187
|
+
* returns < config.autoInlineK hits AND crossRepo is enabled, awaits the PGlite
|
|
188
|
+
* HNSW cross-repo path and merges (source-labeled, deduped by checkpointId). The
|
|
189
|
+
* recallMaxTokens cap + windowDedupe apply to the merged set so cross-repo can
|
|
190
|
+
* never net-inflate the window. Cross-repo uses a stricter cosine floor
|
|
191
|
+
* (config.crossRepoCosine) than same-repo. Non-fatal: any async failure returns
|
|
192
|
+
* the same-repo result unchanged.
|
|
193
|
+
*/
|
|
194
|
+
export async function doRecallAsync(
|
|
195
|
+
runtime: MegaRuntime,
|
|
196
|
+
config: MegaConfig,
|
|
197
|
+
ctx: ExtensionContext,
|
|
198
|
+
query: string,
|
|
199
|
+
source: "resume" | "command",
|
|
200
|
+
opts: { crossRepo?: boolean } = {},
|
|
201
|
+
): Promise<RecallInjectResult> {
|
|
202
|
+
runtime.bindRepo(ctx.cwd);
|
|
203
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
204
|
+
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
205
|
+
// Sync same-repo first (fast, never blocks).
|
|
206
|
+
const sameRepo = recallAndInline(
|
|
207
|
+
{
|
|
208
|
+
sessionId: sid,
|
|
209
|
+
query,
|
|
210
|
+
limit: config.autoInlineK,
|
|
211
|
+
source,
|
|
212
|
+
skipInjected: true,
|
|
213
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
214
|
+
windowDedupe: config.windowDedupe,
|
|
215
|
+
liveWindow,
|
|
216
|
+
dedupSim: config.dedupSim,
|
|
217
|
+
},
|
|
218
|
+
runtime.store,
|
|
219
|
+
);
|
|
220
|
+
if (!config.crossRepoEnabled || !opts.crossRepo) return sameRepo;
|
|
221
|
+
if (sameRepo.toInject.length >= config.autoInlineK) return sameRepo; // same-repo satisfied
|
|
222
|
+
// Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
|
|
223
|
+
try {
|
|
224
|
+
const x = await recallAndInlineAsync(
|
|
225
|
+
{
|
|
226
|
+
sessionId: sid,
|
|
227
|
+
query,
|
|
228
|
+
limit: config.autoInlineK,
|
|
229
|
+
source,
|
|
230
|
+
skipInjected: true,
|
|
231
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
232
|
+
windowDedupe: config.windowDedupe,
|
|
233
|
+
liveWindow,
|
|
234
|
+
dedupSim: config.crossRepoCosine,
|
|
235
|
+
crossRepo: true,
|
|
236
|
+
// F2: resolve the machine-wide index dir via the shared resolver so the
|
|
237
|
+
// cross-repo injected-set dedup works even when MEGACOMPACT_INDEX_DIR is
|
|
238
|
+
// unset. The env var still wins when set (getIndexDir checks it first);
|
|
239
|
+
// the default (~/.mega-compact-index) is the same DB mega-commands and the
|
|
240
|
+
// dashboard read, so injection counts stay consistent. Without this, a
|
|
241
|
+
// bare `process.env` read returns undefined → cross-repo hits re-inject in
|
|
242
|
+
// every new session (the global injected-set is never consulted).
|
|
243
|
+
globalIndexDir: getIndexDir(),
|
|
244
|
+
},
|
|
245
|
+
runtime.store,
|
|
246
|
+
);
|
|
247
|
+
runtime.dashboard.event("recall-crossrepo", {
|
|
248
|
+
source,
|
|
249
|
+
query: query.slice(0, 120),
|
|
250
|
+
injected: x.toInject.length,
|
|
251
|
+
sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
|
|
252
|
+
});
|
|
253
|
+
// Merge, dedup by checkpointId, respect the same token cap by reformatting.
|
|
254
|
+
const seen = new Set(
|
|
255
|
+
sameRepo.toInject.map((h) => h.checkpoint.checkpointId),
|
|
256
|
+
);
|
|
257
|
+
const merged = [...sameRepo.toInject];
|
|
258
|
+
for (const h of x.toInject) {
|
|
259
|
+
if (!seen.has(h.checkpoint.checkpointId)) {
|
|
260
|
+
merged.push(h);
|
|
261
|
+
seen.add(h.checkpoint.checkpointId);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
const block = merged.length ? formatRecallBlock(merged) : "";
|
|
265
|
+
if (merged.length > 0) {
|
|
266
|
+
let sumTokens = 0;
|
|
267
|
+
for (const h of merged) sumTokens += h.checkpoint.tokenEstimate;
|
|
268
|
+
runtime.rt.recallInjections += merged.length;
|
|
269
|
+
runtime.rt.cacheHitTokens += sumTokens;
|
|
270
|
+
incRecallInjected(merged.length, runtime.currentStateDir);
|
|
271
|
+
incCacheHitTokens(sumTokens, runtime.currentStateDir);
|
|
272
|
+
}
|
|
273
|
+
return {
|
|
274
|
+
toInject: merged,
|
|
275
|
+
report: merged.map(
|
|
276
|
+
(h) =>
|
|
277
|
+
` • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`,
|
|
278
|
+
),
|
|
279
|
+
block,
|
|
280
|
+
empty: merged.length === 0,
|
|
281
|
+
// H1: merged cross-repo result reuses the same-repo pass's telemetry.
|
|
282
|
+
hydeInfo: sameRepo.hydeInfo,
|
|
283
|
+
recallMetrics: sameRepo.recallMetrics,
|
|
284
|
+
};
|
|
285
|
+
} catch {
|
|
286
|
+
return sameRepo; // cross-repo failure → same-repo only (non-fatal)
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* Extract the live-window message texts from the session manager (Fix C),
|
|
292
|
+
* for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
|
|
293
|
+
* error so recall falls back to unbounded (still correct, just no dedupe).
|
|
294
|
+
* Mirrors recentUserQuery's use of sessionEntryToContextMessages.
|
|
295
|
+
*/
|
|
296
|
+
export function extractLiveWindow(ctx: ExtensionContext): string[] {
|
|
297
|
+
try {
|
|
298
|
+
const entries = ctx.sessionManager.getEntries();
|
|
299
|
+
const texts: string[] = [];
|
|
300
|
+
for (const e of entries) {
|
|
301
|
+
for (const m of sessionEntryToContextMessages(e)) {
|
|
302
|
+
const c = (m as { content?: unknown }).content;
|
|
303
|
+
if (typeof c === "string") texts.push(c);
|
|
304
|
+
else if (Array.isArray(c))
|
|
305
|
+
texts.push(c.map((b: { text?: string }) => b.text ?? "").join(" "));
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
return texts;
|
|
309
|
+
} catch {
|
|
310
|
+
return [];
|
|
311
|
+
}
|
|
312
|
+
}
|
|
@@ -1,310 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* recall.ts — unified Layer-5 recall pipeline.
|
|
2
|
+
* recall.ts — shell re-export for the unified Layer-5 recall pipeline (3WF-3 split).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* Delegate-shell pattern: the implementation lives in ./recall/impl.ts (kept
|
|
5
|
+
* under the 300-line soft cap). All public symbols are re-exported here so
|
|
6
|
+
* `export * from "./mega-pipeline/recall.js"` (mega-pipeline.ts) and any direct
|
|
7
|
+
* importers keep resolving with byte-identical names.
|
|
7
8
|
*/
|
|
8
9
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
formatRecallBlock,
|
|
15
|
-
type RecallInjectResult,
|
|
16
|
-
} from "../../src/recall.js";
|
|
17
|
-
import { normalizeSessionId } from "../../src/store.js";
|
|
18
|
-
import {
|
|
19
|
-
incRecallInjected,
|
|
20
|
-
incCacheHitTokens,
|
|
21
|
-
getIndexDir,
|
|
22
|
-
} from "../../src/store/sqlite.js";
|
|
23
|
-
import {
|
|
24
|
-
ensureConversationIdFor,
|
|
25
|
-
recordTurnWrite,
|
|
26
|
-
recordRecallWrite,
|
|
27
|
-
} from "../mega-turn-store.js";
|
|
28
|
-
import { type MegaRuntime, C } from "../mega-runtime.js";
|
|
29
|
-
import { recordRecallLatency } from "../mega-runtime/vc-observer.js";
|
|
30
|
-
import type { MegaConfig } from "../mega-config.js";
|
|
31
|
-
|
|
32
|
-
/**
|
|
33
|
-
* Unified recall (Layer 5). The ONE path that injects. Returns the recall
|
|
34
|
-
* result; callers decide whether to stage it for before_agent_start (resume)
|
|
35
|
-
* or report it (command).
|
|
36
|
-
*/
|
|
37
|
-
export function doRecall(
|
|
38
|
-
runtime: MegaRuntime,
|
|
39
|
-
config: MegaConfig,
|
|
40
|
-
ctx: ExtensionContext,
|
|
41
|
-
query: string,
|
|
42
|
-
source: "resume" | "command",
|
|
43
|
-
) {
|
|
44
|
-
runtime.bindRepo(ctx.cwd);
|
|
45
|
-
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
46
|
-
// Live window text for inline dedupe (Fix C): drop recalled checkpoints that
|
|
47
|
-
// are already resident in the session, so recall never re-injects context the
|
|
48
|
-
// model can already see. Best-effort — an empty window just skips dedupe.
|
|
49
|
-
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
50
|
-
const recallStartMs = Date.now();
|
|
51
|
-
const result = recallAndInline(
|
|
52
|
-
{
|
|
53
|
-
sessionId: sid,
|
|
54
|
-
query,
|
|
55
|
-
limit: config.autoInlineK,
|
|
56
|
-
source,
|
|
57
|
-
skipInjected: true,
|
|
58
|
-
recallMaxTokens: config.recallMaxTokens,
|
|
59
|
-
windowDedupe: config.windowDedupe,
|
|
60
|
-
liveWindow,
|
|
61
|
-
dedupSim: config.dedupSim,
|
|
62
|
-
},
|
|
63
|
-
runtime.store,
|
|
64
|
-
);
|
|
65
|
-
runtime.dashboard.event("recall", {
|
|
66
|
-
source,
|
|
67
|
-
query: query.slice(0, 120),
|
|
68
|
-
injected: result.toInject.length,
|
|
69
|
-
empty: result.empty,
|
|
70
|
-
});
|
|
71
|
-
if (config.ragRecallMetrics && result.hydeInfo) {
|
|
72
|
-
runtime.dashboard.event("hyde_executed", {
|
|
73
|
-
sessionId: sid,
|
|
74
|
-
ran: result.hydeInfo.ran,
|
|
75
|
-
skipped: result.hydeInfo.skipped,
|
|
76
|
-
reason: result.hydeInfo.reason,
|
|
77
|
-
hypotheticalDoc: result.hydeInfo.hypotheticalDoc.slice(0, 400),
|
|
78
|
-
generationMs: result.hydeInfo.generationMs,
|
|
79
|
-
rawHitCount: result.hydeInfo.rawHitCount,
|
|
80
|
-
hydeHitCount: result.hydeInfo.hydeHitCount,
|
|
81
|
-
fusedHitCount: result.hydeInfo.fusedHitCount,
|
|
82
|
-
lift: result.hydeInfo.lift,
|
|
83
|
-
});
|
|
84
|
-
}
|
|
85
|
-
if (config.ragRecallMetrics && result.recallMetrics) {
|
|
86
|
-
runtime.dashboard.event("recall_metrics", {
|
|
87
|
-
sessionId: sid,
|
|
88
|
-
hitCount: result.recallMetrics.hitCount,
|
|
89
|
-
score: result.recallMetrics.score,
|
|
90
|
-
pass: result.recallMetrics.pass,
|
|
91
|
-
relevance: result.recallMetrics.relevance,
|
|
92
|
-
coverage: result.recallMetrics.coverage,
|
|
93
|
-
diversity: result.recallMetrics.diversity,
|
|
94
|
-
specificity: result.recallMetrics.specificity,
|
|
95
|
-
});
|
|
96
|
-
}
|
|
97
|
-
if (!result.empty && result.toInject.length > 0) {
|
|
98
|
-
const top = result.toInject[0];
|
|
99
|
-
const scorePct = Math.round((top.score ?? 0) * 100);
|
|
100
|
-
const files = top.checkpoint.filesModified ?? [];
|
|
101
|
-
const label = files.length
|
|
102
|
-
? files
|
|
103
|
-
.map((f) => f.split("/").pop() ?? f)
|
|
104
|
-
.slice(0, 2)
|
|
105
|
-
.join(", ")
|
|
106
|
-
: top.checkpoint.checkpointId;
|
|
107
|
-
runtime.pushTicker(
|
|
108
|
-
`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`,
|
|
109
|
-
);
|
|
110
|
-
runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
|
|
111
|
-
}
|
|
112
|
-
let sumTokens = 0;
|
|
113
|
-
for (const h of result.toInject) sumTokens += h.checkpoint.tokenEstimate;
|
|
114
|
-
if (result.toInject.length > 0) {
|
|
115
|
-
runtime.rt.recallInjections += result.toInject.length;
|
|
116
|
-
runtime.rt.cacheHitTokens += sumTokens;
|
|
117
|
-
incRecallInjected(result.toInject.length, runtime.currentStateDir);
|
|
118
|
-
incCacheHitTokens(sumTokens, runtime.currentStateDir);
|
|
119
|
-
}
|
|
120
|
-
// S43: record recall provenance — which checkpoints/summaries served this
|
|
121
|
-
// turn, their score + source path. Linked to the turn row written at
|
|
122
|
-
// turn_end via the conversation+turnIndex. Best-effort + non-fatal.
|
|
123
|
-
// Persists telemetry (HyDE + recall metrics) even when recall returned
|
|
124
|
-
// no hits, so empty-recall HyDE invocations are still visible in the
|
|
125
|
-
// dashboard Turns/Metrics tabs.
|
|
126
|
-
const hasTelemetry =
|
|
127
|
-
result.hydeInfo != null || result.recallMetrics != null;
|
|
128
|
-
if (result.toInject.length > 0 || hasTelemetry) {
|
|
129
|
-
try {
|
|
130
|
-
const convId = ensureConversationIdFor(
|
|
131
|
-
config,
|
|
132
|
-
sid,
|
|
133
|
-
runtime.currentStateDir,
|
|
134
|
-
);
|
|
135
|
-
const turnId = recordTurnWrite(
|
|
136
|
-
config,
|
|
137
|
-
{
|
|
138
|
-
conversationId: convId,
|
|
139
|
-
sessionId: sid,
|
|
140
|
-
turnIndex: runtime.currentTurn,
|
|
141
|
-
role: "assistant",
|
|
142
|
-
startedAt: Date.now(),
|
|
143
|
-
hyde: result.hydeInfo ?? undefined,
|
|
144
|
-
recallMetrics: result.recallMetrics ?? undefined,
|
|
145
|
-
},
|
|
146
|
-
runtime.currentStateDir,
|
|
147
|
-
);
|
|
148
|
-
if (result.toInject.length > 0) {
|
|
149
|
-
recordRecallWrite(
|
|
150
|
-
config,
|
|
151
|
-
turnId,
|
|
152
|
-
result.toInject.map((h) => ({
|
|
153
|
-
checkpointId: h.checkpoint.checkpointId,
|
|
154
|
-
score: h.score,
|
|
155
|
-
source:
|
|
156
|
-
h.raptorLevel !== undefined
|
|
157
|
-
? "raptor"
|
|
158
|
-
: h.repoId
|
|
159
|
-
? "cross-repo"
|
|
160
|
-
: "flat",
|
|
161
|
-
raptorLevel: h.raptorLevel,
|
|
162
|
-
})),
|
|
163
|
-
runtime.currentStateDir,
|
|
164
|
-
);
|
|
165
|
-
}
|
|
166
|
-
} catch {
|
|
167
|
-
/* non-fatal: recall provenance never breaks the recall path */
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
// VC0A: record recall latency on the eval observer (mode A) so the dashboard
|
|
171
|
-
// histogram reflects real data. No-op when the observer is absent (flag off /
|
|
172
|
-
// construction failure).
|
|
173
|
-
try {
|
|
174
|
-
recordRecallLatency(runtime, Date.now() - recallStartMs, sid, 0);
|
|
175
|
-
} catch {
|
|
176
|
-
/* non-fatal: latency recording never breaks recall */
|
|
177
|
-
}
|
|
178
|
-
return result;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
/**
|
|
182
|
-
* S17: async recall with optional cross-repo augmentation. Used on resume
|
|
183
|
-
* (session_start) and /mega-recall --cross-repo — NEVER from the mid-turn
|
|
184
|
-
* context handler (that stays sync). Runs the sync same-repo scan first; if it
|
|
185
|
-
* returns < config.autoInlineK hits AND crossRepo is enabled, awaits the PGlite
|
|
186
|
-
* HNSW cross-repo path and merges (source-labeled, deduped by checkpointId). The
|
|
187
|
-
* recallMaxTokens cap + windowDedupe apply to the merged set so cross-repo can
|
|
188
|
-
* never net-inflate the window. Cross-repo uses a stricter cosine floor
|
|
189
|
-
* (config.crossRepoCosine) than same-repo. Non-fatal: any async failure returns
|
|
190
|
-
* the same-repo result unchanged.
|
|
191
|
-
*/
|
|
192
|
-
export async function doRecallAsync(
|
|
193
|
-
runtime: MegaRuntime,
|
|
194
|
-
config: MegaConfig,
|
|
195
|
-
ctx: ExtensionContext,
|
|
196
|
-
query: string,
|
|
197
|
-
source: "resume" | "command",
|
|
198
|
-
opts: { crossRepo?: boolean } = {},
|
|
199
|
-
): Promise<RecallInjectResult> {
|
|
200
|
-
runtime.bindRepo(ctx.cwd);
|
|
201
|
-
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
202
|
-
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
203
|
-
// Sync same-repo first (fast, never blocks).
|
|
204
|
-
const sameRepo = recallAndInline(
|
|
205
|
-
{
|
|
206
|
-
sessionId: sid,
|
|
207
|
-
query,
|
|
208
|
-
limit: config.autoInlineK,
|
|
209
|
-
source,
|
|
210
|
-
skipInjected: true,
|
|
211
|
-
recallMaxTokens: config.recallMaxTokens,
|
|
212
|
-
windowDedupe: config.windowDedupe,
|
|
213
|
-
liveWindow,
|
|
214
|
-
dedupSim: config.dedupSim,
|
|
215
|
-
},
|
|
216
|
-
runtime.store,
|
|
217
|
-
);
|
|
218
|
-
if (!config.crossRepoEnabled || !opts.crossRepo) return sameRepo;
|
|
219
|
-
if (sameRepo.toInject.length >= config.autoInlineK) return sameRepo; // same-repo satisfied
|
|
220
|
-
// Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
|
|
221
|
-
try {
|
|
222
|
-
const x = await recallAndInlineAsync(
|
|
223
|
-
{
|
|
224
|
-
sessionId: sid,
|
|
225
|
-
query,
|
|
226
|
-
limit: config.autoInlineK,
|
|
227
|
-
source,
|
|
228
|
-
skipInjected: true,
|
|
229
|
-
recallMaxTokens: config.recallMaxTokens,
|
|
230
|
-
windowDedupe: config.windowDedupe,
|
|
231
|
-
liveWindow,
|
|
232
|
-
dedupSim: config.crossRepoCosine,
|
|
233
|
-
crossRepo: true,
|
|
234
|
-
// F2: resolve the machine-wide index dir via the shared resolver so the
|
|
235
|
-
// cross-repo injected-set dedup works even when MEGACOMPACT_INDEX_DIR is
|
|
236
|
-
// unset. The env var still wins when set (getIndexDir checks it first);
|
|
237
|
-
// the default (~/.mega-compact-index) is the same DB mega-commands and the
|
|
238
|
-
// dashboard read, so injection counts stay consistent. Without this, a
|
|
239
|
-
// bare `process.env` read returns undefined → cross-repo hits re-inject in
|
|
240
|
-
// every new session (the global injected-set is never consulted).
|
|
241
|
-
globalIndexDir: getIndexDir(),
|
|
242
|
-
},
|
|
243
|
-
runtime.store,
|
|
244
|
-
);
|
|
245
|
-
runtime.dashboard.event("recall-crossrepo", {
|
|
246
|
-
source,
|
|
247
|
-
query: query.slice(0, 120),
|
|
248
|
-
injected: x.toInject.length,
|
|
249
|
-
sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
|
|
250
|
-
});
|
|
251
|
-
// Merge, dedup by checkpointId, respect the same token cap by reformatting.
|
|
252
|
-
const seen = new Set(
|
|
253
|
-
sameRepo.toInject.map((h) => h.checkpoint.checkpointId),
|
|
254
|
-
);
|
|
255
|
-
const merged = [...sameRepo.toInject];
|
|
256
|
-
for (const h of x.toInject) {
|
|
257
|
-
if (!seen.has(h.checkpoint.checkpointId)) {
|
|
258
|
-
merged.push(h);
|
|
259
|
-
seen.add(h.checkpoint.checkpointId);
|
|
260
|
-
}
|
|
261
|
-
}
|
|
262
|
-
const block = merged.length ? formatRecallBlock(merged) : "";
|
|
263
|
-
if (merged.length > 0) {
|
|
264
|
-
let sumTokens = 0;
|
|
265
|
-
for (const h of merged) sumTokens += h.checkpoint.tokenEstimate;
|
|
266
|
-
runtime.rt.recallInjections += merged.length;
|
|
267
|
-
runtime.rt.cacheHitTokens += sumTokens;
|
|
268
|
-
incRecallInjected(merged.length, runtime.currentStateDir);
|
|
269
|
-
incCacheHitTokens(sumTokens, runtime.currentStateDir);
|
|
270
|
-
}
|
|
271
|
-
return {
|
|
272
|
-
toInject: merged,
|
|
273
|
-
report: merged.map(
|
|
274
|
-
(h) =>
|
|
275
|
-
` • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`,
|
|
276
|
-
),
|
|
277
|
-
block,
|
|
278
|
-
empty: merged.length === 0,
|
|
279
|
-
// H1: merged cross-repo result reuses the same-repo pass's telemetry.
|
|
280
|
-
hydeInfo: sameRepo.hydeInfo,
|
|
281
|
-
recallMetrics: sameRepo.recallMetrics,
|
|
282
|
-
};
|
|
283
|
-
} catch {
|
|
284
|
-
return sameRepo; // cross-repo failure → same-repo only (non-fatal)
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
/**
|
|
289
|
-
* Extract the live-window message texts from the session manager (Fix C),
|
|
290
|
-
* for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
|
|
291
|
-
* error so recall falls back to unbounded (still correct, just no dedupe).
|
|
292
|
-
* Mirrors recentUserQuery's use of sessionEntryToContextMessages.
|
|
293
|
-
*/
|
|
294
|
-
function extractLiveWindow(ctx: ExtensionContext): string[] {
|
|
295
|
-
try {
|
|
296
|
-
const entries = ctx.sessionManager.getEntries();
|
|
297
|
-
const texts: string[] = [];
|
|
298
|
-
for (const e of entries) {
|
|
299
|
-
for (const m of sessionEntryToContextMessages(e)) {
|
|
300
|
-
const c = (m as { content?: unknown }).content;
|
|
301
|
-
if (typeof c === "string") texts.push(c);
|
|
302
|
-
else if (Array.isArray(c))
|
|
303
|
-
texts.push(c.map((b: { text?: string }) => b.text ?? "").join(" "));
|
|
304
|
-
}
|
|
305
|
-
}
|
|
306
|
-
return texts;
|
|
307
|
-
} catch {
|
|
308
|
-
return [];
|
|
309
|
-
}
|
|
310
|
-
}
|
|
10
|
+
export {
|
|
11
|
+
doRecall,
|
|
12
|
+
doRecallAsync,
|
|
13
|
+
extractLiveWindow,
|
|
14
|
+
} from "./recall/impl.js";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-mega-compact",
|
|
3
|
-
"version": "0.20.
|
|
3
|
+
"version": "0.20.87",
|
|
4
4
|
"description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BSD-3-Clause",
|
package/src/config.ts
CHANGED
|
@@ -144,6 +144,18 @@ export const RAG_HYDE_ENABLED = (): boolean =>
|
|
|
144
144
|
/** Spec 1: vbrainstorm visual design migration for the dashboard. */
|
|
145
145
|
export const NEW_UI = (): boolean => ragEnabled("MEGACOMPACT_NEW_UI");
|
|
146
146
|
|
|
147
|
+
// ---------------------------------------------------------------------------
|
|
148
|
+
// 3WF-3 same-repo recall cosine floor. SEPARATE from the S17 cross-repo floor
|
|
149
|
+
// (config.crossRepoCosine, default 0.90 — stricter, cross-repo only). This is
|
|
150
|
+
// the same-repo floor the 3-source validator applies to the top winner. A low
|
|
151
|
+
// default (0.12) keeps recall permissive within a repo while still rejecting
|
|
152
|
+
// effectively-unrelated hits. Call-time read so tests can set the env per-test.
|
|
153
|
+
// ---------------------------------------------------------------------------
|
|
154
|
+
|
|
155
|
+
/** Same-repo recall cosine floor: top winner must be >= this to be injected. */
|
|
156
|
+
export const RECALL_MIN_COSINE = (): number =>
|
|
157
|
+
Number(process.env.MEGACOMPACT_RECALL_MIN_COSINE ?? "0.12");
|
|
158
|
+
|
|
147
159
|
// ---------------------------------------------------------------------------
|
|
148
160
|
// Vector-cortex flags + breaker constants (VC0A+). Positive sprint flags,
|
|
149
161
|
// default ON, `=0`/`_DISABLED` off. Re-exported from src/config/vector-cortex.ts
|