pi-mega-compact 0.20.86 → 0.20.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.js +9 -0
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +1 -0
- package/dist/extensions/mega-config.js +6 -0
- package/dist/extensions/mega-pipeline/recall/impl.js +258 -0
- package/dist/extensions/mega-pipeline/recall.js +6 -253
- package/dist/src/config.js +9 -0
- package/dist/src/recall/readonly.js +39 -0
- package/dist/src/recall/recall3wf.fixture.js +67 -0
- package/dist/src/recall/validator.js +113 -0
- package/dist/src/recall/vote.js +217 -0
- package/dist/src/store/sqlite/fts5-search.js +26 -0
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +1 -0
- package/extensions/mega-config-types.ts +5 -0
- package/extensions/mega-config.ts +6 -0
- package/extensions/mega-pipeline/recall/impl.ts +312 -0
- package/extensions/mega-pipeline/recall.ts +10 -306
- package/package.json +1 -1
- package/src/config.ts +12 -0
- package/src/failback/types.ts +28 -0
- package/src/recall/readonly.ts +57 -0
- package/src/recall/recall3wf.fixture.ts +87 -0
- package/src/recall/validator.ts +150 -0
- package/src/recall/vote.ts +240 -0
- package/src/store/sqlite/fts5-search.ts +40 -0
package/dist/config.js
CHANGED
|
@@ -110,6 +110,15 @@ export const RAG_HYDE_ENABLED = () => ragEnabled("MEGACOMPACT_HYDE");
|
|
|
110
110
|
/** Spec 1: vbrainstorm visual design migration for the dashboard. */
|
|
111
111
|
export const NEW_UI = () => ragEnabled("MEGACOMPACT_NEW_UI");
|
|
112
112
|
// ---------------------------------------------------------------------------
|
|
113
|
+
// 3WF-3 same-repo recall cosine floor. SEPARATE from the S17 cross-repo floor
|
|
114
|
+
// (config.crossRepoCosine, default 0.90 — stricter, cross-repo only). This is
|
|
115
|
+
// the same-repo floor the 3-source validator applies to the top winner. A low
|
|
116
|
+
// default (0.12) keeps recall permissive within a repo while still rejecting
|
|
117
|
+
// effectively-unrelated hits. Call-time read so tests can set the env per-test.
|
|
118
|
+
// ---------------------------------------------------------------------------
|
|
119
|
+
/** Same-repo recall cosine floor: top winner must be >= this to be injected. */
|
|
120
|
+
export const RECALL_MIN_COSINE = () => Number(process.env.MEGACOMPACT_RECALL_MIN_COSINE ?? "0.12");
|
|
121
|
+
// ---------------------------------------------------------------------------
|
|
113
122
|
// Vector-cortex flags + breaker constants (VC0A+). Positive sprint flags,
|
|
114
123
|
// default ON, `=0`/`_DISABLED` off. Re-exported from src/config/vector-cortex.ts
|
|
115
124
|
// so root consumers share one source of truth.
|
|
@@ -93,6 +93,7 @@ export const SETTINGS = [
|
|
|
93
93
|
num("MEGACOMPACT_L2_THRESHOLD", "L2 Cosine Threshold", "L2 semantic dedup firing point", 0.85, 0, 1),
|
|
94
94
|
num("MEGACOMPACT_L1_JACCARD", "L1 Jaccard Threshold", "L1 MinHash near-dup threshold", 0.8, 0, 1),
|
|
95
95
|
num("MEGACOMPACT_DEDUP_SIM", "Dedup Similarity", "Legacy content-similarity fallback", 0.9, 0, 1),
|
|
96
|
+
num("MEGACOMPACT_RECALL_MIN_COSINE", "Recall Min Cosine (same-repo)", "3WF-3 same-repo floor the 3-source validator applies to the top winner (cross-repo 0.90 stays separate)", 0.12, 0, 1),
|
|
96
97
|
num("MEGACOMPACT_MMR_LAMBDA", "MMR Lambda", "Maximal Marginal Relevance diversity", 0.5, 0, 1),
|
|
97
98
|
num("MEGACOMPACT_SEMDEDUP_COSINE", "SemDeDup Cosine", "Offline SemDeDup pair threshold", 0.95, 0, 1),
|
|
98
99
|
num("MEGACOMPACT_CONSOLIDATE_COSINE", "Consolidate Cosine", "Memory consolidation merge threshold", 0.7, 0, 1),
|
|
@@ -181,6 +181,12 @@ export function loadConfig() {
|
|
|
181
181
|
autoWikiEnabled: envBool("MEGACOMPACT_AUTO_WIKI", true),
|
|
182
182
|
crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
|
|
183
183
|
crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
|
|
184
|
+
// 3WF-3: SAME-repo recall cosine floor applied by the 3-source validator to
|
|
185
|
+
// the top winner. SEPARATE from crossRepoCosine (S17, default 0.90, stricter
|
|
186
|
+
// and cross-repo only). This same-repo floor is permissive by default (0.12)
|
|
187
|
+
// so recall still surfaces loosely-relevant within-repo context while
|
|
188
|
+
// rejecting effectively-unrelated hits. Mirrors src/config.ts RECALL_MIN_COSINE.
|
|
189
|
+
recallMinCosine: Number(process.env.MEGACOMPACT_RECALL_MIN_COSINE ?? "0.12"),
|
|
184
190
|
memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),
|
|
185
191
|
memoryReviewInterval: envFlag("MEGACOMPACT_MEMORY_REVIEW_INTERVAL", 10),
|
|
186
192
|
recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* recall/impl.ts — unified Layer-5 recall pipeline implementation (3WF-3 split).
|
|
3
|
+
*
|
|
4
|
+
* Behavior is UNCHANGED from the pre-split recall.ts. `doRecall` is the ONE path
|
|
5
|
+
* that injects (sync). `doRecallAsync` augments with optional cross-repo HNSW
|
|
6
|
+
* on resume / /mega-recall --cross-repo. Both mutate the shared MegaRuntime
|
|
7
|
+
* (token accounting, ticker, dashboard events). The shell recall.ts re-exports
|
|
8
|
+
* these names so `export * from "./mega-pipeline/recall.js"` stays byte-stable.
|
|
9
|
+
*/
|
|
10
|
+
import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
11
|
+
import { recallAndInline, recallAndInlineAsync, formatRecallBlock, } from "../../../src/recall.js";
|
|
12
|
+
import { normalizeSessionId } from "../../../src/store.js";
|
|
13
|
+
import { incRecallInjected, incCacheHitTokens, getIndexDir, } from "../../../src/store/sqlite.js";
|
|
14
|
+
import { ensureConversationIdFor, recordTurnWrite, recordRecallWrite, } from "../../mega-turn-store.js";
|
|
15
|
+
import { C } from "../../mega-runtime.js";
|
|
16
|
+
import { recordRecallLatency } from "../../mega-runtime/vc-observer.js";
|
|
17
|
+
/**
|
|
18
|
+
* Unified recall (Layer 5). The ONE path that injects. Returns the recall
|
|
19
|
+
* result; callers decide whether to stage it for before_agent_start (resume)
|
|
20
|
+
* or report it (command).
|
|
21
|
+
*/
|
|
22
|
+
export function doRecall(runtime, config, ctx, query, source) {
|
|
23
|
+
runtime.bindRepo(ctx.cwd);
|
|
24
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
25
|
+
// Live window text for inline dedupe (Fix C): drop recalled checkpoints that
|
|
26
|
+
// are already resident in the session, so recall never re-injects context the
|
|
27
|
+
// model can already see. Best-effort — an empty window just skips dedupe.
|
|
28
|
+
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
29
|
+
const recallStartMs = Date.now();
|
|
30
|
+
const result = recallAndInline({
|
|
31
|
+
sessionId: sid,
|
|
32
|
+
query,
|
|
33
|
+
limit: config.autoInlineK,
|
|
34
|
+
source,
|
|
35
|
+
skipInjected: true,
|
|
36
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
37
|
+
windowDedupe: config.windowDedupe,
|
|
38
|
+
liveWindow,
|
|
39
|
+
dedupSim: config.dedupSim,
|
|
40
|
+
}, runtime.store);
|
|
41
|
+
runtime.dashboard.event("recall", {
|
|
42
|
+
source,
|
|
43
|
+
query: query.slice(0, 120),
|
|
44
|
+
injected: result.toInject.length,
|
|
45
|
+
empty: result.empty,
|
|
46
|
+
});
|
|
47
|
+
if (config.ragRecallMetrics && result.hydeInfo) {
|
|
48
|
+
runtime.dashboard.event("hyde_executed", {
|
|
49
|
+
sessionId: sid,
|
|
50
|
+
ran: result.hydeInfo.ran,
|
|
51
|
+
skipped: result.hydeInfo.skipped,
|
|
52
|
+
reason: result.hydeInfo.reason,
|
|
53
|
+
hypotheticalDoc: result.hydeInfo.hypotheticalDoc.slice(0, 400),
|
|
54
|
+
generationMs: result.hydeInfo.generationMs,
|
|
55
|
+
rawHitCount: result.hydeInfo.rawHitCount,
|
|
56
|
+
hydeHitCount: result.hydeInfo.hydeHitCount,
|
|
57
|
+
fusedHitCount: result.hydeInfo.fusedHitCount,
|
|
58
|
+
lift: result.hydeInfo.lift,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
if (config.ragRecallMetrics && result.recallMetrics) {
|
|
62
|
+
runtime.dashboard.event("recall_metrics", {
|
|
63
|
+
sessionId: sid,
|
|
64
|
+
hitCount: result.recallMetrics.hitCount,
|
|
65
|
+
score: result.recallMetrics.score,
|
|
66
|
+
pass: result.recallMetrics.pass,
|
|
67
|
+
relevance: result.recallMetrics.relevance,
|
|
68
|
+
coverage: result.recallMetrics.coverage,
|
|
69
|
+
diversity: result.recallMetrics.diversity,
|
|
70
|
+
specificity: result.recallMetrics.specificity,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
if (!result.empty && result.toInject.length > 0) {
|
|
74
|
+
const top = result.toInject[0];
|
|
75
|
+
const scorePct = Math.round((top.score ?? 0) * 100);
|
|
76
|
+
const files = top.checkpoint.filesModified ?? [];
|
|
77
|
+
const label = files.length
|
|
78
|
+
? files
|
|
79
|
+
.map((f) => f.split("/").pop() ?? f)
|
|
80
|
+
.slice(0, 2)
|
|
81
|
+
.join(", ")
|
|
82
|
+
: top.checkpoint.checkpointId;
|
|
83
|
+
runtime.pushTicker(`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`);
|
|
84
|
+
runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
|
|
85
|
+
}
|
|
86
|
+
let sumTokens = 0;
|
|
87
|
+
for (const h of result.toInject)
|
|
88
|
+
sumTokens += h.checkpoint.tokenEstimate;
|
|
89
|
+
if (result.toInject.length > 0) {
|
|
90
|
+
runtime.rt.recallInjections += result.toInject.length;
|
|
91
|
+
runtime.rt.cacheHitTokens += sumTokens;
|
|
92
|
+
incRecallInjected(result.toInject.length, runtime.currentStateDir);
|
|
93
|
+
incCacheHitTokens(sumTokens, runtime.currentStateDir);
|
|
94
|
+
}
|
|
95
|
+
// S43: record recall provenance — which checkpoints/summaries served this
|
|
96
|
+
// turn, their score + source path. Linked to the turn row written at
|
|
97
|
+
// turn_end via the conversation+turnIndex. Best-effort + non-fatal.
|
|
98
|
+
// Persists telemetry (HyDE + recall metrics) even when recall returned
|
|
99
|
+
// no hits, so empty-recall HyDE invocations are still visible in the
|
|
100
|
+
// dashboard Turns/Metrics tabs.
|
|
101
|
+
const hasTelemetry = result.hydeInfo != null || result.recallMetrics != null;
|
|
102
|
+
if (result.toInject.length > 0 || hasTelemetry) {
|
|
103
|
+
try {
|
|
104
|
+
const convId = ensureConversationIdFor(config, sid, runtime.currentStateDir);
|
|
105
|
+
const turnId = recordTurnWrite(config, {
|
|
106
|
+
conversationId: convId,
|
|
107
|
+
sessionId: sid,
|
|
108
|
+
turnIndex: runtime.currentTurn,
|
|
109
|
+
role: "assistant",
|
|
110
|
+
startedAt: Date.now(),
|
|
111
|
+
hyde: result.hydeInfo ?? undefined,
|
|
112
|
+
recallMetrics: result.recallMetrics ?? undefined,
|
|
113
|
+
}, runtime.currentStateDir);
|
|
114
|
+
if (result.toInject.length > 0) {
|
|
115
|
+
recordRecallWrite(config, turnId, result.toInject.map((h) => ({
|
|
116
|
+
checkpointId: h.checkpoint.checkpointId,
|
|
117
|
+
score: h.score,
|
|
118
|
+
source: h.raptorLevel !== undefined
|
|
119
|
+
? "raptor"
|
|
120
|
+
: h.repoId
|
|
121
|
+
? "cross-repo"
|
|
122
|
+
: "flat",
|
|
123
|
+
raptorLevel: h.raptorLevel,
|
|
124
|
+
})), runtime.currentStateDir);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
/* non-fatal: recall provenance never breaks the recall path */
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
// VC0A: record recall latency on the eval observer (mode A) so the dashboard
|
|
132
|
+
// histogram reflects real data. No-op when the observer is absent (flag off /
|
|
133
|
+
// construction failure).
|
|
134
|
+
try {
|
|
135
|
+
recordRecallLatency(runtime, Date.now() - recallStartMs, sid, 0);
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
/* non-fatal: latency recording never breaks recall */
|
|
139
|
+
}
|
|
140
|
+
return result;
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* S17: async recall with optional cross-repo augmentation. Used on resume
|
|
144
|
+
* (session_start) and /mega-recall --cross-repo — NEVER from the mid-turn
|
|
145
|
+
* context handler (that stays sync). Runs the sync same-repo scan first; if it
|
|
146
|
+
* returns < config.autoInlineK hits AND crossRepo is enabled, awaits the PGlite
|
|
147
|
+
* HNSW cross-repo path and merges (source-labeled, deduped by checkpointId). The
|
|
148
|
+
* recallMaxTokens cap + windowDedupe apply to the merged set so cross-repo can
|
|
149
|
+
* never net-inflate the window. Cross-repo uses a stricter cosine floor
|
|
150
|
+
* (config.crossRepoCosine) than same-repo. Non-fatal: any async failure returns
|
|
151
|
+
* the same-repo result unchanged.
|
|
152
|
+
*/
|
|
153
|
+
export async function doRecallAsync(runtime, config, ctx, query, source, opts = {}) {
|
|
154
|
+
runtime.bindRepo(ctx.cwd);
|
|
155
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
156
|
+
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
157
|
+
// Sync same-repo first (fast, never blocks).
|
|
158
|
+
const sameRepo = recallAndInline({
|
|
159
|
+
sessionId: sid,
|
|
160
|
+
query,
|
|
161
|
+
limit: config.autoInlineK,
|
|
162
|
+
source,
|
|
163
|
+
skipInjected: true,
|
|
164
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
165
|
+
windowDedupe: config.windowDedupe,
|
|
166
|
+
liveWindow,
|
|
167
|
+
dedupSim: config.dedupSim,
|
|
168
|
+
}, runtime.store);
|
|
169
|
+
if (!config.crossRepoEnabled || !opts.crossRepo)
|
|
170
|
+
return sameRepo;
|
|
171
|
+
if (sameRepo.toInject.length >= config.autoInlineK)
|
|
172
|
+
return sameRepo; // same-repo satisfied
|
|
173
|
+
// Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
|
|
174
|
+
try {
|
|
175
|
+
const x = await recallAndInlineAsync({
|
|
176
|
+
sessionId: sid,
|
|
177
|
+
query,
|
|
178
|
+
limit: config.autoInlineK,
|
|
179
|
+
source,
|
|
180
|
+
skipInjected: true,
|
|
181
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
182
|
+
windowDedupe: config.windowDedupe,
|
|
183
|
+
liveWindow,
|
|
184
|
+
dedupSim: config.crossRepoCosine,
|
|
185
|
+
crossRepo: true,
|
|
186
|
+
// F2: resolve the machine-wide index dir via the shared resolver so the
|
|
187
|
+
// cross-repo injected-set dedup works even when MEGACOMPACT_INDEX_DIR is
|
|
188
|
+
// unset. The env var still wins when set (getIndexDir checks it first);
|
|
189
|
+
// the default (~/.mega-compact-index) is the same DB mega-commands and the
|
|
190
|
+
// dashboard read, so injection counts stay consistent. Without this, a
|
|
191
|
+
// bare `process.env` read returns undefined → cross-repo hits re-inject in
|
|
192
|
+
// every new session (the global injected-set is never consulted).
|
|
193
|
+
globalIndexDir: getIndexDir(),
|
|
194
|
+
}, runtime.store);
|
|
195
|
+
runtime.dashboard.event("recall-crossrepo", {
|
|
196
|
+
source,
|
|
197
|
+
query: query.slice(0, 120),
|
|
198
|
+
injected: x.toInject.length,
|
|
199
|
+
sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
|
|
200
|
+
});
|
|
201
|
+
// Merge, dedup by checkpointId, respect the same token cap by reformatting.
|
|
202
|
+
const seen = new Set(sameRepo.toInject.map((h) => h.checkpoint.checkpointId));
|
|
203
|
+
const merged = [...sameRepo.toInject];
|
|
204
|
+
for (const h of x.toInject) {
|
|
205
|
+
if (!seen.has(h.checkpoint.checkpointId)) {
|
|
206
|
+
merged.push(h);
|
|
207
|
+
seen.add(h.checkpoint.checkpointId);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
const block = merged.length ? formatRecallBlock(merged) : "";
|
|
211
|
+
if (merged.length > 0) {
|
|
212
|
+
let sumTokens = 0;
|
|
213
|
+
for (const h of merged)
|
|
214
|
+
sumTokens += h.checkpoint.tokenEstimate;
|
|
215
|
+
runtime.rt.recallInjections += merged.length;
|
|
216
|
+
runtime.rt.cacheHitTokens += sumTokens;
|
|
217
|
+
incRecallInjected(merged.length, runtime.currentStateDir);
|
|
218
|
+
incCacheHitTokens(sumTokens, runtime.currentStateDir);
|
|
219
|
+
}
|
|
220
|
+
return {
|
|
221
|
+
toInject: merged,
|
|
222
|
+
report: merged.map((h) => ` • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`),
|
|
223
|
+
block,
|
|
224
|
+
empty: merged.length === 0,
|
|
225
|
+
// H1: merged cross-repo result reuses the same-repo pass's telemetry.
|
|
226
|
+
hydeInfo: sameRepo.hydeInfo,
|
|
227
|
+
recallMetrics: sameRepo.recallMetrics,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
return sameRepo; // cross-repo failure → same-repo only (non-fatal)
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Extract the live-window message texts from the session manager (Fix C),
|
|
236
|
+
* for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
|
|
237
|
+
* error so recall falls back to unbounded (still correct, just no dedupe).
|
|
238
|
+
* Mirrors recentUserQuery's use of sessionEntryToContextMessages.
|
|
239
|
+
*/
|
|
240
|
+
export function extractLiveWindow(ctx) {
|
|
241
|
+
try {
|
|
242
|
+
const entries = ctx.sessionManager.getEntries();
|
|
243
|
+
const texts = [];
|
|
244
|
+
for (const e of entries) {
|
|
245
|
+
for (const m of sessionEntryToContextMessages(e)) {
|
|
246
|
+
const c = m.content;
|
|
247
|
+
if (typeof c === "string")
|
|
248
|
+
texts.push(c);
|
|
249
|
+
else if (Array.isArray(c))
|
|
250
|
+
texts.push(c.map((b) => b.text ?? "").join(" "));
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
return texts;
|
|
254
|
+
}
|
|
255
|
+
catch {
|
|
256
|
+
return [];
|
|
257
|
+
}
|
|
258
|
+
}
|
|
@@ -1,256 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* recall.ts — unified Layer-5 recall pipeline.
|
|
2
|
+
* recall.ts — shell re-export for the unified Layer-5 recall pipeline (3WF-3 split).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* Delegate-shell pattern: the implementation lives in ./recall/impl.ts (kept
|
|
5
|
+
* under the 300-line soft cap). All public symbols are re-exported here so
|
|
6
|
+
* `export * from "./mega-pipeline/recall.js"` (mega-pipeline.ts) and any direct
|
|
7
|
+
* importers keep resolving with byte-identical names.
|
|
7
8
|
*/
|
|
8
|
-
|
|
9
|
-
import { recallAndInline, recallAndInlineAsync, formatRecallBlock, } from "../../src/recall.js";
|
|
10
|
-
import { normalizeSessionId } from "../../src/store.js";
|
|
11
|
-
import { incRecallInjected, incCacheHitTokens, getIndexDir, } from "../../src/store/sqlite.js";
|
|
12
|
-
import { ensureConversationIdFor, recordTurnWrite, recordRecallWrite, } from "../mega-turn-store.js";
|
|
13
|
-
import { C } from "../mega-runtime.js";
|
|
14
|
-
import { recordRecallLatency } from "../mega-runtime/vc-observer.js";
|
|
15
|
-
/**
|
|
16
|
-
* Unified recall (Layer 5). The ONE path that injects. Returns the recall
|
|
17
|
-
* result; callers decide whether to stage it for before_agent_start (resume)
|
|
18
|
-
* or report it (command).
|
|
19
|
-
*/
|
|
20
|
-
export function doRecall(runtime, config, ctx, query, source) {
|
|
21
|
-
runtime.bindRepo(ctx.cwd);
|
|
22
|
-
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
23
|
-
// Live window text for inline dedupe (Fix C): drop recalled checkpoints that
|
|
24
|
-
// are already resident in the session, so recall never re-injects context the
|
|
25
|
-
// model can already see. Best-effort — an empty window just skips dedupe.
|
|
26
|
-
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
27
|
-
const recallStartMs = Date.now();
|
|
28
|
-
const result = recallAndInline({
|
|
29
|
-
sessionId: sid,
|
|
30
|
-
query,
|
|
31
|
-
limit: config.autoInlineK,
|
|
32
|
-
source,
|
|
33
|
-
skipInjected: true,
|
|
34
|
-
recallMaxTokens: config.recallMaxTokens,
|
|
35
|
-
windowDedupe: config.windowDedupe,
|
|
36
|
-
liveWindow,
|
|
37
|
-
dedupSim: config.dedupSim,
|
|
38
|
-
}, runtime.store);
|
|
39
|
-
runtime.dashboard.event("recall", {
|
|
40
|
-
source,
|
|
41
|
-
query: query.slice(0, 120),
|
|
42
|
-
injected: result.toInject.length,
|
|
43
|
-
empty: result.empty,
|
|
44
|
-
});
|
|
45
|
-
if (config.ragRecallMetrics && result.hydeInfo) {
|
|
46
|
-
runtime.dashboard.event("hyde_executed", {
|
|
47
|
-
sessionId: sid,
|
|
48
|
-
ran: result.hydeInfo.ran,
|
|
49
|
-
skipped: result.hydeInfo.skipped,
|
|
50
|
-
reason: result.hydeInfo.reason,
|
|
51
|
-
hypotheticalDoc: result.hydeInfo.hypotheticalDoc.slice(0, 400),
|
|
52
|
-
generationMs: result.hydeInfo.generationMs,
|
|
53
|
-
rawHitCount: result.hydeInfo.rawHitCount,
|
|
54
|
-
hydeHitCount: result.hydeInfo.hydeHitCount,
|
|
55
|
-
fusedHitCount: result.hydeInfo.fusedHitCount,
|
|
56
|
-
lift: result.hydeInfo.lift,
|
|
57
|
-
});
|
|
58
|
-
}
|
|
59
|
-
if (config.ragRecallMetrics && result.recallMetrics) {
|
|
60
|
-
runtime.dashboard.event("recall_metrics", {
|
|
61
|
-
sessionId: sid,
|
|
62
|
-
hitCount: result.recallMetrics.hitCount,
|
|
63
|
-
score: result.recallMetrics.score,
|
|
64
|
-
pass: result.recallMetrics.pass,
|
|
65
|
-
relevance: result.recallMetrics.relevance,
|
|
66
|
-
coverage: result.recallMetrics.coverage,
|
|
67
|
-
diversity: result.recallMetrics.diversity,
|
|
68
|
-
specificity: result.recallMetrics.specificity,
|
|
69
|
-
});
|
|
70
|
-
}
|
|
71
|
-
if (!result.empty && result.toInject.length > 0) {
|
|
72
|
-
const top = result.toInject[0];
|
|
73
|
-
const scorePct = Math.round((top.score ?? 0) * 100);
|
|
74
|
-
const files = top.checkpoint.filesModified ?? [];
|
|
75
|
-
const label = files.length
|
|
76
|
-
? files
|
|
77
|
-
.map((f) => f.split("/").pop() ?? f)
|
|
78
|
-
.slice(0, 2)
|
|
79
|
-
.join(", ")
|
|
80
|
-
: top.checkpoint.checkpointId;
|
|
81
|
-
runtime.pushTicker(`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`);
|
|
82
|
-
runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
|
|
83
|
-
}
|
|
84
|
-
let sumTokens = 0;
|
|
85
|
-
for (const h of result.toInject)
|
|
86
|
-
sumTokens += h.checkpoint.tokenEstimate;
|
|
87
|
-
if (result.toInject.length > 0) {
|
|
88
|
-
runtime.rt.recallInjections += result.toInject.length;
|
|
89
|
-
runtime.rt.cacheHitTokens += sumTokens;
|
|
90
|
-
incRecallInjected(result.toInject.length, runtime.currentStateDir);
|
|
91
|
-
incCacheHitTokens(sumTokens, runtime.currentStateDir);
|
|
92
|
-
}
|
|
93
|
-
// S43: record recall provenance — which checkpoints/summaries served this
|
|
94
|
-
// turn, their score + source path. Linked to the turn row written at
|
|
95
|
-
// turn_end via the conversation+turnIndex. Best-effort + non-fatal.
|
|
96
|
-
// Persists telemetry (HyDE + recall metrics) even when recall returned
|
|
97
|
-
// no hits, so empty-recall HyDE invocations are still visible in the
|
|
98
|
-
// dashboard Turns/Metrics tabs.
|
|
99
|
-
const hasTelemetry = result.hydeInfo != null || result.recallMetrics != null;
|
|
100
|
-
if (result.toInject.length > 0 || hasTelemetry) {
|
|
101
|
-
try {
|
|
102
|
-
const convId = ensureConversationIdFor(config, sid, runtime.currentStateDir);
|
|
103
|
-
const turnId = recordTurnWrite(config, {
|
|
104
|
-
conversationId: convId,
|
|
105
|
-
sessionId: sid,
|
|
106
|
-
turnIndex: runtime.currentTurn,
|
|
107
|
-
role: "assistant",
|
|
108
|
-
startedAt: Date.now(),
|
|
109
|
-
hyde: result.hydeInfo ?? undefined,
|
|
110
|
-
recallMetrics: result.recallMetrics ?? undefined,
|
|
111
|
-
}, runtime.currentStateDir);
|
|
112
|
-
if (result.toInject.length > 0) {
|
|
113
|
-
recordRecallWrite(config, turnId, result.toInject.map((h) => ({
|
|
114
|
-
checkpointId: h.checkpoint.checkpointId,
|
|
115
|
-
score: h.score,
|
|
116
|
-
source: h.raptorLevel !== undefined
|
|
117
|
-
? "raptor"
|
|
118
|
-
: h.repoId
|
|
119
|
-
? "cross-repo"
|
|
120
|
-
: "flat",
|
|
121
|
-
raptorLevel: h.raptorLevel,
|
|
122
|
-
})), runtime.currentStateDir);
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
catch {
|
|
126
|
-
/* non-fatal: recall provenance never breaks the recall path */
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
// VC0A: record recall latency on the eval observer (mode A) so the dashboard
|
|
130
|
-
// histogram reflects real data. No-op when the observer is absent (flag off /
|
|
131
|
-
// construction failure).
|
|
132
|
-
try {
|
|
133
|
-
recordRecallLatency(runtime, Date.now() - recallStartMs, sid, 0);
|
|
134
|
-
}
|
|
135
|
-
catch {
|
|
136
|
-
/* non-fatal: latency recording never breaks recall */
|
|
137
|
-
}
|
|
138
|
-
return result;
|
|
139
|
-
}
|
|
140
|
-
/**
|
|
141
|
-
* S17: async recall with optional cross-repo augmentation. Used on resume
|
|
142
|
-
* (session_start) and /mega-recall --cross-repo — NEVER from the mid-turn
|
|
143
|
-
* context handler (that stays sync). Runs the sync same-repo scan first; if it
|
|
144
|
-
* returns < config.autoInlineK hits AND crossRepo is enabled, awaits the PGlite
|
|
145
|
-
* HNSW cross-repo path and merges (source-labeled, deduped by checkpointId). The
|
|
146
|
-
* recallMaxTokens cap + windowDedupe apply to the merged set so cross-repo can
|
|
147
|
-
* never net-inflate the window. Cross-repo uses a stricter cosine floor
|
|
148
|
-
* (config.crossRepoCosine) than same-repo. Non-fatal: any async failure returns
|
|
149
|
-
* the same-repo result unchanged.
|
|
150
|
-
*/
|
|
151
|
-
export async function doRecallAsync(runtime, config, ctx, query, source, opts = {}) {
|
|
152
|
-
runtime.bindRepo(ctx.cwd);
|
|
153
|
-
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
154
|
-
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
155
|
-
// Sync same-repo first (fast, never blocks).
|
|
156
|
-
const sameRepo = recallAndInline({
|
|
157
|
-
sessionId: sid,
|
|
158
|
-
query,
|
|
159
|
-
limit: config.autoInlineK,
|
|
160
|
-
source,
|
|
161
|
-
skipInjected: true,
|
|
162
|
-
recallMaxTokens: config.recallMaxTokens,
|
|
163
|
-
windowDedupe: config.windowDedupe,
|
|
164
|
-
liveWindow,
|
|
165
|
-
dedupSim: config.dedupSim,
|
|
166
|
-
}, runtime.store);
|
|
167
|
-
if (!config.crossRepoEnabled || !opts.crossRepo)
|
|
168
|
-
return sameRepo;
|
|
169
|
-
if (sameRepo.toInject.length >= config.autoInlineK)
|
|
170
|
-
return sameRepo; // same-repo satisfied
|
|
171
|
-
// Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
|
|
172
|
-
try {
|
|
173
|
-
const x = await recallAndInlineAsync({
|
|
174
|
-
sessionId: sid,
|
|
175
|
-
query,
|
|
176
|
-
limit: config.autoInlineK,
|
|
177
|
-
source,
|
|
178
|
-
skipInjected: true,
|
|
179
|
-
recallMaxTokens: config.recallMaxTokens,
|
|
180
|
-
windowDedupe: config.windowDedupe,
|
|
181
|
-
liveWindow,
|
|
182
|
-
dedupSim: config.crossRepoCosine,
|
|
183
|
-
crossRepo: true,
|
|
184
|
-
// F2: resolve the machine-wide index dir via the shared resolver so the
|
|
185
|
-
// cross-repo injected-set dedup works even when MEGACOMPACT_INDEX_DIR is
|
|
186
|
-
// unset. The env var still wins when set (getIndexDir checks it first);
|
|
187
|
-
// the default (~/.mega-compact-index) is the same DB mega-commands and the
|
|
188
|
-
// dashboard read, so injection counts stay consistent. Without this, a
|
|
189
|
-
// bare `process.env` read returns undefined → cross-repo hits re-inject in
|
|
190
|
-
// every new session (the global injected-set is never consulted).
|
|
191
|
-
globalIndexDir: getIndexDir(),
|
|
192
|
-
}, runtime.store);
|
|
193
|
-
runtime.dashboard.event("recall-crossrepo", {
|
|
194
|
-
source,
|
|
195
|
-
query: query.slice(0, 120),
|
|
196
|
-
injected: x.toInject.length,
|
|
197
|
-
sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
|
|
198
|
-
});
|
|
199
|
-
// Merge, dedup by checkpointId, respect the same token cap by reformatting.
|
|
200
|
-
const seen = new Set(sameRepo.toInject.map((h) => h.checkpoint.checkpointId));
|
|
201
|
-
const merged = [...sameRepo.toInject];
|
|
202
|
-
for (const h of x.toInject) {
|
|
203
|
-
if (!seen.has(h.checkpoint.checkpointId)) {
|
|
204
|
-
merged.push(h);
|
|
205
|
-
seen.add(h.checkpoint.checkpointId);
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
const block = merged.length ? formatRecallBlock(merged) : "";
|
|
209
|
-
if (merged.length > 0) {
|
|
210
|
-
let sumTokens = 0;
|
|
211
|
-
for (const h of merged)
|
|
212
|
-
sumTokens += h.checkpoint.tokenEstimate;
|
|
213
|
-
runtime.rt.recallInjections += merged.length;
|
|
214
|
-
runtime.rt.cacheHitTokens += sumTokens;
|
|
215
|
-
incRecallInjected(merged.length, runtime.currentStateDir);
|
|
216
|
-
incCacheHitTokens(sumTokens, runtime.currentStateDir);
|
|
217
|
-
}
|
|
218
|
-
return {
|
|
219
|
-
toInject: merged,
|
|
220
|
-
report: merged.map((h) => ` • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`),
|
|
221
|
-
block,
|
|
222
|
-
empty: merged.length === 0,
|
|
223
|
-
// H1: merged cross-repo result reuses the same-repo pass's telemetry.
|
|
224
|
-
hydeInfo: sameRepo.hydeInfo,
|
|
225
|
-
recallMetrics: sameRepo.recallMetrics,
|
|
226
|
-
};
|
|
227
|
-
}
|
|
228
|
-
catch {
|
|
229
|
-
return sameRepo; // cross-repo failure → same-repo only (non-fatal)
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
/**
|
|
233
|
-
* Extract the live-window message texts from the session manager (Fix C),
|
|
234
|
-
* for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
|
|
235
|
-
* error so recall falls back to unbounded (still correct, just no dedupe).
|
|
236
|
-
* Mirrors recentUserQuery's use of sessionEntryToContextMessages.
|
|
237
|
-
*/
|
|
238
|
-
function extractLiveWindow(ctx) {
|
|
239
|
-
try {
|
|
240
|
-
const entries = ctx.sessionManager.getEntries();
|
|
241
|
-
const texts = [];
|
|
242
|
-
for (const e of entries) {
|
|
243
|
-
for (const m of sessionEntryToContextMessages(e)) {
|
|
244
|
-
const c = m.content;
|
|
245
|
-
if (typeof c === "string")
|
|
246
|
-
texts.push(c);
|
|
247
|
-
else if (Array.isArray(c))
|
|
248
|
-
texts.push(c.map((b) => b.text ?? "").join(" "));
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
return texts;
|
|
252
|
-
}
|
|
253
|
-
catch {
|
|
254
|
-
return [];
|
|
255
|
-
}
|
|
256
|
-
}
|
|
9
|
+
export { doRecall, doRecallAsync, extractLiveWindow, } from "./recall/impl.js";
|
package/dist/src/config.js
CHANGED
|
@@ -110,6 +110,15 @@ export const RAG_HYDE_ENABLED = () => ragEnabled("MEGACOMPACT_HYDE");
|
|
|
110
110
|
/** Spec 1: vbrainstorm visual design migration for the dashboard. */
|
|
111
111
|
export const NEW_UI = () => ragEnabled("MEGACOMPACT_NEW_UI");
|
|
112
112
|
// ---------------------------------------------------------------------------
|
|
113
|
+
// 3WF-3 same-repo recall cosine floor. SEPARATE from the S17 cross-repo floor
|
|
114
|
+
// (config.crossRepoCosine, default 0.90 — stricter, cross-repo only). This is
|
|
115
|
+
// the same-repo floor the 3-source validator applies to the top winner. A low
|
|
116
|
+
// default (0.12) keeps recall permissive within a repo while still rejecting
|
|
117
|
+
// effectively-unrelated hits. Call-time read so tests can set the env per-test.
|
|
118
|
+
// ---------------------------------------------------------------------------
|
|
119
|
+
/** Same-repo recall cosine floor: top winner must be >= this to be injected. */
|
|
120
|
+
export const RECALL_MIN_COSINE = () => Number(process.env.MEGACOMPACT_RECALL_MIN_COSINE ?? "0.12");
|
|
121
|
+
// ---------------------------------------------------------------------------
|
|
113
122
|
// Vector-cortex flags + breaker constants (VC0A+). Positive sprint flags,
|
|
114
123
|
// default ON, `=0`/`_DISABLED` off. Re-exported from src/config/vector-cortex.ts
|
|
115
124
|
// so root consumers share one source of truth.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* recall/readonly.ts — read-only recall variant (3WF-3 Source A).
|
|
3
|
+
*
|
|
4
|
+
* A pure search+rank seam wrapping `engine.recall`'s RAW `hits` path. It is the
|
|
5
|
+
* canonical read-only entry point going forward (triggerGuard.ts still inlines
|
|
6
|
+
* `recall(...).hits` for its own need; this module is additive and does NOT
|
|
7
|
+
* refactor it).
|
|
8
|
+
*
|
|
9
|
+
* HARD contract (QA): this module MUST NOT call `vectorMarkInjected`, must NOT
|
|
10
|
+
* write any turn/recall rows, and must NOT emit S43 telemetry. It only searches
|
|
11
|
+
* and returns hits for the vote. RecallAndInline's inject loop is the ONLY place
|
|
12
|
+
* the injected-set is mutated; keying the vote on raw `hits` (skipInjected:false
|
|
13
|
+
* => hits === newHits) is deliberate — `newHits` is post-`skipInjected` filter,
|
|
14
|
+
* which would distort overlap appearance.
|
|
15
|
+
*
|
|
16
|
+
* Non-fatal: any failure returns [] so the caller degrades to other sources.
|
|
17
|
+
* Pi-agnostic: no pi runtime imports.
|
|
18
|
+
*/
|
|
19
|
+
import { recall } from "../engine.js";
|
|
20
|
+
/**
|
|
21
|
+
* Raw, read-only recall hits for the 3-source vote. Returns `engine.recall`'s
|
|
22
|
+
* RAW `.hits` (skipInjected:false => equals the unfiltered vector result). No
|
|
23
|
+
* injected-set mutation, no turn writes, no telemetry. Returns [] on failure.
|
|
24
|
+
*/
|
|
25
|
+
export function recallRawHits(opts, store) {
|
|
26
|
+
try {
|
|
27
|
+
const result = recall({
|
|
28
|
+
sessionId: opts.sessionId,
|
|
29
|
+
query: opts.query,
|
|
30
|
+
limit: opts.limit ?? 3,
|
|
31
|
+
skipInjected: false,
|
|
32
|
+
}, store);
|
|
33
|
+
return result.hits;
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
// Non-fatal: never break the agent loop. Degrade to other sources.
|
|
37
|
+
return [];
|
|
38
|
+
}
|
|
39
|
+
}
|