pi-mega-compact 0.20.86 → 0.20.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.js +9 -0
- package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +1 -0
- package/dist/extensions/mega-config.js +6 -0
- package/dist/extensions/mega-events/context-handler/injectionConfirm.fixture.js +63 -0
- package/dist/extensions/mega-events/context-handler/injectionConfirm.js +107 -0
- package/dist/extensions/mega-events/context-handler/triggerGuard.js +11 -17
- package/dist/extensions/mega-events/context-handler.js +17 -1
- package/dist/extensions/mega-pipeline/recall/impl.js +258 -0
- package/dist/extensions/mega-pipeline/recall.js +6 -253
- package/dist/src/config.js +9 -0
- package/dist/src/failback/floor.js +35 -0
- package/dist/src/recall/readonly.js +39 -0
- package/dist/src/recall/recall3wf.fixture.js +67 -0
- package/dist/src/recall/validator.js +99 -0
- package/dist/src/recall/vote.js +217 -0
- package/dist/src/store/sqlite/fts5-search.js +26 -0
- package/extensions/dashboard-server/routes-rag-settings-helpers.ts +1 -0
- package/extensions/mega-config-types.ts +5 -0
- package/extensions/mega-config.ts +6 -0
- package/extensions/mega-events/context-handler/injectionConfirm.fixture.ts +90 -0
- package/extensions/mega-events/context-handler/injectionConfirm.ts +168 -0
- package/extensions/mega-events/context-handler/triggerGuard.ts +14 -22
- package/extensions/mega-events/context-handler.ts +16 -1
- package/extensions/mega-pipeline/recall/impl.ts +312 -0
- package/extensions/mega-pipeline/recall.ts +10 -306
- package/package.json +1 -1
- package/src/config.ts +12 -0
- package/src/failback/floor.ts +71 -0
- package/src/failback/types.ts +44 -0
- package/src/recall/readonly.ts +57 -0
- package/src/recall/recall3wf.fixture.ts +87 -0
- package/src/recall/validator.ts +137 -0
- package/src/recall/vote.ts +240 -0
- package/src/store/sqlite/fts5-search.ts +40 -0
package/dist/config.js
CHANGED
|
@@ -110,6 +110,15 @@ export const RAG_HYDE_ENABLED = () => ragEnabled("MEGACOMPACT_HYDE");
|
|
|
110
110
|
/** Spec 1: vbrainstorm visual design migration for the dashboard. */
|
|
111
111
|
export const NEW_UI = () => ragEnabled("MEGACOMPACT_NEW_UI");
|
|
112
112
|
// ---------------------------------------------------------------------------
|
|
113
|
+
// 3WF-3 same-repo recall cosine floor. SEPARATE from the S17 cross-repo floor
|
|
114
|
+
// (config.crossRepoCosine, default 0.90 — stricter, cross-repo only). This is
|
|
115
|
+
// the same-repo floor the 3-source validator applies to the top winner. A low
|
|
116
|
+
// default (0.12) keeps recall permissive within a repo while still rejecting
|
|
117
|
+
// effectively-unrelated hits. Call-time read so tests can set the env per-test.
|
|
118
|
+
// ---------------------------------------------------------------------------
|
|
119
|
+
/** Same-repo recall cosine floor: top winner must be >= this to be injected. */
|
|
120
|
+
export const RECALL_MIN_COSINE = () => Number(process.env.MEGACOMPACT_RECALL_MIN_COSINE ?? "0.12");
|
|
121
|
+
// ---------------------------------------------------------------------------
|
|
113
122
|
// Vector-cortex flags + breaker constants (VC0A+). Positive sprint flags,
|
|
114
123
|
// default ON, `=0`/`_DISABLED` off. Re-exported from src/config/vector-cortex.ts
|
|
115
124
|
// so root consumers share one source of truth.
|
|
@@ -93,6 +93,7 @@ export const SETTINGS = [
|
|
|
93
93
|
num("MEGACOMPACT_L2_THRESHOLD", "L2 Cosine Threshold", "L2 semantic dedup firing point", 0.85, 0, 1),
|
|
94
94
|
num("MEGACOMPACT_L1_JACCARD", "L1 Jaccard Threshold", "L1 MinHash near-dup threshold", 0.8, 0, 1),
|
|
95
95
|
num("MEGACOMPACT_DEDUP_SIM", "Dedup Similarity", "Legacy content-similarity fallback", 0.9, 0, 1),
|
|
96
|
+
num("MEGACOMPACT_RECALL_MIN_COSINE", "Recall Min Cosine (same-repo)", "3WF-3 same-repo floor the 3-source validator applies to the top winner (cross-repo 0.90 stays separate)", 0.12, 0, 1),
|
|
96
97
|
num("MEGACOMPACT_MMR_LAMBDA", "MMR Lambda", "Maximal Marginal Relevance diversity", 0.5, 0, 1),
|
|
97
98
|
num("MEGACOMPACT_SEMDEDUP_COSINE", "SemDeDup Cosine", "Offline SemDeDup pair threshold", 0.95, 0, 1),
|
|
98
99
|
num("MEGACOMPACT_CONSOLIDATE_COSINE", "Consolidate Cosine", "Memory consolidation merge threshold", 0.7, 0, 1),
|
|
@@ -181,6 +181,12 @@ export function loadConfig() {
|
|
|
181
181
|
autoWikiEnabled: envBool("MEGACOMPACT_AUTO_WIKI", true),
|
|
182
182
|
crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
|
|
183
183
|
crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
|
|
184
|
+
// 3WF-3: SAME-repo recall cosine floor applied by the 3-source validator to
|
|
185
|
+
// the top winner. SEPARATE from crossRepoCosine (S17, default 0.90, stricter
|
|
186
|
+
// and cross-repo only). This same-repo floor is permissive by default (0.12)
|
|
187
|
+
// so recall still surfaces loosely-relevant within-repo context while
|
|
188
|
+
// rejecting effectively-unrelated hits. Mirrors src/config.ts RECALL_MIN_COSINE.
|
|
189
|
+
recallMinCosine: Number(process.env.MEGACOMPACT_RECALL_MIN_COSINE ?? "0.12"),
|
|
184
190
|
memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),
|
|
185
191
|
memoryReviewInterval: envFlag("MEGACOMPACT_MEMORY_REVIEW_INTERVAL", 10),
|
|
186
192
|
recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* context-handler/injectionConfirm.fixture.ts — shared fixtures for the 3WF-4
|
|
3
|
+
* InjectionConfirm tests.
|
|
4
|
+
*
|
|
5
|
+
* Split out so each test file stays under the extensions/300-soft-cap the way
|
|
6
|
+
* src/recall/recall3wf.fixture.ts does for 3WF-3. These are REAL fixtures, not
|
|
7
|
+
* mocks/stubs: a REAL VectorStore over a temp stateDir with REAL checkpoints
|
|
8
|
+
* persisted via compactSession; the MegaRuntime is a minimal typed stub exposing
|
|
9
|
+
* only the fields confirmInjection touches (store, pendingRecallBlock,
|
|
10
|
+
* pendingMemoryRecallBlock, appendEvent), matching the triggerGuard/thrashGuard
|
|
11
|
+
* test conventions.
|
|
12
|
+
*/
|
|
13
|
+
import { mkdtempSync } from "node:fs";
|
|
14
|
+
import { tmpdir } from "node:os";
|
|
15
|
+
import { join } from "node:path";
|
|
16
|
+
import { VectorStore } from "../../../src/vectorStore.js";
|
|
17
|
+
import { compactSession } from "../../../src/engine.js";
|
|
18
|
+
/** Real EngineMessage fixture. */
|
|
19
|
+
export function msg(role, text) {
|
|
20
|
+
return { role, text };
|
|
21
|
+
}
|
|
22
|
+
/** A user-role AgentMessage carrying `text` (the tail-block shape). */
|
|
23
|
+
export function userMsg(text) {
|
|
24
|
+
return { role: "user", content: text, timestamp: 1 };
|
|
25
|
+
}
|
|
26
|
+
/** Fresh isolated state dir per VectorStore. */
|
|
27
|
+
export function freshStore() {
|
|
28
|
+
const dir = mkdtempSync(join(tmpdir(), "mc-inject-"));
|
|
29
|
+
return { store: new VectorStore({ dedupSim: 0.9, stateDir: dir }), dir };
|
|
30
|
+
}
|
|
31
|
+
/** Persist N distinct checkpoints with ascending timestamps. */
|
|
32
|
+
export function seed(store, topics, sid = "sess_inject") {
|
|
33
|
+
topics.forEach((t, i) => {
|
|
34
|
+
compactSession({
|
|
35
|
+
sessionId: sid,
|
|
36
|
+
messages: [msg("user", t), msg("assistant", "ok")],
|
|
37
|
+
keepFrom: 2,
|
|
38
|
+
timestamp: i + 1,
|
|
39
|
+
}, store);
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
/** Minimal MegaRuntime stub exposing only the confirmInjection touch-points. */
|
|
43
|
+
export function runtimeStub(store, over = {}) {
|
|
44
|
+
const events = [];
|
|
45
|
+
const runtime = {
|
|
46
|
+
store,
|
|
47
|
+
pendingRecallBlock: over.pendingRecallBlock,
|
|
48
|
+
pendingMemoryRecallBlock: over.pendingMemoryRecallBlock,
|
|
49
|
+
perfTurnStart: undefined,
|
|
50
|
+
rt: { recallInjectedThisTurn: false },
|
|
51
|
+
appendEvent: (name, payload) => {
|
|
52
|
+
events.push({ name, payload });
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
return { runtime, events };
|
|
56
|
+
}
|
|
57
|
+
/** Config stub: only the flags confirmInjection reads. */
|
|
58
|
+
export function configStub(over = {}) {
|
|
59
|
+
return {
|
|
60
|
+
threeWayFailback: over.threeWayFailback ?? true,
|
|
61
|
+
recallTailInject: over.recallTailInject ?? true,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { vectorList } from "../../../src/vectorStore.js";
|
|
2
|
+
import { normalizeSessionId } from "../../../src/store.js";
|
|
3
|
+
import { buildFloorBlock } from "../../../src/failback/floor.js";
|
|
4
|
+
import { withRecallTail } from "../recall-tail.js";
|
|
5
|
+
import { messageContentText } from "./messageText.js";
|
|
6
|
+
/**
|
|
7
|
+
* The marker substring used to locate a staged block inside a message. The
|
|
8
|
+
* block's first non-empty line, capped, so prompt reshapes (cache striping /
|
|
9
|
+
* message separation) that regroup messages cannot defeat the match, while a
|
|
10
|
+
* genuinely dropped block still fails it.
|
|
11
|
+
*/
|
|
12
|
+
export function blockMarker(block) {
|
|
13
|
+
const line = block
|
|
14
|
+
.split("\n")
|
|
15
|
+
.map((l) => l.trim())
|
|
16
|
+
.find((l) => l.length > 0);
|
|
17
|
+
return (line ?? "").slice(0, 80);
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* PURE decision function: did the staged block land in this view, and if not,
|
|
21
|
+
* which rung should repair it? Takes the already-extracted message texts so it
|
|
22
|
+
* stays free of pi types and is directly unit-testable.
|
|
23
|
+
*/
|
|
24
|
+
export function decideInjection(input, messageTexts, hasPendingBlocks) {
|
|
25
|
+
const marker = input.staged ? blockMarker(input.staged) : "";
|
|
26
|
+
// Nothing staged => nothing to assert; NOT a miss. Injecting a floor here
|
|
27
|
+
// would push provenance text into sessions that never had recall to lose.
|
|
28
|
+
if (!marker)
|
|
29
|
+
return { landed: true, recovered: "none" };
|
|
30
|
+
const landed = messageTexts.some((t) => t.includes(marker));
|
|
31
|
+
if (landed)
|
|
32
|
+
return { landed: true, recovered: "none" };
|
|
33
|
+
// Absent: recompose when the runtime still holds pending blocks, else floor.
|
|
34
|
+
return { landed: false, recovered: hasPendingBlocks ? "recomposed" : "floor" };
|
|
35
|
+
}
|
|
36
|
+
/** Append `text` as a user-role tail message (same shape as recall-tail.ts). */
|
|
37
|
+
function withFloorTail(view, text) {
|
|
38
|
+
const tailMsg = {
|
|
39
|
+
role: "user",
|
|
40
|
+
content: text,
|
|
41
|
+
timestamp: Date.now(),
|
|
42
|
+
};
|
|
43
|
+
return { messages: [...view.messages, tailMsg] };
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Thin caller: verify (and if needed repair) one composed view. Returns the view
|
|
47
|
+
* to actually return from the handler. `sessionId` sources the floor checkpoints.
|
|
48
|
+
*
|
|
49
|
+
* The recompose rung deliberately re-appends via `withRecallTail` onto the
|
|
50
|
+
* ALREADY-COMPOSED view rather than re-running `buildTailResult`: the reshape
|
|
51
|
+
* stages (cache striping / message separation) are the realistic way a tail
|
|
52
|
+
* message gets regrouped away, and re-running the same composition would
|
|
53
|
+
* reproduce the same loss. Appending after the reshape is the actual repair, and
|
|
54
|
+
* it keeps the PREVENT-PI-001/002 tail-append invariant (a single user-role
|
|
55
|
+
* message after a complete prefix can never split a toolCall/toolResult pair).
|
|
56
|
+
*/
|
|
57
|
+
export function confirmInjection(runtime, config, view, sessionId) {
|
|
58
|
+
try {
|
|
59
|
+
if (!config.threeWayFailback)
|
|
60
|
+
return view;
|
|
61
|
+
// What the tail composition was supposed to inject. BOTH staged blocks
|
|
62
|
+
// count: withRecallTail joins recall + memory blocks into one tail
|
|
63
|
+
// message, so either one going missing is a real injection failure.
|
|
64
|
+
const staged = runtime.pendingRecallBlock ?? runtime.pendingMemoryRecallBlock ?? null;
|
|
65
|
+
// Can the recompose rung actually re-append? Only when a block is still
|
|
66
|
+
// staged on the runtime. When it is not (blocks consumed between
|
|
67
|
+
// composition and this check), or when withRecallTail declines to append,
|
|
68
|
+
// the ladder falls through to the floor rung below.
|
|
69
|
+
const hasPending = runtime.pendingRecallBlock != null ||
|
|
70
|
+
runtime.pendingMemoryRecallBlock != null;
|
|
71
|
+
// Legacy prepend mode: the block is not expected in the message list —
|
|
72
|
+
// verify our composed return value contains it instead (A3 degrade path).
|
|
73
|
+
if (!config.recallTailInject) {
|
|
74
|
+
const composed = view.messages.map(messageContentText).join("\n");
|
|
75
|
+
const marker = staged ? blockMarker(staged) : "";
|
|
76
|
+
runtime.appendEvent("injection_confirmed", {
|
|
77
|
+
mode: "prepend",
|
|
78
|
+
landed: marker ? composed.includes(marker) : true,
|
|
79
|
+
});
|
|
80
|
+
return view;
|
|
81
|
+
}
|
|
82
|
+
const verdict = decideInjection({ staged, tailMode: true }, view.messages.map(messageContentText), hasPending);
|
|
83
|
+
if (verdict.landed) {
|
|
84
|
+
runtime.appendEvent("injection_confirmed", { mode: "tail", landed: true });
|
|
85
|
+
return view;
|
|
86
|
+
}
|
|
87
|
+
if (verdict.recovered === "recomposed") {
|
|
88
|
+
const rebuilt = withRecallTail(view.messages, runtime, config);
|
|
89
|
+
// withRecallTail returns the input array unchanged on failure; only
|
|
90
|
+
// treat a genuine append as a recovery.
|
|
91
|
+
if (rebuilt.length > view.messages.length) {
|
|
92
|
+
runtime.appendEvent("injection_recovered", { via: "recomposed" });
|
|
93
|
+
return { messages: rebuilt };
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
const floor = buildFloorBlock(vectorList(runtime.store, normalizeSessionId(sessionId)));
|
|
97
|
+
runtime.appendEvent("injection_recovered", {
|
|
98
|
+
via: "floor",
|
|
99
|
+
basis: floor.basis,
|
|
100
|
+
});
|
|
101
|
+
return withFloorTail(view, floor.text);
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
// Non-fatal: return the unverified view (pre-sprint behavior).
|
|
105
|
+
return view;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
@@ -2,6 +2,7 @@ import { normalizeSessionId } from "../../../src/store.js";
|
|
|
2
2
|
import { recall } from "../../../src/engine.js";
|
|
3
3
|
import { formatRecallBlock } from "../../../src/recall.js";
|
|
4
4
|
import { vectorStats, vectorList } from "../../../src/vectorStore.js";
|
|
5
|
+
import { buildFloorBlock as sharedFloorBlock, FLOOR_UNAVAILABLE_TEXT, } from "../../../src/failback/floor.js";
|
|
5
6
|
import { recentUserQuery } from "../../mega-runtime.js";
|
|
6
7
|
/** One-shot completion marker per MegaRuntime (dies with the runtime). */
|
|
7
8
|
const guardDone = new WeakMap();
|
|
@@ -61,26 +62,19 @@ export function runTriggerGuard(runtime, config, ctx) {
|
|
|
61
62
|
/* never throws; best-effort guard */
|
|
62
63
|
}
|
|
63
64
|
}
|
|
64
|
-
/**
|
|
65
|
+
/**
|
|
66
|
+
* Build the provenance floor string from the session's newest checkpoint.
|
|
67
|
+
*
|
|
68
|
+
* 3WF-4: the text construction moved to the SHARED pure builder
|
|
69
|
+
* (src/failback/floor.ts) — this keeps the store read (`vectorList`, unfiltered,
|
|
70
|
+
* exactly as 3WF-1 shipped) and the string return type, so output is
|
|
71
|
+
* byte-identical to the pre-refactor version.
|
|
72
|
+
*/
|
|
65
73
|
function buildFloorBlock(runtime, sid) {
|
|
66
74
|
try {
|
|
67
|
-
|
|
68
|
-
let newest = cps[0];
|
|
69
|
-
for (const cp of cps) {
|
|
70
|
-
if (!newest || (cp.timestamp ?? 0) > (newest.timestamp ?? 0))
|
|
71
|
-
newest = cp;
|
|
72
|
-
}
|
|
73
|
-
const summary = newest?.summary?.trim();
|
|
74
|
-
if (summary) {
|
|
75
|
-
return ("The following compacted context is the most recent checkpoint from " +
|
|
76
|
-
"this session (recall found no query-relevant match):\n\n" + summary);
|
|
77
|
-
}
|
|
78
|
-
return ("This session has compacted context but recall could not surface a " +
|
|
79
|
-
"checkpoint relevant to the current request; the most recent checkpoint " +
|
|
80
|
-
"summary is unavailable.");
|
|
75
|
+
return sharedFloorBlock(vectorList(runtime.store, sid)).text;
|
|
81
76
|
}
|
|
82
77
|
catch {
|
|
83
|
-
return
|
|
84
|
-
"checkpoint relevant to the current request.");
|
|
78
|
+
return FLOOR_UNAVAILABLE_TEXT;
|
|
85
79
|
}
|
|
86
80
|
}
|
|
@@ -2,6 +2,7 @@ import { estimateSessionTokens } from "../../src/tokens.js";
|
|
|
2
2
|
import { piCompactWouldNoop } from "../mega-pipeline.js";
|
|
3
3
|
import { buildTailResult } from "./context-handler/tailResult.js";
|
|
4
4
|
import { runTriggerGuard } from "./context-handler/triggerGuard.js";
|
|
5
|
+
import { confirmInjection } from "./context-handler/injectionConfirm.js";
|
|
5
6
|
import { persistEpochAndMaintain } from "./context-handler/afterCompact.js";
|
|
6
7
|
import { appendMirrorAndLedger } from "./context-handler/dbMirrorAppend.js";
|
|
7
8
|
import { evaluateGate, thrashGuardBlocks } from "./context-handler/gateCheck.js";
|
|
@@ -43,7 +44,22 @@ export function registerContextHandler(pi, runtime, config) {
|
|
|
43
44
|
// tail message at any view-return point. Returns undefined when nothing
|
|
44
45
|
// is staged (or the flag is OFF) so the caller falls through to its
|
|
45
46
|
// normal return.
|
|
46
|
-
const
|
|
47
|
+
const composeTail = buildTailResult(runtime, config, messages);
|
|
48
|
+
// 3WF-4 InjectionConfirm: wrap the tail factory so EVERY return point of
|
|
49
|
+
// this handler (gate / replay / debounce / thrash-guard / pipeline /
|
|
50
|
+
// live-trim) is verified — the staged block's marker must be present in
|
|
51
|
+
// the message list pi will send (tail mode), else we re-compose from the
|
|
52
|
+
// runtime's pending blocks and finally fall back to the shared floor.
|
|
53
|
+
// A composition that yields nothing staged (undefined) is passed through
|
|
54
|
+
// untouched, so flag-OFF and no-recall paths are byte-identical.
|
|
55
|
+
const tailResult = config.threeWayFailback
|
|
56
|
+
? (msgs) => {
|
|
57
|
+
const view = composeTail(msgs);
|
|
58
|
+
if (!view)
|
|
59
|
+
return view;
|
|
60
|
+
return confirmInjection(runtime, config, view, ctx.sessionManager.getSessionId());
|
|
61
|
+
}
|
|
62
|
+
: composeTail;
|
|
47
63
|
// Always track context for the dashboard/widget, even when auto is off.
|
|
48
64
|
// (v0.8 regression: !config.auto gate sat above this, leaving ctx stats
|
|
49
65
|
// null -> widget '?% / ?/?' when auto disabled. Track first, THEN gate.)
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* recall/impl.ts — unified Layer-5 recall pipeline implementation (3WF-3 split).
|
|
3
|
+
*
|
|
4
|
+
* Behavior is UNCHANGED from the pre-split recall.ts. `doRecall` is the ONE path
|
|
5
|
+
* that injects (sync). `doRecallAsync` augments with optional cross-repo HNSW
|
|
6
|
+
* on resume / /mega-recall --cross-repo. Both mutate the shared MegaRuntime
|
|
7
|
+
* (token accounting, ticker, dashboard events). The shell recall.ts re-exports
|
|
8
|
+
* these names so `export * from "./mega-pipeline/recall.js"` stays byte-stable.
|
|
9
|
+
*/
|
|
10
|
+
import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
|
|
11
|
+
import { recallAndInline, recallAndInlineAsync, formatRecallBlock, } from "../../../src/recall.js";
|
|
12
|
+
import { normalizeSessionId } from "../../../src/store.js";
|
|
13
|
+
import { incRecallInjected, incCacheHitTokens, getIndexDir, } from "../../../src/store/sqlite.js";
|
|
14
|
+
import { ensureConversationIdFor, recordTurnWrite, recordRecallWrite, } from "../../mega-turn-store.js";
|
|
15
|
+
import { C } from "../../mega-runtime.js";
|
|
16
|
+
import { recordRecallLatency } from "../../mega-runtime/vc-observer.js";
|
|
17
|
+
/**
|
|
18
|
+
* Unified recall (Layer 5). The ONE path that injects. Returns the recall
|
|
19
|
+
* result; callers decide whether to stage it for before_agent_start (resume)
|
|
20
|
+
* or report it (command).
|
|
21
|
+
*/
|
|
22
|
+
export function doRecall(runtime, config, ctx, query, source) {
|
|
23
|
+
runtime.bindRepo(ctx.cwd);
|
|
24
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
25
|
+
// Live window text for inline dedupe (Fix C): drop recalled checkpoints that
|
|
26
|
+
// are already resident in the session, so recall never re-injects context the
|
|
27
|
+
// model can already see. Best-effort — an empty window just skips dedupe.
|
|
28
|
+
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
29
|
+
const recallStartMs = Date.now();
|
|
30
|
+
const result = recallAndInline({
|
|
31
|
+
sessionId: sid,
|
|
32
|
+
query,
|
|
33
|
+
limit: config.autoInlineK,
|
|
34
|
+
source,
|
|
35
|
+
skipInjected: true,
|
|
36
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
37
|
+
windowDedupe: config.windowDedupe,
|
|
38
|
+
liveWindow,
|
|
39
|
+
dedupSim: config.dedupSim,
|
|
40
|
+
}, runtime.store);
|
|
41
|
+
runtime.dashboard.event("recall", {
|
|
42
|
+
source,
|
|
43
|
+
query: query.slice(0, 120),
|
|
44
|
+
injected: result.toInject.length,
|
|
45
|
+
empty: result.empty,
|
|
46
|
+
});
|
|
47
|
+
if (config.ragRecallMetrics && result.hydeInfo) {
|
|
48
|
+
runtime.dashboard.event("hyde_executed", {
|
|
49
|
+
sessionId: sid,
|
|
50
|
+
ran: result.hydeInfo.ran,
|
|
51
|
+
skipped: result.hydeInfo.skipped,
|
|
52
|
+
reason: result.hydeInfo.reason,
|
|
53
|
+
hypotheticalDoc: result.hydeInfo.hypotheticalDoc.slice(0, 400),
|
|
54
|
+
generationMs: result.hydeInfo.generationMs,
|
|
55
|
+
rawHitCount: result.hydeInfo.rawHitCount,
|
|
56
|
+
hydeHitCount: result.hydeInfo.hydeHitCount,
|
|
57
|
+
fusedHitCount: result.hydeInfo.fusedHitCount,
|
|
58
|
+
lift: result.hydeInfo.lift,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
if (config.ragRecallMetrics && result.recallMetrics) {
|
|
62
|
+
runtime.dashboard.event("recall_metrics", {
|
|
63
|
+
sessionId: sid,
|
|
64
|
+
hitCount: result.recallMetrics.hitCount,
|
|
65
|
+
score: result.recallMetrics.score,
|
|
66
|
+
pass: result.recallMetrics.pass,
|
|
67
|
+
relevance: result.recallMetrics.relevance,
|
|
68
|
+
coverage: result.recallMetrics.coverage,
|
|
69
|
+
diversity: result.recallMetrics.diversity,
|
|
70
|
+
specificity: result.recallMetrics.specificity,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
if (!result.empty && result.toInject.length > 0) {
|
|
74
|
+
const top = result.toInject[0];
|
|
75
|
+
const scorePct = Math.round((top.score ?? 0) * 100);
|
|
76
|
+
const files = top.checkpoint.filesModified ?? [];
|
|
77
|
+
const label = files.length
|
|
78
|
+
? files
|
|
79
|
+
.map((f) => f.split("/").pop() ?? f)
|
|
80
|
+
.slice(0, 2)
|
|
81
|
+
.join(", ")
|
|
82
|
+
: top.checkpoint.checkpointId;
|
|
83
|
+
runtime.pushTicker(`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`);
|
|
84
|
+
runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
|
|
85
|
+
}
|
|
86
|
+
let sumTokens = 0;
|
|
87
|
+
for (const h of result.toInject)
|
|
88
|
+
sumTokens += h.checkpoint.tokenEstimate;
|
|
89
|
+
if (result.toInject.length > 0) {
|
|
90
|
+
runtime.rt.recallInjections += result.toInject.length;
|
|
91
|
+
runtime.rt.cacheHitTokens += sumTokens;
|
|
92
|
+
incRecallInjected(result.toInject.length, runtime.currentStateDir);
|
|
93
|
+
incCacheHitTokens(sumTokens, runtime.currentStateDir);
|
|
94
|
+
}
|
|
95
|
+
// S43: record recall provenance — which checkpoints/summaries served this
|
|
96
|
+
// turn, their score + source path. Linked to the turn row written at
|
|
97
|
+
// turn_end via the conversation+turnIndex. Best-effort + non-fatal.
|
|
98
|
+
// Persists telemetry (HyDE + recall metrics) even when recall returned
|
|
99
|
+
// no hits, so empty-recall HyDE invocations are still visible in the
|
|
100
|
+
// dashboard Turns/Metrics tabs.
|
|
101
|
+
const hasTelemetry = result.hydeInfo != null || result.recallMetrics != null;
|
|
102
|
+
if (result.toInject.length > 0 || hasTelemetry) {
|
|
103
|
+
try {
|
|
104
|
+
const convId = ensureConversationIdFor(config, sid, runtime.currentStateDir);
|
|
105
|
+
const turnId = recordTurnWrite(config, {
|
|
106
|
+
conversationId: convId,
|
|
107
|
+
sessionId: sid,
|
|
108
|
+
turnIndex: runtime.currentTurn,
|
|
109
|
+
role: "assistant",
|
|
110
|
+
startedAt: Date.now(),
|
|
111
|
+
hyde: result.hydeInfo ?? undefined,
|
|
112
|
+
recallMetrics: result.recallMetrics ?? undefined,
|
|
113
|
+
}, runtime.currentStateDir);
|
|
114
|
+
if (result.toInject.length > 0) {
|
|
115
|
+
recordRecallWrite(config, turnId, result.toInject.map((h) => ({
|
|
116
|
+
checkpointId: h.checkpoint.checkpointId,
|
|
117
|
+
score: h.score,
|
|
118
|
+
source: h.raptorLevel !== undefined
|
|
119
|
+
? "raptor"
|
|
120
|
+
: h.repoId
|
|
121
|
+
? "cross-repo"
|
|
122
|
+
: "flat",
|
|
123
|
+
raptorLevel: h.raptorLevel,
|
|
124
|
+
})), runtime.currentStateDir);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
/* non-fatal: recall provenance never breaks the recall path */
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
// VC0A: record recall latency on the eval observer (mode A) so the dashboard
|
|
132
|
+
// histogram reflects real data. No-op when the observer is absent (flag off /
|
|
133
|
+
// construction failure).
|
|
134
|
+
try {
|
|
135
|
+
recordRecallLatency(runtime, Date.now() - recallStartMs, sid, 0);
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
/* non-fatal: latency recording never breaks recall */
|
|
139
|
+
}
|
|
140
|
+
return result;
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* S17: async recall with optional cross-repo augmentation. Used on resume
|
|
144
|
+
* (session_start) and /mega-recall --cross-repo — NEVER from the mid-turn
|
|
145
|
+
* context handler (that stays sync). Runs the sync same-repo scan first; if it
|
|
146
|
+
* returns < config.autoInlineK hits AND crossRepo is enabled, awaits the PGlite
|
|
147
|
+
* HNSW cross-repo path and merges (source-labeled, deduped by checkpointId). The
|
|
148
|
+
* recallMaxTokens cap + windowDedupe apply to the merged set so cross-repo can
|
|
149
|
+
* never net-inflate the window. Cross-repo uses a stricter cosine floor
|
|
150
|
+
* (config.crossRepoCosine) than same-repo. Non-fatal: any async failure returns
|
|
151
|
+
* the same-repo result unchanged.
|
|
152
|
+
*/
|
|
153
|
+
export async function doRecallAsync(runtime, config, ctx, query, source, opts = {}) {
|
|
154
|
+
runtime.bindRepo(ctx.cwd);
|
|
155
|
+
const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
|
|
156
|
+
const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
|
|
157
|
+
// Sync same-repo first (fast, never blocks).
|
|
158
|
+
const sameRepo = recallAndInline({
|
|
159
|
+
sessionId: sid,
|
|
160
|
+
query,
|
|
161
|
+
limit: config.autoInlineK,
|
|
162
|
+
source,
|
|
163
|
+
skipInjected: true,
|
|
164
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
165
|
+
windowDedupe: config.windowDedupe,
|
|
166
|
+
liveWindow,
|
|
167
|
+
dedupSim: config.dedupSim,
|
|
168
|
+
}, runtime.store);
|
|
169
|
+
if (!config.crossRepoEnabled || !opts.crossRepo)
|
|
170
|
+
return sameRepo;
|
|
171
|
+
if (sameRepo.toInject.length >= config.autoInlineK)
|
|
172
|
+
return sameRepo; // same-repo satisfied
|
|
173
|
+
// Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
|
|
174
|
+
try {
|
|
175
|
+
const x = await recallAndInlineAsync({
|
|
176
|
+
sessionId: sid,
|
|
177
|
+
query,
|
|
178
|
+
limit: config.autoInlineK,
|
|
179
|
+
source,
|
|
180
|
+
skipInjected: true,
|
|
181
|
+
recallMaxTokens: config.recallMaxTokens,
|
|
182
|
+
windowDedupe: config.windowDedupe,
|
|
183
|
+
liveWindow,
|
|
184
|
+
dedupSim: config.crossRepoCosine,
|
|
185
|
+
crossRepo: true,
|
|
186
|
+
// F2: resolve the machine-wide index dir via the shared resolver so the
|
|
187
|
+
// cross-repo injected-set dedup works even when MEGACOMPACT_INDEX_DIR is
|
|
188
|
+
// unset. The env var still wins when set (getIndexDir checks it first);
|
|
189
|
+
// the default (~/.mega-compact-index) is the same DB mega-commands and the
|
|
190
|
+
// dashboard read, so injection counts stay consistent. Without this, a
|
|
191
|
+
// bare `process.env` read returns undefined → cross-repo hits re-inject in
|
|
192
|
+
// every new session (the global injected-set is never consulted).
|
|
193
|
+
globalIndexDir: getIndexDir(),
|
|
194
|
+
}, runtime.store);
|
|
195
|
+
runtime.dashboard.event("recall-crossrepo", {
|
|
196
|
+
source,
|
|
197
|
+
query: query.slice(0, 120),
|
|
198
|
+
injected: x.toInject.length,
|
|
199
|
+
sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
|
|
200
|
+
});
|
|
201
|
+
// Merge, dedup by checkpointId, respect the same token cap by reformatting.
|
|
202
|
+
const seen = new Set(sameRepo.toInject.map((h) => h.checkpoint.checkpointId));
|
|
203
|
+
const merged = [...sameRepo.toInject];
|
|
204
|
+
for (const h of x.toInject) {
|
|
205
|
+
if (!seen.has(h.checkpoint.checkpointId)) {
|
|
206
|
+
merged.push(h);
|
|
207
|
+
seen.add(h.checkpoint.checkpointId);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
const block = merged.length ? formatRecallBlock(merged) : "";
|
|
211
|
+
if (merged.length > 0) {
|
|
212
|
+
let sumTokens = 0;
|
|
213
|
+
for (const h of merged)
|
|
214
|
+
sumTokens += h.checkpoint.tokenEstimate;
|
|
215
|
+
runtime.rt.recallInjections += merged.length;
|
|
216
|
+
runtime.rt.cacheHitTokens += sumTokens;
|
|
217
|
+
incRecallInjected(merged.length, runtime.currentStateDir);
|
|
218
|
+
incCacheHitTokens(sumTokens, runtime.currentStateDir);
|
|
219
|
+
}
|
|
220
|
+
return {
|
|
221
|
+
toInject: merged,
|
|
222
|
+
report: merged.map((h) => ` • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`),
|
|
223
|
+
block,
|
|
224
|
+
empty: merged.length === 0,
|
|
225
|
+
// H1: merged cross-repo result reuses the same-repo pass's telemetry.
|
|
226
|
+
hydeInfo: sameRepo.hydeInfo,
|
|
227
|
+
recallMetrics: sameRepo.recallMetrics,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
return sameRepo; // cross-repo failure → same-repo only (non-fatal)
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Extract the live-window message texts from the session manager (Fix C),
|
|
236
|
+
* for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
|
|
237
|
+
* error so recall falls back to unbounded (still correct, just no dedupe).
|
|
238
|
+
* Mirrors recentUserQuery's use of sessionEntryToContextMessages.
|
|
239
|
+
*/
|
|
240
|
+
export function extractLiveWindow(ctx) {
|
|
241
|
+
try {
|
|
242
|
+
const entries = ctx.sessionManager.getEntries();
|
|
243
|
+
const texts = [];
|
|
244
|
+
for (const e of entries) {
|
|
245
|
+
for (const m of sessionEntryToContextMessages(e)) {
|
|
246
|
+
const c = m.content;
|
|
247
|
+
if (typeof c === "string")
|
|
248
|
+
texts.push(c);
|
|
249
|
+
else if (Array.isArray(c))
|
|
250
|
+
texts.push(c.map((b) => b.text ?? "").join(" "));
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
return texts;
|
|
254
|
+
}
|
|
255
|
+
catch {
|
|
256
|
+
return [];
|
|
257
|
+
}
|
|
258
|
+
}
|