pi-mega-compact 0.20.86 → 0.20.88

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/config.js +9 -0
  2. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +1 -0
  3. package/dist/extensions/mega-config.js +6 -0
  4. package/dist/extensions/mega-events/context-handler/injectionConfirm.fixture.js +63 -0
  5. package/dist/extensions/mega-events/context-handler/injectionConfirm.js +107 -0
  6. package/dist/extensions/mega-events/context-handler/triggerGuard.js +11 -17
  7. package/dist/extensions/mega-events/context-handler.js +17 -1
  8. package/dist/extensions/mega-pipeline/recall/impl.js +258 -0
  9. package/dist/extensions/mega-pipeline/recall.js +6 -253
  10. package/dist/src/config.js +9 -0
  11. package/dist/src/failback/floor.js +35 -0
  12. package/dist/src/recall/readonly.js +39 -0
  13. package/dist/src/recall/recall3wf.fixture.js +67 -0
  14. package/dist/src/recall/validator.js +99 -0
  15. package/dist/src/recall/vote.js +217 -0
  16. package/dist/src/store/sqlite/fts5-search.js +26 -0
  17. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +1 -0
  18. package/extensions/mega-config-types.ts +5 -0
  19. package/extensions/mega-config.ts +6 -0
  20. package/extensions/mega-events/context-handler/injectionConfirm.fixture.ts +90 -0
  21. package/extensions/mega-events/context-handler/injectionConfirm.ts +168 -0
  22. package/extensions/mega-events/context-handler/triggerGuard.ts +14 -22
  23. package/extensions/mega-events/context-handler.ts +16 -1
  24. package/extensions/mega-pipeline/recall/impl.ts +312 -0
  25. package/extensions/mega-pipeline/recall.ts +10 -306
  26. package/package.json +1 -1
  27. package/src/config.ts +12 -0
  28. package/src/failback/floor.ts +71 -0
  29. package/src/failback/types.ts +44 -0
  30. package/src/recall/readonly.ts +57 -0
  31. package/src/recall/recall3wf.fixture.ts +87 -0
  32. package/src/recall/validator.ts +137 -0
  33. package/src/recall/vote.ts +240 -0
  34. package/src/store/sqlite/fts5-search.ts +40 -0
@@ -0,0 +1,71 @@
1
+ /**
2
+ * src/failback/floor.ts — the SHARED pure provenance-floor builder (3WF-4).
3
+ *
4
+ * Consolidation refactor (ZERO behavior change). Before this module the same
5
+ * floor text was built twice:
6
+ * (a) `extensions/mega-events/context-handler/triggerGuard.ts` (3WF-1) —
7
+ * read checkpoints via `vectorList` (unfiltered), returned a bare string;
8
+ * (b) `src/recall/validator.ts` (3WF-3) — read checkpoints via
9
+ * `listCheckpoints` filtered to `dedupStatus !== "removed"`, returned a
10
+ * `FloorBlock`.
11
+ * The three text variants were byte-identical between the two; only the
12
+ * checkpoint READ differed. So this module takes the already-read checkpoint
13
+ * list from the caller (staying pure — no store, no pi types, no I/O) and each
14
+ * call site keeps its own read semantics. Output is byte-identical to both.
15
+ *
16
+ * 3WF-4's InjectionConfirm is the third consumer: when neither the message list
17
+ * nor the runtime's pending blocks yield a block, it needs the SAME last-resort
18
+ * floor text rather than a fourth copy.
19
+ *
20
+ * Non-fatal by construction: every branch returns a `FloorBlock`; the `none`
21
+ * basis carries the shortest text (used when the checkpoint read itself threw).
22
+ */
23
+ import type { StoredCheckpoint } from "../store.js";
24
+ import type { FloorBlock } from "./types.js";
25
+
26
+ /** Floor text when the newest checkpoint summary is available (prefix). */
27
+ const WITH_SUMMARY_PREFIX =
28
+ "The following compacted context is the most recent checkpoint from " +
29
+ "this session (recall found no query-relevant match):\n\n";
30
+
31
+ /** Floor text when checkpoints exist but no usable summary does. */
32
+ const NO_SUMMARY_TEXT =
33
+ "This session has compacted context but recall could not surface a " +
34
+ "checkpoint relevant to the current request; the most recent checkpoint " +
35
+ "summary is unavailable.";
36
+
37
+ /** Floor text when the checkpoint read itself failed (hard last resort). */
38
+ export const FLOOR_UNAVAILABLE_TEXT =
39
+ "This session has compacted context but recall could not surface a " +
40
+ "checkpoint relevant to the current request.";
41
+
42
+ /** The newest checkpoint by timestamp (first element wins ties, as before). */
43
+ export function newestCheckpoint(
44
+ cps: readonly StoredCheckpoint[],
45
+ ): StoredCheckpoint | undefined {
46
+ let newest = cps[0];
47
+ for (const cp of cps) {
48
+ if (!newest || (cp.timestamp ?? 0) > (newest.timestamp ?? 0)) newest = cp;
49
+ }
50
+ return newest;
51
+ }
52
+
53
+ /**
54
+ * Build the provenance floor from an already-read checkpoint list. Pure: the
55
+ * caller owns the read (and any dedup-status filtering), so both legacy call
56
+ * sites keep byte-identical output.
57
+ */
58
+ export function buildFloorBlock(
59
+ cps: readonly StoredCheckpoint[],
60
+ ): FloorBlock {
61
+ const summary = newestCheckpoint(cps)?.summary?.trim();
62
+ if (summary) {
63
+ return { text: WITH_SUMMARY_PREFIX + summary, basis: "lastCheckpoint" };
64
+ }
65
+ return { text: NO_SUMMARY_TEXT, basis: "lastCheckpoint" };
66
+ }
67
+
68
+ /** The hard last-resort floor (checkpoint read unavailable or threw). */
69
+ export function unavailableFloorBlock(): FloorBlock {
70
+ return { text: FLOOR_UNAVAILABLE_TEXT, basis: "none" };
71
+ }
@@ -88,3 +88,47 @@ export interface ThrashGuardState {
88
88
  /** ms epoch at which the guard was armed. */
89
89
  armedAt: number;
90
90
  }
91
+
92
+ /**
93
+ * A single recall candidate surfaced by one of the three independent, read-only
94
+ * recall sources in the 3WF-3 vote. `score` is the raw per-source score; the
95
+ * voter normalizes each source to a comparable 0..1 before combining.
96
+ */
97
+ export interface RecallCandidate {
98
+ /** Checkpoint id named by the source. */
99
+ checkpointId: string;
100
+ /** Raw per-source relevance score (scale depends on `source`). */
101
+ score: number;
102
+ /** Which independent source named this candidate. */
103
+ source: "vector" | "fts5" | "recency";
104
+ }
105
+
106
+ /**
107
+ * Verdict of an InjectionConfirm pass (3WF-4). There is NO prompt readback API
108
+ * in pi, so the only verifiable proxy for what the provider will receive is the
109
+ * pre-LLM message list (`ContextEvent.messages`, transformed). `landed` is true
110
+ * when the staged block's text was found there (tail mode) or in the composed
111
+ * return string (legacy prepend mode). `recovered` records which repair rung
112
+ * this event used: none (already landed), recomposed (rebuilt from the runtime's
113
+ * pending blocks), or floor (last-resort provenance text).
114
+ */
115
+ export interface InjectionVerdict {
116
+ /** True when the staged block text is present in the verified view. */
117
+ landed: boolean;
118
+ /** Which repair rung ran to make the block present. */
119
+ recovered: "none" | "recomposed" | "floor";
120
+ }
121
+
122
+ /**
123
+ * Outcome of the three-source recall vote. `winners` are the agreed candidates
124
+ * (ranked), `votes` counts how many distinct sources named each checkpointId,
125
+ * and `divergentSources` lists the sources that contributed no winner.
126
+ */
127
+ export interface VoteResult {
128
+ /** Agreed candidates, ranked best-first. */
129
+ winners: RecallCandidate[];
130
+ /** Per-checkpointId vote count (1..3 distinct sources). */
131
+ votes: Record<string, number>;
132
+ /** Source names that produced no winning candidate. */
133
+ divergentSources: string[];
134
+ }
@@ -0,0 +1,57 @@
1
+ /**
2
+ * recall/readonly.ts — read-only recall variant (3WF-3 Source A).
3
+ *
4
+ * A pure search+rank seam wrapping `engine.recall`'s RAW `hits` path. It is the
5
+ * canonical read-only entry point going forward (triggerGuard.ts still inlines
6
+ * `recall(...).hits` for its own need; this module is additive and does NOT
7
+ * refactor it).
8
+ *
9
+ * HARD contract (QA): this module MUST NOT call `vectorMarkInjected`, must NOT
10
+ * write any turn/recall rows, and must NOT emit S43 telemetry. It only searches
11
+ * and returns hits for the vote. RecallAndInline's inject loop is the ONLY place
12
+ * the injected-set is mutated; keying the vote on raw `hits` (skipInjected:false
13
+ * => hits === newHits) is deliberate — `newHits` is post-`skipInjected` filter,
14
+ * which would distort overlap appearance.
15
+ *
16
+ * Non-fatal: any failure returns [] so the caller degrades to other sources.
17
+ * Pi-agnostic: no pi runtime imports.
18
+ */
19
+ import { recall } from "../engine.js";
20
+ import type { VectorStore } from "../vectorStore.js";
21
+ import type { SearchHit } from "../vectorStore.js";
22
+
23
+ /** Options for the read-only recall seam. */
24
+ export interface ReadonlyRecallOptions {
25
+ /** Normalized session id. */
26
+ sessionId: string;
27
+ /** Recall query text. */
28
+ query: string;
29
+ /** Max hits to return (default 3). */
30
+ limit?: number;
31
+ }
32
+
33
+ /**
34
+ * Raw, read-only recall hits for the 3-source vote. Returns `engine.recall`'s
35
+ * RAW `.hits` (skipInjected:false => equals the unfiltered vector result). No
36
+ * injected-set mutation, no turn writes, no telemetry. Returns [] on failure.
37
+ */
38
+ export function recallRawHits(
39
+ opts: ReadonlyRecallOptions,
40
+ store: VectorStore,
41
+ ): SearchHit[] {
42
+ try {
43
+ const result = recall(
44
+ {
45
+ sessionId: opts.sessionId,
46
+ query: opts.query,
47
+ limit: opts.limit ?? 3,
48
+ skipInjected: false,
49
+ },
50
+ store,
51
+ );
52
+ return result.hits;
53
+ } catch {
54
+ // Non-fatal: never break the agent loop. Degrade to other sources.
55
+ return [];
56
+ }
57
+ }
@@ -0,0 +1,87 @@
1
+ /**
2
+ * src/recall/recall3wf.fixture.ts — shared fixtures for the 3WF-3 recall tests.
3
+ *
4
+ * Split out of recall3wf.test.ts (which crossed the src 300 soft cap) so each
5
+ * test file stays under the limit. These are REAL fixtures, not mocks/stubs:
6
+ * a REAL VectorStore over a temp stateDir, REAL checkpoints persisted via
7
+ * compactSession, and readers that go through the SAME working path the
8
+ * extension uses (recallRawHits -> vectorSearch -> listCheckpoints, and
9
+ * vectorWasInjected), mirroring the proven triggerGuard test pattern.
10
+ */
11
+ import { mkdtempSync } from "node:fs";
12
+ import { tmpdir } from "node:os";
13
+ import { join } from "node:path";
14
+
15
+ import { VectorStore } from "../vectorStore.js";
16
+ import { compactSession } from "../engine.js";
17
+ import { recallAndInline } from "../recall.js";
18
+ import { recallRawHits } from "./readonly.js";
19
+ import { openStore } from "../store/sqlite/utils.js";
20
+ import { initSchema } from "../store/sqlite/schema.js";
21
+
22
+ /** Real EngineMessage fixture. */
23
+ export function msg(role: "user" | "assistant", text: string): any {
24
+ return { role, text };
25
+ }
26
+
27
+ /** Fresh isolated state dir per VectorStore. */
28
+ export function freshStore(): { store: VectorStore; dir: string } {
29
+ const dir = mkdtempSync(join(tmpdir(), "mc-3wf-"));
30
+ return { store: new VectorStore({ dedupSim: 0.9, stateDir: dir }), dir };
31
+ }
32
+
33
+ /** Persist N distinct checkpoints with distinct content + ascending timestamps. */
34
+ export function seed(store: VectorStore, topics: string[], sid = "sess_3wf"): void {
35
+ topics.forEach((t, i) => {
36
+ compactSession(
37
+ {
38
+ sessionId: sid,
39
+ messages: [msg("user", t), msg("assistant", "ok")],
40
+ keepFrom: 2,
41
+ timestamp: i + 1,
42
+ },
43
+ store,
44
+ );
45
+ });
46
+ }
47
+
48
+ /** Checkpoint ids via the real search path (vectorSearch -> listCheckpoints). */
49
+ export function checkpointIds(store: VectorStore, sid: string, query: string): string[] {
50
+ return recallRawHits({ sessionId: sid, query, limit: 10 }, store).map(
51
+ (h) => h.checkpoint.checkpointId,
52
+ );
53
+ }
54
+
55
+ /** Run the real recallAndInline path with skipInjected:false so nothing is
56
+ * marked and the block reflects the search result exactly (deterministic). */
57
+ export function recallAndInlineCapture(
58
+ sid: string,
59
+ query: string,
60
+ store: VectorStore,
61
+ ): { block: string; empty: boolean; toInject: unknown[] } {
62
+ const r = recallAndInline(
63
+ { sessionId: sid, query, limit: 3, source: "command", skipInjected: false, windowDedupe: false },
64
+ store,
65
+ );
66
+ return { block: r.block, empty: r.empty, toInject: r.toInject };
67
+ }
68
+
69
+ /** Count recall-provenance rows (turn_recall) for a session via raw SQL reader. */
70
+ export function countTurnRecallRows(store: VectorStore, sid: string): number {
71
+ try {
72
+ const reader = openStore(store.stateDir);
73
+ // Ensure the turns/turn_recall tables exist so a 0-count is meaningful
74
+ // (a write on the new path would be visible, not masked by a missing table).
75
+ initSchema(reader);
76
+ const row = reader
77
+ .prepare(
78
+ `SELECT COUNT(*) AS n FROM turn_recall tr
79
+ JOIN turns t ON t.id = tr.turn_id
80
+ WHERE t.session_id = ?`,
81
+ )
82
+ .get(sid) as { n: number };
83
+ return row?.n ?? 0;
84
+ } catch {
85
+ return 0;
86
+ }
87
+ }
@@ -0,0 +1,137 @@
1
+ /**
2
+ * recall/validator.ts — independent candidate validator (3WF-3).
3
+ *
4
+ * Judges candidates handed to it; it MUST NOT call any search itself. Given the
5
+ * ranked vote winners + the live-window text (already extracted by the caller,
6
+ * since src/ cannot import pi types), it walks the winners in order and returns
7
+ * the first that passes BOTH gates:
8
+ *
9
+ * 1. Cosine floor: the winner's score >= the same-repo floor (default 0.12,
10
+ * env MEGACOMPACT_RECALL_MIN_COSINE). The cross-repo 0.90 floor
11
+ * (config.crossRepoCosine) is SEPARATE and intentionally untouched.
12
+ * 2. Not already resident in the live window: reuse recall/sync.ts's exact
13
+ * comparison — embed each live message, embed the checkpoint summary, and
14
+ * treat the checkpoint as resident when cosineSimilarity >= dedupSim. We
15
+ * reuse that metric rather than inventing a new one.
16
+ *
17
+ * On a failing candidate it advances to the next-ranked winner. If ALL fail it
18
+ * returns the provenance floor (FloorBlock built from the newest checkpoint —
19
+ * pure over checkpoints, same semantics as triggerGuard's buildFloorBlock).
20
+ *
21
+ * Non-fatal throughout: any error degrades to the next candidate / the floor.
22
+ * Pi-agnostic: no pi runtime imports.
23
+ */
24
+ import { defaultEmbedder, cosineSimilarity } from "../embedder.js";
25
+ // SQLite store, NOT src/store.ts's legacy gzipped-JSON DR reader (that returns
26
+ // [] for live sessions). Mirrors vector-search.ts / tieredRouter.ts.
27
+ import { listCheckpoints } from "../store/sqlite.js";
28
+ import { RECALL_MIN_COSINE } from "../config.js";
29
+ import type { VectorStore } from "../vectorStore.js";
30
+ import type { RecallCandidate, FloorBlock } from "../failback/types.js";
31
+ import {
32
+ buildFloorBlock as sharedFloorBlock,
33
+ unavailableFloorBlock,
34
+ } from "../failback/floor.js";
35
+
36
+ /** Options for the recall validator. */
37
+ export interface ValidateOptions {
38
+ /** Normalized session id (for floor-block construction). */
39
+ sessionId: string;
40
+ /**
41
+ * The recall query. Required for a TRUE cosine gate: only `source:"vector"`
42
+ * candidates carry a cosine in `score` (fts5 carries BM25, recency carries a
43
+ * freshness rank), so comparing a raw mixed-scale score against a cosine
44
+ * floor would be meaningless. When supplied, the validator re-derives each
45
+ * candidate's cosine against the query locally (embedder only — never a
46
+ * search call, so the "independent of all three search calls" contract
47
+ * holds). When omitted, only `vector` candidates can clear the gate.
48
+ */
49
+ query?: string;
50
+ /** Live-window message texts already extracted by the caller. */
51
+ liveWindow?: string[];
52
+ /** Dedup similarity threshold for the live-window resident check. */
53
+ dedupSim?: number;
54
+ }
55
+
56
+ /** A validated winner, or the provenance floor when all candidates fail. */
57
+ export type ValidationOutcome =
58
+ | { kind: "candidate"; candidate: RecallCandidate }
59
+ | { kind: "floor"; floor: FloorBlock };
60
+
61
+ /**
62
+ * Build the provenance floor block from the session's newest checkpoint.
63
+ *
64
+ * 3WF-4: the text construction moved to the SHARED pure builder
65
+ * (src/failback/floor.ts). This wrapper keeps THIS call site's read semantics —
66
+ * `listCheckpoints` filtered to `dedupStatus !== "removed"` — so the output is
67
+ * byte-identical to the pre-refactor 3WF-3 version.
68
+ */
69
+ function buildFloorBlock(sessionId: string, store: VectorStore): FloorBlock {
70
+ try {
71
+ const cps = listCheckpoints(sessionId, store.stateDir).filter(
72
+ (c) => c.dedupStatus !== "removed",
73
+ );
74
+ return sharedFloorBlock(cps);
75
+ } catch {
76
+ return unavailableFloorBlock();
77
+ }
78
+ }
79
+
80
+ /**
81
+ * Validate the ranked vote winners, returning the first that passes both gates,
82
+ * or the provenance floor if none do. Does NOT mutate the injected set, does NOT
83
+ * write turns, does NOT emit telemetry. Non-fatal.
84
+ */
85
+ export function validateRecall(
86
+ winners: RecallCandidate[],
87
+ opts: ValidateOptions,
88
+ store: VectorStore,
89
+ ): ValidationOutcome {
90
+ const floor = RECALL_MIN_COSINE();
91
+ const dedupSim = opts.dedupSim ?? 0.9;
92
+ const embedder = defaultEmbedder();
93
+ const liveVecs = (opts.liveWindow ?? []).map((m) => embedder.embed(m));
94
+ // One checkpoint read for the whole pass (both gates share it).
95
+ const cps = listCheckpoints(opts.sessionId, store.stateDir);
96
+ const cpById = new Map(cps.map((c) => [c.checkpointId, c]));
97
+ const queryVec = opts.query ? embedder.embed(opts.query) : null;
98
+
99
+ for (const cand of winners) {
100
+ try {
101
+ const cp = cpById.get(cand.checkpointId);
102
+
103
+ // Gate 1: same-repo COSINE floor. `cand.score` is only a cosine for
104
+ // source "vector"; fts5 (BM25) and recency (freshness rank) live on
105
+ // other scales, so for those we re-derive the true cosine locally from
106
+ // the query + checkpoint embedding. No search call is made.
107
+ let cosine: number;
108
+ if (cand.source === "vector") {
109
+ cosine = cand.score;
110
+ } else if (queryVec && cp) {
111
+ cosine = cosineSimilarity(queryVec, embedder.embed(cp.summary));
112
+ } else {
113
+ // No comparable cosine available => cannot clear a cosine gate.
114
+ continue;
115
+ }
116
+ if (cosine < floor) continue;
117
+
118
+ // Gate 2: not already resident in the live window.
119
+ if (liveVecs.length > 0) {
120
+ if (!cp) continue; // cannot verify => skip rather than risk re-inject
121
+ const hitVec = embedder.embed(cp.summary);
122
+ const resident = liveVecs.some(
123
+ (v) => cosineSimilarity(v, hitVec) >= dedupSim,
124
+ );
125
+ if (resident) continue;
126
+ }
127
+
128
+ return { kind: "candidate", candidate: cand };
129
+ } catch {
130
+ // Non-fatal: skip this candidate, try the next.
131
+ continue;
132
+ }
133
+ }
134
+
135
+ // All candidates rejected -> provenance floor.
136
+ return { kind: "floor", floor: buildFloorBlock(opts.sessionId, store) };
137
+ }
@@ -0,0 +1,240 @@
1
+ /**
2
+ * recall/vote.ts — the 3-independent-source recall vote (3WF-3).
3
+ *
4
+ * Three INDEPENDENT, read-only sources name candidate checkpoints:
5
+ * A vector — raw semantic hits (recall/readonly.ts), cosine 0..1 scale.
6
+ * B fts5 — BM25 trigram hits (hydrated to checkpointIds), FTS5 BM25 scale
7
+ * (negative score = better match; ranking order is what matters).
8
+ * C recency — the N freshest checkpoints by timestamp, query-INDEPENDENT.
9
+ * (NOT turn_recall / TurnReader — that would just echo already-
10
+ * injected content, not an independent signal.)
11
+ *
12
+ * Each source is a different score scale, so they are NOT directly comparable.
13
+ * We normalize each source's scores to a 0..1 scale (per-source min-max) BEFORE
14
+ * combining. Averaging raw cosine (0..1) with raw BM25 (arbitrary negative
15
+ * magnitude) or a recency rank would be meaningless — the largest-magnitude
16
+ * scale would always dominate. Normalization makes each source a peer voter.
17
+ *
18
+ * Overlap rule: a checkpoint named by >=2 of 3 distinct sources short-circuits
19
+ * as a winner. Fallback (no 2/3 majority): rank all candidates by the
20
+ * cross-source MEAN of their normalized scores.
21
+ *
22
+ * Non-fatal throughout. Pi-agnostic: no pi runtime imports.
23
+ */
24
+ import { openStore } from "../store/sqlite/utils.js";
25
+ import { fts5SearchScoped, hydrateFts5Hits } from "../store/sqlite/fts5-search.js";
26
+ // The SQLite store is the source of truth (src/store.ts's same-named helper
27
+ // reads the LEGACY gzipped-JSON DR snapshot, which is empty for live sessions —
28
+ // importing it here would silently starve sources B and C). Mirrors the import
29
+ // in vector-search.ts + tieredRouter.ts.
30
+ import { listCheckpoints } from "../store/sqlite.js";
31
+ import { computeContentDigest } from "../dedup/digest.js";
32
+ import { recallRawHits } from "./readonly.js";
33
+ import { Logger } from "../log.js";
34
+ import type { VectorStore, SearchHit } from "../vectorStore.js";
35
+ import type { RecallCandidate, VoteResult } from "../failback/types.js";
36
+
37
+ /** Options for the three-source recall vote. */
38
+ export interface VoteOptions {
39
+ /** Normalized session id. */
40
+ sessionId: string;
41
+ /** Recall query text. */
42
+ query: string;
43
+ /** Max vector/fts5 hits to consider (default 3). */
44
+ limit?: number;
45
+ /** How many freshest checkpoints source C contributes (default = limit). */
46
+ recencyCount?: number;
47
+ }
48
+
49
+ /** Per-source normalization: map raw scores to 0..1 via min-max within source. */
50
+ function normalizeScores(scores: number[]): Map<number, number> {
51
+ const map = new Map<number, number>();
52
+ if (scores.length === 0) return map;
53
+ const min = Math.min(...scores);
54
+ const max = Math.max(...scores);
55
+ const span = max - min;
56
+ scores.forEach((s, i) => {
57
+ map.set(i, span === 0 ? 1 : (s - min) / span);
58
+ });
59
+ return map;
60
+ }
61
+
62
+ /**
63
+ * Run the three-source recall vote. Returns agreement winners + a per-id vote
64
+ * count + the names of sources that produced no winning candidate. Non-fatal:
65
+ * any search failure degrades to the remaining sources (empty winners allowed).
66
+ */
67
+ export function voteRecall(opts: VoteOptions, store: VectorStore): VoteResult {
68
+ const logger = new Logger();
69
+ const limit = opts.limit ?? 3;
70
+ const recencyCount = opts.recencyCount ?? limit;
71
+
72
+ // ── Source A: vector (raw hits, cosine 0..1). ────────────────────────────
73
+ const vectorCands: RecallCandidate[] = recallRawHits(
74
+ { sessionId: opts.sessionId, query: opts.query, limit },
75
+ store,
76
+ ).map((h: SearchHit) => ({
77
+ checkpointId: h.checkpoint.checkpointId,
78
+ score: h.score,
79
+ source: "vector" as const,
80
+ }));
81
+
82
+ // ── Source B: fts5 (BM25, hydrated to checkpointIds). ─────────────────────
83
+ // Dedup on L0 content digest so the SAME normalized text under two ids does
84
+ // not double-vote — collapse to one candidate per digest.
85
+ const fts5Cands: RecallCandidate[] = (() => {
86
+ try {
87
+ const reader = openStore(store.stateDir);
88
+ const hits = fts5SearchScoped(opts.query, reader, opts.sessionId, limit);
89
+ // FTS5 returns scores ordered best-first (bm25 asc); lower = better.
90
+ // We keep the raw score (negative-is-better) and flip in normalization.
91
+ const hydrated = hydrateFts5Hits(hits, opts.sessionId, store.stateDir);
92
+ // Dedup on checkpointId so the same checkpoint cannot double-count, AND
93
+ // on the L0 CONTENT digest so identical normalized text stored under two
94
+ // different ids collapses to one vote. The digest is taken over the
95
+ // joined `summary` (real content): hashing the id string would be a
96
+ // no-op tier, since ids are unique by definition.
97
+ const seenDigest = new Set<string>();
98
+ const seenId = new Set<string>();
99
+ const out: RecallCandidate[] = [];
100
+ for (const h of hydrated) {
101
+ if (seenId.has(h.checkpointId)) continue;
102
+ seenId.add(h.checkpointId);
103
+ const digest = computeContentDigest(h.summary).contentHash;
104
+ if (seenDigest.has(digest)) continue;
105
+ seenDigest.add(digest);
106
+ out.push({ checkpointId: h.checkpointId, score: h.score, source: "fts5" });
107
+ }
108
+ return out;
109
+ } catch {
110
+ return [];
111
+ }
112
+ })();
113
+
114
+ // ── Source C: recency (N freshest checkpoints, query-independent). ────────
115
+ // Timestamp-ordered; the lower the index the fresher. Score = recency rank
116
+ // (fresh = high) so normalization treats newest as best.
117
+ const recencyCands: RecallCandidate[] = (() => {
118
+ try {
119
+ const cps = listCheckpoints(opts.sessionId, store.stateDir)
120
+ .filter((c) => c.dedupStatus !== "removed")
121
+ .sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
122
+ .slice(0, recencyCount);
123
+ return cps.map((cp, i) => ({
124
+ checkpointId: cp.checkpointId,
125
+ // Fresher => higher raw score (recency rank). Normalized below.
126
+ score: cps.length - i,
127
+ source: "recency" as const,
128
+ }));
129
+ } catch {
130
+ return [];
131
+ }
132
+ })();
133
+
134
+ const sources: { name: string; cands: RecallCandidate[] }[] = [
135
+ { name: "vector", cands: vectorCands },
136
+ { name: "fts5", cands: fts5Cands },
137
+ { name: "recency", cands: recencyCands },
138
+ ];
139
+
140
+ // Per-source normalization to 0..1 so the three scales are comparable.
141
+ const perSource = sources.map((s) => ({
142
+ name: s.name,
143
+ norm: normalizeScores(s.cands.map((c) => c.score)),
144
+ }));
145
+
146
+ // Aggregate: best normalized score per source per checkpointId + vote count.
147
+ const bestScoreBySource = new Map<string, Map<string, number>>();
148
+ const seenIds = new Set<string>();
149
+ for (const src of sources) {
150
+ const norm = perSource.find((p) => p.name === src.name)!.norm;
151
+ const bestByCp = new Map<string, number>();
152
+ src.cands.forEach((c, i) => {
153
+ const n = norm.get(i) ?? 0;
154
+ const prev = bestByCp.get(c.checkpointId);
155
+ if (prev === undefined || n > prev) bestByCp.set(c.checkpointId, n);
156
+ seenIds.add(c.checkpointId);
157
+ });
158
+ bestScoreBySource.set(src.name, bestByCp);
159
+ }
160
+
161
+ // Vote count = number of DISTINCT sources naming each checkpointId.
162
+ const votes: Record<string, number> = {};
163
+ const sumByCp = new Map<string, number>();
164
+ for (const id of seenIds) {
165
+ let count = 0;
166
+ let sum = 0;
167
+ for (const src of sources) {
168
+ const m = bestScoreBySource.get(src.name)!;
169
+ if (m.has(id)) {
170
+ count++;
171
+ sum += m.get(id)!;
172
+ }
173
+ }
174
+ votes[id] = count;
175
+ sumByCp.set(id, sum);
176
+ }
177
+
178
+ /** Mean normalized score across the sources that named `id`. */
179
+ const meanScore = (id: string): number =>
180
+ (sumByCp.get(id) ?? 0) / (votes[id] ?? 1);
181
+
182
+ // Short-circuit: >=2 of 3 distinct sources => winner (agreement). Ranked by
183
+ // vote count first (stronger agreement wins), then by mean normalized score —
184
+ // the validator consumes this list in order and takes the first that passes,
185
+ // so the ordering IS the ranking and must not be Set-insertion order.
186
+ const winners: RecallCandidate[] = [];
187
+ const divergent = new Set<string>(sources.map((s) => s.name));
188
+ const agreed = [...seenIds]
189
+ .filter((id) => (votes[id] ?? 0) >= 2)
190
+ .sort((a, b) => (votes[b] ?? 0) - (votes[a] ?? 0) || meanScore(b) - meanScore(a));
191
+ for (const id of agreed) {
192
+ const cand = (() => {
193
+ for (const src of sources) {
194
+ const c = src.cands.find((x) => x.checkpointId === id);
195
+ if (c) return c;
196
+ }
197
+ return null;
198
+ })();
199
+ if (cand) winners.push(cand);
200
+ }
201
+
202
+ if (winners.length === 0) {
203
+ // Fallback (no 2-of-3 agreement): rank by cross-source MEAN normalized score.
204
+ const ranked = [...seenIds].sort((a, b) => meanScore(b) - meanScore(a));
205
+ for (const id of ranked) {
206
+ const cand = (() => {
207
+ for (const src of sources) {
208
+ const c = src.cands.find((x) => x.checkpointId === id);
209
+ if (c) return c;
210
+ }
211
+ return null;
212
+ })();
213
+ if (cand) winners.push(cand);
214
+ }
215
+ }
216
+
217
+ // Divergence = a source that named NONE of the winning checkpoints. This must
218
+ // be computed per SOURCE against the winning ID SET, not from `winner.source`
219
+ // (a winner is one candidate object carrying a single source label, so an id
220
+ // agreed on by all three sources would still credit only one of them).
221
+ const winningIds = new Set(winners.map((w) => w.checkpointId));
222
+ for (const src of sources) {
223
+ if (src.cands.some((c) => winningIds.has(c.checkpointId))) {
224
+ divergent.delete(src.name);
225
+ }
226
+ }
227
+
228
+ if (divergent.size > 0) {
229
+ logger.info("recall_vote_divergence", {
230
+ divergentSources: [...divergent],
231
+ winnerCount: winners.length,
232
+ });
233
+ }
234
+
235
+ return {
236
+ winners,
237
+ votes,
238
+ divergentSources: [...divergent],
239
+ };
240
+ }