pi-mega-compact 0.20.86 → 0.20.87

package/src/recall/readonly.ts

@@ -0,0 +1,57 @@
+/**
+ * recall/readonly.ts — read-only recall variant (3WF-3 Source A).
+ *
+ * A pure search+rank seam wrapping `engine.recall`'s RAW `hits` path. It is the
+ * canonical read-only entry point going forward (triggerGuard.ts still inlines
+ * `recall(...).hits` for its own need; this module is additive and does NOT
+ * refactor it).
+ *
+ * HARD contract (QA): this module MUST NOT call `vectorMarkInjected`, must NOT
+ * write any turn/recall rows, and must NOT emit S43 telemetry. It only searches
+ * and returns hits for the vote. RecallAndInline's inject loop is the ONLY place
+ * the injected-set is mutated; keying the vote on raw `hits` (skipInjected:false
+ * => hits === newHits) is deliberate — `newHits` is post-`skipInjected` filter,
+ * which would distort overlap appearance.
+ *
+ * Non-fatal: any failure returns [] so the caller degrades to other sources.
+ * Pi-agnostic: no pi runtime imports.
+ */
+import { recall } from "../engine.js";
+import type { VectorStore } from "../vectorStore.js";
+import type { SearchHit } from "../vectorStore.js";
+
+/** Options for the read-only recall seam. */
+export interface ReadonlyRecallOptions {
+	/** Normalized session id. */
+	sessionId: string;
+	/** Recall query text. */
+	query: string;
+	/** Max hits to return (default 3). */
+	limit?: number;
+}
+
+/**
+ * Raw, read-only recall hits for the 3-source vote. Returns `engine.recall`'s
+ * RAW `.hits` (skipInjected:false => equals the unfiltered vector result). No
+ * injected-set mutation, no turn writes, no telemetry. Returns [] on failure.
+ */
+export function recallRawHits(
+	opts: ReadonlyRecallOptions,
+	store: VectorStore,
+): SearchHit[] {
+	try {
+		const result = recall(
+			{
+				sessionId: opts.sessionId,
+				query: opts.query,
+				limit: opts.limit ?? 3,
+				skipInjected: false,
+			},
+			store,
+		);
+		return result.hits;
+	} catch {
+		// Non-fatal: never break the agent loop. Degrade to other sources.
+		return [];
+	}
+}

package/src/recall/recall3wf.fixture.ts

@@ -0,0 +1,87 @@
+/**
+ * src/recall/recall3wf.fixture.ts — shared fixtures for the 3WF-3 recall tests.
+ *
+ * Split out of recall3wf.test.ts (which crossed the src 300 soft cap) so each
+ * test file stays under the limit. These are REAL fixtures, not mocks/stubs:
+ * a REAL VectorStore over a temp stateDir, REAL checkpoints persisted via
+ * compactSession, and readers that go through the SAME working path the
+ * extension uses (recallRawHits -> vectorSearch -> listCheckpoints, and
+ * vectorWasInjected), mirroring the proven triggerGuard test pattern.
+ */
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { VectorStore } from "../vectorStore.js";
+import { compactSession } from "../engine.js";
+import { recallAndInline } from "../recall.js";
+import { recallRawHits } from "./readonly.js";
+import { openStore } from "../store/sqlite/utils.js";
+import { initSchema } from "../store/sqlite/schema.js";
+
+/** Real EngineMessage fixture. */
+export function msg(role: "user" | "assistant", text: string): any {
+	return { role, text };
+}
+
+/** Fresh isolated state dir per VectorStore. */
+export function freshStore(): { store: VectorStore; dir: string } {
+	const dir = mkdtempSync(join(tmpdir(), "mc-3wf-"));
+	return { store: new VectorStore({ dedupSim: 0.9, stateDir: dir }), dir };
+}
+
+/** Persist N distinct checkpoints with distinct content + ascending timestamps. */
+export function seed(store: VectorStore, topics: string[], sid = "sess_3wf"): void {
+	topics.forEach((t, i) => {
+		compactSession(
+			{
+				sessionId: sid,
+				messages: [msg("user", t), msg("assistant", "ok")],
+				keepFrom: 2,
+				timestamp: i + 1,
+			},
+			store,
+		);
+	});
+}
+
+/** Checkpoint ids via the real search path (vectorSearch -> listCheckpoints). */
+export function checkpointIds(store: VectorStore, sid: string, query: string): string[] {
+	return recallRawHits({ sessionId: sid, query, limit: 10 }, store).map(
+		(h) => h.checkpoint.checkpointId,
+	);
+}
+
+/** Run the real recallAndInline path with skipInjected:false so nothing is
+ *  marked and the block reflects the search result exactly (deterministic). */
+export function recallAndInlineCapture(
+	sid: string,
+	query: string,
+	store: VectorStore,
+): { block: string; empty: boolean; toInject: unknown[] } {
+	const r = recallAndInline(
+		{ sessionId: sid, query, limit: 3, source: "command", skipInjected: false, windowDedupe: false },
+		store,
+	);
+	return { block: r.block, empty: r.empty, toInject: r.toInject };
+}
+
+/** Count recall-provenance rows (turn_recall) for a session via raw SQL reader. */
+export function countTurnRecallRows(store: VectorStore, sid: string): number {
+	try {
+		const reader = openStore(store.stateDir);
+		// Ensure the turns/turn_recall tables exist so a 0-count is meaningful
+		// (a write on the new path would be visible, not masked by a missing table).
+		initSchema(reader);
+		const row = reader
+			.prepare(
+				`SELECT COUNT(*) AS n FROM turn_recall tr
+				 JOIN turns t ON t.id = tr.turn_id
+				 WHERE t.session_id = ?`,
+			)
+			.get(sid) as { n: number };
+		return row?.n ?? 0;
+	} catch {
+		return 0;
+	}
+}

package/src/recall/validator.ts

@@ -0,0 +1,150 @@
+/**
+ * recall/validator.ts — independent candidate validator (3WF-3).
+ *
+ * Judges candidates handed to it; it MUST NOT call any search itself. Given the
+ * ranked vote winners + the live-window text (already extracted by the caller,
+ * since src/ cannot import pi types), it walks the winners in order and returns
+ * the first that passes BOTH gates:
+ *
+ *   1. Cosine floor: the winner's score >= the same-repo floor (default 0.12,
+ *      env MEGACOMPACT_RECALL_MIN_COSINE). The cross-repo 0.90 floor
+ *      (config.crossRepoCosine) is SEPARATE and intentionally untouched.
+ *   2. Not already resident in the live window: reuse recall/sync.ts's exact
+ *      comparison — embed each live message, embed the checkpoint summary, and
+ *      treat the checkpoint as resident when cosineSimilarity >= dedupSim. We
+ *      reuse that metric rather than inventing a new one.
+ *
+ * On a failing candidate it advances to the next-ranked winner. If ALL fail it
+ * returns the provenance floor (FloorBlock built from the newest checkpoint —
+ * pure over checkpoints, same semantics as triggerGuard's buildFloorBlock).
+ *
+ * Non-fatal throughout: any error degrades to the next candidate / the floor.
+ * Pi-agnostic: no pi runtime imports.
+ */
+import { defaultEmbedder, cosineSimilarity } from "../embedder.js";
+// SQLite store, NOT src/store.ts's legacy gzipped-JSON DR reader (that returns
+// [] for live sessions). Mirrors vector-search.ts / tieredRouter.ts.
+import { listCheckpoints } from "../store/sqlite.js";
+import { RECALL_MIN_COSINE } from "../config.js";
+import type { VectorStore } from "../vectorStore.js";
+import type { RecallCandidate, FloorBlock } from "../failback/types.js";
+
+/** Options for the recall validator. */
+export interface ValidateOptions {
+	/** Normalized session id (for floor-block construction). */
+	sessionId: string;
+	/**
+	 * The recall query. Required for a TRUE cosine gate: only `source:"vector"`
+	 * candidates carry a cosine in `score` (fts5 carries BM25, recency carries a
+	 * freshness rank), so comparing a raw mixed-scale score against a cosine
+	 * floor would be meaningless. When supplied, the validator re-derives each
+	 * candidate's cosine against the query locally (embedder only — never a
+	 * search call, so the "independent of all three search calls" contract
+	 * holds). When omitted, only `vector` candidates can clear the gate.
+	 */
+	query?: string;
+	/** Live-window message texts already extracted by the caller. */
+	liveWindow?: string[];
+	/** Dedup similarity threshold for the live-window resident check. */
+	dedupSim?: number;
+}
+
+/** A validated winner, or the provenance floor when all candidates fail. */
+export type ValidationOutcome =
+	| { kind: "candidate"; candidate: RecallCandidate }
+	| { kind: "floor"; floor: FloorBlock };
+
+/** Build the provenance floor block from the session's newest checkpoint. */
+function buildFloorBlock(sessionId: string, store: VectorStore): FloorBlock {
+	try {
+		const cps = listCheckpoints(sessionId, store.stateDir).filter(
+			(c) => c.dedupStatus !== "removed",
+		);
+		let newest = cps[0];
+		for (const cp of cps) {
+			if (!newest || (cp.timestamp ?? 0) > (newest.timestamp ?? 0)) newest = cp;
+		}
+		const summary = newest?.summary?.trim();
+		if (summary) {
+			return {
+				text:
+					"The following compacted context is the most recent checkpoint from " +
+					"this session (recall found no query-relevant match):\n\n" + summary,
+				basis: "lastCheckpoint",
+			};
+		}
+		return {
+			text:
+				"This session has compacted context but recall could not surface a " +
+				"checkpoint relevant to the current request; the most recent checkpoint " +
+				"summary is unavailable.",
+			basis: "lastCheckpoint",
+		};
+	} catch {
+		return {
+			text:
+				"This session has compacted context but recall could not surface a " +
+				"checkpoint relevant to the current request.",
+			basis: "none",
+		};
+	}
+}
+
+/**
+ * Validate the ranked vote winners, returning the first that passes both gates,
+ * or the provenance floor if none do. Does NOT mutate the injected set, does NOT
+ * write turns, does NOT emit telemetry. Non-fatal.
+ */
+export function validateRecall(
+	winners: RecallCandidate[],
+	opts: ValidateOptions,
+	store: VectorStore,
+): ValidationOutcome {
+	const floor = RECALL_MIN_COSINE();
+	const dedupSim = opts.dedupSim ?? 0.9;
+	const embedder = defaultEmbedder();
+	const liveVecs = (opts.liveWindow ?? []).map((m) => embedder.embed(m));
+	// One checkpoint read for the whole pass (both gates share it).
+	const cps = listCheckpoints(opts.sessionId, store.stateDir);
+	const cpById = new Map(cps.map((c) => [c.checkpointId, c]));
+	const queryVec = opts.query ? embedder.embed(opts.query) : null;
+
+	for (const cand of winners) {
+		try {
+			const cp = cpById.get(cand.checkpointId);
+
+			// Gate 1: same-repo COSINE floor. `cand.score` is only a cosine for
+			// source "vector"; fts5 (BM25) and recency (freshness rank) live on
+			// other scales, so for those we re-derive the true cosine locally from
+			// the query + checkpoint embedding. No search call is made.
+			let cosine: number;
+			if (cand.source === "vector") {
+				cosine = cand.score;
+			} else if (queryVec && cp) {
+				cosine = cosineSimilarity(queryVec, embedder.embed(cp.summary));
+			} else {
+				// No comparable cosine available => cannot clear a cosine gate.
+				continue;
+			}
+			if (cosine < floor) continue;
+
+			// Gate 2: not already resident in the live window.
+			if (liveVecs.length > 0) {
+				if (!cp) continue; // cannot verify => skip rather than risk re-inject
+				const hitVec = embedder.embed(cp.summary);
+				const resident = liveVecs.some(
+					(v) => cosineSimilarity(v, hitVec) >= dedupSim,
+				);
+				if (resident) continue;
+			}
+
+			return { kind: "candidate", candidate: cand };
+		} catch {
+			// Non-fatal: skip this candidate, try the next.
+			continue;
+		}
+	}
+
+	// All candidates rejected -> provenance floor.
+	return { kind: "floor", floor: buildFloorBlock(opts.sessionId, store) };
+}

package/src/recall/vote.ts

@@ -0,0 +1,240 @@
+/**
+ * recall/vote.ts — the 3-independent-source recall vote (3WF-3).
+ *
+ * Three INDEPENDENT, read-only sources name candidate checkpoints:
+ *   A vector  — raw semantic hits (recall/readonly.ts), cosine 0..1 scale.
+ *   B fts5    — BM25 trigram hits (hydrated to checkpointIds), FTS5 BM25 scale
+ *               (negative score = better match; ranking order is what matters).
+ *   C recency — the N freshest checkpoints by timestamp, query-INDEPENDENT.
+ *               (NOT turn_recall / TurnReader — that would just echo already-
+ *               injected content, not an independent signal.)
+ *
+ * Each source is a different score scale, so they are NOT directly comparable.
+ * We normalize each source's scores to a 0..1 scale (per-source min-max) BEFORE
+ * combining. Averaging raw cosine (0..1) with raw BM25 (arbitrary negative
+ * magnitude) or a recency rank would be meaningless — the largest-magnitude
+ * scale would always dominate. Normalization makes each source a peer voter.
+ *
+ * Overlap rule: a checkpoint named by >=2 of 3 distinct sources short-circuits
+ * as a winner. Fallback (no 2/3 majority): rank all candidates by the
+ * cross-source MEAN of their normalized scores.
+ *
+ * Non-fatal throughout. Pi-agnostic: no pi runtime imports.
+ */
+import { openStore } from "../store/sqlite/utils.js";
+import { fts5SearchScoped, hydrateFts5Hits } from "../store/sqlite/fts5-search.js";
+// The SQLite store is the source of truth (src/store.ts's same-named helper
+// reads the LEGACY gzipped-JSON DR snapshot, which is empty for live sessions —
+// importing it here would silently starve sources B and C). Mirrors the import
+// in vector-search.ts + tieredRouter.ts.
+import { listCheckpoints } from "../store/sqlite.js";
+import { computeContentDigest } from "../dedup/digest.js";
+import { recallRawHits } from "./readonly.js";
+import { Logger } from "../log.js";
+import type { VectorStore, SearchHit } from "../vectorStore.js";
+import type { RecallCandidate, VoteResult } from "../failback/types.js";
+
+/** Options for the three-source recall vote. */
+export interface VoteOptions {
+	/** Normalized session id. */
+	sessionId: string;
+	/** Recall query text. */
+	query: string;
+	/** Max vector/fts5 hits to consider (default 3). */
+	limit?: number;
+	/** How many freshest checkpoints source C contributes (default = limit). */
+	recencyCount?: number;
+}
+
+/** Per-source normalization: map raw scores to 0..1 via min-max within source. */
+function normalizeScores(scores: number[]): Map<number, number> {
+	const map = new Map<number, number>();
+	if (scores.length === 0) return map;
+	const min = Math.min(...scores);
+	const max = Math.max(...scores);
+	const span = max - min;
+	scores.forEach((s, i) => {
+		map.set(i, span === 0 ? 1 : (s - min) / span);
+	});
+	return map;
+}
+
+/**
+ * Run the three-source recall vote. Returns agreement winners + a per-id vote
+ * count + the names of sources that produced no winning candidate. Non-fatal:
+ * any search failure degrades to the remaining sources (empty winners allowed).
+ */
+export function voteRecall(opts: VoteOptions, store: VectorStore): VoteResult {
+	const logger = new Logger();
+	const limit = opts.limit ?? 3;
+	const recencyCount = opts.recencyCount ?? limit;
+
+	// ── Source A: vector (raw hits, cosine 0..1). ────────────────────────────
+	const vectorCands: RecallCandidate[] = recallRawHits(
+		{ sessionId: opts.sessionId, query: opts.query, limit },
+		store,
+	).map((h: SearchHit) => ({
+		checkpointId: h.checkpoint.checkpointId,
+		score: h.score,
+		source: "vector" as const,
+	}));
+
+	// ── Source B: fts5 (BM25, hydrated to checkpointIds). ─────────────────────
+	// Dedup on L0 content digest so the SAME normalized text under two ids does
+	// not double-vote — collapse to one candidate per digest.
+	const fts5Cands: RecallCandidate[] = (() => {
+		try {
+			const reader = openStore(store.stateDir);
+			const hits = fts5SearchScoped(opts.query, reader, opts.sessionId, limit);
+			// FTS5 returns scores ordered best-first (bm25 asc); lower = better.
+			// We keep the raw score (negative-is-better) and flip in normalization.
+			const hydrated = hydrateFts5Hits(hits, opts.sessionId, store.stateDir);
+			// Dedup on checkpointId so the same checkpoint cannot double-count, AND
+			// on the L0 CONTENT digest so identical normalized text stored under two
+			// different ids collapses to one vote. The digest is taken over the
+			// joined `summary` (real content): hashing the id string would be a
+			// no-op tier, since ids are unique by definition.
+			const seenDigest = new Set<string>();
+			const seenId = new Set<string>();
+			const out: RecallCandidate[] = [];
+			for (const h of hydrated) {
+				if (seenId.has(h.checkpointId)) continue;
+				seenId.add(h.checkpointId);
+				const digest = computeContentDigest(h.summary).contentHash;
+				if (seenDigest.has(digest)) continue;
+				seenDigest.add(digest);
+				out.push({ checkpointId: h.checkpointId, score: h.score, source: "fts5" });
+			}
+			return out;
+		} catch {
+			return [];
+		}
+	})();
+
+	// ── Source C: recency (N freshest checkpoints, query-independent). ────────
+	// Timestamp-ordered; the lower the index the fresher. Score = recency rank
+	// (fresh = high) so normalization treats newest as best.
+	const recencyCands: RecallCandidate[] = (() => {
+		try {
+			const cps = listCheckpoints(opts.sessionId, store.stateDir)
+				.filter((c) => c.dedupStatus !== "removed")
+				.sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
+				.slice(0, recencyCount);
+			return cps.map((cp, i) => ({
+				checkpointId: cp.checkpointId,
+				// Fresher => higher raw score (recency rank). Normalized below.
+				score: cps.length - i,
+				source: "recency" as const,
+			}));
+		} catch {
+			return [];
+		}
+	})();
+
+	const sources: { name: string; cands: RecallCandidate[] }[] = [
+		{ name: "vector", cands: vectorCands },
+		{ name: "fts5", cands: fts5Cands },
+		{ name: "recency", cands: recencyCands },
+	];
+
+	// Per-source normalization to 0..1 so the three scales are comparable.
+	const perSource = sources.map((s) => ({
+		name: s.name,
+		norm: normalizeScores(s.cands.map((c) => c.score)),
+	}));
+
+	// Aggregate: best normalized score per source per checkpointId + vote count.
+	const bestScoreBySource = new Map<string, Map<string, number>>();
+	const seenIds = new Set<string>();
+	for (const src of sources) {
+		const norm = perSource.find((p) => p.name === src.name)!.norm;
+		const bestByCp = new Map<string, number>();
+		src.cands.forEach((c, i) => {
+			const n = norm.get(i) ?? 0;
+			const prev = bestByCp.get(c.checkpointId);
+			if (prev === undefined || n > prev) bestByCp.set(c.checkpointId, n);
+			seenIds.add(c.checkpointId);
+		});
+		bestScoreBySource.set(src.name, bestByCp);
+	}
+
+	// Vote count = number of DISTINCT sources naming each checkpointId.
+	const votes: Record<string, number> = {};
+	const sumByCp = new Map<string, number>();
+	for (const id of seenIds) {
+		let count = 0;
+		let sum = 0;
+		for (const src of sources) {
+			const m = bestScoreBySource.get(src.name)!;
+			if (m.has(id)) {
+				count++;
+				sum += m.get(id)!;
+			}
+		}
+		votes[id] = count;
+		sumByCp.set(id, sum);
+	}
+
+	/** Mean normalized score across the sources that named `id`. */
+	const meanScore = (id: string): number =>
+		(sumByCp.get(id) ?? 0) / (votes[id] ?? 1);
+
+	// Short-circuit: >=2 of 3 distinct sources => winner (agreement). Ranked by
+	// vote count first (stronger agreement wins), then by mean normalized score —
+	// the validator consumes this list in order and takes the first that passes,
+	// so the ordering IS the ranking and must not be Set-insertion order.
+	const winners: RecallCandidate[] = [];
+	const divergent = new Set<string>(sources.map((s) => s.name));
+	const agreed = [...seenIds]
+		.filter((id) => (votes[id] ?? 0) >= 2)
+		.sort((a, b) => (votes[b] ?? 0) - (votes[a] ?? 0) || meanScore(b) - meanScore(a));
+	for (const id of agreed) {
+		const cand = (() => {
+			for (const src of sources) {
+				const c = src.cands.find((x) => x.checkpointId === id);
+				if (c) return c;
+			}
+			return null;
+		})();
+		if (cand) winners.push(cand);
+	}
+
+	if (winners.length === 0) {
+		// Fallback (no 2-of-3 agreement): rank by cross-source MEAN normalized score.
+		const ranked = [...seenIds].sort((a, b) => meanScore(b) - meanScore(a));
+		for (const id of ranked) {
+			const cand = (() => {
+				for (const src of sources) {
+					const c = src.cands.find((x) => x.checkpointId === id);
+					if (c) return c;
+				}
+				return null;
+			})();
+			if (cand) winners.push(cand);
+		}
+	}
+
+	// Divergence = a source that named NONE of the winning checkpoints. This must
+	// be computed per SOURCE against the winning ID SET, not from `winner.source`
+	// (a winner is one candidate object carrying a single source label, so an id
+	// agreed on by all three sources would still credit only one of them).
+	const winningIds = new Set(winners.map((w) => w.checkpointId));
+	for (const src of sources) {
+		if (src.cands.some((c) => winningIds.has(c.checkpointId))) {
+			divergent.delete(src.name);
+		}
+	}
+
+	if (divergent.size > 0) {
+		logger.info("recall_vote_divergence", {
+			divergentSources: [...divergent],
+			winnerCount: winners.length,
+		});
+	}
+
+	return {
+		winners,
+		votes,
+		divergentSources: [...divergent],
+	};
+}

package/src/store/sqlite/fts5-search.ts

@@ -13,6 +13,7 @@
  */
 
 import type { DatabaseSync } from "node:sqlite";
+import { listCheckpoints } from "./checkpoints.js";
 
 export interface Fts5Hit {
 	/** checkpoint id (chkpt_001 etc.) from the context_chunks_trgm row. */
@@ -88,3 +89,42 @@ export function fts5SearchScoped(
 	// No session filter — plain FTS5 search across all sessions.
 	return fts5Search(query, reader, limit);
 }
+
+/** A hydrated FTS5 hit: the id + BM25 score joined with its checkpoint summary. */
+export interface HydratedFts5Hit {
+	/** checkpoint id the FTS5 row referred to. */
+	checkpointId: string;
+	/** BM25 relevance score carried over from the FTS5 row. */
+	score: number;
+	/** The joined checkpoint's summary text (content basis for L0 digest dedup). */
+	summary: string;
+}
+
+/**
+ * Hydrate FTS5 hits (which only carry checkpoint `id` + BM25 `score`, with NO
+ * checkpoint object) by joining against the real stored checkpoints. Mirrors the
+ * private `hydrateHits` in tieredRouter.ts (listCheckpoints → Map on
+ * checkpointId → join, dropping ids with no checkpoint).
+ *
+ * The joined `summary` is returned alongside the id because the 3WF-3 voter
+ * dedups candidates on the L0 CONTENT digest — hashing the id string instead
+ * would be a no-op tier (ids are already unique), silently disabling the check.
+ *
+ * @param hits      Raw Fts5Hit rows (id + BM25 score).
+ * @param sessionId Session scope used to fetch the checkpoint set.
+ * @param stateDir  On-disk state dir for the sync store.
+ */
+export function hydrateFts5Hits(
+	hits: Fts5Hit[],
+	sessionId: string,
+	stateDir: string,
+): HydratedFts5Hit[] {
+	const cps = listCheckpoints(sessionId, stateDir);
+	const cpMap = new Map(cps.map((cp) => [cp.checkpointId, cp]));
+	const out: HydratedFts5Hit[] = [];
+	for (const h of hits) {
+		const cp = cpMap.get(h.id);
+		if (cp) out.push({ checkpointId: h.id, score: h.score, summary: cp.summary });
+	}
+	return out;
+}