pi-context-engine 0.1.4 → 0.2.0

package/src/retrieval/reranker.ts

@@ -0,0 +1,112 @@
+/**
+ * R10: opt-in local cross-encoder reranker (plan: docs/IMPLEMENTATION-PLAN.md).
+ *
+ * Why opt-in and OFF by default: reranking evidence is split — a core gain in
+ * CodeRAG-style pipelines, but measured harmful for SE tasks when it costs
+ * ~800 ms per query. Only this package's own eval can decide, and the model
+ * download (~80 MB) must never happen implicitly in a package that markets
+ * "one npm command".
+ *
+ * The reranker reorders the fused candidate list (capped); path penalty, file
+ * diversity and the marginal stop are applied AFTER it, unchanged. If the
+ * model cannot load, retrieval silently falls back to the fused order — a
+ * missing optional download degrades, never breaks.
+ *
+ * Implementation notes (PR #7 review): we do NOT go through the
+ * text-classification pipeline. Two reasons:
+ *
+ * 1. Its input is a plain string — there is no `{text, text_pair}` shape, so
+ *    pair scoring via the pipeline throws and permanently disables reranking.
+ * 2. Cross-encoder rerankers (ms-marco-MiniLM) emit ONE logit; the pipeline
+ *    softmaxes it, so every document scores exactly 1.0 and no reordering
+ *    can happen. Sentence-transformers' CrossEncoder applies a sigmoid to
+ *    the raw logit instead — we do the same, on raw model logits.
+ */
+
+import { transformersCacheDir } from "../indexer/embedder.js";
+
+/** Minimal interface so unit tests can inject a stub (no model download). */
+export interface RerankerLike {
+	rerank(query: string, docs: string[]): Promise<number[]>;
+}
+
+/** Structural slice of @huggingface/transformers we need (kept `any`-light). */
+interface TransformersModule {
+	AutoTokenizer: {
+		from_pretrained: (model: string, options?: Record<string, unknown>) => Promise<TokenizerLike>;
+	};
+	AutoModelForSequenceClassification: {
+		from_pretrained: (model: string, options?: Record<string, unknown>) => Promise<ModelLike>;
+	};
+	env: Record<string, unknown>;
+}
+
+interface TokenizerLike {
+	(text: string, options: Record<string, unknown>): unknown;
+}
+
+interface ModelLike {
+	(inputs: unknown): Promise<{ logits: { data: Float32Array | number[] } }>;
+}
+
+type LoadedReranker = { tokenizer: TokenizerLike; model: ModelLike };
+
+export class LocalReranker implements RerankerLike {
+	private loaded: LoadedReranker | null = null;
+	private loading: Promise<LoadedReranker | null> | null = null;
+	/** Set after a load failure; every later call short-circuits to no-op. */
+	failed = false;
+
+	constructor(
+		readonly model: string,
+		private readonly loadTransformers: () => Promise<TransformersModule> = async () =>
+			(await import("@huggingface/transformers")) as unknown as TransformersModule,
+	) {}
+
+	async rerank(query: string, docs: string[]): Promise<number[]> {
+		if (this.failed || docs.length === 0) return [];
+		try {
+			const loaded = await this.ensureLoaded();
+			if (!loaded) return [];
+			const { tokenizer, model } = loaded;
+			const scores: number[] = [];
+			// One pair per call: 30 sequential CPU inferences are well inside
+			// the latency budget this opt-in feature accepts, and per-pair
+			// truncation keeps a long doc from padding the whole batch.
+			for (const doc of docs) {
+				const inputs = tokenizer(query, { text_pair: doc, padding: true, truncation: true });
+				const out = await model(inputs);
+				const logit = Number(out.logits.data[0] ?? 0);
+				// Sigmoid, not softmax: single-logit cross-encoders must keep
+				// their ranking signal (softmax over one logit is constant 1).
+				scores.push(1 / (1 + Math.exp(-logit)));
+			}
+			return scores;
+		} catch {
+			this.failed = true;
+			return [];
+		}
+	}
+
+	private ensureLoaded(): Promise<LoadedReranker | null> {
+		if (this.loaded) return Promise.resolve(this.loaded);
+		if (!this.loading) {
+			this.loading = (async () => {
+				const mod = await this.loadTransformers();
+				// Same models dir as the embedder, so opt-in reranking never
+				// downloads into a stray ./Xenova folder inside the project.
+				mod.env.cacheDir = transformersCacheDir();
+				const tokenizer = await mod.AutoTokenizer.from_pretrained(this.model);
+				const model = await mod.AutoModelForSequenceClassification.from_pretrained(this.model, {
+					quantized: true,
+				});
+				this.loaded = { tokenizer, model };
+				return this.loaded;
+			})().catch(() => {
+				this.failed = true;
+				return null;
+			});
+		}
+		return this.loading;
+	}
+}

package/src/retrieval/retriever.ts

@@ -5,8 +5,10 @@
 import type { Chunk } from "../types.js";
 import type { LocalBackend } from "../storage/local-backend.js";
 import type { Embedder } from "../indexer/embedder.js";
+import { parseContextHeaderUnit } from "../indexer/context-header.js";
 import { ConfidenceScorer } from "./confidence.js";
-import { QueryParser, type ParsedQuery } from "./query-parser.js";
+import { elaborateQuery, QueryParser, type ParsedQuery } from "./query-parser.js";
+import type { RerankerLike } from "./reranker.js";
 
 const DEPRIORITISED_PATHS = ["tests/", "test_", "docs/", "spec", "plan"];
 const RRF_K = 60;
@@ -14,13 +16,52 @@ const CONFIDENCE_WEIGHT = 0.5;
 const MAX_CHUNKS_PER_FILE = 3;
 const FTS_BOOST_CODE_LOOKUP = 1.5;
 const WORST_COSINE_DISTANCE = 2.0;
+/** R2: reweight near-ties toward the chunk that defines an exact query symbol. */
+const SYMBOL_MATCH_BONUS = 0.06;
+/** R5: structured paths — symbol nodes and explicit file hints as extra RRF lists. */
+const SYMBOL_LIST_MAX = 20;
+const HINT_FILES_MAX = 2;
+const HINT_CHUNKS_PER_FILE = 10;
+const HINT_PATH_BOOST = 1.5;
+/** R6: small-to-big serving — parent module + adjacent chunk for top hits. */
+const EXPAND_FILES_MAX = 3;
+const EXPAND_NEIGHBOR_GAP = 30;
+const EXPAND_SIBLINGS_LOOKUP = 16;
+/** R7: max multiplicative boost from the file PageRank prior (hub files). */
+const PAGE_RANK_WEIGHT = 0.1;
+/** R8: auto-elaborated candidates fuse at half weight and never outscore a raw hit. */
+const ELABORATE_WEIGHT = 0.5;
+/** R10: opt-in reranker reorders at most this many fused candidates. */
+const RERANK_TOPN = 30;
+/** R10: blend weight of the reranker score vs the fused score. */
+const RERANK_WEIGHT = 0.7;
+
+/** R8: deterministic query-quality signals exposed to the engine/tool layer. */
+export interface QuerySignals {
+	/** Best final score after fusion (0..1). */
+	topScore: number;
+	/** Score drop from the top hit to the best hit from a DIFFERENT file. */
+	margin: number;
+	/** Jaccard overlap of vector-top-10 and FTS-top-10 ids (0..1). */
+	agreement: number;
+	/** Distinct candidates that reached fusion. */
+	candidates: number;
+}
 
 export interface RetrieveOptions {
 	topK?: number;
 	confidenceThreshold?: number;
 	maxTokens?: number | null;
 	marginalRatio?: number;
-	statsOut?: { candidates?: number; selected?: number; droppedLowValue?: number };
+	statsOut?: {
+		candidates?: number;
+		selected?: number;
+		droppedLowValue?: number;
+		/** R8: filled on every call; consumed by the confidence line in results. */
+		querySignals?: QuerySignals;
+		/** R9: files ranked 11-15 but not served — refine-hint material. */
+		nearMissFiles?: string[];
+	};
 }
 
 export class HybridRetriever {
@@ -28,10 +69,20 @@ export class HybridRetriever {
 	private embedder: Embedder;
 	private scorer = new ConfidenceScorer();
 	private parser = new QueryParser();
+	private expandContext: boolean;
+	private autoElaborate: boolean;
+	private reranker: RerankerLike | null;
 
-	constructor(backend: LocalBackend, embedder: Embedder) {
+	constructor(
+		backend: LocalBackend,
+		embedder: Embedder,
+		options: { expandContext?: boolean; autoElaborate?: boolean; reranker?: RerankerLike | null } = {},
+	) {
 		this.backend = backend;
 		this.embedder = embedder;
+		this.expandContext = options.expandContext ?? true;
+		this.autoElaborate = options.autoElaborate ?? false;
+		this.reranker = options.reranker ?? null;
 	}
 
 	async retrieve(query: string, options: RetrieveOptions = {}): Promise<Chunk[]> {
@@ -66,8 +117,77 @@ export class HybridRetriever {
 			}
 		}
 
+		// ── R5: structured paths ───────────────────────────────────────────────
+		// Two deterministic lists fused alongside vector and BM25: exact symbol
+		// hits from the graph index (so an identifier query hits the definition,
+		// not prose about it) and the chunks of files the query names outright.
+		// Evidence: CodeRAG's multi-path retrieval is a core reason it beats
+		// single-similarity baselines on repo-level tasks.
+		const symbolRanks = new Map<string, number>();
+		const hintRanks = new Map<string, number>();
+		if (parsed.keywords.length > 0) {
+			this.backend.findSymbols(parsed.keywords, SYMBOL_LIST_MAX).forEach((node, i) => {
+				symbolRanks.set(node.id, i);
+			});
+		}
+		if (parsed.fileHints.length > 0) {
+			const files = Object.keys(this.backend.fileChunkCounts());
+			let rank = 0;
+			for (const hint of parsed.fileHints.slice(0, 3)) {
+				const matches = files.filter((f) => f.includes(hint)).sort().slice(0, HINT_FILES_MAX);
+				for (const file of matches) {
+					for (const chunk of this.backend.chunksByFile(file, HINT_CHUNKS_PER_FILE)) {
+						if (!hintRanks.has(chunk.id)) hintRanks.set(chunk.id, rank++);
+					}
+				}
+			}
+		}
+		const structuredOnlyIds = [...symbolRanks.keys(), ...hintRanks.keys()].filter((id) => !chunkMap.has(id));
+		if (structuredOnlyIds.length > 0) {
+			for (const chunk of this.backend.getChunksByIds(structuredOnlyIds)) chunkMap.set(chunk.id, chunk);
+		}
+
+		// ── R8: optional deterministic elaboration ────────────────────────────
+		// A stopword-stripped variant enriched with terms from near-miss FTS
+		// hits, searched AGAIN and fused as an extra candidate source at half
+		// weight. Only ids the raw lists missed are eligible, so every raw
+		// candidate's score (and their order among themselves) is untouched —
+		// elaboration can only add, never demote. Default off; the eval runs
+		// with it off so the gate measures the un-elaborated baseline.
+		const elaboratedRanks = new Map<string, number>();
+		if (this.autoElaborate && parsed.keywords.length > 0) {
+			try {
+				const nearMiss = this.backend.getChunksByIds([...ftsIds.keys()].slice(0, 5));
+				const variant = elaborateQuery(query, nearMiss);
+				if (variant) {
+					const variantEmbedding = await this.embedder.embedQuery(variant);
+					const seenRaw = (id: string): boolean => vectorRanks.has(id) || ftsIds.has(id);
+					this.backend.vectorSearch(variantEmbedding, topK * 3).forEach((c, rank) => {
+						if (!seenRaw(c.id) && !elaboratedRanks.has(c.id)) elaboratedRanks.set(c.id, rank);
+					});
+					this.backend.ftsSearch(variant, topK * 3).forEach((r, i) => {
+						if (!seenRaw(r.id) && !elaboratedRanks.has(r.id)) elaboratedRanks.set(r.id, i);
+					});
+					const elabOnlyIds = [...elaboratedRanks.keys()].filter((id) => !chunkMap.has(id));
+					if (elabOnlyIds.length > 0) {
+						for (const chunk of this.backend.getChunksByIds(elabOnlyIds)) chunkMap.set(chunk.id, chunk);
+					}
+				}
+			} catch {
+				// elaboration is best-effort
+			}
+		}
+
 		const ftsWeight = parsed.intent === "code_lookup" ? FTS_BOOST_CODE_LOOKUP : 1.0;
-		const allIds = new Set([...vectorRanks.keys(), ...ftsIds.keys()]);
+		const symbolWeight = parsed.intent === "code_lookup" ? FTS_BOOST_CODE_LOOKUP : 1.0;
+		const pageRanks = this.backend.pageRanks();
+		const allIds = new Set([
+			...vectorRanks.keys(),
+			...ftsIds.keys(),
+			...symbolRanks.keys(),
+			...hintRanks.keys(),
+			...elaboratedRanks.keys(),
+		]);
 		const rrfScores = new Map<string, number>();
 		for (const id of allIds) {
 			let score = 0;
@@ -75,6 +195,12 @@ export class HybridRetriever {
 			if (vRank !== undefined) score += 1 / (RRF_K + vRank);
 			const fRank = ftsIds.get(id);
 			if (fRank !== undefined) score += ftsWeight * (1 / (RRF_K + fRank));
+			const sRank = symbolRanks.get(id);
+			if (sRank !== undefined) score += symbolWeight * (1 / (RRF_K + sRank));
+			const hRank = hintRanks.get(id);
+			if (hRank !== undefined) score += HINT_PATH_BOOST * (1 / (RRF_K + hRank));
+			const eRank = elaboratedRanks.get(id);
+			if (eRank !== undefined) score += ELABORATE_WEIGHT * (1 / (RRF_K + eRank));
 			rrfScores.set(id, score);
 		}
 		const maxRrf = Math.max(0, ...rrfScores.values());
@@ -95,12 +221,55 @@ export class HybridRetriever {
 			const confScore = this.scorer.score(chunk, normalisedDistance, keywordDistance);
 			const normalisedRrf = maxRrf > 0 ? rrfScore / maxRrf : 0;
 			let finalScore = CONFIDENCE_WEIGHT * confScore + (1 - CONFIDENCE_WEIGHT) * normalisedRrf;
+			finalScore += symbolBonus(chunk, parsed);
+			// R7: bounded PageRank prior. Structured-importance signal (CodeRank:
+			// PageRank over the code graph lifts entry points and hubs above
+			// equally-matching leaves). Multiplicative and capped so it can only
+			// break ties between near-equal candidates, never override evidence.
+			const prior = pageRanks.get(chunk.filePath) ?? 0;
+			if (prior > 0) finalScore *= 1 + PAGE_RANK_WEIGHT * prior;
 			finalScore = applyPathPenalty(chunk.filePath, finalScore);
 			chunk.confidenceScore = finalScore;
 			scored.push({ chunk, score: finalScore });
 		}
 
 		scored.sort((a, b) => b.score - a.score);
+
+		// ── R10: opt-in rerank of the fused list ─────────────────────────────
+		// Reorders the top candidates only; penalties, dedupe, diversity and
+		// the marginal stop below are untouched. A failed/missing model leaves
+		// the fused order in place (rerank() returns []).
+		if (this.reranker && scored.length > 1) {
+			const topN = scored.slice(0, RERANK_TOPN);
+			const docs = topN.map((s) => s.chunk.compressedContent || s.chunk.content);
+			let rerankScores: number[] = [];
+			try {
+				rerankScores = await this.reranker.rerank(query, docs);
+			} catch {
+				// reranking is best-effort: model missing/offline → fused order stands
+			}
+			if (rerankScores.length === topN.length) {
+				const min = Math.min(...rerankScores);
+				const max = Math.max(...rerankScores);
+				const maxOld = topN[0]!.score || 1;
+				if (max > min) {
+					for (let i = 0; i < topN.length; i++) {
+						const norm = (rerankScores[i]! - min) / (max - min);
+						const blended = RERANK_WEIGHT * norm + (1 - RERANK_WEIGHT) * (topN[i]!.score / maxOld);
+						topN[i]!.score = blended;
+						topN[i]!.chunk.confidenceScore = blended;
+					}
+					// PR #7 review: a reranked candidate can blend down to ~0.3 while
+					// unreranked rank-31+ entries keep their raw ~0.7 scores — a full
+					// sort would promote unjudged tail hits OVER judged ones. Cap the
+					// tail strictly below the blended floor so reranking can only
+					// reorder within the judged set.
+					capTailBelowRerankedFloor(scored, RERANK_TOPN);
+					scored.sort((a, b) => b.score - a.score);
+				}
+			}
+		}
+
 		const deduped = dedupeOverlaps(scored);
 		const candidates = deduped.length;
 
@@ -108,22 +277,81 @@ export class HybridRetriever {
 		if (filtered.length === 0 && deduped.length > 0) filtered = deduped.slice(0, 1);
 		const droppedLowValue = candidates - filtered.length;
 
-		// File diversity + marginal stop.
-		const topScore = filtered.length > 0 ? filtered[0]!.score : 0;
+		// ── R8: query-quality signals ────────────────────────────────────────
+		// Deterministic inputs for the confidence line: absolute top score,
+		// the margin to the best hit from a different file (single-file results
+		// are fragile), vector/FTS agreement (two independent retrievers
+		// landing on the same chunks is the strongest quality signal), and the
+		// candidate count floor (tiny candidate pools are unreliable).
+		const firstHit = filtered[0]?.chunk;
+		let nextFileScore = -1;
+		if (firstHit) {
+			for (const s of filtered) {
+				if (s.chunk.filePath !== firstHit.filePath) {
+					nextFileScore = s.score;
+					break;
+				}
+			}
+		}
+		const vTop = new Set([...vectorRanks.keys()].slice(0, 10));
+		const fTop = new Set([...ftsIds.keys()].slice(0, 10));
+		let agreeInter = 0;
+		for (const id of vTop) if (fTop.has(id)) agreeInter += 1;
+		const agreeUnion = new Set([...vTop, ...fTop]).size;
+		const querySignals: QuerySignals = {
+			topScore: filtered[0]?.score ?? 0,
+			margin: firstHit ? (nextFileScore >= 0 ? filtered[0]!.score - nextFileScore : filtered[0]!.score) : 0,
+			agreement: agreeUnion > 0 ? agreeInter / agreeUnion : 0,
+			candidates: allIds.size,
+		};
+
+		// File diversity + marginal stop, with R6 small-to-big expansion: the
+		// first hit from a file pulls in its module parent and the closest
+		// neighbouring chunk so the agent gets surrounding context in ONE call
+		// instead of a follow-up expand_chunk (cross-file context length is the
+		// dominant quality factor in the controlled chunking study). Extras obey
+		// the same per-file cap: they ride along with an accepted hit, they never
+		// displace other files.
+		// Compare against a REPRESENTATIVE top score (3rd best), not the single
+		// best: one outlier at 0.9 used to push everything from rank ~5 onward
+		// below 0.75×top and cut true hits that raw ranking put at 5-15 (seen on
+		// five eval queries). The stop exists to shed junk tails, not to trust a
+		// lucky first candidate.
+		const topScore = filtered.length > 0 ? filtered[Math.min(2, filtered.length - 1)]!.score : 0;
 		const fileCounts = new Map<string, number>();
 		const diverse: Chunk[] = [];
+		const diverseKeys = new Set<string>();
+		const expandedFiles = new Set<string>();
 		let marginalDropped = 0;
+		const push = (chunk: Chunk): boolean => {
+			const key = `${chunk.filePath}:${chunk.startLine}-${chunk.endLine}`;
+			if (diverseKeys.has(key)) return false;
+			const count = fileCounts.get(chunk.filePath) ?? 0;
+			if (count >= MAX_CHUNKS_PER_FILE) return false;
+			diverseKeys.add(key);
+			diverse.push(chunk);
+			fileCounts.set(chunk.filePath, count + 1);
+			return true;
+		};
+
 		for (let idx = 0; idx < filtered.length; idx++) {
 			const { chunk, score } = filtered[idx]!;
 			if (diverse.length > 0 && marginalRatio > 0 && score < marginalRatio * topScore) {
 				marginalDropped = filtered.length - idx;
 				break;
 			}
-			const count = fileCounts.get(chunk.filePath) ?? 0;
-			if (count < MAX_CHUNKS_PER_FILE) {
-				diverse.push(chunk);
-				fileCounts.set(chunk.filePath, count + 1);
-				if (diverse.length >= topK) break;
+			if (diverse.length >= topK) break;
+			if (!push(chunk)) continue;
+			if (
+				this.expandContext &&
+				!expandedFiles.has(chunk.filePath) &&
+				expandedFiles.size < EXPAND_FILES_MAX
+			) {
+				expandedFiles.add(chunk.filePath);
+				for (const extra of this.expansionChunks(chunk)) {
+					if (diverse.length >= topK) break;
+					push(extra);
+				}
 			}
 		}
 		let ranked = diverse;
@@ -149,7 +377,21 @@ export class HybridRetriever {
 			}
 		}
 
-		const stats = { candidates, selected: 0, droppedLowValue: droppedLowValue + marginalDropped };
+		// R9: near-miss files for refine hints — ranked 11-15 after dedupe but
+		// not served. Surfaced instead of silently vanishing so the tool result
+		// can point the agent one step deeper at near-zero token cost.
+		const servedFiles = new Set(diverse.map((c) => c.filePath));
+		const nearMissFiles = [...new Set(filtered.slice(10, 15).map((s) => s.chunk.filePath))]
+			.filter((f) => !servedFiles.has(f))
+			.slice(0, 2);
+
+		const stats = {
+			candidates,
+			selected: 0,
+			droppedLowValue: droppedLowValue + marginalDropped,
+			querySignals,
+			nearMissFiles,
+		};
 		if (maxTokens === null) {
 			stats.selected = ranked.length;
 			if (options.statsOut) Object.assign(options.statsOut, stats);
@@ -175,6 +417,40 @@ export class HybridRetriever {
 		if (options.statsOut) Object.assign(options.statsOut, stats);
 		return packed;
 	}
+	/**
+	 * R6: parent context for an accepted hit — the file's module chunk plus the
+	 * closest function/class chunk within a 30-line gap. Looked up from storage
+	 * (cheap, ≤16 rows) so it works for hits that arrived via any path.
+	 */
+	private expansionChunks(hit: Chunk): Chunk[] {
+		try {
+			const siblings = this.backend.chunksByFile(hit.filePath, EXPAND_SIBLINGS_LOOKUP);
+			const moduleChunk = siblings.find(
+				(c) => c.chunkType === "module" && !(c.startLine === hit.startLine && c.endLine === hit.endLine),
+			);
+			let neighbor: Chunk | undefined;
+			let bestGap = Number.POSITIVE_INFINITY;
+			for (const c of siblings) {
+				if (c.chunkType === "module" || c.id === hit.id) continue;
+				const gap =
+					c.startLine > hit.endLine
+						? c.startLine - hit.endLine - 1
+						: hit.startLine > c.endLine
+							? hit.startLine - c.endLine - 1
+							: -1;
+				if (gap >= 0 && gap <= EXPAND_NEIGHBOR_GAP && gap < bestGap) {
+					bestGap = gap;
+					neighbor = c;
+				}
+			}
+			const out: Chunk[] = [];
+			if (moduleChunk) out.push(moduleChunk);
+			if (neighbor) out.push(neighbor);
+			return out;
+		} catch {
+			return []; // expansion is best-effort
+		}
+	}
 }
 
 function chunkTokenCountFor(chunk: Chunk): number {
@@ -191,12 +467,33 @@ function applyPathPenalty(filePath: string, score: number): number {
 	return score;
 }
 
+/**
+ * PR #7 review fix: after blending rerank scores into the top-N, cap every
+ * unreranked tail entry strictly below the blended floor (monotonically
+ * decreasing, preserving the tail's own order). Without this, a full sort
+ * promotes unjudged rank-31+ hits over reranked candidates whose blend
+ * dropped low. Exported for unit testing.
+ */
+export function capTailBelowRerankedFloor(scored: { score: number }[], topN: number): void {
+	if (scored.length <= topN) return;
+	let floor = Math.min(...scored.slice(0, topN).map((s) => s.score));
+	for (let i = topN; i < scored.length; i++) {
+		floor = Math.min(floor - 1e-6, scored[i]!.score);
+		scored[i]!.score = floor;
+	}
+}
+
 function dedupeOverlaps(scored: { chunk: Chunk; score: number }[]): { chunk: Chunk; score: number }[] {
 	const kept: { chunk: Chunk; score: number }[] = [];
 	for (const { chunk, score } of scored) {
 		let duplicate = false;
+		// Module chunks are the file's parent context: their span legitimately
+		// covers every function chunk, so overlap-based dedupe would delete the
+		// whole point of R6. Only leaf-vs-leaf overlaps are duplicates.
+		const isParent = chunk.chunkType === "module";
 		for (const k of kept) {
 			const kc = k.chunk;
+			if (isParent || kc.chunkType === "module") continue;
 			if (kc.filePath !== chunk.filePath) continue;
 			const overlap =
 				Math.min(chunk.endLine, kc.endLine) - Math.max(chunk.startLine, kc.startLine) + 1;
@@ -212,6 +509,36 @@ function dedupeOverlaps(scored: { chunk: Chunk; score: number }[]): { chunk: Chu
 	return kept;
 }
 
+/**
+ * R2c: when a query names an exact symbol (`atomicWriteText`,
+ * `HybridRetriever`), the chunk whose unit *defines* that name should win
+ * near-ties against chunks that merely discuss it. High precision by design:
+ * whole-segment, case-insensitive equality only, small constant bonus, so it
+ * never outvotes real relevance evidence.
+ */
+function symbolBonus(chunk: Chunk, parsed: ParsedQuery): number {
+	const unit =
+		typeof chunk.metadata["unit"] === "string"
+			? (chunk.metadata["unit"] as string)
+			: parseContextHeaderUnit(chunk.content);
+	if (!unit) return 0;
+	const segments = unit.split(".").map((s) => s.toLowerCase());
+	for (const kw of parsed.keywords) {
+		const k = kw.toLowerCase();
+		if (k.length < 3) continue;
+		if (segments.includes(k)) return SYMBOL_MATCH_BONUS;
+	}
+	return 0;
+}
+
+/**
+ * Keyword evidence as COVERAGE, not any-hit. One generic word ("result",
+ * "list") used to earn the same full credit as naming the actual identifier,
+ * which let loosely-related chunks score like precise matches. Identifiers
+ * (camelCase / snake_case / acronyms) weigh 3x: they are rare and specific,
+ * which is exactly what code search rewards (see R2 evidence: BM25 and exact
+ * matching dominate code retrieval).
+ */
 function estimateKeywordDistance(chunk: Chunk, parsed: ParsedQuery): number {
 	if (parsed.fileHints.length > 0) {
 		for (const hint of parsed.fileHints) {
@@ -219,8 +546,18 @@ function estimateKeywordDistance(chunk: Chunk, parsed: ParsedQuery): number {
 		}
 	}
 	const content = chunk.content.toLowerCase();
+	let total = 0;
+	let covered = 0;
 	for (const keyword of parsed.keywords) {
-		if (content.includes(keyword.toLowerCase())) return 0;
+		const weight = identifierLike(keyword) ? 3 : 1;
+		total += weight;
+		if (content.includes(keyword.toLowerCase())) covered += weight;
 	}
-	return 2;
+	if (total === 0) return 2;
+	// 0 = every keyword present, 2 = none present (the old miss value).
+	return (1 - covered / total) * 2;
+}
+
+function identifierLike(keyword: string): boolean {
+	return keyword.length >= 3 && (/[A-Z]/.test(keyword) || keyword.includes("_"));
 }

package/src/storage/fts-store.ts

@@ -9,6 +9,17 @@ import type { Chunk } from "../types.js";
 
 const MAX_CONTENT_CHARS = 5_000;
 
+/**
+ * BM25 column weights (table order: id, content, file_path, language,
+ * chunk_type). Weighting `file_path` above `content` makes filename stems
+ * in a query ("grammar rules" → `src/memory/grammar.ts`) decisive, while a
+ * single language/chunk-type token can never carry a match on its own.
+ * Evidence: BM25 is the strongest single retriever for code overall
+ * (arXiv:2605.14503 finding 6) and contextual BM25 was the bigger half of
+ * Anthropic's contextual-retrieval gain.
+ */
+const BM25_WEIGHTS = "0.0, 1.0, 3.0, 0.5, 0.5";
+
 // Common English words dropped so natural-language phrasing doesn't drown the
 // signal terms. If every token is a stopword we fall back to using them all.
 const STOPWORDS = new Set([
@@ -19,16 +30,43 @@ const STOPWORDS = new Set([
 ]);
 
 /**
- * Build a safe FTS5 MATCH expression from free-form user input. Each token is
- * individually quoted (internal quotes doubled) which neutralises FTS5
- * operators; tokens are OR-joined so BM25 still ranks multi-token hits higher.
+ * Build a safe FTS5 MATCH expression from free-form user input.
+ *
+ * - Quoted spans become FTS5 phrases and are AND-constrained: `"error
+ *   handling"` only matches when the words are adjacent, which is what a
+ *   quoted string means to a user. Code symbols work because FTS5's
+ *   tokenizer splits them the same way on both sides (`encode_token` →
+ *   `encode token`).
+ * - Unquoted tokens are OR-joined (BM25 still ranks multi-token hits higher)
+ *   so a natural-language query keeps matching whatever it can.
+ * - Stopwords are dropped; when every token is one, they are used as-is.
+ *
  * Returns "" when nothing usable remains — callers must skip the query.
  */
 export function escapeFts5(query: string): string {
-	const tokens = query.split(/\s+/).filter((t) => /[a-zA-Z0-9]/.test(t));
+	const phrases: string[] = [];
+	const rest = query.replace(/"([^"]+)"/g, (_match, inner: string) => {
+		// Tokenize the phrase the way FTS5's tokenizer will have indexed it
+		// (unicode61: letters/digits are tokenchars, underscores and punctuation
+		// split — `encode_token` indexes as `encode token`). Unicode-aware so
+		// diacritics survive ("café" stays "café"; the tokenizer folds accents
+		// on BOTH sides, so ASCII-splitting them away made quoted phrases miss
+		// content the unquoted query could find).
+		const words = inner
+			.split(/[^\p{L}\p{N}]+/u)
+			.filter((w) => /[\p{L}\p{N}]/u.test(w));
+		if (words.length > 0) phrases.push(`"${words.join(" ")}"`);
+		return " ";
+	});
+
+	const tokens = rest.split(/\s+/).filter((t) => /[\p{L}\p{N}]/u.test(t));
 	let kept = tokens.filter((t) => !STOPWORDS.has(t.toLowerCase()));
 	if (kept.length === 0) kept = tokens;
-	return kept.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
+	const tokenExpr = kept.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
+
+	if (phrases.length > 0 && tokenExpr) return `${phrases.join(" AND ")} AND (${tokenExpr})`;
+	if (phrases.length > 0) return phrases.join(" AND ");
+	return tokenExpr;
 }
 
 type Sqlite = SqliteHandle;
@@ -72,7 +110,11 @@ export class FTSStore {
 		if (!matchExpr) return [];
 		try {
 			const rows = this.db
-				.prepare("SELECT id, rank FROM chunks_fts WHERE chunks_fts MATCH ? ORDER BY rank LIMIT ?")
+				.prepare(
+					`SELECT id, bm25(chunks_fts, ${BM25_WEIGHTS}) AS rank
+					 FROM chunks_fts WHERE chunks_fts MATCH ?
+					 ORDER BY rank LIMIT ?`,
+				)
 				.all(matchExpr, topK) as { id: string; rank: number }[];
 			return rows;
 		} catch {