pi-mega-compact 0.21.1 → 0.21.3

package/src/recall/vote.ts

@@ -46,16 +46,32 @@ export interface VoteOptions {
 	recencyCount?: number;
 }
 
-/** Per-source normalization: map raw scores to 0..1 via min-max within source. */
-function normalizeScores(scores: number[]): Map<number, number> {
+/**
+ * Per-source normalization: map raw scores to 0..1 via min-max within source.
+ *
+ * E1 follow-up (PR #18 review): non-finite scores (NaN/±Infinity) are DROPPED
+ * before the min/max fold — a single NaN silently propagates through Math.min/
+ * max and turns EVERY normalized score of that source into NaN (verified),
+ * poisoning the whole 3-source quorum. Dropping the bad entry degrades that
+ * source gracefully instead. Exported so the guard is unit-testable directly.
+ */
+export function normalizeScores(scores: number[]): Map<number, number> {
 	const map = new Map<number, number>();
-	if (scores.length === 0) return map;
-	const min = Math.min(...scores);
-	const max = Math.max(...scores);
-	const span = max - min;
+	const finite: { idx: number; s: number }[] = [];
 	scores.forEach((s, i) => {
-		map.set(i, span === 0 ? 1 : (s - min) / span);
+		if (Number.isFinite(s)) finite.push({ idx: i, s });
 	});
+	if (finite.length === 0) return map;
+	let min = Infinity;
+	let max = -Infinity;
+	for (const { s } of finite) {
+		if (s < min) min = s;
+		if (s > max) max = s;
+	}
+	const span = max - min;
+	for (const { idx, s } of finite) {
+		map.set(idx, span === 0 ? 1 : (s - min) / span);
+	}
 	return map;
 }
 
@@ -150,7 +166,12 @@ export function voteRecall(opts: VoteOptions, store: VectorStore): VoteResult {
 		const norm = perSource.find((p) => p.name === src.name)!.norm;
 		const bestByCp = new Map<string, number>();
 		src.cands.forEach((c, i) => {
-			const n = norm.get(i) ?? 0;
+			// E1 follow-up: normalizeScores dropped non-finite scores; such a hit is
+			// NOT a valid nomination — skip it entirely instead of defaulting it to
+			// 0 (which would still name the checkpoint and let it rank last into the
+			// fallback ranking).
+			const n = norm.get(i);
+			if (n === undefined) return;
 			const prev = bestByCp.get(c.checkpointId);
 			if (prev === undefined || n > prev) bestByCp.set(c.checkpointId, n);
 			seenIds.add(c.checkpointId);

package/src/store/sqlite/fts5-search.ts

@@ -124,7 +124,13 @@ export function hydrateFts5Hits(
 	const out: HydratedFts5Hit[] = [];
 	for (const h of hits) {
 		const cp = cpMap.get(h.id);
-		if (cp) out.push({ checkpointId: h.id, score: h.score, summary: cp.summary });
+		// H1 follow-up (PR #18 review): exclude SemDeDup-'removed' rows. The FTS5
+		// index is NOT pruned by vectorSemDedup, so a removed row can still MATCH;
+		// hydrating it would let the fts5 voter NAME a dead checkpoint (a removed
+		// row plus a recency vote is a 2/3 agreement that passes the validator).
+		// Mirrors vectorSearch's read-time filter.
+		if (cp && cp.dedupStatus !== "removed")
+			out.push({ checkpointId: h.id, score: h.score, summary: cp.summary });
 	}
 	return out;
 }

package/src/vector-read.ts

@@ -100,8 +100,11 @@ export function vectorDedupe(
     : regionHashOrText;
   const state = loadSessionState(sid, stateDir);
   if (state.storedRegionHashes.includes(hash)) return true;
+  // H1 follow-up (PR #18 review): a SemDeDup-'removed' row's regionHash must not
+  // report "already represented" — its content is excluded from recall, so the
+  // incoming region is NOT deduplicated in any retrievable sense.
   return listCheckpoints(sid, stateDir).some(
-    (c) => c.regionHash === hash,
+    (c) => c.dedupStatus !== "removed" && c.regionHash === hash,
   );
 }
 
@@ -157,7 +160,11 @@ export function vectorTopSimilar(store: VectorStore, sessionId: string, n: numbe
   const current = ordered[ordered.length - 1];
 
   const scored: SearchHit[] = ordered
-    .filter((cp) => cp.checkpointId !== current.checkpointId)
+    .filter(
+      (cp) =>
+        cp.checkpointId !== current.checkpointId &&
+        cp.dedupStatus !== "removed", // H1 follow-up: mirror vectorSearch's filter
+    )
     .map((cp) => ({
       checkpoint: cp,
       score: cosineSimilarity(current.embedding, cp.embedding),

package/src/vectorStore/add.ts

@@ -54,7 +54,13 @@ export function addCheckpoint(store: VectorStore, input: AddInput): AddResult {
 	const t0 = Date.now();
 	const sessionId = normalizeSessionId(input.sessionId);
 	const regionHash = computeRegionHash(input.regionText);
-	const all = listCheckpoints(sessionId, store.stateDir);
+	// H1: exclude SemDeDup-'removed' rows from dedup matching. search() already
+	// filters these, but add() did NOT — so an L0/L1/L2 match against a previously
+	// removed duplicate would upsertCheckpoint it back to active, resurrecting it
+	// into recall and defeating SemDeDup.
+	const all = listCheckpoints(sessionId, store.stateDir).filter(
+		(cp) => cp.dedupStatus !== "removed",
+	);
 	// Honest "tokens saved" base for this region. For a deduped add the whole
 	// original region is discarded (nothing new stored); for a new checkpoint
 	// we persist (orig − stored). Falls back to stored when orig is unknown.