pi-mega-compact 0.4.0

package/src/config/dedup.ts

@@ -0,0 +1,120 @@
+/**
+ * config/dedup.ts — SINGLE SOURCE OF TRUTH for dedup tier flags + thresholds
+ * (Sprint 14, Phase 7).
+ *
+ * Every tier flag and threshold that was previously an inline default in
+ * vectorStore.ts / the RAPTOR modules is defined HERE and only here (QA #8: no
+ * duplicated threshold across modules). Values are read from MEGACOMPACT_* env
+ * at load, with the file defaults below as the fallback (which reproduce the
+ * Sprint 13 behavior — all tiers active, nothing MARK_ONLY).
+ *
+ * MARK_ONLY semantics (QA ops): a tier in MARK_ONLY still RUNS and RECORDS its
+ * decision (so we keep the data + can replay), but does NOT collapse/dedup — a
+ * safe partial-rollout / auto-degrade state.
+ *
+ * PREVENT-PI-004: pure config, no network. Booleans/numbers only.
+ */
+
+function envBool(name: string, def: boolean): boolean {
+  const v = process.env[name];
+  if (v === undefined) return def;
+  return v === "true" || v === "1";
+}
+
+function envNum(name: string, def: number): number {
+  const v = process.env[name];
+  if (v === undefined) return def;
+  const n = Number(v);
+  return Number.isFinite(n) ? n : def;
+}
+
+export interface DedupConfigShape {
+  // Tier enable flags.
+  L0_ENABLED: boolean;
+  L1_ENABLED: boolean;
+  L2_ENABLED: boolean;
+  RAPTOR_ENABLED: boolean;
+  // MARK_ONLY per tier: run + record, never collapse.
+  MARK_ONLY_L0: boolean;
+  MARK_ONLY_L1: boolean;
+  MARK_ONLY_L2: boolean;
+  // Embedder selection.
+  MINILM_EMBEDDER: boolean;
+  // Thresholds.
+  L2_COSINE: number; // semantic dedup firing point
+  L1_JACCARD: number; // MinHash/LSH near-dup verification
+  DEDUP_SIM: number; // legacy content-similarity fallback
+  MMR_LAMBDA: number; // retrieval diversity
+  SEMDEDUP_COSINE: number; // offline SemDeDup pair threshold
+  // Caps / budgets.
+  SIMILARITY_BUDGET_MS: number;
+  L1_VERIFY_BUDGET_MS: number;
+  L1_CANDIDATE_CAP: number;
+  RAPTOR_BUDGET_MS: number;
+  RAPTOR_CLUSTERS_PER_LEVEL: number;
+  RAPTOR_CONSISTENCY: number;
+  // Monitoring / alerting.
+  FP_RATE_L0: number; // FP alert threshold for exact tier
+  FP_RATE_L1L2: number; // FP alert threshold for fuzzy tiers
+  ALERT_WINDOW_MS: number;
+  P95_BUDGET_MS: number; // canary p95 budget per tier
+}
+
+/** Read the current dedup config from env (file defaults reproduce Sprint 13). */
+export function loadDedupConfig(): DedupConfigShape {
+  return {
+    L0_ENABLED: envBool("MEGACOMPACT_L0_ENABLED", true),
+    L1_ENABLED: envBool("MEGACOMPACT_L1_ENABLED", true),
+    L2_ENABLED: envBool("MEGACOMPACT_L2_ENABLED", true),
+    RAPTOR_ENABLED: envBool("MEGACOMPACT_RAPTOR_ENABLED", false), // shadow by default
+    MARK_ONLY_L0: envBool("MEGACOMPACT_MARK_ONLY_L0", false),
+    MARK_ONLY_L1: envBool("MEGACOMPACT_MARK_ONLY_L1", false),
+    MARK_ONLY_L2: envBool("MEGACOMPACT_MARK_ONLY_L2", false),
+    MINILM_EMBEDDER: envBool("MEGACOMPACT_MINILM", false),
+    L2_COSINE: envNum("MEGACOMPACT_L2_THRESHOLD", 0.85),
+    L1_JACCARD: envNum("MEGACOMPACT_L1_JACCARD", 0.8),
+    DEDUP_SIM: envNum("MEGACOMPACT_DEDUP_SIM", 0.9),
+    MMR_LAMBDA: envNum("MEGACOMPACT_MMR_LAMBDA", 0.5),
+    SEMDEDUP_COSINE: envNum("MEGACOMPACT_SEMDEDUP_COSINE", 0.95),
+    SIMILARITY_BUDGET_MS: envNum("MEGACOMPACT_SIMILARITY_BUDGET_MS", 50),
+    L1_VERIFY_BUDGET_MS: envNum("MEGACOMPACT_L1_VERIFY_BUDGET_MS", 20),
+    L1_CANDIDATE_CAP: envNum("MEGACOMPACT_L1_CANDIDATE_CAP", 100),
+    RAPTOR_BUDGET_MS: envNum("MEGACOMPACT_RAPTOR_BUDGET_MS", 5000),
+    RAPTOR_CLUSTERS_PER_LEVEL: envNum("MEGACOMPACT_RAPTOR_CLUSTERS", 5),
+    RAPTOR_CONSISTENCY: envNum("MEGACOMPACT_RAPTOR_CONSISTENCY", 0.6),
+    FP_RATE_L0: envNum("MEGACOMPACT_FP_RATE_L0", 0.01),
+    FP_RATE_L1L2: envNum("MEGACOMPACT_FP_RATE_L1L2", 0.05),
+    ALERT_WINDOW_MS: envNum("MEGACOMPACT_ALERT_WINDOW_MS", 600_000),
+    P95_BUDGET_MS: envNum("MEGACOMPACT_P95_BUDGET_MS", 100),
+  };
+}
+
+/**
+ * The default config snapshot (read once at import). Callers that need to honor
+ * runtime env changes (tests) should call loadDedupConfig() directly; the live
+ * add()/search() path reads this snapshot but accepts an override for testing.
+ */
+export const DedupConfig: DedupConfigShape = loadDedupConfig();
+
+/** Tier identifiers used in monitoring + alerting. */
+export type DedupTier = "L0" | "L1" | "L2" | "RAPTOR";
+
+/** Is a given tier enabled (and not merely MARK_ONLY)? */
+export function tierEnabled(cfg: DedupConfigShape, tier: DedupTier): boolean {
+  switch (tier) {
+    case "L0": return cfg.L0_ENABLED;
+    case "L1": return cfg.L1_ENABLED;
+    case "L2": return cfg.L2_ENABLED;
+    case "RAPTOR": return cfg.RAPTOR_ENABLED;
+  }
+}
+
+/** Is a given tier in MARK_ONLY (record, don't collapse)? */
+export function tierMarkOnly(cfg: DedupConfigShape, tier: DedupTier): boolean {
+  switch (tier) {
+    case "L0": return cfg.MARK_ONLY_L0;
+    case "L1": return cfg.MARK_ONLY_L1;
+    case "L2": return cfg.MARK_ONLY_L2;
+    case "RAPTOR": return false; // RAPTOR has its own shadow mode
+  }
+}

package/src/config.ts

@@ -0,0 +1,15 @@
+/**
+ * config.ts — shared default paths/constants for the mega-compact engine.
+ *
+ * Kept tiny and dependency-free so both the extension entry and unit tests can
+ * import it without pulling in pi runtime types.
+ */
+
+import { join } from "node:path";
+import { homedir } from "node:os";
+
+/** Default on-disk location for checkpoints + session state. */
+export const STATE_DIR_DEFAULT = join(homedir(), ".pi", "agent", "extensions", "pi-mega-compact");
+
+/** Pi custom message / entry type used as the dedup sentinel. */
+export const MARKER_TYPE = "mega-compact-marker";

package/src/dedup/dedup.test.ts

@@ -0,0 +1,46 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { normalize, stripAnsi } from "./normalize.js";
+import { computeContentDigest, CONTENT_HASH_VERSION } from "./digest.js";
+
+test("normalize collapses whitespace/newline variants to one form (Sprint 9)", () => {
+  // Sprint 9 normalizes whitespace/newlines/ANSI (not case — that is Sprint 10).
+  const variants = [
+    "foo  bar",
+    "foo bar",
+    "foo\tbar",
+    "foo\nbar",
+    "  foo   bar  ",
+    "foo\r\nbar",
+  ];
+  const digests = variants.map((v) => computeContentDigest(v).contentHash);
+  const unique = new Set(digests);
+  assert.equal(unique.size, 1, "all whitespace/newline variants must collapse to one digest");
+});
+
+test("normalize is idempotent", () => {
+  const input = "  Hello   World  \n";
+  assert.equal(normalize(normalize(input)), normalize(input));
+});
+
+test("stripAnsi removes terminal color codes", () => {
+  const colored = "err\x1b[31m fatal\x1b[0m boom";
+  assert.equal(stripAnsi(colored), "err fatal boom");
+});
+
+test("computeContentDigest emits full 64-hex dual hashes + version", () => {
+  const d = computeContentDigest("the same region text");
+  assert.equal(d.contentHash.length, 64);
+  assert.equal(d.contentHash2.length, 64);
+  assert.equal(d.contentHashVersion, CONTENT_HASH_VERSION);
+  assert.equal(d.normalizedText, "the same region text");
+  // Secondary is an independent view (reversed) so it differs from primary.
+  assert.notEqual(d.contentHash, d.contentHash2);
+});
+
+test("dual-hash: distinct content yields a distinct pair (both must agree to dedup)", () => {
+  const a = computeContentDigest("region about authentication");
+  const b = computeContentDigest("region about authorization");
+  assert.notEqual(a.contentHash, b.contentHash);
+  assert.notEqual(a.contentHash2, b.contentHash2);
+});

package/src/dedup/digest.ts

@@ -0,0 +1,40 @@
+/**
+ * digest.ts — content-addressable digest (Sprint 9).
+ *
+ * Dual-hash design (QA #2 spirit, local): a primary + a secondary hash guard
+ * against a single-hash collision silently merging distinct content. The L0 dedup
+ * key is `(content_hash, content_hash2)` — both must agree to declare a duplicate.
+ *
+ * `content_hash` is the full 64-hex SHA-256 of the normalized text. `content_hash2`
+ * is a second independent view (SHA-256 of the reversed normalized text) so a
+ * collision on one alone does not dedup. `content_hash_version` lets Sprint 11/12
+ * plug in stronger digests later without breaking old rows.
+ */
+
+import { createHash } from "node:crypto";
+import { normalize } from "./normalize.js";
+
+export const CONTENT_HASH_VERSION = 1;
+
+export interface ContentDigest {
+  contentHash: string;
+  contentHash2: string;
+  contentHashVersion: number;
+  normalizedText: string;
+}
+
+/** Compute the canonical content digest for a (possibly raw) region text. */
+export function computeContentDigest(text: string): ContentDigest {
+  const normalizedText = normalize(text);
+  const contentHash = createHash("sha256").update(normalizedText).digest("hex");
+  // Secondary: hash the reversed normalized string so it's an independent view.
+  const contentHash2 = createHash("sha256")
+    .update(normalizedText.split("").reverse().join(""))
+    .digest("hex");
+  return {
+    contentHash,
+    contentHash2,
+    contentHashVersion: CONTENT_HASH_VERSION,
+    normalizedText,
+  };
+}

package/src/dedup/l1-lsh.ts

@@ -0,0 +1,67 @@
+/**
+ * l1-lsh.ts — Locality-Sensitive Hashing banding over MinHash signatures
+ * (Sprint 11).
+ *
+ * Splits the 256-slot signature into `BANDS` bands of `ROWS_PER_BAND` rows. Each
+ * band is hashed into a bucket key; rows sharing a band are likely near-duplicates.
+ * The bucket key INCLUDES the session_id so candidates are scoped per session
+ * (deterministic, no cross-session leakage — QA determinism fix).
+ *
+ * All hashing is deterministic given the signature + seed (PREVENT-PI-004, pure).
+ */
+
+import { SIGNATURE_VERSION, NUM_HASHES } from "./l1-minhash.js";
+
+export const BANDS = 64;
+export const ROWS_PER_BAND = 4; // 64 * 4 = 256 slots (matches minhashSignature length)
+
+/** Stable 32-bit FNV-1a (buffer form) for band hashing. */
+function fnv1aBuf(buf: Buffer): number {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < buf.length; i++) {
+    h ^= buf[i];
+    h = Math.imul(h, 0x01000193);
+  }
+  return h >>> 0;
+}
+
+/**
+ * Compute the list of LSH bucket keys for a signature within a session.
+ * Deterministic: same (session_id, signature) → same keys, every run.
+ */
+export function lshBands(
+  signature: number[],
+  sessionId: string,
+  version: number = SIGNATURE_VERSION,
+): string[] {
+  if (signature.length < BANDS * ROWS_PER_BAND) {
+    throw new Error(
+      `lshBands: signature length ${signature.length} < required ${BANDS * ROWS_PER_BAND}`,
+    );
+  }
+  const keys: string[] = [];
+  const seedPrefix = Buffer.from(`${sessionId}|${version}|`, "utf-8");
+  for (let band = 0; band < BANDS; band++) {
+    const start = band * ROWS_PER_BAND;
+    const slice = signature.slice(start, start + ROWS_PER_BAND);
+    const body = Buffer.allocUnsafe(ROWS_PER_BAND * 4);
+    for (let r = 0; r < ROWS_PER_BAND; r++) {
+      body.writeUInt32LE(slice[r] >>> 0, r * 4);
+    }
+    const combined = Buffer.concat([seedPrefix, body]);
+    keys.push(`b${band}:${fnv1aBuf(combined).toString(16)}`);
+  }
+  return keys;
+}
+
+/** Convenience: derive bands directly from text (used by callers without a cached sig). */
+export function bandsForText(
+  signature: number[],
+  sessionId: string,
+  version: number = SIGNATURE_VERSION,
+): string[] {
+  if (signature.length !== NUM_HASHES) {
+    throw new Error(`bandsForText: expected signature length ${NUM_HASHES}, got ${signature.length}`);
+  }
+  return lshBands(signature, sessionId, version);
+}

package/src/dedup/l1-minhash.ts

@@ -0,0 +1,90 @@
+/**
+ * l1-minhash.ts — MinHash signatures for L1 near-duplicate detection (Sprint 11).
+ *
+ * Uses UNIVERSAL HASHING (QA #3), not the broken permutation scheme from the
+ * generic dedup plan. Each of the 256 hash functions is `h_i(x) = (a_i·x + b_i)
+ * mod p` with a fixed prime `p` and per-index coefficients derived from a pinned
+ * seed (0xDEADBEEF). This makes signatures DETERMINISTIC across process restarts
+ * (a hard requirement — non-determinism silently breaks dedup).
+ *
+ * `signatureVersion` lets Sprint 12+ swap the scheme without invalidating stored
+ * signatures: old buckets keep their version, new ones get the new one.
+ *
+ * Pure compute, no deps, no network (PREVENT-PI-004).
+ */
+
+import { normalize } from "./normalize.js";
+
+export const SIGNATURE_VERSION = 1;
+export const NUM_HASHES = 256;
+export const SHINGLE_SIZE = 5; // char 5-grams
+const MAX_SHINGLES = 50_000; // QA #7/#15 complexity cap
+const SEED = 0xdeadbeef;
+const P = 2147483647; // 2^31 - 1, Mersenne prime
+
+/** Per-index universal-hashing coefficients, derived deterministically from SEED. */
+function coeffA(i: number): number {
+  return (SEED + i * 2 + 1) % P;
+}
+function coeffB(i: number): number {
+  return (SEED * 3 + i * 7 + 13) % P;
+}
+
+/** Stable 32-bit FNV-1a of a shingle (the `x` fed to the universal hashes). */
+function shingleHash(gram: string): number {
+  let h = 0x811c9dc5;
+  for (let k = 0; k < gram.length; k++) {
+    h ^= gram.charCodeAt(k);
+    h = Math.imul(h, 0x01000193);
+  }
+  return h >>> 0;
+}
+
+/** Char n-gram shingle set (deduped) of normalized text, capped at MAX_SHINGLES. */
+export function shingles(text: string, size = SHINGLE_SIZE): number[] {
+  const norm = normalize(text);
+  if (norm.length === 0) return [];
+  const set = new Set<number>();
+  if (norm.length < size) {
+    set.add(shingleHash(norm));
+  } else {
+    for (let i = 0; i + size <= norm.length; i++) {
+      set.add(shingleHash(norm.slice(i, i + size)));
+      if (set.size >= MAX_SHINGLES) break;
+    }
+  }
+  return [...set];
+}
+
+/**
+ * Compute the 256-element MinHash signature of a text. Each slot is the minimum,
+ * over all shingles, of the i-th universal hash. Empty text → all-P sentinel.
+ */
+export function minhashSignature(text: string): number[] {
+  const grams = shingles(text);
+  const sig = new Array<number>(NUM_HASHES).fill(P);
+  if (grams.length === 0) return sig;
+  for (let i = 0; i < NUM_HASHES; i++) {
+    const a = coeffA(i);
+    const b = coeffB(i);
+    let min = P;
+    for (const x of grams) {
+      // (a*x + b) mod p — use Number math; a,x < 2^31 so a*x < 2^62, within
+      // double-precision integer range (2^53) only if reduced; reduce a*x first.
+      const ax = (a * (x % P)) % P;
+      const h = (ax + b) % P;
+      if (h < min) min = h;
+    }
+    sig[i] = min;
+  }
+  return sig;
+}
+
+/** Estimated Jaccard similarity of two signatures (fraction of equal slots). */
+export function signatureSimilarity(a: number[], b: number[]): number {
+  const n = Math.min(a.length, b.length);
+  if (n === 0) return 0;
+  let equal = 0;
+  for (let i = 0; i < n; i++) if (a[i] === b[i]) equal++;
+  return equal / n;
+}

package/src/dedup/l1-verify.ts

@@ -0,0 +1,55 @@
+/**
+ * l1-verify.ts — trigram-similarity verification gate for L1 near-duplicates
+ * (Sprint 11).
+ *
+ * After LSH cheaply retrieves candidate chunk_ids, we apply a REAL similarity
+ * check before declaring a duplicate. This is the `pg_trgm`-equivalent final
+ * gate: we compute the trigram (character 3-gram) Jaccard / overlap similarity
+ * between the new normalized text and each candidate's normalized text.
+ *
+ * We compute it in TS (not via FTS5 MATCH, which is boolean) so we get a stable
+ * [0,1] score to threshold against (0.85 per spec). The FTS5 `trigram` table is
+ * still maintained for future query-path use, but verification is pure TS so its
+ * result is deterministic and unit-testable (PREVENT-PI-004).
+ */
+
+import { normalize } from "./normalize.js";
+
+const TRIGRAM_SIZE = 3;
+export const L1_VERIFY_THRESHOLD = 0.85;
+
+/** Extract the set of character trigrams from normalized text. */
+function trigrams(text: string): Set<string> {
+  const norm = normalize(text);
+  const set = new Set<string>();
+  if (norm.length === 0) return set;
+  if (norm.length < TRIGRAM_SIZE) {
+    set.add(norm);
+    return set;
+  }
+  for (let i = 0; i + TRIGRAM_SIZE <= norm.length; i++) {
+    set.add(norm.slice(i, i + TRIGRAM_SIZE));
+  }
+  return set;
+}
+
+/**
+ * Trigram similarity in [0,1]. We use overlap coefficient (|A∩B| / min(|A|,|B|))
+ * which, like pg_trgm's `similarity`, is robust when one text is a substring of
+ * the other — better than Jaccard for the near-dup "one-word edit" case.
+ */
+export function trigramSimilarity(a: string, b: string): number {
+  const ta = trigrams(a);
+  const tb = trigrams(b);
+  if (ta.size === 0 || tb.size === 0) return 0;
+  let inter = 0;
+  const smaller = ta.size <= tb.size ? ta : tb;
+  const larger = smaller === ta ? tb : ta;
+  for (const g of smaller) if (larger.has(g)) inter++;
+  return inter / smaller.size;
+}
+
+/** True when `a` and `b` are near-duplicates under the L1 threshold. */
+export function isNearDuplicate(a: string, b: string, threshold = L1_VERIFY_THRESHOLD): boolean {
+  return trigramSimilarity(a, b) >= threshold;
+}

package/src/dedup/l1.test.ts

@@ -0,0 +1,57 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { minhashSignature, shingles, signatureSimilarity, NUM_HASHES } from "./l1-minhash.js";
+import { lshBands, BANDS, ROWS_PER_BAND } from "./l1-lsh.js";
+import { trigramSimilarity, isNearDuplicate, L1_VERIFY_THRESHOLD } from "./l1-verify.js";
+
+test("minhashSignature is deterministic across calls (same input → same sig)", () => {
+  const a = minhashSignature("the authentication module handles login securely");
+  const b = minhashSignature("the authentication module handles login securely");
+  assert.deepEqual(a, b);
+  assert.equal(a.length, NUM_HASHES);
+});
+
+test("minhashSignature of near-identical text is more similar than of unrelated text", () => {
+  const s1 = minhashSignature("user logged in and viewed the dashboard");
+  const s2 = minhashSignature("user logged in and viewed the dashboard page"); // one word added
+  const s3 = minhashSignature("the compiler optimized the hot loop aggressively");
+  assert.ok(signatureSimilarity(s1, s2) > signatureSimilarity(s1, s3));
+});
+
+test("lshBands produces BANDS keys and is stable per session", () => {
+  const sig = minhashSignature("some text to band");
+  const k1 = lshBands(sig, "sess_a", 1);
+  const k2 = lshBands(sig, "sess_a", 1);
+  assert.equal(k1.length, BANDS);
+  assert.equal(BANDS * ROWS_PER_BAND, NUM_HASHES);
+  assert.deepEqual(k1, k2);
+  // Different session → different bucket keys (scoped, deterministic).
+  const k3 = lshBands(sig, "sess_b", 1);
+  assert.notDeepEqual(k1, k3);
+});
+
+test("lshBands keys are stable across restarts (no entropy source)", () => {
+  // Recomputed in a fresh call path — determinism is structural, not time-based.
+  const sig = minhashSignature("deterministic bucket key check");
+  const first = lshBands(sig, "sess_x", 1);
+  const again = lshBands(sig, "sess_x", 1);
+  assert.deepEqual(first, again);
+});
+
+test("trigramSimilarity is 1 for identical, high for one-word-edit, low for unrelated", () => {
+  const a = "the quick brown fox jumps";
+  assert.equal(trigramSimilarity(a, a), 1);
+  assert.ok(trigramSimilarity(a, "the quick brown fox jumps over") >= L1_VERIFY_THRESHOLD);
+  assert.ok(trigramSimilarity(a, "a completely different sentence about databases") < 0.5);
+});
+
+test("isNearDuplicate thresholds at 0.85", () => {
+  assert.equal(isNearDuplicate("user fixed the parser bug", "user fixed the parser bug today"), true);
+  assert.equal(isNearDuplicate("alpha beta gamma", "totally different words here"), false);
+});
+
+test("shingles are capped at 50K (complexity guard)", () => {
+  const huge = "x".repeat(200_000);
+  const sh = shingles(huge);
+  assert.ok(sh.length <= 50_000);
+});

package/src/dedup/mmr.ts

@@ -0,0 +1,54 @@
+/**
+ * mmr.ts — Maximal Marginal Relevance reranking for retrieval diversity
+ * (Sprint 12, QA #10).
+ *
+ * After a relevance-ranked candidate list, MMR reorders so we don't inject a
+ * cluster of near-identical checkpoints. Each step picks the candidate that
+ * maximizes `λ·relevance − (1−λ)·maxSimToAlreadySelected`, balancing relevance
+ * against redundancy. λ=0.5 is the default (equal weight).
+ *
+ * Pure function over cosine similarities — no deps, no network (PREVENT-PI-004).
+ */
+
+import type { Vector } from "../embedder.js";
+import { cosineSimilarity } from "../embedder.js";
+
+export const MMR_LAMBDA = 0.5;
+
+export interface MmrItem<T> {
+  item: T;
+  vector: Vector; // embedding used for redundancy scoring
+  relevance: number; // base relevance score (e.g. query cosine)
+}
+
+/**
+ * Rerank `items` by MMR. Returns the items in MMR order, capped at `k`.
+ * `lambda` balances relevance vs diversity (1 = pure relevance, 0 = max diversity).
+ */
+export function mmrRerank<T>(items: MmrItem<T>[], k: number, lambda = MMR_LAMBDA): T[] {
+  if (items.length === 0) return [];
+  const remaining = [...items];
+  const selected: MmrItem<T>[] = [];
+  const cap = Math.min(k, items.length);
+
+  while (selected.length < cap && remaining.length > 0) {
+    let bestIdx = 0;
+    let bestScore = -Infinity;
+    for (let i = 0; i < remaining.length; i++) {
+      const cand = remaining[i];
+      // Max similarity to already-selected (redundancy penalty).
+      let maxSimToSelected = 0;
+      for (const sel of selected) {
+        const sim = cosineSimilarity(cand.vector, sel.vector);
+        if (sim > maxSimToSelected) maxSimToSelected = sim;
+      }
+      const mmr = lambda * cand.relevance - (1 - lambda) * maxSimToSelected;
+      if (mmr > bestScore) {
+        bestScore = mmr;
+        bestIdx = i;
+      }
+    }
+    selected.push(remaining.splice(bestIdx, 1)[0]);
+  }
+  return selected.map((s) => s.item);
+}

package/src/dedup/normalize.ts

@@ -0,0 +1,41 @@
+/**
+ * normalize.ts — text normalization for content-addressable dedup (Sprint 9).
+ *
+ * The L0 dedup key is `sha256(normalize(text))`, so normalization decides which
+ * surface variants collapse to the same checkpoint. Pure, synchronous, no deps.
+ *
+ * Steps (order matters):
+ *   1. strip ANSI escape sequences (terminal color codes leak into tool output)
+ *   2. Unicode NFC (canonical composition — "e" + combining accent == "é")
+ *   3. case-fold (NFKC Cf + toLowerCase) so "Foo"/"foo"/"FOO" collapse (Sprint 10 L0 upgrade)
+ *   4. normalize newlines (CRLF/CR → LF)
+ *   5. collapse runs of whitespace to a single space, trim ends
+ *   6. cap at 32K chars (bounds hashing cost on pathological inputs — QA #7/#15)
+ */
+
+const MAX_CHARS = 32_768;
+
+// ANSI/VT100 escape sequence: ESC (0x1B) [ ...params... [ -/]* final-byte.
+// Built from the code point so the source file contains no literal escape byte.
+const ESC = String.fromCharCode(0x1b);
+const ANSI_RE = new RegExp(ESC + "\\[[0-?]*[ -/]*[@-~]", "g");
+
+/** Strip ANSI/VT100 escape sequences. */
+export function stripAnsi(text: string): string {
+  return text.replace(ANSI_RE, "");
+}
+
+/**
+ * Normalize text to its canonical dedup form. Deterministic and idempotent:
+ * `normalize(normalize(x)) === normalize(x)`.
+ */
+export function normalize(text: string): string {
+  if (!text) return "";
+  let out = stripAnsi(text);
+  out = out.normalize("NFC");
+  out = out.toLocaleLowerCase(); // case-fold so "Foo"/"FOO" collapse to one key
+  out = out.replace(/\r\n?/g, "\n"); // CRLF / CR → LF
+  out = out.replace(/\s+/g, " ").trim();
+  if (out.length > MAX_CHARS) out = out.slice(0, MAX_CHARS);
+  return out;
+}