pi-mega-compact 0.16.3 → 0.17.1

package/dist/config/vector-cortex.js

@@ -0,0 +1,91 @@
+/**
+ * config/vector-cortex.ts — vector-cortex feature flags + breaker/triad constants.
+ *
+ * Every sprint ships one positive `MEGACOMPACT_<SPRINT>` flag, default ON and
+ * `=0`/`_DISABLED` off. Flag-OFF is byte-identical to the predecessor sprint's
+ * behavior (for VC0A: mode C — observer absent, zero evaluation writes).
+ *
+ * The breaker/triad constants (TRIAD_RESILIENCE.md) live here so VC0C consumes
+ * them without re-declaring the ownership boundary. Pi-agnostic, dependency-free.
+ */
+/** Positive sprint flag: `=0` or `_DISABLED=true` disables (default ON). */
+function sprintFlag(name) {
+    const v = process.env[name];
+    if (v === "0" || v === "false")
+        return false;
+    const disabled = process.env[name + "_DISABLED"];
+    if (disabled === "true" || disabled === "1")
+        return false;
+    return true;
+}
+/** VC0A — baseline observability (MetricEventV1 / AnnotationV1). Default ON. */
+export const VC0A_ENABLED = () => sprintFlag("MEGACOMPACT_VC0A");
+/**
+ * VC0B — replay correctness (ReplayCutV2 / ReplayReportV2, M3 effective-cut-v2).
+ * Default ON. `MEGACOMPACT_VC0B=0` disables and is byte-identical to the
+ * predecessor (legacy capped-replay behavior preserved; the v2 cut/replay is
+ * only consulted on the vector-cortex path).
+ */
+export const VC0B_ENABLED = () => sprintFlag("MEGACOMPACT_VC0B");
+/**
+ * VC1A — canonical byte events (EventV2 / EventCodec).
+ * Default ON. `MEGACOMPACT_VC1A=0` disables and is byte-identical to the
+ * predecessor (mode C: ledger absent, current transcript codec unchanged).
+ * The single real consumer is the ledger emit seam (`ledger/emit.ts`): flag OFF
+ * gates zero observability writes.
+ */
+export const VC1A_ENABLED = () => sprintFlag("MEGACOMPACT_VC1A");
+/**
+ * VC1B — occurrence ledger + tool identity + compat journal (LedgerReader/
+ * Writer/Admin, CompatJournalV1, M2 occurrence-v2 migration).
+ * Default ON. `MEGACOMPACT_VC1B=0` disables and is byte-identical to the
+ * predecessor (mode C: the neutral occurrence ledger is not written, no
+ * journal rows, zero `vector_cortex_occurrence_appended` emissions). The real
+ * consumers are the ledger write integration seam and the compat-journal
+ * switch seam.
+ */
+export const VC1B_ENABLED = () => sprintFlag("MEGACOMPACT_VC1B");
+/**
+ * VC0C — live safety envelope (TriadResult / Breaker / KillDecision + durable
+ * spool). Default ON. `MEGACOMPACT_VC0C=0` disables and is byte-identical to
+ * the predecessor (mode C: selected before provider invocation, unchanged host
+ * transcript, breaker/spool idle and emitting nothing). The single real
+ * consumer is the resilience emit seam + the safety adapter's triad selection.
+ */
+export const VC0C_ENABLED = () => sprintFlag("MEGACOMPACT_VC0C");
+/**
+ * VC1C — cross-language conformance v2 (FixtureManifestV2 / DowngradeReport /
+ * MinHashV2 + M4 minhash-v2 migration).
+ * Default ON. `MEGACOMPACT_VC1C=0` disables and is byte-identical to the
+ * predecessor (mode C: a v2 conformance runner that accepts authority fixtures
+ * and the manifest validator idle; the sync dedup scan stays on the v1 path;
+ * zero `vector_cortex_*` VC1C emissions). The real consumers are the conformance
+ * emit seam, the minhash-v2 backfill seam and the downgrade-export seam.
+ */
+export const VC1C_ENABLED = () => sprintFlag("MEGACOMPACT_VC1C");
+// ---------------------------------------------------------------------------
+// Breaker state machine constants (TRIAD_RESILIENCE.md §breaker).
+// Rolled numbers for one 60s window; VC0C consumes these at its breaker seam.
+// ---------------------------------------------------------------------------
+/** Rolling eligibility window (milliseconds). */
+export const BREAKER_WINDOW_MS = 60_000;
+/** Minimum attempts before a breaker may trip or promote. */
+export const BREAKER_MIN_ATTEMPTS = 20;
+/** Performance trip: ≥ this many failures in window, or ≥ this fraction. */
+export const BREAKER_PERF_FAILURES = 5;
+export const BREAKER_PERF_FAILURE_RATE = 0.1;
+/** Correctness trip trips on the first correctness failure. */
+export const BREAKER_CORRECTNESS_FAILURES = 1;
+/** Cooldown before an open breaker may probe (milliseconds). */
+export const BREAKER_COOLDOWN_MS = 30_000;
+/** Consecutive successful probes required to advance a state. */
+export const BREAKER_PROBE_COUNT = 3;
+/** Exponential retry base: 30s * 2^attempt, capped, ±10% jitter. */
+export const BREAKER_RETRY_BASE_MS = 30_000;
+export const BREAKER_RETRY_CAP_MS = 15 * 60_000;
+export const BREAKER_RETRY_JITTER = 0.1;
+/** Promotion hysteresis: failure rate must be < this and p95 within budget. */
+export const BREAKER_HYSTERESIS_FAILURE_RATE = 0.02;
+export const BREAKER_HYSTERESIS_BUDGET_P95_MS = 50;
+/** Minimum healthy residence before a further promotion (milliseconds). */
+export const BREAKER_MIN_HEALTHY_RESIDENCE_MS = 5 * 60_000;

package/dist/dedup/digest.js

@@ -0,0 +1,30 @@
+/**
+ * digest.ts — content-addressable digest (Sprint 9).
+ *
+ * Dual-hash design (QA #2 spirit, local): a primary + a secondary hash guard
+ * against a single-hash collision silently merging distinct content. The L0 dedup
+ * key is `(content_hash, content_hash2)` — both must agree to declare a duplicate.
+ *
+ * `content_hash` is the full 64-hex SHA-256 of the normalized text. `content_hash2`
+ * is a second independent view (SHA-256 of the reversed normalized text) so a
+ * collision on one alone does not dedup. `content_hash_version` lets Sprint 11/12
+ * plug in stronger digests later without breaking old rows.
+ */
+import { createHash } from "node:crypto";
+import { normalize } from "./normalize.js";
+export const CONTENT_HASH_VERSION = 1;
+/** Compute the canonical content digest for a (possibly raw) region text. */
+export function computeContentDigest(text) {
+    const normalizedText = normalize(text);
+    const contentHash = createHash("sha256").update(normalizedText).digest("hex");
+    // Secondary: hash the reversed normalized string so it's an independent view.
+    const contentHash2 = createHash("sha256")
+        .update(normalizedText.split("").reverse().join(""))
+        .digest("hex");
+    return {
+        contentHash,
+        contentHash2,
+        contentHashVersion: CONTENT_HASH_VERSION,
+        normalizedText,
+    };
+}

package/dist/dedup/l1-lsh-v2.js

@@ -0,0 +1,60 @@
+/**
+ * l1-lsh-v2.ts — Locality-Sensitive Hashing banding over MinHashV2 signatures
+ * (VC1C, M4 minhash-v2).
+ *
+ * Splits the frozen 2048-byte v2 signature (256 x u64 LE slots) into 64 bands
+ * of FOUR u64 values each. Each band yields a deterministic bucket key derived
+ * from its 4 u64 little-endian bytes (32 bytes) hashed with a 64-bit FNV-1a,
+ * scoped by session id and the frozen v2 version tag so buckets never collide
+ * across sessions or the v1 path. Deterministic: same (sessionId,
+ * signatureBytes) -> same 64 bucket keys, every run, every language.
+ *
+ * Mixed v1/v2 comparison and bucket mixing is REJECTED upstream with
+ * `MINHASH_VERSION_MISMATCH` (see minhash-v2 migration); this module only ever
+ * bands a v2 signature.
+ *
+ * Pure compute, no deps, no storage, no network (PREVENT-PI-004 / PREVENT-011).
+ */
+import { MINHASH_VERSION, NUM_HASHES_V2, U64_BYTES, } from "./l1-minhash-v2.js";
+/** Number of bands (frozen). */
+export const BANDS_V2 = 64;
+/** u64 slots per band (frozen): 64 * 4 = 256 total slots. */
+export const VALUES_PER_BAND_V2 = 4;
+/** Band byte length: 4 u64 LE = 32 bytes. */
+export const BAND_BYTES_V2 = VALUES_PER_BAND_V2 * U64_BYTES;
+/** Total signature byte length this scheme expects (256 * 8 = 2048). */
+export const SIGNATURE_BYTES_EXPECTED_V2 = NUM_HASHES_V2 * U64_BYTES;
+const MASK64 = 0xffffffffffffffffn;
+/** 64-bit FNV-1a over a byte buffer. */
+function fnv1a64Bytes(buf) {
+    let h = 0xcbf29ce484222325n;
+    for (let i = 0; i < buf.length; i++) {
+        h ^= BigInt(buf[i] ?? 0);
+        h = (h * 0x100000001b3n) & MASK64;
+    }
+    return h;
+}
+/**
+ * Compute the 64 LSH bucket keys for a v2 signature within a session. `bytes`
+ * is the 2048-byte little-endian signature (see encodeSignatureV2). Each bucket
+ * key is the hex of an 8-byte little-endian FNV-1a-64 over the band's 32 bytes
+ * prefixed by `<sessionId>|v2|band-` — so the "bucket bytes" are deterministic
+ * and reproducible across languages.
+ */
+export function lshBandsV2(bytes, sessionId, version = MINHASH_VERSION) {
+    if (bytes.length !== SIGNATURE_BYTES_EXPECTED_V2) {
+        throw new Error(`lshBandsV2: expected ${SIGNATURE_BYTES_EXPECTED_V2} signature bytes, got ${bytes.length}`);
+    }
+    const keys = [];
+    const prefix = Buffer.from(`${sessionId}|v${version}|`, "utf8");
+    for (let band = 0; band < BANDS_V2; band++) {
+        const start = band * BAND_BYTES_V2;
+        const bandBytes = bytes.slice(start, start + BAND_BYTES_V2);
+        const combined = Buffer.concat([prefix, Buffer.from(bandBytes)]);
+        const h = fnv1a64Bytes(combined);
+        const out = Buffer.allocUnsafe(U64_BYTES);
+        out.writeBigUInt64LE(h, 0);
+        keys.push(`b${band}:${out.toString("hex")}`);
+    }
+    return keys;
+}

package/dist/dedup/l1-lsh.js

@@ -0,0 +1,52 @@
+/**
+ * l1-lsh.ts — Locality-Sensitive Hashing banding over MinHash signatures
+ * (Sprint 11).
+ *
+ * Splits the 256-slot signature into `BANDS` bands of `ROWS_PER_BAND` rows. Each
+ * band is hashed into a bucket key; rows sharing a band are likely near-duplicates.
+ * The bucket key INCLUDES the session_id so candidates are scoped per session
+ * (deterministic, no cross-session leakage — QA determinism fix).
+ *
+ * All hashing is deterministic given the signature + seed (PREVENT-PI-004, pure).
+ */
+import { SIGNATURE_VERSION, NUM_HASHES } from "./l1-minhash.js";
+export const BANDS = 64;
+export const ROWS_PER_BAND = 4; // 64 * 4 = 256 slots (matches minhashSignature length)
+/** Stable 32-bit FNV-1a (buffer form) for band hashing. */
+function fnv1aBuf(buf) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < buf.length; i++) {
+        h ^= buf[i];
+        h = Math.imul(h, 0x01000193);
+    }
+    return h >>> 0;
+}
+/**
+ * Compute the list of LSH bucket keys for a signature within a session.
+ * Deterministic: same (session_id, signature) → same keys, every run.
+ */
+export function lshBands(signature, sessionId, version = SIGNATURE_VERSION) {
+    if (signature.length < BANDS * ROWS_PER_BAND) {
+        throw new Error(`lshBands: signature length ${signature.length} < required ${BANDS * ROWS_PER_BAND}`);
+    }
+    const keys = [];
+    const seedPrefix = Buffer.from(`${sessionId}|${version}|`, "utf-8");
+    for (let band = 0; band < BANDS; band++) {
+        const start = band * ROWS_PER_BAND;
+        const slice = signature.slice(start, start + ROWS_PER_BAND);
+        const body = Buffer.allocUnsafe(ROWS_PER_BAND * 4);
+        for (let r = 0; r < ROWS_PER_BAND; r++) {
+            body.writeUInt32LE(slice[r] >>> 0, r * 4);
+        }
+        const combined = Buffer.concat([seedPrefix, body]);
+        keys.push(`b${band}:${fnv1aBuf(combined).toString(16)}`);
+    }
+    return keys;
+}
+/** Convenience: derive bands directly from text (used by callers without a cached sig). */
+export function bandsForText(signature, sessionId, version = SIGNATURE_VERSION) {
+    if (signature.length !== NUM_HASHES) {
+        throw new Error(`bandsForText: expected signature length ${NUM_HASHES}, got ${signature.length}`);
+    }
+    return lshBands(signature, sessionId, version);
+}

package/dist/dedup/l1-minhash-v2.js

@@ -0,0 +1,193 @@
+/**
+ * l1-minhash-v2.ts — MinHash V2 signatures (VC1C, M4 minhash-v2).
+ *
+ * The v1 path (l1-minhash.ts) uses 32-bit universal hashing with `p=2^31-1`
+ * and Number math, which is exact because a*x < 2^62 can overflow double
+ * precision (>= 2^53) and is reduced too eagerly. MinHashV2 freezes a v2
+ * scheme that is CROSS-LANGUAGE deterministic and byte-exact:
+ *
+ *   - normalization: the stable `normalize()` from normalize.ts (NFC, NFKC
+ *     case-fold, CRLF->LF, whitespace collapse, 32K cap);
+ *   - shingles: windows of FIVE CODE POINTS over the normalized text
+ *     (SHINGLE_CP=5), each shingle hashed to a u64 with 64-bit FNV-1a over its
+ *     UTF-8 bytes;
+ *   - 256 PUBLISHED unsigned (a_i, b_i) seed pairs, derived deterministically
+ *     via splitmix64 and listed in
+ *     conformance/vector-cortex/v2/minhash/seeds-v2.json;
+ *   - p = 2^61 - 1 (2305843009213693951), EXACT BigInt multiply/modulo — a_i*x
+ *     can reach ~2^124 so Number math would silently corrupt high-bit products;
+ *   - signature = 256 slots, slot i = min over shingles of (a_i*x + b_i) mod p;
+ *     the empty signature is all-`p` sentinel slots;
+ *   - encoding: 256 x u64 LITTLE-ENDIAN bytes (2048 bytes total);
+ *   - bands: 64 bands of FOUR u64 slots each (256 total), see l1-lsh-v2.ts.
+ *
+ * NEVER compares a v2 signature against a v1 signature — cross-version compare
+ * is rejected with `MINHASH_VERSION_MISMATCH` (see minhash-v2 migration).
+ *
+ * Pure compute, no deps, no storage, no network (PREVENT-PI-004 / PREVENT-011).
+ */
+import { normalize } from "./normalize.js";
+/** MinHashV2 scheme version tag (frozen). */
+export const MINHASH_VERSION = 2;
+/** The published count of slots / seed pairs (frozen). */
+export const NUM_HASHES_V2 = 256;
+/** Five-code-point shingle length (frozen). */
+export const SHINGLE_CP = 5;
+/** p = 2^61 - 1 (Mersenne prime), as exact BigInt. */
+export const P_V2 = (1n << 61n) - 1n;
+/** Little-endian bytes per u64 signature slot. */
+export const U64_BYTES = 8;
+/** Signature byte length: 256 slots x 8 LE bytes. */
+export const SIGNATURE_BYTES_V2 = NUM_HASHES_V2 * U64_BYTES;
+/** Fixed splitmix64 initial state so the seed table is reproducible everywhere. */
+export const SEED_STATE_INIT = 0x5eedc0def001ba11n & 0xffffffffffffffffn;
+/** splitmix64 mixing constants (frozen). */
+const SM_GOLDEN = 0x9e3779b97f4a7c15n;
+const SM_M1 = 0xbf58476d1ce4e5b9n;
+const SM_M2 = 0x94d049bb133111ebn;
+const MASK64 = 0xffffffffffffffffn;
+function mix(z0) {
+    let z = z0;
+    z = (z ^ (z >> 30n)) * SM_M1;
+    z &= MASK64;
+    z = (z ^ (z >> 27n)) * SM_M2;
+    z &= MASK64;
+    return z ^ (z >> 31n);
+}
+/** Create a splitmix64 PRNG from a fixed unsigned 64-bit state. */
+export function splitmix64(state = SEED_STATE_INIT) {
+    let s = state & MASK64;
+    return () => {
+        s = (s + SM_GOLDEN) & MASK64;
+        return mix(s) & MASK64;
+    };
+}
+/**
+ * Derive the frozen 256 published (a_i, b_i) seed pairs. Each pair maps two
+ * splitmix64 draws into [1, p-1]. Deterministic and cross-language reproducible
+ * from SEED_STATE_INIT — mirrors `seeds-v2.json`.
+ *
+ * The table is frozen: computed ONCE at module load and reused for every call.
+ * Recomputing the 256 splitmix64 pairs (and re-walking all shingles) per
+ * signature/backfill row is wasteful for a large v1 index backfill, and the
+ * table is immutable by construction, so a single shared instance is safe and
+ * byte-identical. The returned sub-arrays share the frozen pair objects; they
+ * are themselves frozen so no caller can mutate the shared cache.
+ */
+export function minhashV2Seeds(count = NUM_HASHES_V2) {
+    return FROZEN_SEEDS.slice(0, count);
+}
+/** Singleton seed table, computed exactly once and frozen (see minhashV2Seeds). */
+const FROZEN_SEEDS = (() => {
+    const prng = splitmix64();
+    const pairs = [];
+    for (let i = 0; i < NUM_HASHES_V2; i++) {
+        const a = (prng() % (P_V2 - 1n)) + 1n;
+        const b = (prng() % (P_V2 - 1n)) + 1n;
+        pairs.push(Object.freeze({ a, b }));
+    }
+    return Object.freeze(pairs);
+})();
+/** 64-bit FNV-1a over a UTF-8 string (for shingle hashing). */
+function fnv1a64String(s) {
+    let h = 0xcbf29ce484222325n;
+    const bytes = Buffer.from(s, "utf8");
+    for (let i = 0; i < bytes.length; i++) {
+        h ^= BigInt(bytes[i] ?? 0);
+        h = (h * 0x100000001b3n) & MASK64;
+    }
+    return h;
+}
+/**
+ * Five-code-point shingle set (deduped, capped) of normalized text as u64
+ * hashes. Iterates over code points (not UTF-16 code units) so astral-plane
+ * characters do not split a shingle.
+ */
+export function shinglesV2(text, shingleCp = SHINGLE_CP, maxShingles = 50_000) {
+    const norm = normalize(text);
+    if (norm.length === 0)
+        return [];
+    const cps = Array.from(norm); // code points
+    const set = new Set();
+    if (cps.length < shingleCp) {
+        set.add(fnv1a64String(norm));
+    }
+    else {
+        for (let i = 0; i + shingleCp <= cps.length; i++) {
+            const sh = cps.slice(i, i + shingleCp).join("");
+            set.add(fnv1a64String(sh));
+            if (set.size >= maxShingles)
+                break;
+        }
+    }
+    return [...set];
+}
+/**
+ * Compute the 256-slot MinHashV2 signature of a text as an array of exact u64
+ * BigInt values (each in [0, p], p = sentinel when empty). Uses EXACT BigInt
+ * multiply/modulo so high-bit products (a*x ~ up to 2^124) are never corrupted.
+ */
+export function minhashV2Signature(text) {
+    const grams = shinglesV2(text);
+    const seeds = minhashV2Seeds();
+    const sig = new Array(NUM_HASHES_V2).fill(P_V2);
+    if (grams.length === 0)
+        return sig;
+    for (let i = 0; i < NUM_HASHES_V2; i++) {
+        const { a, b } = seeds[i];
+        let min = P_V2;
+        for (const x of grams) {
+            const h = (a * x + b) % P_V2;
+            if (h < min)
+                min = h;
+        }
+        sig[i] = min;
+    }
+    return sig;
+}
+/** Encode a 256-slot signature to 2048 bytes as 256 x u64 little-endian. */
+export function encodeSignatureV2(sig) {
+    if (sig.length !== NUM_HASHES_V2) {
+        throw new Error(`encodeSignatureV2: expected ${NUM_HASHES_V2} slots, got ${sig.length}`);
+    }
+    const buf = Buffer.allocUnsafe(SIGNATURE_BYTES_V2);
+    for (let i = 0; i < NUM_HASHES_V2; i++) {
+        buf.writeBigUInt64LE(sig[i], i * U64_BYTES);
+    }
+    return new Uint8Array(buf);
+}
+/**
+ * Estimated Jaccard similarity of two v2 signatures (fraction of equal slots).
+ *
+ * THIS IS v2-ONLY. Cross-version compare is rejected INTERNALLY with
+ * `MINHASH_VERSION_MISMATCH`: both inputs must be exact 256-slot v2 signatures,
+ * i.e. every slot a `bigint` in [0, P_V2] (the empty sentinel is exactly P_V2).
+ * A v1 signature is a `number[]` of u31 values, so passing a v1 array fails the
+ * bigint type/range check and is rejected — a mixed v1/v2 compare can never
+ * produce a similarity number. Never call this with v1 arrays.
+ */
+export function signatureSimilarityV2(a, b) {
+    assertV2Signature(a);
+    assertV2Signature(b);
+    const n = Math.min(a.length, b.length);
+    if (n === 0)
+        return 0;
+    let equal = 0;
+    for (let i = 0; i < n; i++)
+        if (a[i] === b[i])
+            equal++;
+    return equal / n;
+}
+/** Reject any array that is not an exact 256-slot v2 signature (bigint in [0, P_V2]). */
+function assertV2Signature(sig) {
+    if (sig.length !== NUM_HASHES_V2) {
+        throw new Error(MINHASH_VERSION_MISMATCH_MSG);
+    }
+    for (const s of sig) {
+        if (typeof s !== "bigint" || s < 0n || s > P_V2) {
+            throw new Error(MINHASH_VERSION_MISMATCH_MSG);
+        }
+    }
+}
+/** Frozen cross-version-reject message shared by the v2-only function guards. */
+const MINHASH_VERSION_MISMATCH_MSG = "MINHASH_VERSION_MISMATCH";

package/dist/dedup/l1-minhash.js

@@ -0,0 +1,91 @@
+/**
+ * l1-minhash.ts — MinHash signatures for L1 near-duplicate detection (Sprint 11).
+ *
+ * Uses UNIVERSAL HASHING (QA #3), not the broken permutation scheme from the
+ * generic dedup plan. Each of the 256 hash functions is `h_i(x) = (a_i·x + b_i)
+ * mod p` with a fixed prime `p` and per-index coefficients derived from a pinned
+ * seed (0xDEADBEEF). This makes signatures DETERMINISTIC across process restarts
+ * (a hard requirement — non-determinism silently breaks dedup).
+ *
+ * `signatureVersion` lets Sprint 12+ swap the scheme without invalidating stored
+ * signatures: old buckets keep their version, new ones get the new one.
+ *
+ * Pure compute, no deps, no network (PREVENT-PI-004).
+ */
+import { normalize } from "./normalize.js";
+export const SIGNATURE_VERSION = 1;
+export const NUM_HASHES = 256;
+export const SHINGLE_SIZE = 5; // char 5-grams
+const MAX_SHINGLES = 50_000; // QA #7/#15 complexity cap
+const SEED = 0xdeadbeef;
+const P = 2147483647; // 2^31 - 1, Mersenne prime
+/** Per-index universal-hashing coefficients, derived deterministically from SEED. */
+function coeffA(i) {
+    return (SEED + i * 2 + 1) % P;
+}
+function coeffB(i) {
+    return (SEED * 3 + i * 7 + 13) % P;
+}
+/** Stable 32-bit FNV-1a of a shingle (the `x` fed to the universal hashes). */
+function shingleHash(gram) {
+    let h = 0x811c9dc5;
+    for (let k = 0; k < gram.length; k++) {
+        h ^= gram.charCodeAt(k);
+        h = Math.imul(h, 0x01000193);
+    }
+    return h >>> 0;
+}
+/** Char n-gram shingle set (deduped) of normalized text, capped at MAX_SHINGLES. */
+export function shingles(text, size = SHINGLE_SIZE) {
+    const norm = normalize(text);
+    if (norm.length === 0)
+        return [];
+    const set = new Set();
+    if (norm.length < size) {
+        set.add(shingleHash(norm));
+    }
+    else {
+        for (let i = 0; i + size <= norm.length; i++) {
+            set.add(shingleHash(norm.slice(i, i + size)));
+            if (set.size >= MAX_SHINGLES)
+                break;
+        }
+    }
+    return [...set];
+}
+/**
+ * Compute the 256-element MinHash signature of a text. Each slot is the minimum,
+ * over all shingles, of the i-th universal hash. Empty text → all-P sentinel.
+ */
+export function minhashSignature(text) {
+    const grams = shingles(text);
+    const sig = new Array(NUM_HASHES).fill(P);
+    if (grams.length === 0)
+        return sig;
+    for (let i = 0; i < NUM_HASHES; i++) {
+        const a = coeffA(i);
+        const b = coeffB(i);
+        let min = P;
+        for (const x of grams) {
+            // (a*x + b) mod p — use Number math; a,x < 2^31 so a*x < 2^62, within
+            // double-precision integer range (2^53) only if reduced; reduce a*x first.
+            const ax = (a * (x % P)) % P;
+            const h = (ax + b) % P;
+            if (h < min)
+                min = h;
+        }
+        sig[i] = min;
+    }
+    return sig;
+}
+/** Estimated Jaccard similarity of two signatures (fraction of equal slots). */
+export function signatureSimilarity(a, b) {
+    const n = Math.min(a.length, b.length);
+    if (n === 0)
+        return 0;
+    let equal = 0;
+    for (let i = 0; i < n; i++)
+        if (a[i] === b[i])
+            equal++;
+    return equal / n;
+}

package/dist/dedup/l1-verify.js

@@ -0,0 +1,54 @@
+/**
+ * l1-verify.ts — trigram-similarity verification gate for L1 near-duplicates
+ * (Sprint 11).
+ *
+ * After LSH cheaply retrieves candidate chunk_ids, we apply a REAL similarity
+ * check before declaring a duplicate. This is the `pg_trgm`-equivalent final
+ * gate: we compute the trigram (character 3-gram) Jaccard / overlap similarity
+ * between the new normalized text and each candidate's normalized text.
+ *
+ * We compute it in TS (not via FTS5 MATCH, which is boolean) so we get a stable
+ * [0,1] score to threshold against (0.85 per spec). The FTS5 `trigram` table is
+ * still maintained for future query-path use, but verification is pure TS so its
+ * result is deterministic and unit-testable (PREVENT-PI-004).
+ */
+import { normalize } from "./normalize.js";
+const TRIGRAM_SIZE = 3;
+export const L1_VERIFY_THRESHOLD = 0.85;
+/** Extract the set of character trigrams from normalized text. */
+function trigrams(text) {
+    const norm = normalize(text);
+    const set = new Set();
+    if (norm.length === 0)
+        return set;
+    if (norm.length < TRIGRAM_SIZE) {
+        set.add(norm);
+        return set;
+    }
+    for (let i = 0; i + TRIGRAM_SIZE <= norm.length; i++) {
+        set.add(norm.slice(i, i + TRIGRAM_SIZE));
+    }
+    return set;
+}
+/**
+ * Trigram similarity in [0,1]. We use overlap coefficient (|A∩B| / min(|A|,|B|))
+ * which, like pg_trgm's `similarity`, is robust when one text is a substring of
+ * the other — better than Jaccard for the near-dup "one-word edit" case.
+ */
+export function trigramSimilarity(a, b) {
+    const ta = trigrams(a);
+    const tb = trigrams(b);
+    if (ta.size === 0 || tb.size === 0)
+        return 0;
+    let inter = 0;
+    const smaller = ta.size <= tb.size ? ta : tb;
+    const larger = smaller === ta ? tb : ta;
+    for (const g of smaller)
+        if (larger.has(g))
+            inter++;
+    return inter / smaller.size;
+}
+/** True when `a` and `b` are near-duplicates under the L1 threshold. */
+export function isNearDuplicate(a, b, threshold = L1_VERIFY_THRESHOLD) {
+    return trigramSimilarity(a, b) >= threshold;
+}

package/dist/dedup/mmr.js

@@ -0,0 +1,45 @@
+/**
+ * mmr.ts — Maximal Marginal Relevance reranking for retrieval diversity
+ * (Sprint 12, QA #10).
+ *
+ * After a relevance-ranked candidate list, MMR reorders so we don't inject a
+ * cluster of near-identical checkpoints. Each step picks the candidate that
+ * maximizes `λ·relevance − (1−λ)·maxSimToAlreadySelected`, balancing relevance
+ * against redundancy. λ=0.5 is the default (equal weight).
+ *
+ * Pure function over cosine similarities — no deps, no network (PREVENT-PI-004).
+ */
+import { cosineSimilarity } from "../embedder.js";
+export const MMR_LAMBDA = 0.5;
+/**
+ * Rerank `items` by MMR. Returns the items in MMR order, capped at `k`.
+ * `lambda` balances relevance vs diversity (1 = pure relevance, 0 = max diversity).
+ */
+export function mmrRerank(items, k, lambda = MMR_LAMBDA) {
+    if (items.length === 0)
+        return [];
+    const remaining = [...items];
+    const selected = [];
+    const cap = Math.min(k, items.length);
+    while (selected.length < cap && remaining.length > 0) {
+        let bestIdx = 0;
+        let bestScore = -Infinity;
+        for (let i = 0; i < remaining.length; i++) {
+            const cand = remaining[i];
+            // Max similarity to already-selected (redundancy penalty).
+            let maxSimToSelected = 0;
+            for (const sel of selected) {
+                const sim = cosineSimilarity(cand.vector, sel.vector);
+                if (sim > maxSimToSelected)
+                    maxSimToSelected = sim;
+            }
+            const mmr = lambda * cand.relevance - (1 - lambda) * maxSimToSelected;
+            if (mmr > bestScore) {
+                bestScore = mmr;
+                bestIdx = i;
+            }
+        }
+        selected.push(remaining.splice(bestIdx, 1)[0]);
+    }
+    return selected.map((s) => s.item);
+}

package/dist/dedup/normalize.js

@@ -0,0 +1,39 @@
+/**
+ * normalize.ts — text normalization for content-addressable dedup (Sprint 9).
+ *
+ * The L0 dedup key is `sha256(normalize(text))`, so normalization decides which
+ * surface variants collapse to the same checkpoint. Pure, synchronous, no deps.
+ *
+ * Steps (order matters):
+ *   1. strip ANSI escape sequences (terminal color codes leak into tool output)
+ *   2. Unicode NFC (canonical composition — "e" + combining accent == "é")
+ *   3. case-fold (NFKC Cf + toLowerCase) so "Foo"/"foo"/"FOO" collapse (Sprint 10 L0 upgrade)
+ *   4. normalize newlines (CRLF/CR → LF)
+ *   5. collapse runs of whitespace to a single space, trim ends
+ *   6. cap at 32K chars (bounds hashing cost on pathological inputs — QA #7/#15)
+ */
+const MAX_CHARS = 32_768;
+// ANSI/VT100 escape sequence: ESC (0x1B) [ ...params... [ -/]* final-byte.
+// Built from the code point so the source file contains no literal escape byte.
+const ESC = String.fromCharCode(0x1b);
+const ANSI_RE = new RegExp(ESC + "\\[[0-?]*[ -/]*[@-~]", "g");
+/** Strip ANSI/VT100 escape sequences. */
+export function stripAnsi(text) {
+    return text.replace(ANSI_RE, "");
+}
+/**
+ * Normalize text to its canonical dedup form. Deterministic and idempotent:
+ * `normalize(normalize(x)) === normalize(x)`.
+ */
+export function normalize(text) {
+    if (!text)
+        return "";
+    let out = stripAnsi(text);
+    out = out.normalize("NFC");
+    out = out.toLowerCase(); // case-fold so "Foo"/"FOO" collapse to one key (locale-independent: byte-exact across languages)
+    out = out.replace(/\r\n?/g, "\n"); // CRLF / CR → LF
+    out = out.replace(/\s+/g, " ").trim();
+    if (out.length > MAX_CHARS)
+        out = out.slice(0, MAX_CHARS);
+    return out;
+}