pi-mega-compact 0.4.5 → 0.4.7

package/dist/src/compact.test.js

@@ -0,0 +1,78 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { formatCompactSummary, summarizeMessages, mergeCompactSummaries, shouldCompact, autoCompactCheck, collectKeyFiles, inferPendingWork, extractFileCandidates, } from "./compact.js";
+import { estimateSessionTokens } from "./tokens.js";
+function user(text) { return { role: "user", text }; }
+function assistant(text) { return { role: "assistant", text }; }
+function toolUse(name, input) { return { role: "assistant", text: "", toolName: name, input }; }
+function toolResult(name, output) { return { role: "tool", text: "", toolName: name, output }; }
+test("formatCompactSummary strips analysis and formats summary block (claw-code parity)", () => {
+    const summary = "<analysis>scratch</analysis>\n<summary>Kept work</summary>";
+    assert.equal(formatCompactSummary(summary), "Summary:\nKept work");
+});
+test("leaves small sessions unchanged (shouldCompact false)", () => {
+    const messages = [user("hello")];
+    assert.equal(shouldCompact(messages, 1, 4), false);
+});
+test("compacts older messages into a summary with Scope + timeline", () => {
+    const messages = [
+        user("one ".repeat(200)),
+        assistant("two ".repeat(200)),
+        toolResult("bash", "ok ".repeat(200)),
+        assistant("recent"),
+    ];
+    assert.equal(shouldCompact(messages, 1, 2), true);
+    const summary = summarizeMessages(messages.slice(0, 2));
+    const formatted = formatCompactSummary(summary);
+    assert.ok(formatted.includes("Scope:"));
+    assert.ok(formatted.includes("Key timeline:"));
+});
+test("merge keeps previous compacted context when compacting again", () => {
+    const first = summarizeMessages([
+        user("Investigate src/compact.ts"),
+        assistant("I will inspect the compact flow."),
+    ]);
+    const second = summarizeMessages([
+        user("Also update src/boundary.ts"),
+        assistant("Next: preserve prior summary context."),
+    ]);
+    const merged = mergeCompactSummaries(first, second);
+    assert.ok(merged.includes("Previously compacted context:"));
+    assert.ok(merged.includes("Newly compacted context:"));
+    assert.ok(merged.includes("src/boundary.ts"));
+});
+test("infers pending work from recent messages", () => {
+    const pending = inferPendingWork([
+        user("done"),
+        assistant("Next: update tests and follow up on remaining CLI polish."),
+    ]);
+    assert.equal(pending.length, 1);
+    assert.ok(pending[0].includes("Next: update tests"));
+});
+test("extracts key files from message content", () => {
+    const files = collectKeyFiles([
+        user("Update src/compact.ts and extensions/mega-compact.ts next."),
+    ]);
+    assert.ok(files.includes("src/compact.ts"));
+    assert.ok(files.includes("extensions/mega-compact.ts"));
+});
+test("extractFileCandidates ignores plain words and non-interesting extensions", () => {
+    const files = extractFileCandidates("look at foo/bar.png and src/x.ts and justaword");
+    assert.deepEqual(files, ["src/x.ts"]);
+});
+test("summarizeMessages lists tool names sorted + deduped", () => {
+    const summary = summarizeMessages([toolUse("search", "{}"), toolUse("bash", "{}"), toolResult("search", "ok")]);
+    assert.ok(summary.includes("Tools mentioned: bash, search."));
+});
+test("autoCompactCheck reports utilization and threshold gate", () => {
+    const under = autoCompactCheck(10000, 50000);
+    assert.equal(under.shouldCompact, false);
+    assert.equal(under.utilizationPct, 20);
+    const over = autoCompactCheck(60000, 50000);
+    assert.equal(over.shouldCompact, true);
+});
+test("token estimator counts text + tool payloads", () => {
+    const total = estimateSessionTokens([user("abcd"), toolResult("bash", "abcd")]);
+    // "abcd"/4+1 = 2 for user; tool: name "bash"/4+1=2 plus output "abcd"/4+1=2 => 4
+    assert.equal(total, 2 + 4);
+});

package/dist/src/config/dedup.js

@@ -0,0 +1,81 @@
+/**
+ * config/dedup.ts — SINGLE SOURCE OF TRUTH for dedup tier flags + thresholds
+ * (Sprint 14, Phase 7).
+ *
+ * Every tier flag and threshold that was previously an inline default in
+ * vectorStore.ts / the RAPTOR modules is defined HERE and only here (QA #8: no
+ * duplicated threshold across modules). Values are read from MEGACOMPACT_* env
+ * at load, with the file defaults below as the fallback (which reproduce the
+ * Sprint 13 behavior — all tiers active, nothing MARK_ONLY).
+ *
+ * MARK_ONLY semantics (QA ops): a tier in MARK_ONLY still RUNS and RECORDS its
+ * decision (so we keep the data + can replay), but does NOT collapse/dedup — a
+ * safe partial-rollout / auto-degrade state.
+ *
+ * PREVENT-PI-004: pure config, no network. Booleans/numbers only.
+ */
+function envBool(name, def) {
+    const v = process.env[name];
+    if (v === undefined)
+        return def;
+    return v === "true" || v === "1";
+}
+function envNum(name, def) {
+    const v = process.env[name];
+    if (v === undefined)
+        return def;
+    const n = Number(v);
+    return Number.isFinite(n) ? n : def;
+}
+/** Read the current dedup config from env (file defaults reproduce Sprint 13). */
+export function loadDedupConfig() {
+    return {
+        L0_ENABLED: envBool("MEGACOMPACT_L0_ENABLED", true),
+        L1_ENABLED: envBool("MEGACOMPACT_L1_ENABLED", true),
+        L2_ENABLED: envBool("MEGACOMPACT_L2_ENABLED", true),
+        RAPTOR_ENABLED: envBool("MEGACOMPACT_RAPTOR_ENABLED", false), // shadow by default
+        MARK_ONLY_L0: envBool("MEGACOMPACT_MARK_ONLY_L0", false),
+        MARK_ONLY_L1: envBool("MEGACOMPACT_MARK_ONLY_L1", false),
+        MARK_ONLY_L2: envBool("MEGACOMPACT_MARK_ONLY_L2", false),
+        MINILM_EMBEDDER: envBool("MEGACOMPACT_MINILM", false),
+        L2_COSINE: envNum("MEGACOMPACT_L2_THRESHOLD", 0.85),
+        L1_JACCARD: envNum("MEGACOMPACT_L1_JACCARD", 0.8),
+        DEDUP_SIM: envNum("MEGACOMPACT_DEDUP_SIM", 0.9),
+        MMR_LAMBDA: envNum("MEGACOMPACT_MMR_LAMBDA", 0.5),
+        SEMDEDUP_COSINE: envNum("MEGACOMPACT_SEMDEDUP_COSINE", 0.95),
+        SIMILARITY_BUDGET_MS: envNum("MEGACOMPACT_SIMILARITY_BUDGET_MS", 50),
+        L1_VERIFY_BUDGET_MS: envNum("MEGACOMPACT_L1_VERIFY_BUDGET_MS", 20),
+        L1_CANDIDATE_CAP: envNum("MEGACOMPACT_L1_CANDIDATE_CAP", 100),
+        RAPTOR_BUDGET_MS: envNum("MEGACOMPACT_RAPTOR_BUDGET_MS", 5000),
+        RAPTOR_CLUSTERS_PER_LEVEL: envNum("MEGACOMPACT_RAPTOR_CLUSTERS", 5),
+        RAPTOR_CONSISTENCY: envNum("MEGACOMPACT_RAPTOR_CONSISTENCY", 0.6),
+        FP_RATE_L0: envNum("MEGACOMPACT_FP_RATE_L0", 0.01),
+        FP_RATE_L1L2: envNum("MEGACOMPACT_FP_RATE_L1L2", 0.05),
+        ALERT_WINDOW_MS: envNum("MEGACOMPACT_ALERT_WINDOW_MS", 600_000),
+        P95_BUDGET_MS: envNum("MEGACOMPACT_P95_BUDGET_MS", 100),
+    };
+}
+/**
+ * The default config snapshot (read once at import). Callers that need to honor
+ * runtime env changes (tests) should call loadDedupConfig() directly; the live
+ * add()/search() path reads this snapshot but accepts an override for testing.
+ */
+export const DedupConfig = loadDedupConfig();
+/** Is a given tier enabled (and not merely MARK_ONLY)? */
+export function tierEnabled(cfg, tier) {
+    switch (tier) {
+        case "L0": return cfg.L0_ENABLED;
+        case "L1": return cfg.L1_ENABLED;
+        case "L2": return cfg.L2_ENABLED;
+        case "RAPTOR": return cfg.RAPTOR_ENABLED;
+    }
+}
+/** Is a given tier in MARK_ONLY (record, don't collapse)? */
+export function tierMarkOnly(cfg, tier) {
+    switch (tier) {
+        case "L0": return cfg.MARK_ONLY_L0;
+        case "L1": return cfg.MARK_ONLY_L1;
+        case "L2": return cfg.MARK_ONLY_L2;
+        case "RAPTOR": return false; // RAPTOR has its own shadow mode
+    }
+}

package/dist/src/config.js

@@ -0,0 +1,12 @@
+/**
+ * config.ts — shared default paths/constants for the mega-compact engine.
+ *
+ * Kept tiny and dependency-free so both the extension entry and unit tests can
+ * import it without pulling in pi runtime types.
+ */
+import { join } from "node:path";
+import { homedir } from "node:os";
+/** Default on-disk location for checkpoints + session state. */
+export const STATE_DIR_DEFAULT = join(homedir(), ".pi", "agent", "extensions", "pi-mega-compact");
+/** Pi custom message / entry type used as the dedup sentinel. */
+export const MARKER_TYPE = "mega-compact-marker";

package/dist/src/dedup/dedup.test.js

@@ -0,0 +1,41 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { normalize, stripAnsi } from "./normalize.js";
+import { computeContentDigest, CONTENT_HASH_VERSION } from "./digest.js";
+test("normalize collapses whitespace/newline variants to one form (Sprint 9)", () => {
+    // Sprint 9 normalizes whitespace/newlines/ANSI (not case — that is Sprint 10).
+    const variants = [
+        "foo  bar",
+        "foo bar",
+        "foo\tbar",
+        "foo\nbar",
+        "  foo   bar  ",
+        "foo\r\nbar",
+    ];
+    const digests = variants.map((v) => computeContentDigest(v).contentHash);
+    const unique = new Set(digests);
+    assert.equal(unique.size, 1, "all whitespace/newline variants must collapse to one digest");
+});
+test("normalize is idempotent", () => {
+    const input = "  Hello   World  \n";
+    assert.equal(normalize(normalize(input)), normalize(input));
+});
+test("stripAnsi removes terminal color codes", () => {
+    const colored = "err\x1b[31m fatal\x1b[0m boom";
+    assert.equal(stripAnsi(colored), "err fatal boom");
+});
+test("computeContentDigest emits full 64-hex dual hashes + version", () => {
+    const d = computeContentDigest("the same region text");
+    assert.equal(d.contentHash.length, 64);
+    assert.equal(d.contentHash2.length, 64);
+    assert.equal(d.contentHashVersion, CONTENT_HASH_VERSION);
+    assert.equal(d.normalizedText, "the same region text");
+    // Secondary is an independent view (reversed) so it differs from primary.
+    assert.notEqual(d.contentHash, d.contentHash2);
+});
+test("dual-hash: distinct content yields a distinct pair (both must agree to dedup)", () => {
+    const a = computeContentDigest("region about authentication");
+    const b = computeContentDigest("region about authorization");
+    assert.notEqual(a.contentHash, b.contentHash);
+    assert.notEqual(a.contentHash2, b.contentHash2);
+});

package/dist/src/dedup/digest.js

@@ -0,0 +1,30 @@
+/**
+ * digest.ts — content-addressable digest (Sprint 9).
+ *
+ * Dual-hash design (QA #2 spirit, local): a primary + a secondary hash guard
+ * against a single-hash collision silently merging distinct content. The L0 dedup
+ * key is `(content_hash, content_hash2)` — both must agree to declare a duplicate.
+ *
+ * `content_hash` is the full 64-hex SHA-256 of the normalized text. `content_hash2`
+ * is a second independent view (SHA-256 of the reversed normalized text) so a
+ * collision on one alone does not dedup. `content_hash_version` lets Sprint 11/12
+ * plug in stronger digests later without breaking old rows.
+ */
+import { createHash } from "node:crypto";
+import { normalize } from "./normalize.js";
+export const CONTENT_HASH_VERSION = 1;
+/** Compute the canonical content digest for a (possibly raw) region text. */
+export function computeContentDigest(text) {
+    const normalizedText = normalize(text);
+    const contentHash = createHash("sha256").update(normalizedText).digest("hex");
+    // Secondary: hash the reversed normalized string so it's an independent view.
+    const contentHash2 = createHash("sha256")
+        .update(normalizedText.split("").reverse().join(""))
+        .digest("hex");
+    return {
+        contentHash,
+        contentHash2,
+        contentHashVersion: CONTENT_HASH_VERSION,
+        normalizedText,
+    };
+}

package/dist/src/dedup/l1-lsh.js

@@ -0,0 +1,52 @@
+/**
+ * l1-lsh.ts — Locality-Sensitive Hashing banding over MinHash signatures
+ * (Sprint 11).
+ *
+ * Splits the 256-slot signature into `BANDS` bands of `ROWS_PER_BAND` rows. Each
+ * band is hashed into a bucket key; rows sharing a band are likely near-duplicates.
+ * The bucket key INCLUDES the session_id so candidates are scoped per session
+ * (deterministic, no cross-session leakage — QA determinism fix).
+ *
+ * All hashing is deterministic given the signature + seed (PREVENT-PI-004, pure).
+ */
+import { SIGNATURE_VERSION, NUM_HASHES } from "./l1-minhash.js";
+export const BANDS = 64;
+export const ROWS_PER_BAND = 4; // 64 * 4 = 256 slots (matches minhashSignature length)
+/** Stable 32-bit FNV-1a (buffer form) for band hashing. */
+function fnv1aBuf(buf) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < buf.length; i++) {
+        h ^= buf[i];
+        h = Math.imul(h, 0x01000193);
+    }
+    return h >>> 0;
+}
+/**
+ * Compute the list of LSH bucket keys for a signature within a session.
+ * Deterministic: same (session_id, signature) → same keys, every run.
+ */
+export function lshBands(signature, sessionId, version = SIGNATURE_VERSION) {
+    if (signature.length < BANDS * ROWS_PER_BAND) {
+        throw new Error(`lshBands: signature length ${signature.length} < required ${BANDS * ROWS_PER_BAND}`);
+    }
+    const keys = [];
+    const seedPrefix = Buffer.from(`${sessionId}|${version}|`, "utf-8");
+    for (let band = 0; band < BANDS; band++) {
+        const start = band * ROWS_PER_BAND;
+        const slice = signature.slice(start, start + ROWS_PER_BAND);
+        const body = Buffer.allocUnsafe(ROWS_PER_BAND * 4);
+        for (let r = 0; r < ROWS_PER_BAND; r++) {
+            body.writeUInt32LE(slice[r] >>> 0, r * 4);
+        }
+        const combined = Buffer.concat([seedPrefix, body]);
+        keys.push(`b${band}:${fnv1aBuf(combined).toString(16)}`);
+    }
+    return keys;
+}
+/** Convenience: derive bands directly from text (used by callers without a cached sig). */
+export function bandsForText(signature, sessionId, version = SIGNATURE_VERSION) {
+    if (signature.length !== NUM_HASHES) {
+        throw new Error(`bandsForText: expected signature length ${NUM_HASHES}, got ${signature.length}`);
+    }
+    return lshBands(signature, sessionId, version);
+}

package/dist/src/dedup/l1-minhash.js

@@ -0,0 +1,91 @@
+/**
+ * l1-minhash.ts — MinHash signatures for L1 near-duplicate detection (Sprint 11).
+ *
+ * Uses UNIVERSAL HASHING (QA #3), not the broken permutation scheme from the
+ * generic dedup plan. Each of the 256 hash functions is `h_i(x) = (a_i·x + b_i)
+ * mod p` with a fixed prime `p` and per-index coefficients derived from a pinned
+ * seed (0xDEADBEEF). This makes signatures DETERMINISTIC across process restarts
+ * (a hard requirement — non-determinism silently breaks dedup).
+ *
+ * `signatureVersion` lets Sprint 12+ swap the scheme without invalidating stored
+ * signatures: old buckets keep their version, new ones get the new one.
+ *
+ * Pure compute, no deps, no network (PREVENT-PI-004).
+ */
+import { normalize } from "./normalize.js";
+export const SIGNATURE_VERSION = 1;
+export const NUM_HASHES = 256;
+export const SHINGLE_SIZE = 5; // char 5-grams
+const MAX_SHINGLES = 50_000; // QA #7/#15 complexity cap
+const SEED = 0xdeadbeef;
+const P = 2147483647; // 2^31 - 1, Mersenne prime
+/** Per-index universal-hashing coefficients, derived deterministically from SEED. */
+function coeffA(i) {
+    return (SEED + i * 2 + 1) % P;
+}
+function coeffB(i) {
+    return (SEED * 3 + i * 7 + 13) % P;
+}
+/** Stable 32-bit FNV-1a of a shingle (the `x` fed to the universal hashes). */
+function shingleHash(gram) {
+    let h = 0x811c9dc5;
+    for (let k = 0; k < gram.length; k++) {
+        h ^= gram.charCodeAt(k);
+        h = Math.imul(h, 0x01000193);
+    }
+    return h >>> 0;
+}
+/** Char n-gram shingle set (deduped) of normalized text, capped at MAX_SHINGLES. */
+export function shingles(text, size = SHINGLE_SIZE) {
+    const norm = normalize(text);
+    if (norm.length === 0)
+        return [];
+    const set = new Set();
+    if (norm.length < size) {
+        set.add(shingleHash(norm));
+    }
+    else {
+        for (let i = 0; i + size <= norm.length; i++) {
+            set.add(shingleHash(norm.slice(i, i + size)));
+            if (set.size >= MAX_SHINGLES)
+                break;
+        }
+    }
+    return [...set];
+}
+/**
+ * Compute the 256-element MinHash signature of a text. Each slot is the minimum,
+ * over all shingles, of the i-th universal hash. Empty text → all-P sentinel.
+ */
+export function minhashSignature(text) {
+    const grams = shingles(text);
+    const sig = new Array(NUM_HASHES).fill(P);
+    if (grams.length === 0)
+        return sig;
+    for (let i = 0; i < NUM_HASHES; i++) {
+        const a = coeffA(i);
+        const b = coeffB(i);
+        let min = P;
+        for (const x of grams) {
+            // (a*x + b) mod p — use Number math; a,x < 2^31 so a*x < 2^62, within
+            // double-precision integer range (2^53) only if reduced; reduce a*x first.
+            const ax = (a * (x % P)) % P;
+            const h = (ax + b) % P;
+            if (h < min)
+                min = h;
+        }
+        sig[i] = min;
+    }
+    return sig;
+}
+/** Estimated Jaccard similarity of two signatures (fraction of equal slots). */
+export function signatureSimilarity(a, b) {
+    const n = Math.min(a.length, b.length);
+    if (n === 0)
+        return 0;
+    let equal = 0;
+    for (let i = 0; i < n; i++)
+        if (a[i] === b[i])
+            equal++;
+    return equal / n;
+}

package/dist/src/dedup/l1-verify.js

@@ -0,0 +1,54 @@
+/**
+ * l1-verify.ts — trigram-similarity verification gate for L1 near-duplicates
+ * (Sprint 11).
+ *
+ * After LSH cheaply retrieves candidate chunk_ids, we apply a REAL similarity
+ * check before declaring a duplicate. This is the `pg_trgm`-equivalent final
+ * gate: we compute the trigram (character 3-gram) Jaccard / overlap similarity
+ * between the new normalized text and each candidate's normalized text.
+ *
+ * We compute it in TS (not via FTS5 MATCH, which is boolean) so we get a stable
+ * [0,1] score to threshold against (0.85 per spec). The FTS5 `trigram` table is
+ * still maintained for future query-path use, but verification is pure TS so its
+ * result is deterministic and unit-testable (PREVENT-PI-004).
+ */
+import { normalize } from "./normalize.js";
+const TRIGRAM_SIZE = 3;
+export const L1_VERIFY_THRESHOLD = 0.85;
+/** Extract the set of character trigrams from normalized text. */
+function trigrams(text) {
+    const norm = normalize(text);
+    const set = new Set();
+    if (norm.length === 0)
+        return set;
+    if (norm.length < TRIGRAM_SIZE) {
+        set.add(norm);
+        return set;
+    }
+    for (let i = 0; i + TRIGRAM_SIZE <= norm.length; i++) {
+        set.add(norm.slice(i, i + TRIGRAM_SIZE));
+    }
+    return set;
+}
+/**
+ * Trigram similarity in [0,1]. We use overlap coefficient (|A∩B| / min(|A|,|B|))
+ * which, like pg_trgm's `similarity`, is robust when one text is a substring of
+ * the other — better than Jaccard for the near-dup "one-word edit" case.
+ */
+export function trigramSimilarity(a, b) {
+    const ta = trigrams(a);
+    const tb = trigrams(b);
+    if (ta.size === 0 || tb.size === 0)
+        return 0;
+    let inter = 0;
+    const smaller = ta.size <= tb.size ? ta : tb;
+    const larger = smaller === ta ? tb : ta;
+    for (const g of smaller)
+        if (larger.has(g))
+            inter++;
+    return inter / smaller.size;
+}
+/** True when `a` and `b` are near-duplicates under the L1 threshold. */
+export function isNearDuplicate(a, b, threshold = L1_VERIFY_THRESHOLD) {
+    return trigramSimilarity(a, b) >= threshold;
+}

package/dist/src/dedup/l1.test.js

@@ -0,0 +1,50 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { minhashSignature, shingles, signatureSimilarity, NUM_HASHES } from "./l1-minhash.js";
+import { lshBands, BANDS, ROWS_PER_BAND } from "./l1-lsh.js";
+import { trigramSimilarity, isNearDuplicate, L1_VERIFY_THRESHOLD } from "./l1-verify.js";
+test("minhashSignature is deterministic across calls (same input → same sig)", () => {
+    const a = minhashSignature("the authentication module handles login securely");
+    const b = minhashSignature("the authentication module handles login securely");
+    assert.deepEqual(a, b);
+    assert.equal(a.length, NUM_HASHES);
+});
+test("minhashSignature of near-identical text is more similar than of unrelated text", () => {
+    const s1 = minhashSignature("user logged in and viewed the dashboard");
+    const s2 = minhashSignature("user logged in and viewed the dashboard page"); // one word added
+    const s3 = minhashSignature("the compiler optimized the hot loop aggressively");
+    assert.ok(signatureSimilarity(s1, s2) > signatureSimilarity(s1, s3));
+});
+test("lshBands produces BANDS keys and is stable per session", () => {
+    const sig = minhashSignature("some text to band");
+    const k1 = lshBands(sig, "sess_a", 1);
+    const k2 = lshBands(sig, "sess_a", 1);
+    assert.equal(k1.length, BANDS);
+    assert.equal(BANDS * ROWS_PER_BAND, NUM_HASHES);
+    assert.deepEqual(k1, k2);
+    // Different session → different bucket keys (scoped, deterministic).
+    const k3 = lshBands(sig, "sess_b", 1);
+    assert.notDeepEqual(k1, k3);
+});
+test("lshBands keys are stable across restarts (no entropy source)", () => {
+    // Recomputed in a fresh call path — determinism is structural, not time-based.
+    const sig = minhashSignature("deterministic bucket key check");
+    const first = lshBands(sig, "sess_x", 1);
+    const again = lshBands(sig, "sess_x", 1);
+    assert.deepEqual(first, again);
+});
+test("trigramSimilarity is 1 for identical, high for one-word-edit, low for unrelated", () => {
+    const a = "the quick brown fox jumps";
+    assert.equal(trigramSimilarity(a, a), 1);
+    assert.ok(trigramSimilarity(a, "the quick brown fox jumps over") >= L1_VERIFY_THRESHOLD);
+    assert.ok(trigramSimilarity(a, "a completely different sentence about databases") < 0.5);
+});
+test("isNearDuplicate thresholds at 0.85", () => {
+    assert.equal(isNearDuplicate("user fixed the parser bug", "user fixed the parser bug today"), true);
+    assert.equal(isNearDuplicate("alpha beta gamma", "totally different words here"), false);
+});
+test("shingles are capped at 50K (complexity guard)", () => {
+    const huge = "x".repeat(200_000);
+    const sh = shingles(huge);
+    assert.ok(sh.length <= 50_000);
+});

package/dist/src/dedup/mmr.js

@@ -0,0 +1,45 @@
+/**
+ * mmr.ts — Maximal Marginal Relevance reranking for retrieval diversity
+ * (Sprint 12, QA #10).
+ *
+ * After a relevance-ranked candidate list, MMR reorders so we don't inject a
+ * cluster of near-identical checkpoints. Each step picks the candidate that
+ * maximizes `λ·relevance − (1−λ)·maxSimToAlreadySelected`, balancing relevance
+ * against redundancy. λ=0.5 is the default (equal weight).
+ *
+ * Pure function over cosine similarities — no deps, no network (PREVENT-PI-004).
+ */
+import { cosineSimilarity } from "../embedder.js";
+export const MMR_LAMBDA = 0.5;
+/**
+ * Rerank `items` by MMR. Returns the items in MMR order, capped at `k`.
+ * `lambda` balances relevance vs diversity (1 = pure relevance, 0 = max diversity).
+ */
+export function mmrRerank(items, k, lambda = MMR_LAMBDA) {
+    if (items.length === 0)
+        return [];
+    const remaining = [...items];
+    const selected = [];
+    const cap = Math.min(k, items.length);
+    while (selected.length < cap && remaining.length > 0) {
+        let bestIdx = 0;
+        let bestScore = -Infinity;
+        for (let i = 0; i < remaining.length; i++) {
+            const cand = remaining[i];
+            // Max similarity to already-selected (redundancy penalty).
+            let maxSimToSelected = 0;
+            for (const sel of selected) {
+                const sim = cosineSimilarity(cand.vector, sel.vector);
+                if (sim > maxSimToSelected)
+                    maxSimToSelected = sim;
+            }
+            const mmr = lambda * cand.relevance - (1 - lambda) * maxSimToSelected;
+            if (mmr > bestScore) {
+                bestScore = mmr;
+                bestIdx = i;
+            }
+        }
+        selected.push(remaining.splice(bestIdx, 1)[0]);
+    }
+    return selected.map((s) => s.item);
+}

package/dist/src/dedup/normalize.js

@@ -0,0 +1,39 @@
+/**
+ * normalize.ts — text normalization for content-addressable dedup (Sprint 9).
+ *
+ * The L0 dedup key is `sha256(normalize(text))`, so normalization decides which
+ * surface variants collapse to the same checkpoint. Pure, synchronous, no deps.
+ *
+ * Steps (order matters):
+ *   1. strip ANSI escape sequences (terminal color codes leak into tool output)
+ *   2. Unicode NFC (canonical composition — "e" + combining accent == "é")
+ *   3. case-fold (NFKC Cf + toLowerCase) so "Foo"/"foo"/"FOO" collapse (Sprint 10 L0 upgrade)
+ *   4. normalize newlines (CRLF/CR → LF)
+ *   5. collapse runs of whitespace to a single space, trim ends
+ *   6. cap at 32K chars (bounds hashing cost on pathological inputs — QA #7/#15)
+ */
+const MAX_CHARS = 32_768;
+// ANSI/VT100 escape sequence: ESC (0x1B) [ ...params... [ -/]* final-byte.
+// Built from the code point so the source file contains no literal escape byte.
+const ESC = String.fromCharCode(0x1b);
+const ANSI_RE = new RegExp(ESC + "\\[[0-?]*[ -/]*[@-~]", "g");
+/** Strip ANSI/VT100 escape sequences. */
+export function stripAnsi(text) {
+    return text.replace(ANSI_RE, "");
+}
+/**
+ * Normalize text to its canonical dedup form. Deterministic and idempotent:
+ * `normalize(normalize(x)) === normalize(x)`.
+ */
+export function normalize(text) {
+    if (!text)
+        return "";
+    let out = stripAnsi(text);
+    out = out.normalize("NFC");
+    out = out.toLocaleLowerCase(); // case-fold so "Foo"/"FOO" collapse to one key
+    out = out.replace(/\r\n?/g, "\n"); // CRLF / CR → LF
+    out = out.replace(/\s+/g, " ").trim();
+    if (out.length > MAX_CHARS)
+        out = out.slice(0, MAX_CHARS);
+    return out;
+}

package/dist/src/dedup/raptor/guardrails.js

@@ -0,0 +1,83 @@
+/**
+ * guardrails.ts — hallucination defense for RAPTOR summary nodes (Sprint 13, QA #16).
+ *
+ * Four layers gate a candidate summary before it may be marked high-quality:
+ *   1. Claim grounding — every claim in the summary maps to source text
+ *      (no entity/claim appears that isn't supported by a source chunk).
+ *   2. Entity coverage — fraction of summary entities that are present in source.
+ *   3. Consistency — cosine(reEmbed(summary), cluster centroid) ≥ threshold.
+ *      This is the HARD gate: a low score means the summary drifted from the
+ *      source cluster, so we fall back to extractive (never serve a low-quality
+ *      LLM summary).
+ *   4. Quality markers — 'high' | 'low' | 'extractive_fallback' assigned from the
+ *      above.
+ *
+ * Pure functions, no network, no model. The consistency check uses the caller's
+ * embedder (the same local Embedder used everywhere else).
+ */
+import { cosineSimilarity } from "../../embedder.js";
+const ENTITY_RE = /\b([A-Z][a-zA-Z0-9_]{2,}|[a-z_]+_[a-z_]+|\d{2,})\b/g;
+/** Extract candidate "entities"/tokens from a summary for grounding checks. */
+export function extractEntities(text) {
+    const out = new Set();
+    for (const m of text.matchAll(ENTITY_RE))
+        out.add(m[1].toLowerCase());
+    return [...out];
+}
+/** Lowercase word set from a body of source text (for grounding lookups). */
+export function sourceTokenSet(sources) {
+    const set = new Set();
+    for (const s of sources)
+        for (const w of s.toLowerCase().split(/\W+/))
+            if (w)
+                set.add(w);
+    return set;
+}
+/**
+ * Verify a summary against its sources + centroid.
+ *
+ * Faithfulness (QA #16): consistency is the hard gate. If the summary embedding
+ * is insufficiently similar to the cluster centroid, the summary is NOT faithful
+ * to the source — mark it 'extractive_fallback' so callers fall back to the
+ * deterministic extractive summary instead of serving a drifted LLM summary.
+ *
+ * grounding: every summary entity must appear in the source token set. A single
+ * un-grounded entity fails grounding (caught hallucination).
+ */
+export function applyHallucinationGuardrails(input) {
+    const threshold = input.consistencyThreshold ?? 0.6;
+    const sourceTokens = input.sourceTokens;
+    // Layer 1 + 2: entity grounding & coverage.
+    const entities = extractEntities(input.summary);
+    let groundedCount = 0;
+    for (const e of entities) {
+        if (sourceTokens.has(e))
+            groundedCount++;
+    }
+    const entityCoverage = entities.length === 0 ? 1 : groundedCount / entities.length;
+    const grounded = entities.length === 0 || groundedCount === entities.length;
+    // Layer 3: consistency re-embed.
+    const summEmbed = input.embedder.embed(input.summary);
+    const consistency = cosineSimilarity(summEmbed, input.centroid);
+    // Layer 4: quality marker decision.
+    if (!grounded || consistency < threshold) {
+        return {
+            marker: "extractive_fallback",
+            entityCoverage,
+            consistency,
+            grounded,
+            reason: !grounded
+                ? "ungrounded entity in summary"
+                : `consistency ${consistency.toFixed(2)} < ${threshold} (drift from source)`,
+        };
+    }
+    const marker = entityCoverage >= 0.7 ? "high" : "low";
+    return { marker, entityCoverage, consistency, grounded, reason: "ok" };
+}
+/**
+ * Convenience: build a fixture summary that is deliberately un-grounded (used by
+ * tests to prove the guardrail CATCHES a hallucination). Not used in production.
+ */
+export function makeUngroundedSummary(realSource, fakeEntity) {
+    return `${realSource.slice(0, 60)} The quarterly revenue doubled to ${fakeEntity}.`;
+}