pi-mega-compact 0.17.0 → 0.17.1

package/dist/dedup/raptor/summarizer.js

@@ -0,0 +1,86 @@
+/**
+ * summarizer.ts — per-cluster summary for the RAPTOR tree (Sprint 13, Phase 6).
+ *
+ * Default: pure extractive (deterministic, zero network, zero model) reusing
+ * extractive.ts. Optional: a LOCAL Ollama model (llama3.2:3b by default) when
+ * MEGACOMPACT_RAPTOR_MODEL is set — localhost only, same PREVENT-PI-004
+ * exception class as HttpEmbedder/the dashboard. No remote API is ever called.
+ *
+ * The summarizer returns structured text + a token estimate. Faithfulness is
+ * enforced downstream by guardrails.ts — this module only produces candidates.
+ */
+import { extractiveSummarize } from "../../extractive.js";
+import { estimateBlockTokens } from "../../tokens.js";
+import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: localhost-only user-spawned Ollama server (BYO local model, never remote)
+/** The local Ollama endpoint (loopback). Read lazily so tests can avoid it. */
+function ollamaEndpoint() {
+    const model = process.env.MEGACOMPACT_RAPTOR_MODEL;
+    if (!model)
+        return null;
+    const base = process.env.MEGACOMPACT_RAPTOR_URL ?? "http://127.0.0.1:11434"; // guardrails-allow PREVENT-PI-004: loopback-only Ollama endpoint for local RAPTOR summarization
+    // Guard: only loopback is permitted (remote Ollama would violate PREVENT-PI-004).
+    if (!/^https?:\/\/(localhost|127\.0\.0\.1)([:/]|$)/.test(base)) {
+        throw new Error(`MEGACOMPACT_RAPTOR_URL must be localhost/127.0.0.1 (got ${base}). ` +
+            `Remote Ollama is not allowed (PREVENT-PI-004).`);
+    }
+    return { url: `${base}/api/generate`, model };
+}
+/**
+ * Extractive summarization of a cluster's source messages. Deterministic and
+ * fully local — the on-by-default path.
+ */
+export function extractiveClusterSummary(messages) {
+    const s = extractiveSummarize(messages);
+    return { summary: s.topicSummary, tokenEstimate: s.tokenEstimate };
+}
+/**
+ * Build a summary for one cluster of source messages.
+ *
+ * Uses local Ollama when MEGACOMPACT_RAPTOR_MODEL is set (localhost-only);
+ * otherwise falls back to deterministic extractive. The `fetch` is a localhost
+ * call inside the PREVENT-PI-004 exception — annotated accordingly.
+ */
+export function summarizeCluster(messages) {
+    const ollama = ollamaEndpoint();
+    if (!ollama)
+        return extractiveClusterSummary(messages);
+    return ollamaSummarize(messages, ollama);
+}
+function ollamaSummarize(messages, ollama) {
+    // The fetch below is localhost-only (loopback Ollama) — the PREVENT-PI-004
+    // sanctioned local-model exceptions (same class as /dashboard, HttpEmbedder).
+    const prompt = messages.map((m) => `${m.role}: ${m.text}`).join("\n");
+    // Synchronous bridge: spawnSync an inline worker so the call blocks without
+    // deadlocking fetch (mirrors HttpEmbedder — Atomics.wait on main thread would
+    // hang). A blocked main thread cannot pump the socket.
+    const WORKER = String.raw `
+    const url = process.env.R_URL, model = process.env.R_MODEL, prompt = process.env.R_PROMPT;
+    try {
+      const r = await fetch(url, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model, prompt, stream: false }) }); // guardrails-allow PREVENT-PI-004: localhost-only user-spawned Ollama server (BYO local model, never remote)
+      const j = await r.json();
+      process.stdout.write(JSON.stringify({ ok: r.ok, text: j.response || "" }));
+    } catch (e) {
+      process.stdout.write(JSON.stringify({ ok: false, error: String(e && e.message ? e.message : e) }));
+    }
+  `;
+    const res = spawnSync(process.execPath, ["-e", WORKER], {
+        encoding: "utf8",
+        timeout: 30_000,
+        env: { ...process.env, R_URL: ollama.url, R_MODEL: ollama.model, R_PROMPT: prompt },
+    });
+    let parsed = { ok: false, error: "no response" };
+    if (typeof res.stdout === "string" && res.stdout.length > 0) {
+        try {
+            parsed = JSON.parse(res.stdout);
+        }
+        catch {
+            parsed = { ok: false, error: "bad json" };
+        }
+    }
+    if (!parsed.ok || !parsed.text) {
+        // Ollama unavailable → deterministic extractive fallback (never fail the build).
+        return extractiveClusterSummary(messages);
+    }
+    const summary = parsed.text.trim();
+    return { summary, tokenEstimate: estimateBlockTokens(summary) };
+}

package/dist/dedup/raptor/tree.js

@@ -0,0 +1,202 @@
+/**
+ * tree.ts — RAPTOR hierarchical summary-tree builder (Sprint 13, Phase 6).
+ *
+ * Builds a multi-level tree of summary nodes over leaf chunks: leaves are the
+ * original regions; each higher level summarizes clusters of the level below
+ * until a single root remains. QA ops: a wall-clock budget guard — on exhaustion
+ * we build an extractive fallback root. <10 leaves → a single summary node.
+ *
+ * Node model (kept simple + flat): every RaptorNode stores the LIST OF LEAF IDS
+ * it ultimately covers in `children` (not a mix of node/leaf ids). So the node
+ * map holds ONLY internal summary nodes — never per-leaf wrappers — which is
+ * what makes RAPTOR consolidate (nodes.size << leaves) and makes retrieval's
+ * leaf walk trivial.
+ *
+ * PREVENT-PI-004: no network here. summarizeCluster() may call a localhost
+ * Ollama (annotated in summarizer.ts); extractive is the default.
+ */
+import { kmeanspp, meanVector } from "./kmeans.js";
+import { summarizeCluster } from "./summarizer.js";
+import { applyHallucinationGuardrails, sourceTokenSet } from "./guardrails.js";
+const DEFAULT_BUDGET_MS = 5000;
+const DEFAULT_CLUSTERS = 5;
+function defaultNextId(level, index) {
+    return `r${level}_${index}`;
+}
+function summarizeInto(item, centroid, embedder, consistencyThreshold) {
+    let summary = summarizeCluster(item.messages);
+    const guard = applyHallucinationGuardrails({
+        summary: summary.summary,
+        sources: item.sources,
+        centroid,
+        embedder,
+        sourceTokens: sourceTokenSet(item.sources),
+        consistencyThreshold,
+    });
+    if (guard.marker === "extractive_fallback") {
+        summary = summarizeCluster(item.messages); // deterministic extractive text
+    }
+    return {
+        summary: summary.summary,
+        tokenEstimate: summary.tokenEstimate,
+        qualityMarker: guard.marker === "extractive_fallback" ? "low" : guard.marker,
+    };
+}
+/**
+ * Build a RAPTOR tree from leaf chunks. Synchronous; guarded by an elapsed-time
+ * budget. Returns a tree whose `nodes` map holds ONLY internal summary nodes.
+ */
+export function buildRaptorTree(leaves, opts) {
+    const embedder = opts.embedder;
+    const budgetMs = opts.budgetMs ?? DEFAULT_BUDGET_MS;
+    const clustersPerLevel = opts.clustersPerLevel ?? DEFAULT_CLUSTERS;
+    const nextId = opts.nextId ?? defaultNextId;
+    const now = opts.now ?? (() => Date.now());
+    const start = now();
+    const within = () => now() - start <= budgetMs;
+    const nodes = new Map();
+    // <10 leaves → single summary root (no hierarchy needed).
+    if (leaves.length < 10) {
+        const item = {
+            id: "root",
+            parentId: null,
+            embedding: meanVector(leaves.map((l) => l.embedding)),
+            leafIds: leaves.map((l) => l.id),
+            messages: leaves.flatMap((l) => l.messages),
+            sources: leaves.map((l) => l.sourceText),
+        };
+        const centroid = item.embedding;
+        const { summary, tokenEstimate, qualityMarker } = summarizeInto(item, centroid, embedder, opts.consistencyThreshold);
+        const rootId = nextId(0, 0);
+        nodes.set(rootId, {
+            id: rootId,
+            level: 0,
+            parentId: null,
+            children: item.leafIds,
+            summary,
+            embedding: centroid,
+            qualityMarker,
+            tokenEstimate,
+        });
+        return { nodes, rootId, levels: 1, timedOut: false };
+    }
+    let currentLevel = leaves.map((l) => ({
+        id: l.id,
+        parentId: null,
+        embedding: l.embedding,
+        leafIds: [l.id],
+        messages: l.messages,
+        sources: [l.sourceText],
+    }));
+    let level = 0;
+    while (currentLevel.length > 1) {
+        if (!within())
+            return extractiveFallbackRoot(leaves, nodes, nextId);
+        // Once we're down to a handful of items, collapse them all into one root.
+        // (k === currentLevel.length would make every item its own singleton
+        // cluster and never shrink — an infinite loop until the budget blows.)
+        if (currentLevel.length <= clustersPerLevel) {
+            const merged = {
+                id: "merge",
+                parentId: null,
+                embedding: meanVector(currentLevel.map((c) => c.embedding)),
+                leafIds: currentLevel.flatMap((c) => c.leafIds),
+                messages: currentLevel.flatMap((c) => c.messages),
+                sources: currentLevel.flatMap((c) => c.sources),
+            };
+            const centroid = merged.embedding;
+            const { summary, tokenEstimate, qualityMarker } = summarizeInto(merged, centroid, embedder, opts.consistencyThreshold);
+            const rootId = nextId(level + 1, 0);
+            nodes.set(rootId, {
+                id: rootId,
+                level: level + 1,
+                parentId: null,
+                children: merged.leafIds,
+                summary,
+                embedding: centroid,
+                qualityMarker,
+                tokenEstimate,
+            });
+            // Populate parentId for the internal nodes being merged into this root.
+            // currentLevel items with ids in `nodes` are internal summary nodes (level
+            // >= 1); raw leaf ids are not in `nodes` and are correctly skipped.
+            for (const c of currentLevel) {
+                const child = nodes.get(c.id);
+                if (child && child.id !== rootId && child.parentId === null) {
+                    child.parentId = rootId;
+                }
+            }
+            return { nodes, rootId, levels: level + 2, timedOut: false };
+        }
+        const k = Math.max(1, Math.min(clustersPerLevel, currentLevel.length));
+        const clustered = kmeanspp(currentLevel.map((c) => c.embedding), k, { seed: 0x1234 + level });
+        const groups = Array.from({ length: clustered.k }, () => []);
+        clustered.assignments.forEach((c, i) => groups[c].push(currentLevel[i]));
+        const nextLevel = [];
+        for (let g = 0; g < groups.length; g++) {
+            const group = groups[g];
+            if (group.length === 0)
+                continue;
+            const merged = {
+                id: nextId(level + 1, g),
+                parentId: null,
+                embedding: clustered.centroids[g],
+                leafIds: group.flatMap((c) => c.leafIds),
+                messages: group.flatMap((c) => c.messages),
+                sources: group.flatMap((c) => c.sources),
+            };
+            const { summary, tokenEstimate, qualityMarker } = summarizeInto(merged, merged.embedding, embedder, opts.consistencyThreshold);
+            nodes.set(merged.id, {
+                id: merged.id,
+                level: level + 1,
+                parentId: null,
+                children: merged.leafIds,
+                summary,
+                embedding: merged.embedding,
+                qualityMarker,
+                tokenEstimate,
+            });
+            // Populate parentId for the internal nodes being absorbed into this
+            // parent summary. Group members with ids in `nodes` are internal summary
+            // nodes (level >= 1); raw leaf ids are not in `nodes` (per-leaf wrappers
+            // are intentionally absent) and are correctly skipped — leaf→summary
+            // walks go through the parent's `children` list instead.
+            for (const c of group) {
+                const child = nodes.get(c.id);
+                if (child && child.id !== merged.id) {
+                    child.parentId = merged.id;
+                }
+            }
+            nextLevel.push(merged);
+        }
+        currentLevel = nextLevel;
+        level++;
+    }
+    const root = currentLevel[0];
+    const rootId = root ? root.id : null;
+    return {
+        nodes,
+        rootId,
+        levels: level + 1,
+        timedOut: false,
+    };
+}
+/**
+ * Budget-exceeded fallback: build a single deterministic extractive root over
+ * all leaves and mark it low quality. Keeps a valid (if shallow) tree.
+ */
+function extractiveFallbackRoot(leaves, nodes, nextId) {
+    const summary = summarizeCluster(leaves.flatMap((l) => l.messages));
+    const rootId = nextId(99, 0);
+    nodes.set(rootId, {
+        id: rootId,
+        level: 99,
+        parentId: null,
+        children: leaves.map((l) => l.id),
+        summary: summary.summary,
+        embedding: meanVector(leaves.map((l) => l.embedding)),
+        qualityMarker: "low",
+        tokenEstimate: summary.tokenEstimate,
+    });
+    return { nodes, rootId, levels: 100, timedOut: true };
+}

package/dist/dedup/topk.js

@@ -0,0 +1,60 @@
+/**
+ * topk.ts — min-heap based top-K selection (Sprint 12, QA #4).
+ *
+ * Replaces the O(N log N) full `.sort()` in search() with an O(N log k) heap,
+ * which matters once a session holds thousands of checkpoints. Generic over any
+ * scored item with a numeric `score`.
+ *
+ * Pure, no deps, no network (PREVENT-PI-004).
+ */
+/**
+ * Return the `k` highest-scoring items (stable insertion order on ties).
+ * O(N log k) — a bounded min-heap of size k.
+ */
+export function topK(items, k) {
+    if (k <= 0)
+        return [];
+    if (items.length <= k)
+        return [...items].sort((a, b) => b.score - a.score);
+    // Min-heap of the current top-k, stored as a flat array of Scored<T>.
+    const heap = [];
+    const push = (e) => {
+        heap.push(e);
+        let i = heap.length - 1;
+        while (i > 0) {
+            const parent = (i - 1) >> 1;
+            if (heap[parent].score <= heap[i].score)
+                break;
+            [heap[parent], heap[i]] = [heap[i], heap[parent]];
+            i = parent;
+        }
+    };
+    const siftDown = (start) => {
+        let i = start;
+        for (;;) {
+            const l = 2 * i + 1;
+            const r = 2 * i + 2;
+            let smallest = i;
+            if (l < heap.length && heap[l].score < heap[smallest].score)
+                smallest = l;
+            if (r < heap.length && heap[r].score < heap[smallest].score)
+                smallest = r;
+            if (smallest === i)
+                break;
+            [heap[smallest], heap[i]] = [heap[i], heap[smallest]];
+            i = smallest;
+        }
+    };
+    for (const it of items) {
+        if (heap.length < k) {
+            push(it);
+        }
+        else if (it.score > heap[0].score) {
+            // Replace the current minimum with this better-scoring item, then sift
+            // it down to restore the min-heap invariant.
+            heap[0] = it;
+            siftDown(0);
+        }
+    }
+    return heap.sort((a, b) => b.score - a.score);
+}

package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js

@@ -137,6 +137,7 @@ export const SETTINGS = [
             boolDirect("MEGACOMPACT_VC1A", "VC1A Canonical Byte Events", "EventV2 byte-authority ledger codec (original bytes + SHA-256, strict UTF-8, derived NFC) and canonical validator (EVT_DIGEST_MISMATCH / EVT_UTF8_TAG_INVALID / EVT_DUPLICATE_ID). OFF = mode C, transcript codec unchanged, byte-identical.", true),
             boolDirect("MEGACOMPACT_VC1B", "VC1B Occurrence Ledger + Tool Identity", "Neutral occurrence ledger (LedgerReader/Writer/Admin + CompatJournalV1): per-session monotonic seq, tool result references one earlier call, uniqueness by (eventId,digest) only, and the M2 copy-validate-switch downgrade journal. OFF = mode C, ledger unwritten, byte-identical.", true),
             boolDirect("MEGACOMPACT_VC0C", "VC0C Live Safety Envelope", "TriadResult/Breaker live circuit breaker (60s window, 20 attempts, 30s cooldown, 3 probes, 5min healthy residence) + durable spool before provider invocation; manual reset clears cooldown but never evidence. OFF = mode C, unchanged transcript, byte-identical.", true),
+            boolDirect("MEGACOMPACT_VC1C", "VC1C Cross-Language Conformance v2", "FixtureManifestV2 canonical manifest validator + DowngradeReport deterministic downgrade export + MinHashV2 exact big-integer signatures and the M4 copy/validate/switch minhash-v2 migration (seed table frozen, cross-language byte-exact). OFF = mode C, v1 sync dedup scan unchanged, byte-identical.", true),
         ],
     },
     {

package/dist/src/config/vector-cortex.js

@@ -53,6 +53,16 @@ export const VC1B_ENABLED = () => sprintFlag("MEGACOMPACT_VC1B");
  * consumer is the resilience emit seam + the safety adapter's triad selection.
  */
 export const VC0C_ENABLED = () => sprintFlag("MEGACOMPACT_VC0C");
+/**
+ * VC1C — cross-language conformance v2 (FixtureManifestV2 / DowngradeReport /
+ * MinHashV2 + M4 minhash-v2 migration).
+ * Default ON. `MEGACOMPACT_VC1C=0` disables and is byte-identical to the
+ * predecessor (mode C: a v2 conformance runner that accepts authority fixtures
+ * and the manifest validator idle; the sync dedup scan stays on the v1 path;
+ * zero `vector_cortex_*` VC1C emissions). The real consumers are the conformance
+ * emit seam, the minhash-v2 backfill seam and the downgrade-export seam.
+ */
+export const VC1C_ENABLED = () => sprintFlag("MEGACOMPACT_VC1C");
 // ---------------------------------------------------------------------------
 // Breaker state machine constants (TRIAD_RESILIENCE.md §breaker).
 // Rolled numbers for one 60s window; VC0C consumes these at its breaker seam.

package/dist/src/config.js

@@ -114,4 +114,4 @@ export const NEW_UI = () => ragEnabled("MEGACOMPACT_NEW_UI");
 // default ON, `=0`/`_DISABLED` off. Re-exported from src/config/vector-cortex.ts
 // so root consumers share one source of truth.
 // ---------------------------------------------------------------------------
-export { VC0A_ENABLED, VC0B_ENABLED, VC1A_ENABLED, VC0C_ENABLED, VC1B_ENABLED, BREAKER_WINDOW_MS, BREAKER_MIN_ATTEMPTS, BREAKER_PERF_FAILURES, BREAKER_PERF_FAILURE_RATE, BREAKER_CORRECTNESS_FAILURES, BREAKER_COOLDOWN_MS, BREAKER_PROBE_COUNT, BREAKER_RETRY_BASE_MS, BREAKER_RETRY_CAP_MS, BREAKER_RETRY_JITTER, BREAKER_HYSTERESIS_FAILURE_RATE, BREAKER_HYSTERESIS_BUDGET_P95_MS, BREAKER_MIN_HEALTHY_RESIDENCE_MS, } from "./config/vector-cortex.js";
+export { VC0A_ENABLED, VC0B_ENABLED, VC1A_ENABLED, VC0C_ENABLED, VC1B_ENABLED, VC1C_ENABLED, BREAKER_WINDOW_MS, BREAKER_MIN_ATTEMPTS, BREAKER_PERF_FAILURES, BREAKER_PERF_FAILURE_RATE, BREAKER_CORRECTNESS_FAILURES, BREAKER_COOLDOWN_MS, BREAKER_PROBE_COUNT, BREAKER_RETRY_BASE_MS, BREAKER_RETRY_CAP_MS, BREAKER_RETRY_JITTER, BREAKER_HYSTERESIS_FAILURE_RATE, BREAKER_HYSTERESIS_BUDGET_P95_MS, BREAKER_MIN_HEALTHY_RESIDENCE_MS, } from "./config/vector-cortex.js";

package/dist/src/dedup/l1-lsh-v2.js

@@ -0,0 +1,60 @@
+/**
+ * l1-lsh-v2.ts — Locality-Sensitive Hashing banding over MinHashV2 signatures
+ * (VC1C, M4 minhash-v2).
+ *
+ * Splits the frozen 2048-byte v2 signature (256 x u64 LE slots) into 64 bands
+ * of FOUR u64 values each. Each band yields a deterministic bucket key derived
+ * from its 4 u64 little-endian bytes (32 bytes) hashed with a 64-bit FNV-1a,
+ * scoped by session id and the frozen v2 version tag so buckets never collide
+ * across sessions or the v1 path. Deterministic: same (sessionId,
+ * signatureBytes) -> same 64 bucket keys, every run, every language.
+ *
+ * Mixed v1/v2 comparison and bucket mixing is REJECTED upstream with
+ * `MINHASH_VERSION_MISMATCH` (see minhash-v2 migration); this module only ever
+ * bands a v2 signature.
+ *
+ * Pure compute, no deps, no storage, no network (PREVENT-PI-004 / PREVENT-011).
+ */
+import { MINHASH_VERSION, NUM_HASHES_V2, U64_BYTES, } from "./l1-minhash-v2.js";
+/** Number of bands (frozen). */
+export const BANDS_V2 = 64;
+/** u64 slots per band (frozen): 64 * 4 = 256 total slots. */
+export const VALUES_PER_BAND_V2 = 4;
+/** Band byte length: 4 u64 LE = 32 bytes. */
+export const BAND_BYTES_V2 = VALUES_PER_BAND_V2 * U64_BYTES;
+/** Total signature byte length this scheme expects (256 * 8 = 2048). */
+export const SIGNATURE_BYTES_EXPECTED_V2 = NUM_HASHES_V2 * U64_BYTES;
+const MASK64 = 0xffffffffffffffffn;
+/** 64-bit FNV-1a over a byte buffer. */
+function fnv1a64Bytes(buf) {
+    let h = 0xcbf29ce484222325n;
+    for (let i = 0; i < buf.length; i++) {
+        h ^= BigInt(buf[i] ?? 0);
+        h = (h * 0x100000001b3n) & MASK64;
+    }
+    return h;
+}
+/**
+ * Compute the 64 LSH bucket keys for a v2 signature within a session. `bytes`
+ * is the 2048-byte little-endian signature (see encodeSignatureV2). Each bucket
+ * key is the hex of an 8-byte little-endian FNV-1a-64 over the band's 32 bytes
+ * prefixed by `<sessionId>|v2|band-` — so the "bucket bytes" are deterministic
+ * and reproducible across languages.
+ */
+export function lshBandsV2(bytes, sessionId, version = MINHASH_VERSION) {
+    if (bytes.length !== SIGNATURE_BYTES_EXPECTED_V2) {
+        throw new Error(`lshBandsV2: expected ${SIGNATURE_BYTES_EXPECTED_V2} signature bytes, got ${bytes.length}`);
+    }
+    const keys = [];
+    const prefix = Buffer.from(`${sessionId}|v${version}|`, "utf8");
+    for (let band = 0; band < BANDS_V2; band++) {
+        const start = band * BAND_BYTES_V2;
+        const bandBytes = bytes.slice(start, start + BAND_BYTES_V2);
+        const combined = Buffer.concat([prefix, Buffer.from(bandBytes)]);
+        const h = fnv1a64Bytes(combined);
+        const out = Buffer.allocUnsafe(U64_BYTES);
+        out.writeBigUInt64LE(h, 0);
+        keys.push(`b${band}:${out.toString("hex")}`);
+    }
+    return keys;
+}

package/dist/src/dedup/l1-minhash-v2.js

@@ -0,0 +1,193 @@
+/**
+ * l1-minhash-v2.ts — MinHash V2 signatures (VC1C, M4 minhash-v2).
+ *
+ * The v1 path (l1-minhash.ts) uses 32-bit universal hashing with `p=2^31-1`
+ * and Number math, which is exact because a*x < 2^62 can overflow double
+ * precision (>= 2^53) and is reduced too eagerly. MinHashV2 freezes a v2
+ * scheme that is CROSS-LANGUAGE deterministic and byte-exact:
+ *
+ *   - normalization: the stable `normalize()` from normalize.ts (NFC, NFKC
+ *     case-fold, CRLF->LF, whitespace collapse, 32K cap);
+ *   - shingles: windows of FIVE CODE POINTS over the normalized text
+ *     (SHINGLE_CP=5), each shingle hashed to a u64 with 64-bit FNV-1a over its
+ *     UTF-8 bytes;
+ *   - 256 PUBLISHED unsigned (a_i, b_i) seed pairs, derived deterministically
+ *     via splitmix64 and listed in
+ *     conformance/vector-cortex/v2/minhash/seeds-v2.json;
+ *   - p = 2^61 - 1 (2305843009213693951), EXACT BigInt multiply/modulo — a_i*x
+ *     can reach ~2^124 so Number math would silently corrupt high-bit products;
+ *   - signature = 256 slots, slot i = min over shingles of (a_i*x + b_i) mod p;
+ *     the empty signature is all-`p` sentinel slots;
+ *   - encoding: 256 x u64 LITTLE-ENDIAN bytes (2048 bytes total);
+ *   - bands: 64 bands of FOUR u64 slots each (256 total), see l1-lsh-v2.ts.
+ *
+ * NEVER compares a v2 signature against a v1 signature — cross-version compare
+ * is rejected with `MINHASH_VERSION_MISMATCH` (see minhash-v2 migration).
+ *
+ * Pure compute, no deps, no storage, no network (PREVENT-PI-004 / PREVENT-011).
+ */
+import { normalize } from "./normalize.js";
+/** MinHashV2 scheme version tag (frozen). */
+export const MINHASH_VERSION = 2;
+/** The published count of slots / seed pairs (frozen). */
+export const NUM_HASHES_V2 = 256;
+/** Five-code-point shingle length (frozen). */
+export const SHINGLE_CP = 5;
+/** p = 2^61 - 1 (Mersenne prime), as exact BigInt. */
+export const P_V2 = (1n << 61n) - 1n;
+/** Little-endian bytes per u64 signature slot. */
+export const U64_BYTES = 8;
+/** Signature byte length: 256 slots x 8 LE bytes. */
+export const SIGNATURE_BYTES_V2 = NUM_HASHES_V2 * U64_BYTES;
+/** Fixed splitmix64 initial state so the seed table is reproducible everywhere. */
+export const SEED_STATE_INIT = 0x5eedc0def001ba11n & 0xffffffffffffffffn;
+/** splitmix64 mixing constants (frozen). */
+const SM_GOLDEN = 0x9e3779b97f4a7c15n;
+const SM_M1 = 0xbf58476d1ce4e5b9n;
+const SM_M2 = 0x94d049bb133111ebn;
+const MASK64 = 0xffffffffffffffffn;
+function mix(z0) {
+    let z = z0;
+    z = (z ^ (z >> 30n)) * SM_M1;
+    z &= MASK64;
+    z = (z ^ (z >> 27n)) * SM_M2;
+    z &= MASK64;
+    return z ^ (z >> 31n);
+}
+/** Create a splitmix64 PRNG from a fixed unsigned 64-bit state. */
+export function splitmix64(state = SEED_STATE_INIT) {
+    let s = state & MASK64;
+    return () => {
+        s = (s + SM_GOLDEN) & MASK64;
+        return mix(s) & MASK64;
+    };
+}
+/**
+ * Derive the frozen 256 published (a_i, b_i) seed pairs. Each pair maps two
+ * splitmix64 draws into [1, p-1]. Deterministic and cross-language reproducible
+ * from SEED_STATE_INIT — mirrors `seeds-v2.json`.
+ *
+ * The table is frozen: computed ONCE at module load and reused for every call.
+ * Recomputing the 256 splitmix64 pairs (and re-walking all shingles) per
+ * signature/backfill row is wasteful for a large v1 index backfill, and the
+ * table is immutable by construction, so a single shared instance is safe and
+ * byte-identical. The returned sub-arrays share the frozen pair objects; they
+ * are themselves frozen so no caller can mutate the shared cache.
+ */
+export function minhashV2Seeds(count = NUM_HASHES_V2) {
+    return FROZEN_SEEDS.slice(0, count);
+}
+/** Singleton seed table, computed exactly once and frozen (see minhashV2Seeds). */
+const FROZEN_SEEDS = (() => {
+    const prng = splitmix64();
+    const pairs = [];
+    for (let i = 0; i < NUM_HASHES_V2; i++) {
+        const a = (prng() % (P_V2 - 1n)) + 1n;
+        const b = (prng() % (P_V2 - 1n)) + 1n;
+        pairs.push(Object.freeze({ a, b }));
+    }
+    return Object.freeze(pairs);
+})();
+/** 64-bit FNV-1a over a UTF-8 string (for shingle hashing). */
+function fnv1a64String(s) {
+    let h = 0xcbf29ce484222325n;
+    const bytes = Buffer.from(s, "utf8");
+    for (let i = 0; i < bytes.length; i++) {
+        h ^= BigInt(bytes[i] ?? 0);
+        h = (h * 0x100000001b3n) & MASK64;
+    }
+    return h;
+}
+/**
+ * Five-code-point shingle set (deduped, capped) of normalized text as u64
+ * hashes. Iterates over code points (not UTF-16 code units) so astral-plane
+ * characters do not split a shingle.
+ */
+export function shinglesV2(text, shingleCp = SHINGLE_CP, maxShingles = 50_000) {
+    const norm = normalize(text);
+    if (norm.length === 0)
+        return [];
+    const cps = Array.from(norm); // code points
+    const set = new Set();
+    if (cps.length < shingleCp) {
+        set.add(fnv1a64String(norm));
+    }
+    else {
+        for (let i = 0; i + shingleCp <= cps.length; i++) {
+            const sh = cps.slice(i, i + shingleCp).join("");
+            set.add(fnv1a64String(sh));
+            if (set.size >= maxShingles)
+                break;
+        }
+    }
+    return [...set];
+}
+/**
+ * Compute the 256-slot MinHashV2 signature of a text as an array of exact u64
+ * BigInt values (each in [0, p], p = sentinel when empty). Uses EXACT BigInt
+ * multiply/modulo so high-bit products (a*x ~ up to 2^124) are never corrupted.
+ */
+export function minhashV2Signature(text) {
+    const grams = shinglesV2(text);
+    const seeds = minhashV2Seeds();
+    const sig = new Array(NUM_HASHES_V2).fill(P_V2);
+    if (grams.length === 0)
+        return sig;
+    for (let i = 0; i < NUM_HASHES_V2; i++) {
+        const { a, b } = seeds[i];
+        let min = P_V2;
+        for (const x of grams) {
+            const h = (a * x + b) % P_V2;
+            if (h < min)
+                min = h;
+        }
+        sig[i] = min;
+    }
+    return sig;
+}
+/** Encode a 256-slot signature to 2048 bytes as 256 x u64 little-endian. */
+export function encodeSignatureV2(sig) {
+    if (sig.length !== NUM_HASHES_V2) {
+        throw new Error(`encodeSignatureV2: expected ${NUM_HASHES_V2} slots, got ${sig.length}`);
+    }
+    const buf = Buffer.allocUnsafe(SIGNATURE_BYTES_V2);
+    for (let i = 0; i < NUM_HASHES_V2; i++) {
+        buf.writeBigUInt64LE(sig[i], i * U64_BYTES);
+    }
+    return new Uint8Array(buf);
+}
+/**
+ * Estimated Jaccard similarity of two v2 signatures (fraction of equal slots).
+ *
+ * THIS IS v2-ONLY. Cross-version compare is rejected INTERNALLY with
+ * `MINHASH_VERSION_MISMATCH`: both inputs must be exact 256-slot v2 signatures,
+ * i.e. every slot a `bigint` in [0, P_V2] (the empty sentinel is exactly P_V2).
+ * A v1 signature is a `number[]` of u31 values, so passing a v1 array fails the
+ * bigint type/range check and is rejected — a mixed v1/v2 compare can never
+ * produce a similarity number. Never call this with v1 arrays.
+ */
+export function signatureSimilarityV2(a, b) {
+    assertV2Signature(a);
+    assertV2Signature(b);
+    const n = Math.min(a.length, b.length);
+    if (n === 0)
+        return 0;
+    let equal = 0;
+    for (let i = 0; i < n; i++)
+        if (a[i] === b[i])
+            equal++;
+    return equal / n;
+}
+/** Reject any array that is not an exact 256-slot v2 signature (bigint in [0, P_V2]). */
+function assertV2Signature(sig) {
+    if (sig.length !== NUM_HASHES_V2) {
+        throw new Error(MINHASH_VERSION_MISMATCH_MSG);
+    }
+    for (const s of sig) {
+        if (typeof s !== "bigint" || s < 0n || s > P_V2) {
+            throw new Error(MINHASH_VERSION_MISMATCH_MSG);
+        }
+    }
+}
+/** Frozen cross-version-reject message shared by the v2-only function guards. */
+const MINHASH_VERSION_MISMATCH_MSG = "MINHASH_VERSION_MISMATCH";

package/dist/src/dedup/normalize.js

@@ -30,7 +30,7 @@ export function normalize(text) {
         return "";
     let out = stripAnsi(text);
     out = out.normalize("NFC");
-    out = out.toLocaleLowerCase(); // case-fold so "Foo"/"FOO" collapse to one key
+    out = out.toLowerCase(); // case-fold so "Foo"/"FOO" collapse to one key (locale-independent: byte-exact across languages)
     out = out.replace(/\r\n?/g, "\n"); // CRLF / CR → LF
     out = out.replace(/\s+/g, " ").trim();
     if (out.length > MAX_CHARS)