pikelet-wasm 0.7.0

package/complete/inline-transformer.mjs

@@ -0,0 +1,205 @@
+import { createWordPiece } from './wordpiece.mjs';
+
+const decoder = new TextDecoder();
+
+// The wasm kernel compiles these in (encoder-spike/encoder.cpp `constexpr`
+// block) and walks the weight blob with a bare running cursor — it never
+// reads the declaration. Declaration/blob skew is therefore only catchable
+// here, host-side, before the first forward.
+export const KERNEL_LAYOUT = { V: 30522, P: 512, T: 2, D: 384, F: 1536, L: 6, B: 64, H: 12 };
+const KERNEL_MAX_SEQ = 128;
+
+// Byte size the kernel's fill_layout() will consume for a given layout:
+// each quantized matrix is rows*cols u8 plus per-block f32 scales and
+// offsets; biases and layer norms are plain f32.
+export function expectedBlobBytes(layout = KERNEL_LAYOUT) {
+  const { V, P, T, D, F, L, B } = layout;
+  const quant = (rows, cols) => rows * cols + rows * (cols / B) * 4 * 2;
+  const layer = 4 * quant(D, D) + 4 * (D * 4)   // wq,wk,wv,wo + their biases
+    + 2 * (D * 4)                                // ln1 gamma/beta
+    + quant(F, D) + F * 4                        // wu + bu
+    + quant(D, F) + D * 4                        // wd + bd
+    + 2 * (D * 4);                               // ln2 gamma/beta
+  return quant(V, D) + quant(P, D) + quant(T, D) + 2 * (D * 4) + L * layer;
+}
+
+export function parseInlineTransformerEncoder(encoderBytes) {
+  const view = new DataView(encoderBytes.buffer, encoderBytes.byteOffset, encoderBytes.byteLength);
+  const declLen = view.getUint32(0, true);
+  const vocabLen = view.getUint32(4, true);
+  const blobLen = view.getUint32(8, true);
+  if (12 + declLen + vocabLen + blobLen !== encoderBytes.length) {
+    throw new Error('.pancake inline-encoder layout is inconsistent');
+  }
+  const declaration = JSON.parse(decoder.decode(encoderBytes.subarray(12, 12 + declLen)));
+  const blob = encoderBytes.subarray(12 + declLen + vocabLen);
+  const layout = declaration.layout || {};
+  for (const key of Object.keys(KERNEL_LAYOUT)) {
+    if (layout[key] !== KERNEL_LAYOUT[key]) {
+      throw new Error(`.pancake inline-encoder declares layout ${key}=${layout[key]}; `
+        + `the compiled kernel requires ${key}=${KERNEL_LAYOUT[key]}`);
+    }
+  }
+  if (declaration.dim !== KERNEL_LAYOUT.D) {
+    throw new Error(`.pancake inline-encoder declares dim ${declaration.dim}; the kernel emits ${KERNEL_LAYOUT.D}`);
+  }
+  const expected = expectedBlobBytes(layout);
+  if (blob.length !== expected) {
+    throw new Error(`.pancake inline-encoder blob is ${blob.length} bytes but its declared layout implies ${expected}`);
+  }
+  return {
+    declaration,
+    vocabText: decoder.decode(encoderBytes.subarray(12 + declLen, 12 + declLen + vocabLen)),
+    blob,
+  };
+}
+
+export async function createInlineTransformerEmbedder({ declaration, vocabText, blob, createEncoder, verify = true }) {
+  const vocabLines = vocabText.split('\n');
+  const vocabSize = vocabLines[vocabLines.length - 1] === '' ? vocabLines.length - 1 : vocabLines.length;
+  if (vocabSize > KERNEL_LAYOUT.V) {
+    throw new Error(`inline-encoder vocab has ${vocabSize} entries; the kernel's embedding table holds ${KERNEL_LAYOUT.V}`);
+  }
+  const tokenizer = createWordPiece(vocabText);
+  const EM = await createEncoder();
+  const dim = declaration.dim;
+  const maxSeq = Math.min(declaration.maxTokens || KERNEL_MAX_SEQ, KERNEL_MAX_SEQ);
+  const blobPtr = EM._malloc(blob.length);
+  EM.HEAPU8.set(blob, blobPtr);
+  const idsPtr = EM._malloc(maxSeq * 4);
+  const hiddenPtr = EM._malloc(maxSeq * dim * 4);
+
+  async function embed(text) {
+    const allIds = tokenizer.encode(String(text || ''));
+    // Inputs longer than maxSeq are mean-pooled across [CLS]…[SEP]-framed
+    // windows instead of truncated, so long chunks keep their tail content.
+    const interior = allIds.slice(1, -1);
+    const windowLen = maxSeq - 2;
+    const windows = Math.max(1, Math.ceil(interior.length / windowLen));
+    const pooled = new Float32Array(dim);
+    let pooledTokens = 0;
+    for (let w = 0; w < windows; w++) {
+      const tokenIds = [allIds[0], ...interior.slice(w * windowLen, (w + 1) * windowLen), allIds[allIds.length - 1]];
+      new Int32Array(EM.HEAP32.buffer, idsPtr, tokenIds.length).set(tokenIds);
+      const rc = EM._encoder_forward(blobPtr, idsPtr, tokenIds.length, hiddenPtr, 0);
+      if (rc !== tokenIds.length) throw new Error(`inline encoder failed: ${rc}`);
+      const hidden = new Float32Array(EM.HEAPF32.buffer, hiddenPtr, tokenIds.length * dim);
+      for (let t = 0; t < tokenIds.length; t++) {
+        for (let d = 0; d < dim; d++) pooled[d] += hidden[t * dim + d];
+      }
+      pooledTokens += tokenIds.length;
+    }
+    let norm = 0;
+    for (let d = 0; d < dim; d++) {
+      pooled[d] /= pooledTokens;
+      norm += pooled[d] ** 2;
+    }
+    norm = Math.sqrt(norm);
+    for (let d = 0; d < dim; d++) pooled[d] /= norm;
+    return { vector: pooled, text, windows };
+  }
+
+  const embedder = {
+    declaration,
+    dim,
+    maxSeq,
+    embed,
+    dispose() {
+      EM._free(hiddenPtr);
+      EM._free(idsPtr);
+      EM._free(blobPtr);
+    },
+  };
+  if (verify && Array.isArray(declaration.testVectors) && declaration.testVectors.length > 0) {
+    try {
+      await verifyInlineTestVectors(embedder);
+    } catch (err) {
+      embedder.dispose();
+      throw err;
+    }
+  }
+  return embedder;
+}
+
+// Conformance probes for kind-3 declarations (contract section 4.4 mode 1:
+// an inline encoder carries verification vectors). The last probe exceeds
+// the kernel window so the windowed mean-pool path stays pinned alongside
+// the single-window path.
+export const INLINE_TEST_VECTOR_TEXTS = [
+  'how do volcanoes form and why do they erupt',
+  'which treaty ended the first world war',
+  'The water cycle describes how water moves between the oceans, the atmosphere, and the land. '
+    + 'Heat from the sun evaporates water from the surface of the sea, and plants release more through their leaves. '
+    + 'The rising vapor cools as it climbs, condensing into droplets that gather into clouds. '
+    + 'When the droplets grow too heavy they fall as rain, snow, or hail, depending on the temperature of the air they pass through. '
+    + 'Some of the water soaks into the soil and filters down to recharge underground aquifers, '
+    + 'while the rest runs off into streams and rivers that carry it back toward the coast. '
+    + 'Along the way it erodes rock, deposits sediment, and sustains wetlands, forests, and farmland. '
+    + 'Glaciers and ice sheets hold a share of it frozen for centuries before releasing meltwater in spring. '
+    + 'Eventually nearly every drop returns to the ocean, where the sun lifts it again and the whole journey repeats.',
+];
+
+export async function buildInlineTestVectors(embedder, texts = INLINE_TEST_VECTOR_TEXTS) {
+  const out = [];
+  for (const text of texts) {
+    const { vector, windows } = await embedder.embed(text);
+    out.push({
+      text,
+      windows,
+      embedding: Array.from(vector, (v) => Number(v.toFixed(6))),
+      tolerance: 1e-3,
+    });
+  }
+  return out;
+}
+
+// A verification vector's expected embedding must itself be verifiable:
+// exactly dim real, finite numbers. A malformed entry (string, null, NaN,
+// missing component, wrong length) would otherwise make every comparison
+// NaN, and `NaN > maxDiff` is false — the vector would "pass" without a
+// single meaningful comparison. Shared by the kind-2 host verifier and the
+// kind-3 inline verifier.
+export function validateExpectedEmbedding(values, dim, label) {
+  if (!Array.isArray(values) || values.length !== dim) {
+    throw new Error(`${label}: expected embedding must be an array of ${dim} numbers, got ${Array.isArray(values) ? `length ${values.length}` : typeof values}`);
+  }
+  for (let d = 0; d < dim; d++) {
+    if (typeof values[d] !== 'number' || !Number.isFinite(values[d])) {
+      throw new Error(`${label}: expected embedding component ${d} is not a finite number`);
+    }
+  }
+}
+
+export function validateTestVectorTolerance(tolerance, label) {
+  if (tolerance === undefined) return 1e-3;
+  if (typeof tolerance !== 'number' || !Number.isFinite(tolerance) || tolerance <= 0 || tolerance >= 1) {
+    throw new Error(`${label}: tolerance is implausible`);
+  }
+  return tolerance;
+}
+
+export async function verifyInlineTestVectors(embedder) {
+  for (const tv of embedder.declaration.testVectors) {
+    if (!tv || typeof tv.text !== 'string') {
+      throw new Error('inline encoder verification vector is malformed (no text)');
+    }
+    const label = `inline encoder verification vector "${tv.text.slice(0, 40)}…"`;
+    const { vector, windows } = await embedder.embed(tv.text);
+    validateExpectedEmbedding(tv.embedding, vector.length, label);
+    const tolerance = validateTestVectorTolerance(tv.tolerance, label);
+    if (tv.windows !== undefined && windows !== tv.windows) {
+      throw new Error(`inline encoder verification: "${tv.text.slice(0, 40)}…" pooled over ${windows} window(s); `
+        + `the declaration recorded ${tv.windows}`);
+    }
+    let maxDiff = 0;
+    for (let d = 0; d < vector.length; d++) {
+      const diff = Math.abs(vector[d] - tv.embedding[d]);
+      if (diff > maxDiff) maxDiff = diff;
+    }
+    if (!(maxDiff <= tolerance)) {
+      throw new Error(`inline encoder failed its declaration's verification vector "${tv.text.slice(0, 40)}…": `
+        + `max component diff ${maxDiff.toExponential(2)} exceeds tolerance ${tolerance}`);
+    }
+  }
+  return { checked: embedder.declaration.testVectors.length };
+}

package/complete/lexical.mjs

@@ -0,0 +1,264 @@
+// Reader for the lexical index segment (kind 5, layout bm25-v1) — the
+// static inverted index a complete artifact MAY carry for hybrid retrieval.
+// Byte layout and hashing mirror buildLexicalSegment in builder.mjs; this
+// module must stay runtime-neutral (browsers, Workers, Node).
+//
+// The phase-1 reader receives the whole hash-verified segment; the format
+// itself is lazy-friendly (fixed-width hash-sorted term table, absolute
+// offsets) so a range-reading opener can replace this one without a format
+// revision.
+
+const BM25_K1 = 1.2;
+const BM25_B = 0.75;
+
+function fnv1a32(str, seed) {
+    let h = 0x811c9dc5 ^ seed;
+    for (let i = 0; i < str.length; i++) {
+        h ^= str.charCodeAt(i);
+        h = Math.imul(h, 0x01000193);
+    }
+    return h >>> 0;
+}
+
+// Query tokens follow the builder's shape (lowercase [a-z0-9']+, 2..32
+// chars). No stopword list ships or is needed: a token the builder skipped
+// simply finds no postings.
+const tokenize = (text) => (String(text).toLowerCase().match(/[a-z0-9']+/g) || [])
+    .filter((w) => w.length >= 2 && w.length <= 32);
+
+// Lazy opener for wiki-scale lexical segments: only the 64-byte header and
+// the doclen array (4 bytes per record — 4 MiB at a million chunks) become
+// resident; term lookups and postings stay remote. Term lookup exploits the
+// format's own structure: fnv hashes are uniformly distributed and the term
+// table is sorted by them, so the table is an implicit interpolation index —
+// predict the entry position from the hash value, read a ~512-entry window
+// (12 KiB), and the term is almost always inside on the first read, with
+// bounded re-aims for the tail. A query term then costs ~1 window read plus
+// 1 postings read, issued in parallel across the query's terms.
+//
+// Integrity stance is transitional, like format-1 sketch rows: the segment
+// is committed via the manifest digest but lazy reads are not individually
+// verified (the eager opener verifies the whole segment; per-page digests
+// are the format-v2 path if this gap needs closing).
+export async function openLexicalIndexLazy(read, segLength) {
+    const headerBytes = await read(0, 64);
+    if (!(headerBytes instanceof Uint8Array) || headerBytes.length < 64 || segLength < 64) {
+        throw new Error('.pancake lexical segment is too short');
+    }
+    const hv = new DataView(headerBytes.buffer, headerBytes.byteOffset, headerBytes.byteLength);
+    const version = hv.getUint32(0, true);
+    if (version !== 1) throw new Error(`unsupported lexical segment version ${version}`);
+    const docCount = hv.getUint32(4, true);
+    const termCount = hv.getUint32(8, true);
+    const totalTokens = Number(hv.getBigUint64(12, true));
+    const doclenOffset = hv.getUint32(20, true);
+    const termTableOffset = hv.getUint32(24, true);
+    const postingsOffset = hv.getUint32(28, true);
+    const postingsBytes = Number(hv.getBigUint64(32, true));
+    if (doclenOffset !== 64
+        || termTableOffset !== doclenOffset + 4 * docCount
+        || postingsOffset !== termTableOffset + 24 * termCount
+        || postingsOffset + postingsBytes !== segLength) {
+        throw new Error('.pancake lexical segment regions do not tile the segment');
+    }
+    const avgdl = docCount > 0 ? Math.max(1, totalTokens / docCount) : 1;
+    const doclenBytes = docCount > 0 ? await read(doclenOffset, 4 * docCount) : new Uint8Array(0);
+    const doclenView = new DataView(doclenBytes.buffer, doclenBytes.byteOffset, doclenBytes.byteLength);
+    const doclenOf = (id) => doclenView.getUint32(4 * id, true);
+
+    const WINDOW = 512;
+    const windowCache = new Map(); // startIdx -> entry bytes (small LRU)
+    async function readWindow(startIdx, count) {
+        const key = `${startIdx}:${count}`;
+        const hit = windowCache.get(key);
+        if (hit) return hit;
+        const bytes = await read(termTableOffset + 24 * startIdx, 24 * count);
+        if (windowCache.size >= 64) windowCache.delete(windowCache.keys().next().value);
+        windowCache.set(key, bytes);
+        return bytes;
+    }
+    const entryAt = (bytes, i) => {
+        const view = new DataView(bytes.buffer, bytes.byteOffset + 24 * i, 24);
+        return {
+            hashLo: view.getUint32(0, true),
+            hashHi: view.getUint32(4, true),
+            postingsRel: Number(view.getBigUint64(8, true)),
+            postingsLen: view.getUint32(16, true),
+            df: view.getUint32(20, true),
+        };
+    };
+    const cmp = (aHi, aLo, bHi, bLo) => (aHi !== bHi ? (aHi < bHi ? -1 : 1) : aLo === bLo ? 0 : (aLo < bLo ? -1 : 1));
+
+    async function findTerm(hashLo, hashHi) {
+        if (termCount === 0) return null;
+        let loIdx = 0;
+        let hiIdx = termCount - 1;
+        let loHash = 0;
+        let hiHash = 0xffffffff;
+        for (let iter = 0; iter < 10 && loIdx <= hiIdx; iter++) {
+            const span = hiHash > loHash ? hiHash - loHash : 1;
+            let guess = loIdx + Math.floor((hiIdx - loIdx) * ((hashHi - loHash) / span));
+            guess = Math.min(hiIdx, Math.max(loIdx, guess));
+            const start = Math.min(Math.max(loIdx, guess - (WINDOW >> 1)), Math.max(loIdx, hiIdx - WINDOW + 1));
+            const count = Math.min(WINDOW, hiIdx - start + 1);
+            const bytes = await readWindow(start, count);
+            const first = entryAt(bytes, 0);
+            const last = entryAt(bytes, count - 1);
+            if (cmp(hashHi, hashLo, first.hashHi, first.hashLo) < 0) {
+                hiIdx = start - 1;
+                hiHash = first.hashHi;
+                continue;
+            }
+            if (cmp(hashHi, hashLo, last.hashHi, last.hashLo) > 0) {
+                loIdx = start + count;
+                loHash = last.hashHi;
+                continue;
+            }
+            let a = 0;
+            let b = count - 1;
+            while (a <= b) {
+                const mid = (a + b) >> 1;
+                const e = entryAt(bytes, mid);
+                const c = cmp(hashHi, hashLo, e.hashHi, e.hashLo);
+                if (c === 0) return e;
+                if (c < 0) b = mid - 1;
+                else a = mid + 1;
+            }
+            return null; // window brackets the hash; absence is definitive
+        }
+        return null;
+    }
+
+    return {
+        docCount,
+        termCount,
+        totalTokens,
+        lazy: true,
+        async search(text, n) {
+            const terms = [...new Set(tokenize(text))];
+            if (!terms.length || docCount === 0) return [];
+            const found = (await Promise.all(terms.map((t) => findTerm(fnv1a32(t, 0), fnv1a32(t, 0x9e3779b9)))))
+                .filter(Boolean);
+            if (!found.length) return [];
+            const postings = await Promise.all(found.map((e) => read(postingsOffset + e.postingsRel, e.postingsLen)));
+            const scores = new Map();
+            found.forEach((e, i) => {
+                const bytes = postings[i];
+                const idf = Math.log(1 + (docCount - e.df + 0.5) / (e.df + 0.5));
+                const cursor = { at: 0, end: bytes.length };
+                const readVarint = () => {
+                    let value = 0;
+                    let shift = 0;
+                    for (;;) {
+                        if (cursor.at >= cursor.end) throw new Error('.pancake lexical postings truncated');
+                        const b = bytes[cursor.at++];
+                        value += (b & 0x7f) * 2 ** shift;
+                        if (!(b & 0x80)) return value;
+                        shift += 7;
+                        if (shift > 35) throw new Error('.pancake lexical postings varint overflow');
+                    }
+                };
+                let docId = 0;
+                for (let p = 0; p < e.df; p++) {
+                    docId = p === 0 ? readVarint() : docId + readVarint();
+                    const tf = readVarint();
+                    if (docId >= docCount) throw new Error('.pancake lexical postings doc id out of range');
+                    const dl = doclenOf(docId);
+                    const norm = tf + BM25_K1 * (1 - BM25_B + (BM25_B * dl) / avgdl);
+                    scores.set(docId, (scores.get(docId) || 0) + (idf * tf * (BM25_K1 + 1)) / norm);
+                }
+            });
+            return [...scores.entries()]
+                .map(([id, score]) => ({ id, score }))
+                .sort((a, b) => (b.score - a.score) || (a.id - b.id))
+                .slice(0, n);
+        },
+    };
+}
+
+export function openLexicalIndex(bytes) {
+    if (!(bytes instanceof Uint8Array) || bytes.length < 64) {
+        throw new Error('.pancake lexical segment is too short');
+    }
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const version = view.getUint32(0, true);
+    if (version !== 1) throw new Error(`unsupported lexical segment version ${version}`);
+    const docCount = view.getUint32(4, true);
+    const termCount = view.getUint32(8, true);
+    const totalTokens = Number(view.getBigUint64(12, true));
+    const doclenOffset = view.getUint32(20, true);
+    const termTableOffset = view.getUint32(24, true);
+    const postingsOffset = view.getUint32(28, true);
+    const postingsBytes = Number(view.getBigUint64(32, true));
+    if (doclenOffset !== 64
+        || termTableOffset !== doclenOffset + 4 * docCount
+        || postingsOffset !== termTableOffset + 24 * termCount
+        || postingsOffset + postingsBytes !== bytes.length) {
+        throw new Error('.pancake lexical segment regions do not tile the segment');
+    }
+    const avgdl = docCount > 0 ? Math.max(1, totalTokens / docCount) : 1;
+
+    const entryAt = (i) => termTableOffset + 24 * i;
+    function findTerm(hashLo, hashHi) {
+        let lo = 0;
+        let hi = termCount - 1;
+        while (lo <= hi) {
+            const mid = (lo + hi) >> 1;
+            const eLo = view.getUint32(entryAt(mid), true);
+            const eHi = view.getUint32(entryAt(mid) + 4, true);
+            if (eHi === hashHi && eLo === hashLo) return mid;
+            if (eHi < hashHi || (eHi === hashHi && eLo < hashLo)) lo = mid + 1;
+            else hi = mid - 1;
+        }
+        return -1;
+    }
+
+    function readVarint(cursor) {
+        let value = 0;
+        let shift = 0;
+        for (;;) {
+            if (cursor.at >= cursor.end) throw new Error('.pancake lexical postings truncated');
+            const b = bytes[cursor.at++];
+            value += (b & 0x7f) * 2 ** shift;
+            if (!(b & 0x80)) return value;
+            shift += 7;
+            if (shift > 35) throw new Error('.pancake lexical postings varint overflow');
+        }
+    }
+
+    return {
+        docCount,
+        termCount,
+        totalTokens,
+        // BM25 over the query's unique tokens; returns [{id, score}] sorted
+        // by score descending (ties by id), at most n entries.
+        search(text, n) {
+            const terms = [...new Set(tokenize(text))];
+            if (!terms.length || docCount === 0) return [];
+            const scores = new Map();
+            for (const term of terms) {
+                const i = findTerm(fnv1a32(term, 0), fnv1a32(term, 0x9e3779b9));
+                if (i < 0) continue;
+                const rel = Number(view.getBigUint64(entryAt(i) + 8, true));
+                const len = view.getUint32(entryAt(i) + 16, true);
+                const df = view.getUint32(entryAt(i) + 20, true);
+                if (rel + len > postingsBytes) throw new Error('.pancake lexical postings out of bounds');
+                const idf = Math.log(1 + (docCount - df + 0.5) / (df + 0.5));
+                const cursor = { at: postingsOffset + rel, end: postingsOffset + rel + len };
+                let docId = 0;
+                for (let p = 0; p < df; p++) {
+                    docId = p === 0 ? readVarint(cursor) : docId + readVarint(cursor);
+                    const tf = readVarint(cursor);
+                    if (docId >= docCount) throw new Error('.pancake lexical postings doc id out of range');
+                    const dl = view.getUint32(doclenOffset + 4 * docId, true);
+                    const norm = tf + BM25_K1 * (1 - BM25_B + (BM25_B * dl) / avgdl);
+                    scores.set(docId, (scores.get(docId) || 0) + (idf * tf * (BM25_K1 + 1)) / norm);
+                }
+            }
+            return [...scores.entries()]
+                .map(([id, score]) => ({ id, score }))
+                .sort((a, b) => (b.score - a.score) || (a.id - b.id))
+                .slice(0, n);
+        },
+    };
+}

package/complete/retrieval-abstention.mjs

@@ -0,0 +1,119 @@
+// Client-side abstention scorer for the wiki pack. Mirrors the math in
+// ../../calibrate_abstention.mjs exactly: retrieval signals (d0, margin,
+// mean10) plus the corpus-vocabulary known-token fraction from the shipped
+// bloom filter, standardized and passed through the fitted logistic model.
+// Verdict semantics: 'answer' (strong match), 'weak' (closest match is
+// distant — shown with a caveat), 'abstain' (nothing useful in the pack).
+//
+// Assets calibrated by pikelet's self-templates-v2+ may carry
+// an additional grounding term (asset.coverage): the fraction of the query's
+// content words that appear in the top retrieved passage's text. Every base
+// feature measures topic similarity, so "the corpus discusses this area" and
+// "this passage answers this question" are indistinguishable without it. The
+// term is serialized outside features[]/weights[] deliberately: a reader
+// that predates it scores the topic-only model against the same thresholds
+// (a conservative degradation) instead of hitting an unknown feature name
+// and computing NaN. Word rules (min length, stopwords) ship in the asset so
+// builder and reader cannot drift.
+
+export function createAbstentionScorer(asset, bloomBytes) {
+    if (!asset || !Array.isArray(asset.weights) || !asset.thresholds) return null;
+    const coverageCfg = asset.coverage && typeof asset.coverage.weight === 'number'
+        && Number.isFinite(asset.coverage.mean) && Number.isFinite(asset.coverage.std)
+        ? asset.coverage : null;
+    const coverageStopwords = coverageCfg ? new Set(coverageCfg.stopwords || []) : null;
+    const coverageMinLen = coverageCfg ? (coverageCfg.minWordLen || 3) : 3;
+    // Words the corpus uses everywhere ground any query that mentions them,
+    // so they are excluded from coverage; the builder ships them as a bloom.
+    const commonBloom = coverageCfg?.commonBloom?.base64
+        ? (typeof Buffer !== 'undefined'
+            ? new Uint8Array(Buffer.from(coverageCfg.commonBloom.base64, 'base64'))
+            : Uint8Array.from(atob(coverageCfg.commonBloom.base64), (c) => c.charCodeAt(0)))
+        : null;
+    const commonBits = coverageCfg?.commonBloom?.bits;
+    const isCommon = commonBloom ? (w) => SEEDS.every((seed) => {
+        let h = 0x811c9dc5 ^ seed;
+        for (let i = 0; i < w.length; i++) { h ^= w.charCodeAt(i); h = Math.imul(h, 0x01000193); }
+        const bit = (h >>> 0) % commonBits;
+        return (commonBloom[bit >> 3] >> (bit & 7)) & 1;
+    }) : null;
+
+    // Max over the scored passages, with corpus-common words counted at
+    // reduced weight — mirrors create-pancake-search/src/calibrate.mjs
+    // coverageFrac exactly (the weight ships in the asset).
+    const commonWordWeight = coverageCfg?.commonWordWeight ?? 1 / 3;
+    function coverageFrac(text, passageTexts) {
+        const content = (String(text).toLowerCase().match(/[a-z0-9']+/g) || [])
+            .filter((w) => w.length >= coverageMinLen && !coverageStopwords.has(w));
+        if (!content.length) return 0;
+        const weights = content.map((w) => (isCommon && isCommon(w) ? commonWordWeight : 1));
+        const weightSum = weights.reduce((a, c) => a + c, 0);
+        let best = 0;
+        for (const passageText of passageTexts || []) {
+            const passage = new Set(String(passageText || '').toLowerCase().match(/[a-z0-9']+/g) || []);
+            const present = (w) => passage.has(w) || passage.has(`${w}s`) || passage.has(`${w}es`)
+                || (w.endsWith('s') && passage.has(w.slice(0, -1)));
+            const grounded = content.reduce((sum, w, i) => sum + (present(w) ? weights[i] : 0), 0);
+            best = Math.max(best, grounded / weightSum);
+        }
+        return best;
+    }
+    const bloom = new Uint8Array(bloomBytes);
+    const bits = asset.vocabBloom.bits;
+
+    function fnv1a(str, seed) {
+        let h = 0x811c9dc5 ^ seed;
+        for (let i = 0; i < str.length; i++) {
+            h ^= str.charCodeAt(i);
+            h = Math.imul(h, 0x01000193);
+        }
+        return (h >>> 0) % bits;
+    }
+    const SEEDS = [0, 0x9e3779b9];
+
+    function knownFrac(text) {
+        const words = text.toLowerCase().match(/[a-z0-9']+/g) || [];
+        if (!words.length) return 0;
+        let known = 0;
+        for (const w of words) {
+            const hit = SEEDS.every((seed) => {
+                const bit = fnv1a(w, seed);
+                return (bloom[bit >> 3] >> (bit & 7)) & 1;
+            });
+            if (hit) known++;
+        }
+        return known / words.length;
+    }
+
+    return {
+        // The caller must hydrate the top passagesNeeded results' text
+        // before scoring when usesPassage is true; the coverage term reads
+        // them (older assets scored one passage; topK now defaults to it).
+        usesPassage: !!coverageCfg,
+        passagesNeeded: coverageCfg ? (coverageCfg.topK || 1) : 0,
+        score(queryText, results, passageTexts) {
+            const d0 = results.length ? results[0].distance : 1;
+            const margin = results.length > 1
+                ? results[Math.min(4, results.length - 1)].distance - d0 : 0;
+            const mean10 = results.length
+                ? results.reduce((s, r) => s + r.distance, 0) / results.length : 1;
+            const signals = { d0, margin, mean10, known_frac: knownFrac(queryText) };
+            let z = asset.bias;
+            asset.features.forEach((f, j) => {
+                z += ((signals[f] - asset.standardize.mean[f]) / asset.standardize.std[f]) * asset.weights[j];
+            });
+            if (coverageCfg) {
+                signals.coverage1 = coverageFrac(queryText, Array.isArray(passageTexts) ? passageTexts : [passageTexts]);
+                z += ((signals.coverage1 - coverageCfg.mean) / (coverageCfg.std || 1)) * coverageCfg.weight;
+            }
+            // A malformed asset (unknown feature name, non-numeric term) must
+            // degrade to unscored, never to a NaN that compares false against
+            // both thresholds and answers everything.
+            if (!Number.isFinite(z)) return { p: null, verdict: 'unscored', signals };
+            const p = 1 / (1 + Math.exp(-z));
+            const verdict = p < asset.thresholds.hard ? 'abstain'
+                : p < asset.thresholds.weak ? 'weak' : 'answer';
+            return { p, verdict, signals };
+        },
+    };
+}