pikelet-wasm 0.7.0 → 0.8.1

package/complete/inline-transformer.mjs

@@ -7,7 +7,11 @@ const decoder = new TextDecoder();
 // reads the declaration. Declaration/blob skew is therefore only catchable
 // here, host-side, before the first forward.
 export const KERNEL_LAYOUT = { V: 30522, P: 512, T: 2, D: 384, F: 1536, L: 6, B: 64, H: 12 };
-const KERNEL_MAX_SEQ = 128;
+const KERNEL_MAX_SEQ = 512;
+// Smallest window that still frames interior tokens between [CLS] and [SEP]
+// with room to make progress; see the maxTokens check in
+// parseInlineTransformerEncoder().
+const MIN_DECLARED_MAX_TOKENS = 8;
 
 // Byte size the kernel's fill_layout() will consume for a given layout:
 // each quantized matrix is rows*cols u8 plus per-block f32 scales and
@@ -29,23 +33,39 @@ export function parseInlineTransformerEncoder(encoderBytes) {
   const vocabLen = view.getUint32(4, true);
   const blobLen = view.getUint32(8, true);
   if (12 + declLen + vocabLen + blobLen !== encoderBytes.length) {
-    throw new Error('.pancake inline-encoder layout is inconsistent');
+    throw new Error('.pikelet inline-encoder layout is inconsistent');
   }
   const declaration = JSON.parse(decoder.decode(encoderBytes.subarray(12, 12 + declLen)));
   const blob = encoderBytes.subarray(12 + declLen + vocabLen);
   const layout = declaration.layout || {};
   for (const key of Object.keys(KERNEL_LAYOUT)) {
     if (layout[key] !== KERNEL_LAYOUT[key]) {
-      throw new Error(`.pancake inline-encoder declares layout ${key}=${layout[key]}; `
+      throw new Error(`.pikelet inline-encoder declares layout ${key}=${layout[key]}; `
         + `the compiled kernel requires ${key}=${KERNEL_LAYOUT[key]}`);
     }
   }
   if (declaration.dim !== KERNEL_LAYOUT.D) {
-    throw new Error(`.pancake inline-encoder declares dim ${declaration.dim}; the kernel emits ${KERNEL_LAYOUT.D}`);
+    throw new Error(`.pikelet inline-encoder declares dim ${declaration.dim}; the kernel emits ${KERNEL_LAYOUT.D}`);
+  }
+  // maxTokens sizes the kernel's ids/hidden heap buffers, and the windowing
+  // arithmetic (windowLen = maxSeq - 2) assumes it leaves room for [CLS] and
+  // [SEP] plus interior tokens. A pack declaring 1 makes windowLen negative,
+  // so interior.slice(w * windowLen, (w + 1) * windowLen) returns nearly the
+  // whole sequence and the forward pass writes far past both allocations; 2
+  // makes windowLen 0, so the window count is Infinity. Reject the whole
+  // range here, at parse, so nothing is allocated from an unusable value.
+  // The floor is well under any value the toolchain emits (the builder
+  // declares 512; the BEIR encoder's own --max-tokens floor is 16).
+  if (declaration.maxTokens !== undefined && declaration.maxTokens !== null) {
+    const declared = declaration.maxTokens;
+    if (!Number.isInteger(declared) || declared < MIN_DECLARED_MAX_TOKENS || declared > KERNEL_MAX_SEQ) {
+      throw new Error(`.pikelet inline-encoder declares maxTokens ${declared}; `
+        + `must be an integer in [${MIN_DECLARED_MAX_TOKENS}, ${KERNEL_MAX_SEQ}]`);
+    }
   }
   const expected = expectedBlobBytes(layout);
   if (blob.length !== expected) {
-    throw new Error(`.pancake inline-encoder blob is ${blob.length} bytes but its declared layout implies ${expected}`);
+    throw new Error(`.pikelet inline-encoder blob is ${blob.length} bytes but its declared layout implies ${expected}`);
   }
   return {
     declaration,
@@ -63,27 +83,52 @@ export async function createInlineTransformerEmbedder({ declaration, vocabText,
   const tokenizer = createWordPiece(vocabText);
   const EM = await createEncoder();
   const dim = declaration.dim;
-  const maxSeq = Math.min(declaration.maxTokens || KERNEL_MAX_SEQ, KERNEL_MAX_SEQ);
+  // Clamped on BOTH sides: parseInlineTransformerEncoder() rejects an
+  // out-of-range declaration, but this function is also called directly (the
+  // BEIR ladder, encoder ablations), so the buffer sizing must not depend on
+  // that check having run.
+  const maxSeq = Math.min(
+    Math.max(MIN_DECLARED_MAX_TOKENS, Math.trunc(declaration.maxTokens) || KERNEL_MAX_SEQ),
+    KERNEL_MAX_SEQ,
+  );
   const blobPtr = EM._malloc(blob.length);
   EM.HEAPU8.set(blob, blobPtr);
   const idsPtr = EM._malloc(maxSeq * 4);
   const hiddenPtr = EM._malloc(maxSeq * dim * 4);
 
+  const usesClsPooling = declaration.pooling === 'cls';
+
   async function embed(text) {
     const allIds = tokenizer.encode(String(text || ''));
-    // Inputs longer than maxSeq are mean-pooled across [CLS]…[SEP]-framed
-    // windows instead of truncated, so long chunks keep their tail content.
+    // Inputs longer than maxSeq are windowed across [CLS]…[SEP]-framed
+    // chunks instead of truncated, so long chunks keep their tail content.
     const interior = allIds.slice(1, -1);
     const windowLen = maxSeq - 2;
     const windows = Math.max(1, Math.ceil(interior.length / windowLen));
     const pooled = new Float32Array(dim);
     let pooledTokens = 0;
-    for (let w = 0; w < windows; w++) {
+    // CLS pooling has no defined multi-window semantics (unlike mean
+    // pooling, there's no natural way to combine multiple windows' [CLS]
+    // tokens into one), so only the first window is encoded; long
+    // documents lose tail content under this pooling mode — mean pooling
+    // is the better choice for a corpus with chunks near or past maxSeq.
+    const windowsToEncode = usesClsPooling ? 1 : windows;
+    for (let w = 0; w < windowsToEncode; w++) {
       const tokenIds = [allIds[0], ...interior.slice(w * windowLen, (w + 1) * windowLen), allIds[allIds.length - 1]];
+      // The windowing above must never produce more tokens than the buffers
+      // were sized for; fail closed instead of writing past them.
+      if (tokenIds.length > maxSeq) {
+        throw new Error(`inline encoder window is ${tokenIds.length} tokens for a ${maxSeq}-token buffer`);
+      }
       new Int32Array(EM.HEAP32.buffer, idsPtr, tokenIds.length).set(tokenIds);
       const rc = EM._encoder_forward(blobPtr, idsPtr, tokenIds.length, hiddenPtr, 0);
       if (rc !== tokenIds.length) throw new Error(`inline encoder failed: ${rc}`);
       const hidden = new Float32Array(EM.HEAPF32.buffer, hiddenPtr, tokenIds.length * dim);
+      if (usesClsPooling) {
+        for (let d = 0; d < dim; d++) pooled[d] = hidden[d];
+        pooledTokens = 1;
+        break;
+      }
       for (let t = 0; t < tokenIds.length; t++) {
         for (let d = 0; d < dim; d++) pooled[d] += hidden[t * dim + d];
       }
@@ -91,7 +136,7 @@ export async function createInlineTransformerEmbedder({ declaration, vocabText,
     }
     let norm = 0;
     for (let d = 0; d < dim; d++) {
-      pooled[d] /= pooledTokens;
+      if (!usesClsPooling) pooled[d] /= pooledTokens;
       norm += pooled[d] ** 2;
     }
     norm = Math.sqrt(norm);
@@ -99,11 +144,75 @@ export async function createInlineTransformerEmbedder({ declaration, vocabText,
     return { vector: pooled, text, windows };
   }
 
+  // Per-word vectors from the same forward pass embed() already runs — the
+  // kernel's outHidden buffer holds every token's final-layer hidden state
+  // (encoder.cpp's encoder_forward does no pooling itself; JS pools it),
+  // so this is not a second, slower embedding path: it is the same
+  // forward pass, with the per-token buffer copied out and pooled per
+  // WORD (via encodeWithWordMap's word alignment, merging WordPiece
+  // sub-tokens like "quant"+"##ization" into one vector) instead of
+  // discarded after sentence-level pooling. Used for MaxSim-style
+  // token-level grounding (complete/retrieval-abstention.mjs, calibrate.
+  // mjs), an alternative to coverage1's exact/stemmed word match that
+  // measures semantic closeness instead of lexical identity — so a query
+  // and passage that use different words for the same idea can still
+  // ground each other, the case coverage1 structurally cannot handle.
+  // Windowing/CLS-pooling limits mirror embed(): a word that falls past
+  // the first window under CLS pooling, or past the last window under
+  // mean pooling's cap, has no vector and is left out of the result.
+  async function embedWords(text) {
+    const { ids: allIds, wordOf: allWordOf, words } = tokenizer.encodeWithWordMap(String(text || ''));
+    const interior = allIds.slice(1, -1);
+    const interiorWordOf = allWordOf.slice(1, -1);
+    const windowLen = maxSeq - 2;
+    const windows = Math.max(1, Math.ceil(interior.length / windowLen));
+    const windowsToEncode = usesClsPooling ? 1 : windows;
+    // Accumulate per-word sums, then average — a word split into several
+    // WordPiece pieces gets one vector, not several.
+    const wordSums = words.map(() => new Float64Array(dim));
+    const wordCounts = words.map(() => 0);
+    for (let w = 0; w < windowsToEncode; w++) {
+      const idSlice = interior.slice(w * windowLen, (w + 1) * windowLen);
+      const wordOfSlice = interiorWordOf.slice(w * windowLen, (w + 1) * windowLen);
+      const tokenIds = [allIds[0], ...idSlice, allIds[allIds.length - 1]];
+      // The windowing above must never produce more tokens than the buffers
+      // were sized for; fail closed instead of writing past them.
+      if (tokenIds.length > maxSeq) {
+        throw new Error(`inline encoder window is ${tokenIds.length} tokens for a ${maxSeq}-token buffer`);
+      }
+      new Int32Array(EM.HEAP32.buffer, idsPtr, tokenIds.length).set(tokenIds);
+      const rc = EM._encoder_forward(blobPtr, idsPtr, tokenIds.length, hiddenPtr, 0);
+      if (rc !== tokenIds.length) throw new Error(`inline encoder failed: ${rc}`);
+      const hidden = new Float32Array(EM.HEAPF32.buffer, hiddenPtr, tokenIds.length * dim);
+      // Offset by 1: tokenIds[0] is [CLS], so idSlice's tokens start at
+      // hidden row 1, matching wordOfSlice's own indexing into idSlice.
+      for (let t = 0; t < wordOfSlice.length; t++) {
+        const wi = wordOfSlice[t];
+        if (wi === -1) continue;
+        const row = (t + 1) * dim;
+        for (let d = 0; d < dim; d++) wordSums[wi][d] += hidden[row + d];
+        wordCounts[wi]++;
+      }
+    }
+    const vectors = new Map();
+    for (let wi = 0; wi < words.length; wi++) {
+      if (wordCounts[wi] === 0) continue;
+      const v = new Float32Array(dim);
+      let norm = 0;
+      for (let d = 0; d < dim; d++) { v[d] = wordSums[wi][d] / wordCounts[wi]; norm += v[d] ** 2; }
+      norm = Math.sqrt(norm) || 1;
+      for (let d = 0; d < dim; d++) v[d] /= norm;
+      vectors.set(words[wi], v);
+    }
+    return vectors;
+  }
+
   const embedder = {
     declaration,
     dim,
     maxSeq,
     embed,
+    embedWords,
     dispose() {
       EM._free(hiddenPtr);
       EM._free(idsPtr);

package/complete/lexical.mjs

@@ -43,7 +43,7 @@ const tokenize = (text) => (String(text).toLowerCase().match(/[a-z0-9']+/g) || [
 export async function openLexicalIndexLazy(read, segLength) {
     const headerBytes = await read(0, 64);
     if (!(headerBytes instanceof Uint8Array) || headerBytes.length < 64 || segLength < 64) {
-        throw new Error('.pancake lexical segment is too short');
+        throw new Error('.pikelet lexical segment is too short');
     }
     const hv = new DataView(headerBytes.buffer, headerBytes.byteOffset, headerBytes.byteLength);
     const version = hv.getUint32(0, true);
@@ -59,7 +59,7 @@ export async function openLexicalIndexLazy(read, segLength) {
         || termTableOffset !== doclenOffset + 4 * docCount
         || postingsOffset !== termTableOffset + 24 * termCount
         || postingsOffset + postingsBytes !== segLength) {
-        throw new Error('.pancake lexical segment regions do not tile the segment');
+        throw new Error('.pikelet lexical segment regions do not tile the segment');
     }
     const avgdl = docCount > 0 ? Math.max(1, totalTokens / docCount) : 1;
     const doclenBytes = docCount > 0 ? await read(doclenOffset, 4 * docCount) : new Uint8Array(0);
@@ -150,19 +150,19 @@ export async function openLexicalIndexLazy(read, segLength) {
                     let value = 0;
                     let shift = 0;
                     for (;;) {
-                        if (cursor.at >= cursor.end) throw new Error('.pancake lexical postings truncated');
+                        if (cursor.at >= cursor.end) throw new Error('.pikelet lexical postings truncated');
                         const b = bytes[cursor.at++];
                         value += (b & 0x7f) * 2 ** shift;
                         if (!(b & 0x80)) return value;
                         shift += 7;
-                        if (shift > 35) throw new Error('.pancake lexical postings varint overflow');
+                        if (shift > 35) throw new Error('.pikelet lexical postings varint overflow');
                     }
                 };
                 let docId = 0;
                 for (let p = 0; p < e.df; p++) {
                     docId = p === 0 ? readVarint() : docId + readVarint();
                     const tf = readVarint();
-                    if (docId >= docCount) throw new Error('.pancake lexical postings doc id out of range');
+                    if (docId >= docCount) throw new Error('.pikelet lexical postings doc id out of range');
                     const dl = doclenOf(docId);
                     const norm = tf + BM25_K1 * (1 - BM25_B + (BM25_B * dl) / avgdl);
                     scores.set(docId, (scores.get(docId) || 0) + (idf * tf * (BM25_K1 + 1)) / norm);
@@ -178,7 +178,7 @@ export async function openLexicalIndexLazy(read, segLength) {
 
 export function openLexicalIndex(bytes) {
     if (!(bytes instanceof Uint8Array) || bytes.length < 64) {
-        throw new Error('.pancake lexical segment is too short');
+        throw new Error('.pikelet lexical segment is too short');
     }
     const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
     const version = view.getUint32(0, true);
@@ -194,7 +194,7 @@ export function openLexicalIndex(bytes) {
         || termTableOffset !== doclenOffset + 4 * docCount
         || postingsOffset !== termTableOffset + 24 * termCount
         || postingsOffset + postingsBytes !== bytes.length) {
-        throw new Error('.pancake lexical segment regions do not tile the segment');
+        throw new Error('.pikelet lexical segment regions do not tile the segment');
     }
     const avgdl = docCount > 0 ? Math.max(1, totalTokens / docCount) : 1;
 
@@ -217,12 +217,12 @@ export function openLexicalIndex(bytes) {
         let value = 0;
         let shift = 0;
         for (;;) {
-            if (cursor.at >= cursor.end) throw new Error('.pancake lexical postings truncated');
+            if (cursor.at >= cursor.end) throw new Error('.pikelet lexical postings truncated');
             const b = bytes[cursor.at++];
             value += (b & 0x7f) * 2 ** shift;
             if (!(b & 0x80)) return value;
             shift += 7;
-            if (shift > 35) throw new Error('.pancake lexical postings varint overflow');
+            if (shift > 35) throw new Error('.pikelet lexical postings varint overflow');
         }
     }
 
@@ -242,14 +242,14 @@ export function openLexicalIndex(bytes) {
                 const rel = Number(view.getBigUint64(entryAt(i) + 8, true));
                 const len = view.getUint32(entryAt(i) + 16, true);
                 const df = view.getUint32(entryAt(i) + 20, true);
-                if (rel + len > postingsBytes) throw new Error('.pancake lexical postings out of bounds');
+                if (rel + len > postingsBytes) throw new Error('.pikelet lexical postings out of bounds');
                 const idf = Math.log(1 + (docCount - df + 0.5) / (df + 0.5));
                 const cursor = { at: postingsOffset + rel, end: postingsOffset + rel + len };
                 let docId = 0;
                 for (let p = 0; p < df; p++) {
                     docId = p === 0 ? readVarint(cursor) : docId + readVarint(cursor);
                     const tf = readVarint(cursor);
-                    if (docId >= docCount) throw new Error('.pancake lexical postings doc id out of range');
+                    if (docId >= docCount) throw new Error('.pikelet lexical postings doc id out of range');
                     const dl = view.getUint32(doclenOffset + 4 * docId, true);
                     const norm = tf + BM25_K1 * (1 - BM25_B + (BM25_B * dl) / avgdl);
                     scores.set(docId, (scores.get(docId) || 0) + (idf * tf * (BM25_K1 + 1)) / norm);

package/complete/retrieval-abstention.mjs

@@ -5,16 +5,34 @@
 // Verdict semantics: 'answer' (strong match), 'weak' (closest match is
 // distant — shown with a caveat), 'abstain' (nothing useful in the pack).
 //
-// Assets calibrated by pikelet's self-templates-v2+ may carry
-// an additional grounding term (asset.coverage): the fraction of the query's
-// content words that appear in the top retrieved passage's text. Every base
-// feature measures topic similarity, so "the corpus discusses this area" and
-// "this passage answers this question" are indistinguishable without it. The
-// term is serialized outside features[]/weights[] deliberately: a reader
-// that predates it scores the topic-only model against the same thresholds
-// (a conservative degradation) instead of hitting an unknown feature name
-// and computing NaN. Word rules (min length, stopwords) ship in the asset so
+// Assets calibrated by pikelet's self-templates-v2+ may carry an additional
+// grounding term (asset.coverage): the fraction of the query's content
+// words that appear in the top retrieved passage's text (coverage1,
+// exact/stemmed match). Every base feature measures topic similarity, so
+// "the corpus discusses this area" and "this passage answers this
+// question" are indistinguishable without a grounding term. The term is
+// serialized outside features[]/weights[] deliberately: a reader that
+// predates it scores the topic-only model against the same thresholds (a
+// conservative degradation) instead of hitting an unknown feature name and
+// computing NaN. Word rules (min length, stopwords) ship in the asset so
 // builder and reader cannot drift.
+//
+// A semantic word-cosine grounding term (maxSim1, folded into grounding1
+// via Math.max with coverage1, gated by asset.coverage.useMaxSim) was
+// tried twice and reverted twice — see pikelet/src/calibrate.mjs's
+// GROUNDING_FEAT design note for the two failure modes (an unrescaled
+// version inflated the fitted threshold and caused widespread false
+// abstention; a rescaled/damped version fixed that but still made
+// real-query behavior worse than plain coverage1 alone). The mechanism
+// itself was also measured to cost 200-400 individual per-word encoder
+// calls per query once whatever built the calibration asset turned the
+// flag on — a 30-80ms search became 1.5-4.6 seconds, with nothing in the
+// response indicating why. Removed entirely (not just disabled) rather
+// than left as a dormant code path: dead code an external asset can
+// silently reactivate isn't actually dead, and both attempts already
+// failed for a structural reason (see the design note), not a tuning
+// one — there is no scenario where re-adding this via a config flag was
+// going to end differently a third time.
 
 export function createAbstentionScorer(asset, bloomBytes) {
     if (!asset || !Array.isArray(asset.weights) || !asset.thresholds) return null;
@@ -39,24 +57,86 @@ export function createAbstentionScorer(asset, bloomBytes) {
     }) : null;
 
     // Max over the scored passages, with corpus-common words counted at
-    // reduced weight — mirrors create-pancake-search/src/calibrate.mjs
+    // reduced weight — mirrors pikelet/src/calibrate.mjs
     // coverageFrac exactly (the weight ships in the asset).
     const commonWordWeight = coverageCfg?.commonWordWeight ?? 1 / 3;
-    function coverageFrac(text, passageTexts) {
+    // Light suffix stripping so "quantize" grounds against "quantization"
+    // and "configure" against "configuration" — kept byte-identical to
+    // pikelet/src/calibrate.mjs's stem() (same suffix list and order):
+    // present() otherwise only forgave plurals, so any word-form mismatch
+    // at all scored zero coverage for a passage that plainly answers the
+    // query. Deliberately conservative (STEM_MIN_LEN guards short words
+    // like "king" from over-stripping); not a general stemmer.
+    const STEM_MIN_LEN = 4;
+    const STEM_SUFFIXES = ['ization', 'isation', 'ication', 'ation', 'ition', 'tion', 'ing', 'ed', 'ate', 'ize', 'ise'];
+    function stem(w) {
+        for (const suf of STEM_SUFFIXES) {
+            if (w.length - suf.length >= STEM_MIN_LEN && w.endsWith(suf)) return w.slice(0, -suf.length);
+        }
+        if (w.length - 1 >= STEM_MIN_LEN && w.endsWith('e')) return w.slice(0, -1);
+        return w;
+    }
+    // Kept byte-identical to pikelet/src/calibrate.mjs's stripPossessive:
+    // the tokenizer regex keeps the apostrophe, so a possessive ("darcy's")
+    // is one token that never matches a plain mention ("darcy") without
+    // this, in either direction.
+    function stripPossessive(w) {
+        if (w.endsWith("'s")) return w.slice(0, -2);
+        if (w.endsWith("s'")) return w.slice(0, -1);
+        if (w.endsWith("'")) return w.slice(0, -1);
+        return w;
+    }
+    // Kept byte-identical to pikelet/src/calibrate.mjs's isNumeral: a
+    // numeral is exempted from the coverageMinLen floor because it is
+    // often the only token distinguishing two otherwise-identical
+    // passages ("Chamber 4" vs "Chamber 5"), unlike an ordinary short
+    // word the floor exists to drop.
+    const isNumeral = (w) => /^\d+$/.test(w);
+    // A word present only in a passage's heading, not its body, still
+    // grounds the query, at reduced credit — kept byte-identical to
+    // pikelet/src/calibrate.mjs's HEADING_COVERAGE_WEIGHT. Full credit
+    // there is the free-coverage bug body/heading splitting exists to fix
+    // (a title-templated positive's own words ARE the heading); zero
+    // credit is the opposite failure (a real rank-1 hit whose match is in
+    // the heading scoring no coverage at all).
+    const HEADING_COVERAGE_WEIGHT = 0.5;
+    // Returns { value, grounding } rather than writing grounding detail to
+    // a shared variable: score() is async, so a module- or closure-level
+    // "last grounding" write would race between two concurrent queries
+    // against the same open pack if anything ever awaited between this
+    // call and reading it back — the second call's write could land, and
+    // get read back, before the first call's own return. Threading the
+    // value through the return keeps it local to this call regardless.
+    function coverageFrac(text, passages) {
         const content = (String(text).toLowerCase().match(/[a-z0-9']+/g) || [])
-            .filter((w) => w.length >= coverageMinLen && !coverageStopwords.has(w));
-        if (!content.length) return 0;
+            .map(stripPossessive)
+            .filter((w) => (w.length >= coverageMinLen || isNumeral(w)) && !coverageStopwords.has(w));
+        if (!content.length) return { value: 0, grounding: null };
         const weights = content.map((w) => (isCommon && isCommon(w) ? commonWordWeight : 1));
         const weightSum = weights.reduce((a, c) => a + c, 0);
-        let best = 0;
-        for (const passageText of passageTexts || []) {
-            const passage = new Set(String(passageText || '').toLowerCase().match(/[a-z0-9']+/g) || []);
-            const present = (w) => passage.has(w) || passage.has(`${w}s`) || passage.has(`${w}es`)
-                || (w.endsWith('s') && passage.has(w.slice(0, -1)));
-            const grounded = content.reduce((sum, w, i) => sum + (present(w) ? weights[i] : 0), 0);
-            best = Math.max(best, grounded / weightSum);
+        let best = 0; let bestGrounding = null; let passageIndex = 0;
+        for (const { heading, body } of passages || []) {
+            const bodyWords = (String(body || '').toLowerCase().match(/[a-z0-9']+/g) || []).map(stripPossessive);
+            const bodySet = new Set(bodyWords);
+            const bodyStems = new Set(bodyWords.map(stem));
+            const headingSet = new Set((String(heading || '').toLowerCase().match(/[a-z0-9']+/g) || []).map(stripPossessive));
+            const presentIn = (set, stems, w) => set.has(w) || set.has(`${w}s`) || set.has(`${w}es`)
+                || (w.endsWith('s') && set.has(w.slice(0, -1)))
+                || (stems && stems.has(stem(w)));
+            const creditFor = (w) => {
+                if (presentIn(bodySet, bodyStems, w)) return 1;
+                if (presentIn(headingSet, null, w)) return HEADING_COVERAGE_WEIGHT;
+                return 0;
+            };
+            const grounded = content.reduce((sum, w, i) => sum + creditFor(w) * weights[i], 0);
+            const frac = grounded / weightSum;
+            if (frac > best) {
+                best = frac;
+                bestGrounding = { passageIndex, coverage: +frac.toFixed(3), covered: content.filter((w) => creditFor(w) > 0), uncovered: content.filter((w) => creditFor(w) === 0) };
+            }
+            passageIndex++;
         }
-        return best;
+        return { value: best, grounding: bestGrounding };
     }
     const bloom = new Uint8Array(bloomBytes);
     const bits = asset.vocabBloom.bits;
@@ -87,23 +167,48 @@ export function createAbstentionScorer(asset, bloomBytes) {
 
     return {
         // The caller must hydrate the top passagesNeeded results' text
-        // before scoring when usesPassage is true; the coverage term reads
-        // them (older assets scored one passage; topK now defaults to it).
+        // before scoring when usesPassage is true; the coverage/maxSim
+        // terms read them (older assets scored one passage; topK now
+        // defaults to it). maxSim never ships without coverage (calibrate.mjs
+        // fits them together), so coverageCfg.topK already covers both.
         usesPassage: !!coverageCfg,
         passagesNeeded: coverageCfg ? (coverageCfg.topK || 1) : 0,
-        score(queryText, results, passageTexts) {
-            const d0 = results.length ? results[0].distance : 1;
-            const margin = results.length > 1
-                ? results[Math.min(4, results.length - 1)].distance - d0 : 0;
-            const mean10 = results.length
-                ? results.reduce((s, r) => s + r.distance, 0) / results.length : 1;
+        async score(queryText, results, passageTexts) {
+            // Fit at build time (pikelet/src/calibrate.mjs) always scores a
+            // fixed top-10 window (K = min(10, candidates)), independent of
+            // whatever k a caller later passes to query(). Slicing to the
+            // caller's k here before windowing would shrink margin/mean10/
+            // coverage's inputs at small k and change the verdict for an
+            // identical retrieval — the window, not the caller's k, is what
+            // must match the fit.
+            const top = results.slice(0, Math.min(10, results.length));
+            const d0 = top.length ? top[0].distance : 1;
+            const margin = top.length > 1
+                ? top[Math.min(4, top.length - 1)].distance - d0 : 0;
+            const mean10 = top.length
+                ? top.reduce((s, r) => s + r.distance, 0) / top.length : 1;
             const signals = { d0, margin, mean10, known_frac: knownFrac(queryText) };
             let z = asset.bias;
             asset.features.forEach((f, j) => {
                 z += ((signals[f] - asset.standardize.mean[f]) / asset.standardize.std[f]) * asset.weights[j];
             });
+            // Local to this call (not module/closure state — see
+            // coverageFrac's comment) so concurrent queries against the
+            // same open pack can never see each other's grounding detail.
+            let grounding = null;
             if (coverageCfg) {
-                signals.coverage1 = coverageFrac(queryText, Array.isArray(passageTexts) ? passageTexts : [passageTexts]);
+                // passageTexts is hydrated by the caller for the fixed
+                // top-COVERAGE_TOP_PASSAGES window (see passagesNeeded
+                // below), independent of the caller's k, matching
+                // calibrate.mjs's top.slice(0, COVERAGE_TOP_PASSAGES).
+                const passages = Array.isArray(passageTexts) ? passageTexts : [passageTexts];
+                const coverage = coverageFrac(queryText, passages);
+                signals.coverage1 = coverage.value;
+                grounding = coverage.grounding;
+                // grounding1 is coverage1 alone — see this file's header
+                // comment and calibrate.mjs's GROUNDING_FEAT design note
+                // for why a semantic (maxSim1) blend was tried twice and
+                // removed both times.
                 z += ((signals.coverage1 - coverageCfg.mean) / (coverageCfg.std || 1)) * coverageCfg.weight;
             }
             // A malformed asset (unknown feature name, non-numeric term) must
@@ -113,7 +218,7 @@ export function createAbstentionScorer(asset, bloomBytes) {
             const p = 1 / (1 + Math.exp(-z));
             const verdict = p < asset.thresholds.hard ? 'abstain'
                 : p < asset.thresholds.weak ? 'weak' : 'answer';
-            return { p, verdict, signals };
+            return { p, verdict, signals, grounding };
         },
     };
 }

package/complete/sources.mjs

@@ -1,3 +1,5 @@
+import { pikeletError, PIKELET_ERROR_CODES } from '../src/errors/pikelet-errors.js';
+
 export function httpRangeSource(url, options = {}) {
     const stats = { requests: 0, bytes: 0, acceptRanges: null, etag: null, fullFallback: false, retries: 0, redirects: 0 };
     const maxFullFallbackBytes = options.maxFullFallbackBytes ?? 64 * 1024 * 1024;
@@ -139,6 +141,25 @@ export function httpRangeSource(url, options = {}) {
                 return body;
             }
             if (response.status === 200) {
+                // If-Range has a defined meaning: a 200 instead of the
+                // requested 206 means either the host ignores Range, or the
+                // pinned ETag no longer matches (the artifact was replaced).
+                // Those need different handling — slices of a replaced file
+                // don't match what the manifest describes, and format-2's
+                // lexical segment above 8 MiB carries only a whole-segment
+                // digest, so postings read from it would go unverified.
+                // Only fall back when the ETag still matches (or neither
+                // side has one to compare).
+                if (stats.etag) {
+                    const responseEtag = response.headers.get('etag');
+                    if (responseEtag !== stats.etag) {
+                        throw pikeletError(
+                            PIKELET_ERROR_CODES.ARTIFACT_CHANGED,
+                            `artifact changed on the host: expected ETag ${stats.etag}, got ${responseEtag || '(missing)'}`,
+                            { expectedEtag: stats.etag, actualEtag: responseEtag },
+                        );
+                    }
+                }
                 // An unknown size must refuse, not pass the gate: with no
                 // Content-Length (chunked body) and init() never called,
                 // size would be 0 and the cap check silently vacuous.
@@ -177,7 +198,7 @@ export function httpRangeSource(url, options = {}) {
                     for (const chunk of chunks) { full.set(chunk, at); at += chunk.byteLength; }
                 }
                 stats.fullFallback = true;
-                console.warn('.pancake host does not honor Range; fell back to a one-time full download');
+                console.warn('.pikelet host does not honor Range; fell back to a one-time full download');
                 return full.subarray(offset, offset + length);
             }
             throw new Error(`range read failed: HTTP ${response.status}`);

package/complete/wordpiece.mjs

@@ -64,5 +64,27 @@ export function createWordPiece(vocabText) {
       ids.push(SEP);
       return ids;
     },
+    // Same as encode(), but also reports which basic (whitespace/punct-
+    // split) word each output token belongs to, as a same-length array of
+    // word indices (CLS and SEP get -1 — they belong to no word). A word
+    // that WordPiece splits into several sub-tokens (e.g. "quantization"
+    // -> "quant" + "##ization") reports the same word index for all of
+    // them, so a caller building one vector per word can average or
+    // otherwise pool the sub-token hidden states back into a single
+    // word-level vector — needed for MaxSim-style token-level grounding,
+    // where coverage must be measured in words, not WordPiece pieces.
+    encodeWithWordMap(text) {
+      const ids = [CLS];
+      const wordOf = [-1];
+      const words = basicTokenize(text);
+      words.forEach((word, wi) => {
+        const pieces = wordpiece(word);
+        ids.push(...pieces);
+        for (let i = 0; i < pieces.length; i++) wordOf.push(wi);
+      });
+      ids.push(SEP);
+      wordOf.push(-1);
+      return { ids, wordOf, words };
+    },
   };
 }