pikelet-wasm 0.8.0 → 0.8.1

package/complete/index.mjs

@@ -25,7 +25,7 @@ import { loadStudentModel, embedTextWithStudent } from './student-embedder.mjs';
 import { computeMatchQuality, computePreSearchAbstention } from './student-abstention.mjs';
 import { createAbstentionScorer } from './retrieval-abstention.mjs';
 import { openLexicalIndex, openLexicalIndexLazy } from './lexical.mjs';
-import { PikeletSketchArtifact } from '../pikelet-artifact.js';
+import { PikeletSketchArtifact } from '../src/artifact/pikelet-artifact.js';
 import { MAGIC, HEADER_BYTES, TABLE_ENTRY_BYTES, KINDS, KIND_NAMES } from './format.mjs';
 
 import {
@@ -41,6 +41,8 @@ export { httpRangeSource } from './sources.mjs';
 // in-memory lexical searcher over the segment bytes already assembled by
 // buildLexicalSegment, before the artifact is written to disk.
 export { openLexicalIndex } from './lexical.mjs';
+export { fuseCandidates, FUSION_DEFAULTS } from './fusion.mjs';
+import { fuseCandidates } from './fusion.mjs';
 export {
     KERNEL_LAYOUT, expectedBlobBytes, parseInlineTransformerEncoder,
     createInlineTransformerEmbedder, INLINE_TEST_VECTOR_TEXTS,
@@ -75,9 +77,21 @@ const RECORD_CACHE = 256;
 // its fixed-size header is what format 2 commits to in the manifest.
 const SKETCH_HEADER_BYTES = 256;
 // Hybrid retrieval: BM25 candidates fetched from the lexical segment per
-// query, and the reciprocal-rank-fusion constant (the standard untuned 60).
+// query; the fusion rule and its constants live in fusion.mjs, shared with
+// the calibrator so fit-time rankings match what is served.
 const LEXICAL_CANDIDATES = 24;
-const RRF_K = 60;
+// scoreQuality's coverage term hydrates the fused top passagesNeeded
+// records (asset.coverage.topK, 5 by default — see calibrate.mjs's
+// COVERAGE_TOP_PASSAGES) once retrieval returns. The scorer that knows
+// the real number is created inside retrievalScorer()'s own closure, not
+// reachable from here, so this is a fixed approximation used to decide
+// how many of fusedFull's records to include in the batched hydration
+// below, alongside phrase-pinning's own candidates — hydrating a couple
+// more than coverage strictly needs just costs a little extra bandwidth
+// on a cache entry that goes unread; hydrating fewer only narrows how
+// much of coverage's later hydration hits warm cache. Either way is
+// harmless: it's a batch-sizing choice, not a correctness one.
+const COVERAGE_BATCH_COUNT = 5;
 // Only BM25 hits within this fraction of the top lexical score join fusion.
 // Common query terms give every matching document a near-tied score (idf
 // collapses them flat); a wide cutoff let that tied mass ride into fusion on
@@ -125,13 +139,23 @@ function toHex(bytes) {
 const isSha256Hex = (value) => typeof value === 'string' && /^[0-9a-f]{64}$/.test(value);
 
 // SHA-256 via WebCrypto (browser, workerd, Node 18+); node:crypto fallback
-// for older Node. Fails closed: verification is not optional in this reader.
+// for older Node, gated on an actual Node runtime check rather than just
+// "WebCrypto is missing" — a browser can lack crypto.subtle too (privacy
+// settings, a hardened build), and a bare dynamic import('node:crypto')
+// there isn't a Node module resolution failure, it's a cross-origin module
+// fetch a browser refuses outright, surfacing as an opaque module-loading
+// error instead of the clear message below. Fails closed either way:
+// verification is not optional in this reader.
 async function sha256hex(bytes) {
     if (globalThis.crypto?.subtle) {
         return toHex(new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256', bytes)));
     }
-    const { createHash } = await import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:crypto');
-    return createHash('sha256').update(bytes).digest('hex');
+    if (typeof process !== 'undefined' && process.versions?.node) {
+        const { createHash } = await import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:crypto');
+        return createHash('sha256').update(bytes).digest('hex');
+    }
+    throw new Error('No crypto backend available (crypto.subtle is missing) — '
+        + 'cannot verify artifact integrity in this environment');
 }
 
 function asUint8Array(bytes) {
@@ -628,12 +652,25 @@ export async function openPikeletFile(input, options = {}) {
                 // record that fails its digest must fail the query, not
                 // skew its verdict.
                 const passageSource = fusedHits?.length ? fusedHits : hits;
+                // A record's text starts with its own heading echoed as the
+                // first line (ingest.mjs's section-to-chunk join). A
+                // heading match still grounds the query but at reduced
+                // credit — see retrieval-abstention.mjs's coverageFrac and
+                // calibrate.mjs's HEADING_COVERAGE_WEIGHT, which this split
+                // must match exactly (build-time and serve-time coverage
+                // must agree, or the fitted thresholds are scoring a
+                // different signal than the one they were calibrated
+                // against).
+                const splitHeadingBody = (text) => {
+                    const nl = String(text || '').indexOf('\n');
+                    return nl === -1 ? { heading: text, body: '' } : { heading: text.slice(0, nl), body: text.slice(nl + 1) };
+                };
                 const topTexts = scorer.usesPassage && passageSource.length
                     ? (await Promise.all(passageSource.slice(0, scorer.passagesNeeded || 1)
-                        .map((hit) => hydrate(hit.id)))).map((record) => record?.text)
+                        .map((hit) => hydrate(hit.id)))).map((record) => splitHeadingBody(record?.text))
                     : [];
-                const scored = scorer.score(context.text, hits, topTexts);
-                return { match_quality: VERDICTS[scored.verdict] || scored.verdict, confidence: scored.p };
+                const scored = await scorer.score(context.text, hits, topTexts);
+                return { match_quality: VERDICTS[scored.verdict] || scored.verdict, confidence: scored.p, grounding: scored.grounding || null };
             };
         };
         if (qiKind === 1) {
@@ -695,7 +732,7 @@ export async function openPikeletFile(input, options = {}) {
                 } catch (err) {
                     throw new Error('kind-3 artifact requires the inline-transformer kernels at '
                         + 'complete/encoder-kernels/ (pikelet-wasm/complete); run '
-                        + 'examples/05-one-file-search/encoder-spike/build-encoder.sh to rebuild them',
+                        + 'examples/one-file-search/encoder-spike/build-encoder.sh to rebuild them',
                     { cause: err });
                 }
             }
@@ -813,7 +850,7 @@ export async function openPikeletFile(input, options = {}) {
                 // Hidden from bundlers like the node encoder kernels: the
                 // web entrypoint's ?url wasm imports would break a browser
                 // build that statically pulled this in.
-                const { default: Pikelet } = await import(/* webpackIgnore: true */ /* @vite-ignore */ '../pikelet.node.mjs');
+                const { default: Pikelet } = await import(/* webpackIgnore: true */ /* @vite-ignore */ '../src/pikelet.node.mjs');
                 await sketch.fullyResident;
                 const created = await Pikelet.createSketchScanner(sketch, {
                     maxRerank: Math.min(sketch.count, SCANNER_MAX_RERANK),
@@ -1076,13 +1113,51 @@ export async function openPikeletFile(input, options = {}) {
                         fusedFull = lexicalHits.map((h) => byId.get(h.id)).filter(Boolean);
                     } else if (retrieval !== 'augmented' && lexicalHits.length) {
                         const lexRank = new Map(lexicalHits.map((h, i) => [h.id, i]));
-                        fusedFull = searched
-                            .map((hit, vRank) => ({
-                                hit,
-                                score: 1 / (RRF_K + vRank) + (lexRank.has(hit.id) ? 1 / (RRF_K + lexRank.get(hit.id)) : 0),
-                            }))
-                            .sort((a, b) => (b.score - a.score) || (a.hit.distance - b.hit.distance))
-                            .map((entry) => entry.hit);
+                        fusedFull = fuseCandidates(searched, lexicalHits.map((h) => h.id));
+                        // One batched hydration for everything this block and
+                        // scoreQuality's coverage term (below) will need, in
+                        // place of two separate staged fetches (phrase-pin's
+                        // own, then a later cold start for coverage): over a
+                        // remote range-read source each hydrate() a caller
+                        // hasn't already paid for is its own network round
+                        // trip, and sequential/staged awaits pay that cost
+                        // once per stage instead of once per query. The
+                        // union of phrase-pin's candidates (top lexical
+                        // hits) and coverage's (fusedFull's top few — phrase
+                        // pinning only ever reorders that window, never
+                        // changes its membership) is known right here, so
+                        // both stages can share one round trip. A record
+                        // that fails to hydrate here does not fail the
+                        // query: the id is just missing from the map, and
+                        // each site below falls through exactly as it would
+                        // on a cold cache miss (recs[i] undefined; coverage
+                        // sees fewer usable passages) — hydrate() raising
+                        // for a bad id is still a real failure, but only
+                        // once a site actually needs that specific record,
+                        // not speculatively for every candidate touched here.
+                        const phraseCandidates = lexicalHits.slice(0, 3);
+                        const coverageCandidates = fusedFull.slice(0, COVERAGE_BATCH_COUNT);
+                        const batchIds = [...new Set([...phraseCandidates, ...coverageCandidates].map((h) => h.id))];
+                        const batch = new Map();
+                        await Promise.all(batchIds.map(async (id) => {
+                            try { batch.set(id, await hydrate(id)); } catch { /* see comment above */ }
+                        }));
+                        // Phrase pinning: a verbatim occurrence of the query (>= 3
+                        // content words) in one of the top lexical hits is the
+                        // strongest evidence of support there is; rank it first.
+                        const norm = (t) => String(t || '').toLowerCase().replace(/[^a-z0-9' ]+/g, ' ').replace(/\s+/g, ' ').trim();
+                        const phrase = norm(trimmed);
+                        if (phrase.split(' ').length >= 3) {
+                            for (const h of phraseCandidates) {
+                                if (norm(batch.get(h.id)?.text).includes(phrase)) {
+                                    fusedFull = [searched.find((x) => x.id === h.id), ...fusedFull.filter((x) => x.id !== h.id)].filter(Boolean);
+                                    break;
+                                }
+                            }
+                        }
+                        // Rank transparency: a consumer can see a verbatim hit sitting
+                        // at fused rank 3 behind two semantic near-misses.
+                        for (const [i, hit] of fusedFull.entries()) { hit.fusedRank = i + 1; hit.lexicalRank = lexRank.has(hit.id) ? lexRank.get(hit.id) + 1 : null; hit.vectorRank = searched.findIndex((x) => x.id === hit.id) + 1; }
                     }
                     fused = retrieval === 'augmented' ? null : fusedFull?.slice(0, k) ?? null;
                 }
@@ -1111,9 +1186,14 @@ export async function openPikeletFile(input, options = {}) {
                 // `distance` field cannot overwrite them (reserved names).
                 const results = await Promise.all(returned.map(async (hit) => {
                     const record = await hydrate(hit.id);
-                    return { ...record, id: hit.id, distance: hit.distance };
+                    return { ...record, id: hit.id, distance: hit.distance, ...(hit.fusedRank ? { fusedRank: hit.fusedRank, lexicalRank: hit.lexicalRank, vectorRank: hit.vectorRank } : {}) };
                 }));
-                return { matchQuality: quality.match_quality, confidence: quality.confidence, results };
+                // Grounding transparency: which record's text grounded the verdict
+                // and which query words it covered, so a consumer can tell a
+                // topical match ("much money darcy make year") from an answer.
+                const passageOrder = fusedFull?.length ? fusedFull : (searched ?? hits);
+                const grounding = quality.grounding ? { ...quality.grounding, recordId: passageOrder[quality.grounding.passageIndex]?.id ?? null } : null;
+                return { matchQuality: quality.match_quality, confidence: quality.confidence, results, ...(grounding ? { grounding } : {}) };
             },
 
             /** Hydrate one corpus record by id (verified per record on format 2). */

package/complete/inline-transformer.mjs

@@ -8,6 +8,10 @@ const decoder = new TextDecoder();
 // here, host-side, before the first forward.
 export const KERNEL_LAYOUT = { V: 30522, P: 512, T: 2, D: 384, F: 1536, L: 6, B: 64, H: 12 };
 const KERNEL_MAX_SEQ = 512;
+// Smallest window that still frames interior tokens between [CLS] and [SEP]
+// with room to make progress; see the maxTokens check in
+// parseInlineTransformerEncoder().
+const MIN_DECLARED_MAX_TOKENS = 8;
 
 // Byte size the kernel's fill_layout() will consume for a given layout:
 // each quantized matrix is rows*cols u8 plus per-block f32 scales and
@@ -43,6 +47,22 @@ export function parseInlineTransformerEncoder(encoderBytes) {
   if (declaration.dim !== KERNEL_LAYOUT.D) {
     throw new Error(`.pikelet inline-encoder declares dim ${declaration.dim}; the kernel emits ${KERNEL_LAYOUT.D}`);
   }
+  // maxTokens sizes the kernel's ids/hidden heap buffers, and the windowing
+  // arithmetic (windowLen = maxSeq - 2) assumes it leaves room for [CLS] and
+  // [SEP] plus interior tokens. A pack declaring 1 makes windowLen negative,
+  // so interior.slice(w * windowLen, (w + 1) * windowLen) returns nearly the
+  // whole sequence and the forward pass writes far past both allocations; 2
+  // makes windowLen 0, so the window count is Infinity. Reject the whole
+  // range here, at parse, so nothing is allocated from an unusable value.
+  // The floor is well under any value the toolchain emits (the builder
+  // declares 512; the BEIR encoder's own --max-tokens floor is 16).
+  if (declaration.maxTokens !== undefined && declaration.maxTokens !== null) {
+    const declared = declaration.maxTokens;
+    if (!Number.isInteger(declared) || declared < MIN_DECLARED_MAX_TOKENS || declared > KERNEL_MAX_SEQ) {
+      throw new Error(`.pikelet inline-encoder declares maxTokens ${declared}; `
+        + `must be an integer in [${MIN_DECLARED_MAX_TOKENS}, ${KERNEL_MAX_SEQ}]`);
+    }
+  }
   const expected = expectedBlobBytes(layout);
   if (blob.length !== expected) {
     throw new Error(`.pikelet inline-encoder blob is ${blob.length} bytes but its declared layout implies ${expected}`);
@@ -63,7 +83,14 @@ export async function createInlineTransformerEmbedder({ declaration, vocabText,
   const tokenizer = createWordPiece(vocabText);
   const EM = await createEncoder();
   const dim = declaration.dim;
-  const maxSeq = Math.min(declaration.maxTokens || KERNEL_MAX_SEQ, KERNEL_MAX_SEQ);
+  // Clamped on BOTH sides: parseInlineTransformerEncoder() rejects an
+  // out-of-range declaration, but this function is also called directly (the
+  // BEIR ladder, encoder ablations), so the buffer sizing must not depend on
+  // that check having run.
+  const maxSeq = Math.min(
+    Math.max(MIN_DECLARED_MAX_TOKENS, Math.trunc(declaration.maxTokens) || KERNEL_MAX_SEQ),
+    KERNEL_MAX_SEQ,
+  );
   const blobPtr = EM._malloc(blob.length);
   EM.HEAPU8.set(blob, blobPtr);
   const idsPtr = EM._malloc(maxSeq * 4);
@@ -88,6 +115,11 @@ export async function createInlineTransformerEmbedder({ declaration, vocabText,
     const windowsToEncode = usesClsPooling ? 1 : windows;
     for (let w = 0; w < windowsToEncode; w++) {
       const tokenIds = [allIds[0], ...interior.slice(w * windowLen, (w + 1) * windowLen), allIds[allIds.length - 1]];
+      // The windowing above must never produce more tokens than the buffers
+      // were sized for; fail closed instead of writing past them.
+      if (tokenIds.length > maxSeq) {
+        throw new Error(`inline encoder window is ${tokenIds.length} tokens for a ${maxSeq}-token buffer`);
+      }
       new Int32Array(EM.HEAP32.buffer, idsPtr, tokenIds.length).set(tokenIds);
       const rc = EM._encoder_forward(blobPtr, idsPtr, tokenIds.length, hiddenPtr, 0);
       if (rc !== tokenIds.length) throw new Error(`inline encoder failed: ${rc}`);
@@ -112,11 +144,75 @@ export async function createInlineTransformerEmbedder({ declaration, vocabText,
     return { vector: pooled, text, windows };
   }
 
+  // Per-word vectors from the same forward pass embed() already runs — the
+  // kernel's outHidden buffer holds every token's final-layer hidden state
+  // (encoder.cpp's encoder_forward does no pooling itself; JS pools it),
+  // so this is not a second, slower embedding path: it is the same
+  // forward pass, with the per-token buffer copied out and pooled per
+  // WORD (via encodeWithWordMap's word alignment, merging WordPiece
+  // sub-tokens like "quant"+"##ization" into one vector) instead of
+  // discarded after sentence-level pooling. Used for MaxSim-style
+  // token-level grounding (complete/retrieval-abstention.mjs, calibrate.
+  // mjs), an alternative to coverage1's exact/stemmed word match that
+  // measures semantic closeness instead of lexical identity — so a query
+  // and passage that use different words for the same idea can still
+  // ground each other, the case coverage1 structurally cannot handle.
+  // Windowing/CLS-pooling limits mirror embed(): a word that falls past
+  // the first window under CLS pooling, or past the last window under
+  // mean pooling's cap, has no vector and is left out of the result.
+  async function embedWords(text) {
+    const { ids: allIds, wordOf: allWordOf, words } = tokenizer.encodeWithWordMap(String(text || ''));
+    const interior = allIds.slice(1, -1);
+    const interiorWordOf = allWordOf.slice(1, -1);
+    const windowLen = maxSeq - 2;
+    const windows = Math.max(1, Math.ceil(interior.length / windowLen));
+    const windowsToEncode = usesClsPooling ? 1 : windows;
+    // Accumulate per-word sums, then average — a word split into several
+    // WordPiece pieces gets one vector, not several.
+    const wordSums = words.map(() => new Float64Array(dim));
+    const wordCounts = words.map(() => 0);
+    for (let w = 0; w < windowsToEncode; w++) {
+      const idSlice = interior.slice(w * windowLen, (w + 1) * windowLen);
+      const wordOfSlice = interiorWordOf.slice(w * windowLen, (w + 1) * windowLen);
+      const tokenIds = [allIds[0], ...idSlice, allIds[allIds.length - 1]];
+      // The windowing above must never produce more tokens than the buffers
+      // were sized for; fail closed instead of writing past them.
+      if (tokenIds.length > maxSeq) {
+        throw new Error(`inline encoder window is ${tokenIds.length} tokens for a ${maxSeq}-token buffer`);
+      }
+      new Int32Array(EM.HEAP32.buffer, idsPtr, tokenIds.length).set(tokenIds);
+      const rc = EM._encoder_forward(blobPtr, idsPtr, tokenIds.length, hiddenPtr, 0);
+      if (rc !== tokenIds.length) throw new Error(`inline encoder failed: ${rc}`);
+      const hidden = new Float32Array(EM.HEAPF32.buffer, hiddenPtr, tokenIds.length * dim);
+      // Offset by 1: tokenIds[0] is [CLS], so idSlice's tokens start at
+      // hidden row 1, matching wordOfSlice's own indexing into idSlice.
+      for (let t = 0; t < wordOfSlice.length; t++) {
+        const wi = wordOfSlice[t];
+        if (wi === -1) continue;
+        const row = (t + 1) * dim;
+        for (let d = 0; d < dim; d++) wordSums[wi][d] += hidden[row + d];
+        wordCounts[wi]++;
+      }
+    }
+    const vectors = new Map();
+    for (let wi = 0; wi < words.length; wi++) {
+      if (wordCounts[wi] === 0) continue;
+      const v = new Float32Array(dim);
+      let norm = 0;
+      for (let d = 0; d < dim; d++) { v[d] = wordSums[wi][d] / wordCounts[wi]; norm += v[d] ** 2; }
+      norm = Math.sqrt(norm) || 1;
+      for (let d = 0; d < dim; d++) v[d] /= norm;
+      vectors.set(words[wi], v);
+    }
+    return vectors;
+  }
+
   const embedder = {
     declaration,
     dim,
     maxSeq,
     embed,
+    embedWords,
     dispose() {
       EM._free(hiddenPtr);
       EM._free(idsPtr);

package/complete/retrieval-abstention.mjs

@@ -5,16 +5,34 @@
 // Verdict semantics: 'answer' (strong match), 'weak' (closest match is
 // distant — shown with a caveat), 'abstain' (nothing useful in the pack).
 //
-// Assets calibrated by pikelet's self-templates-v2+ may carry
-// an additional grounding term (asset.coverage): the fraction of the query's
-// content words that appear in the top retrieved passage's text. Every base
-// feature measures topic similarity, so "the corpus discusses this area" and
-// "this passage answers this question" are indistinguishable without it. The
-// term is serialized outside features[]/weights[] deliberately: a reader
-// that predates it scores the topic-only model against the same thresholds
-// (a conservative degradation) instead of hitting an unknown feature name
-// and computing NaN. Word rules (min length, stopwords) ship in the asset so
+// Assets calibrated by pikelet's self-templates-v2+ may carry an additional
+// grounding term (asset.coverage): the fraction of the query's content
+// words that appear in the top retrieved passage's text (coverage1,
+// exact/stemmed match). Every base feature measures topic similarity, so
+// "the corpus discusses this area" and "this passage answers this
+// question" are indistinguishable without a grounding term. The term is
+// serialized outside features[]/weights[] deliberately: a reader that
+// predates it scores the topic-only model against the same thresholds (a
+// conservative degradation) instead of hitting an unknown feature name and
+// computing NaN. Word rules (min length, stopwords) ship in the asset so
 // builder and reader cannot drift.
+//
+// A semantic word-cosine grounding term (maxSim1, folded into grounding1
+// via Math.max with coverage1, gated by asset.coverage.useMaxSim) was
+// tried twice and reverted twice — see pikelet/src/calibrate.mjs's
+// GROUNDING_FEAT design note for the two failure modes (an unrescaled
+// version inflated the fitted threshold and caused widespread false
+// abstention; a rescaled/damped version fixed that but still made
+// real-query behavior worse than plain coverage1 alone). The mechanism
+// itself was also measured to cost 200-400 individual per-word encoder
+// calls per query once whatever built the calibration asset turned the
+// flag on — a 30-80ms search became 1.5-4.6 seconds, with nothing in the
+// response indicating why. Removed entirely (not just disabled) rather
+// than left as a dormant code path: dead code an external asset can
+// silently reactivate isn't actually dead, and both attempts already
+// failed for a structural reason (see the design note), not a tuning
+// one — there is no scenario where re-adding this via a config flag was
+// going to end differently a third time.
 
 export function createAbstentionScorer(asset, bloomBytes) {
     if (!asset || !Array.isArray(asset.weights) || !asset.thresholds) return null;
@@ -42,21 +60,83 @@ export function createAbstentionScorer(asset, bloomBytes) {
     // reduced weight — mirrors pikelet/src/calibrate.mjs
     // coverageFrac exactly (the weight ships in the asset).
     const commonWordWeight = coverageCfg?.commonWordWeight ?? 1 / 3;
-    function coverageFrac(text, passageTexts) {
+    // Light suffix stripping so "quantize" grounds against "quantization"
+    // and "configure" against "configuration" — kept byte-identical to
+    // pikelet/src/calibrate.mjs's stem() (same suffix list and order):
+    // present() otherwise only forgave plurals, so any word-form mismatch
+    // at all scored zero coverage for a passage that plainly answers the
+    // query. Deliberately conservative (STEM_MIN_LEN guards short words
+    // like "king" from over-stripping); not a general stemmer.
+    const STEM_MIN_LEN = 4;
+    const STEM_SUFFIXES = ['ization', 'isation', 'ication', 'ation', 'ition', 'tion', 'ing', 'ed', 'ate', 'ize', 'ise'];
+    function stem(w) {
+        for (const suf of STEM_SUFFIXES) {
+            if (w.length - suf.length >= STEM_MIN_LEN && w.endsWith(suf)) return w.slice(0, -suf.length);
+        }
+        if (w.length - 1 >= STEM_MIN_LEN && w.endsWith('e')) return w.slice(0, -1);
+        return w;
+    }
+    // Kept byte-identical to pikelet/src/calibrate.mjs's stripPossessive:
+    // the tokenizer regex keeps the apostrophe, so a possessive ("darcy's")
+    // is one token that never matches a plain mention ("darcy") without
+    // this, in either direction.
+    function stripPossessive(w) {
+        if (w.endsWith("'s")) return w.slice(0, -2);
+        if (w.endsWith("s'")) return w.slice(0, -1);
+        if (w.endsWith("'")) return w.slice(0, -1);
+        return w;
+    }
+    // Kept byte-identical to pikelet/src/calibrate.mjs's isNumeral: a
+    // numeral is exempted from the coverageMinLen floor because it is
+    // often the only token distinguishing two otherwise-identical
+    // passages ("Chamber 4" vs "Chamber 5"), unlike an ordinary short
+    // word the floor exists to drop.
+    const isNumeral = (w) => /^\d+$/.test(w);
+    // A word present only in a passage's heading, not its body, still
+    // grounds the query, at reduced credit — kept byte-identical to
+    // pikelet/src/calibrate.mjs's HEADING_COVERAGE_WEIGHT. Full credit
+    // there is the free-coverage bug body/heading splitting exists to fix
+    // (a title-templated positive's own words ARE the heading); zero
+    // credit is the opposite failure (a real rank-1 hit whose match is in
+    // the heading scoring no coverage at all).
+    const HEADING_COVERAGE_WEIGHT = 0.5;
+    // Returns { value, grounding } rather than writing grounding detail to
+    // a shared variable: score() is async, so a module- or closure-level
+    // "last grounding" write would race between two concurrent queries
+    // against the same open pack if anything ever awaited between this
+    // call and reading it back — the second call's write could land, and
+    // get read back, before the first call's own return. Threading the
+    // value through the return keeps it local to this call regardless.
+    function coverageFrac(text, passages) {
         const content = (String(text).toLowerCase().match(/[a-z0-9']+/g) || [])
-            .filter((w) => w.length >= coverageMinLen && !coverageStopwords.has(w));
-        if (!content.length) return 0;
+            .map(stripPossessive)
+            .filter((w) => (w.length >= coverageMinLen || isNumeral(w)) && !coverageStopwords.has(w));
+        if (!content.length) return { value: 0, grounding: null };
         const weights = content.map((w) => (isCommon && isCommon(w) ? commonWordWeight : 1));
         const weightSum = weights.reduce((a, c) => a + c, 0);
-        let best = 0;
-        for (const passageText of passageTexts || []) {
-            const passage = new Set(String(passageText || '').toLowerCase().match(/[a-z0-9']+/g) || []);
-            const present = (w) => passage.has(w) || passage.has(`${w}s`) || passage.has(`${w}es`)
-                || (w.endsWith('s') && passage.has(w.slice(0, -1)));
-            const grounded = content.reduce((sum, w, i) => sum + (present(w) ? weights[i] : 0), 0);
-            best = Math.max(best, grounded / weightSum);
+        let best = 0; let bestGrounding = null; let passageIndex = 0;
+        for (const { heading, body } of passages || []) {
+            const bodyWords = (String(body || '').toLowerCase().match(/[a-z0-9']+/g) || []).map(stripPossessive);
+            const bodySet = new Set(bodyWords);
+            const bodyStems = new Set(bodyWords.map(stem));
+            const headingSet = new Set((String(heading || '').toLowerCase().match(/[a-z0-9']+/g) || []).map(stripPossessive));
+            const presentIn = (set, stems, w) => set.has(w) || set.has(`${w}s`) || set.has(`${w}es`)
+                || (w.endsWith('s') && set.has(w.slice(0, -1)))
+                || (stems && stems.has(stem(w)));
+            const creditFor = (w) => {
+                if (presentIn(bodySet, bodyStems, w)) return 1;
+                if (presentIn(headingSet, null, w)) return HEADING_COVERAGE_WEIGHT;
+                return 0;
+            };
+            const grounded = content.reduce((sum, w, i) => sum + creditFor(w) * weights[i], 0);
+            const frac = grounded / weightSum;
+            if (frac > best) {
+                best = frac;
+                bestGrounding = { passageIndex, coverage: +frac.toFixed(3), covered: content.filter((w) => creditFor(w) > 0), uncovered: content.filter((w) => creditFor(w) === 0) };
+            }
+            passageIndex++;
         }
-        return best;
+        return { value: best, grounding: bestGrounding };
     }
     const bloom = new Uint8Array(bloomBytes);
     const bits = asset.vocabBloom.bits;
@@ -87,11 +167,13 @@ export function createAbstentionScorer(asset, bloomBytes) {
 
     return {
         // The caller must hydrate the top passagesNeeded results' text
-        // before scoring when usesPassage is true; the coverage term reads
-        // them (older assets scored one passage; topK now defaults to it).
+        // before scoring when usesPassage is true; the coverage/maxSim
+        // terms read them (older assets scored one passage; topK now
+        // defaults to it). maxSim never ships without coverage (calibrate.mjs
+        // fits them together), so coverageCfg.topK already covers both.
         usesPassage: !!coverageCfg,
         passagesNeeded: coverageCfg ? (coverageCfg.topK || 1) : 0,
-        score(queryText, results, passageTexts) {
+        async score(queryText, results, passageTexts) {
             // Fit at build time (pikelet/src/calibrate.mjs) always scores a
             // fixed top-10 window (K = min(10, candidates)), independent of
             // whatever k a caller later passes to query(). Slicing to the
@@ -110,12 +192,23 @@ export function createAbstentionScorer(asset, bloomBytes) {
             asset.features.forEach((f, j) => {
                 z += ((signals[f] - asset.standardize.mean[f]) / asset.standardize.std[f]) * asset.weights[j];
             });
+            // Local to this call (not module/closure state — see
+            // coverageFrac's comment) so concurrent queries against the
+            // same open pack can never see each other's grounding detail.
+            let grounding = null;
             if (coverageCfg) {
                 // passageTexts is hydrated by the caller for the fixed
                 // top-COVERAGE_TOP_PASSAGES window (see passagesNeeded
                 // below), independent of the caller's k, matching
                 // calibrate.mjs's top.slice(0, COVERAGE_TOP_PASSAGES).
-                signals.coverage1 = coverageFrac(queryText, Array.isArray(passageTexts) ? passageTexts : [passageTexts]);
+                const passages = Array.isArray(passageTexts) ? passageTexts : [passageTexts];
+                const coverage = coverageFrac(queryText, passages);
+                signals.coverage1 = coverage.value;
+                grounding = coverage.grounding;
+                // grounding1 is coverage1 alone — see this file's header
+                // comment and calibrate.mjs's GROUNDING_FEAT design note
+                // for why a semantic (maxSim1) blend was tried twice and
+                // removed both times.
                 z += ((signals.coverage1 - coverageCfg.mean) / (coverageCfg.std || 1)) * coverageCfg.weight;
             }
             // A malformed asset (unknown feature name, non-numeric term) must
@@ -125,7 +218,7 @@ export function createAbstentionScorer(asset, bloomBytes) {
             const p = 1 / (1 + Math.exp(-z));
             const verdict = p < asset.thresholds.hard ? 'abstain'
                 : p < asset.thresholds.weak ? 'weak' : 'answer';
-            return { p, verdict, signals };
+            return { p, verdict, signals, grounding };
         },
     };
 }

package/complete/sources.mjs

@@ -1,4 +1,4 @@
-import { pikeletError, PIKELET_ERROR_CODES } from '../pikelet-errors.js';
+import { pikeletError, PIKELET_ERROR_CODES } from '../src/errors/pikelet-errors.js';
 
 export function httpRangeSource(url, options = {}) {
     const stats = { requests: 0, bytes: 0, acceptRanges: null, etag: null, fullFallback: false, retries: 0, redirects: 0 };

package/complete/wordpiece.mjs

@@ -64,5 +64,27 @@ export function createWordPiece(vocabText) {
       ids.push(SEP);
       return ids;
     },
+    // Same as encode(), but also reports which basic (whitespace/punct-
+    // split) word each output token belongs to, as a same-length array of
+    // word indices (CLS and SEP get -1 — they belong to no word). A word
+    // that WordPiece splits into several sub-tokens (e.g. "quantization"
+    // -> "quant" + "##ization") reports the same word index for all of
+    // them, so a caller building one vector per word can average or
+    // otherwise pool the sub-token hidden states back into a single
+    // word-level vector — needed for MaxSim-style token-level grounding,
+    // where coverage must be measured in words, not WordPiece pieces.
+    encodeWithWordMap(text) {
+      const ids = [CLS];
+      const wordOf = [-1];
+      const words = basicTokenize(text);
+      words.forEach((word, wi) => {
+        const pieces = wordpiece(word);
+        ids.push(...pieces);
+        for (let i = 0; i < pieces.length; i++) wordOf.push(wi);
+      });
+      ids.push(SEP);
+      wordOf.push(-1);
+      return { ids, wordOf, words };
+    },
   };
 }