pikelet-wasm 0.8.0 → 0.9.0

package/complete/layer-rebase.mjs

@@ -0,0 +1,218 @@
+// `rebase` — LAYERED_PROFILE.md 6.2.
+//
+// Two producers appending to the same parent create a fork. Both chains are
+// valid; the ref names one head. The other layer is re-based onto it: its rows,
+// sketches, records, postings and bloom are copied verbatim (nothing is
+// re-embedded, local ids do not change), and its rowBase, bitset, supersessions
+// and evaluation material are recomputed.
+//
+// The correctness rule the spec was revised twice to get right:
+//
+//   A rebase replays what the layer ITSELF did -- its delta over its old
+//   parent, and the edges physically in its own segment -- and NEVER its
+//   cumulative bitset. T_L contains every deletion any ancestor on the old
+//   branch performed; replaying it would import those ancestors' operations
+//   into a branch that never performed them, and worse, import an ancestor's
+//   tombstone without that ancestor's supersession edge: half of one operation.
+//
+// To compute the delta, the rebase MUST open L's ORIGINAL parent. If that
+// artifact is unavailable the delta is unknowable and the rebase must refuse --
+// it cannot be reconstructed from L alone.
+
+import { zeroExtend, bitsetBytesFor, testBit, popcountBytes } from './tombstones.mjs';
+import { canonicalJson } from './layer-manifest.mjs';
+
+export const ON_CONFLICT = Object.freeze({ REFUSE: 'refuse', SKIP: 'skip', KEEP_BOTH: 'keep-both' });
+export const ON_FOREIGN = Object.freeze({ REFUSE: 'refuse', DROP: 'drop' });
+
+/**
+ * The fork point: the longest common prefix of L's original parent chain and
+ * H's chain, compared layer by layer from the base BY IDENTITY, and the total
+ * row count of that prefix.
+ *
+ * Every id below forkRowBase names the same record in both histories. Every id
+ * at or above it belongs to a layer the two histories do not share — fork
+ * siblings assign overlapping id ranges to different records, so such an id is
+ * foreign.
+ *
+ * @param {Array<{identity: string, records: number}>} oldChain base-first, L's ancestors
+ * @param {Array<{identity: string, records: number}>} newChain base-first, H's chain
+ */
+export function forkPoint(oldChain, newChain) {
+    let i = 0;
+    let forkRowBase = 0;
+    while (i < oldChain.length && i < newChain.length
+           && oldChain[i].identity === newChain[i].identity) {
+        forkRowBase += oldChain[i].records;
+        i += 1;
+    }
+    if (i === 0) {
+        throw new Error('the two histories share no base: a rebase requires H.baseIdentity == L.baseIdentity (6.2)');
+    }
+    return { sharedPrefix: i, forkRowBase };
+}
+
+/**
+ * Preconditions a rebase must satisfy before it constructs any bytes (6.2).
+ * They are producer preconditions precisely so a rebase never emits a layer a
+ * reader would refuse.
+ */
+export function checkRebasePreconditions({ layerManifest, headChainDecl, layerChainDecl, headIsLegacyBase = false, headBaseIdentity, layerBaseIdentity }) {
+    if (headBaseIdentity !== layerBaseIdentity) {
+        throw new Error(`rebase refuses: the head's base ${String(headBaseIdentity).slice(0, 12)}… is not the layer's base ${String(layerBaseIdentity).slice(0, 12)}… (6.2)`);
+    }
+    // Same base is NOT enough. Two depth-1 forks of a legacy base can each have
+    // asserted a DIFFERENT declaration, and copying L's layer.ingest onto H's
+    // history would emit a layer a reader must reject.
+    if (headIsLegacyBase) {
+        // H is the legacy depth-0 base itself and has no declaration yet, so a
+        // rebased depth-1 layer MAY introduce L's asserted declaration.
+        if (layerChainDecl && !layerChainDecl.asserted) {
+            throw new Error('rebase refuses: the head is a legacy base with no declaration, but the layer\'s declaration is not asserted (6.2)');
+        }
+        return { ingest: layerManifest.layer.ingest, ingestAsserted: true };
+    }
+    if (!headChainDecl || !layerChainDecl) {
+        throw new Error('rebase refuses: an ingestion declaration is missing on one of the two histories (6.1, 6.2)');
+    }
+    if (headChainDecl.canonical !== layerChainDecl.canonical) {
+        throw new Error('rebase refuses: the two histories disagree on the chain ingestion declaration; a copied layer.ingest would be rejected by a reader (6.2)');
+    }
+    return { ingest: layerManifest.layer.ingest, ingestAsserted: headChainDecl.asserted };
+}
+
+/**
+ * Plan a rebase.
+ *
+ * @param {object} spec
+ * @param {Uint8Array} spec.layerBitset        T_L, the layer's own cumulative bitset
+ * @param {Uint8Array} spec.oldParentBitset    T_oldParent — REQUIRED; the delta is
+ *   unknowable without it, and a rebase must refuse rather than guess.
+ * @param {number} spec.oldRowBase             L.layer.rowBase
+ * @param {number} spec.layerRecords           R_L
+ * @param {Array<[number, number]>} spec.layerSupersessions  S_L, physically recorded in L
+ * @param {number} spec.newRowBase             H.rowBase + H.records
+ * @param {Uint8Array} spec.headBitset         T_H
+ * @param {Map<number, Array<{depth:number,newId:number}>>} spec.headSupersessions  S_H
+ * @param {number} spec.forkRowBase
+ * @param {'refuse'|'drop'} [spec.onForeign]
+ * @param {'refuse'|'skip'|'keep-both'} [spec.onConflict]
+ */
+export function planRebase({
+    layerBitset, oldParentBitset, oldRowBase, layerRecords, layerSupersessions = [],
+    newRowBase, headBitset, headSupersessions = new Map(), forkRowBase,
+    onForeign = ON_FOREIGN.REFUSE, onConflict = ON_CONFLICT.REFUSE,
+}) {
+    if (!(oldParentBitset instanceof Uint8Array)) {
+        throw new Error('rebase refuses: the layer\'s original parent is unavailable, so its delta over that parent is unknowable (6.2)');
+    }
+    if (!Object.values(ON_FOREIGN).includes(onForeign)) throw new Error(`--on-foreign must be one of ${Object.values(ON_FOREIGN).join(', ')}`);
+    if (!Object.values(ON_CONFLICT).includes(onConflict)) throw new Error(`--on-conflict must be one of ${Object.values(ON_CONFLICT).join(', ')}`);
+
+    // ΔT_L = T_L \ zeroExtend(T_oldParent, L.rowBase) — never T_L as a whole.
+    const inheritedOld = zeroExtend(oldParentBitset, oldRowBase);
+    const delta = [];
+    for (let id = 0; id < oldRowBase; id++) {
+        if (testBit(layerBitset, id) && !testBit(inheritedOld, id)) delta.push(id);
+    }
+
+    // Two kinds of id, treated oppositely (6.2). Ids L OWNS — its records and
+    // the y side of every edge — are offsets into its own range and must be
+    // translated. Ids L REFERENCES — ΔT_L and the x side — are ancestor ids,
+    // stable only below the fork point.
+    const translateY = (oldY) => newRowBase + (oldY - oldRowBase);
+
+    // Foreign ids: at or above forkRowBase they belong to a layer the two
+    // histories do not share. The test applies to ΔT_L, not to T_L: bits L
+    // inherited from its old branch are not L's to carry anywhere.
+    const foreignDeletions = delta.filter((x) => x >= forkRowBase);
+    const foreignEdges = layerSupersessions.filter(([x]) => x >= forkRowBase);
+    if ((foreignDeletions.length || foreignEdges.length) && onForeign === ON_FOREIGN.REFUSE) {
+        const ids = [...new Set([...foreignDeletions, ...foreignEdges.map(([x]) => x)])].sort((a, b) => a - b);
+        throw new Error(`rebase refuses: ${ids.length} foreign id(s) at or above the fork point ${forkRowBase} name different records in the two histories: ${ids.join(', ')} (pass --on-foreign drop to discard them) (6.2)`);
+    }
+    const survivingDeletions = delta.filter((x) => x < forkRowBase);
+    const survivingEdges = layerSupersessions.filter(([x]) => x < forkRowBase);
+    const droppedForeign = { deletions: foreignDeletions, edges: foreignEdges.map(([x]) => x) };
+
+    // Conflicts, deterministically (6.2).
+    const conflicts = [];
+    const recordedEdges = [];
+    const skippedEdges = [];
+    for (const [x, oldY] of survivingEdges) {
+        const newY = translateY(oldY);
+        // Every newId a rebased layer writes MUST lie in its new range.
+        if (newY < newRowBase || newY >= newRowBase + layerRecords) {
+            throw new Error(`rebase refuses: translated newId ${newY} for oldId ${x} falls outside the layer's new range [${newRowBase}, ${newRowBase + layerRecords}) (6.2)`);
+        }
+        const headEdges = headSupersessions.get(x);
+        const headY = headEdges && headEdges.length ? headEdges[headEdges.length - 1].newId : null;
+        if (headY !== null && headY !== newY) {
+            // H already records a DIFFERENT successor for x.
+            if (onConflict === ON_CONFLICT.REFUSE) { conflicts.push(x); continue; }
+            if (onConflict === ON_CONFLICT.SKIP) { skippedEdges.push(x); continue; }
+            // keep-both: L's edge is recorded as an additional edge, which makes
+            // 3.5's "all edges" view non-trivial — H's edge is the immediate
+            // successor and L's the current one (greatest depth wins).
+            recordedEdges.push([x, newY]);
+            continue;
+        }
+        // x not tombstoned in H, or tombstoned with no edge: recorded either
+        // way — L adds information H lacked.
+        recordedEdges.push([x, newY]);
+    }
+    if (conflicts.length) {
+        throw new Error(`rebase refuses: ${conflicts.length} supersession conflict(s) — the head already records a different successor for id(s) ${conflicts.join(', ')} (pass --on-conflict skip or keep-both) (6.2)`);
+    }
+
+    // The rebased bitset is zeroExtend(T_H, newRowBase) with the surviving
+    // ΔT_L bits set. A deletion already in T_H is a no-op, not a conflict:
+    // deletion is idempotent.
+    const bitset = new Uint8Array(bitsetBytesFor(newRowBase));
+    bitset.set(zeroExtend(headBitset, newRowBase).subarray(0, bitset.length));
+    const newlySet = [];
+    const setBit = (id) => {
+        if (id >= newRowBase) {
+            throw new Error(`rebase refuses: id ${id} is at or above the new rowBase ${newRowBase}`);
+        }
+        if (testBit(bitset, id)) return false;
+        bitset[id >> 3] |= 1 << (id & 7);
+        newlySet.push(id);
+        return true;
+    };
+    for (const x of survivingDeletions) setBit(x);
+    for (const [x] of recordedEdges) setBit(x);
+
+    // Terminal rule: an empty rebase emits NOTHING (6.2). If the layer owns no
+    // records and its surviving deletions set no bit the head does not already
+    // have, emitting would manufacture exactly the artifact 6.1 forbids and 4.3
+    // rejects — records 0 with no increase in tombstoneCount.
+    if (layerRecords === 0 && newlySet.length === 0) {
+        return {
+            emit: false,
+            reason: 'the rebased operation is empty: the target history already represents every deletion this layer performed (6.2)',
+            droppedForeign, skippedEdges,
+        };
+    }
+
+    return {
+        emit: true,
+        newRowBase,
+        records: layerRecords,
+        bitset,
+        tombstones: popcountBytes(bitset),
+        delta: survivingDeletions,
+        newlySet: newlySet.sort((a, b) => a - b),
+        supersessions: recordedEdges.sort((a, b) => a[0] - b[0]),
+        droppedForeign,
+        skippedEdges,
+        // Chain-level golden queries carry expected GLOBAL ids, which are wrong
+        // after a rebase: L's own ids moved and the surrounding chain changed.
+        // A rebase MUST either regenerate them against the rebased chain or drop
+        // them explicitly; it MUST NOT copy them (6.2).
+        goldenQueries: null,
+        goldenQueriesDropped: 'rebase',
+    };
+}
+
+export { canonicalJson };

package/complete/lexical.mjs

@@ -40,6 +40,19 @@ const tokenize = (text) => (String(text).toLowerCase().match(/[a-z0-9']+/g) || [
 // is committed via the manifest digest but lazy reads are not individually
 // verified (the eager opener verifies the whole segment; per-page digests
 // are the format-v2 path if this gap needs closing).
+// The df a term is scored with: the chain-global one from options.df when the
+// caller supplied it (LAYERED_PROFILE.md 5.4), else the index's own. A global
+// df below the local one is impossible for a real sum, so it is refused rather
+// than producing a negative-count idf.
+function globalDf(options, term, localDf) {
+    const g = options.df instanceof Map ? options.df.get(term) : undefined;
+    if (g === undefined) return localDf;
+    if (!Number.isSafeInteger(g) || g < localDf) {
+        throw new Error(`lexical options.df for "${term}" (${g}) is below this index's own df (${localDf})`);
+    }
+    return g;
+}
+
 export async function openLexicalIndexLazy(read, segLength) {
     const headerBytes = await read(0, 64);
     if (!(headerBytes instanceof Uint8Array) || headerBytes.length < 64 || segLength < 64) {
@@ -134,17 +147,46 @@ export async function openLexicalIndexLazy(read, segLength) {
         termCount,
         totalTokens,
         lazy: true,
-        async search(text, n) {
+        // This index's df for each of the query's terms (absent terms omitted).
+        async documentFrequencies(text) {
+            const terms = [...new Set(tokenize(text))];
+            const out = new Map();
+            await Promise.all(terms.map(async (t) => {
+                const e = await findTerm(fnv1a32(t, 0), fnv1a32(t, 0x9e3779b9));
+                if (e) out.set(t, e.df);
+            }));
+            return out;
+        },
+        // `options.exclude(id)` drops a row BEFORE the cap, and
+        // `options.stats` supplies chain-global {docCount, avgdl} for IDF and
+        // length normalization. Both exist for the layered profile
+        // (LAYERED_PROFILE.md 5.3, 5.4): a chain used to cap each tier at n
+        // and mask afterwards, so a tier whose top-n were all tombstoned
+        // contributed nothing while live rows below the cap were discarded;
+        // and each tier scored BM25 against its own docCount, making scores
+        // from different tiers incomparable before fusion. `options.df`
+        // (Map term -> df) is the chain-global df summed over tiers from each
+        // tier's documentFrequencies(); a term absent from it keeps this
+        // index's own df.
+        async search(text, n, options = {}) {
+            const exclude = typeof options.exclude === 'function' ? options.exclude : null;
+            const statDocCount = Number.isFinite(options.stats?.docCount) && options.stats.docCount > 0
+                ? options.stats.docCount : docCount;
+            const statAvgdl = Number.isFinite(options.stats?.avgdl) && options.stats.avgdl > 0
+                ? options.stats.avgdl : avgdl;
             const terms = [...new Set(tokenize(text))];
             if (!terms.length || docCount === 0) return [];
-            const found = (await Promise.all(terms.map((t) => findTerm(fnv1a32(t, 0), fnv1a32(t, 0x9e3779b9)))))
-                .filter(Boolean);
+            const found = (await Promise.all(terms.map(async (t) => {
+                const e = await findTerm(fnv1a32(t, 0), fnv1a32(t, 0x9e3779b9));
+                return e && { ...e, term: t };
+            }))).filter(Boolean);
             if (!found.length) return [];
             const postings = await Promise.all(found.map((e) => read(postingsOffset + e.postingsRel, e.postingsLen)));
             const scores = new Map();
             found.forEach((e, i) => {
                 const bytes = postings[i];
-                const idf = Math.log(1 + (docCount - e.df + 0.5) / (e.df + 0.5));
+                const df = globalDf(options, e.term, e.df);
+                const idf = Math.log(1 + (statDocCount - df + 0.5) / (df + 0.5));
                 const cursor = { at: 0, end: bytes.length };
                 const readVarint = () => {
                     let value = 0;
@@ -163,8 +205,9 @@ export async function openLexicalIndexLazy(read, segLength) {
                     docId = p === 0 ? readVarint() : docId + readVarint();
                     const tf = readVarint();
                     if (docId >= docCount) throw new Error('.pikelet lexical postings doc id out of range');
+                    if (exclude && exclude(docId)) continue;
                     const dl = doclenOf(docId);
-                    const norm = tf + BM25_K1 * (1 - BM25_B + (BM25_B * dl) / avgdl);
+                    const norm = tf + BM25_K1 * (1 - BM25_B + (BM25_B * dl) / statAvgdl);
                     scores.set(docId, (scores.get(docId) || 0) + (idf * tf * (BM25_K1 + 1)) / norm);
                 }
             });
@@ -230,9 +273,34 @@ export function openLexicalIndex(bytes) {
         docCount,
         termCount,
         totalTokens,
+        // This index's df for each of the query's terms (absent terms omitted).
+        documentFrequencies(text) {
+            const out = new Map();
+            for (const term of new Set(tokenize(text))) {
+                const i = findTerm(fnv1a32(term, 0), fnv1a32(term, 0x9e3779b9));
+                if (i >= 0) out.set(term, view.getUint32(entryAt(i) + 20, true));
+            }
+            return out;
+        },
         // BM25 over the query's unique tokens; returns [{id, score}] sorted
         // by score descending (ties by id), at most n entries.
-        search(text, n) {
+        // `options.exclude(id)` drops a row BEFORE the cap, and
+        // `options.stats` supplies chain-global {docCount, avgdl} for IDF and
+        // length normalization. Both exist for the layered profile
+        // (LAYERED_PROFILE.md 5.3, 5.4): a chain used to cap each tier at n
+        // and mask afterwards, so a tier whose top-n were all tombstoned
+        // contributed nothing while live rows below the cap were discarded;
+        // and each tier scored BM25 against its own docCount, making scores
+        // from different tiers incomparable before fusion. `options.df`
+        // (Map term -> df) is the chain-global df summed over tiers from each
+        // tier's documentFrequencies(); a term absent from it keeps this
+        // index's own df.
+        search(text, n, options = {}) {
+            const exclude = typeof options.exclude === 'function' ? options.exclude : null;
+            const statDocCount = Number.isFinite(options.stats?.docCount) && options.stats.docCount > 0
+                ? options.stats.docCount : docCount;
+            const statAvgdl = Number.isFinite(options.stats?.avgdl) && options.stats.avgdl > 0
+                ? options.stats.avgdl : avgdl;
             const terms = [...new Set(tokenize(text))];
             if (!terms.length || docCount === 0) return [];
             const scores = new Map();
@@ -243,15 +311,17 @@ export function openLexicalIndex(bytes) {
                 const len = view.getUint32(entryAt(i) + 16, true);
                 const df = view.getUint32(entryAt(i) + 20, true);
                 if (rel + len > postingsBytes) throw new Error('.pikelet lexical postings out of bounds');
-                const idf = Math.log(1 + (docCount - df + 0.5) / (df + 0.5));
+                const scoringDf = globalDf(options, term, df);
+                const idf = Math.log(1 + (statDocCount - scoringDf + 0.5) / (scoringDf + 0.5));
                 const cursor = { at: postingsOffset + rel, end: postingsOffset + rel + len };
                 let docId = 0;
                 for (let p = 0; p < df; p++) {
                     docId = p === 0 ? readVarint(cursor) : docId + readVarint(cursor);
                     const tf = readVarint(cursor);
                     if (docId >= docCount) throw new Error('.pikelet lexical postings doc id out of range');
+                    if (exclude && exclude(docId)) continue;
                     const dl = view.getUint32(doclenOffset + 4 * docId, true);
-                    const norm = tf + BM25_K1 * (1 - BM25_B + (BM25_B * dl) / avgdl);
+                    const norm = tf + BM25_K1 * (1 - BM25_B + (BM25_B * dl) / statAvgdl);
                     scores.set(docId, (scores.get(docId) || 0) + (idf * tf * (BM25_K1 + 1)) / norm);
                 }
             }

package/complete/retrieval-abstention.mjs

@@ -5,16 +5,34 @@
 // Verdict semantics: 'answer' (strong match), 'weak' (closest match is
 // distant — shown with a caveat), 'abstain' (nothing useful in the pack).
 //
-// Assets calibrated by pikelet's self-templates-v2+ may carry
-// an additional grounding term (asset.coverage): the fraction of the query's
-// content words that appear in the top retrieved passage's text. Every base
-// feature measures topic similarity, so "the corpus discusses this area" and
-// "this passage answers this question" are indistinguishable without it. The
-// term is serialized outside features[]/weights[] deliberately: a reader
-// that predates it scores the topic-only model against the same thresholds
-// (a conservative degradation) instead of hitting an unknown feature name
-// and computing NaN. Word rules (min length, stopwords) ship in the asset so
+// Assets calibrated by pikelet's self-templates-v2+ may carry an additional
+// grounding term (asset.coverage): the fraction of the query's content
+// words that appear in the top retrieved passage's text (coverage1,
+// exact/stemmed match). Every base feature measures topic similarity, so
+// "the corpus discusses this area" and "this passage answers this
+// question" are indistinguishable without a grounding term. The term is
+// serialized outside features[]/weights[] deliberately: a reader that
+// predates it scores the topic-only model against the same thresholds (a
+// conservative degradation) instead of hitting an unknown feature name and
+// computing NaN. Word rules (min length, stopwords) ship in the asset so
 // builder and reader cannot drift.
+//
+// A semantic word-cosine grounding term (maxSim1, folded into grounding1
+// via Math.max with coverage1, gated by asset.coverage.useMaxSim) was
+// tried twice and reverted twice — see pikelet/src/calibrate.mjs's
+// GROUNDING_FEAT design note for the two failure modes (an unrescaled
+// version inflated the fitted threshold and caused widespread false
+// abstention; a rescaled/damped version fixed that but still made
+// real-query behavior worse than plain coverage1 alone). The mechanism
+// itself was also measured to cost 200-400 individual per-word encoder
+// calls per query once whatever built the calibration asset turned the
+// flag on — a 30-80ms search became 1.5-4.6 seconds, with nothing in the
+// response indicating why. Removed entirely (not just disabled) rather
+// than left as a dormant code path: dead code an external asset can
+// silently reactivate isn't actually dead, and both attempts already
+// failed for a structural reason (see the design note), not a tuning
+// one — there is no scenario where re-adding this via a config flag was
+// going to end differently a third time.
 
 export function createAbstentionScorer(asset, bloomBytes) {
     if (!asset || !Array.isArray(asset.weights) || !asset.thresholds) return null;
@@ -42,21 +60,83 @@ export function createAbstentionScorer(asset, bloomBytes) {
     // reduced weight — mirrors pikelet/src/calibrate.mjs
     // coverageFrac exactly (the weight ships in the asset).
     const commonWordWeight = coverageCfg?.commonWordWeight ?? 1 / 3;
-    function coverageFrac(text, passageTexts) {
+    // Light suffix stripping so "quantize" grounds against "quantization"
+    // and "configure" against "configuration" — kept byte-identical to
+    // pikelet/src/calibrate.mjs's stem() (same suffix list and order):
+    // present() otherwise only forgave plurals, so any word-form mismatch
+    // at all scored zero coverage for a passage that plainly answers the
+    // query. Deliberately conservative (STEM_MIN_LEN guards short words
+    // like "king" from over-stripping); not a general stemmer.
+    const STEM_MIN_LEN = 4;
+    const STEM_SUFFIXES = ['ization', 'isation', 'ication', 'ation', 'ition', 'tion', 'ing', 'ed', 'ate', 'ize', 'ise'];
+    function stem(w) {
+        for (const suf of STEM_SUFFIXES) {
+            if (w.length - suf.length >= STEM_MIN_LEN && w.endsWith(suf)) return w.slice(0, -suf.length);
+        }
+        if (w.length - 1 >= STEM_MIN_LEN && w.endsWith('e')) return w.slice(0, -1);
+        return w;
+    }
+    // Kept byte-identical to pikelet/src/calibrate.mjs's stripPossessive:
+    // the tokenizer regex keeps the apostrophe, so a possessive ("darcy's")
+    // is one token that never matches a plain mention ("darcy") without
+    // this, in either direction.
+    function stripPossessive(w) {
+        if (w.endsWith("'s")) return w.slice(0, -2);
+        if (w.endsWith("s'")) return w.slice(0, -1);
+        if (w.endsWith("'")) return w.slice(0, -1);
+        return w;
+    }
+    // Kept byte-identical to pikelet/src/calibrate.mjs's isNumeral: a
+    // numeral is exempted from the coverageMinLen floor because it is
+    // often the only token distinguishing two otherwise-identical
+    // passages ("Chamber 4" vs "Chamber 5"), unlike an ordinary short
+    // word the floor exists to drop.
+    const isNumeral = (w) => /^\d+$/.test(w);
+    // A word present only in a passage's heading, not its body, still
+    // grounds the query, at reduced credit — kept byte-identical to
+    // pikelet/src/calibrate.mjs's HEADING_COVERAGE_WEIGHT. Full credit
+    // there is the free-coverage bug body/heading splitting exists to fix
+    // (a title-templated positive's own words ARE the heading); zero
+    // credit is the opposite failure (a real rank-1 hit whose match is in
+    // the heading scoring no coverage at all).
+    const HEADING_COVERAGE_WEIGHT = 0.5;
+    // Returns { value, grounding } rather than writing grounding detail to
+    // a shared variable: score() is async, so a module- or closure-level
+    // "last grounding" write would race between two concurrent queries
+    // against the same open pack if anything ever awaited between this
+    // call and reading it back — the second call's write could land, and
+    // get read back, before the first call's own return. Threading the
+    // value through the return keeps it local to this call regardless.
+    function coverageFrac(text, passages) {
         const content = (String(text).toLowerCase().match(/[a-z0-9']+/g) || [])
-            .filter((w) => w.length >= coverageMinLen && !coverageStopwords.has(w));
-        if (!content.length) return 0;
+            .map(stripPossessive)
+            .filter((w) => (w.length >= coverageMinLen || isNumeral(w)) && !coverageStopwords.has(w));
+        if (!content.length) return { value: 0, grounding: null };
         const weights = content.map((w) => (isCommon && isCommon(w) ? commonWordWeight : 1));
         const weightSum = weights.reduce((a, c) => a + c, 0);
-        let best = 0;
-        for (const passageText of passageTexts || []) {
-            const passage = new Set(String(passageText || '').toLowerCase().match(/[a-z0-9']+/g) || []);
-            const present = (w) => passage.has(w) || passage.has(`${w}s`) || passage.has(`${w}es`)
-                || (w.endsWith('s') && passage.has(w.slice(0, -1)));
-            const grounded = content.reduce((sum, w, i) => sum + (present(w) ? weights[i] : 0), 0);
-            best = Math.max(best, grounded / weightSum);
+        let best = 0; let bestGrounding = null; let passageIndex = 0;
+        for (const { heading, body } of passages || []) {
+            const bodyWords = (String(body || '').toLowerCase().match(/[a-z0-9']+/g) || []).map(stripPossessive);
+            const bodySet = new Set(bodyWords);
+            const bodyStems = new Set(bodyWords.map(stem));
+            const headingSet = new Set((String(heading || '').toLowerCase().match(/[a-z0-9']+/g) || []).map(stripPossessive));
+            const presentIn = (set, stems, w) => set.has(w) || set.has(`${w}s`) || set.has(`${w}es`)
+                || (w.endsWith('s') && set.has(w.slice(0, -1)))
+                || (stems && stems.has(stem(w)));
+            const creditFor = (w) => {
+                if (presentIn(bodySet, bodyStems, w)) return 1;
+                if (presentIn(headingSet, null, w)) return HEADING_COVERAGE_WEIGHT;
+                return 0;
+            };
+            const grounded = content.reduce((sum, w, i) => sum + creditFor(w) * weights[i], 0);
+            const frac = grounded / weightSum;
+            if (frac > best) {
+                best = frac;
+                bestGrounding = { passageIndex, coverage: +frac.toFixed(3), covered: content.filter((w) => creditFor(w) > 0), uncovered: content.filter((w) => creditFor(w) === 0) };
+            }
+            passageIndex++;
         }
-        return best;
+        return { value: best, grounding: bestGrounding };
     }
     const bloom = new Uint8Array(bloomBytes);
     const bits = asset.vocabBloom.bits;
@@ -71,14 +151,25 @@ export function createAbstentionScorer(asset, bloomBytes) {
     }
     const SEEDS = [0, 0x9e3779b9];
 
-    function knownFrac(text) {
+    // `override` is the layered profile's union bloom (LAYERED_PROFILE.md
+    // 4.5): the bitwise OR of the base's bloom and every mounted layer's,
+    // which a chain MUST score against rather than the base's alone.
+    // Identical geometry is enforced where the union is built, and re-checked
+    // here because reading a different-sized bloom with these `bits` would
+    // silently probe the wrong positions rather than fail.
+    function knownFrac(text, override) {
+        const table = override || bloom;
+        if (table !== bloom && table.length !== bloom.length) {
+            throw new Error(`abstention: bloom override is ${table.length} bytes but the fit's is ${bloom.length}; `
+                + 'known_frac would probe the wrong bits');
+        }
         const words = text.toLowerCase().match(/[a-z0-9']+/g) || [];
         if (!words.length) return 0;
         let known = 0;
         for (const w of words) {
             const hit = SEEDS.every((seed) => {
                 const bit = fnv1a(w, seed);
-                return (bloom[bit >> 3] >> (bit & 7)) & 1;
+                return (table[bit >> 3] >> (bit & 7)) & 1;
             });
             if (hit) known++;
         }
@@ -87,11 +178,13 @@ export function createAbstentionScorer(asset, bloomBytes) {
 
     return {
         // The caller must hydrate the top passagesNeeded results' text
-        // before scoring when usesPassage is true; the coverage term reads
-        // them (older assets scored one passage; topK now defaults to it).
+        // before scoring when usesPassage is true; the coverage/maxSim
+        // terms read them (older assets scored one passage; topK now
+        // defaults to it). maxSim never ships without coverage (calibrate.mjs
+        // fits them together), so coverageCfg.topK already covers both.
         usesPassage: !!coverageCfg,
         passagesNeeded: coverageCfg ? (coverageCfg.topK || 1) : 0,
-        score(queryText, results, passageTexts) {
+        async score(queryText, results, passageTexts, options = {}) {
             // Fit at build time (pikelet/src/calibrate.mjs) always scores a
             // fixed top-10 window (K = min(10, candidates)), independent of
             // whatever k a caller later passes to query(). Slicing to the
@@ -105,17 +198,28 @@ export function createAbstentionScorer(asset, bloomBytes) {
                 ? top[Math.min(4, top.length - 1)].distance - d0 : 0;
             const mean10 = top.length
                 ? top.reduce((s, r) => s + r.distance, 0) / top.length : 1;
-            const signals = { d0, margin, mean10, known_frac: knownFrac(queryText) };
+            const signals = { d0, margin, mean10, known_frac: knownFrac(queryText, options.vocabBloom) };
             let z = asset.bias;
             asset.features.forEach((f, j) => {
                 z += ((signals[f] - asset.standardize.mean[f]) / asset.standardize.std[f]) * asset.weights[j];
             });
+            // Local to this call (not module/closure state — see
+            // coverageFrac's comment) so concurrent queries against the
+            // same open pack can never see each other's grounding detail.
+            let grounding = null;
             if (coverageCfg) {
                 // passageTexts is hydrated by the caller for the fixed
                 // top-COVERAGE_TOP_PASSAGES window (see passagesNeeded
                 // below), independent of the caller's k, matching
                 // calibrate.mjs's top.slice(0, COVERAGE_TOP_PASSAGES).
-                signals.coverage1 = coverageFrac(queryText, Array.isArray(passageTexts) ? passageTexts : [passageTexts]);
+                const passages = Array.isArray(passageTexts) ? passageTexts : [passageTexts];
+                const coverage = coverageFrac(queryText, passages);
+                signals.coverage1 = coverage.value;
+                grounding = coverage.grounding;
+                // grounding1 is coverage1 alone — see this file's header
+                // comment and calibrate.mjs's GROUNDING_FEAT design note
+                // for why a semantic (maxSim1) blend was tried twice and
+                // removed both times.
                 z += ((signals.coverage1 - coverageCfg.mean) / (coverageCfg.std || 1)) * coverageCfg.weight;
             }
             // A malformed asset (unknown feature name, non-numeric term) must
@@ -125,7 +229,7 @@ export function createAbstentionScorer(asset, bloomBytes) {
             const p = 1 / (1 + Math.exp(-z));
             const verdict = p < asset.thresholds.hard ? 'abstain'
                 : p < asset.thresholds.weak ? 'weak' : 'answer';
-            return { p, verdict, signals };
+            return { p, verdict, signals, grounding };
         },
     };
 }

package/complete/sources.mjs

@@ -1,4 +1,4 @@
-import { pikeletError, PIKELET_ERROR_CODES } from '../pikelet-errors.js';
+import { pikeletError, PIKELET_ERROR_CODES } from '../src/errors/pikelet-errors.js';
 
 export function httpRangeSource(url, options = {}) {
     const stats = { requests: 0, bytes: 0, acceptRanges: null, etag: null, fullFallback: false, retries: 0, redirects: 0 };