pikelet-wasm 0.8.0 → 0.9.0

package/complete/index.mjs

@@ -25,7 +25,7 @@ import { loadStudentModel, embedTextWithStudent } from './student-embedder.mjs';
 import { computeMatchQuality, computePreSearchAbstention } from './student-abstention.mjs';
 import { createAbstentionScorer } from './retrieval-abstention.mjs';
 import { openLexicalIndex, openLexicalIndexLazy } from './lexical.mjs';
-import { PikeletSketchArtifact } from '../pikelet-artifact.js';
+import { PikeletSketchArtifact } from '../src/artifact/pikelet-artifact.js';
 import { MAGIC, HEADER_BYTES, TABLE_ENTRY_BYTES, KINDS, KIND_NAMES } from './format.mjs';
 
 import {
@@ -41,6 +41,8 @@ export { httpRangeSource } from './sources.mjs';
 // in-memory lexical searcher over the segment bytes already assembled by
 // buildLexicalSegment, before the artifact is written to disk.
 export { openLexicalIndex } from './lexical.mjs';
+export { fuseCandidates, FUSION_DEFAULTS } from './fusion.mjs';
+import { fuseCandidates } from './fusion.mjs';
 export {
     KERNEL_LAYOUT, expectedBlobBytes, parseInlineTransformerEncoder,
     createInlineTransformerEmbedder, INLINE_TEST_VECTOR_TEXTS,
@@ -75,16 +77,28 @@ const RECORD_CACHE = 256;
 // its fixed-size header is what format 2 commits to in the manifest.
 const SKETCH_HEADER_BYTES = 256;
 // Hybrid retrieval: BM25 candidates fetched from the lexical segment per
-// query, and the reciprocal-rank-fusion constant (the standard untuned 60).
-const LEXICAL_CANDIDATES = 24;
-const RRF_K = 60;
+// query; the fusion rule and its constants live in fusion.mjs, shared with
+// the calibrator so fit-time rankings match what is served.
+export const LEXICAL_CANDIDATES = 24;
+// scoreQuality's coverage term hydrates the fused top passagesNeeded
+// records (asset.coverage.topK, 5 by default — see calibrate.mjs's
+// COVERAGE_TOP_PASSAGES) once retrieval returns. The scorer that knows
+// the real number is created inside retrievalScorer()'s own closure, not
+// reachable from here, so this is a fixed approximation used to decide
+// how many of fusedFull's records to include in the batched hydration
+// below, alongside phrase-pinning's own candidates — hydrating a couple
+// more than coverage strictly needs just costs a little extra bandwidth
+// on a cache entry that goes unread; hydrating fewer only narrows how
+// much of coverage's later hydration hits warm cache. Either way is
+// harmless: it's a batch-sizing choice, not a correctness one.
+const COVERAGE_BATCH_COUNT = 5;
 // Only BM25 hits within this fraction of the top lexical score join fusion.
 // Common query terms give every matching document a near-tied score (idf
 // collapses them flat); a wide cutoff let that tied mass ride into fusion on
 // a technicality. Kept identical to pikelet/src/calibrate.mjs's
 // LEXICAL_CUTOFF so calibration is fit against the same fusion the reader
 // actually serves.
-const LEXICAL_CUTOFF = 1.5;
+export const LEXICAL_CUTOFF = 1.5;
 // Query-interpretation segments above this size open lazily (kind 3's
 // inline encoder is ~25 MiB); smaller ones keep the one-read eager path.
 const LAZY_QI_BYTES = 4 * 1024 * 1024;
@@ -125,13 +139,23 @@ function toHex(bytes) {
 const isSha256Hex = (value) => typeof value === 'string' && /^[0-9a-f]{64}$/.test(value);
 
 // SHA-256 via WebCrypto (browser, workerd, Node 18+); node:crypto fallback
-// for older Node. Fails closed: verification is not optional in this reader.
+// for older Node, gated on an actual Node runtime check rather than just
+// "WebCrypto is missing" — a browser can lack crypto.subtle too (privacy
+// settings, a hardened build), and a bare dynamic import('node:crypto')
+// there isn't a Node module resolution failure, it's a cross-origin module
+// fetch a browser refuses outright, surfacing as an opaque module-loading
+// error instead of the clear message below. Fails closed either way:
+// verification is not optional in this reader.
 async function sha256hex(bytes) {
     if (globalThis.crypto?.subtle) {
         return toHex(new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256', bytes)));
     }
-    const { createHash } = await import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:crypto');
-    return createHash('sha256').update(bytes).digest('hex');
+    if (typeof process !== 'undefined' && process.versions?.node) {
+        const { createHash } = await import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:crypto');
+        return createHash('sha256').update(bytes).digest('hex');
+    }
+    throw new Error('No crypto backend available (crypto.subtle is missing) — '
+        + 'cannot verify artifact integrity in this environment');
 }
 
 function asUint8Array(bytes) {
@@ -258,7 +282,7 @@ function toFloat32(vector, dim, what) {
     return v;
 }
 
-function base64Bytes(text) {
+export function base64Bytes(text) {
     if (typeof text !== 'string') return new Uint8Array(0);
     return typeof Buffer !== 'undefined'
         ? Buffer.from(text, 'base64')
@@ -307,6 +331,169 @@ export async function verifyHostEncoder(declaration, encodeQuery, dim) {
  * { read(offset, length), size? } range source (any runtime).
  * Returns { query(text, {k}), info(), evaluation(), close() }.
  */
+/**
+ * Read a chain member's manifest and segment table without opening the corpus,
+ * sketch or lexical machinery — LAYERED_PROFILE.md 4.3.
+ *
+ * A tombstone-only layer carries only a query-interp (kind 4) and a tombstones
+ * segment. It is a chain member but not a search tier (3.1), so the full open
+ * has nothing to do for it and every corpus/index step would need a branch.
+ * The chain reader uses this instead; the header, manifest identity and each
+ * segment digest are verified exactly as the full open verifies them.
+ *
+ * Returns { tombstoneOnly, tier, close }. When `tombstoneOnly` is false the
+ * caller should close this and use openPikeletFile() instead: the member is a
+ * search tier and needs the real open.
+ */
+/**
+ * Read and identity-verify just a member's manifest, without opening its
+ * segments. 5.1.1's locator walk needs this: to find a chain's ancestors from
+ * the head alone it must read each member's `layer.parent` and stop at the
+ * base, and paying for a full open per hop only to discard it is wasteful when
+ * the walk may then reject the location.
+ *
+ * Returns { identity, manifest, isLayer, parent }. The identity check is the
+ * same one openPikeletFile makes, so a tampered manifest fails here too.
+ */
+export async function readMemberManifest(input, options = {}) {
+    const source = typeof input === 'string' ? await fileSource(input) : input;
+    const owned = typeof input === 'string';
+    const maxReadBytes = resolveBudget(options.maxReadBytes, 'maxReadBytes', DEFAULT_OPEN_READ_BYTES);
+    try {
+        const header = await readChecked(source, 0, HEADER_BYTES, 'header', maxReadBytes);
+        const hv = viewOf(header);
+        if (hv.getUint32(0, true) !== MAGIC) throw new Error('not a .pikelet file (bad magic)');
+        const manifestBytes = hv.getUint32(8, true);
+        const identity = [...new Uint8Array(header.buffer, header.byteOffset + 24, 32)]
+            .map((b) => b.toString(16).padStart(2, '0')).join('');
+        const fileBytes = u64(hv, 16, 'header fileBytes');
+        const manifestBuf = await readChecked(source, HEADER_BYTES, manifestBytes, 'manifest', maxReadBytes, fileBytes);
+        if (await sha256hex(manifestBuf) !== identity) {
+            throw new Error('.pikelet manifest failed identity verification');
+        }
+        const manifest = JSON.parse(decoder.decode(manifestBuf));
+        const isLayer = manifest.profile === 'pikelet-layer-v1';
+        return {
+            identity,
+            manifest,
+            isLayer,
+            parent: isLayer && manifest.layer?.parent ? { ...manifest.layer.parent } : null,
+        };
+    } finally {
+        if (owned && source.close) await source.close();
+    }
+}
+
+/**
+ * Validate a segment table against the identity-committed manifest and return
+ * kind -> {offset, length, sha256}. The table sits OUTSIDE the identity, so
+ * every entry must agree with the manifest entry at the same index: length,
+ * digest shape, contiguous 16-aligned placement inside the file, and kind. A
+ * known kind appears at most once. Unknown kinds are skipped (spec 3.3), but
+ * the manifest must not name them as a kind this reader knows. Shared by the
+ * full open and the chain-member shell so neither can drift from the other.
+ */
+function parseSegmentTable(table, manifest, segmentCount, manifestBytes, fileBytes) {
+    const tableView = viewOf(table);
+    const segments = new Map();
+    let expectedOffset = align16(HEADER_BYTES + manifestBytes + segmentCount * TABLE_ENTRY_BYTES);
+    for (let i = 0; i < segmentCount; i++) {
+        const entry = i * TABLE_ENTRY_BYTES;
+        const kindNumber = tableView.getUint32(entry, true);
+        const kind = KIND_NAMES[kindNumber];
+        const offset = u64(tableView, entry + 8, `segment ${i} offset`);
+        const length = u64(tableView, entry + 16, `segment ${i} length`);
+        const declared = manifest.segments[i];
+        if (!declared || typeof declared !== 'object' || declared.bytes !== length || !isSha256Hex(declared.sha256)
+            || offset !== expectedOffset || !Number.isSafeInteger(offset + length) || offset + length > fileBytes) {
+            throw new Error(`.pikelet segment table disagrees with manifest at entry ${i}`);
+        }
+        if (kind === undefined) {
+            if (typeof declared.kind !== 'string' || declared.kind in KINDS) {
+                throw new Error(`.pikelet segment table entry ${i} has unknown kind ${kindNumber} but the manifest names it ${declared.kind}`);
+            }
+        } else {
+            if (declared.kind !== kind) {
+                throw new Error(`.pikelet segment table disagrees with manifest at entry ${i}`);
+            }
+            if (segments.has(kind)) throw new Error(`.pikelet carries more than one ${kind} segment`);
+            segments.set(kind, { offset, length, sha256: declared.sha256 });
+        }
+        expectedOffset = align16(offset + length);
+    }
+    return segments;
+}
+
+export async function readChainMemberShell(input, options = {}) {
+    const source = typeof input === 'string' ? await fileSource(input) : input;
+    const owned = typeof input === 'string';
+    const maxReadBytes = resolveBudget(options.maxReadBytes, 'maxReadBytes', DEFAULT_OPEN_READ_BYTES);
+    const close = async () => { if (owned && source.close) await source.close(); };
+    try {
+        const header = await readChecked(source, 0, HEADER_BYTES, 'header', maxReadBytes);
+        const hv = viewOf(header);
+        if (hv.getUint32(0, true) !== MAGIC) throw new Error('not a .pikelet file (bad magic)');
+        const formatVersion = hv.getUint32(4, true);
+        const manifestBytes = hv.getUint32(8, true);
+        const segmentCount = hv.getUint32(12, true);
+        const identity = [...new Uint8Array(header.buffer, header.byteOffset + 24, 32)]
+            .map((b) => b.toString(16).padStart(2, '0')).join('');
+        const fileBytes = u64(hv, 16, 'header fileBytes');
+        if (manifestBytes < 2 || manifestBytes > MAX_MANIFEST_BYTES || segmentCount < 1 || segmentCount > MAX_SEGMENTS
+            || HEADER_BYTES + manifestBytes + segmentCount * TABLE_ENTRY_BYTES > fileBytes) {
+            throw new Error('.pikelet header is implausible');
+        }
+        const manifestBuf = await readChecked(source, HEADER_BYTES, manifestBytes, 'manifest', maxReadBytes, fileBytes);
+        if (await sha256hex(manifestBuf) !== identity) {
+            throw new Error('.pikelet manifest failed identity verification');
+        }
+        const manifest = JSON.parse(decoder.decode(manifestBuf));
+        if (manifest.profile !== 'pikelet-layer-v1' || formatVersion !== 2) {
+            return { tombstoneOnly: false, tier: null, close };
+        }
+        if (manifest.layer?.records !== 0) {
+            return { tombstoneOnly: false, tier: null, close };
+        }
+        if (!Array.isArray(manifest.segments) || manifest.segments.length !== segmentCount) {
+            throw new Error('.pikelet manifest segment list disagrees with the header segment count');
+        }
+        // The same table checks as the full open. This path used to keep only
+        // length and digest shape: it never compared the table's kind with the
+        // manifest's and let a later duplicate overwrite an earlier one, so
+        // whoever controls the table bytes (a mirror) could relabel another
+        // committed segment as the tombstone mask without changing identity.
+        const table = await readChecked(source, HEADER_BYTES + manifestBytes, segmentCount * TABLE_ENTRY_BYTES, 'segment table', maxReadBytes, fileBytes);
+        const segments = parseSegmentTable(table, manifest, segmentCount, manifestBytes, fileBytes);
+        for (const required of ['query-interp', 'tombstones']) {
+            if (!segments.has(required)) throw new Error(`a tombstone-only layer is missing the ${required} segment`);
+        }
+        // 4.3: a tombstone-only layer carries no records, so it MUST NOT carry
+        // the segments that would hold them.
+        for (const forbidden of ['index', 'corpus', 'lexical']) {
+            if (segments.has(forbidden)) throw new Error(`a tombstone-only layer must not carry a ${forbidden} segment (4.3)`);
+        }
+        return {
+            tombstoneOnly: true,
+            tier: {
+                manifest, identity, recordCount: 0, dim: manifest.dim,
+                sketch: null, lexicalIndex: null, hydrate: null,
+                encoderInfo: null, segments,
+                async readSegmentBytes(seg) {
+                    const bytes = await readChecked(source, seg.offset, seg.length, 'chain segment', maxReadBytes, fileBytes);
+                    if (await sha256hex(bytes) !== seg.sha256) {
+                        throw new Error('.pikelet chain segment failed hash verification');
+                    }
+                    return bytes;
+                },
+            },
+            close,
+        };
+    } catch (err) {
+        await close().catch(() => {});
+        throw err;
+    }
+}
+
 export async function openPikeletFile(input, options = {}) {
     const source = typeof input === 'string' ? await fileSource(input) : input;
     const owned = typeof input === 'string';
@@ -330,6 +517,10 @@ export async function openPikeletFile(input, options = {}) {
     // Kind-3 lazy encoder: the pending fetch+init promise. close() settles
     // it before disposing so an in-flight prefetch cannot leak its kernel.
     let encoderPending = null;
+    // Set by the kind-3 branch below. Hoisted because `append` needs the base's
+    // own encoder through the __chainTier seam (6.1), and the branch's own
+    // ensureEmbedder is scoped to that branch.
+    let resolveEmbedder = null;
     const settleEncoder = async () => {
         if (encoderPending) { try { await encoderPending; } catch { /* surfaced to queries */ } }
     };
@@ -389,8 +580,15 @@ export async function openPikeletFile(input, options = {}) {
         try { manifest = JSON.parse(decoder.decode(manifestBuf)); } catch (err) {
             throw new Error('.pikelet manifest is not valid JSON', { cause: err });
         }
+        // 4.1 of the layered profile: a layer carries `pikelet-layer-v1` under
+        // formatVersion 2, and a reader MUST refuse it by default — a layer
+        // opened alone would silently serve a fraction of a corpus. Only the
+        // chain reader, which opens every ancestor and verifies the hash chain,
+        // passes asChainMember to accept one.
+        const isLayerProfile = manifest?.profile === 'pikelet-layer-v1' && formatVersion === 2;
+        const acceptLayer = isLayerProfile && options.asChainMember === true;
         if (!manifest || typeof manifest !== 'object'
-            || (manifest.profile !== profile && manifest.profile !== LEGACY_PROFILES[formatVersion])) {
+            || (manifest.profile !== profile && manifest.profile !== LEGACY_PROFILES[formatVersion] && !acceptLayer)) {
             throw new Error(`unsupported profile ${manifest?.profile} for format version ${formatVersion}`);
         }
         if (!Array.isArray(manifest.segments) || manifest.segments.length !== segmentCount) {
@@ -405,37 +603,97 @@ export async function openPikeletFile(input, options = {}) {
         }
 
         const table = await readChecked(source, HEADER_BYTES + manifestBytes, tableBytes, 'segment table', maxReadBytes, fileBytes);
-        const tableView = viewOf(table);
-        const segments = new Map();
-        let expectedOffset = align16(HEADER_BYTES + manifestBytes + tableBytes);
-        for (let i = 0; i < segmentCount; i++) {
-            const entry = i * TABLE_ENTRY_BYTES;
-            const kindNumber = tableView.getUint32(entry, true);
-            const kind = KIND_NAMES[kindNumber];
-            const offset = u64(tableView, entry + 8, `segment ${i} offset`);
-            const length = u64(tableView, entry + 16, `segment ${i} length`);
-            const declared = manifest.segments[i];
-            if (!declared || typeof declared !== 'object' || declared.bytes !== length || !isSha256Hex(declared.sha256)
-                || offset !== expectedOffset || !Number.isSafeInteger(offset + length) || offset + length > fileBytes) {
-                throw new Error(`.pikelet segment table disagrees with manifest at entry ${i}`);
+        const segments = parseSegmentTable(table, manifest, segmentCount, manifestBytes, fileBytes);
+        // The evaluation segment, read and hash-verified. Named so both the
+        // reader's own evaluation() and the __chainTier seam can reach it: a
+        // chain has no evaluation segment of its own, and `verify_pack` called
+        // `search.evaluation()` on a chain reader that had no such method,
+        // which threw a TypeError instead of reporting the base's goldens.
+        const readEvaluationSegment = async () => {
+            const ev = segments.get('evaluation');
+            if (!ev) return null;
+            const bytes = await readChecked(source, ev.offset, ev.length, 'evaluation segment', maxReadBytes, fileBytes);
+            if (await sha256hex(bytes) !== ev.sha256) {
+                throw new Error('.pikelet evaluation segment failed hash verification');
             }
-            if (kind === undefined) {
-                // Unknown kinds are skipped, not failed (spec 3.3); they are
-                // still committed through the manifest, so the declared name
-                // must at least not claim to be a kind this reader knows.
-                if (typeof declared.kind !== 'string' || declared.kind in KINDS) {
-                    throw new Error(`.pikelet segment table entry ${i} has unknown kind ${kindNumber} but the manifest names it ${declared.kind}`);
-                }
-            } else {
-                if (declared.kind !== kind) {
-                    throw new Error(`.pikelet segment table disagrees with manifest at entry ${i}`);
+            let parsed;
+            try { parsed = JSON.parse(decoder.decode(bytes)); } catch (err) {
+                throw new Error('.pikelet evaluation segment is not valid JSON', { cause: err });
+            }
+            if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+                throw new Error('.pikelet evaluation segment must be a JSON object');
+            }
+            return parsed;
+        };
+        // The lineage segment (kind 7, LAYERED_PROFILE.md 6.3), parsed and
+        // validated on first use. Present only on a base compacted from a
+        // chain. buildLineageSegment wrote it and parseLineageSegment validated
+        // it, but nothing on the read side ever reached either: a real
+        // compacted pack could not translate a citation, which is the whole
+        // point of the segment ("what lets a consumer treat a translated
+        // citation as evidence rather than as a hint").
+        //
+        // Cached because the validation is not free and the segment is
+        // immutable; the cache holds the parse, never a translation.
+        let lineageCache;
+        const readLineageSegment = async () => {
+            if (lineageCache !== undefined) return lineageCache;
+            const seg = segments.get('lineage');
+            if (!seg) { lineageCache = null; return null; }
+            const bytes = await readChecked(source, seg.offset, seg.length, 'lineage segment', maxReadBytes, fileBytes);
+            if (await sha256hex(bytes) !== seg.sha256) {
+                throw new Error('.pikelet lineage segment failed hash verification');
+            }
+            const { parseLineageSegment } = await import('./layer-compact.mjs');
+            // Cross-checked against the manifest: the parser refuses a segment
+            // whose last head is not compactedFrom.identity, or whose counts
+            // disagree, so a tampered pair fails rather than translating.
+            lineageCache = parseLineageSegment(bytes, manifest.compactedFrom || null);
+            return lineageCache;
+        };
+
+        // oldId -> new dense id, by 6.3's rule: a surviving id loses one
+        // position per tombstone below it. The per-byte prefix table is built
+        // once and reused, because the naive form rescans the bitset from zero
+        // on every call -- O(id) per citation, quadratic over a run of them on
+        // a large pack.
+        let compactPrefix;
+        const compactMapFor = (lineage) => {
+            if (compactPrefix === undefined) {
+                const bytes = lineage.bitset.length;
+                compactPrefix = new Uint32Array(bytes + 1);
+                for (let b = 0; b < bytes; b++) {
+                    let n = 0;
+                    for (let k = 0; k < 8; k++) if ((lineage.bitset[b] >> k) & 1) n++;
+                    compactPrefix[b + 1] = compactPrefix[b] + n;
                 }
-                if (segments.has(kind)) throw new Error(`.pikelet carries more than one ${kind} segment`);
-                segments.set(kind, { offset, length, sha256: declared.sha256 });
             }
-            expectedOffset = align16(offset + length);
+            return (oldId) => {
+                if (oldId < 0 || oldId >= lineage.oldRowTotal) return null;
+                if ((lineage.bitset[oldId >> 3] >> (oldId & 7)) & 1) return null;
+                const byte = oldId >> 3;
+                let below = compactPrefix[byte];
+                for (let k = 0; k < (oldId & 7); k++) {
+                    if ((lineage.bitset[byte] >> k) & 1) below++;
+                }
+                return oldId - below;
+            };
+        };
+
+        // Required segments are per-profile (layered profile 4.3). A layer with
+        // records carries a tombstones segment on top of the complete set; a
+        // TOMBSTONE-ONLY layer has neither an index nor a corpus segment and is
+        // therefore not opened through this function at all — the chain reader
+        // reads its two segments directly (see openPikeletChain), because
+        // threading "no index, no corpus" through the whole of this open would
+        // put a branch on every step of the single-file path.
+        if (acceptLayer && manifest.layer?.records === 0) {
+            throw new Error('a tombstone-only layer is read by the chain reader, not opened as a member');
         }
-        for (const required of ['index', 'corpus', 'query-interp']) {
+        const requiredSegments = acceptLayer
+            ? ['index', 'corpus', 'query-interp', 'tombstones']
+            : ['index', 'corpus', 'query-interp'];
+        for (const required of requiredSegments) {
             if (!segments.has(required)) throw new Error(`.pikelet is missing the ${required} segment`);
         }
 
@@ -611,7 +869,7 @@ export async function openPikeletFile(input, options = {}) {
         const retrievalScorer = () => {
             const scorer = createAbstentionScorer(calibrationJson.asset, base64Bytes(calibrationJson.vocabBloomBase64));
             const VERDICTS = { answer: 'strong', weak: 'weak', abstain: 'none' };
-            return async (hits, context, fusedHits) => {
+            return async (hits, context, fusedHits, scoreOptions) => {
                 if (!scorer) return { match_quality: 'unscored' };
                 // The coverage term grounds the verdict in the top passages'
                 // text, so those records hydrate before scoring. Hydrated
@@ -628,12 +886,36 @@ export async function openPikeletFile(input, options = {}) {
                 // record that fails its digest must fail the query, not
                 // skew its verdict.
                 const passageSource = fusedHits?.length ? fusedHits : hits;
-                const topTexts = scorer.usesPassage && passageSource.length
-                    ? (await Promise.all(passageSource.slice(0, scorer.passagesNeeded || 1)
-                        .map((hit) => hydrate(hit.id)))).map((record) => record?.text)
-                    : [];
-                const scored = scorer.score(context.text, hits, topTexts);
-                return { match_quality: VERDICTS[scored.verdict] || scored.verdict, confidence: scored.p };
+                // A record's text starts with its own heading echoed as the
+                // first line (ingest.mjs's section-to-chunk join). A
+                // heading match still grounds the query but at reduced
+                // credit — see retrieval-abstention.mjs's coverageFrac and
+                // calibrate.mjs's HEADING_COVERAGE_WEIGHT, which this split
+                // must match exactly (build-time and serve-time coverage
+                // must agree, or the fitted thresholds are scoring a
+                // different signal than the one they were calibrated
+                // against).
+                const splitHeadingBody = (text) => {
+                    const nl = String(text || '').indexOf('\n');
+                    return nl === -1 ? { heading: text, body: '' } : { heading: text.slice(0, nl), body: text.slice(nl + 1) };
+                };
+                // A chain supplies the passages itself (context.passages): its
+                // ids are GLOBAL, and this hydrate only knows this member's
+                // local ids, so letting it fetch them would throw on any id a
+                // layer owns. Single-file callers pass none and it hydrates.
+                const topTexts = Array.isArray(context.passages)
+                    ? context.passages.slice(0, scorer.passagesNeeded || 1).map(splitHeadingBody)
+                    : scorer.usesPassage && passageSource.length
+                        ? (await Promise.all(passageSource.slice(0, scorer.passagesNeeded || 1)
+                            .map((hit) => hydrate(hit.id)))).map((record) => splitHeadingBody(record?.text))
+                        : [];
+                // scoreOptions.vocabBloom is the chain's union bloom (4.5).
+                // Absent for a single-file pack, where the fit's own bloom is
+                // the whole vocabulary.
+                const scored = await scorer.score(context.text, hits, topTexts, {
+                    vocabBloom: scoreOptions?.vocabBloom,
+                });
+                return { match_quality: VERDICTS[scored.verdict] || scored.verdict, confidence: scored.p, grounding: scored.grounding || null };
             };
         };
         if (qiKind === 1) {
@@ -695,7 +977,7 @@ export async function openPikeletFile(input, options = {}) {
                 } catch (err) {
                     throw new Error('kind-3 artifact requires the inline-transformer kernels at '
                         + 'complete/encoder-kernels/ (pikelet-wasm/complete); run '
-                        + 'examples/05-one-file-search/encoder-spike/build-encoder.sh to rebuild them',
+                        + 'examples/one-file-search/encoder-spike/build-encoder.sh to rebuild them',
                     { cause: err });
                 }
             }
@@ -728,6 +1010,7 @@ export async function openPikeletFile(input, options = {}) {
                 return { embedder, declaration };
             };
             const ensureEmbedder = () => (encoderPending ??= loadEmbedder());
+            resolveEmbedder = ensureEmbedder;
             // Until the encoder arrives, info() serves the identity-verified
             // manifest declaration; encoderVerified stays null (unknown, not
             // unverified) and flips once test vectors have run.
@@ -750,6 +1033,17 @@ export async function openPikeletFile(input, options = {}) {
                 return { vector: toFloat32(vector, dim, 'inline transformer encoder'), text };
             };
             scoreQuality = retrievalScorer();
+        } else if (qiKind === 4 && acceptLayer) {
+            // Kind 4, `inherited-v1` (layered profile 4.5): a layer carries a
+            // COMMITMENT to the base's encoder, never encoder bytes. This open
+            // therefore installs no embedder — the chain reader resolves the
+            // encoder from the base it opened and validates the commitment
+            // itself (layer-reader.mjs), and a query on this member alone is
+            // refused rather than answered with a missing encoder.
+            embed = async () => {
+                throw new Error('a layer inherits its encoder from the base: query the chain, not the layer (4.5)');
+            };
+            scoreQuality = null;
         } else {
             throw new Error(`unsupported query-interpretation kind ${qiKind}`);
         }
@@ -813,7 +1107,7 @@ export async function openPikeletFile(input, options = {}) {
                 // Hidden from bundlers like the node encoder kernels: the
                 // web entrypoint's ?url wasm imports would break a browser
                 // build that statically pulled this in.
-                const { default: Pikelet } = await import(/* webpackIgnore: true */ /* @vite-ignore */ '../pikelet.node.mjs');
+                const { default: Pikelet } = await import(/* webpackIgnore: true */ /* @vite-ignore */ '../src/pikelet.node.mjs');
                 await sketch.fullyResident;
                 const created = await Pikelet.createSketchScanner(sketch, {
                     maxRerank: Math.min(sketch.count, SCANNER_MAX_RERANK),
@@ -892,16 +1186,13 @@ export async function openPikeletFile(input, options = {}) {
             return record;
         };
 
-        const hydrate = async (id) => {
+        // One record's stored bytes, read and (on format 2) verified against
+        // its per-record digest. hydrate() parses these; `compact` (6.3)
+        // copies them verbatim through the chain's __recordBytes seam.
+        const readRecordBytes = async (id) => {
             if (!Number.isSafeInteger(id) || id < 0 || id >= recordCount) {
                 throw new Error(`.pikelet result id ${id} is outside the corpus`);
             }
-            if (recordCache.has(id)) {
-                const cached = recordCache.get(id);
-                recordCache.delete(id);
-                recordCache.set(id, cached);
-                return cached;
-            }
             const start = recordOffsets[id];
             const end = recordOffsets[id + 1];
             const bytes = await readChecked(source, corpus.offset + start, end - start, `corpus record ${id}`, maxRecordBytes, fileBytes);
@@ -913,6 +1204,20 @@ export async function openPikeletFile(input, options = {}) {
                     throw new Error(`.pikelet corpus record ${id} failed integrity verification`);
                 }
             }
+            return bytes;
+        };
+
+        const hydrate = async (id) => {
+            if (!Number.isSafeInteger(id) || id < 0 || id >= recordCount) {
+                throw new Error(`.pikelet result id ${id} is outside the corpus`);
+            }
+            if (recordCache.has(id)) {
+                const cached = recordCache.get(id);
+                recordCache.delete(id);
+                recordCache.set(id, cached);
+                return cached;
+            }
+            const bytes = await readRecordBytes(id);
             let record;
             try { record = JSON.parse(decoder.decode(bytes)); } catch (err) {
                 throw new Error(`.pikelet corpus record ${id} is not valid JSON`, { cause: err });
@@ -937,6 +1242,16 @@ export async function openPikeletFile(input, options = {}) {
                     identity,
                     formatVersion,
                     profile,
+                    // 6.3: present when this base was compacted from a chain,
+                    // and the key to whether citation() can say anything. It
+                    // was in the manifest all along and reported nowhere, so a
+                    // compacted pack looked indistinguishable from a freshly
+                    // compiled one.
+                    compactedFrom: manifest.compactedFrom
+                        && typeof manifest.compactedFrom === 'object'
+                        && !Array.isArray(manifest.compactedFrom)
+                        ? { ...manifest.compactedFrom }
+                        : null,
                     // Human-readable pack name from the identity-verified
                     // manifest (compile records it under corpus.provenance);
                     // null when the builder recorded none.
@@ -1076,13 +1391,51 @@ export async function openPikeletFile(input, options = {}) {
                         fusedFull = lexicalHits.map((h) => byId.get(h.id)).filter(Boolean);
                     } else if (retrieval !== 'augmented' && lexicalHits.length) {
                         const lexRank = new Map(lexicalHits.map((h, i) => [h.id, i]));
-                        fusedFull = searched
-                            .map((hit, vRank) => ({
-                                hit,
-                                score: 1 / (RRF_K + vRank) + (lexRank.has(hit.id) ? 1 / (RRF_K + lexRank.get(hit.id)) : 0),
-                            }))
-                            .sort((a, b) => (b.score - a.score) || (a.hit.distance - b.hit.distance))
-                            .map((entry) => entry.hit);
+                        fusedFull = fuseCandidates(searched, lexicalHits.map((h) => h.id));
+                        // One batched hydration for everything this block and
+                        // scoreQuality's coverage term (below) will need, in
+                        // place of two separate staged fetches (phrase-pin's
+                        // own, then a later cold start for coverage): over a
+                        // remote range-read source each hydrate() a caller
+                        // hasn't already paid for is its own network round
+                        // trip, and sequential/staged awaits pay that cost
+                        // once per stage instead of once per query. The
+                        // union of phrase-pin's candidates (top lexical
+                        // hits) and coverage's (fusedFull's top few — phrase
+                        // pinning only ever reorders that window, never
+                        // changes its membership) is known right here, so
+                        // both stages can share one round trip. A record
+                        // that fails to hydrate here does not fail the
+                        // query: the id is just missing from the map, and
+                        // each site below falls through exactly as it would
+                        // on a cold cache miss (recs[i] undefined; coverage
+                        // sees fewer usable passages) — hydrate() raising
+                        // for a bad id is still a real failure, but only
+                        // once a site actually needs that specific record,
+                        // not speculatively for every candidate touched here.
+                        const phraseCandidates = lexicalHits.slice(0, 3);
+                        const coverageCandidates = fusedFull.slice(0, COVERAGE_BATCH_COUNT);
+                        const batchIds = [...new Set([...phraseCandidates, ...coverageCandidates].map((h) => h.id))];
+                        const batch = new Map();
+                        await Promise.all(batchIds.map(async (id) => {
+                            try { batch.set(id, await hydrate(id)); } catch { /* see comment above */ }
+                        }));
+                        // Phrase pinning: a verbatim occurrence of the query (>= 3
+                        // content words) in one of the top lexical hits is the
+                        // strongest evidence of support there is; rank it first.
+                        const norm = (t) => String(t || '').toLowerCase().replace(/[^a-z0-9' ]+/g, ' ').replace(/\s+/g, ' ').trim();
+                        const phrase = norm(trimmed);
+                        if (phrase.split(' ').length >= 3) {
+                            for (const h of phraseCandidates) {
+                                if (norm(batch.get(h.id)?.text).includes(phrase)) {
+                                    fusedFull = [searched.find((x) => x.id === h.id), ...fusedFull.filter((x) => x.id !== h.id)].filter(Boolean);
+                                    break;
+                                }
+                            }
+                        }
+                        // Rank transparency: a consumer can see a verbatim hit sitting
+                        // at fused rank 3 behind two semantic near-misses.
+                        for (const [i, hit] of fusedFull.entries()) { hit.fusedRank = i + 1; hit.lexicalRank = lexRank.has(hit.id) ? lexRank.get(hit.id) + 1 : null; hit.vectorRank = searched.findIndex((x) => x.id === hit.id) + 1; }
                     }
                     fused = retrieval === 'augmented' ? null : fusedFull?.slice(0, k) ?? null;
                 }
@@ -1111,9 +1464,14 @@ export async function openPikeletFile(input, options = {}) {
                 // `distance` field cannot overwrite them (reserved names).
                 const results = await Promise.all(returned.map(async (hit) => {
                     const record = await hydrate(hit.id);
-                    return { ...record, id: hit.id, distance: hit.distance };
+                    return { ...record, id: hit.id, distance: hit.distance, ...(hit.fusedRank ? { fusedRank: hit.fusedRank, lexicalRank: hit.lexicalRank, vectorRank: hit.vectorRank } : {}) };
                 }));
-                return { matchQuality: quality.match_quality, confidence: quality.confidence, results };
+                // Grounding transparency: which record's text grounded the verdict
+                // and which query words it covered, so a consumer can tell a
+                // topical match ("much money darcy make year") from an answer.
+                const passageOrder = fusedFull?.length ? fusedFull : (searched ?? hits);
+                const grounding = quality.grounding ? { ...quality.grounding, recordId: passageOrder[quality.grounding.passageIndex]?.id ?? null } : null;
+                return { matchQuality: quality.match_quality, confidence: quality.confidence, results, ...(grounding ? { grounding } : {}) };
             },
 
             /** Hydrate one corpus record by id (verified per record on format 2). */
@@ -1135,20 +1493,166 @@ export async function openPikeletFile(input, options = {}) {
 
             async evaluation() {
                 assertOpen();
-                const ev = segments.get('evaluation');
-                if (!ev) return null;
-                const bytes = await readChecked(source, ev.offset, ev.length, 'evaluation segment', maxReadBytes, fileBytes);
-                if (await sha256hex(bytes) !== ev.sha256) {
-                    throw new Error('.pikelet evaluation segment failed hash verification');
-                }
-                let parsed;
-                try { parsed = JSON.parse(decoder.decode(bytes)); } catch (err) {
-                    throw new Error('.pikelet evaluation segment is not valid JSON', { cause: err });
-                }
-                if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
-                    throw new Error('.pikelet evaluation segment must be a JSON object');
-                }
-                return parsed;
+                return readEvaluationSegment();
+            },
+
+            /**
+             * 6.3: translate a citation made against any head of the chain this
+             * base was compacted from. Returns null when this pack carries no
+             * lineage segment, i.e. it was not compacted from a chain.
+             *
+             * Forward translation only, and the shape says so: `{ok: true,
+             * newId, via}` for a record that survives, `{ok: true,
+             * deleted: true, ...}` for one that does not, and `{ok: false,
+             * reason}` for an id or identity this history cannot speak to.
+             * It answers what an old id BECAME, never what an old head served
+             * — the old record's bytes are not in this file (see the spec's
+             * "Translation is not dereference").
+             */
+            async citation(identity, id) {
+                assertOpen();
+                const lineage = await readLineageSegment();
+                if (!lineage) return null;
+                const { translateCitation } = await import('./layer-compact.mjs');
+                // oldId -> new dense id, by the spec's own rule: a live id
+                // loses one position per tombstone below it. Derived from the
+                // segment's bitset rather than passed in, so a caller cannot
+                // supply a mapping that disagrees with the bytes.
+                return translateCitation(lineage, compactMapFor(lineage), identity, id);
+            },
+
+            /**
+             * The parsed lineage segment, or null. Exposed so a consumer can
+             * enumerate the heads a citation may name without guessing.
+             */
+            async lineage() {
+                assertOpen();
+                const parsed = await readLineageSegment();
+                if (!parsed) return null;
+                return {
+                    heads: parsed.heads.map((h) => ({ ...h })),
+                    oldRowTotal: parsed.oldRowTotal,
+                    tombstoneCount: parsed.tombstoneCount,
+                    supersessions: parsed.supersessions.map((e) => [...e]),
+                };
+            },
+
+            /**
+             * Internal seam for the layered profile (LAYERED_PROFILE.md 5.1
+             * step 3): a chain reader mounts each member through this same
+             * open, then needs each search tier's sketch, lexical index and
+             * hydration to merge across tiers (5.3). Exposing them keeps one
+             * container parser rather than a second one — decision 1.
+             *
+             * Deliberately not part of the documented reader API: the names
+             * carry no compatibility promise and a caller outside this package
+             * should use query()/record() instead.
+             */
+            get __chainTier() {
+                assertOpen();
+                return {
+                    manifest,
+                    identity,
+                    recordCount,
+                    dim,
+                    sketch,
+                    lexicalIndex,
+                    hydrate,
+                    readRecordBytes,
+                    encoderInfo,
+                    segments,
+                    // The query-interpretation kind, verbatim from the
+                    // segment header. `append` needs it to decide
+                    // appendability (6.1), and reverse-mapping it from
+                    // encoderInfo.kind loses kind 2, whose encoderInfo is
+                    // shaped by the host declaration rather than a fixed name.
+                    qiKind,
+                    // This member's own evaluation segment, hash-verified.
+                    // A chain surfaces the base's through it (6.2).
+                    evaluation: readEvaluationSegment,
+                    // The tier's staged WASM scan kernel, or null before
+                    // background staging resolves. A chain queries
+                    // sketch.search directly, so without passing this through
+                    // every tier fell back to the JS scan: on the 456k-record
+                    // wiki base that is ~450 ms a query against ~50 ms with
+                    // the kernel. A getter, not a captured value, because
+                    // staging completes after open() returns.
+                    get scanner() { return scanner; },
+                    // The base's OWN encoder, for `append` (6.1): "a kind-3
+                    // base carries the teacher, and append loads it from the
+                    // base's own query-interp segment — no model download, no
+                    // external dependency: a pack is sufficient to compile its
+                    // own successors." Resolving it here rather than letting a
+                    // producer construct one is what makes substituting a
+                    // different encoder impossible.
+                    //
+                    // Returns null for kinds that cannot embed passages: kind 1
+                    // carries a query encoder only (6.1 refuses it), and kind 4
+                    // is a layer, which inherits.
+                    async passageEmbedder() {
+                        assertOpen();
+                        if (qiKind !== 3 || !resolveEmbedder) return null;
+                        const { embedder, declaration } = await resolveEmbedder();
+                        return {
+                            declaration,
+                            // The passage prefix is part of the contract the
+                            // base was built with; applying the wrong one (or
+                            // none) silently produces vectors in a different
+                            // space from the base's.
+                            embed: async (text) => {
+                                const prefix = declaration.prefixPolicy?.passage || '';
+                                const { vector } = await embedder.embed(`${prefix}${text}`);
+                                return vector;
+                            },
+                            // A refit (6.3 step 4) embeds generated QUERIES,
+                            // which take the query prefix, not the passage one:
+                            // the asymmetry is the encoder's contract and
+                            // applying the wrong side puts the fit's probes in
+                            // a different space from the corpus it scores.
+                            embedQuery: async (text) => {
+                                const prefix = declaration.prefixPolicy?.query || '';
+                                const { vector } = await embedder.embed(`${prefix}${text}`);
+                                return vector;
+                            },
+                            // Word-level vectors for the comparison feature.
+                            // Unprefixed by design -- a single word is not a
+                            // query needing an instruction prefix. Null when
+                            // the kernel cannot do it, which the caller reads
+                            // as "fit without that feature".
+                            embedWords: typeof embedder.embedWords === 'function'
+                                ? (text) => embedder.embedWords(text)
+                                : null,
+                        };
+                    },
+                    // The base's own abstention scorer, for a chain serving
+                    // under `inherited` (5.5): "the base's
+                    // retrieval-signals-v1 scorer runs unchanged over the
+                    // merged searched window and fused passages, with the union
+                    // bloom of 4.5". Null when the base carries no fit, which
+                    // is what makes a chain report `none` rather than silently
+                    // omitting the verdict.
+                    scoreQuality: typeof scoreQuality === 'function' ? scoreQuality : null,
+                    // The base's fit, for a chain's calibration status (5.5).
+                    // It lives in the query-interp segment's calibration
+                    // region, not in the manifest — `asset` present means the
+                    // base carries a retrieval-signals fit, and
+                    // `asset.driftLimit` is the producer's declared validity
+                    // envelope when it declared one.
+                    calibrationJson,
+                    // Raw segment bytes for the chain's structural parsing
+                    // (tombstones, kind-4). Bounded and digest-checked exactly
+                    // as any other segment read: the digest comes from the
+                    // identity-verified manifest, so bytes that fail it never
+                    // reach a parser.
+                    async readSegmentBytes(seg) {
+                        assertOpen();
+                        const bytes = await readChecked(source, seg.offset, seg.length, 'chain segment', maxReadBytes, fileBytes);
+                        if (await sha256hex(bytes) !== seg.sha256) {
+                            throw new Error('.pikelet chain segment failed hash verification');
+                        }
+                        return bytes;
+                    },
+                };
             },
 
             async close() {