pikelet-wasm 0.8.0 → 0.9.0

package/complete/inline-transformer.mjs

@@ -8,6 +8,10 @@ const decoder = new TextDecoder();
 // here, host-side, before the first forward.
 export const KERNEL_LAYOUT = { V: 30522, P: 512, T: 2, D: 384, F: 1536, L: 6, B: 64, H: 12 };
 const KERNEL_MAX_SEQ = 512;
+// Smallest window that still frames interior tokens between [CLS] and [SEP]
+// with room to make progress; see the maxTokens check in
+// parseInlineTransformerEncoder().
+const MIN_DECLARED_MAX_TOKENS = 8;
 
 // Byte size the kernel's fill_layout() will consume for a given layout:
 // each quantized matrix is rows*cols u8 plus per-block f32 scales and
@@ -43,6 +47,22 @@ export function parseInlineTransformerEncoder(encoderBytes) {
   if (declaration.dim !== KERNEL_LAYOUT.D) {
     throw new Error(`.pikelet inline-encoder declares dim ${declaration.dim}; the kernel emits ${KERNEL_LAYOUT.D}`);
   }
+  // maxTokens sizes the kernel's ids/hidden heap buffers, and the windowing
+  // arithmetic (windowLen = maxSeq - 2) assumes it leaves room for [CLS] and
+  // [SEP] plus interior tokens. A pack declaring 1 makes windowLen negative,
+  // so interior.slice(w * windowLen, (w + 1) * windowLen) returns nearly the
+  // whole sequence and the forward pass writes far past both allocations; 2
+  // makes windowLen 0, so the window count is Infinity. Reject the whole
+  // range here, at parse, so nothing is allocated from an unusable value.
+  // The floor is well under any value the toolchain emits (the builder
+  // declares 512; the BEIR encoder's own --max-tokens floor is 16).
+  if (declaration.maxTokens !== undefined && declaration.maxTokens !== null) {
+    const declared = declaration.maxTokens;
+    if (!Number.isInteger(declared) || declared < MIN_DECLARED_MAX_TOKENS || declared > KERNEL_MAX_SEQ) {
+      throw new Error(`.pikelet inline-encoder declares maxTokens ${declared}; `
+        + `must be an integer in [${MIN_DECLARED_MAX_TOKENS}, ${KERNEL_MAX_SEQ}]`);
+    }
+  }
   const expected = expectedBlobBytes(layout);
   if (blob.length !== expected) {
     throw new Error(`.pikelet inline-encoder blob is ${blob.length} bytes but its declared layout implies ${expected}`);
@@ -63,7 +83,14 @@ export async function createInlineTransformerEmbedder({ declaration, vocabText,
   const tokenizer = createWordPiece(vocabText);
   const EM = await createEncoder();
   const dim = declaration.dim;
-  const maxSeq = Math.min(declaration.maxTokens || KERNEL_MAX_SEQ, KERNEL_MAX_SEQ);
+  // Clamped on BOTH sides: parseInlineTransformerEncoder() rejects an
+  // out-of-range declaration, but this function is also called directly (the
+  // BEIR ladder, encoder ablations), so the buffer sizing must not depend on
+  // that check having run.
+  const maxSeq = Math.min(
+    Math.max(MIN_DECLARED_MAX_TOKENS, Math.trunc(declaration.maxTokens) || KERNEL_MAX_SEQ),
+    KERNEL_MAX_SEQ,
+  );
   const blobPtr = EM._malloc(blob.length);
   EM.HEAPU8.set(blob, blobPtr);
   const idsPtr = EM._malloc(maxSeq * 4);
@@ -88,6 +115,11 @@ export async function createInlineTransformerEmbedder({ declaration, vocabText,
     const windowsToEncode = usesClsPooling ? 1 : windows;
     for (let w = 0; w < windowsToEncode; w++) {
       const tokenIds = [allIds[0], ...interior.slice(w * windowLen, (w + 1) * windowLen), allIds[allIds.length - 1]];
+      // The windowing above must never produce more tokens than the buffers
+      // were sized for; fail closed instead of writing past them.
+      if (tokenIds.length > maxSeq) {
+        throw new Error(`inline encoder window is ${tokenIds.length} tokens for a ${maxSeq}-token buffer`);
+      }
       new Int32Array(EM.HEAP32.buffer, idsPtr, tokenIds.length).set(tokenIds);
       const rc = EM._encoder_forward(blobPtr, idsPtr, tokenIds.length, hiddenPtr, 0);
       if (rc !== tokenIds.length) throw new Error(`inline encoder failed: ${rc}`);
@@ -112,11 +144,75 @@ export async function createInlineTransformerEmbedder({ declaration, vocabText,
     return { vector: pooled, text, windows };
   }
 
+  // Per-word vectors from the same forward pass embed() already runs — the
+  // kernel's outHidden buffer holds every token's final-layer hidden state
+  // (encoder.cpp's encoder_forward does no pooling itself; JS pools it),
+  // so this is not a second, slower embedding path: it is the same
+  // forward pass, with the per-token buffer copied out and pooled per
+  // WORD (via encodeWithWordMap's word alignment, merging WordPiece
+  // sub-tokens like "quant"+"##ization" into one vector) instead of
+  // discarded after sentence-level pooling. Used for MaxSim-style
+  // token-level grounding (complete/retrieval-abstention.mjs, calibrate.
+  // mjs), an alternative to coverage1's exact/stemmed word match that
+  // measures semantic closeness instead of lexical identity — so a query
+  // and passage that use different words for the same idea can still
+  // ground each other, the case coverage1 structurally cannot handle.
+  // Windowing/CLS-pooling limits mirror embed(): a word that falls past
+  // the first window under CLS pooling, or past the last window under
+  // mean pooling's cap, has no vector and is left out of the result.
+  async function embedWords(text) {
+    const { ids: allIds, wordOf: allWordOf, words } = tokenizer.encodeWithWordMap(String(text || ''));
+    const interior = allIds.slice(1, -1);
+    const interiorWordOf = allWordOf.slice(1, -1);
+    const windowLen = maxSeq - 2;
+    const windows = Math.max(1, Math.ceil(interior.length / windowLen));
+    const windowsToEncode = usesClsPooling ? 1 : windows;
+    // Accumulate per-word sums, then average — a word split into several
+    // WordPiece pieces gets one vector, not several.
+    const wordSums = words.map(() => new Float64Array(dim));
+    const wordCounts = words.map(() => 0);
+    for (let w = 0; w < windowsToEncode; w++) {
+      const idSlice = interior.slice(w * windowLen, (w + 1) * windowLen);
+      const wordOfSlice = interiorWordOf.slice(w * windowLen, (w + 1) * windowLen);
+      const tokenIds = [allIds[0], ...idSlice, allIds[allIds.length - 1]];
+      // The windowing above must never produce more tokens than the buffers
+      // were sized for; fail closed instead of writing past them.
+      if (tokenIds.length > maxSeq) {
+        throw new Error(`inline encoder window is ${tokenIds.length} tokens for a ${maxSeq}-token buffer`);
+      }
+      new Int32Array(EM.HEAP32.buffer, idsPtr, tokenIds.length).set(tokenIds);
+      const rc = EM._encoder_forward(blobPtr, idsPtr, tokenIds.length, hiddenPtr, 0);
+      if (rc !== tokenIds.length) throw new Error(`inline encoder failed: ${rc}`);
+      const hidden = new Float32Array(EM.HEAPF32.buffer, hiddenPtr, tokenIds.length * dim);
+      // Offset by 1: tokenIds[0] is [CLS], so idSlice's tokens start at
+      // hidden row 1, matching wordOfSlice's own indexing into idSlice.
+      for (let t = 0; t < wordOfSlice.length; t++) {
+        const wi = wordOfSlice[t];
+        if (wi === -1) continue;
+        const row = (t + 1) * dim;
+        for (let d = 0; d < dim; d++) wordSums[wi][d] += hidden[row + d];
+        wordCounts[wi]++;
+      }
+    }
+    const vectors = new Map();
+    for (let wi = 0; wi < words.length; wi++) {
+      if (wordCounts[wi] === 0) continue;
+      const v = new Float32Array(dim);
+      let norm = 0;
+      for (let d = 0; d < dim; d++) { v[d] = wordSums[wi][d] / wordCounts[wi]; norm += v[d] ** 2; }
+      norm = Math.sqrt(norm) || 1;
+      for (let d = 0; d < dim; d++) v[d] /= norm;
+      vectors.set(words[wi], v);
+    }
+    return vectors;
+  }
+
   const embedder = {
     declaration,
     dim,
     maxSeq,
     embed,
+    embedWords,
     dispose() {
       EM._free(hiddenPtr);
       EM._free(idsPtr);

package/complete/layer-append.mjs

@@ -0,0 +1,246 @@
+// `append` — LAYERED_PROFILE.md 6.1.
+//
+// Compiles one layer on top of a mounted chain: new records, a cumulative
+// tombstone bitset, and a commitment to the parent's identity. The hard parts
+// are not the bytes but the preconditions, so they are checked first and
+// separately, and each refusal names the rule it enforces.
+//
+// Two things the complete profile never promised are required to append:
+//
+//   1. An encoder that can embed PASSAGES. A kind-3 base carries the teacher,
+//      so a pack is sufficient to compile its own successors. A kind-2 base
+//      names its encoder and the host supplies it. A KIND-1 base carries a
+//      corpus-distilled QUERY encoder whose passages were embedded by a teacher
+//      the artifact neither carries nor verifies — so a kind-1 base is readable
+//      but NOT appendable, and append MUST refuse it.
+//   2. An ingestion declaration, because record granularity is part of what the
+//      calibration saw and MUST NOT change mid-chain. A base compiled after
+//      this profile carries `corpus.ingest`; an older one does not, and append
+//      MUST refuse unless the operator asserts one.
+//
+// This module does the planning and assembly that is testable in-process. It
+// takes already-embedded vectors rather than embedding itself, so it stays
+// environment-neutral and so a test can drive it without a 24 MiB encoder.
+
+import { buildTombstoneSegment, zeroExtend, bitsetBytesFor, testBit, popcountBytes, MAX_RECORDS } from './tombstones.mjs';
+import { buildInheritedQuerySegment } from './layer-encoder.mjs';
+import { canonicalJson } from './layer-manifest.mjs';
+
+// 6.1: the producer default is 8 — "an operational recommendation until the
+// range-proof measurements of section 13 say otherwise; the format ceiling of
+// 16 is enforced regardless". 13.5 still lists it as unmeasured.
+export const PRODUCER_MAX_DEPTH = 8;
+export const FORMAT_MAX_DEPTH = 16;
+
+/**
+ * Decide whether a base can be appended to, and how its encoder is obtained.
+ * Separated from the assembly because it is the part with a normative answer
+ * for every base ever released.
+ *
+ * @param {{qiKind: number, corpusIngest: object|null, assertIngest: object|null}} spec
+ * @returns {{appendable: true, encoderSource: 'pack'|'host', ingest: object, ingestAsserted: boolean}}
+ */
+export function planAppendability({ qiKind, corpusIngest = null, assertIngest = null }) {
+    if (qiKind === 1) {
+        throw new Error('a kind-1 base carries a corpus-distilled query encoder whose passages were embedded by a teacher the artifact does not carry: readable, but not appendable (6.1)');
+    }
+    if (qiKind === 4) {
+        throw new Error('a layer is not a base: append to the chain\'s head, whose base supplies the encoder (4.5)');
+    }
+    let encoderSource;
+    if (qiKind === 3) encoderSource = 'pack';       // the pack carries its teacher
+    else if (qiKind === 2) encoderSource = 'host';  // the host supplies the named encoder
+    else throw new Error(`unsupported query-interpretation kind ${qiKind} for append`);
+
+    if (corpusIngest !== null && corpusIngest !== undefined) {
+        if (assertIngest) {
+            throw new Error('the base declares corpus.ingest; --assert-ingest would override an artifact\'s own declaration (6.1)');
+        }
+        return { appendable: true, encoderSource, ingest: corpusIngest, ingestAsserted: false };
+    }
+    if (!assertIngest) {
+        throw new Error('the base carries no corpus.ingest, so record granularity cannot be enforced; supply --assert-ingest to state it (6.1). The assertion is the operator\'s statement, not the artifact\'s.');
+    }
+    if (typeof assertIngest !== 'object' || Array.isArray(assertIngest)) {
+        throw new Error('--assert-ingest must be a JSON object');
+    }
+    if ('ingestAsserted' in assertIngest) {
+        throw new Error('the asserted declaration must not contain ingestAsserted: it is a sibling field, so asserting must not change the declaration\'s canonical form (4.1)');
+    }
+    return { appendable: true, encoderSource, ingest: assertIngest, ingestAsserted: true };
+}
+
+/**
+ * Plan a layer's ids, mask and supersessions — step 4 and 5 of 6.1.
+ *
+ * @param {{parentRowBase: number, parentRecords: number, parentBitset: Uint8Array,
+ *          newRecordCount: number, remove?: number[], supersede?: Array<[number, string|number]>,
+ *          maxDepth?: number, parentDepth: number}} spec
+ */
+export function planLayer({
+    parentRowBase, parentRecords, parentBitset,
+    newRecordCount, remove = [], supersede = [],
+    parentDepth, maxDepth = PRODUCER_MAX_DEPTH,
+}) {
+    const depth = parentDepth + 1;
+    if (depth > FORMAT_MAX_DEPTH) {
+        throw new Error(`appending would reach depth ${depth}, past the format ceiling of ${FORMAT_MAX_DEPTH} (2.7)`);
+    }
+    if (depth > maxDepth) {
+        throw new Error(`appending would reach depth ${depth}, past --max-depth ${maxDepth}; compact the chain instead (6.1)`);
+    }
+    const rowBase = parentRowBase + parentRecords;
+    if (!Number.isSafeInteger(rowBase) || rowBase < 0) throw new Error('parent row arithmetic is not a safe integer');
+    // 3.3: a producer MUST refuse an append whose rowBase + records would
+    // exceed the id ceiling. There is no other mechanism.
+    if (rowBase + newRecordCount > MAX_RECORDS) {
+        throw new Error(`appending ${newRecordCount} records at rowBase ${rowBase} would exhaust the id space (${MAX_RECORDS}); compact the chain (3.3)`);
+    }
+
+    // Step 5: the cumulative bitset is zeroExtend(T_parent, rowBase), then
+    // every --remove id and every --supersede old id.
+    const inherited = zeroExtend(parentBitset, rowBase);
+    const bitset = new Uint8Array(bitsetBytesFor(rowBase));
+    bitset.set(inherited.subarray(0, bitset.length));
+    const introduced = [];
+    const setOne = (id, what) => {
+        if (!Number.isSafeInteger(id) || id < 0) throw new Error(`${what} id ${id} is not a valid id`);
+        // A layer cannot retract its own records: ids at or above rowBase are
+        // out of range by construction (3.4).
+        if (id >= rowBase) {
+            throw new Error(`${what} id ${id} is at or above this layer's rowBase ${rowBase}: a layer cannot retract its own records (3.4)`);
+        }
+        if (testBit(bitset, id)) return false;   // already deleted is a no-op
+        bitset[id >> 3] |= 1 << (id & 7);
+        introduced.push(id);
+        return true;
+    };
+    for (const id of remove) setOne(id, '--remove');
+
+    // Each pair is [oldId, target]. `target` is the CALLER's way of naming
+    // which new record supersedes oldId: a number is a resolved local index
+    // into this layer's own records, which the producer computed by matching
+    // the --supersede path against the chunks that path produced. A
+    // non-numeric target (a raw path the producer could not resolve) falls
+    // back to positional assignment, which is what this did for every pair —
+    // so `--supersede 5=a.md --supersede 9=b.md` bound 5 to whichever chunk
+    // happened to land first, regardless of which file produced it.
+    const supersessions = [];
+    let nextNewId = rowBase;
+    const claimed = new Set();
+    for (const [oldId, target] of supersede) {
+        let newId;
+        if (Number.isSafeInteger(target)) {
+            if (target < 0 || target >= newRecordCount) {
+                throw new Error(`--supersede resolved to local record ${target}, outside this layer's ${newRecordCount} record(s) (3.5)`);
+            }
+            newId = rowBase + target;
+            if (claimed.has(newId)) {
+                throw new Error(`--supersede maps two old ids onto new record ${newId}: each new record supersedes at most one predecessor (3.5)`);
+            }
+        } else {
+            while (claimed.has(nextNewId)) nextNewId += 1;
+            if (nextNewId >= rowBase + newRecordCount) {
+                throw new Error('more --supersede pairs than new records: every newId must lie in this layer\'s own id range (3.5)');
+            }
+            newId = nextNewId;
+            nextNewId += 1;
+        }
+        setOne(oldId, '--supersede');
+        supersessions.push([oldId, newId]);
+        claimed.add(newId);
+    }
+    // 3.5: within one layer each oldId appears at most once.
+    const seen = new Set();
+    for (const [oldId] of supersessions) {
+        if (seen.has(oldId)) throw new Error(`--supersede names id ${oldId} more than once: an oldId has at most one edge per layer (3.5)`);
+        seen.add(oldId);
+    }
+
+    // 6.1: "A layer must do something." A layer that introduces neither a
+    // record nor a new tombstone would still lengthen every chain carrying it,
+    // cost a mount round trip and count against the depth limits, for no
+    // corpus meaning. This is also why 4.3 requires a tombstone-only layer's
+    // tombstoneCount to EXCEED its parent's.
+    if (newRecordCount === 0 && introduced.length === 0) {
+        throw new Error('refusing to emit a layer that introduces neither a record nor a new tombstone (6.1): --remove of already-deleted ids alone produces nothing');
+    }
+
+    return {
+        depth, rowBase, records: newRecordCount,
+        bitset, tombstones: popcountBytes(bitset),
+        delta: introduced.sort((a, b) => a - b),
+        supersessions,
+        tombstoneOnly: newRecordCount === 0,
+    };
+}
+
+/**
+ * Assemble the layer's manifest fields and its tombstone + kind-4 segments.
+ * The caller supplies the index/corpus/lexical segment bytes, since building
+ * those is the existing builder's job.
+ */
+export function buildLayerSegments({ plan, baseIdentity, parentIdentity, baseQueryInterpSha256, parentLocator = null, ingest, ingestAsserted, dim, metric, encoder, corpus, layerBloom = null }) {
+    const tombstones = buildTombstoneSegment({
+        rowBase: plan.rowBase,
+        tombstonedIds: idsSetIn(plan.bitset, plan.rowBase),
+        supersessions: plan.supersessions,
+        records: plan.records,
+    });
+    const queryInterp = buildInheritedQuerySegment({
+        baseIdentity, queryInterpSha256: baseQueryInterpSha256,
+        vocabBloom: layerBloom ? layerBloom.geometry : null,
+        vocabBloomBytes: layerBloom ? layerBloom.bytes : null,
+    });
+    const manifestFields = {
+        profile: 'pikelet-layer-v1',
+        dim, metric, encoder,
+        layer: {
+            parent: { identity: parentIdentity, ...(parentLocator ? { locator: parentLocator } : {}) },
+            baseIdentity,
+            depth: plan.depth,
+            rowBase: plan.rowBase,
+            records: plan.records,
+            tombstones: plan.tombstones,
+            supersessions: plan.supersessions.length,
+            ingest,
+            // 4.1: a SIBLING of layer.ingest, never a member — asserting must
+            // not change the declaration's canonical form.
+            ...(ingestAsserted ? { ingestAsserted: true } : {}),
+        },
+        // A tombstone-only layer still carries a corpus object, with records 0,
+        // and omits the layout fields (4.1).
+        corpus: plan.tombstoneOnly ? { records: 0, ...(corpus?.provenance ? { provenance: corpus.provenance } : {}) } : corpus,
+        // A layer with records carries an index object; assemblePikeletFile
+        // fills in headerSha256 from the sketch bytes, but the object must be
+        // present for the manifest to be a valid layer manifest (4.1). A
+        // tombstone-only layer omits it, having no index segment (4.3).
+        ...(plan.tombstoneOnly ? {} : { index: {} }),
+    };
+    return { manifestFields, tombstones, queryInterp };
+}
+
+/**
+ * The drift an append would produce, and whether the producer may emit it.
+ * 6.1: append MUST refuse a layer whose drift would exceed the chain's
+ * effective limit unless --allow-drift is given.
+ */
+export function checkAppendDrift({ baseRecords, appendedBefore, newRecords, headTombstones, effectiveLimit, allowDrift = false }) {
+    if (!Number.isSafeInteger(baseRecords) || baseRecords <= 0) {
+        throw new Error('drift needs a base with at least one record');
+    }
+    const drift = (appendedBefore + newRecords + headTombstones) / baseRecords;
+    if (drift > effectiveLimit && !allowDrift) {
+        throw new Error(`this append would take calibrationDrift to ${drift.toFixed(4)}, past the chain's effective limit ${effectiveLimit}; pass --allow-drift to emit it anyway (the chain then serves unscored) (6.1)`);
+    }
+    return { drift, exceedsLimit: drift > effectiveLimit };
+}
+
+/** Every id set in a bitset, ascending. */
+export function idsSetIn(bitset, rowBase) {
+    const out = [];
+    for (let id = 0; id < rowBase; id++) if (testBit(bitset, id)) out.push(id);
+    return out;
+}
+
+export { canonicalJson };

package/complete/layer-chain.mjs

@@ -0,0 +1,284 @@
+// Chain state — LAYERED_PROFILE.md 5.1 (steps 4-5), 5.2, 5.3, 5.5, 5.6.
+//
+// Everything a mounted chain needs that is not bytes: the ordered interval
+// table of search tiers, id ownership, the head mask projected onto each tier's
+// local range, the union supersession map, calibration drift, and the ancestry
+// used to scope citations.
+//
+// Pure: the caller supplies already-verified per-member facts. That keeps the
+// whole of 5.2/5.3/5.6 testable without building files, and keeps the sharp
+// edges in one place.
+
+import { testBit, bitsetBytesFor } from './tombstones.mjs';
+
+/**
+ * A **search tier** (3.1) is the base, or any layer with records >= 1: a chain
+ * member that owns ids. Tombstone-only layers are chain members but not search
+ * tiers, and take no part in id resolution.
+ *
+ * Every per-tier construction in section 5 ranges over search tiers, so the
+ * base is always included — "for every layer" would literally have excluded it,
+ * which is the bug the term exists to prevent.
+ */
+export function buildIntervalTable(members) {
+    if (!Array.isArray(members) || members.length === 0) {
+        throw new Error('chain has no members');
+    }
+    if (!members[0] || members[0].depth !== 0) {
+        throw new Error('the first chain member must be the base (depth 0)');
+    }
+    const tiers = [];
+    let expectedRowBase = 0;
+    let expectedDepth = 0;
+    for (const m of members) {
+        if (m.depth !== expectedDepth) {
+            throw new Error(`chain member at index ${expectedDepth} declares depth ${m.depth}`);
+        }
+        const rowBase = m.depth === 0 ? 0 : m.rowBase;
+        if (rowBase !== expectedRowBase) {
+            throw new Error(`chain member at depth ${m.depth} has rowBase ${rowBase}, expected ${expectedRowBase}`);
+        }
+        if (!Number.isSafeInteger(m.records) || m.records < 0) {
+            throw new Error(`chain member at depth ${m.depth} has a bad record count`);
+        }
+        if (m.records > 0) {
+            tiers.push({
+                tier: tiers.length,
+                depth: m.depth,
+                identity: m.identity,
+                rowBase,
+                records: m.records,
+                end: rowBase + m.records,
+            });
+        }
+        expectedRowBase = rowBase + m.records;
+        expectedDepth += 1;
+    }
+    if (!tiers.length) throw new Error('a chain must have at least one search tier');
+    // Search tiers' rowBase values are strictly increasing, since each owns at
+    // least one id. This is the property that makes binary search valid over
+    // THIS table and invalid over all members (5.2).
+    for (let i = 1; i < tiers.length; i++) {
+        if (tiers[i].rowBase <= tiers[i - 1].rowBase) {
+            throw new Error('search tier rowBase values must be strictly increasing');
+        }
+    }
+    return { tiers, rowTotal: expectedRowBase };
+}
+
+/**
+ * `owner(id)` — 5.2. The unique search tier whose interval contains `id`.
+ *
+ * Binary search over the SEARCH TIERS only. Searching "the largest rowBase <=
+ * id" over all chain members is wrong: consecutive tombstone-only layers give
+ * the next layer the same rowBase (3.3), so rowBase is not a unique key over
+ * members and such a search can land on a member that owns no ids at all.
+ */
+export function ownerOf(table, id) {
+    if (!Number.isSafeInteger(id) || id < 0) return null;
+    const { tiers } = table;
+    let lo = 0;
+    let hi = tiers.length - 1;
+    while (lo <= hi) {
+        const mid = (lo + hi) >> 1;
+        const t = tiers[mid];
+        if (id < t.rowBase) hi = mid - 1;
+        else if (id >= t.end) lo = mid + 1;
+        else return { tier: t, localId: id - t.rowBase };
+    }
+    // Ids outside every interval are invalid and MUST be rejected (5.2).
+    return null;
+}
+
+/**
+ * Project the head's bitset onto each search tier's local range (5.3):
+ * `mask_j[i] = T_head[rowBase_j + i]`, computed once at mount.
+ *
+ * Returned as one Uint8Array per tier, LSB-first, so a tier's scan can test a
+ * local row with one bit test and no arithmetic against the global id space.
+ */
+export function projectMasks(table, headBitset) {
+    const masks = [];
+    for (const t of table.tiers) {
+        const bytes = bitsetBytesFor(t.records);
+        const mask = new Uint8Array(bytes);
+        if (headBitset && headBitset.length) {
+            for (let i = 0; i < t.records; i++) {
+                if (testBit(headBitset, t.rowBase + i)) mask[i >> 3] |= 1 << (i & 7);
+            }
+        }
+        masks.push(mask);
+    }
+    return masks;
+}
+
+/** Live record count: rowTotal minus the head's tombstone popcount. */
+export function liveCount(table, headTombstoneCount) {
+    return table.rowTotal - (headTombstoneCount || 0);
+}
+
+/**
+ * The union supersession map (5.1 step 4, 3.5). Each layer's segment carries
+ * only the edges that layer recorded; the reader forms the union at mount.
+ *
+ * Edges are ordered by the DEPTH of the layer that recorded them — never by
+ * segment order, file order or id — so "earliest" and "latest" are always
+ * unique (within one layer an oldId has at most one edge).
+ */
+export function buildSupersessionMap(perLayer) {
+    const map = new Map();
+    for (const { depth, supersessions } of perLayer) {
+        for (const [oldId, newId] of supersessions || []) {
+            if (!map.has(oldId)) map.set(oldId, []);
+            map.get(oldId).push({ depth, newId });
+        }
+    }
+    for (const edges of map.values()) {
+        edges.sort((a, b) => a.depth - b.depth);
+        for (let i = 1; i < edges.length; i++) {
+            if (edges[i].depth === edges[i - 1].depth) {
+                throw new Error(`two supersessions for the same oldId at depth ${edges[i].depth}: within one layer an oldId has at most one edge`);
+            }
+        }
+    }
+    return map;
+}
+
+/**
+ * The three views of 3.5.
+ *  - immediate: the edge recorded at the LEAST depth
+ *  - all:       every edge, ascending by depth
+ *  - current:   start from the GREATEST-depth edge and repeatedly follow each
+ *               reached id's greatest-depth edge until an id with none
+ *
+ * Cycles are structurally impossible (newId >= rowBase_j > oldId, so edges
+ * always point to higher ids), but the walk is bounded anyway: a corrupted map
+ * must not hang a reader.
+ */
+export function resolveSupersession(map, id) {
+    const edges = map.get(id);
+    if (!edges || !edges.length) return { immediate: null, all: [], current: null };
+    const immediate = edges[0].newId;
+    const all = edges.map((e) => e.newId);
+    let current = edges[edges.length - 1].newId;
+    const seen = new Set([id, current]);
+    for (let hops = 0; hops < 64; hops++) {
+        const next = map.get(current);
+        if (!next || !next.length) break;
+        const candidate = next[next.length - 1].newId;
+        if (seen.has(candidate)) break;
+        seen.add(candidate);
+        current = candidate;
+    }
+    return { immediate, all, current };
+}
+
+/**
+ * Calibration drift (3.1):
+ *   (sum of layer records for depth >= 1 + head tombstones) / base records
+ *
+ * It counts HISTORY, not current divergence: a record appended and later
+ * tombstoned counts twice, and a supersession counts as one append plus one
+ * tombstone. Intentionally conservative — the fit's population lost a member it
+ * had seen and gained one it had not.
+ */
+export function calibrationDrift(members, headTombstoneCount) {
+    const base = members[0];
+    if (!base || base.depth !== 0) throw new Error('calibrationDrift needs the base as the first member');
+    if (!Number.isSafeInteger(base.records) || base.records <= 0) {
+        throw new Error('calibrationDrift needs a base with at least one record');
+    }
+    let appended = 0;
+    for (const m of members) if (m.depth >= 1) appended += m.records;
+    return (appended + (headTombstoneCount || 0)) / base.records;
+}
+
+/**
+ * Calibration status (5.5). The effective limit is
+ * `min(producerEnvelope, readerLimit)`: a host can always be stricter than the
+ * artifact, and an artifact can never make a host less strict.
+ *
+ * An absent producer envelope is +Infinity (no claim is no constraint), and the
+ * reader's limit MUST be finite, so an artifact that declares nothing is still
+ * gated.
+ */
+export function calibrationStatus({ drift, producerEnvelope, readerLimit, baseHasFit, everyLayerShipsBloom }) {
+    if (!Number.isFinite(readerLimit)) {
+        throw new Error('readerLimit MUST be finite (5.5)');
+    }
+    const envelope = Number.isFinite(producerEnvelope) ? producerEnvelope : Infinity;
+    const effectiveLimit = Math.min(envelope, readerLimit);
+    if (!baseHasFit) {
+        return { status: 'none', drift, effectiveLimit, producerEnvelope: envelope, readerLimit };
+    }
+    // A layer whose calibration region is `none` while the base carries a fit
+    // degrades the chain to unscored (4.5, 5.5).
+    const status = (drift <= effectiveLimit && everyLayerShipsBloom) ? 'inherited' : 'drift-exceeded';
+    return { status, drift, effectiveLimit, producerEnvelope: envelope, readerLimit };
+}
+
+/**
+ * Global lexical statistics over search tiers (5.4). Tombstoned records are
+ * counted in N, avgdl and df: removing them would require reading every posting
+ * list at mount.
+ *
+ * The resulting scores differ from a from-scratch build over the live set. The
+ * drift gate of 5.5 is a change-volume gate and implies no bound on that
+ * difference; 13.3 measured it at under 0.005 nDCG@10 up to 25% drift on two
+ * BEIR datasets, which is why no correction table ships. A reader MUST report
+ * `statsIncludeTombstoned: true`.
+ */
+export function globalLexicalStats(perTier) {
+    let N = 0;
+    let totalTokens = 0;
+    for (const t of perTier) {
+        if (!Number.isSafeInteger(t.docCount) || t.docCount < 0) throw new Error('a tier has a bad docCount');
+        N += t.docCount;
+        totalTokens += t.totalTokens;
+    }
+    return {
+        N,
+        totalTokens,
+        avgdl: N > 0 ? Math.max(1, totalTokens / N) : 1,
+        statsIncludeTombstoned: true,
+    };
+}
+
+/** df(t) summed across tiers, and the idf the reader scores with (5.4). */
+export function idfFor(N, df) {
+    return Math.log(1 + (N - df + 0.5) / (df + 0.5));
+}
+
+/**
+ * Ancestry for citation scoping (5.6): identity -> { depth, rowTotal }.
+ *
+ * A bare id is an id in the mounted head; a citation is (identity, id), and it
+ * resolves only along the mounted ancestry. A citation against a head that is
+ * on this ancestry is valid when its id existed when that head was current —
+ * that is, `id < rowTotal(thatHead)`.
+ */
+export function buildAncestry(members) {
+    const ancestry = new Map();
+    let rowTotal = 0;
+    for (const m of members) {
+        rowTotal += m.records;
+        ancestry.set(m.identity, { depth: m.depth, rowTotal });
+    }
+    return ancestry;
+}
+
+/**
+ * Resolve a citation (identity, id) against the mounted ancestry (5.6).
+ * Returns a reason rather than throwing, because "not on this history" is a
+ * legitimate answer a consumer needs to be able to act on.
+ */
+export function resolveCitation(ancestry, identity, id) {
+    const entry = ancestry.get(identity);
+    if (!entry) return { ok: false, reason: 'not-on-this-history' };
+    if (!Number.isSafeInteger(id) || id < 0) return { ok: false, reason: 'invalid-id' };
+    // An id at or above that head's rowTotal did not exist when it was current,
+    // so a citation naming it is invalid under that identity even though the id
+    // is valid in the current head.
+    if (id >= entry.rowTotal) return { ok: false, reason: 'id-postdates-that-head' };
+    return { ok: true, depth: entry.depth, id };
+}