pikelet-wasm 0.8.1 → 0.9.0

package/complete/layer-rebase.mjs

@@ -0,0 +1,218 @@
+// `rebase` — LAYERED_PROFILE.md 6.2.
+//
+// Two producers appending to the same parent create a fork. Both chains are
+// valid; the ref names one head. The other layer is re-based onto it: its rows,
+// sketches, records, postings and bloom are copied verbatim (nothing is
+// re-embedded, local ids do not change), and its rowBase, bitset, supersessions
+// and evaluation material are recomputed.
+//
+// The correctness rule the spec was revised twice to get right:
+//
+//   A rebase replays what the layer ITSELF did -- its delta over its old
+//   parent, and the edges physically in its own segment -- and NEVER its
+//   cumulative bitset. T_L contains every deletion any ancestor on the old
+//   branch performed; replaying it would import those ancestors' operations
+//   into a branch that never performed them, and worse, import an ancestor's
+//   tombstone without that ancestor's supersession edge: half of one operation.
+//
+// To compute the delta, the rebase MUST open L's ORIGINAL parent. If that
+// artifact is unavailable the delta is unknowable and the rebase must refuse --
+// it cannot be reconstructed from L alone.
+
+import { zeroExtend, bitsetBytesFor, testBit, popcountBytes } from './tombstones.mjs';
+import { canonicalJson } from './layer-manifest.mjs';
+
+export const ON_CONFLICT = Object.freeze({ REFUSE: 'refuse', SKIP: 'skip', KEEP_BOTH: 'keep-both' });
+export const ON_FOREIGN = Object.freeze({ REFUSE: 'refuse', DROP: 'drop' });
+
+/**
+ * The fork point: the longest common prefix of L's original parent chain and
+ * H's chain, compared layer by layer from the base BY IDENTITY, and the total
+ * row count of that prefix.
+ *
+ * Every id below forkRowBase names the same record in both histories. Every id
+ * at or above it belongs to a layer the two histories do not share — fork
+ * siblings assign overlapping id ranges to different records, so such an id is
+ * foreign.
+ *
+ * @param {Array<{identity: string, records: number}>} oldChain base-first, L's ancestors
+ * @param {Array<{identity: string, records: number}>} newChain base-first, H's chain
+ */
+export function forkPoint(oldChain, newChain) {
+    let i = 0;
+    let forkRowBase = 0;
+    while (i < oldChain.length && i < newChain.length
+           && oldChain[i].identity === newChain[i].identity) {
+        forkRowBase += oldChain[i].records;
+        i += 1;
+    }
+    if (i === 0) {
+        throw new Error('the two histories share no base: a rebase requires H.baseIdentity == L.baseIdentity (6.2)');
+    }
+    return { sharedPrefix: i, forkRowBase };
+}
+
+/**
+ * Preconditions a rebase must satisfy before it constructs any bytes (6.2).
+ * They are producer preconditions precisely so a rebase never emits a layer a
+ * reader would refuse.
+ */
+export function checkRebasePreconditions({ layerManifest, headChainDecl, layerChainDecl, headIsLegacyBase = false, headBaseIdentity, layerBaseIdentity }) {
+    if (headBaseIdentity !== layerBaseIdentity) {
+        throw new Error(`rebase refuses: the head's base ${String(headBaseIdentity).slice(0, 12)}… is not the layer's base ${String(layerBaseIdentity).slice(0, 12)}… (6.2)`);
+    }
+    // Same base is NOT enough. Two depth-1 forks of a legacy base can each have
+    // asserted a DIFFERENT declaration, and copying L's layer.ingest onto H's
+    // history would emit a layer a reader must reject.
+    if (headIsLegacyBase) {
+        // H is the legacy depth-0 base itself and has no declaration yet, so a
+        // rebased depth-1 layer MAY introduce L's asserted declaration.
+        if (layerChainDecl && !layerChainDecl.asserted) {
+            throw new Error('rebase refuses: the head is a legacy base with no declaration, but the layer\'s declaration is not asserted (6.2)');
+        }
+        return { ingest: layerManifest.layer.ingest, ingestAsserted: true };
+    }
+    if (!headChainDecl || !layerChainDecl) {
+        throw new Error('rebase refuses: an ingestion declaration is missing on one of the two histories (6.1, 6.2)');
+    }
+    if (headChainDecl.canonical !== layerChainDecl.canonical) {
+        throw new Error('rebase refuses: the two histories disagree on the chain ingestion declaration; a copied layer.ingest would be rejected by a reader (6.2)');
+    }
+    return { ingest: layerManifest.layer.ingest, ingestAsserted: headChainDecl.asserted };
+}
+
+/**
+ * Plan a rebase.
+ *
+ * @param {object} spec
+ * @param {Uint8Array} spec.layerBitset        T_L, the layer's own cumulative bitset
+ * @param {Uint8Array} spec.oldParentBitset    T_oldParent — REQUIRED; the delta is
+ *   unknowable without it, and a rebase must refuse rather than guess.
+ * @param {number} spec.oldRowBase             L.layer.rowBase
+ * @param {number} spec.layerRecords           R_L
+ * @param {Array<[number, number]>} spec.layerSupersessions  S_L, physically recorded in L
+ * @param {number} spec.newRowBase             H.rowBase + H.records
+ * @param {Uint8Array} spec.headBitset         T_H
+ * @param {Map<number, Array<{depth:number,newId:number}>>} spec.headSupersessions  S_H
+ * @param {number} spec.forkRowBase
+ * @param {'refuse'|'drop'} [spec.onForeign]
+ * @param {'refuse'|'skip'|'keep-both'} [spec.onConflict]
+ */
+export function planRebase({
+    layerBitset, oldParentBitset, oldRowBase, layerRecords, layerSupersessions = [],
+    newRowBase, headBitset, headSupersessions = new Map(), forkRowBase,
+    onForeign = ON_FOREIGN.REFUSE, onConflict = ON_CONFLICT.REFUSE,
+}) {
+    if (!(oldParentBitset instanceof Uint8Array)) {
+        throw new Error('rebase refuses: the layer\'s original parent is unavailable, so its delta over that parent is unknowable (6.2)');
+    }
+    if (!Object.values(ON_FOREIGN).includes(onForeign)) throw new Error(`--on-foreign must be one of ${Object.values(ON_FOREIGN).join(', ')}`);
+    if (!Object.values(ON_CONFLICT).includes(onConflict)) throw new Error(`--on-conflict must be one of ${Object.values(ON_CONFLICT).join(', ')}`);
+
+    // ΔT_L = T_L \ zeroExtend(T_oldParent, L.rowBase) — never T_L as a whole.
+    const inheritedOld = zeroExtend(oldParentBitset, oldRowBase);
+    const delta = [];
+    for (let id = 0; id < oldRowBase; id++) {
+        if (testBit(layerBitset, id) && !testBit(inheritedOld, id)) delta.push(id);
+    }
+
+    // Two kinds of id, treated oppositely (6.2). Ids L OWNS — its records and
+    // the y side of every edge — are offsets into its own range and must be
+    // translated. Ids L REFERENCES — ΔT_L and the x side — are ancestor ids,
+    // stable only below the fork point.
+    const translateY = (oldY) => newRowBase + (oldY - oldRowBase);
+
+    // Foreign ids: at or above forkRowBase they belong to a layer the two
+    // histories do not share. The test applies to ΔT_L, not to T_L: bits L
+    // inherited from its old branch are not L's to carry anywhere.
+    const foreignDeletions = delta.filter((x) => x >= forkRowBase);
+    const foreignEdges = layerSupersessions.filter(([x]) => x >= forkRowBase);
+    if ((foreignDeletions.length || foreignEdges.length) && onForeign === ON_FOREIGN.REFUSE) {
+        const ids = [...new Set([...foreignDeletions, ...foreignEdges.map(([x]) => x)])].sort((a, b) => a - b);
+        throw new Error(`rebase refuses: ${ids.length} foreign id(s) at or above the fork point ${forkRowBase} name different records in the two histories: ${ids.join(', ')} (pass --on-foreign drop to discard them) (6.2)`);
+    }
+    const survivingDeletions = delta.filter((x) => x < forkRowBase);
+    const survivingEdges = layerSupersessions.filter(([x]) => x < forkRowBase);
+    const droppedForeign = { deletions: foreignDeletions, edges: foreignEdges.map(([x]) => x) };
+
+    // Conflicts, deterministically (6.2).
+    const conflicts = [];
+    const recordedEdges = [];
+    const skippedEdges = [];
+    for (const [x, oldY] of survivingEdges) {
+        const newY = translateY(oldY);
+        // Every newId a rebased layer writes MUST lie in its new range.
+        if (newY < newRowBase || newY >= newRowBase + layerRecords) {
+            throw new Error(`rebase refuses: translated newId ${newY} for oldId ${x} falls outside the layer's new range [${newRowBase}, ${newRowBase + layerRecords}) (6.2)`);
+        }
+        const headEdges = headSupersessions.get(x);
+        const headY = headEdges && headEdges.length ? headEdges[headEdges.length - 1].newId : null;
+        if (headY !== null && headY !== newY) {
+            // H already records a DIFFERENT successor for x.
+            if (onConflict === ON_CONFLICT.REFUSE) { conflicts.push(x); continue; }
+            if (onConflict === ON_CONFLICT.SKIP) { skippedEdges.push(x); continue; }
+            // keep-both: L's edge is recorded as an additional edge, which makes
+            // 3.5's "all edges" view non-trivial — H's edge is the immediate
+            // successor and L's the current one (greatest depth wins).
+            recordedEdges.push([x, newY]);
+            continue;
+        }
+        // x not tombstoned in H, or tombstoned with no edge: recorded either
+        // way — L adds information H lacked.
+        recordedEdges.push([x, newY]);
+    }
+    if (conflicts.length) {
+        throw new Error(`rebase refuses: ${conflicts.length} supersession conflict(s) — the head already records a different successor for id(s) ${conflicts.join(', ')} (pass --on-conflict skip or keep-both) (6.2)`);
+    }
+
+    // The rebased bitset is zeroExtend(T_H, newRowBase) with the surviving
+    // ΔT_L bits set. A deletion already in T_H is a no-op, not a conflict:
+    // deletion is idempotent.
+    const bitset = new Uint8Array(bitsetBytesFor(newRowBase));
+    bitset.set(zeroExtend(headBitset, newRowBase).subarray(0, bitset.length));
+    const newlySet = [];
+    const setBit = (id) => {
+        if (id >= newRowBase) {
+            throw new Error(`rebase refuses: id ${id} is at or above the new rowBase ${newRowBase}`);
+        }
+        if (testBit(bitset, id)) return false;
+        bitset[id >> 3] |= 1 << (id & 7);
+        newlySet.push(id);
+        return true;
+    };
+    for (const x of survivingDeletions) setBit(x);
+    for (const [x] of recordedEdges) setBit(x);
+
+    // Terminal rule: an empty rebase emits NOTHING (6.2). If the layer owns no
+    // records and its surviving deletions set no bit the head does not already
+    // have, emitting would manufacture exactly the artifact 6.1 forbids and 4.3
+    // rejects — records 0 with no increase in tombstoneCount.
+    if (layerRecords === 0 && newlySet.length === 0) {
+        return {
+            emit: false,
+            reason: 'the rebased operation is empty: the target history already represents every deletion this layer performed (6.2)',
+            droppedForeign, skippedEdges,
+        };
+    }
+
+    return {
+        emit: true,
+        newRowBase,
+        records: layerRecords,
+        bitset,
+        tombstones: popcountBytes(bitset),
+        delta: survivingDeletions,
+        newlySet: newlySet.sort((a, b) => a - b),
+        supersessions: recordedEdges.sort((a, b) => a[0] - b[0]),
+        droppedForeign,
+        skippedEdges,
+        // Chain-level golden queries carry expected GLOBAL ids, which are wrong
+        // after a rebase: L's own ids moved and the surrounding chain changed.
+        // A rebase MUST either regenerate them against the rebased chain or drop
+        // them explicitly; it MUST NOT copy them (6.2).
+        goldenQueries: null,
+        goldenQueriesDropped: 'rebase',
+    };
+}
+
+export { canonicalJson };

package/complete/lexical.mjs

@@ -40,6 +40,19 @@ const tokenize = (text) => (String(text).toLowerCase().match(/[a-z0-9']+/g) || [
 // is committed via the manifest digest but lazy reads are not individually
 // verified (the eager opener verifies the whole segment; per-page digests
 // are the format-v2 path if this gap needs closing).
+// The df a term is scored with: the chain-global one from options.df when the
+// caller supplied it (LAYERED_PROFILE.md 5.4), else the index's own. A global
+// df below the local one is impossible for a real sum, so it is refused rather
+// than producing a negative-count idf.
+function globalDf(options, term, localDf) {
+    const g = options.df instanceof Map ? options.df.get(term) : undefined;
+    if (g === undefined) return localDf;
+    if (!Number.isSafeInteger(g) || g < localDf) {
+        throw new Error(`lexical options.df for "${term}" (${g}) is below this index's own df (${localDf})`);
+    }
+    return g;
+}
+
 export async function openLexicalIndexLazy(read, segLength) {
     const headerBytes = await read(0, 64);
     if (!(headerBytes instanceof Uint8Array) || headerBytes.length < 64 || segLength < 64) {
@@ -134,17 +147,46 @@ export async function openLexicalIndexLazy(read, segLength) {
         termCount,
         totalTokens,
         lazy: true,
-        async search(text, n) {
+        // This index's df for each of the query's terms (absent terms omitted).
+        async documentFrequencies(text) {
+            const terms = [...new Set(tokenize(text))];
+            const out = new Map();
+            await Promise.all(terms.map(async (t) => {
+                const e = await findTerm(fnv1a32(t, 0), fnv1a32(t, 0x9e3779b9));
+                if (e) out.set(t, e.df);
+            }));
+            return out;
+        },
+        // `options.exclude(id)` drops a row BEFORE the cap, and
+        // `options.stats` supplies chain-global {docCount, avgdl} for IDF and
+        // length normalization. Both exist for the layered profile
+        // (LAYERED_PROFILE.md 5.3, 5.4): a chain used to cap each tier at n
+        // and mask afterwards, so a tier whose top-n were all tombstoned
+        // contributed nothing while live rows below the cap were discarded;
+        // and each tier scored BM25 against its own docCount, making scores
+        // from different tiers incomparable before fusion. `options.df`
+        // (Map term -> df) is the chain-global df summed over tiers from each
+        // tier's documentFrequencies(); a term absent from it keeps this
+        // index's own df.
+        async search(text, n, options = {}) {
+            const exclude = typeof options.exclude === 'function' ? options.exclude : null;
+            const statDocCount = Number.isFinite(options.stats?.docCount) && options.stats.docCount > 0
+                ? options.stats.docCount : docCount;
+            const statAvgdl = Number.isFinite(options.stats?.avgdl) && options.stats.avgdl > 0
+                ? options.stats.avgdl : avgdl;
             const terms = [...new Set(tokenize(text))];
             if (!terms.length || docCount === 0) return [];
-            const found = (await Promise.all(terms.map((t) => findTerm(fnv1a32(t, 0), fnv1a32(t, 0x9e3779b9)))))
-                .filter(Boolean);
+            const found = (await Promise.all(terms.map(async (t) => {
+                const e = await findTerm(fnv1a32(t, 0), fnv1a32(t, 0x9e3779b9));
+                return e && { ...e, term: t };
+            }))).filter(Boolean);
             if (!found.length) return [];
             const postings = await Promise.all(found.map((e) => read(postingsOffset + e.postingsRel, e.postingsLen)));
             const scores = new Map();
             found.forEach((e, i) => {
                 const bytes = postings[i];
-                const idf = Math.log(1 + (docCount - e.df + 0.5) / (e.df + 0.5));
+                const df = globalDf(options, e.term, e.df);
+                const idf = Math.log(1 + (statDocCount - df + 0.5) / (df + 0.5));
                 const cursor = { at: 0, end: bytes.length };
                 const readVarint = () => {
                     let value = 0;
@@ -163,8 +205,9 @@ export async function openLexicalIndexLazy(read, segLength) {
                     docId = p === 0 ? readVarint() : docId + readVarint();
                     const tf = readVarint();
                     if (docId >= docCount) throw new Error('.pikelet lexical postings doc id out of range');
+                    if (exclude && exclude(docId)) continue;
                     const dl = doclenOf(docId);
-                    const norm = tf + BM25_K1 * (1 - BM25_B + (BM25_B * dl) / avgdl);
+                    const norm = tf + BM25_K1 * (1 - BM25_B + (BM25_B * dl) / statAvgdl);
                     scores.set(docId, (scores.get(docId) || 0) + (idf * tf * (BM25_K1 + 1)) / norm);
                 }
             });
@@ -230,9 +273,34 @@ export function openLexicalIndex(bytes) {
         docCount,
         termCount,
         totalTokens,
+        // This index's df for each of the query's terms (absent terms omitted).
+        documentFrequencies(text) {
+            const out = new Map();
+            for (const term of new Set(tokenize(text))) {
+                const i = findTerm(fnv1a32(term, 0), fnv1a32(term, 0x9e3779b9));
+                if (i >= 0) out.set(term, view.getUint32(entryAt(i) + 20, true));
+            }
+            return out;
+        },
         // BM25 over the query's unique tokens; returns [{id, score}] sorted
         // by score descending (ties by id), at most n entries.
-        search(text, n) {
+        // `options.exclude(id)` drops a row BEFORE the cap, and
+        // `options.stats` supplies chain-global {docCount, avgdl} for IDF and
+        // length normalization. Both exist for the layered profile
+        // (LAYERED_PROFILE.md 5.3, 5.4): a chain used to cap each tier at n
+        // and mask afterwards, so a tier whose top-n were all tombstoned
+        // contributed nothing while live rows below the cap were discarded;
+        // and each tier scored BM25 against its own docCount, making scores
+        // from different tiers incomparable before fusion. `options.df`
+        // (Map term -> df) is the chain-global df summed over tiers from each
+        // tier's documentFrequencies(); a term absent from it keeps this
+        // index's own df.
+        search(text, n, options = {}) {
+            const exclude = typeof options.exclude === 'function' ? options.exclude : null;
+            const statDocCount = Number.isFinite(options.stats?.docCount) && options.stats.docCount > 0
+                ? options.stats.docCount : docCount;
+            const statAvgdl = Number.isFinite(options.stats?.avgdl) && options.stats.avgdl > 0
+                ? options.stats.avgdl : avgdl;
             const terms = [...new Set(tokenize(text))];
             if (!terms.length || docCount === 0) return [];
             const scores = new Map();
@@ -243,15 +311,17 @@ export function openLexicalIndex(bytes) {
                 const len = view.getUint32(entryAt(i) + 16, true);
                 const df = view.getUint32(entryAt(i) + 20, true);
                 if (rel + len > postingsBytes) throw new Error('.pikelet lexical postings out of bounds');
-                const idf = Math.log(1 + (docCount - df + 0.5) / (df + 0.5));
+                const scoringDf = globalDf(options, term, df);
+                const idf = Math.log(1 + (statDocCount - scoringDf + 0.5) / (scoringDf + 0.5));
                 const cursor = { at: postingsOffset + rel, end: postingsOffset + rel + len };
                 let docId = 0;
                 for (let p = 0; p < df; p++) {
                     docId = p === 0 ? readVarint(cursor) : docId + readVarint(cursor);
                     const tf = readVarint(cursor);
                     if (docId >= docCount) throw new Error('.pikelet lexical postings doc id out of range');
+                    if (exclude && exclude(docId)) continue;
                     const dl = view.getUint32(doclenOffset + 4 * docId, true);
-                    const norm = tf + BM25_K1 * (1 - BM25_B + (BM25_B * dl) / avgdl);
+                    const norm = tf + BM25_K1 * (1 - BM25_B + (BM25_B * dl) / statAvgdl);
                     scores.set(docId, (scores.get(docId) || 0) + (idf * tf * (BM25_K1 + 1)) / norm);
                 }
             }

package/complete/retrieval-abstention.mjs

@@ -151,14 +151,25 @@ export function createAbstentionScorer(asset, bloomBytes) {
     }
     const SEEDS = [0, 0x9e3779b9];
 
-    function knownFrac(text) {
+    // `override` is the layered profile's union bloom (LAYERED_PROFILE.md
+    // 4.5): the bitwise OR of the base's bloom and every mounted layer's,
+    // which a chain MUST score against rather than the base's alone.
+    // Identical geometry is enforced where the union is built, and re-checked
+    // here because reading a different-sized bloom with these `bits` would
+    // silently probe the wrong positions rather than fail.
+    function knownFrac(text, override) {
+        const table = override || bloom;
+        if (table !== bloom && table.length !== bloom.length) {
+            throw new Error(`abstention: bloom override is ${table.length} bytes but the fit's is ${bloom.length}; `
+                + 'known_frac would probe the wrong bits');
+        }
         const words = text.toLowerCase().match(/[a-z0-9']+/g) || [];
         if (!words.length) return 0;
         let known = 0;
         for (const w of words) {
             const hit = SEEDS.every((seed) => {
                 const bit = fnv1a(w, seed);
-                return (bloom[bit >> 3] >> (bit & 7)) & 1;
+                return (table[bit >> 3] >> (bit & 7)) & 1;
             });
             if (hit) known++;
         }
@@ -173,7 +184,7 @@ export function createAbstentionScorer(asset, bloomBytes) {
         // fits them together), so coverageCfg.topK already covers both.
         usesPassage: !!coverageCfg,
         passagesNeeded: coverageCfg ? (coverageCfg.topK || 1) : 0,
-        async score(queryText, results, passageTexts) {
+        async score(queryText, results, passageTexts, options = {}) {
             // Fit at build time (pikelet/src/calibrate.mjs) always scores a
             // fixed top-10 window (K = min(10, candidates)), independent of
             // whatever k a caller later passes to query(). Slicing to the
@@ -187,7 +198,7 @@ export function createAbstentionScorer(asset, bloomBytes) {
                 ? top[Math.min(4, top.length - 1)].distance - d0 : 0;
             const mean10 = top.length
                 ? top.reduce((s, r) => s + r.distance, 0) / top.length : 1;
-            const signals = { d0, margin, mean10, known_frac: knownFrac(queryText) };
+            const signals = { d0, margin, mean10, known_frac: knownFrac(queryText, options.vocabBloom) };
             let z = asset.bias;
             asset.features.forEach((f, j) => {
                 z += ((signals[f] - asset.standardize.mean[f]) / asset.standardize.std[f]) * asset.weights[j];