pikelet-wasm 0.10.0 → 0.11.0

package/complete/index.d.ts

@@ -41,6 +41,11 @@ export interface HttpRangeSource extends CompleteRangeSource {
 
 export declare function httpRangeSource(url: string, options?: {
   preferredParallelism?: number;
+  /**
+   * Rows of the vector rerank closer than this many bytes are fetched in
+   * one request, reading the bytes between them. Default 256 KiB. A pack's
+   * manifest recommendedGap applies only when it is wider.
+   */
   preferredGapBytes?: number;
   /** Cap on the one-time full download when a host ignores Range, enforced on received bytes. Default 64 MiB. */
   maxFullFallbackBytes?: number;
@@ -64,8 +69,33 @@ export declare function httpRangeSource(url: string, options?: {
    * the child's origin and directory (LAYERED_PROFILE.md 5.1.1).
    */
   redirectGuard?: (target: string) => void;
+  /**
+   * The fetch used for every request. Default nodeFetch under Node
+   * (the package's node export condition), else globalThis.fetch. A runtime
+   * whose fetch transparently decodes response bodies can pass one that does
+   * not.
+   */
+  fetch?: (input: string, init?: {
+    method?: string;
+    headers?: Record<string, string>;
+    redirect?: 'follow' | 'manual' | 'error';
+  }) => Promise<any>;
 }): HttpRangeSource;
 
+/**
+ * Node only (the 'node' export condition): a fetch over node:http/node:https
+ * that never decompresses, sends Accept-Encoding: identity and refuses any
+ * other Content-Encoding. httpRangeSource defaults to it under Node.
+ */
+export declare function nodeFetch(input: string, init?: {
+  method?: 'GET' | 'HEAD';
+  headers?: Record<string, string>;
+  redirect?: 'follow' | 'manual' | 'error';
+  signal?: unknown;
+  /** Inflate a gzip/deflate body a server sent despite identity (crawled HTML); Brotli/zstd still refused. */
+  inflateGzip?: boolean;
+}): Promise<any>;
+
 export interface CompleteQueryResult {
   /**
    * The pack's calibrated verdict on this retrieval. On packs calibrated
@@ -84,6 +114,22 @@ export interface CompleteQueryResult {
    * other way round.
    */
   results: Array<{ id: number; distance: number } & Record<string, unknown>>;
+  /**
+   * Which passage grounded the verdict and which of the query's content
+   * words it contains. Present when a fitted calibrator scored the query
+   * (absent on 'unscored'). An asked-for value in `uncovered` is one no top
+   * passage states, whatever matchQuality says.
+   */
+  grounding?: {
+    /** Id of the passage the coverage score was taken from. */
+    recordId: number | null;
+    /** Index of that passage in the fused (or distance-ordered) window. */
+    passageIndex: number;
+    /** Weighted share of the query's content words the passage contains, 0-1. */
+    coverage: number;
+    covered: string[];
+    uncovered: string[];
+  };
 }
 
 export interface CompleteSearch {
@@ -105,6 +151,10 @@ export interface CompleteSearch {
      */
     license: string | null;
     records: number;
+    /** Rows in the index: equal to records unless a row map groups several rows per record. */
+    indexRows: number;
+    /** The policy behind a multi-row index (COMPLETE_PROFILE.md 3.9), e.g. sentence windows; null otherwise. */
+    rowMap: { kind: string; sentences: number; rows: number } | null;
     dim: number;
     metric: string | number;
     encoder: Record<string, unknown>;
@@ -156,6 +206,13 @@ export interface CompleteSearch {
      */
     residentScan: 'engine' | 'js';
     sampleQueries: string[];
+    /**
+     * The file's parts in file order: 'header' (fixed header, manifest and
+     * segment table) followed by each segment of a known kind ('index',
+     * 'corpus', 'lexical', 'query-interp', 'evaluation', ...). Absolute
+     * byte offsets.
+     */
+    layout: Array<{ kind: string; offset: number; length: number }>;
   };
   /**
    * Verify the index's lazy vector rows against the identity-anchored
@@ -191,6 +248,8 @@ export interface CompleteSearch {
   }): Promise<CompleteQueryResult>;
   /** Hydrate one corpus record by id (verified per record on format 2). */
   record(id: number): Promise<Record<string, unknown>>;
+  /** The absolute byte range one corpus record's stored bytes occupy in the file. */
+  recordRange(id: number): { offset: number; length: number };
   /** The evaluation segment's JSON object, or null when the artifact carries none. */
   evaluation(): Promise<Record<string, unknown> | null>;
   close(): Promise<void>;

package/complete/index.mjs

@@ -25,6 +25,7 @@ import { loadStudentModel, embedTextWithStudent } from './student-embedder.mjs';
 import { computeMatchQuality, computePreSearchAbstention } from './student-abstention.mjs';
 import { createAbstentionScorer } from './retrieval-abstention.mjs';
 import { openLexicalIndex, openLexicalIndexLazy } from './lexical.mjs';
+import { parseRowMapSegment } from './rowmap.mjs';
 import { PikeletSketchArtifact } from '../src/artifact/pikelet-artifact.js';
 import { MAGIC, HEADER_BYTES, TABLE_ENTRY_BYTES, KINDS, KIND_NAMES } from './format.mjs';
 
@@ -769,7 +770,7 @@ export async function openPikeletFile(input, options = {}) {
         // influences a returned result. Small segments (kinds 1/2) keep the
         // one-read eager path unchanged.
         const qiDeferrable = qi.length > LAZY_QI_BYTES;
-        const [qiFirst, sketch, tables, lexicalBytes] = await Promise.all([
+        const [qiFirst, sketch, tables, lexicalBytes, rowMapBytes] = await Promise.all([
             (async () => {
                 if (qiDeferrable) {
                     return readChecked(source, qi.offset, Math.min(16, qi.length), 'query-interp header', maxReadBytes, fileBytes);
@@ -813,6 +814,17 @@ export async function openPikeletFile(input, options = {}) {
                 }
                 return { bytes };
             })() : null,
+            // Row map (kind 8, OPTIONAL): a few bytes per record, read
+            // eagerly and digest-verified. Present when the index holds
+            // several vectors per record (sentence windows).
+            segments.has('rowmap') ? (async () => {
+                const seg = segments.get('rowmap');
+                const bytes = await readChecked(source, seg.offset, seg.length, 'row map segment', maxReadBytes, fileBytes);
+                if (await sha256hex(bytes) !== seg.sha256) {
+                    throw new Error('.pikelet row map segment failed hash verification');
+                }
+                return bytes;
+            })() : null,
         ]);
 
         const lexicalIndex = lexicalBytes
@@ -1057,7 +1069,18 @@ export async function openPikeletFile(input, options = {}) {
 
         // Index: the embedded sketch artifact, opened in the wave above
         // against a segment-windowed source (resident hash verified there).
-        if (sketch.count !== declaredRecords) {
+        // The index binds to the corpus positionally (3.4) unless a row map
+        // (3.9) says how its rows group into records; the manifest names the
+        // policy that produced the rows, so a row map can neither be dropped
+        // nor added without changing the identity.
+        const rowMapPolicy = manifest.index?.rowMap ?? null;
+        if (!!rowMapPolicy !== !!rowMapBytes) {
+            throw new Error(rowMapPolicy
+                ? '.pikelet manifest declares index.rowMap but carries no row map segment'
+                : '.pikelet carries a row map segment its manifest does not declare (index.rowMap)');
+        }
+        const rowMap = rowMapBytes ? parseRowMapSegment(rowMapBytes, declaredRecords, sketch.count) : null;
+        if (!rowMap && sketch.count !== declaredRecords) {
             throw new Error(`.pikelet index count ${sketch.count} != corpus records ${declaredRecords}`);
         }
         if (sketch.dim !== undefined && sketch.dim !== dim) {
@@ -1243,6 +1266,180 @@ export async function openPikeletFile(input, options = {}) {
             if (closed) throw new Error('.pikelet reader is closed');
         };
 
+        // The rerank's default coalescing gap. The identity-verified manifest
+        // may carry a fetch hint: cluster-ordered layouts only coalesce at a
+        // gap at least matched to their cluster geometry (the wiki pack's
+        // k-means layout needs ~16 KiB; a 2 KiB gap squanders it — measured
+        // 440 vs 200 requests/query at C=600). The hint is a floor: a wider
+        // gap merges a superset of the same runs, so a source that prefers
+        // more (HTTP's 256 KiB) keeps its own. undefined leaves the sketch
+        // reader on the source's preference. A chain reads it per tier
+        // through __chainTier.
+        const rerankGapHint = Number.isFinite(manifest.recommendedGap)
+            ? Math.max(manifest.recommendedGap, Number(source.preferredGapBytes) || 0)
+            : undefined;
+
+        // The one retrieval path behind query(): resident sketch scan plus
+        // BM25 candidates, exact rerank, rank fusion and phrase pinning.
+        // Compile-time calibration fits on exactly this (through
+        // __calibrationEvidence below), so the evidence a verdict was fit on
+        // is the evidence query() scores. excludeIds removes records from
+        // both candidate sources before either is cut to size — calibration
+        // ablations use it; query() never does.
+        const retrieveCandidates = async (trimmed, context, k, queryOptions, excludeIds = null) => {
+            let fusedFull = null;
+            // Hybrid retrieval when the artifact carries a lexical
+            // segment: the BM25 top matches join the sketch's exact
+            // rerank as extra candidates (scored by true vector
+            // distance like any candidate — this is what recovers
+            // known-item lookups the sketch scan's top-C missed),
+            // and the final ordering fuses the distance ranking
+            // with the BM25 ranking by reciprocal rank (RRF, k=60,
+            // untuned). Abstention still scores the distance-sorted
+            // top-k exactly as calibrated; fusion affects result
+            // order only. fullRerankOutput returns every reranked
+            // candidate (all already fetched and exactly scored) so
+            // fusion sees the true distance of every lexical match.
+            // Common query terms give every matching document a
+            // small, near-tied BM25 score (idf collapses them);
+            // that flat mass carries no ranking information and
+            // dilutes rank fusion. Keep only hits within a factor
+            // of the best lexical score — for a known-item lookup
+            // that is typically the one document holding the rare
+            // terms.
+            // retrieval: 'hybrid' (default when the artifact
+            // carries a lexical segment) | 'vector' | 'lexical' —
+            // the last two exist for measurement and debugging
+            // (retrieval-quality bakeoffs run all three on one
+            // artifact). Abstention scores the distance-sorted
+            // top-k identically in every mode.
+            // 'augmented' unions the lexical candidates into the
+            // exact rerank but keeps pure distance order — the mode
+            // for workloads scored against exact nearest neighbors,
+            // where RRF's reordering reads as loss by definition.
+            const retrieval = queryOptions.retrieval ?? 'hybrid';
+            if (!['hybrid', 'vector', 'lexical', 'augmented'].includes(retrieval)) {
+                throw new Error(`query() retrieval must be hybrid, vector, lexical, or augmented, got ${retrieval}`);
+            }
+            // An excluded record must not take a candidate slot on either
+            // side, so both sources skip it before cutting to size — BM25
+            // before its top-n sort (scored with the full corpus's term
+            // statistics, the one way this differs from a corpus without
+            // those records) and the sketch before its top-C cut.
+            const excluded = excludeIds && excludeIds.size ? excludeIds : null;
+            let excludeBits = null;
+            if (excluded) {
+                excludeBits = new Uint8Array(Math.ceil(sketch.count / 8));
+                for (const id of excluded) {
+                    // With a row map, excluding a record masks every row it owns.
+                    const [from, to] = rowMap ? [rowMap.offsets[id], rowMap.offsets[id + 1]] : [id, id + 1];
+                    for (let row = from; row < to; row++) excludeBits[row >> 3] |= 1 << (row & 7);
+                }
+            }
+            const lexicalRaw = lexicalIndex && retrieval !== 'vector'
+                ? await lexicalIndex.search(trimmed, LEXICAL_CANDIDATES, excluded ? { exclude: (id) => excluded.has(id) } : {})
+                : [];
+            const lexicalHits = lexicalRaw.filter((h) => h.score >= lexicalRaw[0].score / LEXICAL_CUTOFF);
+            // With a row map (3.9) the sketch searches rows: a record's lexical
+            // match joins the rerank with every row it owns, k is scaled to the
+            // mean rows per record, and the reranked rows collapse to records
+            // below, each scored by its best row.
+            const extraRows = rowMap
+                ? lexicalHits.flatMap((h) => Array.from({ length: rowMap.offsets[h.id + 1] - rowMap.offsets[h.id] }, (_, i) => rowMap.offsets[h.id] + i))
+                : lexicalHits.map((h) => h.id);
+            const searchK = rowMap ? Math.min(sketch.count, Math.ceil(k * sketch.count / recordCount)) : k;
+            const rerankedRows = (await sketch.search(context.vector, searchK, {
+                rerank: queryOptions.rerank,
+                parallelism: queryOptions.parallelism ?? queryOptions.rerankParallelism ?? options.rerankParallelism,
+                gap: queryOptions.gap ?? queryOptions.rerankGap ?? options.rerankGap ?? rerankGapHint,
+                maxRangeBytes: queryOptions.maxRangeBytes ?? options.rerankMaxRangeBytes,
+                // Present once background staging finished; the
+                // sketch reader ignores it for any query whose C
+                // exceeds the scanner's buffers.
+                ...(scanner ? { scanner } : {}),
+                ...(extraRows.length ? { extraCandidates: extraRows } : {}),
+                ...(excludeBits ? { exclude: excludeBits } : {}),
+                // Always return the full reranked candidate list
+                // (already fetched and exactly scored) rather than
+                // truncating to k: abstention is fit at a fixed
+                // top-10 window (calibrate.mjs) independent of the
+                // caller's k, and needs that full window to score
+                // consistently — see retrieval-abstention.mjs.
+                fullRerankOutput: true,
+            })).results;
+            // The rerank output is distance-ordered, so a record's first row
+            // is its best: keeping first occurrences gives the records in
+            // order of their best row's distance.
+            let searched = rerankedRows;
+            if (rowMap) {
+                const seen = new Set();
+                searched = [];
+                for (const row of rerankedRows) {
+                    const id = rowMap.owner[row.id];
+                    if (!seen.has(id)) { seen.add(id); searched.push({ id, distance: row.distance }); }
+                }
+            }
+            const hits = searched.slice(0, k);
+            // Full-window fused order (not k-truncated) for scoring:
+            // coverage grounds its verdict in the passage fusion
+            // actually ranks first, and calibrate.mjs fits it the
+            // same way — a k-truncated fused list would silently
+            // drop the fused top hit for callers requesting a small k.
+            if (retrieval === 'lexical') {
+                const byId = new Map(searched.map((hit) => [hit.id, hit]));
+                fusedFull = lexicalHits.map((h) => byId.get(h.id)).filter(Boolean);
+            } else if (retrieval !== 'augmented' && lexicalHits.length) {
+                const lexRank = new Map(lexicalHits.map((h, i) => [h.id, i]));
+                fusedFull = fuseCandidates(searched, lexicalHits.map((h) => h.id));
+                // One batched hydration for everything this block and
+                // scoreQuality's coverage term (below) will need, in
+                // place of two separate staged fetches (phrase-pin's
+                // own, then a later cold start for coverage): over a
+                // remote range-read source each hydrate() a caller
+                // hasn't already paid for is its own network round
+                // trip, and sequential/staged awaits pay that cost
+                // once per stage instead of once per query. The
+                // union of phrase-pin's candidates (top lexical
+                // hits) and coverage's (fusedFull's top few — phrase
+                // pinning only ever reorders that window, never
+                // changes its membership) is known right here, so
+                // both stages can share one round trip. A record
+                // that fails to hydrate here does not fail the
+                // query: the id is just missing from the map, and
+                // each site below falls through exactly as it would
+                // on a cold cache miss (recs[i] undefined; coverage
+                // sees fewer usable passages) — hydrate() raising
+                // for a bad id is still a real failure, but only
+                // once a site actually needs that specific record,
+                // not speculatively for every candidate touched here.
+                const phraseCandidates = lexicalHits.slice(0, 3);
+                const coverageCandidates = fusedFull.slice(0, COVERAGE_BATCH_COUNT);
+                const batchIds = [...new Set([...phraseCandidates, ...coverageCandidates].map((h) => h.id))];
+                const batch = new Map();
+                await Promise.all(batchIds.map(async (id) => {
+                    try { batch.set(id, await hydrate(id)); } catch { /* see comment above */ }
+                }));
+                // Phrase pinning: a verbatim occurrence of the query (>= 3
+                // content words) in one of the top lexical hits is the
+                // strongest evidence of support there is; rank it first.
+                const norm = (t) => String(t || '').toLowerCase().replace(/[^a-z0-9' ]+/g, ' ').replace(/\s+/g, ' ').trim();
+                const phrase = norm(trimmed);
+                if (phrase.split(' ').length >= 3) {
+                    for (const h of phraseCandidates) {
+                        if (norm(batch.get(h.id)?.text).includes(phrase)) {
+                            fusedFull = [searched.find((x) => x.id === h.id), ...fusedFull.filter((x) => x.id !== h.id)].filter(Boolean);
+                            break;
+                        }
+                    }
+                }
+                // Rank transparency: a consumer can see a verbatim hit sitting
+                // at fused rank 3 behind two semantic near-misses.
+                for (const [i, hit] of fusedFull.entries()) { hit.fusedRank = i + 1; hit.lexicalRank = lexRank.has(hit.id) ? lexRank.get(hit.id) + 1 : null; hit.vectorRank = searched.findIndex((x) => x.id === hit.id) + 1; }
+            }
+            const fused = retrieval === 'augmented' ? null : fusedFull?.slice(0, k) ?? null;
+            return { hits, fused, searched, fusedFull };
+        };
+
         return {
             info() {
                 return {
@@ -1270,6 +1467,11 @@ export async function openPikeletFile(input, options = {}) {
                     license: typeof manifest.corpus?.provenance?.license === 'string'
                         ? manifest.corpus.provenance.license : null,
                     records: recordCount,
+                    // Index rows and the policy that produced them when the
+                    // index holds several vectors per record (row map, 3.9);
+                    // otherwise one row per record and rowMap is null.
+                    indexRows: sketch.count,
+                    rowMap: rowMapPolicy ? { ...rowMapPolicy } : null,
                     dim,
                     metric: manifest.metric,
                     encoder: encoderInfo,
@@ -1303,6 +1505,16 @@ export async function openPikeletFile(input, options = {}) {
                         }
                         : null,
                     sampleQueries: Array.isArray(manifest.sampleQueries) ? manifest.sampleQueries : [],
+                    // Where each part of the file lives, in file order: the
+                    // header (fixed header, manifest and segment table, up to
+                    // the first segment's aligned offset) and every segment of
+                    // a known kind. Lets a viewer attribute range reads to
+                    // what they fetched.
+                    layout: [
+                        { kind: 'header', offset: 0, length: align16(HEADER_BYTES + manifestBytes + tableBytes) },
+                        ...[...segments].map(([kind, seg]) => ({ kind, offset: seg.offset, length: seg.length }))
+                            .sort((a, b) => a.offset - b.offset),
+                    ],
                 };
             },
 
@@ -1328,127 +1540,7 @@ export async function openPikeletFile(input, options = {}) {
                 let fused = null;
                 let searched = null;
                 let fusedFull = null;
-                if (!pre) {
-                    // Hybrid retrieval when the artifact carries a lexical
-                    // segment: the BM25 top matches join the sketch's exact
-                    // rerank as extra candidates (scored by true vector
-                    // distance like any candidate — this is what recovers
-                    // known-item lookups the sketch scan's top-C missed),
-                    // and the final ordering fuses the distance ranking
-                    // with the BM25 ranking by reciprocal rank (RRF, k=60,
-                    // untuned). Abstention still scores the distance-sorted
-                    // top-k exactly as calibrated; fusion affects result
-                    // order only. fullRerankOutput returns every reranked
-                    // candidate (all already fetched and exactly scored) so
-                    // fusion sees the true distance of every lexical match.
-                    // Common query terms give every matching document a
-                    // small, near-tied BM25 score (idf collapses them);
-                    // that flat mass carries no ranking information and
-                    // dilutes rank fusion. Keep only hits within a factor
-                    // of the best lexical score — for a known-item lookup
-                    // that is typically the one document holding the rare
-                    // terms.
-                    // retrieval: 'hybrid' (default when the artifact
-                    // carries a lexical segment) | 'vector' | 'lexical' —
-                    // the last two exist for measurement and debugging
-                    // (retrieval-quality bakeoffs run all three on one
-                    // artifact). Abstention scores the distance-sorted
-                    // top-k identically in every mode.
-                    // 'augmented' unions the lexical candidates into the
-                    // exact rerank but keeps pure distance order — the mode
-                    // for workloads scored against exact nearest neighbors,
-                    // where RRF's reordering reads as loss by definition.
-                    const retrieval = queryOptions.retrieval ?? 'hybrid';
-                    if (!['hybrid', 'vector', 'lexical', 'augmented'].includes(retrieval)) {
-                        throw new Error(`query() retrieval must be hybrid, vector, lexical, or augmented, got ${retrieval}`);
-                    }
-                    const lexicalRaw = lexicalIndex && retrieval !== 'vector'
-                        ? await lexicalIndex.search(trimmed, LEXICAL_CANDIDATES) : [];
-                    const lexicalHits = lexicalRaw.filter((h) => h.score >= lexicalRaw[0].score / LEXICAL_CUTOFF);
-                    searched = (await sketch.search(context.vector, k, {
-                        rerank: queryOptions.rerank,
-                        parallelism: queryOptions.parallelism ?? queryOptions.rerankParallelism ?? options.rerankParallelism,
-                        // The identity-verified manifest may carry a fetch
-                        // hint: cluster-ordered layouts only coalesce at a
-                        // gap matched to their cluster geometry (the wiki
-                        // pack's k-means layout needs ~16 KiB; the default
-                        // 2 KiB squanders it — measured 440 vs 200
-                        // requests/query at C=600).
-                        gap: queryOptions.gap ?? queryOptions.rerankGap ?? options.rerankGap
-                            ?? (Number.isFinite(manifest.recommendedGap) ? manifest.recommendedGap : undefined),
-                        maxRangeBytes: queryOptions.maxRangeBytes ?? options.rerankMaxRangeBytes,
-                        // Present once background staging finished; the
-                        // sketch reader ignores it for any query whose C
-                        // exceeds the scanner's buffers.
-                        ...(scanner ? { scanner } : {}),
-                        ...(lexicalHits.length ? { extraCandidates: lexicalHits.map((h) => h.id) } : {}),
-                        // Always return the full reranked candidate list
-                        // (already fetched and exactly scored) rather than
-                        // truncating to k: abstention is fit at a fixed
-                        // top-10 window (calibrate.mjs) independent of the
-                        // caller's k, and needs that full window to score
-                        // consistently — see retrieval-abstention.mjs.
-                        fullRerankOutput: true,
-                    })).results;
-                    hits = searched.slice(0, k);
-                    // Full-window fused order (not k-truncated) for scoring:
-                    // coverage grounds its verdict in the passage fusion
-                    // actually ranks first, and calibrate.mjs fits it the
-                    // same way — a k-truncated fused list would silently
-                    // drop the fused top hit for callers requesting a small k.
-                    if (retrieval === 'lexical') {
-                        const byId = new Map(searched.map((hit) => [hit.id, hit]));
-                        fusedFull = lexicalHits.map((h) => byId.get(h.id)).filter(Boolean);
-                    } else if (retrieval !== 'augmented' && lexicalHits.length) {
-                        const lexRank = new Map(lexicalHits.map((h, i) => [h.id, i]));
-                        fusedFull = fuseCandidates(searched, lexicalHits.map((h) => h.id));
-                        // One batched hydration for everything this block and
-                        // scoreQuality's coverage term (below) will need, in
-                        // place of two separate staged fetches (phrase-pin's
-                        // own, then a later cold start for coverage): over a
-                        // remote range-read source each hydrate() a caller
-                        // hasn't already paid for is its own network round
-                        // trip, and sequential/staged awaits pay that cost
-                        // once per stage instead of once per query. The
-                        // union of phrase-pin's candidates (top lexical
-                        // hits) and coverage's (fusedFull's top few — phrase
-                        // pinning only ever reorders that window, never
-                        // changes its membership) is known right here, so
-                        // both stages can share one round trip. A record
-                        // that fails to hydrate here does not fail the
-                        // query: the id is just missing from the map, and
-                        // each site below falls through exactly as it would
-                        // on a cold cache miss (recs[i] undefined; coverage
-                        // sees fewer usable passages) — hydrate() raising
-                        // for a bad id is still a real failure, but only
-                        // once a site actually needs that specific record,
-                        // not speculatively for every candidate touched here.
-                        const phraseCandidates = lexicalHits.slice(0, 3);
-                        const coverageCandidates = fusedFull.slice(0, COVERAGE_BATCH_COUNT);
-                        const batchIds = [...new Set([...phraseCandidates, ...coverageCandidates].map((h) => h.id))];
-                        const batch = new Map();
-                        await Promise.all(batchIds.map(async (id) => {
-                            try { batch.set(id, await hydrate(id)); } catch { /* see comment above */ }
-                        }));
-                        // Phrase pinning: a verbatim occurrence of the query (>= 3
-                        // content words) in one of the top lexical hits is the
-                        // strongest evidence of support there is; rank it first.
-                        const norm = (t) => String(t || '').toLowerCase().replace(/[^a-z0-9' ]+/g, ' ').replace(/\s+/g, ' ').trim();
-                        const phrase = norm(trimmed);
-                        if (phrase.split(' ').length >= 3) {
-                            for (const h of phraseCandidates) {
-                                if (norm(batch.get(h.id)?.text).includes(phrase)) {
-                                    fusedFull = [searched.find((x) => x.id === h.id), ...fusedFull.filter((x) => x.id !== h.id)].filter(Boolean);
-                                    break;
-                                }
-                            }
-                        }
-                        // Rank transparency: a consumer can see a verbatim hit sitting
-                        // at fused rank 3 behind two semantic near-misses.
-                        for (const [i, hit] of fusedFull.entries()) { hit.fusedRank = i + 1; hit.lexicalRank = lexRank.has(hit.id) ? lexRank.get(hit.id) + 1 : null; hit.vectorRank = searched.findIndex((x) => x.id === hit.id) + 1; }
-                    }
-                    fused = retrieval === 'augmented' ? null : fusedFull?.slice(0, k) ?? null;
-                }
+                if (!pre) ({ hits, fused, searched, fusedFull } = await retrieveCandidates(trimmed, context, k, queryOptions));
                 // Abstention scores the full reranked candidate window
                 // (searched), not the k-truncated hits: calibrate.mjs fits
                 // its signals at a fixed top-10 (top-5 for coverage)
@@ -1484,12 +1576,42 @@ export async function openPikeletFile(input, options = {}) {
                 return { matchQuality: quality.match_quality, confidence: quality.confidence, results, ...(grounding ? { grounding } : {}) };
             },
 
+            /**
+             * The retrieval evidence query() scores, for compile-time
+             * calibration: `vector` is the distance-ordered rerank window
+             * d0/margin/mean10 read, `fused` the order coverage reads its
+             * passages from. Calibration consumes this instead of
+             * reimplementing retrieval, so a verdict is fit on the evidence
+             * it is later scored on. Not part of the documented reader API.
+             */
+            async __calibrationEvidence(text, { excludeIds = null } = {}) {
+                assertOpen();
+                const trimmed = String(text || '').trim();
+                if (!trimmed) throw new Error('query text is required');
+                // The 10-wide window calibration fits at; any k up to the
+                // pack's rerank depth gives query() the same candidates.
+                const k = Math.min(10, Math.max(recordCount, 1));
+                const { searched, fusedFull } = await retrieveCandidates(trimmed, await embed(trimmed), k, {}, excludeIds);
+                const plain = (list) => list.map((hit) => ({ id: hit.id, distance: hit.distance }));
+                return { vector: plain(searched), fused: plain(fusedFull?.length ? fusedFull : searched) };
+            },
+
             /** Hydrate one corpus record by id (verified per record on format 2). */
             async record(id) {
                 assertOpen();
                 return hydrate(id);
             },
 
+            /** The absolute byte range one corpus record's stored bytes occupy in the file. */
+            recordRange(id) {
+                assertOpen();
+                if (!Number.isSafeInteger(id) || id < 0 || id >= recordCount) {
+                    throw new Error(`.pikelet record id ${id} is outside the corpus`);
+                }
+                const start = recordOffsets[id];
+                return { offset: corpus.offset + start, length: recordOffsets[id + 1] - start };
+            },
+
             /**
              * Full pass over the index's lazy vector rows against the
              * identity-anchored vectorsSha256 (streamed where possible).
@@ -1566,6 +1688,9 @@ export async function openPikeletFile(input, options = {}) {
                     recordCount,
                     dim,
                     sketch,
+                    // A chain maps sketch rows to records positionally, so
+                    // it needs to know when a member's rows are not records.
+                    rowMap,
                     lexicalIndex,
                     hydrate,
                     readRecordBytes,
@@ -1588,6 +1713,10 @@ export async function openPikeletFile(input, options = {}) {
                     // the kernel. A getter, not a captured value, because
                     // staging completes after open() returns.
                     get scanner() { return scanner; },
+                    // The rerank's default coalescing gap for this member
+                    // (manifest hint as a floor under the source's own), so
+                    // a chain fetches each tier's rows as this member would.
+                    rerankGap: rerankGapHint,
                     // The base's OWN encoder, for `append` (6.1): "a kind-3
                     // base carries the teacher, and append loads it from the
                     // base's own query-interp segment — no model download, no

package/complete/index.node.mjs

@@ -0,0 +1,15 @@
+// The Node entry of 'pikelet-wasm/complete': the portable reader, except
+// that httpRangeSource defaults to nodeFetch (node:http/node:https, no
+// automatic decompression, Content-Encoding other than identity refused)
+// instead of the runtime's fetch, which decodes whatever a server labels as
+// encoded. Pass options.fetch to choose otherwise.
+
+import { httpRangeSource as portableHttpRangeSource } from './sources.mjs';
+import { nodeFetch } from './node-fetch.mjs';
+
+export * from './index.mjs';
+export { nodeFetch };
+
+export function httpRangeSource(url, options = {}) {
+    return portableHttpRangeSource(url, { fetch: nodeFetch, ...options });
+}