pikelet-wasm 0.10.0 → 0.10.1

package/CHANGELOG.md

@@ -8,6 +8,45 @@ Through 0.6.0 these were published as `pancake-wasm` and
 packages are pre-1.0: a minor bump may carry breaking changes, and each entry lists them
 first.
 
+## pikelet-wasm 0.10.1 / pikelet 0.10.1 — 2026-10-11
+
+Layered packs now serve the same grounding and fetch tuning as single
+files, HTTP readers make fewer, larger range requests, and lazily read
+keyword indexes cache what they read. Upgrade if you serve layered packs
+over `pikelet mcp`: under 0.10.0 they returned no `grounding`, the fact
+check a relevance verdict leaves to the reader. Both packages move to
+0.10.1 together, and `pikelet` requires `pikelet-wasm` ^0.10.1.
+
+### Compatibility
+
+- **HTTP range sources coalesce rerank rows within 256 KiB by default**
+  (was 16 KiB), and a pack's `recommendedGap` is a floor under the
+  source's own gap rather than a replacement for it. On the 456,153-record
+  Wikipedia pack a query makes about 38 range requests instead of 61 and
+  transfers about 2.3 MiB instead of 0.9 MiB; results are unchanged. Pass
+  `preferredGapBytes` to `httpRangeSource`, or `gap` to `query()`, to
+  choose otherwise.
+
+### Fixed
+
+- **A layered pack returns `grounding`.** The chain reader kept only the
+  inherited scorer's verdict and confidence, so `grounding.covered` and
+  `grounding.uncovered` were missing from chain queries and from MCP
+  `search` sections as soon as a pack gained a layer. `recordId` is the
+  chain-global id of the passage the coverage was taken from.
+- **A layered pack honors fetch tuning.** `gap`, `parallelism` and
+  `maxRangeBytes` (and the mount-level `rerankGap`, `rerankParallelism`
+  and `rerankMaxRangeBytes`) did not reach the tiers, and a tier's
+  manifest `recommendedGap` never applied. Each tier now fetches as its
+  own file would.
+- **Lazily read keyword indexes cache postings and share reads.** A
+  repeated query re-read every postings list, and query terms whose
+  entries share an uncached term-table window each read it. Postings are
+  now cached (4 MiB, by bytes), pending reads are shared, and a failed
+  read is retried by the next query rather than cached.
+- **`CompleteQueryResult` declares `grounding`** in the TypeScript
+  types; `query()` already returned it.
+
 ## pikelet-wasm 0.10.0 / pikelet 0.10.0 — 2026-10-09
 
 `matchQuality` changes meaning for packs compiled with this release: it

package/complete/index.d.ts

@@ -41,6 +41,11 @@ export interface HttpRangeSource extends CompleteRangeSource {
 
 export declare function httpRangeSource(url: string, options?: {
   preferredParallelism?: number;
+  /**
+   * Rows of the vector rerank closer than this many bytes are fetched in
+   * one request, reading the bytes between them. Default 256 KiB. A pack's
+   * manifest recommendedGap applies only when it is wider.
+   */
   preferredGapBytes?: number;
   /** Cap on the one-time full download when a host ignores Range, enforced on received bytes. Default 64 MiB. */
   maxFullFallbackBytes?: number;
@@ -84,6 +89,22 @@ export interface CompleteQueryResult {
    * other way round.
    */
   results: Array<{ id: number; distance: number } & Record<string, unknown>>;
+  /**
+   * Which passage grounded the verdict and which of the query's content
+   * words it contains. Present when a fitted calibrator scored the query
+   * (absent on 'unscored'). An asked-for value in `uncovered` is one no top
+   * passage states, whatever matchQuality says.
+   */
+  grounding?: {
+    /** Id of the passage the coverage score was taken from. */
+    recordId: number | null;
+    /** Index of that passage in the fused (or distance-ordered) window. */
+    passageIndex: number;
+    /** Weighted share of the query's content words the passage contains, 0-1. */
+    coverage: number;
+    covered: string[];
+    uncovered: string[];
+  };
 }
 
 export interface CompleteSearch {

package/complete/index.mjs

@@ -1243,6 +1243,19 @@ export async function openPikeletFile(input, options = {}) {
             if (closed) throw new Error('.pikelet reader is closed');
         };
 
+        // The rerank's default coalescing gap. The identity-verified manifest
+        // may carry a fetch hint: cluster-ordered layouts only coalesce at a
+        // gap at least matched to their cluster geometry (the wiki pack's
+        // k-means layout needs ~16 KiB; a 2 KiB gap squanders it — measured
+        // 440 vs 200 requests/query at C=600). The hint is a floor: a wider
+        // gap merges a superset of the same runs, so a source that prefers
+        // more (HTTP's 256 KiB) keeps its own. undefined leaves the sketch
+        // reader on the source's preference. A chain reads it per tier
+        // through __chainTier.
+        const rerankGapHint = Number.isFinite(manifest.recommendedGap)
+            ? Math.max(manifest.recommendedGap, Number(source.preferredGapBytes) || 0)
+            : undefined;
+
         return {
             info() {
                 return {
@@ -1368,14 +1381,7 @@ export async function openPikeletFile(input, options = {}) {
                     searched = (await sketch.search(context.vector, k, {
                         rerank: queryOptions.rerank,
                         parallelism: queryOptions.parallelism ?? queryOptions.rerankParallelism ?? options.rerankParallelism,
-                        // The identity-verified manifest may carry a fetch
-                        // hint: cluster-ordered layouts only coalesce at a
-                        // gap matched to their cluster geometry (the wiki
-                        // pack's k-means layout needs ~16 KiB; the default
-                        // 2 KiB squanders it — measured 440 vs 200
-                        // requests/query at C=600).
-                        gap: queryOptions.gap ?? queryOptions.rerankGap ?? options.rerankGap
-                            ?? (Number.isFinite(manifest.recommendedGap) ? manifest.recommendedGap : undefined),
+                        gap: queryOptions.gap ?? queryOptions.rerankGap ?? options.rerankGap ?? rerankGapHint,
                         maxRangeBytes: queryOptions.maxRangeBytes ?? options.rerankMaxRangeBytes,
                         // Present once background staging finished; the
                         // sketch reader ignores it for any query whose C
@@ -1588,6 +1594,10 @@ export async function openPikeletFile(input, options = {}) {
                     // the kernel. A getter, not a captured value, because
                     // staging completes after open() returns.
                     get scanner() { return scanner; },
+                    // The rerank's default coalescing gap for this member
+                    // (manifest hint as a floor under the source's own), so
+                    // a chain fetches each tier's rows as this member would.
+                    rerankGap: rerankGapHint,
                     // The base's OWN encoder, for `append` (6.1): "a kind-3
                     // base carries the teacher, and append loads it from the
                     // base's own query-interp segment — no model download, no

package/complete/layer-reader.mjs

@@ -465,6 +465,13 @@ export async function openPikeletChain(members, options = {}) {
                     // tier has no live deletion (maskCounts, computed at mount).
                     const out = await t.reader.sketch.search(vector, Math.min(k, t.records), {
                         rerank: queryOptions.rerank,
+                        // The single-file reader's fetch tuning, with the same
+                        // precedence: the query's own value, then the mount's,
+                        // then this tier's default gap (its manifest hint as a
+                        // floor under its source's preference).
+                        parallelism: queryOptions.parallelism ?? queryOptions.rerankParallelism ?? options.rerankParallelism,
+                        gap: queryOptions.gap ?? queryOptions.rerankGap ?? options.rerankGap ?? t.reader.rerankGap,
+                        maxRangeBytes: queryOptions.maxRangeBytes ?? options.rerankMaxRangeBytes,
                         ...(maskCounts[j] > 0 ? { exclude: masks[j] } : {}),
                         // The tier's own staged scan kernel. Read per query, not
                         // captured at mount: staging finishes in the background
@@ -528,6 +535,7 @@ export async function openPikeletChain(members, options = {}) {
                 // to prevent.
                 let matchQuality = 'unscored';
                 let confidence;
+                let grounding = null;
                 if (calibration.status === 'inherited' && base.scoreQuality) {
                     // Coverage reads the fit's passagesNeeded from the FULL
                     // fused window; top.slice(0, 3) gave it one passage at k=1.
@@ -553,6 +561,13 @@ export async function openPikeletChain(members, options = {}) {
                     if (scored && scored.match_quality) {
                         matchQuality = scored.match_quality;
                         if (Number.isFinite(scored.confidence)) confidence = scored.confidence;
+                        // Which passage grounded the verdict and which query
+                        // words no passage contains, as the single-file
+                        // reader returns it; recordId is the chain-global id
+                        // of the passage coverage scored.
+                        if (scored.grounding) {
+                            grounding = { ...scored.grounding, recordId: passageOrder[scored.grounding.passageIndex]?.id ?? null };
+                        }
                     }
                 } else if (calibration.status === 'none') {
                     matchQuality = 'unscored';
@@ -567,6 +582,7 @@ export async function openPikeletChain(members, options = {}) {
                     matchQuality,
                     ...(confidence === undefined ? {} : { confidence }),
                     results: withheld ? [] : results,
+                    ...(grounding ? { grounding } : {}),
                     identity: chainMembers[chainMembers.length - 1].identity,
                     calibration: {
                         status: calibration.status,

package/complete/lexical.mjs

@@ -80,15 +80,58 @@ export async function openLexicalIndexLazy(read, segLength) {
     const doclenOf = (id) => doclenView.getUint32(4 * id, true);
 
     const WINDOW = 512;
-    const windowCache = new Map(); // startIdx -> entry bytes (small LRU)
-    async function readWindow(startIdx, count) {
+    // Both caches hold the pending read, not its result, so concurrent
+    // lookups of one window (two query terms whose hashes land in it) or one
+    // postings list share a single read; a failed read is evicted so the
+    // next query retries rather than re-throwing a cached rejection.
+    const windowCache = new Map(); // `${startIdx}:${count}` -> Promise<entry bytes> (small LRU)
+    function readWindow(startIdx, count) {
         const key = `${startIdx}:${count}`;
         const hit = windowCache.get(key);
-        if (hit) return hit;
-        const bytes = await read(termTableOffset + 24 * startIdx, 24 * count);
+        if (hit) {
+            windowCache.delete(key);
+            windowCache.set(key, hit);
+            return hit;
+        }
+        const pending = read(termTableOffset + 24 * startIdx, 24 * count);
         if (windowCache.size >= 64) windowCache.delete(windowCache.keys().next().value);
-        windowCache.set(key, bytes);
-        return bytes;
+        windowCache.set(key, pending);
+        pending.catch(() => { if (windowCache.get(key) === pending) windowCache.delete(key); });
+        return pending;
+    }
+    // Postings, LRU by bytes: a repeated or overlapping query re-reads only
+    // the terms it has not seen. A list larger than POSTINGS_CACHE_MAX_ENTRY
+    // (a very common term) is read every time rather than evicting
+    // everything else.
+    const POSTINGS_CACHE_BYTES = 4 * 1024 * 1024;
+    const POSTINGS_CACHE_MAX_ENTRY = 1024 * 1024;
+    const postingsCache = new Map(); // postingsRel -> { pending, bytes }
+    let postingsCachedBytes = 0;
+    function readPostings(rel, len) {
+        const hit = postingsCache.get(rel);
+        if (hit && hit.bytes === len) {
+            postingsCache.delete(rel);
+            postingsCache.set(rel, hit);
+            return hit.pending;
+        }
+        const pending = read(postingsOffset + rel, len);
+        if (len > POSTINGS_CACHE_MAX_ENTRY) return pending;
+        const entry = { pending, bytes: len };
+        if (hit) postingsCachedBytes -= hit.bytes;
+        postingsCache.set(rel, entry);
+        postingsCachedBytes += len;
+        while (postingsCachedBytes > POSTINGS_CACHE_BYTES && postingsCache.size > 1) {
+            const [oldRel, old] = postingsCache.entries().next().value;
+            postingsCache.delete(oldRel);
+            postingsCachedBytes -= old.bytes;
+        }
+        pending.catch(() => {
+            if (postingsCache.get(rel) === entry) {
+                postingsCache.delete(rel);
+                postingsCachedBytes -= len;
+            }
+        });
+        return pending;
     }
     const entryAt = (bytes, i) => {
         const view = new DataView(bytes.buffer, bytes.byteOffset + 24 * i, 24);
@@ -195,7 +238,7 @@ export async function openLexicalIndexLazy(read, segLength) {
                     throw new Error('.pikelet lexical postings out of bounds');
                 }
             }
-            const postings = await Promise.all(found.map((e) => read(postingsOffset + e.postingsRel, e.postingsLen)));
+            const postings = await Promise.all(found.map((e) => readPostings(e.postingsRel, e.postingsLen)));
             const scores = new Map();
             found.forEach((e, i) => {
                 const bytes = postings[i];

package/complete/sources.mjs

@@ -102,7 +102,12 @@ export function httpRangeSource(url, options = {}) {
         url,
         size: undefined,
         preferredParallelism: options.preferredParallelism ?? 32,
-        preferredGapBytes: options.preferredGapBytes ?? 16 * 1024,
+        // Rows closer than this are fetched in one request, reading the
+        // bytes between them. Over HTTP a round trip costs far more than
+        // the bytes: on the 648.5 MB wiki pack in Chromium, 256 KiB took
+        // 61 -> 38 requests and about 2.8 -> 2.2 s per query against 16 KiB,
+        // at 0.9 -> 2.3 MiB transferred.
+        preferredGapBytes: options.preferredGapBytes ?? 256 * 1024,
 
         async init() {
             await resolveTarget();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pikelet-wasm",
-  "version": "0.10.0",
+  "version": "0.10.1",
   "description": "Vector search engine and .pikelet artifact readers/builders, in WebAssembly. Compile a corpus into one self-contained, hash-verified search file, or use the underlying HNSW engine directly. Runs in Node.js, browsers, and Cloudflare Workers with no native dependencies.",
   "main": "src/pikelet.js",
   "types": "src/pikelet.d.ts",