pikelet-wasm 0.9.2 → 0.9.3

package/CHANGELOG.md

@@ -8,6 +8,67 @@ Through 0.6.0 these were published as `pancake-wasm` and
 packages are pre-1.0: a minor bump may carry breaking changes, and each entry lists them
 first.
 
+## pikelet-wasm 0.9.3 / pikelet 0.9.3 — 2026-10-06
+
+Hardening for readers that mount packs from other publishers or hosts:
+the index header commitment now covers the bytes actually parsed, and a
+pack can no longer steer lazy reads, demand an exhaustive rerank, or turn
+a malformed abstention asset into confident answers. Upgrade if you mount
+packs you did not build, including over `pikelet mcp`. Both packages move
+to 0.9.3 together, and `pikelet` requires `pikelet-wasm` ^0.9.3 so that
+`pikelet mcp` serves through the hardened reader.
+
+### Compatibility
+
+- **A pack that declares a rerank depth over 4096 is served at 4096 by
+  default**, unless the caller passes `rerank`. No published pack is
+  affected.
+- **A pack whose abstention asset has malformed `thresholds` or
+  `coverage.topK` now reports `unscored`** instead of answering. Packs
+  built by `compile` are not affected.
+- **`compile` declares at most 4096** for a pack over 4096 records whose
+  sweep does not meet its recall target, and warns. Other packs compile
+  as under 0.9.2.
+
+### Fixed
+
+- **The index header commitment binds the bytes the reader parses.** The
+  complete reader checked `manifest.index.headerSha256` on its own read of
+  the sketch header, while the sketch parsed a separate read of the same
+  range. A host answering the two reads differently could pass the check
+  with the genuine header and have a forged one parsed, together with the
+  integrity anchors it carries, replacing the index under a pinned
+  identity. `PikeletSketchArtifact.open()` now takes
+  `expectedHeaderSha256` and checks the header it parses.
+- **Lazily read keyword segments cannot steer reads or inflate them.**
+  Lexical segments over 8 MiB are read on demand without per-read
+  verification. Reads are now confined to the segment's window, and each
+  term entry is checked before its postings are fetched: inside the
+  postings region, `df` no larger than the corpus, and a length a postings
+  list of that `df` can occupy. `info().lexical.integrity` reports
+  `'unverified-lazy-reads'` for such segments and `'segment-sha256'` for
+  eagerly verified ones.
+- **A pack's declared rerank depth is capped at 4096 by default.** The
+  sketch header's `recommendedRerank` is a producer hint; a value near the
+  row count made every query fetch, hash and rerank the whole vectors
+  region. Readers default to at most 4096, keep the declared value as
+  `declaredRecommendedRerank`, and still honor an explicit `rerank`. The
+  published wiki, astro-docs and rust-book packs declare 200, 40 and 30.
+- **`compile` and `append` no longer declare a rerank depth readers will not
+  use.** `compile`'s rerank sweep fell back to the full row count when no
+  depth up to 640 met its recall target; it now stops at the same 4096
+  ceiling, records the recall measured there, and logs a warning when the
+  target was not met. `append` caps a layer's exhaustive hint the same way.
+  `measureRecommendedRerank` returns `targetReached` and `rerankCeiling`,
+  and `pikelet-wasm/artifact` exports `MAX_RECOMMENDED_RERANK`.
+- **A malformed abstention asset serves `unscored`.** Missing or
+  non-numeric `thresholds` compared false against every score, so the pack
+  answered every query as if calibrated; out-of-order or out-of-range ones
+  gave verdicts the fit never defined. A `coverage.topK` outside 1–10 is
+  refused too, since each counted passage is a record read on every query.
+  The scorer now rejects such an asset and the pack reports `unscored`.
+  Assets written by `compile` are unaffected.
+
 ## pikelet-wasm 0.9.2 / pikelet 0.9.2 — 2026-10-01
 
 Makes the layered update path work as documented: `append` on a freshly

package/complete/builder.d.ts

@@ -71,6 +71,10 @@ export declare function measureRecommendedRerank(input: {
 }): Promise<{
   recommendedRerank: number;
   recall: number;
+  /** False when no rung up to rerankCeiling reached targetRecall; recommendedRerank is then the ceiling. */
+  targetReached: boolean;
+  /** min(row count, MAX_RECOMMENDED_RERANK): the deepest rerank the sweep measures. */
+  rerankCeiling: number;
   k: number;
   targetRecall: number;
   queries: number;

package/complete/builder.mjs

@@ -351,6 +351,12 @@ export function buildInlineTransformerEncoderSegment({ declaration, vocabBytes,
 // targetRecall — plus the whole curve for the evaluation segment. Queries
 // are the corpus's own float embeddings when the caller has them, else rows
 // dequantized out of the snapshot.
+//
+// The sweep stops at the reader's default ceiling (MAX_RECOMMENDED_RERANK):
+// it used to fall back to the full row count, a depth readers no longer
+// honor by default, so a pack could record a recall it would not be served
+// at. The last rung is the ceiling, measured like any other, and
+// targetReached says whether the target was met below it.
 export async function measureRecommendedRerank({
   artifactModule,
   sketchBytes,
@@ -387,9 +393,10 @@ export async function measureRecommendedRerank({
     for (const q of queries) {
       truth.push(new Set((await sketch.search(q, topK, { rerank: count })).results.map((r) => r.id)));
     }
+    const rerankCeiling = Math.min(count, artifactModule.MAX_RECOMMENDED_RERANK ?? count);
     const ladder = (sweep || [10, 15, 20, 30, 40, 60, 80, 120, 160, 240, 320, 480, 640])
-      .filter((c) => c >= topK && c < count);
-    ladder.push(count);
+      .filter((c) => c >= topK && c < rerankCeiling);
+    ladder.push(rerankCeiling);
     const curve = [];
     let recommendedRerank = null;
     let recommendedRecall = null;
@@ -409,13 +416,16 @@ export async function measureRecommendedRerank({
         recommendedRecall = recall;
       }
     }
-    if (recommendedRerank === null) {
-      recommendedRerank = count;
+    const targetReached = recommendedRerank !== null;
+    if (!targetReached) {
+      recommendedRerank = rerankCeiling;
       recommendedRecall = curve[curve.length - 1].recall;
     }
     return {
       recommendedRerank,
       recall: recommendedRecall,
+      targetReached,
+      rerankCeiling,
       k: topK,
       targetRecall,
       queries: queries.length,

package/complete/index.d.ts

@@ -134,7 +134,13 @@ export interface CompleteSearch {
      * order fuses distance and BM25 rankings by reciprocal rank). Null on
      * vector-only artifacts.
      */
-    lexical: { terms: number; docCount: number; lazy?: true } | null;
+    lexical: {
+        terms: number;
+        docCount: number;
+        /** 'segment-sha256' when read eagerly and verified whole; 'unverified-lazy-reads' for lazily opened segments. */
+        integrity: 'segment-sha256' | 'unverified-lazy-reads';
+        lazy?: true;
+    } | null;
     /**
      * 'engine' once a WASM scan kernel serves the resident scan
      * (auto-staged or injected via options.sketchScanner); 'js' before

package/complete/index.mjs

@@ -780,7 +780,12 @@ export async function openPikeletFile(input, options = {}) {
                 }
                 return bytes;
             })(),
-            PikeletSketchArtifact.open(windowSource(source, idx.offset, idx.length), { maxReadBytes }),
+            // Format 2's header commitment is checked inside open(), against
+            // the same bytes the sketch parses (see expectedHeaderSha256).
+            PikeletSketchArtifact.open(windowSource(source, idx.offset, idx.length), {
+                maxReadBytes,
+                ...(perRecord ? { expectedHeaderSha256: manifest.index.headerSha256 } : {}),
+            }),
             readChecked(source, corpus.offset, tablesBytes, 'corpus tables', maxReadBytes, fileBytes),
             // Lexical index (kind 5, OPTIONAL). Small segments: eager
             // whole-segment read, digest-verified like the query-interp
@@ -791,7 +796,15 @@ export async function openPikeletFile(input, options = {}) {
             segments.has('lexical') ? (async () => {
                 const seg = segments.get('lexical');
                 if (seg.length > LAZY_LEXICAL_BYTES) {
-                    const lexRead = (off, len) => readChecked(source, seg.offset + off, len, 'lexical segment', maxReadBytes, fileBytes);
+                    // Confined to the segment's own window: lazy reads are
+                    // not individually verified, so no field they carry may
+                    // steer a read into another segment.
+                    const lexRead = (off, len) => {
+                        if (!Number.isSafeInteger(off) || !Number.isSafeInteger(len) || off < 0 || len < 0 || off + len > seg.length) {
+                            throw new Error('.pikelet lexical segment read is outside the segment window');
+                        }
+                        return readChecked(source, seg.offset + off, len, 'lexical segment', maxReadBytes, fileBytes);
+                    };
                     return { lazyIndex: await openLexicalIndexLazy(lexRead, seg.length) };
                 }
                 const bytes = await readChecked(source, seg.offset, seg.length, 'lexical segment', maxReadBytes, fileBytes);
@@ -800,12 +813,6 @@ export async function openPikeletFile(input, options = {}) {
                 }
                 return { bytes };
             })() : null,
-            perRecord ? (async () => {
-                const headerBytes = await readChecked(source, idx.offset, SKETCH_HEADER_BYTES, 'index header', maxReadBytes, fileBytes);
-                if (await sha256hex(headerBytes) !== manifest.index.headerSha256) {
-                    throw new Error('.pikelet index header failed hash verification against the manifest');
-                }
-            })() : null,
         ]);
 
         const lexicalIndex = lexicalBytes
@@ -1289,6 +1296,9 @@ export async function openPikeletFile(input, options = {}) {
                         ? {
                             terms: lexicalIndex.termCount,
                             docCount: lexicalIndex.docCount,
+                            // Lazy reads are bounded to the segment but not
+                            // individually verified (LAZY_LEXICAL_BYTES).
+                            integrity: lexicalIndex.lazy ? 'unverified-lazy-reads' : 'segment-sha256',
                             ...(lexicalIndex.lazy ? { lazy: true } : {}),
                         }
                         : null,

package/complete/lexical.mjs

@@ -181,6 +181,20 @@ export async function openLexicalIndexLazy(read, segLength) {
                 return e && { ...e, term: t };
             }))).filter(Boolean);
             if (!found.length) return [];
+            // Term entries arrive by lazy read, unverified (see the stance
+            // note above), so each one is held to the segment before its
+            // postings are fetched: inside the postings region (spec 3.8, as
+            // the eager reader enforces), a df no larger than the corpus, and
+            // a length a df-entry list can actually occupy (a doc-gap and a
+            // tf varint, each at most 6 bytes). Without these, one forged
+            // entry could steer a read anywhere in the file or ask for
+            // hundreds of MiB per query term.
+            for (const e of found) {
+                if (!Number.isSafeInteger(e.postingsRel) || e.postingsRel + e.postingsLen > postingsBytes
+                    || e.df > docCount || e.postingsLen > 12 * e.df) {
+                    throw new Error('.pikelet lexical postings out of bounds');
+                }
+            }
             const postings = await Promise.all(found.map((e) => read(postingsOffset + e.postingsRel, e.postingsLen)));
             const scores = new Map();
             found.forEach((e, i) => {

package/complete/retrieval-abstention.mjs

@@ -34,11 +34,25 @@
 // one — there is no scenario where re-adding this via a config flag was
 // going to end differently a third time.
 
+// Upper bound on asset.coverage.topK. calibrate.mjs ships 5; every passage
+// counted here is a record hydrated on every query, so a pack must not be
+// able to turn one query into a hydration of its whole rerank window.
+export const MAX_COVERAGE_TOP_K = 10;
+
 export function createAbstentionScorer(asset, bloomBytes) {
     if (!asset || !Array.isArray(asset.weights) || !asset.thresholds) return null;
+    // Thresholds that are missing, non-numeric or out of order compare false
+    // against every p and would answer everything; a malformed asset must
+    // degrade to unscored instead (callers map a null scorer to it).
+    const { hard, weak } = asset.thresholds;
+    if (!Number.isFinite(hard) || !Number.isFinite(weak) || hard < 0 || hard > weak || weak > 1) return null;
     const coverageCfg = asset.coverage && typeof asset.coverage.weight === 'number'
         && Number.isFinite(asset.coverage.mean) && Number.isFinite(asset.coverage.std)
         ? asset.coverage : null;
+    if (coverageCfg && coverageCfg.topK !== undefined
+        && !(Number.isInteger(coverageCfg.topK) && coverageCfg.topK >= 1 && coverageCfg.topK <= MAX_COVERAGE_TOP_K)) {
+        return null;
+    }
     const coverageStopwords = coverageCfg ? new Set(coverageCfg.stopwords || []) : null;
     const coverageMinLen = coverageCfg ? (coverageCfg.minWordLen || 3) : 3;
     // Words the corpus uses everywhere ground any query that mentions them,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pikelet-wasm",
-  "version": "0.9.2",
+  "version": "0.9.3",
   "description": "Vector search engine and .pikelet artifact readers/builders, in WebAssembly. Compile a corpus into one self-contained, hash-verified search file, or use the underlying HNSW engine directly. Runs in Node.js, browsers, and Cloudflare Workers with no native dependencies.",
   "main": "src/pikelet.js",
   "types": "src/pikelet.d.ts",

package/src/artifact/pikelet-artifact-sketch.js

@@ -37,6 +37,12 @@ const {
 const SKETCH_MAGIC = 0x31415350; // PSA1
 const SKETCH_HEADER_BYTES = 256;
 const SKETCH_KIND_U8 = 1;
+// Ceiling on the header's recommendedRerank as a reader default. The field
+// is a producer hint, and a hostile (or exhaustive-fallback) value up to the
+// row count would make every query fetch, hash and rerank the whole vectors
+// region — at wiki scale, minutes per query with every check passing. An
+// explicit options.rerank is the caller's own choice and is not capped here.
+const MAX_RECOMMENDED_RERANK = 4096;
 
 // The affine table feeds every rerank distance; a NaN/Infinity entry makes
 // that row's distance non-finite and lets it sit in the result set
@@ -331,6 +337,28 @@ class PikeletSketchArtifact {
         if (header.byteLength !== SKETCH_HEADER_BYTES) {
             throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact header is truncated');
         }
+        // options.expectedHeaderSha256 (hex): a container that commits to this
+        // header (the complete profile's manifest.index.headerSha256) checks
+        // it HERE, against the very bytes parsed below. Checking a separate
+        // read instead would let a host serve the genuine header to that read
+        // and a rewritten one (with a self-consistent residentSha256 and page
+        // table) to this one, replacing the index under a pinned identity.
+        if (options.expectedHeaderSha256 !== undefined) {
+            const expected = options.expectedHeaderSha256;
+            if (typeof expected !== 'string' || !/^[0-9a-f]{64}$/.test(expected)) {
+                throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'SketchArtifact.open() expectedHeaderSha256 must be 64 lowercase hex characters');
+            }
+            const digest = await sha256BytesAsync(header);
+            if (!digest) {
+                throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
+                    'Sketch artifact header verification requested but no crypto backend is available');
+            }
+            let hex = '';
+            for (const b of digest) hex += b.toString(16).padStart(2, '0');
+            if (hex !== expected) {
+                throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, '.pikelet index header failed hash verification against the manifest');
+            }
+        }
         const view = new DataView(header.buffer, header.byteOffset, header.byteLength);
         if (view.getUint32(0, true) !== SKETCH_MAGIC) {
             throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Not a Pikelet sketch artifact (bad magic)');
@@ -353,7 +381,8 @@ class PikeletSketchArtifact {
         const sketchesOffset = view.getUint32(40, true);
         artifact.vectorsOffset = view.getUint32(44, true);
         const fileBytes = Number(view.getBigUint64(48, true));
-        artifact.recommendedRerank = view.getUint32(120, true);
+        artifact.declaredRecommendedRerank = view.getUint32(120, true);
+        artifact.recommendedRerank = Math.min(artifact.declaredRecommendedRerank, MAX_RECOMMENDED_RERANK);
         // Retained so the lazy tier stays verifiable after open — see
         // verifyVectors(). Copied out of the header read buffer.
         artifact.vectorsSha256 = new Uint8Array(header.subarray(88, 120));
@@ -1131,6 +1160,7 @@ async function createSketchScanner(loadEngine, artifact, options = {}) {
 
 module.exports = {
     PikeletSketchArtifact,
+    MAX_RECOMMENDED_RERANK,
     createSketchScanner,
     buildSketchArtifact,
     buildSketchArtifactBytes,

package/src/artifact/pikelet-artifact.d.ts

@@ -105,5 +105,12 @@ export declare function exportSketchArtifact(
   options?: SketchArtifactBuildOptions
 ): SketchArtifactBuildManifest;
 
+/**
+ * Ceiling on the header's recommendedRerank that readers use as their default
+ * rerank depth. Producers should not declare more: a reader will not honor it
+ * unless the caller passes an explicit rerank.
+ */
+export declare const MAX_RECOMMENDED_RERANK: number;
+
 /** Parse and validate an untrusted uint8 engine snapshot (fails closed). */
 export declare function parseUint8Snapshot(bytes: Uint8Array | ArrayBufferLike): Uint8SnapshotGraph;

package/src/artifact/pikelet-artifact.js

@@ -12,13 +12,14 @@
 const { NodeFileRangeSource, parseUint8Snapshot } = require('./pikelet-artifact-common.js');
 const { PikeletRangeArtifact, buildRangeArtifact, buildRangeArtifactFile } = require('./pikelet-artifact-range.js');
 const {
-    PikeletSketchArtifact, createSketchScanner,
+    PikeletSketchArtifact, MAX_RECOMMENDED_RERANK, createSketchScanner,
     buildSketchArtifact, buildSketchArtifactBytes, buildSketchArtifactFile, exportSketchArtifact,
 } = require('./pikelet-artifact-sketch.js');
 
 module.exports = {
     PikeletRangeArtifact,
     PikeletSketchArtifact,
+    MAX_RECOMMENDED_RERANK,
     createSketchScanner,
     NodeFileRangeSource,
     buildRangeArtifact,