pikelet-wasm 0.7.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/CHANGELOG.md +340 -4
  2. package/NOTICE +5 -0
  3. package/README.md +226 -475
  4. package/complete/builder.d.ts +4 -4
  5. package/complete/builder.mjs +26 -9
  6. package/complete/encoder-kernels/encoder.node.wasm +0 -0
  7. package/complete/encoder-kernels/encoder.wasm +0 -0
  8. package/complete/fusion.mjs +72 -0
  9. package/complete/index.d.ts +34 -7
  10. package/complete/index.mjs +258 -129
  11. package/complete/inline-transformer.mjs +119 -10
  12. package/complete/lexical.mjs +11 -11
  13. package/complete/retrieval-abstention.mjs +136 -31
  14. package/complete/sources.mjs +22 -1
  15. package/complete/wordpiece.mjs +22 -0
  16. package/dist/engine.js +1 -1
  17. package/dist/engine.scalar.js +1 -1
  18. package/dist/engine.scalar.wasm +0 -0
  19. package/dist/engine.wasm +0 -0
  20. package/package.json +47 -82
  21. package/{pancake-artifact-common.js → src/artifact/pikelet-artifact-common.js} +42 -42
  22. package/{pancake-artifact-range.js → src/artifact/pikelet-artifact-range.js} +43 -36
  23. package/{pancake-artifact-sketch.js → src/artifact/pikelet-artifact-sketch.js} +105 -68
  24. package/{pancake-artifact.d.ts → src/artifact/pikelet-artifact.d.ts} +11 -11
  25. package/{pancake-artifact.js → src/artifact/pikelet-artifact.js} +10 -10
  26. package/{pancake-core.js → src/core/pikelet-core.js} +141 -140
  27. package/{pancake-errors.js → src/errors/pikelet-errors.js} +9 -8
  28. package/{pancake.d.ts → src/pikelet.d.ts} +55 -55
  29. package/{pancake.js → src/pikelet.js} +47 -47
  30. package/{pancake.node.mjs → src/pikelet.node.mjs} +49 -49
  31. package/{pancake.web.d.ts → src/pikelet.web.d.ts} +13 -13
  32. package/{pancake.web.mjs → src/pikelet.web.mjs} +24 -24
  33. package/{pancake.workerd.mjs → src/pikelet.workerd.mjs} +22 -22
  34. /package/{pancake-loader.js → src/loader/pikelet-loader.js} +0 -0
@@ -1,4 +1,4 @@
1
- // The one-file reader: open a .pancake (spec/COMPLETE_PROFILE.md) and query
1
+ // The one-file reader: open a .pikelet (spec/COMPLETE_PROFILE.md) and query
2
2
  // it. Environment-neutral — the same module runs in Node (file path or
3
3
  // range source) and the browser (HTTP range source, via a bundler for the
4
4
  // CJS sketch-reader dependency). The query path needs no core HNSW engine:
@@ -8,15 +8,15 @@
8
8
  // engine's SIMD scan kernel (options.sketchScanner — auto-staged in Node,
9
9
  // injected in browsers), which only ever changes speed, never results.
10
10
  //
11
- // const search = await openPancakeFile('pancake-docs.pancake'); // Node
12
- // const search = await openPancakeFile(httpRangeSource(url)); // browser
11
+ // const search = await openPikeletFile('docs.pikelet'); // Node
12
+ // const search = await openPikeletFile(httpRangeSource(url)); // browser
13
13
  // const out = await search.query('how do workers restore snapshots');
14
14
  //
15
15
  // Every byte of the artifact is untrusted input (contract section 7): each
16
16
  // offset, length, and count is validated — safe integer, within the header's
17
17
  // fileBytes and the source's size, under the per-open read budget — before a
18
18
  // read is issued or a buffer allocated, and every read must return exactly
19
- // the bytes asked for. Format 2 files (profile pancake-complete-v2) carry
19
+ // the bytes asked for. Format 2 files (profile pikelet-complete-v2) carry
20
20
  // per-record corpus digests, so each hydrated record is verified on its own
21
21
  // range read; format 1 files verify eager segments whole and hydrate records
22
22
  // under the segment digest only (reported via info().corpusIntegrity).
@@ -25,7 +25,7 @@ import { loadStudentModel, embedTextWithStudent } from './student-embedder.mjs';
25
25
  import { computeMatchQuality, computePreSearchAbstention } from './student-abstention.mjs';
26
26
  import { createAbstentionScorer } from './retrieval-abstention.mjs';
27
27
  import { openLexicalIndex, openLexicalIndexLazy } from './lexical.mjs';
28
- import { PancakeSketchArtifact } from '../pancake-artifact.js';
28
+ import { PikeletSketchArtifact } from '../src/artifact/pikelet-artifact.js';
29
29
  import { MAGIC, HEADER_BYTES, TABLE_ENTRY_BYTES, KINDS, KIND_NAMES } from './format.mjs';
30
30
 
31
31
  import {
@@ -36,9 +36,13 @@ import {
36
36
  } from './inline-transformer.mjs';
37
37
 
38
38
  export { httpRangeSource } from './sources.mjs';
39
- // The project renamed to Pikelet (2026-09); the wire format and this
40
- // reader's original export keep their pancake-era names for
41
- // compatibility, and openPikeletFile is the documented alias.
39
+ // Re-exported for the builder side: calibration fits coverage against the
40
+ // same fused (vector+BM25) ranking the reader serves, which needs an
41
+ // in-memory lexical searcher over the segment bytes already assembled by
42
+ // buildLexicalSegment, before the artifact is written to disk.
43
+ export { openLexicalIndex } from './lexical.mjs';
44
+ export { fuseCandidates, FUSION_DEFAULTS } from './fusion.mjs';
45
+ import { fuseCandidates } from './fusion.mjs';
42
46
  export {
43
47
  KERNEL_LAYOUT, expectedBlobBytes, parseInlineTransformerEncoder,
44
48
  createInlineTransformerEmbedder, INLINE_TEST_VECTOR_TEXTS,
@@ -47,7 +51,13 @@ export {
47
51
  };
48
52
 
49
53
  // Supported container formats: header formatVersion -> manifest profile.
50
- export const SUPPORTED_PROFILES = Object.freeze({ 1: 'pancake-complete-v1', 2: 'pancake-complete-v2' });
54
+ export const SUPPORTED_PROFILES = Object.freeze({ 1: 'pikelet-complete-v1', 2: 'pikelet-complete-v2' });
55
+ // Pre-rename profile strings (2026-09): every .pikelet published before this
56
+ // date — including live GitHub release assets — carries "pancake-complete-
57
+ // vN" in its manifest. Readers accept both spellings indefinitely so an
58
+ // already-published pack does not go dark on a reader upgrade; only
59
+ // SUPPORTED_PROFILES (above) is what a fresh build ever writes.
60
+ const LEGACY_PROFILES = Object.freeze({ 1: 'pancake-complete-v1', 2: 'pancake-complete-v2' });
51
61
  export const CORPUS_LAYOUT_V2 = 'records-v2';
52
62
 
53
63
  // Read budgets. Open-path reads (manifest, segment table, query-interp,
@@ -63,13 +73,32 @@ const MAX_RECORDS = 2 ** 31 - 1;
63
73
  const MAX_PAGE_RECORDS = 65536;
64
74
  const DIGEST_PAGE_CACHE = 64;
65
75
  const RECORD_CACHE = 256;
66
- // The embedded index is a .pancake-sketch artifact (SKETCH_PROFILE.md);
76
+ // The embedded index is a .pikelet-sketch artifact (SKETCH_PROFILE.md);
67
77
  // its fixed-size header is what format 2 commits to in the manifest.
68
78
  const SKETCH_HEADER_BYTES = 256;
69
79
  // Hybrid retrieval: BM25 candidates fetched from the lexical segment per
70
- // query, and the reciprocal-rank-fusion constant (the standard untuned 60).
80
+ // query; the fusion rule and its constants live in fusion.mjs, shared with
81
+ // the calibrator so fit-time rankings match what is served.
71
82
  const LEXICAL_CANDIDATES = 24;
72
- const RRF_K = 60;
83
+ // scoreQuality's coverage term hydrates the fused top passagesNeeded
84
+ // records (asset.coverage.topK, 5 by default — see calibrate.mjs's
85
+ // COVERAGE_TOP_PASSAGES) once retrieval returns. The scorer that knows
86
+ // the real number is created inside retrievalScorer()'s own closure, not
87
+ // reachable from here, so this is a fixed approximation used to decide
88
+ // how many of fusedFull's records to include in the batched hydration
89
+ // below, alongside phrase-pinning's own candidates — hydrating a couple
90
+ // more than coverage strictly needs just costs a little extra bandwidth
91
+ // on a cache entry that goes unread; hydrating fewer only narrows how
92
+ // much of coverage's later hydration hits warm cache. Either way is
93
+ // harmless: it's a batch-sizing choice, not a correctness one.
94
+ const COVERAGE_BATCH_COUNT = 5;
95
+ // Only BM25 hits within this fraction of the top lexical score join fusion.
96
+ // Common query terms give every matching document a near-tied score (idf
97
+ // collapses them flat); a wide cutoff let that tied mass ride into fusion on
98
+ // a technicality. Kept identical to pikelet/src/calibrate.mjs's
99
+ // LEXICAL_CUTOFF so calibration is fit against the same fusion the reader
100
+ // actually serves.
101
+ const LEXICAL_CUTOFF = 1.5;
73
102
  // Query-interpretation segments above this size open lazily (kind 3's
74
103
  // inline encoder is ~25 MiB); smaller ones keep the one-read eager path.
75
104
  const LAZY_QI_BYTES = 4 * 1024 * 1024;
@@ -110,20 +139,30 @@ function toHex(bytes) {
110
139
  const isSha256Hex = (value) => typeof value === 'string' && /^[0-9a-f]{64}$/.test(value);
111
140
 
112
141
  // SHA-256 via WebCrypto (browser, workerd, Node 18+); node:crypto fallback
113
- // for older Node. Fails closed: verification is not optional in this reader.
142
+ // for older Node, gated on an actual Node runtime check rather than just
143
+ // "WebCrypto is missing" — a browser can lack crypto.subtle too (privacy
144
+ // settings, a hardened build), and a bare dynamic import('node:crypto')
145
+ // there isn't a Node module resolution failure, it's a cross-origin module
146
+ // fetch a browser refuses outright, surfacing as an opaque module-loading
147
+ // error instead of the clear message below. Fails closed either way:
148
+ // verification is not optional in this reader.
114
149
  async function sha256hex(bytes) {
115
150
  if (globalThis.crypto?.subtle) {
116
151
  return toHex(new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256', bytes)));
117
152
  }
118
- const { createHash } = await import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:crypto');
119
- return createHash('sha256').update(bytes).digest('hex');
153
+ if (typeof process !== 'undefined' && process.versions?.node) {
154
+ const { createHash } = await import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:crypto');
155
+ return createHash('sha256').update(bytes).digest('hex');
156
+ }
157
+ throw new Error('No crypto backend available (crypto.subtle is missing) — '
158
+ + 'cannot verify artifact integrity in this environment');
120
159
  }
121
160
 
122
161
  function asUint8Array(bytes) {
123
162
  if (bytes instanceof Uint8Array) return bytes;
124
163
  if (bytes instanceof ArrayBuffer) return new Uint8Array(bytes);
125
164
  if (ArrayBuffer.isView(bytes)) return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
126
- throw new Error('.pancake range source returned a non-binary value');
165
+ throw new Error('.pikelet range source returned a non-binary value');
127
166
  }
128
167
 
129
168
  function resolveBudget(value, label, fallback) {
@@ -138,7 +177,7 @@ function resolveBudget(value, label, fallback) {
138
177
  // u64 fields become JS numbers only when they are exactly representable.
139
178
  function u64(view, offset, label) {
140
179
  const big = view.getBigUint64(offset, true);
141
- if (big > MAX_SAFE_BIG) throw new Error(`.pancake ${label} exceeds the safe integer range`);
180
+ if (big > MAX_SAFE_BIG) throw new Error(`.pikelet ${label} exceeds the safe integer range`);
142
181
  return Number(big);
143
182
  }
144
183
 
@@ -148,16 +187,16 @@ function u64(view, offset, label) {
148
187
  function checkRange(source, offset, length, label, limit, fileBytes) {
149
188
  if (!Number.isSafeInteger(offset) || !Number.isSafeInteger(length)
150
189
  || offset < 0 || length < 0 || !Number.isSafeInteger(offset + length)) {
151
- throw new Error(`.pancake ${label} range is out of bounds`);
190
+ throw new Error(`.pikelet ${label} range is out of bounds`);
152
191
  }
153
192
  if (length > limit) {
154
- throw new Error(`.pancake ${label} read of ${length} bytes exceeds the ${limit}-byte read budget`);
193
+ throw new Error(`.pikelet ${label} read of ${length} bytes exceeds the ${limit}-byte read budget`);
155
194
  }
156
195
  if (fileBytes !== undefined && offset + length > fileBytes) {
157
- throw new Error(`.pancake ${label} range extends past the file (${offset}+${length} > ${fileBytes})`);
196
+ throw new Error(`.pikelet ${label} range extends past the file (${offset}+${length} > ${fileBytes})`);
158
197
  }
159
198
  if (Number.isSafeInteger(source.size) && offset + length > source.size) {
160
- throw new Error(`.pancake ${label} range extends past the source (${offset}+${length} > ${source.size})`);
199
+ throw new Error(`.pikelet ${label} range extends past the source (${offset}+${length} > ${source.size})`);
161
200
  }
162
201
  }
163
202
 
@@ -165,7 +204,7 @@ async function readChecked(source, offset, length, label, limit, fileBytes) {
165
204
  checkRange(source, offset, length, label, limit, fileBytes);
166
205
  const bytes = asUint8Array(await source.read(offset, length));
167
206
  if (bytes.length !== length) {
168
- throw new Error(`.pancake ${label} read returned ${bytes.length} of ${length} bytes (truncated or misbehaving source)`);
207
+ throw new Error(`.pikelet ${label} read returned ${bytes.length} of ${length} bytes (truncated or misbehaving source)`);
169
208
  }
170
209
  return bytes;
171
210
  }
@@ -185,7 +224,7 @@ async function fileSource(filePath) {
185
224
  preferredParallelism: Infinity,
186
225
  preferredGapBytes: 2048,
187
226
  async read(offset, length) {
188
- if (fd === null) throw new Error('.pancake file source is closed');
227
+ if (fd === null) throw new Error('.pikelet file source is closed');
189
228
  const buffer = new Uint8Array(length);
190
229
  let bytesRead = 0;
191
230
  while (bytesRead < length) {
@@ -217,7 +256,7 @@ function windowSource(source, offset, length) {
217
256
  preferredGapBytes: source.preferredGapBytes,
218
257
  async read(off, len) {
219
258
  if (!Number.isSafeInteger(off) || !Number.isSafeInteger(len) || off < 0 || len < 0 || off + len > length) {
220
- throw new Error('.pancake index segment read is outside the segment window');
259
+ throw new Error('.pikelet index segment read is outside the segment window');
221
260
  }
222
261
  return source.read(offset + off, len);
223
262
  },
@@ -257,19 +296,19 @@ function base64Bytes(text) {
257
296
  export async function verifyHostEncoder(declaration, encodeQuery, dim) {
258
297
  const vectors = declaration.testVectors;
259
298
  if (!Array.isArray(vectors) || vectors.length === 0) {
260
- throw new Error('.pancake kind-2 declaration carries no verification vectors');
299
+ throw new Error('.pikelet kind-2 declaration carries no verification vectors');
261
300
  }
262
301
  if (declaration.dim !== undefined && declaration.dim !== dim) {
263
- throw new Error(`.pancake kind-2 declaration dim ${declaration.dim} disagrees with manifest dim ${dim}`);
302
+ throw new Error(`.pikelet kind-2 declaration dim ${declaration.dim} disagrees with manifest dim ${dim}`);
264
303
  }
265
304
  for (const tv of vectors) {
266
305
  if (!tv || typeof tv.text !== 'string') {
267
- throw new Error('.pancake kind-2 verification vector is malformed');
306
+ throw new Error('.pikelet kind-2 verification vector is malformed');
268
307
  }
269
308
  // The expected embedding must be dim finite numbers: a malformed
270
309
  // entry would turn every comparison into NaN, and NaN > maxDiff is
271
310
  // false — the vector would "pass" without comparing anything.
272
- const label = `.pancake kind-2 verification vector "${tv.text.slice(0, 40)}…"`;
311
+ const label = `.pikelet kind-2 verification vector "${tv.text.slice(0, 40)}…"`;
273
312
  validateExpectedEmbedding(tv.embedding, dim, label);
274
313
  const tolerance = validateTestVectorTolerance(tv.tolerance, label);
275
314
  const vector = toFloat32(await encodeQuery(tv.text), dim, 'options.encodeQuery');
@@ -288,15 +327,15 @@ export async function verifyHostEncoder(declaration, encodeQuery, dim) {
288
327
  }
289
328
 
290
329
  /**
291
- * Open a .pancake complete artifact from a file path (Node) or a
330
+ * Open a .pikelet complete artifact from a file path (Node) or a
292
331
  * { read(offset, length), size? } range source (any runtime).
293
332
  * Returns { query(text, {k}), info(), evaluation(), close() }.
294
333
  */
295
- export async function openPancakeFile(input, options = {}) {
334
+ export async function openPikeletFile(input, options = {}) {
296
335
  const source = typeof input === 'string' ? await fileSource(input) : input;
297
336
  const owned = typeof input === 'string';
298
337
  if (!source || typeof source.read !== 'function') {
299
- throw new Error('openPancakeFile() requires a file path or a range source with read(offset, length)');
338
+ throw new Error('openPikeletFile() requires a file path or a range source with read(offset, length)');
300
339
  }
301
340
  const maxReadBytes = resolveBudget(options.maxReadBytes, 'maxReadBytes', DEFAULT_OPEN_READ_BYTES);
302
341
  const maxRecordBytes = resolveBudget(options.maxRecordBytes, 'maxRecordBytes', DEFAULT_RECORD_BYTES);
@@ -333,23 +372,23 @@ export async function openPancakeFile(input, options = {}) {
333
372
  const header = await readChecked(source, 0, HEADER_BYTES, 'header', maxReadBytes);
334
373
  const headerView = viewOf(header);
335
374
  if (headerView.getUint32(0, true) !== MAGIC) {
336
- throw new Error('not a .pancake file (bad magic)');
375
+ throw new Error('not a .pikelet file (bad magic)');
337
376
  }
338
377
  const formatVersion = headerView.getUint32(4, true);
339
378
  const profile = SUPPORTED_PROFILES[formatVersion];
340
- if (!profile) throw new Error(`unsupported .pancake format version ${formatVersion}`);
379
+ if (!profile) throw new Error(`unsupported .pikelet format version ${formatVersion}`);
341
380
  const manifestBytes = headerView.getUint32(8, true);
342
381
  const segmentCount = headerView.getUint32(12, true);
343
382
  const fileBytes = u64(headerView, 16, 'header fileBytes');
344
383
  if (manifestBytes < 2 || manifestBytes > MAX_MANIFEST_BYTES || segmentCount < 1 || segmentCount > MAX_SEGMENTS) {
345
- throw new Error('.pancake header is implausible');
384
+ throw new Error('.pikelet header is implausible');
346
385
  }
347
386
  if (Number.isSafeInteger(source.size) && source.size !== fileBytes) {
348
- throw new Error(`.pancake is truncated or padded: header says ${fileBytes} bytes, source has ${source.size}`);
387
+ throw new Error(`.pikelet is truncated or padded: header says ${fileBytes} bytes, source has ${source.size}`);
349
388
  }
350
389
  const tableBytes = segmentCount * TABLE_ENTRY_BYTES;
351
390
  if (HEADER_BYTES + manifestBytes + tableBytes > fileBytes) {
352
- throw new Error('.pancake header places the manifest or segment table past the file');
391
+ throw new Error('.pikelet header places the manifest or segment table past the file');
353
392
  }
354
393
  const identity = toHex(header.subarray(24, 56));
355
394
  // A caller pinning an expected identity (a shelf entry, a lockfile,
@@ -362,30 +401,31 @@ export async function openPancakeFile(input, options = {}) {
362
401
  throw new Error('options.expectedIdentity must be a sha256 hex string');
363
402
  }
364
403
  if (identity !== options.expectedIdentity.toLowerCase()) {
365
- throw new Error(`.pancake identity mismatch: expected ${options.expectedIdentity.toLowerCase()}, found ${identity}`);
404
+ throw new Error(`.pikelet identity mismatch: expected ${options.expectedIdentity.toLowerCase()}, found ${identity}`);
366
405
  }
367
406
  }
368
407
 
369
408
  const manifestBuf = await readChecked(source, HEADER_BYTES, manifestBytes, 'manifest', maxReadBytes, fileBytes);
370
409
  if (await sha256hex(manifestBuf) !== identity) {
371
- throw new Error('.pancake manifest failed identity verification');
410
+ throw new Error('.pikelet manifest failed identity verification');
372
411
  }
373
412
  let manifest;
374
413
  try { manifest = JSON.parse(decoder.decode(manifestBuf)); } catch (err) {
375
- throw new Error('.pancake manifest is not valid JSON', { cause: err });
414
+ throw new Error('.pikelet manifest is not valid JSON', { cause: err });
376
415
  }
377
- if (!manifest || typeof manifest !== 'object' || manifest.profile !== profile) {
416
+ if (!manifest || typeof manifest !== 'object'
417
+ || (manifest.profile !== profile && manifest.profile !== LEGACY_PROFILES[formatVersion])) {
378
418
  throw new Error(`unsupported profile ${manifest?.profile} for format version ${formatVersion}`);
379
419
  }
380
420
  if (!Array.isArray(manifest.segments) || manifest.segments.length !== segmentCount) {
381
- throw new Error('.pancake manifest segment list disagrees with the header segment count');
421
+ throw new Error('.pikelet manifest segment list disagrees with the header segment count');
382
422
  }
383
423
  const dim = manifest.dim;
384
424
  if (!Number.isSafeInteger(dim) || dim < 1 || dim > 65536) {
385
- throw new Error('.pancake manifest dim is implausible');
425
+ throw new Error('.pikelet manifest dim is implausible');
386
426
  }
387
427
  if (!manifest.corpus || typeof manifest.corpus !== 'object') {
388
- throw new Error('.pancake manifest has no corpus block');
428
+ throw new Error('.pikelet manifest has no corpus block');
389
429
  }
390
430
 
391
431
  const table = await readChecked(source, HEADER_BYTES + manifestBytes, tableBytes, 'segment table', maxReadBytes, fileBytes);
@@ -401,26 +441,26 @@ export async function openPancakeFile(input, options = {}) {
401
441
  const declared = manifest.segments[i];
402
442
  if (!declared || typeof declared !== 'object' || declared.bytes !== length || !isSha256Hex(declared.sha256)
403
443
  || offset !== expectedOffset || !Number.isSafeInteger(offset + length) || offset + length > fileBytes) {
404
- throw new Error(`.pancake segment table disagrees with manifest at entry ${i}`);
444
+ throw new Error(`.pikelet segment table disagrees with manifest at entry ${i}`);
405
445
  }
406
446
  if (kind === undefined) {
407
447
  // Unknown kinds are skipped, not failed (spec 3.3); they are
408
448
  // still committed through the manifest, so the declared name
409
449
  // must at least not claim to be a kind this reader knows.
410
450
  if (typeof declared.kind !== 'string' || declared.kind in KINDS) {
411
- throw new Error(`.pancake segment table entry ${i} has unknown kind ${kindNumber} but the manifest names it ${declared.kind}`);
451
+ throw new Error(`.pikelet segment table entry ${i} has unknown kind ${kindNumber} but the manifest names it ${declared.kind}`);
412
452
  }
413
453
  } else {
414
454
  if (declared.kind !== kind) {
415
- throw new Error(`.pancake segment table disagrees with manifest at entry ${i}`);
455
+ throw new Error(`.pikelet segment table disagrees with manifest at entry ${i}`);
416
456
  }
417
- if (segments.has(kind)) throw new Error(`.pancake carries more than one ${kind} segment`);
457
+ if (segments.has(kind)) throw new Error(`.pikelet carries more than one ${kind} segment`);
418
458
  segments.set(kind, { offset, length, sha256: declared.sha256 });
419
459
  }
420
460
  expectedOffset = align16(offset + length);
421
461
  }
422
462
  for (const required of ['index', 'corpus', 'query-interp']) {
423
- if (!segments.has(required)) throw new Error(`.pancake is missing the ${required} segment`);
463
+ if (!segments.has(required)) throw new Error(`.pikelet is missing the ${required} segment`);
424
464
  }
425
465
 
426
466
  // Corpus tables: count + offsets (+ page table for layout v2),
@@ -432,15 +472,15 @@ export async function openPancakeFile(input, options = {}) {
432
472
  const corpus = segments.get('corpus');
433
473
  const declaredRecords = manifest.corpus.records;
434
474
  if (!Number.isSafeInteger(declaredRecords) || declaredRecords < 0 || declaredRecords > MAX_RECORDS) {
435
- throw new Error('.pancake manifest corpus.records is implausible');
475
+ throw new Error('.pikelet manifest corpus.records is implausible');
436
476
  }
437
477
  const layout = manifest.corpus.layout;
438
478
  const perRecord = formatVersion >= 2;
439
479
  if (perRecord && layout !== CORPUS_LAYOUT_V2) {
440
- throw new Error(`.pancake format ${formatVersion} requires corpus layout ${CORPUS_LAYOUT_V2}, manifest says ${layout}`);
480
+ throw new Error(`.pikelet format ${formatVersion} requires corpus layout ${CORPUS_LAYOUT_V2}, manifest says ${layout}`);
441
481
  }
442
482
  if (!perRecord && layout !== undefined) {
443
- throw new Error(`.pancake format 1 carries corpus layout v1, manifest says ${layout}`);
483
+ throw new Error(`.pikelet format 1 carries corpus layout v1, manifest says ${layout}`);
444
484
  }
445
485
  let pageRecords = 0;
446
486
  let pages = 0;
@@ -451,12 +491,12 @@ export async function openPancakeFile(input, options = {}) {
451
491
  if (perRecord) {
452
492
  pageRecords = manifest.corpus.pageRecords;
453
493
  if (!Number.isSafeInteger(pageRecords) || pageRecords < 1 || pageRecords > MAX_PAGE_RECORDS) {
454
- throw new Error('.pancake manifest corpus.pageRecords is implausible');
494
+ throw new Error('.pikelet manifest corpus.pageRecords is implausible');
455
495
  }
456
496
  pages = Math.ceil(declaredRecords / pageRecords);
457
497
  if (manifest.corpus.pages !== pages || !isSha256Hex(manifest.corpus.pageTableSha256)
458
498
  || manifest.corpus.recordDigest !== 'sha256') {
459
- throw new Error('.pancake manifest corpus integrity block is inconsistent');
499
+ throw new Error('.pikelet manifest corpus integrity block is inconsistent');
460
500
  }
461
501
  offsetsAt = 8;
462
502
  tablesBytes = offsetsAt + 8 * (declaredRecords + 1) + 32 * pages;
@@ -468,7 +508,7 @@ export async function openPancakeFile(input, options = {}) {
468
508
  recordsAt = tablesBytes;
469
509
  }
470
510
  if (!Number.isSafeInteger(recordsAt) || recordsAt > corpus.length) {
471
- throw new Error('.pancake manifest corpus.records is implausible for the corpus segment');
511
+ throw new Error('.pikelet manifest corpus.records is implausible for the corpus segment');
472
512
  }
473
513
  // Format 2 commits to the embedded sketch's 256-byte header
474
514
  // (metric/dim/count and the sketch's residentSha256/vectorsSha256),
@@ -476,7 +516,7 @@ export async function openPancakeFile(input, options = {}) {
476
516
  // sketch's self-checks alone verify against fields an attacker who
477
517
  // rewrites the segment also controls.
478
518
  if (perRecord && (!isSha256Hex(manifest.index?.headerSha256) || idx.length < SKETCH_HEADER_BYTES)) {
479
- throw new Error('.pancake manifest carries no index header commitment (index.headerSha256)');
519
+ throw new Error('.pikelet manifest carries no index header commitment (index.headerSha256)');
480
520
  }
481
521
 
482
522
  // After the segment table, every remaining open-path extent is known,
@@ -502,11 +542,11 @@ export async function openPancakeFile(input, options = {}) {
502
542
  }
503
543
  const bytes = await readChecked(source, qi.offset, qi.length, 'query-interp segment', maxReadBytes, fileBytes);
504
544
  if (await sha256hex(bytes) !== qi.sha256) {
505
- throw new Error('.pancake query-interp segment failed hash verification');
545
+ throw new Error('.pikelet query-interp segment failed hash verification');
506
546
  }
507
547
  return bytes;
508
548
  })(),
509
- PancakeSketchArtifact.open(windowSource(source, idx.offset, idx.length), { maxReadBytes }),
549
+ PikeletSketchArtifact.open(windowSource(source, idx.offset, idx.length), { maxReadBytes }),
510
550
  readChecked(source, corpus.offset, tablesBytes, 'corpus tables', maxReadBytes, fileBytes),
511
551
  // Lexical index (kind 5, OPTIONAL). Small segments: eager
512
552
  // whole-segment read, digest-verified like the query-interp
@@ -522,14 +562,14 @@ export async function openPancakeFile(input, options = {}) {
522
562
  }
523
563
  const bytes = await readChecked(source, seg.offset, seg.length, 'lexical segment', maxReadBytes, fileBytes);
524
564
  if (await sha256hex(bytes) !== seg.sha256) {
525
- throw new Error('.pancake lexical segment failed hash verification');
565
+ throw new Error('.pikelet lexical segment failed hash verification');
526
566
  }
527
567
  return { bytes };
528
568
  })() : null,
529
569
  perRecord ? (async () => {
530
570
  const headerBytes = await readChecked(source, idx.offset, SKETCH_HEADER_BYTES, 'index header', maxReadBytes, fileBytes);
531
571
  if (await sha256hex(headerBytes) !== manifest.index.headerSha256) {
532
- throw new Error('.pancake index header failed hash verification against the manifest');
572
+ throw new Error('.pikelet index header failed hash verification against the manifest');
533
573
  }
534
574
  })() : null,
535
575
  ]);
@@ -538,12 +578,12 @@ export async function openPancakeFile(input, options = {}) {
538
578
  ? (lexicalBytes.lazyIndex || openLexicalIndex(lexicalBytes.bytes))
539
579
  : null;
540
580
  if (lexicalIndex && lexicalIndex.docCount !== declaredRecords) {
541
- throw new Error(`.pancake lexical segment covers ${lexicalIndex.docCount} records, corpus declares ${declaredRecords}`);
581
+ throw new Error(`.pikelet lexical segment covers ${lexicalIndex.docCount} records, corpus declares ${declaredRecords}`);
542
582
  }
543
583
 
544
584
  // Query interpretation: version word, kind, two length-prefixed
545
585
  // regions that must tile the segment exactly.
546
- if (qi.length < 16) throw new Error('.pancake query-interp segment is too short');
586
+ if (qi.length < 16) throw new Error('.pikelet query-interp segment is too short');
547
587
  const qiView = viewOf(qiFirst);
548
588
  const qiVersion = qiView.getUint32(0, true);
549
589
  if (qiVersion !== 1) throw new Error(`unsupported query-interpretation version ${qiVersion}`);
@@ -551,7 +591,7 @@ export async function openPancakeFile(input, options = {}) {
551
591
  const encoderLen = qiView.getUint32(8, true);
552
592
  const calibrationLen = qiView.getUint32(12, true);
553
593
  if (16 + encoderLen + calibrationLen !== qi.length) {
554
- throw new Error('.pancake query-interp segment layout is inconsistent');
594
+ throw new Error('.pikelet query-interp segment layout is inconsistent');
555
595
  }
556
596
  // Laziness only pays for kind 3 (the deferrable bulk IS the
557
597
  // encoder); a large segment of any other kind falls back to the
@@ -561,7 +601,7 @@ export async function openPancakeFile(input, options = {}) {
561
601
  if (qiDeferrable && qiKind !== 3) {
562
602
  qiBytes = await readChecked(source, qi.offset, qi.length, 'query-interp segment', maxReadBytes, fileBytes);
563
603
  if (await sha256hex(qiBytes) !== qi.sha256) {
564
- throw new Error('.pancake query-interp segment failed hash verification');
604
+ throw new Error('.pikelet query-interp segment failed hash verification');
565
605
  }
566
606
  qiDeferred = false;
567
607
  }
@@ -574,10 +614,10 @@ export async function openPancakeFile(input, options = {}) {
574
614
  const encoderBytes = qiDeferred ? null : qiBytes.subarray(16, 16 + encoderLen);
575
615
  let calibrationJson;
576
616
  try { calibrationJson = JSON.parse(decoder.decode(calibrationRegion)); } catch (err) {
577
- throw new Error('.pancake calibration is not valid JSON', { cause: err });
617
+ throw new Error('.pikelet calibration is not valid JSON', { cause: err });
578
618
  }
579
619
  if (!calibrationJson || typeof calibrationJson !== 'object') {
580
- throw new Error('.pancake calibration must be a JSON object');
620
+ throw new Error('.pikelet calibration must be a JSON object');
581
621
  }
582
622
 
583
623
  // kind 1 (student-inline-v1): pure-JS inline encoder + feature-stream
@@ -595,20 +635,42 @@ export async function openPancakeFile(input, options = {}) {
595
635
  const retrievalScorer = () => {
596
636
  const scorer = createAbstentionScorer(calibrationJson.asset, base64Bytes(calibrationJson.vocabBloomBase64));
597
637
  const VERDICTS = { answer: 'strong', weak: 'weak', abstain: 'none' };
598
- return async (hits, context) => {
638
+ return async (hits, context, fusedHits) => {
599
639
  if (!scorer) return { match_quality: 'unscored' };
600
640
  // The coverage term grounds the verdict in the top passages'
601
- // text, so those records hydrate before scoring. When the
602
- // query is answered the pages are already hot for result
603
- // hydration; an abstained query costs these extra reads.
604
- // Hydration failures propagate — on format 2 a record that
605
- // fails its digest must fail the query, not skew its verdict.
606
- const topTexts = scorer.usesPassage && hits.length
607
- ? (await Promise.all(hits.slice(0, scorer.passagesNeeded || 1)
608
- .map((hit) => hydrate(hit.id)))).map((record) => record?.text)
641
+ // text, so those records hydrate before scoring. Hydrated
642
+ // from the fused (RRF) order when hybrid retrieval produced
643
+ // one — calibrate.mjs fits coverage against the same fused
644
+ // ranking, and a chunk BM25 ranks first but the embedding
645
+ // ranks far down is exactly the case coverage exists to
646
+ // catch; scoring it against vector-only order would miss it
647
+ // again. Falls back to vector order (hits) when there is no
648
+ // lexical segment or fusion did not change anything. When
649
+ // the query is answered the pages are already hot for
650
+ // result hydration; an abstained query costs these extra
651
+ // reads. Hydration failures propagate — on format 2 a
652
+ // record that fails its digest must fail the query, not
653
+ // skew its verdict.
654
+ const passageSource = fusedHits?.length ? fusedHits : hits;
655
+ // A record's text starts with its own heading echoed as the
656
+ // first line (ingest.mjs's section-to-chunk join). A
657
+ // heading match still grounds the query but at reduced
658
+ // credit — see retrieval-abstention.mjs's coverageFrac and
659
+ // calibrate.mjs's HEADING_COVERAGE_WEIGHT, which this split
660
+ // must match exactly (build-time and serve-time coverage
661
+ // must agree, or the fitted thresholds are scoring a
662
+ // different signal than the one they were calibrated
663
+ // against).
664
+ const splitHeadingBody = (text) => {
665
+ const nl = String(text || '').indexOf('\n');
666
+ return nl === -1 ? { heading: text, body: '' } : { heading: text.slice(0, nl), body: text.slice(nl + 1) };
667
+ };
668
+ const topTexts = scorer.usesPassage && passageSource.length
669
+ ? (await Promise.all(passageSource.slice(0, scorer.passagesNeeded || 1)
670
+ .map((hit) => hydrate(hit.id)))).map((record) => splitHeadingBody(record?.text))
609
671
  : [];
610
- const scored = scorer.score(context.text, hits, topTexts);
611
- return { match_quality: VERDICTS[scored.verdict] || scored.verdict, confidence: scored.p };
672
+ const scored = await scorer.score(context.text, hits, topTexts);
673
+ return { match_quality: VERDICTS[scored.verdict] || scored.verdict, confidence: scored.p, grounding: scored.grounding || null };
612
674
  };
613
675
  };
614
676
  if (qiKind === 1) {
@@ -625,9 +687,9 @@ export async function openPancakeFile(input, options = {}) {
625
687
  } else if (qiKind === 2) {
626
688
  let declaration;
627
689
  try { declaration = JSON.parse(decoder.decode(encoderBytes)); } catch (err) {
628
- throw new Error('.pancake kind-2 encoder declaration is not valid JSON', { cause: err });
690
+ throw new Error('.pikelet kind-2 encoder declaration is not valid JSON', { cause: err });
629
691
  }
630
- if (!declaration || typeof declaration !== 'object') throw new Error('.pancake kind-2 encoder declaration must be a JSON object');
692
+ if (!declaration || typeof declaration !== 'object') throw new Error('.pikelet kind-2 encoder declaration must be a JSON object');
631
693
  encoderInfo = declaration;
632
694
  const encodeQuery = options.encodeQuery;
633
695
  if (encodeQuery !== undefined && typeof encodeQuery !== 'function') {
@@ -641,7 +703,7 @@ export async function openPancakeFile(input, options = {}) {
641
703
  } else if (options.allowUnverifiedEncoder === true) {
642
704
  encoderVerified = false;
643
705
  } else {
644
- throw new Error(`.pancake declares an external encoder (${declaration.model}) without verification vectors, `
706
+ throw new Error(`.pikelet declares an external encoder (${declaration.model}) without verification vectors, `
645
707
  + 'so the host encoder cannot be checked against it (contract section 4.4); '
646
708
  + 'pass options.allowUnverifiedEncoder: true to serve it anyway, marked unverified');
647
709
  }
@@ -650,8 +712,8 @@ export async function openPancakeFile(input, options = {}) {
650
712
  }
651
713
  embed = async (text) => {
652
714
  if (!encodeQuery) {
653
- throw new Error(`.pancake declares an external encoder (${declaration.model}); `
654
- + 'pass options.encodeQuery to openPancakeFile');
715
+ throw new Error(`.pikelet declares an external encoder (${declaration.model}); `
716
+ + 'pass options.encodeQuery to openPikeletFile');
655
717
  }
656
718
  return { vector: toFloat32(await encodeQuery(text), dim, 'options.encodeQuery'), text };
657
719
  };
@@ -670,7 +732,7 @@ export async function openPancakeFile(input, options = {}) {
670
732
  } catch (err) {
671
733
  throw new Error('kind-3 artifact requires the inline-transformer kernels at '
672
734
  + 'complete/encoder-kernels/ (pikelet-wasm/complete); run '
673
- + 'examples/05-one-file-search/encoder-spike/build-encoder.sh to rebuild them',
735
+ + 'examples/one-file-search/encoder-spike/build-encoder.sh to rebuild them',
674
736
  { cause: err });
675
737
  }
676
738
  }
@@ -680,7 +742,7 @@ export async function openPancakeFile(input, options = {}) {
680
742
  if (qiDeferred) {
681
743
  const full = await readChecked(source, qi.offset, qi.length, 'query-interp segment', maxReadBytes, fileBytes);
682
744
  if (await sha256hex(full) !== qi.sha256) {
683
- throw new Error('.pancake query-interp segment failed hash verification');
745
+ throw new Error('.pikelet query-interp segment failed hash verification');
684
746
  }
685
747
  // The header and calibration slices used at open were
686
748
  // unverified; the digest-verified segment must contain
@@ -688,13 +750,13 @@ export async function openPancakeFile(input, options = {}) {
688
750
  // every query must fail.
689
751
  if (!bytesEqual(full.subarray(0, 16), openHeader)
690
752
  || !bytesEqual(full.subarray(16 + encoderLen), calibrationRegion)) {
691
- throw new Error('.pancake query-interp segment bytes changed between open and encoder fetch');
753
+ throw new Error('.pikelet query-interp segment bytes changed between open and encoder fetch');
692
754
  }
693
755
  encBytes = full.subarray(16, 16 + encoderLen);
694
756
  }
695
757
  const { declaration, vocabText, blob } = parseInlineTransformerEncoder(encBytes);
696
758
  if (declaration.dim !== undefined && declaration.dim !== dim) {
697
- throw new Error(`.pancake kind-3 declaration dim ${declaration.dim} disagrees with manifest dim ${dim}`);
759
+ throw new Error(`.pikelet kind-3 declaration dim ${declaration.dim} disagrees with manifest dim ${dim}`);
698
760
  }
699
761
  const embedder = await createInlineTransformerEmbedder({ declaration, vocabText, blob, createEncoder, verify: verifyEncoder });
700
762
  encoderInfo = declaration;
@@ -732,10 +794,10 @@ export async function openPancakeFile(input, options = {}) {
732
794
  // Index: the embedded sketch artifact, opened in the wave above
733
795
  // against a segment-windowed source (resident hash verified there).
734
796
  if (sketch.count !== declaredRecords) {
735
- throw new Error(`.pancake index count ${sketch.count} != corpus records ${declaredRecords}`);
797
+ throw new Error(`.pikelet index count ${sketch.count} != corpus records ${declaredRecords}`);
736
798
  }
737
799
  if (sketch.dim !== undefined && sketch.dim !== dim) {
738
- throw new Error(`.pancake index dim ${sketch.dim} != manifest dim ${dim}`);
800
+ throw new Error(`.pikelet index dim ${sketch.dim} != manifest dim ${dim}`);
739
801
  }
740
802
  // The metric decides the search semantics; the sketch header's copy
741
803
  // must agree with the identity-verified manifest (on format 2 the
@@ -743,7 +805,7 @@ export async function openPancakeFile(input, options = {}) {
743
805
  // is the only binding).
744
806
  if (METRIC_NAMES[sketch.metric] !== undefined && typeof manifest.metric === 'string'
745
807
  && METRIC_NAMES[sketch.metric] !== manifest.metric) {
746
- throw new Error(`.pancake index metric ${METRIC_NAMES[sketch.metric]} != manifest metric ${manifest.metric}`);
808
+ throw new Error(`.pikelet index metric ${METRIC_NAMES[sketch.metric]} != manifest metric ${manifest.metric}`);
747
809
  }
748
810
  if (verifyIndexVectors) {
749
811
  await sketch.verifyVectors();
@@ -788,9 +850,9 @@ export async function openPancakeFile(input, options = {}) {
788
850
  // Hidden from bundlers like the node encoder kernels: the
789
851
  // web entrypoint's ?url wasm imports would break a browser
790
852
  // build that statically pulled this in.
791
- const { default: Pancake } = await import(/* webpackIgnore: true */ /* @vite-ignore */ '../pancake.node.mjs');
853
+ const { default: Pikelet } = await import(/* webpackIgnore: true */ /* @vite-ignore */ '../src/pikelet.node.mjs');
792
854
  await sketch.fullyResident;
793
- const created = await Pancake.createSketchScanner(sketch, {
855
+ const created = await Pikelet.createSketchScanner(sketch, {
794
856
  maxRerank: Math.min(sketch.count, SCANNER_MAX_RERANK),
795
857
  });
796
858
  disposeScanner = () => created.dispose();
@@ -805,16 +867,16 @@ export async function openPancakeFile(input, options = {}) {
805
867
  const tablesView = viewOf(tables);
806
868
  const recordCount = tablesView.getUint32(0, true);
807
869
  if (recordCount !== declaredRecords) {
808
- throw new Error('.pancake corpus count disagrees with manifest');
870
+ throw new Error('.pikelet corpus count disagrees with manifest');
809
871
  }
810
872
  if (perRecord && tablesView.getUint32(4, true) !== pageRecords) {
811
- throw new Error('.pancake corpus page size disagrees with manifest');
873
+ throw new Error('.pikelet corpus page size disagrees with manifest');
812
874
  }
813
875
  const recordOffsets = new Float64Array(recordCount + 1);
814
876
  for (let i = 0; i <= recordCount; i++) {
815
877
  const value = u64(tablesView, offsetsAt + 8 * i, `corpus offset ${i}`);
816
878
  if (value > corpus.length || (i > 0 && value < recordOffsets[i - 1]) || (i === 0 && value !== recordsAt)) {
817
- throw new Error('.pancake corpus offsets are inconsistent');
879
+ throw new Error('.pikelet corpus offsets are inconsistent');
818
880
  }
819
881
  recordOffsets[i] = value;
820
882
  }
@@ -822,7 +884,7 @@ export async function openPancakeFile(input, options = {}) {
822
884
  if (perRecord) {
823
885
  pageTable = tables.slice(offsetsAt + 8 * (recordCount + 1), tablesBytes);
824
886
  if (await sha256hex(pageTable) !== manifest.corpus.pageTableSha256) {
825
- throw new Error('.pancake corpus page table failed hash verification');
887
+ throw new Error('.pikelet corpus page table failed hash verification');
826
888
  }
827
889
  }
828
890
  const corpusIntegrity = perRecord ? (verifyRecords ? 'per-record-sha256' : 'per-record-sha256 (unverified by option)') : 'segment-sha256';
@@ -846,7 +908,7 @@ export async function openPancakeFile(input, options = {}) {
846
908
  `corpus digest page ${page}`, maxReadBytes, fileBytes);
847
909
  if (await sha256hex(bytes) !== toHex(pageTable.subarray(32 * page, 32 * page + 32))) {
848
910
  digestPages.delete(page);
849
- throw new Error(`.pancake corpus digest page ${page} failed hash verification`);
911
+ throw new Error(`.pikelet corpus digest page ${page} failed hash verification`);
850
912
  }
851
913
  return bytes;
852
914
  })();
@@ -869,7 +931,7 @@ export async function openPancakeFile(input, options = {}) {
869
931
 
870
932
  const hydrate = async (id) => {
871
933
  if (!Number.isSafeInteger(id) || id < 0 || id >= recordCount) {
872
- throw new Error(`.pancake result id ${id} is outside the corpus`);
934
+ throw new Error(`.pikelet result id ${id} is outside the corpus`);
873
935
  }
874
936
  if (recordCache.has(id)) {
875
937
  const cached = recordCache.get(id);
@@ -885,15 +947,15 @@ export async function openPancakeFile(input, options = {}) {
885
947
  const digests = await pageDigests(page);
886
948
  const at = 32 * (id - page * pageRecords);
887
949
  if (await sha256hex(bytes) !== toHex(digests.subarray(at, at + 32))) {
888
- throw new Error(`.pancake corpus record ${id} failed integrity verification`);
950
+ throw new Error(`.pikelet corpus record ${id} failed integrity verification`);
889
951
  }
890
952
  }
891
953
  let record;
892
954
  try { record = JSON.parse(decoder.decode(bytes)); } catch (err) {
893
- throw new Error(`.pancake corpus record ${id} is not valid JSON`, { cause: err });
955
+ throw new Error(`.pikelet corpus record ${id} is not valid JSON`, { cause: err });
894
956
  }
895
957
  if (!record || typeof record !== 'object' || Array.isArray(record)) {
896
- throw new Error(`.pancake corpus record ${id} is not a JSON object`);
958
+ throw new Error(`.pikelet corpus record ${id} is not a JSON object`);
897
959
  }
898
960
  return cacheRecord(id, record);
899
961
  };
@@ -903,7 +965,7 @@ export async function openPancakeFile(input, options = {}) {
903
965
  // use after close instead of reading through a freed encoder.
904
966
  let closed = false;
905
967
  const assertOpen = () => {
906
- if (closed) throw new Error('.pancake reader is closed');
968
+ if (closed) throw new Error('.pikelet reader is closed');
907
969
  };
908
970
 
909
971
  return {
@@ -976,6 +1038,8 @@ export async function openPancakeFile(input, options = {}) {
976
1038
  const pre = preScore(context);
977
1039
  let hits = [];
978
1040
  let fused = null;
1041
+ let searched = null;
1042
+ let fusedFull = null;
979
1043
  if (!pre) {
980
1044
  // Hybrid retrieval when the artifact carries a lexical
981
1045
  // segment: the BM25 top matches join the sketch's exact
@@ -1012,8 +1076,8 @@ export async function openPancakeFile(input, options = {}) {
1012
1076
  }
1013
1077
  const lexicalRaw = lexicalIndex && retrieval !== 'vector'
1014
1078
  ? await lexicalIndex.search(trimmed, LEXICAL_CANDIDATES) : [];
1015
- const lexicalHits = lexicalRaw.filter((h) => h.score >= lexicalRaw[0].score / 3);
1016
- const searched = (await sketch.search(context.vector, k, {
1079
+ const lexicalHits = lexicalRaw.filter((h) => h.score >= lexicalRaw[0].score / LEXICAL_CUTOFF);
1080
+ searched = (await sketch.search(context.vector, k, {
1017
1081
  rerank: queryOptions.rerank,
1018
1082
  parallelism: queryOptions.parallelism ?? queryOptions.rerankParallelism ?? options.rerankParallelism,
1019
1083
  // The identity-verified manifest may carry a fetch
@@ -1029,39 +1093,107 @@ export async function openPancakeFile(input, options = {}) {
1029
1093
  // sketch reader ignores it for any query whose C
1030
1094
  // exceeds the scanner's buffers.
1031
1095
  ...(scanner ? { scanner } : {}),
1032
- ...(lexicalHits.length ? {
1033
- extraCandidates: lexicalHits.map((h) => h.id),
1034
- fullRerankOutput: true,
1035
- } : {}),
1096
+ ...(lexicalHits.length ? { extraCandidates: lexicalHits.map((h) => h.id) } : {}),
1097
+ // Always return the full reranked candidate list
1098
+ // (already fetched and exactly scored) rather than
1099
+ // truncating to k: abstention is fit at a fixed
1100
+ // top-10 window (calibrate.mjs) independent of the
1101
+ // caller's k, and needs that full window to score
1102
+ // consistently — see retrieval-abstention.mjs.
1103
+ fullRerankOutput: true,
1036
1104
  })).results;
1037
1105
  hits = searched.slice(0, k);
1038
- if (retrieval === 'augmented') {
1039
- fused = null; // distance order, candidates already augmented
1040
- } else if (retrieval === 'lexical') {
1106
+ // Full-window fused order (not k-truncated) for scoring:
1107
+ // coverage grounds its verdict in the passage fusion
1108
+ // actually ranks first, and calibrate.mjs fits it the
1109
+ // same way — a k-truncated fused list would silently
1110
+ // drop the fused top hit for callers requesting a small k.
1111
+ if (retrieval === 'lexical') {
1041
1112
  const byId = new Map(searched.map((hit) => [hit.id, hit]));
1042
- fused = lexicalHits.map((h) => byId.get(h.id)).filter(Boolean).slice(0, k);
1043
- } else if (lexicalHits.length) {
1113
+ fusedFull = lexicalHits.map((h) => byId.get(h.id)).filter(Boolean);
1114
+ } else if (retrieval !== 'augmented' && lexicalHits.length) {
1044
1115
  const lexRank = new Map(lexicalHits.map((h, i) => [h.id, i]));
1045
- fused = searched
1046
- .map((hit, vRank) => ({
1047
- hit,
1048
- score: 1 / (RRF_K + vRank) + (lexRank.has(hit.id) ? 1 / (RRF_K + lexRank.get(hit.id)) : 0),
1049
- }))
1050
- .sort((a, b) => (b.score - a.score) || (a.hit.distance - b.hit.distance))
1051
- .slice(0, k)
1052
- .map((entry) => entry.hit);
1116
+ fusedFull = fuseCandidates(searched, lexicalHits.map((h) => h.id));
1117
+ // One batched hydration for everything this block and
1118
+ // scoreQuality's coverage term (below) will need, in
1119
+ // place of two separate staged fetches (phrase-pin's
1120
+ // own, then a later cold start for coverage): over a
1121
+ // remote range-read source each hydrate() a caller
1122
+ // hasn't already paid for is its own network round
1123
+ // trip, and sequential/staged awaits pay that cost
1124
+ // once per stage instead of once per query. The
1125
+ // union of phrase-pin's candidates (top lexical
1126
+ // hits) and coverage's (fusedFull's top few — phrase
1127
+ // pinning only ever reorders that window, never
1128
+ // changes its membership) is known right here, so
1129
+ // both stages can share one round trip. A record
1130
+ // that fails to hydrate here does not fail the
1131
+ // query: the id is just missing from the map, and
1132
+ // each site below falls through exactly as it would
1133
+ // on a cold cache miss (recs[i] undefined; coverage
1134
+ // sees fewer usable passages) — hydrate() raising
1135
+ // for a bad id is still a real failure, but only
1136
+ // once a site actually needs that specific record,
1137
+ // not speculatively for every candidate touched here.
1138
+ const phraseCandidates = lexicalHits.slice(0, 3);
1139
+ const coverageCandidates = fusedFull.slice(0, COVERAGE_BATCH_COUNT);
1140
+ const batchIds = [...new Set([...phraseCandidates, ...coverageCandidates].map((h) => h.id))];
1141
+ const batch = new Map();
1142
+ await Promise.all(batchIds.map(async (id) => {
1143
+ try { batch.set(id, await hydrate(id)); } catch { /* see comment above */ }
1144
+ }));
1145
+ // Phrase pinning: a verbatim occurrence of the query (>= 3
1146
+ // content words) in one of the top lexical hits is the
1147
+ // strongest evidence of support there is; rank it first.
1148
+ const norm = (t) => String(t || '').toLowerCase().replace(/[^a-z0-9' ]+/g, ' ').replace(/\s+/g, ' ').trim();
1149
+ const phrase = norm(trimmed);
1150
+ if (phrase.split(' ').length >= 3) {
1151
+ for (const h of phraseCandidates) {
1152
+ if (norm(batch.get(h.id)?.text).includes(phrase)) {
1153
+ fusedFull = [searched.find((x) => x.id === h.id), ...fusedFull.filter((x) => x.id !== h.id)].filter(Boolean);
1154
+ break;
1155
+ }
1156
+ }
1157
+ }
1158
+ // Rank transparency: a consumer can see a verbatim hit sitting
1159
+ // at fused rank 3 behind two semantic near-misses.
1160
+ for (const [i, hit] of fusedFull.entries()) { hit.fusedRank = i + 1; hit.lexicalRank = lexRank.has(hit.id) ? lexRank.get(hit.id) + 1 : null; hit.vectorRank = searched.findIndex((x) => x.id === hit.id) + 1; }
1053
1161
  }
1162
+ fused = retrieval === 'augmented' ? null : fusedFull?.slice(0, k) ?? null;
1054
1163
  }
1055
- const quality = pre || await scoreQuality(hits, context);
1056
- const returned = quality.match_quality === 'none' ? [] : (fused || hits);
1164
+ // Abstention scores the full reranked candidate window
1165
+ // (searched), not the k-truncated hits: calibrate.mjs fits
1166
+ // its signals at a fixed top-10 (top-5 for coverage)
1167
+ // regardless of how many results a caller ultimately
1168
+ // requests, so scoring must use the same window to match.
1169
+ // preScore (kind 1) runs before search and has no window to
1170
+ // fix, so it is unaffected. fusedFull (RRF over the same
1171
+ // full window, not k-truncated) is threaded through
1172
+ // separately so coverage grades the passage fusion actually
1173
+ // ranks first, while d0/margin/mean10 still read vector
1174
+ // order (searched) exactly as calibrated.
1175
+ const quality = pre || await scoreQuality(searched ?? hits, context, fusedFull);
1176
+ // A 'none' verdict withholds results by default — the
1177
+ // calibrated abstention signal is doing its job, and most
1178
+ // callers want that. showAbstained is an explicit opt-out
1179
+ // for a caller that wants the raw retrieval anyway (e.g. to
1180
+ // inspect why the classifier abstained); it never changes
1181
+ // matchQuality or confidence, only whether results ships.
1182
+ const returned = quality.match_quality === 'none' && !queryOptions.showAbstained
1183
+ ? [] : (fused || hits);
1057
1184
  // The search's id and distance are authoritative: they are
1058
1185
  // written last so a corpus record carrying its own `id` or
1059
1186
  // `distance` field cannot overwrite them (reserved names).
1060
1187
  const results = await Promise.all(returned.map(async (hit) => {
1061
1188
  const record = await hydrate(hit.id);
1062
- return { ...record, id: hit.id, distance: hit.distance };
1189
+ return { ...record, id: hit.id, distance: hit.distance, ...(hit.fusedRank ? { fusedRank: hit.fusedRank, lexicalRank: hit.lexicalRank, vectorRank: hit.vectorRank } : {}) };
1063
1190
  }));
1064
- return { matchQuality: quality.match_quality, confidence: quality.confidence, results };
1191
+ // Grounding transparency: which record's text grounded the verdict
1192
+ // and which query words it covered, so a consumer can tell a
1193
+ // topical match ("much money darcy make year") from an answer.
1194
+ const passageOrder = fusedFull?.length ? fusedFull : (searched ?? hits);
1195
+ const grounding = quality.grounding ? { ...quality.grounding, recordId: passageOrder[quality.grounding.passageIndex]?.id ?? null } : null;
1196
+ return { matchQuality: quality.match_quality, confidence: quality.confidence, results, ...(grounding ? { grounding } : {}) };
1065
1197
  },
1066
1198
 
1067
1199
  /** Hydrate one corpus record by id (verified per record on format 2). */
@@ -1087,14 +1219,14 @@ export async function openPancakeFile(input, options = {}) {
1087
1219
  if (!ev) return null;
1088
1220
  const bytes = await readChecked(source, ev.offset, ev.length, 'evaluation segment', maxReadBytes, fileBytes);
1089
1221
  if (await sha256hex(bytes) !== ev.sha256) {
1090
- throw new Error('.pancake evaluation segment failed hash verification');
1222
+ throw new Error('.pikelet evaluation segment failed hash verification');
1091
1223
  }
1092
1224
  let parsed;
1093
1225
  try { parsed = JSON.parse(decoder.decode(bytes)); } catch (err) {
1094
- throw new Error('.pancake evaluation segment is not valid JSON', { cause: err });
1226
+ throw new Error('.pikelet evaluation segment is not valid JSON', { cause: err });
1095
1227
  }
1096
1228
  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
1097
- throw new Error('.pancake evaluation segment must be a JSON object');
1229
+ throw new Error('.pikelet evaluation segment must be a JSON object');
1098
1230
  }
1099
1231
  return parsed;
1100
1232
  },
@@ -1119,6 +1251,3 @@ export async function openPancakeFile(input, options = {}) {
1119
1251
  throw err;
1120
1252
  }
1121
1253
  }
1122
-
1123
- // Documented alias (see the rename note at the top of this file).
1124
- export { openPancakeFile as openPikeletFile };