pikelet-wasm 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +340 -4
- package/NOTICE +5 -0
- package/README.md +226 -475
- package/complete/builder.d.ts +4 -4
- package/complete/builder.mjs +26 -9
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/fusion.mjs +72 -0
- package/complete/index.d.ts +34 -7
- package/complete/index.mjs +258 -129
- package/complete/inline-transformer.mjs +119 -10
- package/complete/lexical.mjs +11 -11
- package/complete/retrieval-abstention.mjs +136 -31
- package/complete/sources.mjs +22 -1
- package/complete/wordpiece.mjs +22 -0
- package/dist/engine.js +1 -1
- package/dist/engine.scalar.js +1 -1
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +47 -82
- package/{pancake-artifact-common.js → src/artifact/pikelet-artifact-common.js} +42 -42
- package/{pancake-artifact-range.js → src/artifact/pikelet-artifact-range.js} +43 -36
- package/{pancake-artifact-sketch.js → src/artifact/pikelet-artifact-sketch.js} +105 -68
- package/{pancake-artifact.d.ts → src/artifact/pikelet-artifact.d.ts} +11 -11
- package/{pancake-artifact.js → src/artifact/pikelet-artifact.js} +10 -10
- package/{pancake-core.js → src/core/pikelet-core.js} +141 -140
- package/{pancake-errors.js → src/errors/pikelet-errors.js} +9 -8
- package/{pancake.d.ts → src/pikelet.d.ts} +55 -55
- package/{pancake.js → src/pikelet.js} +47 -47
- package/{pancake.node.mjs → src/pikelet.node.mjs} +49 -49
- package/{pancake.web.d.ts → src/pikelet.web.d.ts} +13 -13
- package/{pancake.web.mjs → src/pikelet.web.mjs} +24 -24
- package/{pancake.workerd.mjs → src/pikelet.workerd.mjs} +22 -22
- /package/{pancake-loader.js → src/loader/pikelet-loader.js} +0 -0
package/complete/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// The one-file reader: open a .
|
|
1
|
+
// The one-file reader: open a .pikelet (spec/COMPLETE_PROFILE.md) and query
|
|
2
2
|
// it. Environment-neutral — the same module runs in Node (file path or
|
|
3
3
|
// range source) and the browser (HTTP range source, via a bundler for the
|
|
4
4
|
// CJS sketch-reader dependency). The query path needs no core HNSW engine:
|
|
@@ -8,15 +8,15 @@
|
|
|
8
8
|
// engine's SIMD scan kernel (options.sketchScanner — auto-staged in Node,
|
|
9
9
|
// injected in browsers), which only ever changes speed, never results.
|
|
10
10
|
//
|
|
11
|
-
// const search = await
|
|
12
|
-
// const search = await
|
|
11
|
+
// const search = await openPikeletFile('docs.pikelet'); // Node
|
|
12
|
+
// const search = await openPikeletFile(httpRangeSource(url)); // browser
|
|
13
13
|
// const out = await search.query('how do workers restore snapshots');
|
|
14
14
|
//
|
|
15
15
|
// Every byte of the artifact is untrusted input (contract section 7): each
|
|
16
16
|
// offset, length, and count is validated — safe integer, within the header's
|
|
17
17
|
// fileBytes and the source's size, under the per-open read budget — before a
|
|
18
18
|
// read is issued or a buffer allocated, and every read must return exactly
|
|
19
|
-
// the bytes asked for. Format 2 files (profile
|
|
19
|
+
// the bytes asked for. Format 2 files (profile pikelet-complete-v2) carry
|
|
20
20
|
// per-record corpus digests, so each hydrated record is verified on its own
|
|
21
21
|
// range read; format 1 files verify eager segments whole and hydrate records
|
|
22
22
|
// under the segment digest only (reported via info().corpusIntegrity).
|
|
@@ -25,7 +25,7 @@ import { loadStudentModel, embedTextWithStudent } from './student-embedder.mjs';
|
|
|
25
25
|
import { computeMatchQuality, computePreSearchAbstention } from './student-abstention.mjs';
|
|
26
26
|
import { createAbstentionScorer } from './retrieval-abstention.mjs';
|
|
27
27
|
import { openLexicalIndex, openLexicalIndexLazy } from './lexical.mjs';
|
|
28
|
-
import {
|
|
28
|
+
import { PikeletSketchArtifact } from '../src/artifact/pikelet-artifact.js';
|
|
29
29
|
import { MAGIC, HEADER_BYTES, TABLE_ENTRY_BYTES, KINDS, KIND_NAMES } from './format.mjs';
|
|
30
30
|
|
|
31
31
|
import {
|
|
@@ -36,9 +36,13 @@ import {
|
|
|
36
36
|
} from './inline-transformer.mjs';
|
|
37
37
|
|
|
38
38
|
export { httpRangeSource } from './sources.mjs';
|
|
39
|
-
//
|
|
40
|
-
//
|
|
41
|
-
//
|
|
39
|
+
// Re-exported for the builder side: calibration fits coverage against the
|
|
40
|
+
// same fused (vector+BM25) ranking the reader serves, which needs an
|
|
41
|
+
// in-memory lexical searcher over the segment bytes already assembled by
|
|
42
|
+
// buildLexicalSegment, before the artifact is written to disk.
|
|
43
|
+
export { openLexicalIndex } from './lexical.mjs';
|
|
44
|
+
export { fuseCandidates, FUSION_DEFAULTS } from './fusion.mjs';
|
|
45
|
+
import { fuseCandidates } from './fusion.mjs';
|
|
42
46
|
export {
|
|
43
47
|
KERNEL_LAYOUT, expectedBlobBytes, parseInlineTransformerEncoder,
|
|
44
48
|
createInlineTransformerEmbedder, INLINE_TEST_VECTOR_TEXTS,
|
|
@@ -47,7 +51,13 @@ export {
|
|
|
47
51
|
};
|
|
48
52
|
|
|
49
53
|
// Supported container formats: header formatVersion -> manifest profile.
|
|
50
|
-
export const SUPPORTED_PROFILES = Object.freeze({ 1: '
|
|
54
|
+
export const SUPPORTED_PROFILES = Object.freeze({ 1: 'pikelet-complete-v1', 2: 'pikelet-complete-v2' });
|
|
55
|
+
// Pre-rename profile strings (2026-09): every .pikelet published before this
|
|
56
|
+
// date — including live GitHub release assets — carries "pancake-complete-
|
|
57
|
+
// vN" in its manifest. Readers accept both spellings indefinitely so an
|
|
58
|
+
// already-published pack does not go dark on a reader upgrade; only
|
|
59
|
+
// SUPPORTED_PROFILES (above) is what a fresh build ever writes.
|
|
60
|
+
const LEGACY_PROFILES = Object.freeze({ 1: 'pancake-complete-v1', 2: 'pancake-complete-v2' });
|
|
51
61
|
export const CORPUS_LAYOUT_V2 = 'records-v2';
|
|
52
62
|
|
|
53
63
|
// Read budgets. Open-path reads (manifest, segment table, query-interp,
|
|
@@ -63,13 +73,32 @@ const MAX_RECORDS = 2 ** 31 - 1;
|
|
|
63
73
|
const MAX_PAGE_RECORDS = 65536;
|
|
64
74
|
const DIGEST_PAGE_CACHE = 64;
|
|
65
75
|
const RECORD_CACHE = 256;
|
|
66
|
-
// The embedded index is a .
|
|
76
|
+
// The embedded index is a .pikelet-sketch artifact (SKETCH_PROFILE.md);
|
|
67
77
|
// its fixed-size header is what format 2 commits to in the manifest.
|
|
68
78
|
const SKETCH_HEADER_BYTES = 256;
|
|
69
79
|
// Hybrid retrieval: BM25 candidates fetched from the lexical segment per
|
|
70
|
-
// query
|
|
80
|
+
// query; the fusion rule and its constants live in fusion.mjs, shared with
|
|
81
|
+
// the calibrator so fit-time rankings match what is served.
|
|
71
82
|
const LEXICAL_CANDIDATES = 24;
|
|
72
|
-
|
|
83
|
+
// scoreQuality's coverage term hydrates the fused top passagesNeeded
|
|
84
|
+
// records (asset.coverage.topK, 5 by default — see calibrate.mjs's
|
|
85
|
+
// COVERAGE_TOP_PASSAGES) once retrieval returns. The scorer that knows
|
|
86
|
+
// the real number is created inside retrievalScorer()'s own closure, not
|
|
87
|
+
// reachable from here, so this is a fixed approximation used to decide
|
|
88
|
+
// how many of fusedFull's records to include in the batched hydration
|
|
89
|
+
// below, alongside phrase-pinning's own candidates — hydrating a couple
|
|
90
|
+
// more than coverage strictly needs just costs a little extra bandwidth
|
|
91
|
+
// on a cache entry that goes unread; hydrating fewer only narrows how
|
|
92
|
+
// much of coverage's later hydration hits warm cache. Either way is
|
|
93
|
+
// harmless: it's a batch-sizing choice, not a correctness one.
|
|
94
|
+
const COVERAGE_BATCH_COUNT = 5;
|
|
95
|
+
// Only BM25 hits within this fraction of the top lexical score join fusion.
|
|
96
|
+
// Common query terms give every matching document a near-tied score (idf
|
|
97
|
+
// collapses them flat); a wide cutoff let that tied mass ride into fusion on
|
|
98
|
+
// a technicality. Kept identical to pikelet/src/calibrate.mjs's
|
|
99
|
+
// LEXICAL_CUTOFF so calibration is fit against the same fusion the reader
|
|
100
|
+
// actually serves.
|
|
101
|
+
const LEXICAL_CUTOFF = 1.5;
|
|
73
102
|
// Query-interpretation segments above this size open lazily (kind 3's
|
|
74
103
|
// inline encoder is ~25 MiB); smaller ones keep the one-read eager path.
|
|
75
104
|
const LAZY_QI_BYTES = 4 * 1024 * 1024;
|
|
@@ -110,20 +139,30 @@ function toHex(bytes) {
|
|
|
110
139
|
const isSha256Hex = (value) => typeof value === 'string' && /^[0-9a-f]{64}$/.test(value);
|
|
111
140
|
|
|
112
141
|
// SHA-256 via WebCrypto (browser, workerd, Node 18+); node:crypto fallback
|
|
113
|
-
// for older Node
|
|
142
|
+
// for older Node, gated on an actual Node runtime check rather than just
|
|
143
|
+
// "WebCrypto is missing" — a browser can lack crypto.subtle too (privacy
|
|
144
|
+
// settings, a hardened build), and a bare dynamic import('node:crypto')
|
|
145
|
+
// there isn't a Node module resolution failure, it's a cross-origin module
|
|
146
|
+
// fetch a browser refuses outright, surfacing as an opaque module-loading
|
|
147
|
+
// error instead of the clear message below. Fails closed either way:
|
|
148
|
+
// verification is not optional in this reader.
|
|
114
149
|
async function sha256hex(bytes) {
|
|
115
150
|
if (globalThis.crypto?.subtle) {
|
|
116
151
|
return toHex(new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256', bytes)));
|
|
117
152
|
}
|
|
118
|
-
|
|
119
|
-
|
|
153
|
+
if (typeof process !== 'undefined' && process.versions?.node) {
|
|
154
|
+
const { createHash } = await import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:crypto');
|
|
155
|
+
return createHash('sha256').update(bytes).digest('hex');
|
|
156
|
+
}
|
|
157
|
+
throw new Error('No crypto backend available (crypto.subtle is missing) — '
|
|
158
|
+
+ 'cannot verify artifact integrity in this environment');
|
|
120
159
|
}
|
|
121
160
|
|
|
122
161
|
function asUint8Array(bytes) {
|
|
123
162
|
if (bytes instanceof Uint8Array) return bytes;
|
|
124
163
|
if (bytes instanceof ArrayBuffer) return new Uint8Array(bytes);
|
|
125
164
|
if (ArrayBuffer.isView(bytes)) return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
126
|
-
throw new Error('.
|
|
165
|
+
throw new Error('.pikelet range source returned a non-binary value');
|
|
127
166
|
}
|
|
128
167
|
|
|
129
168
|
function resolveBudget(value, label, fallback) {
|
|
@@ -138,7 +177,7 @@ function resolveBudget(value, label, fallback) {
|
|
|
138
177
|
// u64 fields become JS numbers only when they are exactly representable.
|
|
139
178
|
function u64(view, offset, label) {
|
|
140
179
|
const big = view.getBigUint64(offset, true);
|
|
141
|
-
if (big > MAX_SAFE_BIG) throw new Error(`.
|
|
180
|
+
if (big > MAX_SAFE_BIG) throw new Error(`.pikelet ${label} exceeds the safe integer range`);
|
|
142
181
|
return Number(big);
|
|
143
182
|
}
|
|
144
183
|
|
|
@@ -148,16 +187,16 @@ function u64(view, offset, label) {
|
|
|
148
187
|
function checkRange(source, offset, length, label, limit, fileBytes) {
|
|
149
188
|
if (!Number.isSafeInteger(offset) || !Number.isSafeInteger(length)
|
|
150
189
|
|| offset < 0 || length < 0 || !Number.isSafeInteger(offset + length)) {
|
|
151
|
-
throw new Error(`.
|
|
190
|
+
throw new Error(`.pikelet ${label} range is out of bounds`);
|
|
152
191
|
}
|
|
153
192
|
if (length > limit) {
|
|
154
|
-
throw new Error(`.
|
|
193
|
+
throw new Error(`.pikelet ${label} read of ${length} bytes exceeds the ${limit}-byte read budget`);
|
|
155
194
|
}
|
|
156
195
|
if (fileBytes !== undefined && offset + length > fileBytes) {
|
|
157
|
-
throw new Error(`.
|
|
196
|
+
throw new Error(`.pikelet ${label} range extends past the file (${offset}+${length} > ${fileBytes})`);
|
|
158
197
|
}
|
|
159
198
|
if (Number.isSafeInteger(source.size) && offset + length > source.size) {
|
|
160
|
-
throw new Error(`.
|
|
199
|
+
throw new Error(`.pikelet ${label} range extends past the source (${offset}+${length} > ${source.size})`);
|
|
161
200
|
}
|
|
162
201
|
}
|
|
163
202
|
|
|
@@ -165,7 +204,7 @@ async function readChecked(source, offset, length, label, limit, fileBytes) {
|
|
|
165
204
|
checkRange(source, offset, length, label, limit, fileBytes);
|
|
166
205
|
const bytes = asUint8Array(await source.read(offset, length));
|
|
167
206
|
if (bytes.length !== length) {
|
|
168
|
-
throw new Error(`.
|
|
207
|
+
throw new Error(`.pikelet ${label} read returned ${bytes.length} of ${length} bytes (truncated or misbehaving source)`);
|
|
169
208
|
}
|
|
170
209
|
return bytes;
|
|
171
210
|
}
|
|
@@ -185,7 +224,7 @@ async function fileSource(filePath) {
|
|
|
185
224
|
preferredParallelism: Infinity,
|
|
186
225
|
preferredGapBytes: 2048,
|
|
187
226
|
async read(offset, length) {
|
|
188
|
-
if (fd === null) throw new Error('.
|
|
227
|
+
if (fd === null) throw new Error('.pikelet file source is closed');
|
|
189
228
|
const buffer = new Uint8Array(length);
|
|
190
229
|
let bytesRead = 0;
|
|
191
230
|
while (bytesRead < length) {
|
|
@@ -217,7 +256,7 @@ function windowSource(source, offset, length) {
|
|
|
217
256
|
preferredGapBytes: source.preferredGapBytes,
|
|
218
257
|
async read(off, len) {
|
|
219
258
|
if (!Number.isSafeInteger(off) || !Number.isSafeInteger(len) || off < 0 || len < 0 || off + len > length) {
|
|
220
|
-
throw new Error('.
|
|
259
|
+
throw new Error('.pikelet index segment read is outside the segment window');
|
|
221
260
|
}
|
|
222
261
|
return source.read(offset + off, len);
|
|
223
262
|
},
|
|
@@ -257,19 +296,19 @@ function base64Bytes(text) {
|
|
|
257
296
|
export async function verifyHostEncoder(declaration, encodeQuery, dim) {
|
|
258
297
|
const vectors = declaration.testVectors;
|
|
259
298
|
if (!Array.isArray(vectors) || vectors.length === 0) {
|
|
260
|
-
throw new Error('.
|
|
299
|
+
throw new Error('.pikelet kind-2 declaration carries no verification vectors');
|
|
261
300
|
}
|
|
262
301
|
if (declaration.dim !== undefined && declaration.dim !== dim) {
|
|
263
|
-
throw new Error(`.
|
|
302
|
+
throw new Error(`.pikelet kind-2 declaration dim ${declaration.dim} disagrees with manifest dim ${dim}`);
|
|
264
303
|
}
|
|
265
304
|
for (const tv of vectors) {
|
|
266
305
|
if (!tv || typeof tv.text !== 'string') {
|
|
267
|
-
throw new Error('.
|
|
306
|
+
throw new Error('.pikelet kind-2 verification vector is malformed');
|
|
268
307
|
}
|
|
269
308
|
// The expected embedding must be dim finite numbers: a malformed
|
|
270
309
|
// entry would turn every comparison into NaN, and NaN > maxDiff is
|
|
271
310
|
// false — the vector would "pass" without comparing anything.
|
|
272
|
-
const label = `.
|
|
311
|
+
const label = `.pikelet kind-2 verification vector "${tv.text.slice(0, 40)}…"`;
|
|
273
312
|
validateExpectedEmbedding(tv.embedding, dim, label);
|
|
274
313
|
const tolerance = validateTestVectorTolerance(tv.tolerance, label);
|
|
275
314
|
const vector = toFloat32(await encodeQuery(tv.text), dim, 'options.encodeQuery');
|
|
@@ -288,15 +327,15 @@ export async function verifyHostEncoder(declaration, encodeQuery, dim) {
|
|
|
288
327
|
}
|
|
289
328
|
|
|
290
329
|
/**
|
|
291
|
-
* Open a .
|
|
330
|
+
* Open a .pikelet complete artifact from a file path (Node) or a
|
|
292
331
|
* { read(offset, length), size? } range source (any runtime).
|
|
293
332
|
* Returns { query(text, {k}), info(), evaluation(), close() }.
|
|
294
333
|
*/
|
|
295
|
-
export async function
|
|
334
|
+
export async function openPikeletFile(input, options = {}) {
|
|
296
335
|
const source = typeof input === 'string' ? await fileSource(input) : input;
|
|
297
336
|
const owned = typeof input === 'string';
|
|
298
337
|
if (!source || typeof source.read !== 'function') {
|
|
299
|
-
throw new Error('
|
|
338
|
+
throw new Error('openPikeletFile() requires a file path or a range source with read(offset, length)');
|
|
300
339
|
}
|
|
301
340
|
const maxReadBytes = resolveBudget(options.maxReadBytes, 'maxReadBytes', DEFAULT_OPEN_READ_BYTES);
|
|
302
341
|
const maxRecordBytes = resolveBudget(options.maxRecordBytes, 'maxRecordBytes', DEFAULT_RECORD_BYTES);
|
|
@@ -333,23 +372,23 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
333
372
|
const header = await readChecked(source, 0, HEADER_BYTES, 'header', maxReadBytes);
|
|
334
373
|
const headerView = viewOf(header);
|
|
335
374
|
if (headerView.getUint32(0, true) !== MAGIC) {
|
|
336
|
-
throw new Error('not a .
|
|
375
|
+
throw new Error('not a .pikelet file (bad magic)');
|
|
337
376
|
}
|
|
338
377
|
const formatVersion = headerView.getUint32(4, true);
|
|
339
378
|
const profile = SUPPORTED_PROFILES[formatVersion];
|
|
340
|
-
if (!profile) throw new Error(`unsupported .
|
|
379
|
+
if (!profile) throw new Error(`unsupported .pikelet format version ${formatVersion}`);
|
|
341
380
|
const manifestBytes = headerView.getUint32(8, true);
|
|
342
381
|
const segmentCount = headerView.getUint32(12, true);
|
|
343
382
|
const fileBytes = u64(headerView, 16, 'header fileBytes');
|
|
344
383
|
if (manifestBytes < 2 || manifestBytes > MAX_MANIFEST_BYTES || segmentCount < 1 || segmentCount > MAX_SEGMENTS) {
|
|
345
|
-
throw new Error('.
|
|
384
|
+
throw new Error('.pikelet header is implausible');
|
|
346
385
|
}
|
|
347
386
|
if (Number.isSafeInteger(source.size) && source.size !== fileBytes) {
|
|
348
|
-
throw new Error(`.
|
|
387
|
+
throw new Error(`.pikelet is truncated or padded: header says ${fileBytes} bytes, source has ${source.size}`);
|
|
349
388
|
}
|
|
350
389
|
const tableBytes = segmentCount * TABLE_ENTRY_BYTES;
|
|
351
390
|
if (HEADER_BYTES + manifestBytes + tableBytes > fileBytes) {
|
|
352
|
-
throw new Error('.
|
|
391
|
+
throw new Error('.pikelet header places the manifest or segment table past the file');
|
|
353
392
|
}
|
|
354
393
|
const identity = toHex(header.subarray(24, 56));
|
|
355
394
|
// A caller pinning an expected identity (a shelf entry, a lockfile,
|
|
@@ -362,30 +401,31 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
362
401
|
throw new Error('options.expectedIdentity must be a sha256 hex string');
|
|
363
402
|
}
|
|
364
403
|
if (identity !== options.expectedIdentity.toLowerCase()) {
|
|
365
|
-
throw new Error(`.
|
|
404
|
+
throw new Error(`.pikelet identity mismatch: expected ${options.expectedIdentity.toLowerCase()}, found ${identity}`);
|
|
366
405
|
}
|
|
367
406
|
}
|
|
368
407
|
|
|
369
408
|
const manifestBuf = await readChecked(source, HEADER_BYTES, manifestBytes, 'manifest', maxReadBytes, fileBytes);
|
|
370
409
|
if (await sha256hex(manifestBuf) !== identity) {
|
|
371
|
-
throw new Error('.
|
|
410
|
+
throw new Error('.pikelet manifest failed identity verification');
|
|
372
411
|
}
|
|
373
412
|
let manifest;
|
|
374
413
|
try { manifest = JSON.parse(decoder.decode(manifestBuf)); } catch (err) {
|
|
375
|
-
throw new Error('.
|
|
414
|
+
throw new Error('.pikelet manifest is not valid JSON', { cause: err });
|
|
376
415
|
}
|
|
377
|
-
if (!manifest || typeof manifest !== 'object'
|
|
416
|
+
if (!manifest || typeof manifest !== 'object'
|
|
417
|
+
|| (manifest.profile !== profile && manifest.profile !== LEGACY_PROFILES[formatVersion])) {
|
|
378
418
|
throw new Error(`unsupported profile ${manifest?.profile} for format version ${formatVersion}`);
|
|
379
419
|
}
|
|
380
420
|
if (!Array.isArray(manifest.segments) || manifest.segments.length !== segmentCount) {
|
|
381
|
-
throw new Error('.
|
|
421
|
+
throw new Error('.pikelet manifest segment list disagrees with the header segment count');
|
|
382
422
|
}
|
|
383
423
|
const dim = manifest.dim;
|
|
384
424
|
if (!Number.isSafeInteger(dim) || dim < 1 || dim > 65536) {
|
|
385
|
-
throw new Error('.
|
|
425
|
+
throw new Error('.pikelet manifest dim is implausible');
|
|
386
426
|
}
|
|
387
427
|
if (!manifest.corpus || typeof manifest.corpus !== 'object') {
|
|
388
|
-
throw new Error('.
|
|
428
|
+
throw new Error('.pikelet manifest has no corpus block');
|
|
389
429
|
}
|
|
390
430
|
|
|
391
431
|
const table = await readChecked(source, HEADER_BYTES + manifestBytes, tableBytes, 'segment table', maxReadBytes, fileBytes);
|
|
@@ -401,26 +441,26 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
401
441
|
const declared = manifest.segments[i];
|
|
402
442
|
if (!declared || typeof declared !== 'object' || declared.bytes !== length || !isSha256Hex(declared.sha256)
|
|
403
443
|
|| offset !== expectedOffset || !Number.isSafeInteger(offset + length) || offset + length > fileBytes) {
|
|
404
|
-
throw new Error(`.
|
|
444
|
+
throw new Error(`.pikelet segment table disagrees with manifest at entry ${i}`);
|
|
405
445
|
}
|
|
406
446
|
if (kind === undefined) {
|
|
407
447
|
// Unknown kinds are skipped, not failed (spec 3.3); they are
|
|
408
448
|
// still committed through the manifest, so the declared name
|
|
409
449
|
// must at least not claim to be a kind this reader knows.
|
|
410
450
|
if (typeof declared.kind !== 'string' || declared.kind in KINDS) {
|
|
411
|
-
throw new Error(`.
|
|
451
|
+
throw new Error(`.pikelet segment table entry ${i} has unknown kind ${kindNumber} but the manifest names it ${declared.kind}`);
|
|
412
452
|
}
|
|
413
453
|
} else {
|
|
414
454
|
if (declared.kind !== kind) {
|
|
415
|
-
throw new Error(`.
|
|
455
|
+
throw new Error(`.pikelet segment table disagrees with manifest at entry ${i}`);
|
|
416
456
|
}
|
|
417
|
-
if (segments.has(kind)) throw new Error(`.
|
|
457
|
+
if (segments.has(kind)) throw new Error(`.pikelet carries more than one ${kind} segment`);
|
|
418
458
|
segments.set(kind, { offset, length, sha256: declared.sha256 });
|
|
419
459
|
}
|
|
420
460
|
expectedOffset = align16(offset + length);
|
|
421
461
|
}
|
|
422
462
|
for (const required of ['index', 'corpus', 'query-interp']) {
|
|
423
|
-
if (!segments.has(required)) throw new Error(`.
|
|
463
|
+
if (!segments.has(required)) throw new Error(`.pikelet is missing the ${required} segment`);
|
|
424
464
|
}
|
|
425
465
|
|
|
426
466
|
// Corpus tables: count + offsets (+ page table for layout v2),
|
|
@@ -432,15 +472,15 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
432
472
|
const corpus = segments.get('corpus');
|
|
433
473
|
const declaredRecords = manifest.corpus.records;
|
|
434
474
|
if (!Number.isSafeInteger(declaredRecords) || declaredRecords < 0 || declaredRecords > MAX_RECORDS) {
|
|
435
|
-
throw new Error('.
|
|
475
|
+
throw new Error('.pikelet manifest corpus.records is implausible');
|
|
436
476
|
}
|
|
437
477
|
const layout = manifest.corpus.layout;
|
|
438
478
|
const perRecord = formatVersion >= 2;
|
|
439
479
|
if (perRecord && layout !== CORPUS_LAYOUT_V2) {
|
|
440
|
-
throw new Error(`.
|
|
480
|
+
throw new Error(`.pikelet format ${formatVersion} requires corpus layout ${CORPUS_LAYOUT_V2}, manifest says ${layout}`);
|
|
441
481
|
}
|
|
442
482
|
if (!perRecord && layout !== undefined) {
|
|
443
|
-
throw new Error(`.
|
|
483
|
+
throw new Error(`.pikelet format 1 carries corpus layout v1, manifest says ${layout}`);
|
|
444
484
|
}
|
|
445
485
|
let pageRecords = 0;
|
|
446
486
|
let pages = 0;
|
|
@@ -451,12 +491,12 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
451
491
|
if (perRecord) {
|
|
452
492
|
pageRecords = manifest.corpus.pageRecords;
|
|
453
493
|
if (!Number.isSafeInteger(pageRecords) || pageRecords < 1 || pageRecords > MAX_PAGE_RECORDS) {
|
|
454
|
-
throw new Error('.
|
|
494
|
+
throw new Error('.pikelet manifest corpus.pageRecords is implausible');
|
|
455
495
|
}
|
|
456
496
|
pages = Math.ceil(declaredRecords / pageRecords);
|
|
457
497
|
if (manifest.corpus.pages !== pages || !isSha256Hex(manifest.corpus.pageTableSha256)
|
|
458
498
|
|| manifest.corpus.recordDigest !== 'sha256') {
|
|
459
|
-
throw new Error('.
|
|
499
|
+
throw new Error('.pikelet manifest corpus integrity block is inconsistent');
|
|
460
500
|
}
|
|
461
501
|
offsetsAt = 8;
|
|
462
502
|
tablesBytes = offsetsAt + 8 * (declaredRecords + 1) + 32 * pages;
|
|
@@ -468,7 +508,7 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
468
508
|
recordsAt = tablesBytes;
|
|
469
509
|
}
|
|
470
510
|
if (!Number.isSafeInteger(recordsAt) || recordsAt > corpus.length) {
|
|
471
|
-
throw new Error('.
|
|
511
|
+
throw new Error('.pikelet manifest corpus.records is implausible for the corpus segment');
|
|
472
512
|
}
|
|
473
513
|
// Format 2 commits to the embedded sketch's 256-byte header
|
|
474
514
|
// (metric/dim/count and the sketch's residentSha256/vectorsSha256),
|
|
@@ -476,7 +516,7 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
476
516
|
// sketch's self-checks alone verify against fields an attacker who
|
|
477
517
|
// rewrites the segment also controls.
|
|
478
518
|
if (perRecord && (!isSha256Hex(manifest.index?.headerSha256) || idx.length < SKETCH_HEADER_BYTES)) {
|
|
479
|
-
throw new Error('.
|
|
519
|
+
throw new Error('.pikelet manifest carries no index header commitment (index.headerSha256)');
|
|
480
520
|
}
|
|
481
521
|
|
|
482
522
|
// After the segment table, every remaining open-path extent is known,
|
|
@@ -502,11 +542,11 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
502
542
|
}
|
|
503
543
|
const bytes = await readChecked(source, qi.offset, qi.length, 'query-interp segment', maxReadBytes, fileBytes);
|
|
504
544
|
if (await sha256hex(bytes) !== qi.sha256) {
|
|
505
|
-
throw new Error('.
|
|
545
|
+
throw new Error('.pikelet query-interp segment failed hash verification');
|
|
506
546
|
}
|
|
507
547
|
return bytes;
|
|
508
548
|
})(),
|
|
509
|
-
|
|
549
|
+
PikeletSketchArtifact.open(windowSource(source, idx.offset, idx.length), { maxReadBytes }),
|
|
510
550
|
readChecked(source, corpus.offset, tablesBytes, 'corpus tables', maxReadBytes, fileBytes),
|
|
511
551
|
// Lexical index (kind 5, OPTIONAL). Small segments: eager
|
|
512
552
|
// whole-segment read, digest-verified like the query-interp
|
|
@@ -522,14 +562,14 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
522
562
|
}
|
|
523
563
|
const bytes = await readChecked(source, seg.offset, seg.length, 'lexical segment', maxReadBytes, fileBytes);
|
|
524
564
|
if (await sha256hex(bytes) !== seg.sha256) {
|
|
525
|
-
throw new Error('.
|
|
565
|
+
throw new Error('.pikelet lexical segment failed hash verification');
|
|
526
566
|
}
|
|
527
567
|
return { bytes };
|
|
528
568
|
})() : null,
|
|
529
569
|
perRecord ? (async () => {
|
|
530
570
|
const headerBytes = await readChecked(source, idx.offset, SKETCH_HEADER_BYTES, 'index header', maxReadBytes, fileBytes);
|
|
531
571
|
if (await sha256hex(headerBytes) !== manifest.index.headerSha256) {
|
|
532
|
-
throw new Error('.
|
|
572
|
+
throw new Error('.pikelet index header failed hash verification against the manifest');
|
|
533
573
|
}
|
|
534
574
|
})() : null,
|
|
535
575
|
]);
|
|
@@ -538,12 +578,12 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
538
578
|
? (lexicalBytes.lazyIndex || openLexicalIndex(lexicalBytes.bytes))
|
|
539
579
|
: null;
|
|
540
580
|
if (lexicalIndex && lexicalIndex.docCount !== declaredRecords) {
|
|
541
|
-
throw new Error(`.
|
|
581
|
+
throw new Error(`.pikelet lexical segment covers ${lexicalIndex.docCount} records, corpus declares ${declaredRecords}`);
|
|
542
582
|
}
|
|
543
583
|
|
|
544
584
|
// Query interpretation: version word, kind, two length-prefixed
|
|
545
585
|
// regions that must tile the segment exactly.
|
|
546
|
-
if (qi.length < 16) throw new Error('.
|
|
586
|
+
if (qi.length < 16) throw new Error('.pikelet query-interp segment is too short');
|
|
547
587
|
const qiView = viewOf(qiFirst);
|
|
548
588
|
const qiVersion = qiView.getUint32(0, true);
|
|
549
589
|
if (qiVersion !== 1) throw new Error(`unsupported query-interpretation version ${qiVersion}`);
|
|
@@ -551,7 +591,7 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
551
591
|
const encoderLen = qiView.getUint32(8, true);
|
|
552
592
|
const calibrationLen = qiView.getUint32(12, true);
|
|
553
593
|
if (16 + encoderLen + calibrationLen !== qi.length) {
|
|
554
|
-
throw new Error('.
|
|
594
|
+
throw new Error('.pikelet query-interp segment layout is inconsistent');
|
|
555
595
|
}
|
|
556
596
|
// Laziness only pays for kind 3 (the deferrable bulk IS the
|
|
557
597
|
// encoder); a large segment of any other kind falls back to the
|
|
@@ -561,7 +601,7 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
561
601
|
if (qiDeferrable && qiKind !== 3) {
|
|
562
602
|
qiBytes = await readChecked(source, qi.offset, qi.length, 'query-interp segment', maxReadBytes, fileBytes);
|
|
563
603
|
if (await sha256hex(qiBytes) !== qi.sha256) {
|
|
564
|
-
throw new Error('.
|
|
604
|
+
throw new Error('.pikelet query-interp segment failed hash verification');
|
|
565
605
|
}
|
|
566
606
|
qiDeferred = false;
|
|
567
607
|
}
|
|
@@ -574,10 +614,10 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
574
614
|
const encoderBytes = qiDeferred ? null : qiBytes.subarray(16, 16 + encoderLen);
|
|
575
615
|
let calibrationJson;
|
|
576
616
|
try { calibrationJson = JSON.parse(decoder.decode(calibrationRegion)); } catch (err) {
|
|
577
|
-
throw new Error('.
|
|
617
|
+
throw new Error('.pikelet calibration is not valid JSON', { cause: err });
|
|
578
618
|
}
|
|
579
619
|
if (!calibrationJson || typeof calibrationJson !== 'object') {
|
|
580
|
-
throw new Error('.
|
|
620
|
+
throw new Error('.pikelet calibration must be a JSON object');
|
|
581
621
|
}
|
|
582
622
|
|
|
583
623
|
// kind 1 (student-inline-v1): pure-JS inline encoder + feature-stream
|
|
@@ -595,20 +635,42 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
595
635
|
const retrievalScorer = () => {
|
|
596
636
|
const scorer = createAbstentionScorer(calibrationJson.asset, base64Bytes(calibrationJson.vocabBloomBase64));
|
|
597
637
|
const VERDICTS = { answer: 'strong', weak: 'weak', abstain: 'none' };
|
|
598
|
-
return async (hits, context) => {
|
|
638
|
+
return async (hits, context, fusedHits) => {
|
|
599
639
|
if (!scorer) return { match_quality: 'unscored' };
|
|
600
640
|
// The coverage term grounds the verdict in the top passages'
|
|
601
|
-
// text, so those records hydrate before scoring.
|
|
602
|
-
//
|
|
603
|
-
//
|
|
604
|
-
//
|
|
605
|
-
//
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
641
|
+
// text, so those records hydrate before scoring. Hydrated
|
|
642
|
+
// from the fused (RRF) order when hybrid retrieval produced
|
|
643
|
+
// one — calibrate.mjs fits coverage against the same fused
|
|
644
|
+
// ranking, and a chunk BM25 ranks first but the embedding
|
|
645
|
+
// ranks far down is exactly the case coverage exists to
|
|
646
|
+
// catch; scoring it against vector-only order would miss it
|
|
647
|
+
// again. Falls back to vector order (hits) when there is no
|
|
648
|
+
// lexical segment or fusion did not change anything. When
|
|
649
|
+
// the query is answered the pages are already hot for
|
|
650
|
+
// result hydration; an abstained query costs these extra
|
|
651
|
+
// reads. Hydration failures propagate — on format 2 a
|
|
652
|
+
// record that fails its digest must fail the query, not
|
|
653
|
+
// skew its verdict.
|
|
654
|
+
const passageSource = fusedHits?.length ? fusedHits : hits;
|
|
655
|
+
// A record's text starts with its own heading echoed as the
|
|
656
|
+
// first line (ingest.mjs's section-to-chunk join). A
|
|
657
|
+
// heading match still grounds the query but at reduced
|
|
658
|
+
// credit — see retrieval-abstention.mjs's coverageFrac and
|
|
659
|
+
// calibrate.mjs's HEADING_COVERAGE_WEIGHT, which this split
|
|
660
|
+
// must match exactly (build-time and serve-time coverage
|
|
661
|
+
// must agree, or the fitted thresholds are scoring a
|
|
662
|
+
// different signal than the one they were calibrated
|
|
663
|
+
// against).
|
|
664
|
+
const splitHeadingBody = (text) => {
|
|
665
|
+
const nl = String(text || '').indexOf('\n');
|
|
666
|
+
return nl === -1 ? { heading: text, body: '' } : { heading: text.slice(0, nl), body: text.slice(nl + 1) };
|
|
667
|
+
};
|
|
668
|
+
const topTexts = scorer.usesPassage && passageSource.length
|
|
669
|
+
? (await Promise.all(passageSource.slice(0, scorer.passagesNeeded || 1)
|
|
670
|
+
.map((hit) => hydrate(hit.id)))).map((record) => splitHeadingBody(record?.text))
|
|
609
671
|
: [];
|
|
610
|
-
const scored = scorer.score(context.text, hits, topTexts);
|
|
611
|
-
return { match_quality: VERDICTS[scored.verdict] || scored.verdict, confidence: scored.p };
|
|
672
|
+
const scored = await scorer.score(context.text, hits, topTexts);
|
|
673
|
+
return { match_quality: VERDICTS[scored.verdict] || scored.verdict, confidence: scored.p, grounding: scored.grounding || null };
|
|
612
674
|
};
|
|
613
675
|
};
|
|
614
676
|
if (qiKind === 1) {
|
|
@@ -625,9 +687,9 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
625
687
|
} else if (qiKind === 2) {
|
|
626
688
|
let declaration;
|
|
627
689
|
try { declaration = JSON.parse(decoder.decode(encoderBytes)); } catch (err) {
|
|
628
|
-
throw new Error('.
|
|
690
|
+
throw new Error('.pikelet kind-2 encoder declaration is not valid JSON', { cause: err });
|
|
629
691
|
}
|
|
630
|
-
if (!declaration || typeof declaration !== 'object') throw new Error('.
|
|
692
|
+
if (!declaration || typeof declaration !== 'object') throw new Error('.pikelet kind-2 encoder declaration must be a JSON object');
|
|
631
693
|
encoderInfo = declaration;
|
|
632
694
|
const encodeQuery = options.encodeQuery;
|
|
633
695
|
if (encodeQuery !== undefined && typeof encodeQuery !== 'function') {
|
|
@@ -641,7 +703,7 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
641
703
|
} else if (options.allowUnverifiedEncoder === true) {
|
|
642
704
|
encoderVerified = false;
|
|
643
705
|
} else {
|
|
644
|
-
throw new Error(`.
|
|
706
|
+
throw new Error(`.pikelet declares an external encoder (${declaration.model}) without verification vectors, `
|
|
645
707
|
+ 'so the host encoder cannot be checked against it (contract section 4.4); '
|
|
646
708
|
+ 'pass options.allowUnverifiedEncoder: true to serve it anyway, marked unverified');
|
|
647
709
|
}
|
|
@@ -650,8 +712,8 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
650
712
|
}
|
|
651
713
|
embed = async (text) => {
|
|
652
714
|
if (!encodeQuery) {
|
|
653
|
-
throw new Error(`.
|
|
654
|
-
+ 'pass options.encodeQuery to
|
|
715
|
+
throw new Error(`.pikelet declares an external encoder (${declaration.model}); `
|
|
716
|
+
+ 'pass options.encodeQuery to openPikeletFile');
|
|
655
717
|
}
|
|
656
718
|
return { vector: toFloat32(await encodeQuery(text), dim, 'options.encodeQuery'), text };
|
|
657
719
|
};
|
|
@@ -670,7 +732,7 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
670
732
|
} catch (err) {
|
|
671
733
|
throw new Error('kind-3 artifact requires the inline-transformer kernels at '
|
|
672
734
|
+ 'complete/encoder-kernels/ (pikelet-wasm/complete); run '
|
|
673
|
-
+ 'examples/
|
|
735
|
+
+ 'examples/one-file-search/encoder-spike/build-encoder.sh to rebuild them',
|
|
674
736
|
{ cause: err });
|
|
675
737
|
}
|
|
676
738
|
}
|
|
@@ -680,7 +742,7 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
680
742
|
if (qiDeferred) {
|
|
681
743
|
const full = await readChecked(source, qi.offset, qi.length, 'query-interp segment', maxReadBytes, fileBytes);
|
|
682
744
|
if (await sha256hex(full) !== qi.sha256) {
|
|
683
|
-
throw new Error('.
|
|
745
|
+
throw new Error('.pikelet query-interp segment failed hash verification');
|
|
684
746
|
}
|
|
685
747
|
// The header and calibration slices used at open were
|
|
686
748
|
// unverified; the digest-verified segment must contain
|
|
@@ -688,13 +750,13 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
688
750
|
// every query must fail.
|
|
689
751
|
if (!bytesEqual(full.subarray(0, 16), openHeader)
|
|
690
752
|
|| !bytesEqual(full.subarray(16 + encoderLen), calibrationRegion)) {
|
|
691
|
-
throw new Error('.
|
|
753
|
+
throw new Error('.pikelet query-interp segment bytes changed between open and encoder fetch');
|
|
692
754
|
}
|
|
693
755
|
encBytes = full.subarray(16, 16 + encoderLen);
|
|
694
756
|
}
|
|
695
757
|
const { declaration, vocabText, blob } = parseInlineTransformerEncoder(encBytes);
|
|
696
758
|
if (declaration.dim !== undefined && declaration.dim !== dim) {
|
|
697
|
-
throw new Error(`.
|
|
759
|
+
throw new Error(`.pikelet kind-3 declaration dim ${declaration.dim} disagrees with manifest dim ${dim}`);
|
|
698
760
|
}
|
|
699
761
|
const embedder = await createInlineTransformerEmbedder({ declaration, vocabText, blob, createEncoder, verify: verifyEncoder });
|
|
700
762
|
encoderInfo = declaration;
|
|
@@ -732,10 +794,10 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
732
794
|
// Index: the embedded sketch artifact, opened in the wave above
|
|
733
795
|
// against a segment-windowed source (resident hash verified there).
|
|
734
796
|
if (sketch.count !== declaredRecords) {
|
|
735
|
-
throw new Error(`.
|
|
797
|
+
throw new Error(`.pikelet index count ${sketch.count} != corpus records ${declaredRecords}`);
|
|
736
798
|
}
|
|
737
799
|
if (sketch.dim !== undefined && sketch.dim !== dim) {
|
|
738
|
-
throw new Error(`.
|
|
800
|
+
throw new Error(`.pikelet index dim ${sketch.dim} != manifest dim ${dim}`);
|
|
739
801
|
}
|
|
740
802
|
// The metric decides the search semantics; the sketch header's copy
|
|
741
803
|
// must agree with the identity-verified manifest (on format 2 the
|
|
@@ -743,7 +805,7 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
743
805
|
// is the only binding).
|
|
744
806
|
if (METRIC_NAMES[sketch.metric] !== undefined && typeof manifest.metric === 'string'
|
|
745
807
|
&& METRIC_NAMES[sketch.metric] !== manifest.metric) {
|
|
746
|
-
throw new Error(`.
|
|
808
|
+
throw new Error(`.pikelet index metric ${METRIC_NAMES[sketch.metric]} != manifest metric ${manifest.metric}`);
|
|
747
809
|
}
|
|
748
810
|
if (verifyIndexVectors) {
|
|
749
811
|
await sketch.verifyVectors();
|
|
@@ -788,9 +850,9 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
788
850
|
// Hidden from bundlers like the node encoder kernels: the
|
|
789
851
|
// web entrypoint's ?url wasm imports would break a browser
|
|
790
852
|
// build that statically pulled this in.
|
|
791
|
-
const { default:
|
|
853
|
+
const { default: Pikelet } = await import(/* webpackIgnore: true */ /* @vite-ignore */ '../src/pikelet.node.mjs');
|
|
792
854
|
await sketch.fullyResident;
|
|
793
|
-
const created = await
|
|
855
|
+
const created = await Pikelet.createSketchScanner(sketch, {
|
|
794
856
|
maxRerank: Math.min(sketch.count, SCANNER_MAX_RERANK),
|
|
795
857
|
});
|
|
796
858
|
disposeScanner = () => created.dispose();
|
|
@@ -805,16 +867,16 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
805
867
|
const tablesView = viewOf(tables);
|
|
806
868
|
const recordCount = tablesView.getUint32(0, true);
|
|
807
869
|
if (recordCount !== declaredRecords) {
|
|
808
|
-
throw new Error('.
|
|
870
|
+
throw new Error('.pikelet corpus count disagrees with manifest');
|
|
809
871
|
}
|
|
810
872
|
if (perRecord && tablesView.getUint32(4, true) !== pageRecords) {
|
|
811
|
-
throw new Error('.
|
|
873
|
+
throw new Error('.pikelet corpus page size disagrees with manifest');
|
|
812
874
|
}
|
|
813
875
|
const recordOffsets = new Float64Array(recordCount + 1);
|
|
814
876
|
for (let i = 0; i <= recordCount; i++) {
|
|
815
877
|
const value = u64(tablesView, offsetsAt + 8 * i, `corpus offset ${i}`);
|
|
816
878
|
if (value > corpus.length || (i > 0 && value < recordOffsets[i - 1]) || (i === 0 && value !== recordsAt)) {
|
|
817
|
-
throw new Error('.
|
|
879
|
+
throw new Error('.pikelet corpus offsets are inconsistent');
|
|
818
880
|
}
|
|
819
881
|
recordOffsets[i] = value;
|
|
820
882
|
}
|
|
@@ -822,7 +884,7 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
822
884
|
if (perRecord) {
|
|
823
885
|
pageTable = tables.slice(offsetsAt + 8 * (recordCount + 1), tablesBytes);
|
|
824
886
|
if (await sha256hex(pageTable) !== manifest.corpus.pageTableSha256) {
|
|
825
|
-
throw new Error('.
|
|
887
|
+
throw new Error('.pikelet corpus page table failed hash verification');
|
|
826
888
|
}
|
|
827
889
|
}
|
|
828
890
|
const corpusIntegrity = perRecord ? (verifyRecords ? 'per-record-sha256' : 'per-record-sha256 (unverified by option)') : 'segment-sha256';
|
|
@@ -846,7 +908,7 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
846
908
|
`corpus digest page ${page}`, maxReadBytes, fileBytes);
|
|
847
909
|
if (await sha256hex(bytes) !== toHex(pageTable.subarray(32 * page, 32 * page + 32))) {
|
|
848
910
|
digestPages.delete(page);
|
|
849
|
-
throw new Error(`.
|
|
911
|
+
throw new Error(`.pikelet corpus digest page ${page} failed hash verification`);
|
|
850
912
|
}
|
|
851
913
|
return bytes;
|
|
852
914
|
})();
|
|
@@ -869,7 +931,7 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
869
931
|
|
|
870
932
|
const hydrate = async (id) => {
|
|
871
933
|
if (!Number.isSafeInteger(id) || id < 0 || id >= recordCount) {
|
|
872
|
-
throw new Error(`.
|
|
934
|
+
throw new Error(`.pikelet result id ${id} is outside the corpus`);
|
|
873
935
|
}
|
|
874
936
|
if (recordCache.has(id)) {
|
|
875
937
|
const cached = recordCache.get(id);
|
|
@@ -885,15 +947,15 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
885
947
|
const digests = await pageDigests(page);
|
|
886
948
|
const at = 32 * (id - page * pageRecords);
|
|
887
949
|
if (await sha256hex(bytes) !== toHex(digests.subarray(at, at + 32))) {
|
|
888
|
-
throw new Error(`.
|
|
950
|
+
throw new Error(`.pikelet corpus record ${id} failed integrity verification`);
|
|
889
951
|
}
|
|
890
952
|
}
|
|
891
953
|
let record;
|
|
892
954
|
try { record = JSON.parse(decoder.decode(bytes)); } catch (err) {
|
|
893
|
-
throw new Error(`.
|
|
955
|
+
throw new Error(`.pikelet corpus record ${id} is not valid JSON`, { cause: err });
|
|
894
956
|
}
|
|
895
957
|
if (!record || typeof record !== 'object' || Array.isArray(record)) {
|
|
896
|
-
throw new Error(`.
|
|
958
|
+
throw new Error(`.pikelet corpus record ${id} is not a JSON object`);
|
|
897
959
|
}
|
|
898
960
|
return cacheRecord(id, record);
|
|
899
961
|
};
|
|
@@ -903,7 +965,7 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
903
965
|
// use after close instead of reading through a freed encoder.
|
|
904
966
|
let closed = false;
|
|
905
967
|
const assertOpen = () => {
|
|
906
|
-
if (closed) throw new Error('.
|
|
968
|
+
if (closed) throw new Error('.pikelet reader is closed');
|
|
907
969
|
};
|
|
908
970
|
|
|
909
971
|
return {
|
|
@@ -976,6 +1038,8 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
976
1038
|
const pre = preScore(context);
|
|
977
1039
|
let hits = [];
|
|
978
1040
|
let fused = null;
|
|
1041
|
+
let searched = null;
|
|
1042
|
+
let fusedFull = null;
|
|
979
1043
|
if (!pre) {
|
|
980
1044
|
// Hybrid retrieval when the artifact carries a lexical
|
|
981
1045
|
// segment: the BM25 top matches join the sketch's exact
|
|
@@ -1012,8 +1076,8 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
1012
1076
|
}
|
|
1013
1077
|
const lexicalRaw = lexicalIndex && retrieval !== 'vector'
|
|
1014
1078
|
? await lexicalIndex.search(trimmed, LEXICAL_CANDIDATES) : [];
|
|
1015
|
-
const lexicalHits = lexicalRaw.filter((h) => h.score >= lexicalRaw[0].score /
|
|
1016
|
-
|
|
1079
|
+
const lexicalHits = lexicalRaw.filter((h) => h.score >= lexicalRaw[0].score / LEXICAL_CUTOFF);
|
|
1080
|
+
searched = (await sketch.search(context.vector, k, {
|
|
1017
1081
|
rerank: queryOptions.rerank,
|
|
1018
1082
|
parallelism: queryOptions.parallelism ?? queryOptions.rerankParallelism ?? options.rerankParallelism,
|
|
1019
1083
|
// The identity-verified manifest may carry a fetch
|
|
@@ -1029,39 +1093,107 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
1029
1093
|
// sketch reader ignores it for any query whose C
|
|
1030
1094
|
// exceeds the scanner's buffers.
|
|
1031
1095
|
...(scanner ? { scanner } : {}),
|
|
1032
|
-
...(lexicalHits.length ? {
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1096
|
+
...(lexicalHits.length ? { extraCandidates: lexicalHits.map((h) => h.id) } : {}),
|
|
1097
|
+
// Always return the full reranked candidate list
|
|
1098
|
+
// (already fetched and exactly scored) rather than
|
|
1099
|
+
// truncating to k: abstention is fit at a fixed
|
|
1100
|
+
// top-10 window (calibrate.mjs) independent of the
|
|
1101
|
+
// caller's k, and needs that full window to score
|
|
1102
|
+
// consistently — see retrieval-abstention.mjs.
|
|
1103
|
+
fullRerankOutput: true,
|
|
1036
1104
|
})).results;
|
|
1037
1105
|
hits = searched.slice(0, k);
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1106
|
+
// Full-window fused order (not k-truncated) for scoring:
|
|
1107
|
+
// coverage grounds its verdict in the passage fusion
|
|
1108
|
+
// actually ranks first, and calibrate.mjs fits it the
|
|
1109
|
+
// same way — a k-truncated fused list would silently
|
|
1110
|
+
// drop the fused top hit for callers requesting a small k.
|
|
1111
|
+
if (retrieval === 'lexical') {
|
|
1041
1112
|
const byId = new Map(searched.map((hit) => [hit.id, hit]));
|
|
1042
|
-
|
|
1043
|
-
} else if (lexicalHits.length) {
|
|
1113
|
+
fusedFull = lexicalHits.map((h) => byId.get(h.id)).filter(Boolean);
|
|
1114
|
+
} else if (retrieval !== 'augmented' && lexicalHits.length) {
|
|
1044
1115
|
const lexRank = new Map(lexicalHits.map((h, i) => [h.id, i]));
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1116
|
+
fusedFull = fuseCandidates(searched, lexicalHits.map((h) => h.id));
|
|
1117
|
+
// One batched hydration for everything this block and
|
|
1118
|
+
// scoreQuality's coverage term (below) will need, in
|
|
1119
|
+
// place of two separate staged fetches (phrase-pin's
|
|
1120
|
+
// own, then a later cold start for coverage): over a
|
|
1121
|
+
// remote range-read source each hydrate() a caller
|
|
1122
|
+
// hasn't already paid for is its own network round
|
|
1123
|
+
// trip, and sequential/staged awaits pay that cost
|
|
1124
|
+
// once per stage instead of once per query. The
|
|
1125
|
+
// union of phrase-pin's candidates (top lexical
|
|
1126
|
+
// hits) and coverage's (fusedFull's top few — phrase
|
|
1127
|
+
// pinning only ever reorders that window, never
|
|
1128
|
+
// changes its membership) is known right here, so
|
|
1129
|
+
// both stages can share one round trip. A record
|
|
1130
|
+
// that fails to hydrate here does not fail the
|
|
1131
|
+
// query: the id is just missing from the map, and
|
|
1132
|
+
// each site below falls through exactly as it would
|
|
1133
|
+
// on a cold cache miss (recs[i] undefined; coverage
|
|
1134
|
+
// sees fewer usable passages) — hydrate() raising
|
|
1135
|
+
// for a bad id is still a real failure, but only
|
|
1136
|
+
// once a site actually needs that specific record,
|
|
1137
|
+
// not speculatively for every candidate touched here.
|
|
1138
|
+
const phraseCandidates = lexicalHits.slice(0, 3);
|
|
1139
|
+
const coverageCandidates = fusedFull.slice(0, COVERAGE_BATCH_COUNT);
|
|
1140
|
+
const batchIds = [...new Set([...phraseCandidates, ...coverageCandidates].map((h) => h.id))];
|
|
1141
|
+
const batch = new Map();
|
|
1142
|
+
await Promise.all(batchIds.map(async (id) => {
|
|
1143
|
+
try { batch.set(id, await hydrate(id)); } catch { /* see comment above */ }
|
|
1144
|
+
}));
|
|
1145
|
+
// Phrase pinning: a verbatim occurrence of the query (>= 3
|
|
1146
|
+
// content words) in one of the top lexical hits is the
|
|
1147
|
+
// strongest evidence of support there is; rank it first.
|
|
1148
|
+
const norm = (t) => String(t || '').toLowerCase().replace(/[^a-z0-9' ]+/g, ' ').replace(/\s+/g, ' ').trim();
|
|
1149
|
+
const phrase = norm(trimmed);
|
|
1150
|
+
if (phrase.split(' ').length >= 3) {
|
|
1151
|
+
for (const h of phraseCandidates) {
|
|
1152
|
+
if (norm(batch.get(h.id)?.text).includes(phrase)) {
|
|
1153
|
+
fusedFull = [searched.find((x) => x.id === h.id), ...fusedFull.filter((x) => x.id !== h.id)].filter(Boolean);
|
|
1154
|
+
break;
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
// Rank transparency: a consumer can see a verbatim hit sitting
|
|
1159
|
+
// at fused rank 3 behind two semantic near-misses.
|
|
1160
|
+
for (const [i, hit] of fusedFull.entries()) { hit.fusedRank = i + 1; hit.lexicalRank = lexRank.has(hit.id) ? lexRank.get(hit.id) + 1 : null; hit.vectorRank = searched.findIndex((x) => x.id === hit.id) + 1; }
|
|
1053
1161
|
}
|
|
1162
|
+
fused = retrieval === 'augmented' ? null : fusedFull?.slice(0, k) ?? null;
|
|
1054
1163
|
}
|
|
1055
|
-
|
|
1056
|
-
|
|
1164
|
+
// Abstention scores the full reranked candidate window
|
|
1165
|
+
// (searched), not the k-truncated hits: calibrate.mjs fits
|
|
1166
|
+
// its signals at a fixed top-10 (top-5 for coverage)
|
|
1167
|
+
// regardless of how many results a caller ultimately
|
|
1168
|
+
// requests, so scoring must use the same window to match.
|
|
1169
|
+
// preScore (kind 1) runs before search and has no window to
|
|
1170
|
+
// fix, so it is unaffected. fusedFull (RRF over the same
|
|
1171
|
+
// full window, not k-truncated) is threaded through
|
|
1172
|
+
// separately so coverage grades the passage fusion actually
|
|
1173
|
+
// ranks first, while d0/margin/mean10 still read vector
|
|
1174
|
+
// order (searched) exactly as calibrated.
|
|
1175
|
+
const quality = pre || await scoreQuality(searched ?? hits, context, fusedFull);
|
|
1176
|
+
// A 'none' verdict withholds results by default — the
|
|
1177
|
+
// calibrated abstention signal is doing its job, and most
|
|
1178
|
+
// callers want that. showAbstained is an explicit opt-out
|
|
1179
|
+
// for a caller that wants the raw retrieval anyway (e.g. to
|
|
1180
|
+
// inspect why the classifier abstained); it never changes
|
|
1181
|
+
// matchQuality or confidence, only whether results ships.
|
|
1182
|
+
const returned = quality.match_quality === 'none' && !queryOptions.showAbstained
|
|
1183
|
+
? [] : (fused || hits);
|
|
1057
1184
|
// The search's id and distance are authoritative: they are
|
|
1058
1185
|
// written last so a corpus record carrying its own `id` or
|
|
1059
1186
|
// `distance` field cannot overwrite them (reserved names).
|
|
1060
1187
|
const results = await Promise.all(returned.map(async (hit) => {
|
|
1061
1188
|
const record = await hydrate(hit.id);
|
|
1062
|
-
return { ...record, id: hit.id, distance: hit.distance };
|
|
1189
|
+
return { ...record, id: hit.id, distance: hit.distance, ...(hit.fusedRank ? { fusedRank: hit.fusedRank, lexicalRank: hit.lexicalRank, vectorRank: hit.vectorRank } : {}) };
|
|
1063
1190
|
}));
|
|
1064
|
-
|
|
1191
|
+
// Grounding transparency: which record's text grounded the verdict
|
|
1192
|
+
// and which query words it covered, so a consumer can tell a
|
|
1193
|
+
// topical match ("much money darcy make year") from an answer.
|
|
1194
|
+
const passageOrder = fusedFull?.length ? fusedFull : (searched ?? hits);
|
|
1195
|
+
const grounding = quality.grounding ? { ...quality.grounding, recordId: passageOrder[quality.grounding.passageIndex]?.id ?? null } : null;
|
|
1196
|
+
return { matchQuality: quality.match_quality, confidence: quality.confidence, results, ...(grounding ? { grounding } : {}) };
|
|
1065
1197
|
},
|
|
1066
1198
|
|
|
1067
1199
|
/** Hydrate one corpus record by id (verified per record on format 2). */
|
|
@@ -1087,14 +1219,14 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
1087
1219
|
if (!ev) return null;
|
|
1088
1220
|
const bytes = await readChecked(source, ev.offset, ev.length, 'evaluation segment', maxReadBytes, fileBytes);
|
|
1089
1221
|
if (await sha256hex(bytes) !== ev.sha256) {
|
|
1090
|
-
throw new Error('.
|
|
1222
|
+
throw new Error('.pikelet evaluation segment failed hash verification');
|
|
1091
1223
|
}
|
|
1092
1224
|
let parsed;
|
|
1093
1225
|
try { parsed = JSON.parse(decoder.decode(bytes)); } catch (err) {
|
|
1094
|
-
throw new Error('.
|
|
1226
|
+
throw new Error('.pikelet evaluation segment is not valid JSON', { cause: err });
|
|
1095
1227
|
}
|
|
1096
1228
|
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
1097
|
-
throw new Error('.
|
|
1229
|
+
throw new Error('.pikelet evaluation segment must be a JSON object');
|
|
1098
1230
|
}
|
|
1099
1231
|
return parsed;
|
|
1100
1232
|
},
|
|
@@ -1119,6 +1251,3 @@ export async function openPancakeFile(input, options = {}) {
|
|
|
1119
1251
|
throw err;
|
|
1120
1252
|
}
|
|
1121
1253
|
}
|
|
1122
|
-
|
|
1123
|
-
// Documented alias (see the rename note at the top of this file).
|
|
1124
|
-
export { openPancakeFile as openPikeletFile };
|