pikelet-wasm 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +340 -4
- package/NOTICE +5 -0
- package/README.md +226 -475
- package/complete/builder.d.ts +4 -4
- package/complete/builder.mjs +26 -9
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/fusion.mjs +72 -0
- package/complete/index.d.ts +34 -7
- package/complete/index.mjs +258 -129
- package/complete/inline-transformer.mjs +119 -10
- package/complete/lexical.mjs +11 -11
- package/complete/retrieval-abstention.mjs +136 -31
- package/complete/sources.mjs +22 -1
- package/complete/wordpiece.mjs +22 -0
- package/dist/engine.js +1 -1
- package/dist/engine.scalar.js +1 -1
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +47 -82
- package/{pancake-artifact-common.js → src/artifact/pikelet-artifact-common.js} +42 -42
- package/{pancake-artifact-range.js → src/artifact/pikelet-artifact-range.js} +43 -36
- package/{pancake-artifact-sketch.js → src/artifact/pikelet-artifact-sketch.js} +105 -68
- package/{pancake-artifact.d.ts → src/artifact/pikelet-artifact.d.ts} +11 -11
- package/{pancake-artifact.js → src/artifact/pikelet-artifact.js} +10 -10
- package/{pancake-core.js → src/core/pikelet-core.js} +141 -140
- package/{pancake-errors.js → src/errors/pikelet-errors.js} +9 -8
- package/{pancake.d.ts → src/pikelet.d.ts} +55 -55
- package/{pancake.js → src/pikelet.js} +47 -47
- package/{pancake.node.mjs → src/pikelet.node.mjs} +49 -49
- package/{pancake.web.d.ts → src/pikelet.web.d.ts} +13 -13
- package/{pancake.web.mjs → src/pikelet.web.mjs} +24 -24
- package/{pancake.workerd.mjs → src/pikelet.workerd.mjs} +22 -22
- /package/{pancake-loader.js → src/loader/pikelet-loader.js} +0 -0
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
'use strict';
|
|
2
|
-
// Sketch artifact profile (.
|
|
3
|
-
//
|
|
2
|
+
// Sketch artifact profile (.pikelet-sketch) — spec/SKETCH_PROFILE.md:
|
|
3
|
+
// PikeletSketchArtifact reader, the sketch-artifact builders, and the
|
|
4
4
|
// engine-backed resident scanner.
|
|
5
|
-
// Split out of
|
|
5
|
+
// Split out of pikelet-artifact.js (the public entry, which re-exports the
|
|
6
6
|
// three parts); see that file for the module map.
|
|
7
7
|
|
|
8
|
-
const {
|
|
8
|
+
const { pikeletError, PIKELET_ERROR_CODES } = require('../errors/pikelet-errors.js');
|
|
9
9
|
const {
|
|
10
10
|
DEFAULT_OPEN_READ_BYTES,
|
|
11
11
|
MAX_COALESCED_RANGE_BYTES,
|
|
@@ -18,15 +18,15 @@ const {
|
|
|
18
18
|
sha256Bytes,
|
|
19
19
|
sha256BytesAsync,
|
|
20
20
|
verifySegmentSha256,
|
|
21
|
-
} = require('./
|
|
21
|
+
} = require('./pikelet-artifact-common.js');
|
|
22
22
|
const {
|
|
23
23
|
normalizeQuery: normalizeQueryVector,
|
|
24
24
|
resolveSearchK,
|
|
25
25
|
resolveOptionalPositiveInt,
|
|
26
|
-
} = require('./
|
|
26
|
+
} = require('./pikelet-artifact-common.js');
|
|
27
27
|
|
|
28
28
|
// ============================================================================
|
|
29
|
-
// Sketch artifact profile (.
|
|
29
|
+
// Sketch artifact profile (.pikelet-sketch) — spec/SKETCH_PROFILE.md
|
|
30
30
|
// ============================================================================
|
|
31
31
|
//
|
|
32
32
|
// Layout: 256-byte header | scales f32[count] | offsets f32[count] |
|
|
@@ -38,9 +38,22 @@ const SKETCH_MAGIC = 0x31415350; // PSA1
|
|
|
38
38
|
const SKETCH_HEADER_BYTES = 256;
|
|
39
39
|
const SKETCH_KIND_U8 = 1;
|
|
40
40
|
|
|
41
|
+
// The affine table feeds every rerank distance; a NaN/Infinity entry makes
|
|
42
|
+
// that row's distance non-finite and lets it sit in the result set
|
|
43
|
+
// unchallenged (NaN never loses a comparison). The digests do not catch
|
|
44
|
+
// this, since the producer signs whatever bytes it wrote, so every path
|
|
45
|
+
// that adopts the table checks it.
|
|
46
|
+
function assertFiniteAffine(scales, offsets) {
|
|
47
|
+
for (let i = 0; i < scales.length; i++) {
|
|
48
|
+
if (!Number.isFinite(scales[i]) || !Number.isFinite(offsets[i])) {
|
|
49
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact affine table has a non-finite scale or offset', { id: i });
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
41
54
|
function buildSketchArtifact(snapshotBytes, outPath, options = {}) {
|
|
42
55
|
if (typeof outPath !== 'string' || outPath.length === 0) {
|
|
43
|
-
throw
|
|
56
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'buildSketchArtifact() requires an output path');
|
|
44
57
|
}
|
|
45
58
|
return exportSketchArtifact(parseUint8Snapshot(snapshotBytes), outPath, options);
|
|
46
59
|
}
|
|
@@ -55,13 +68,13 @@ function exportSketchArtifact(index, outPath, options = {}) {
|
|
|
55
68
|
const sketchBits = options.sketchBits || 4;
|
|
56
69
|
const recommendedRerank = options.recommendedRerank || 0;
|
|
57
70
|
if (![4, 8].includes(sketchBits)) {
|
|
58
|
-
throw
|
|
71
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'sketchBits must be 4 or 8', { sketchBits });
|
|
59
72
|
}
|
|
60
73
|
if (!Number.isInteger(sketchDims) || sketchDims < 1 || dim % sketchDims !== 0) {
|
|
61
|
-
throw
|
|
74
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'sketchDims must divide dim', { sketchDims, dim });
|
|
62
75
|
}
|
|
63
76
|
if (sketchBits === 4 && sketchDims % 2 !== 0) {
|
|
64
|
-
throw
|
|
77
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'sketchDims must be even for 4-bit sketches', { sketchDims });
|
|
65
78
|
}
|
|
66
79
|
// Optional staged-boot micro tier: a second, coarser pooling of the same
|
|
67
80
|
// quantized rows, stored after the full sketches so v1 readers see it
|
|
@@ -78,10 +91,10 @@ function exportSketchArtifact(index, outPath, options = {}) {
|
|
|
78
91
|
const microBits = microDims ? (options.microBits || 4) : 0;
|
|
79
92
|
if (microDims) {
|
|
80
93
|
if (!Number.isInteger(microDims) || microDims < 1 || sketchDims % microDims !== 0 || microDims >= sketchDims) {
|
|
81
|
-
throw
|
|
94
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'microDims must divide sketchDims and be smaller', { microDims, sketchDims });
|
|
82
95
|
}
|
|
83
96
|
if (![4, 8].includes(microBits) || (microBits === 4 && microDims % 2 !== 0)) {
|
|
84
|
-
throw
|
|
97
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'invalid micro sketch encoding', { microBits, microDims });
|
|
85
98
|
}
|
|
86
99
|
}
|
|
87
100
|
// Per-row integrity (format version 2, spec section 2.4): the vectors
|
|
@@ -97,10 +110,20 @@ function exportSketchArtifact(index, outPath, options = {}) {
|
|
|
97
110
|
const rowDigestBytes = options.rowDigestBytes || 16;
|
|
98
111
|
if (rowIntegrity) {
|
|
99
112
|
if (!Number.isInteger(rowsPerBlock) || rowsPerBlock < 1 || rowsPerBlock > 4096) {
|
|
100
|
-
throw
|
|
113
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'rowsPerBlock must be an integer in [1, 4096]', { rowsPerBlock });
|
|
101
114
|
}
|
|
102
|
-
|
|
103
|
-
|
|
115
|
+
// Floor is 16, not the format's 8: the per-row digest is truncated
|
|
116
|
+
// SHA-256, and the claim it backs is that nobody can substitute a
|
|
117
|
+
// different row with the same digest — a second-preimage problem at
|
|
118
|
+
// 8*rowDigestBytes bits. At 8 bytes that's 2^64, a fraction of a
|
|
119
|
+
// second of the Bitcoin network's hash rate; not tamper evidence
|
|
120
|
+
// against anyone who cares. At 16 it's 2^128. The saving from 8 vs
|
|
121
|
+
// 16 bytes on a typical row is a couple percent, so there is no
|
|
122
|
+
// reason to offer the weak end of the range. Readers still accept
|
|
123
|
+
// down to 8 (spec's read-path floor, format.md) for artifacts built
|
|
124
|
+
// before this floor was raised.
|
|
125
|
+
if (!Number.isInteger(rowDigestBytes) || rowDigestBytes < 16 || rowDigestBytes > 32) {
|
|
126
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'rowDigestBytes must be an integer in [16, 32]', { rowDigestBytes });
|
|
104
127
|
}
|
|
105
128
|
}
|
|
106
129
|
const pool = dim / sketchDims;
|
|
@@ -223,7 +246,7 @@ function exportSketchArtifact(index, outPath, options = {}) {
|
|
|
223
246
|
}
|
|
224
247
|
|
|
225
248
|
const manifest = {
|
|
226
|
-
format: '
|
|
249
|
+
format: 'pikelet-sketch-artifact',
|
|
227
250
|
formatVersion: rowIntegrity ? 2 : 1,
|
|
228
251
|
sizeBytes: fileBytes,
|
|
229
252
|
metric: index.metric === 1 ? 'cosine' : 'l2',
|
|
@@ -256,7 +279,7 @@ function buildSketchArtifactFile(snapshotPath, outPath, options = {}) {
|
|
|
256
279
|
return buildSketchArtifact(fs.readFileSync(snapshotPath), outPath, options);
|
|
257
280
|
}
|
|
258
281
|
|
|
259
|
-
class
|
|
282
|
+
class PikeletSketchArtifact {
|
|
260
283
|
constructor(source) {
|
|
261
284
|
this.source = source;
|
|
262
285
|
// Fetched rows live in a byte-budgeted LRU; search correctness never
|
|
@@ -300,25 +323,25 @@ class PancakeSketchArtifact {
|
|
|
300
323
|
|
|
301
324
|
static async open(source, options = {}) {
|
|
302
325
|
if (!source || typeof source.read !== 'function') {
|
|
303
|
-
throw
|
|
326
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'SketchArtifact.open() requires a range source with read(offset, length)');
|
|
304
327
|
}
|
|
305
|
-
const artifact = new
|
|
328
|
+
const artifact = new PikeletSketchArtifact(source);
|
|
306
329
|
artifact.maxReadBytes = resolveMaxReadBytes(options.maxReadBytes);
|
|
307
330
|
const header = await readChecked(source, 0, SKETCH_HEADER_BYTES, 'header');
|
|
308
331
|
if (header.byteLength !== SKETCH_HEADER_BYTES) {
|
|
309
|
-
throw
|
|
332
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact header is truncated');
|
|
310
333
|
}
|
|
311
334
|
const view = new DataView(header.buffer, header.byteOffset, header.byteLength);
|
|
312
335
|
if (view.getUint32(0, true) !== SKETCH_MAGIC) {
|
|
313
|
-
throw
|
|
336
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Not a Pikelet sketch artifact (bad magic)');
|
|
314
337
|
}
|
|
315
338
|
const version = view.getUint32(4, true);
|
|
316
339
|
if (version !== 1 && version !== 2) {
|
|
317
|
-
throw
|
|
340
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported sketch artifact version', { version });
|
|
318
341
|
}
|
|
319
342
|
artifact.formatVersion = version;
|
|
320
343
|
if (view.getUint32(8, true) !== SKETCH_KIND_U8) {
|
|
321
|
-
throw
|
|
344
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported sketch artifact kind');
|
|
322
345
|
}
|
|
323
346
|
artifact.metric = view.getUint32(12, true);
|
|
324
347
|
artifact.dim = view.getUint32(16, true);
|
|
@@ -344,13 +367,13 @@ class PancakeSketchArtifact {
|
|
|
344
367
|
|
|
345
368
|
const { metric, dim, count, sketchDims, sketchBits, vectorsOffset } = artifact;
|
|
346
369
|
if (metric !== 0 && metric !== 1) {
|
|
347
|
-
throw
|
|
370
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported sketch artifact metric', { metric });
|
|
348
371
|
}
|
|
349
372
|
if (dim < 1 || count < 1 || sketchDims < 1 || dim % sketchDims !== 0) {
|
|
350
|
-
throw
|
|
373
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Invalid sketch artifact geometry', { dim, count, sketchDims });
|
|
351
374
|
}
|
|
352
375
|
if (![4, 8].includes(sketchBits) || (sketchBits === 4 && sketchDims % 2 !== 0)) {
|
|
353
|
-
throw
|
|
376
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Invalid sketch encoding', { sketchBits, sketchDims });
|
|
354
377
|
}
|
|
355
378
|
const sketchRowBytes = (sketchDims * sketchBits) / 8;
|
|
356
379
|
// Version 2 interleaves a digest page ahead of each rowsPerBlock rows
|
|
@@ -360,7 +383,7 @@ class PancakeSketchArtifact {
|
|
|
360
383
|
const P = artifact.rowsPerBlock;
|
|
361
384
|
const D = artifact.rowDigestBytes;
|
|
362
385
|
if (!Number.isInteger(P) || P < 1 || P > 4096 || !Number.isInteger(D) || D < 8 || D > 32) {
|
|
363
|
-
throw
|
|
386
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Invalid sketch row-integrity geometry', { rowsPerBlock: P, rowDigestBytes: D });
|
|
364
387
|
}
|
|
365
388
|
artifact.numBlocks = Math.ceil(count / P);
|
|
366
389
|
artifact.pageBytes = P * D;
|
|
@@ -370,7 +393,7 @@ class PancakeSketchArtifact {
|
|
|
370
393
|
if (!Number.isSafeInteger(artifact.vectorsRegionBytes)
|
|
371
394
|
|| pageTableOffset < sketchesOffset + count * sketchRowBytes
|
|
372
395
|
|| pageTableOffset + artifact.numBlocks * 32 > vectorsOffset) {
|
|
373
|
-
throw
|
|
396
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact page table layout is inconsistent', { pageTableOffset });
|
|
374
397
|
}
|
|
375
398
|
} else {
|
|
376
399
|
artifact.numBlocks = 0;
|
|
@@ -384,7 +407,7 @@ class PancakeSketchArtifact {
|
|
|
384
407
|
|| vectorsOffset < sketchesOffset + count * sketchRowBytes
|
|
385
408
|
|| vectorsOffset % 16 !== 0
|
|
386
409
|
|| fileBytes !== vectorsOffset + expectVectors) {
|
|
387
|
-
throw
|
|
410
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact layout is inconsistent');
|
|
388
411
|
}
|
|
389
412
|
let microRowBytes = 0;
|
|
390
413
|
if (artifact.microDims) {
|
|
@@ -393,7 +416,7 @@ class PancakeSketchArtifact {
|
|
|
393
416
|
|| ![4, 8].includes(artifact.microBits) || (artifact.microBits === 4 && artifact.microDims % 2 !== 0)
|
|
394
417
|
|| microOffset !== sketchesOffset + count * sketchRowBytes
|
|
395
418
|
|| microOffset + count * microRowBytes > vectorsOffset) {
|
|
396
|
-
throw
|
|
419
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact micro tier layout is inconsistent', { microDims: artifact.microDims });
|
|
397
420
|
}
|
|
398
421
|
}
|
|
399
422
|
artifact.microOffset = microOffset;
|
|
@@ -402,7 +425,7 @@ class PancakeSketchArtifact {
|
|
|
402
425
|
// still inside the resident prefix residentSha256 covers.
|
|
403
426
|
const tiersEnd = artifact.microDims ? microOffset + count * microRowBytes : sketchesOffset + count * sketchRowBytes;
|
|
404
427
|
if (pageTableOffset !== tiersEnd) {
|
|
405
|
-
throw
|
|
428
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact page table layout is inconsistent', { pageTableOffset, tiersEnd });
|
|
406
429
|
}
|
|
407
430
|
}
|
|
408
431
|
artifact.pageTableOffset = pageTableOffset;
|
|
@@ -418,7 +441,7 @@ class PancakeSketchArtifact {
|
|
|
418
441
|
if (!staged) {
|
|
419
442
|
const resident = await readChecked(source, SKETCH_HEADER_BYTES, vectorsOffset - SKETCH_HEADER_BYTES, 'resident prefix', artifact.maxReadBytes);
|
|
420
443
|
if (resident.byteLength !== vectorsOffset - SKETCH_HEADER_BYTES) {
|
|
421
|
-
throw
|
|
444
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact resident prefix is truncated');
|
|
422
445
|
}
|
|
423
446
|
// Copy into an aligned buffer so the typed-array views are valid
|
|
424
447
|
// regardless of the source's byteOffset.
|
|
@@ -447,7 +470,7 @@ class PancakeSketchArtifact {
|
|
|
447
470
|
]);
|
|
448
471
|
if (affineBytes.byteLength !== sketchesOffset - SKETCH_HEADER_BYTES || microBytes.byteLength !== count * microRowBytes
|
|
449
472
|
|| (version >= 2 && pageTableRead.byteLength !== pageTableBytes)) {
|
|
450
|
-
throw
|
|
473
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact stage-1 read is truncated');
|
|
451
474
|
}
|
|
452
475
|
const affineCopy = new Uint8Array(affineBytes.byteLength);
|
|
453
476
|
affineCopy.set(affineBytes);
|
|
@@ -463,19 +486,20 @@ class PancakeSketchArtifact {
|
|
|
463
486
|
const digest = await sha256BytesAsync(stage1);
|
|
464
487
|
if (!digest) {
|
|
465
488
|
// Verification requested but no crypto backend: fail closed.
|
|
466
|
-
throw
|
|
489
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
467
490
|
'Sketch artifact verification requested but no crypto backend is available; pass verify:false to skip');
|
|
468
491
|
}
|
|
469
492
|
const expected = header.subarray(136, 168);
|
|
470
493
|
for (let b = 0; b < 32; b++) {
|
|
471
494
|
if (digest[b] !== expected[b]) {
|
|
472
|
-
throw
|
|
495
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact stage-1 prefix failed hash verification');
|
|
473
496
|
}
|
|
474
497
|
}
|
|
475
498
|
artifact.residentVerified = true;
|
|
476
499
|
}
|
|
477
500
|
artifact.scales = new Float32Array(affineCopy.buffer, 0, count);
|
|
478
501
|
artifact.offsets = new Float32Array(affineCopy.buffer, count * 4, count);
|
|
502
|
+
assertFiniteAffine(artifact.scales, artifact.offsets);
|
|
479
503
|
artifact.sketches = null;
|
|
480
504
|
artifact.microSketches = microCopy;
|
|
481
505
|
if (pageTableCopy) artifact.pageTable = pageTableCopy;
|
|
@@ -486,7 +510,7 @@ class PancakeSketchArtifact {
|
|
|
486
510
|
artifact.fullyResident = (async () => {
|
|
487
511
|
const rest = await readChecked(source, sketchesOffset, vectorsOffset - sketchesOffset, 'stage-2 sketches', artifact.maxReadBytes);
|
|
488
512
|
if (rest.byteLength !== vectorsOffset - sketchesOffset) {
|
|
489
|
-
throw
|
|
513
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact stage-2 read is truncated');
|
|
490
514
|
}
|
|
491
515
|
const residentCopy = new Uint8Array((sketchesOffset - SKETCH_HEADER_BYTES) + rest.byteLength);
|
|
492
516
|
residentCopy.set(affineCopy, 0);
|
|
@@ -504,8 +528,21 @@ class PancakeSketchArtifact {
|
|
|
504
528
|
}
|
|
505
529
|
|
|
506
530
|
_adoptResident(residentCopy, count, sketchRowBytes, microRowBytes, microOffset) {
|
|
507
|
-
|
|
508
|
-
this.offsets
|
|
531
|
+
// Validate into locals BEFORE touching a single member. Assigning
|
|
532
|
+
// this.scales/this.offsets first and validating after left a rejected
|
|
533
|
+
// stage-2 adopt half-applied: this.sketches still null, this.tier
|
|
534
|
+
// still 'micro', this.microSketches still pointing at the STAGE-1
|
|
535
|
+
// buffer -- while scales/offsets had already moved to stage-2. The
|
|
536
|
+
// staged path swallows the rejection (see fullyResident.catch above),
|
|
537
|
+
// so the artifact stayed open and kept serving micro-tier queries
|
|
538
|
+
// that paired stage-1 sketches against stage-2 affine parameters:
|
|
539
|
+
// silently wrong ranking, which is worse than the non-finite
|
|
540
|
+
// distances this check exists to reject.
|
|
541
|
+
const scales = new Float32Array(residentCopy.buffer, 0, count);
|
|
542
|
+
const offsets = new Float32Array(residentCopy.buffer, count * 4, count);
|
|
543
|
+
assertFiniteAffine(scales, offsets);
|
|
544
|
+
this.scales = scales;
|
|
545
|
+
this.offsets = offsets;
|
|
509
546
|
this.sketches = new Uint8Array(residentCopy.buffer, count * 8, count * sketchRowBytes);
|
|
510
547
|
if (this.microDims) {
|
|
511
548
|
this.microSketches = new Uint8Array(residentCopy.buffer, microOffset - SKETCH_HEADER_BYTES, count * microRowBytes);
|
|
@@ -522,13 +559,13 @@ class PancakeSketchArtifact {
|
|
|
522
559
|
// Verification was requested (callers gate this on verify) but no
|
|
523
560
|
// crypto backend is available. Fail closed rather than admit
|
|
524
561
|
// unverified bytes.
|
|
525
|
-
throw
|
|
562
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
526
563
|
'Sketch artifact verification requested but no crypto backend is available; pass verify:false to skip');
|
|
527
564
|
}
|
|
528
565
|
const expected = header.subarray(56, 88);
|
|
529
566
|
for (let b = 0; b < 32; b++) {
|
|
530
567
|
if (digest[b] !== expected[b]) {
|
|
531
|
-
throw
|
|
568
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact resident prefix failed hash verification');
|
|
532
569
|
}
|
|
533
570
|
}
|
|
534
571
|
this.residentVerified = true;
|
|
@@ -539,7 +576,7 @@ class PancakeSketchArtifact {
|
|
|
539
576
|
// open must release it, or every corrupt artifact leaks an fd.
|
|
540
577
|
const source = new NodeFileRangeSource(filePath);
|
|
541
578
|
try {
|
|
542
|
-
return await
|
|
579
|
+
return await PikeletSketchArtifact.open(source, options);
|
|
543
580
|
} catch (err) {
|
|
544
581
|
await source.close().catch(() => {});
|
|
545
582
|
throw err;
|
|
@@ -607,7 +644,7 @@ class PancakeSketchArtifact {
|
|
|
607
644
|
for (const id of ids) {
|
|
608
645
|
if (rows.has(id)) continue;
|
|
609
646
|
if (!Number.isInteger(id) || id < 0 || id >= this.count) {
|
|
610
|
-
throw
|
|
647
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'row id out of range', { id });
|
|
611
648
|
}
|
|
612
649
|
const cached = this.cachedRow(id);
|
|
613
650
|
if (cached !== undefined) rows.set(id, cached);
|
|
@@ -647,7 +684,7 @@ class PancakeSketchArtifact {
|
|
|
647
684
|
const length = (endId - startId) * dim;
|
|
648
685
|
const bytes = await readChecked(this.source, offset, length, 'row');
|
|
649
686
|
if (bytes.byteLength !== length) {
|
|
650
|
-
throw
|
|
687
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact row read returned a truncated range', { offset, length });
|
|
651
688
|
}
|
|
652
689
|
this.rangeRequests++;
|
|
653
690
|
this.rangeBytes += length;
|
|
@@ -687,7 +724,7 @@ class PancakeSketchArtifact {
|
|
|
687
724
|
for (const id of ids) {
|
|
688
725
|
if (rows.has(id)) continue;
|
|
689
726
|
if (!Number.isInteger(id) || id < 0 || id >= this.count) {
|
|
690
|
-
throw
|
|
727
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'row id out of range', { id });
|
|
691
728
|
}
|
|
692
729
|
const cached = this.cachedRow(id);
|
|
693
730
|
if (cached !== undefined) rows.set(id, cached);
|
|
@@ -730,7 +767,7 @@ class PancakeSketchArtifact {
|
|
|
730
767
|
const length = r.end - r.start;
|
|
731
768
|
const bytes = await readChecked(this.source, r.start, length, 'row block');
|
|
732
769
|
if (bytes.byteLength !== length) {
|
|
733
|
-
throw
|
|
770
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact row read returned a truncated range', { offset: r.start, length });
|
|
734
771
|
}
|
|
735
772
|
this.rangeRequests++;
|
|
736
773
|
this.rangeBytes += length;
|
|
@@ -743,13 +780,13 @@ class PancakeSketchArtifact {
|
|
|
743
780
|
if (this.verifyRows) {
|
|
744
781
|
const pageDigest = await sha256BytesAsync(page);
|
|
745
782
|
if (!pageDigest) {
|
|
746
|
-
throw
|
|
783
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
747
784
|
'Sketch artifact row verification requested but no crypto backend is available; pass verify:false to skip');
|
|
748
785
|
}
|
|
749
786
|
const expected = this.pageTable.subarray(b * 32, b * 32 + 32);
|
|
750
787
|
for (let i = 0; i < 32; i++) {
|
|
751
788
|
if (pageDigest[i] !== expected[i]) {
|
|
752
|
-
throw
|
|
789
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact row digest page failed hash verification', { block: b });
|
|
753
790
|
}
|
|
754
791
|
}
|
|
755
792
|
}
|
|
@@ -761,7 +798,7 @@ class PancakeSketchArtifact {
|
|
|
761
798
|
const slot = page.subarray(idx * rowDigestBytes, (idx + 1) * rowDigestBytes);
|
|
762
799
|
for (let i = 0; i < rowDigestBytes; i++) {
|
|
763
800
|
if (digest[i] !== slot[i]) {
|
|
764
|
-
throw
|
|
801
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact row failed digest verification', { id });
|
|
765
802
|
}
|
|
766
803
|
}
|
|
767
804
|
}
|
|
@@ -802,7 +839,7 @@ class PancakeSketchArtifact {
|
|
|
802
839
|
for (let d = 0; d < dim; d++) norm += query[d] * query[d];
|
|
803
840
|
norm = Math.sqrt(norm);
|
|
804
841
|
if (!(norm > 0) || !Number.isFinite(norm)) {
|
|
805
|
-
throw
|
|
842
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'cosine query must have a nonzero finite norm');
|
|
806
843
|
}
|
|
807
844
|
q = new Float32Array(dim);
|
|
808
845
|
for (let d = 0; d < dim; d++) q[d] = query[d] / norm;
|
|
@@ -828,10 +865,10 @@ class PancakeSketchArtifact {
|
|
|
828
865
|
// declaration: a metric-blind scanner silently loses recall there.
|
|
829
866
|
const scannerMetric = scanner.metric;
|
|
830
867
|
if (scannerMetric !== undefined && scannerMetric !== this.metric) {
|
|
831
|
-
throw
|
|
868
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'scanner metric does not match artifact metric', { scannerMetric, metric: this.metric });
|
|
832
869
|
}
|
|
833
870
|
if (this.metric === 1 && scannerMetric !== 1) {
|
|
834
|
-
throw
|
|
871
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'cosine sketch artifacts require a metric-aware scanner (scanner.metric === 1)', { metric: this.metric });
|
|
835
872
|
}
|
|
836
873
|
ids = scanner.scan(qPool, C);
|
|
837
874
|
} else if (C >= count) {
|
|
@@ -879,13 +916,13 @@ class PancakeSketchArtifact {
|
|
|
879
916
|
// even when the resident scan's top-C missed it.
|
|
880
917
|
if (options.extraCandidates !== undefined) {
|
|
881
918
|
if (!Array.isArray(options.extraCandidates)) {
|
|
882
|
-
throw
|
|
919
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'extraCandidates must be an array of row ids');
|
|
883
920
|
}
|
|
884
921
|
if (!Array.isArray(ids)) ids = Array.from(ids); // a custom scanner may return a typed array
|
|
885
922
|
const have = new Set(ids);
|
|
886
923
|
for (const id of options.extraCandidates) {
|
|
887
924
|
if (!Number.isInteger(id) || id < 0 || id >= count) {
|
|
888
|
-
throw
|
|
925
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'extraCandidates ids must be integers in [0, count)', { id, count });
|
|
889
926
|
}
|
|
890
927
|
if (!have.has(id)) {
|
|
891
928
|
have.add(id);
|
|
@@ -915,7 +952,7 @@ class PancakeSketchArtifact {
|
|
|
915
952
|
}
|
|
916
953
|
exact.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
917
954
|
// Rerank accumulates squared L2; the API contract (README "Distance
|
|
918
|
-
// values") reports Euclidean, matching
|
|
955
|
+
// values") reports Euclidean, matching PikeletIndex.search.
|
|
919
956
|
const sqrtL2 = this.metric !== 1;
|
|
920
957
|
// fullRerankOutput returns every reranked candidate in distance
|
|
921
958
|
// order instead of the top k — the candidates are already fetched
|
|
@@ -936,26 +973,26 @@ class PancakeSketchArtifact {
|
|
|
936
973
|
}
|
|
937
974
|
|
|
938
975
|
// Build a WASM-backed scanner for a sketch artifact's resident tier, usable
|
|
939
|
-
// as the `scanner` option to
|
|
940
|
-
//
|
|
976
|
+
// as the `scanner` option to PikeletSketchArtifact.search(). The engine's
|
|
977
|
+
// pikelet_sketch_scan SIMD kernel runs the O(count*sketchDims) resident scan
|
|
941
978
|
// that is otherwise the browser query bottleneck. `loadEngine` is the
|
|
942
979
|
// entrypoint's own async engine loader (Node/web/workerd all supply one);
|
|
943
980
|
// the sketch tier is staged in the engine heap once at creation.
|
|
944
981
|
async function createSketchScanner(loadEngine, artifact, options = {}) {
|
|
945
982
|
if (typeof loadEngine !== 'function') {
|
|
946
|
-
throw
|
|
983
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'createSketchScanner() requires an engine loader');
|
|
947
984
|
}
|
|
948
|
-
if (!artifact || !(artifact instanceof
|
|
949
|
-
throw
|
|
985
|
+
if (!artifact || !(artifact instanceof PikeletSketchArtifact)) {
|
|
986
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'createSketchScanner() requires a sketch artifact');
|
|
950
987
|
}
|
|
951
988
|
const engine = await loadEngine();
|
|
952
989
|
const { count, metric } = artifact;
|
|
953
990
|
const tierName = options.tier === 'micro' ? 'micro' : 'full';
|
|
954
991
|
if (tierName === 'micro' && !artifact.microDims) {
|
|
955
|
-
throw
|
|
992
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'artifact has no micro tier');
|
|
956
993
|
}
|
|
957
994
|
if (tierName === 'full' && !artifact.sketches) {
|
|
958
|
-
throw
|
|
995
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'full sketches are not resident yet (staged open still in stage 1; await artifact.fullyResident)');
|
|
959
996
|
}
|
|
960
997
|
const sketchDims = tierName === 'micro' ? artifact.microDims : artifact.sketchDims;
|
|
961
998
|
const tierBits = tierName === 'micro' ? artifact.microBits : artifact.sketchBits;
|
|
@@ -984,7 +1021,7 @@ async function createSketchScanner(loadEngine, artifact, options = {}) {
|
|
|
984
1021
|
for (const ptr of [sketchesPtr, scalesPtr, offsetsPtr, queryPtr, outIdsPtr, outDistsPtr]) {
|
|
985
1022
|
if (ptr) engine._emsc_free(ptr);
|
|
986
1023
|
}
|
|
987
|
-
throw
|
|
1024
|
+
throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED, 'sketch scanner heap allocation failed');
|
|
988
1025
|
}
|
|
989
1026
|
engine.HEAPU8.set(expanded, sketchesPtr);
|
|
990
1027
|
engine.HEAPF32.set(artifact.scales, scalesPtr >> 2);
|
|
@@ -997,25 +1034,25 @@ async function createSketchScanner(loadEngine, artifact, options = {}) {
|
|
|
997
1034
|
sketchDims,
|
|
998
1035
|
tier: tierName,
|
|
999
1036
|
scan(pooledQuery, c) {
|
|
1000
|
-
if (disposed) throw
|
|
1037
|
+
if (disposed) throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'sketch scanner disposed');
|
|
1001
1038
|
const query = pooledQuery instanceof Float32Array ? pooledQuery : Float32Array.from(pooledQuery);
|
|
1002
1039
|
// The query is copied into a queryPtr sized for exactly sketchDims
|
|
1003
1040
|
// floats: an over- or under-sized input would read/write outside
|
|
1004
1041
|
// that buffer in the WASM heap. Validate before the copy.
|
|
1005
1042
|
if (query.length !== sketchDims) {
|
|
1006
|
-
throw
|
|
1043
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
1007
1044
|
`scan() pooled query has ${query.length} values, expected ${sketchDims}`,
|
|
1008
1045
|
{ expected: sketchDims, actual: query.length });
|
|
1009
1046
|
}
|
|
1010
1047
|
for (let i = 0; i < query.length; i++) {
|
|
1011
1048
|
if (!Number.isFinite(query[i])) {
|
|
1012
|
-
throw
|
|
1049
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR,
|
|
1013
1050
|
'scan() pooled query contains a non-finite value', { index: i });
|
|
1014
1051
|
}
|
|
1015
1052
|
}
|
|
1016
1053
|
engine.HEAPF32.set(query, queryPtr >> 2);
|
|
1017
1054
|
const topC = Math.min(Math.max(1, Math.trunc(c)), maxC);
|
|
1018
|
-
const n = engine.
|
|
1055
|
+
const n = engine._pikelet_sketch_scan(
|
|
1019
1056
|
sketchesPtr, scalesPtr, offsetsPtr, count, sketchDims, queryPtr, metric, topC, outIdsPtr, outDistsPtr
|
|
1020
1057
|
);
|
|
1021
1058
|
return Array.from(engine.HEAPU32.subarray(outIdsPtr >> 2, (outIdsPtr >> 2) + n));
|
|
@@ -1031,7 +1068,7 @@ async function createSketchScanner(loadEngine, artifact, options = {}) {
|
|
|
1031
1068
|
}
|
|
1032
1069
|
|
|
1033
1070
|
module.exports = {
|
|
1034
|
-
|
|
1071
|
+
PikeletSketchArtifact,
|
|
1035
1072
|
createSketchScanner,
|
|
1036
1073
|
buildSketchArtifact,
|
|
1037
1074
|
buildSketchArtifactBytes,
|
|
@@ -4,16 +4,16 @@
|
|
|
4
4
|
|
|
5
5
|
import type {
|
|
6
6
|
NodeFileRangeSourceConstructor,
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
PikeletRangeArtifactConstructor,
|
|
8
|
+
PikeletSketchArtifact,
|
|
9
|
+
PikeletSketchArtifactConstructor,
|
|
10
10
|
RangeArtifactBuildManifest,
|
|
11
11
|
RangeArtifactBuildOptions,
|
|
12
12
|
SketchArtifactBuildManifest,
|
|
13
13
|
SketchArtifactBuildOptions,
|
|
14
14
|
SketchScanner,
|
|
15
15
|
SketchScannerOptions,
|
|
16
|
-
} from '
|
|
16
|
+
} from '../pikelet.js';
|
|
17
17
|
|
|
18
18
|
/** A parsed uint8 engine snapshot, as returned by {@link parseUint8Snapshot}. */
|
|
19
19
|
export interface Uint8SnapshotGraph {
|
|
@@ -50,8 +50,8 @@ export interface SketchRowSource {
|
|
|
50
50
|
readonly offsets: Float32Array;
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
-
export declare const
|
|
54
|
-
export declare const
|
|
53
|
+
export declare const PikeletRangeArtifact: PikeletRangeArtifactConstructor;
|
|
54
|
+
export declare const PikeletSketchArtifact: PikeletSketchArtifactConstructor;
|
|
55
55
|
export declare const NodeFileRangeSource: NodeFileRangeSourceConstructor;
|
|
56
56
|
|
|
57
57
|
/**
|
|
@@ -60,13 +60,13 @@ export declare const NodeFileRangeSource: NodeFileRangeSourceConstructor;
|
|
|
60
60
|
*/
|
|
61
61
|
export declare function createSketchScanner(
|
|
62
62
|
loadEngine: () => Promise<unknown>,
|
|
63
|
-
artifact:
|
|
63
|
+
artifact: PikeletSketchArtifact,
|
|
64
64
|
options?: SketchScannerOptions
|
|
65
65
|
): Promise<SketchScanner>;
|
|
66
66
|
|
|
67
67
|
/**
|
|
68
|
-
* @deprecated The `.
|
|
69
|
-
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.
|
|
68
|
+
* @deprecated The `.pikelet-range` profile is deprecated
|
|
69
|
+
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.pikelet-sketch`
|
|
70
70
|
* artifact instead. Readers stay supported for existing artifacts.
|
|
71
71
|
*/
|
|
72
72
|
export declare function buildRangeArtifact(
|
|
@@ -76,8 +76,8 @@ export declare function buildRangeArtifact(
|
|
|
76
76
|
): RangeArtifactBuildManifest;
|
|
77
77
|
|
|
78
78
|
/**
|
|
79
|
-
* @deprecated The `.
|
|
80
|
-
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.
|
|
79
|
+
* @deprecated The `.pikelet-range` profile is deprecated
|
|
80
|
+
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.pikelet-sketch`
|
|
81
81
|
* artifact instead. Readers stay supported for existing artifacts.
|
|
82
82
|
*/
|
|
83
83
|
export declare function buildRangeArtifactFile(
|
|
@@ -1,24 +1,24 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
// pikelet-wasm/artifact — the Search Artifact layer (spec/SEARCH_ARTIFACT_CONTRACT.md).
|
|
3
3
|
// Public entry point; the implementation is split by profile:
|
|
4
|
-
//
|
|
4
|
+
// pikelet-artifact-common.js read budgets + range validation, range sources
|
|
5
5
|
// (NodeFileRangeSource), result heap, snapshot
|
|
6
6
|
// parsing (parseUint8Snapshot), SHA-256 helpers
|
|
7
|
-
//
|
|
8
|
-
//
|
|
7
|
+
// pikelet-artifact-range.js .pikelet-range reader (PikeletRangeArtifact) + builder
|
|
8
|
+
// pikelet-artifact-sketch.js .pikelet-sketch reader (PikeletSketchArtifact),
|
|
9
9
|
// builders, and createSketchScanner
|
|
10
|
-
// The export list below is the package's contract (
|
|
10
|
+
// The export list below is the package's contract (pikelet-artifact.d.ts).
|
|
11
11
|
|
|
12
|
-
const { NodeFileRangeSource, parseUint8Snapshot } = require('./
|
|
13
|
-
const {
|
|
12
|
+
const { NodeFileRangeSource, parseUint8Snapshot } = require('./pikelet-artifact-common.js');
|
|
13
|
+
const { PikeletRangeArtifact, buildRangeArtifact, buildRangeArtifactFile } = require('./pikelet-artifact-range.js');
|
|
14
14
|
const {
|
|
15
|
-
|
|
15
|
+
PikeletSketchArtifact, createSketchScanner,
|
|
16
16
|
buildSketchArtifact, buildSketchArtifactBytes, buildSketchArtifactFile, exportSketchArtifact,
|
|
17
|
-
} = require('./
|
|
17
|
+
} = require('./pikelet-artifact-sketch.js');
|
|
18
18
|
|
|
19
19
|
module.exports = {
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
PikeletRangeArtifact,
|
|
21
|
+
PikeletSketchArtifact,
|
|
22
22
|
createSketchScanner,
|
|
23
23
|
NodeFileRangeSource,
|
|
24
24
|
buildRangeArtifact,
|