pikelet-wasm 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -3
- package/NOTICE +5 -0
- package/README.md +397 -454
- package/complete/builder.d.ts +4 -4
- package/complete/builder.mjs +8 -8
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/index.d.ts +11 -7
- package/complete/index.mjs +161 -112
- package/complete/inline-transformer.mjs +22 -9
- package/complete/lexical.mjs +11 -11
- package/complete/retrieval-abstention.mjs +18 -6
- package/complete/sources.mjs +22 -1
- package/dist/engine.js +1 -1
- package/dist/engine.scalar.js +1 -1
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +58 -51
- package/{pancake-artifact-common.js → pikelet-artifact-common.js} +42 -42
- package/{pancake-artifact-range.js → pikelet-artifact-range.js} +36 -36
- package/{pancake-artifact-sketch.js → pikelet-artifact-sketch.js} +76 -66
- package/{pancake-artifact.d.ts → pikelet-artifact.d.ts} +11 -11
- package/{pancake-artifact.js → pikelet-artifact.js} +10 -10
- package/{pancake-core.js → pikelet-core.js} +136 -136
- package/{pancake-errors.js → pikelet-errors.js} +9 -8
- package/{pancake.d.ts → pikelet.d.ts} +55 -55
- package/{pancake.js → pikelet.js} +44 -44
- package/{pancake.node.mjs → pikelet.node.mjs} +46 -46
- package/{pancake.web.d.ts → pikelet.web.d.ts} +13 -13
- package/{pancake.web.mjs → pikelet.web.mjs} +20 -20
- package/{pancake.workerd.mjs → pikelet.workerd.mjs} +18 -18
- /package/{pancake-loader.js → pikelet-loader.js} +0 -0
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
'use strict';
|
|
2
|
-
// Sketch artifact profile (.
|
|
3
|
-
//
|
|
2
|
+
// Sketch artifact profile (.pikelet-sketch) — spec/SKETCH_PROFILE.md:
|
|
3
|
+
// PikeletSketchArtifact reader, the sketch-artifact builders, and the
|
|
4
4
|
// engine-backed resident scanner.
|
|
5
|
-
// Split out of
|
|
5
|
+
// Split out of pikelet-artifact.js (the public entry, which re-exports the
|
|
6
6
|
// three parts); see that file for the module map.
|
|
7
7
|
|
|
8
|
-
const {
|
|
8
|
+
const { pikeletError, PIKELET_ERROR_CODES } = require('./pikelet-errors.js');
|
|
9
9
|
const {
|
|
10
10
|
DEFAULT_OPEN_READ_BYTES,
|
|
11
11
|
MAX_COALESCED_RANGE_BYTES,
|
|
@@ -18,15 +18,15 @@ const {
|
|
|
18
18
|
sha256Bytes,
|
|
19
19
|
sha256BytesAsync,
|
|
20
20
|
verifySegmentSha256,
|
|
21
|
-
} = require('./
|
|
21
|
+
} = require('./pikelet-artifact-common.js');
|
|
22
22
|
const {
|
|
23
23
|
normalizeQuery: normalizeQueryVector,
|
|
24
24
|
resolveSearchK,
|
|
25
25
|
resolveOptionalPositiveInt,
|
|
26
|
-
} = require('./
|
|
26
|
+
} = require('./pikelet-artifact-common.js');
|
|
27
27
|
|
|
28
28
|
// ============================================================================
|
|
29
|
-
// Sketch artifact profile (.
|
|
29
|
+
// Sketch artifact profile (.pikelet-sketch) — spec/SKETCH_PROFILE.md
|
|
30
30
|
// ============================================================================
|
|
31
31
|
//
|
|
32
32
|
// Layout: 256-byte header | scales f32[count] | offsets f32[count] |
|
|
@@ -40,7 +40,7 @@ const SKETCH_KIND_U8 = 1;
|
|
|
40
40
|
|
|
41
41
|
function buildSketchArtifact(snapshotBytes, outPath, options = {}) {
|
|
42
42
|
if (typeof outPath !== 'string' || outPath.length === 0) {
|
|
43
|
-
throw
|
|
43
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'buildSketchArtifact() requires an output path');
|
|
44
44
|
}
|
|
45
45
|
return exportSketchArtifact(parseUint8Snapshot(snapshotBytes), outPath, options);
|
|
46
46
|
}
|
|
@@ -55,13 +55,13 @@ function exportSketchArtifact(index, outPath, options = {}) {
|
|
|
55
55
|
const sketchBits = options.sketchBits || 4;
|
|
56
56
|
const recommendedRerank = options.recommendedRerank || 0;
|
|
57
57
|
if (![4, 8].includes(sketchBits)) {
|
|
58
|
-
throw
|
|
58
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'sketchBits must be 4 or 8', { sketchBits });
|
|
59
59
|
}
|
|
60
60
|
if (!Number.isInteger(sketchDims) || sketchDims < 1 || dim % sketchDims !== 0) {
|
|
61
|
-
throw
|
|
61
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'sketchDims must divide dim', { sketchDims, dim });
|
|
62
62
|
}
|
|
63
63
|
if (sketchBits === 4 && sketchDims % 2 !== 0) {
|
|
64
|
-
throw
|
|
64
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'sketchDims must be even for 4-bit sketches', { sketchDims });
|
|
65
65
|
}
|
|
66
66
|
// Optional staged-boot micro tier: a second, coarser pooling of the same
|
|
67
67
|
// quantized rows, stored after the full sketches so v1 readers see it
|
|
@@ -78,10 +78,10 @@ function exportSketchArtifact(index, outPath, options = {}) {
|
|
|
78
78
|
const microBits = microDims ? (options.microBits || 4) : 0;
|
|
79
79
|
if (microDims) {
|
|
80
80
|
if (!Number.isInteger(microDims) || microDims < 1 || sketchDims % microDims !== 0 || microDims >= sketchDims) {
|
|
81
|
-
throw
|
|
81
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'microDims must divide sketchDims and be smaller', { microDims, sketchDims });
|
|
82
82
|
}
|
|
83
83
|
if (![4, 8].includes(microBits) || (microBits === 4 && microDims % 2 !== 0)) {
|
|
84
|
-
throw
|
|
84
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'invalid micro sketch encoding', { microBits, microDims });
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
87
|
// Per-row integrity (format version 2, spec section 2.4): the vectors
|
|
@@ -97,10 +97,20 @@ function exportSketchArtifact(index, outPath, options = {}) {
|
|
|
97
97
|
const rowDigestBytes = options.rowDigestBytes || 16;
|
|
98
98
|
if (rowIntegrity) {
|
|
99
99
|
if (!Number.isInteger(rowsPerBlock) || rowsPerBlock < 1 || rowsPerBlock > 4096) {
|
|
100
|
-
throw
|
|
100
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'rowsPerBlock must be an integer in [1, 4096]', { rowsPerBlock });
|
|
101
101
|
}
|
|
102
|
-
|
|
103
|
-
|
|
102
|
+
// Floor is 16, not the format's 8: the per-row digest is truncated
|
|
103
|
+
// SHA-256, and the claim it backs is that nobody can substitute a
|
|
104
|
+
// different row with the same digest — a second-preimage problem at
|
|
105
|
+
// 8*rowDigestBytes bits. At 8 bytes that's 2^64, a fraction of a
|
|
106
|
+
// second of the Bitcoin network's hash rate; not tamper evidence
|
|
107
|
+
// against anyone who cares. At 16 it's 2^128. The saving from 8 vs
|
|
108
|
+
// 16 bytes on a typical row is a couple percent, so there is no
|
|
109
|
+
// reason to offer the weak end of the range. Readers still accept
|
|
110
|
+
// down to 8 (spec's read-path floor, format.md) for artifacts built
|
|
111
|
+
// before this floor was raised.
|
|
112
|
+
if (!Number.isInteger(rowDigestBytes) || rowDigestBytes < 16 || rowDigestBytes > 32) {
|
|
113
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'rowDigestBytes must be an integer in [16, 32]', { rowDigestBytes });
|
|
104
114
|
}
|
|
105
115
|
}
|
|
106
116
|
const pool = dim / sketchDims;
|
|
@@ -223,7 +233,7 @@ function exportSketchArtifact(index, outPath, options = {}) {
|
|
|
223
233
|
}
|
|
224
234
|
|
|
225
235
|
const manifest = {
|
|
226
|
-
format: '
|
|
236
|
+
format: 'pikelet-sketch-artifact',
|
|
227
237
|
formatVersion: rowIntegrity ? 2 : 1,
|
|
228
238
|
sizeBytes: fileBytes,
|
|
229
239
|
metric: index.metric === 1 ? 'cosine' : 'l2',
|
|
@@ -256,7 +266,7 @@ function buildSketchArtifactFile(snapshotPath, outPath, options = {}) {
|
|
|
256
266
|
return buildSketchArtifact(fs.readFileSync(snapshotPath), outPath, options);
|
|
257
267
|
}
|
|
258
268
|
|
|
259
|
-
class
|
|
269
|
+
class PikeletSketchArtifact {
|
|
260
270
|
constructor(source) {
|
|
261
271
|
this.source = source;
|
|
262
272
|
// Fetched rows live in a byte-budgeted LRU; search correctness never
|
|
@@ -300,25 +310,25 @@ class PancakeSketchArtifact {
|
|
|
300
310
|
|
|
301
311
|
static async open(source, options = {}) {
|
|
302
312
|
if (!source || typeof source.read !== 'function') {
|
|
303
|
-
throw
|
|
313
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'SketchArtifact.open() requires a range source with read(offset, length)');
|
|
304
314
|
}
|
|
305
|
-
const artifact = new
|
|
315
|
+
const artifact = new PikeletSketchArtifact(source);
|
|
306
316
|
artifact.maxReadBytes = resolveMaxReadBytes(options.maxReadBytes);
|
|
307
317
|
const header = await readChecked(source, 0, SKETCH_HEADER_BYTES, 'header');
|
|
308
318
|
if (header.byteLength !== SKETCH_HEADER_BYTES) {
|
|
309
|
-
throw
|
|
319
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact header is truncated');
|
|
310
320
|
}
|
|
311
321
|
const view = new DataView(header.buffer, header.byteOffset, header.byteLength);
|
|
312
322
|
if (view.getUint32(0, true) !== SKETCH_MAGIC) {
|
|
313
|
-
throw
|
|
323
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Not a Pikelet sketch artifact (bad magic)');
|
|
314
324
|
}
|
|
315
325
|
const version = view.getUint32(4, true);
|
|
316
326
|
if (version !== 1 && version !== 2) {
|
|
317
|
-
throw
|
|
327
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported sketch artifact version', { version });
|
|
318
328
|
}
|
|
319
329
|
artifact.formatVersion = version;
|
|
320
330
|
if (view.getUint32(8, true) !== SKETCH_KIND_U8) {
|
|
321
|
-
throw
|
|
331
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported sketch artifact kind');
|
|
322
332
|
}
|
|
323
333
|
artifact.metric = view.getUint32(12, true);
|
|
324
334
|
artifact.dim = view.getUint32(16, true);
|
|
@@ -344,13 +354,13 @@ class PancakeSketchArtifact {
|
|
|
344
354
|
|
|
345
355
|
const { metric, dim, count, sketchDims, sketchBits, vectorsOffset } = artifact;
|
|
346
356
|
if (metric !== 0 && metric !== 1) {
|
|
347
|
-
throw
|
|
357
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported sketch artifact metric', { metric });
|
|
348
358
|
}
|
|
349
359
|
if (dim < 1 || count < 1 || sketchDims < 1 || dim % sketchDims !== 0) {
|
|
350
|
-
throw
|
|
360
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Invalid sketch artifact geometry', { dim, count, sketchDims });
|
|
351
361
|
}
|
|
352
362
|
if (![4, 8].includes(sketchBits) || (sketchBits === 4 && sketchDims % 2 !== 0)) {
|
|
353
|
-
throw
|
|
363
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Invalid sketch encoding', { sketchBits, sketchDims });
|
|
354
364
|
}
|
|
355
365
|
const sketchRowBytes = (sketchDims * sketchBits) / 8;
|
|
356
366
|
// Version 2 interleaves a digest page ahead of each rowsPerBlock rows
|
|
@@ -360,7 +370,7 @@ class PancakeSketchArtifact {
|
|
|
360
370
|
const P = artifact.rowsPerBlock;
|
|
361
371
|
const D = artifact.rowDigestBytes;
|
|
362
372
|
if (!Number.isInteger(P) || P < 1 || P > 4096 || !Number.isInteger(D) || D < 8 || D > 32) {
|
|
363
|
-
throw
|
|
373
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Invalid sketch row-integrity geometry', { rowsPerBlock: P, rowDigestBytes: D });
|
|
364
374
|
}
|
|
365
375
|
artifact.numBlocks = Math.ceil(count / P);
|
|
366
376
|
artifact.pageBytes = P * D;
|
|
@@ -370,7 +380,7 @@ class PancakeSketchArtifact {
|
|
|
370
380
|
if (!Number.isSafeInteger(artifact.vectorsRegionBytes)
|
|
371
381
|
|| pageTableOffset < sketchesOffset + count * sketchRowBytes
|
|
372
382
|
|| pageTableOffset + artifact.numBlocks * 32 > vectorsOffset) {
|
|
373
|
-
throw
|
|
383
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact page table layout is inconsistent', { pageTableOffset });
|
|
374
384
|
}
|
|
375
385
|
} else {
|
|
376
386
|
artifact.numBlocks = 0;
|
|
@@ -384,7 +394,7 @@ class PancakeSketchArtifact {
|
|
|
384
394
|
|| vectorsOffset < sketchesOffset + count * sketchRowBytes
|
|
385
395
|
|| vectorsOffset % 16 !== 0
|
|
386
396
|
|| fileBytes !== vectorsOffset + expectVectors) {
|
|
387
|
-
throw
|
|
397
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact layout is inconsistent');
|
|
388
398
|
}
|
|
389
399
|
let microRowBytes = 0;
|
|
390
400
|
if (artifact.microDims) {
|
|
@@ -393,7 +403,7 @@ class PancakeSketchArtifact {
|
|
|
393
403
|
|| ![4, 8].includes(artifact.microBits) || (artifact.microBits === 4 && artifact.microDims % 2 !== 0)
|
|
394
404
|
|| microOffset !== sketchesOffset + count * sketchRowBytes
|
|
395
405
|
|| microOffset + count * microRowBytes > vectorsOffset) {
|
|
396
|
-
throw
|
|
406
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact micro tier layout is inconsistent', { microDims: artifact.microDims });
|
|
397
407
|
}
|
|
398
408
|
}
|
|
399
409
|
artifact.microOffset = microOffset;
|
|
@@ -402,7 +412,7 @@ class PancakeSketchArtifact {
|
|
|
402
412
|
// still inside the resident prefix residentSha256 covers.
|
|
403
413
|
const tiersEnd = artifact.microDims ? microOffset + count * microRowBytes : sketchesOffset + count * sketchRowBytes;
|
|
404
414
|
if (pageTableOffset !== tiersEnd) {
|
|
405
|
-
throw
|
|
415
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact page table layout is inconsistent', { pageTableOffset, tiersEnd });
|
|
406
416
|
}
|
|
407
417
|
}
|
|
408
418
|
artifact.pageTableOffset = pageTableOffset;
|
|
@@ -418,7 +428,7 @@ class PancakeSketchArtifact {
|
|
|
418
428
|
if (!staged) {
|
|
419
429
|
const resident = await readChecked(source, SKETCH_HEADER_BYTES, vectorsOffset - SKETCH_HEADER_BYTES, 'resident prefix', artifact.maxReadBytes);
|
|
420
430
|
if (resident.byteLength !== vectorsOffset - SKETCH_HEADER_BYTES) {
|
|
421
|
-
throw
|
|
431
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact resident prefix is truncated');
|
|
422
432
|
}
|
|
423
433
|
// Copy into an aligned buffer so the typed-array views are valid
|
|
424
434
|
// regardless of the source's byteOffset.
|
|
@@ -447,7 +457,7 @@ class PancakeSketchArtifact {
|
|
|
447
457
|
]);
|
|
448
458
|
if (affineBytes.byteLength !== sketchesOffset - SKETCH_HEADER_BYTES || microBytes.byteLength !== count * microRowBytes
|
|
449
459
|
|| (version >= 2 && pageTableRead.byteLength !== pageTableBytes)) {
|
|
450
|
-
throw
|
|
460
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact stage-1 read is truncated');
|
|
451
461
|
}
|
|
452
462
|
const affineCopy = new Uint8Array(affineBytes.byteLength);
|
|
453
463
|
affineCopy.set(affineBytes);
|
|
@@ -463,13 +473,13 @@ class PancakeSketchArtifact {
|
|
|
463
473
|
const digest = await sha256BytesAsync(stage1);
|
|
464
474
|
if (!digest) {
|
|
465
475
|
// Verification requested but no crypto backend: fail closed.
|
|
466
|
-
throw
|
|
476
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
467
477
|
'Sketch artifact verification requested but no crypto backend is available; pass verify:false to skip');
|
|
468
478
|
}
|
|
469
479
|
const expected = header.subarray(136, 168);
|
|
470
480
|
for (let b = 0; b < 32; b++) {
|
|
471
481
|
if (digest[b] !== expected[b]) {
|
|
472
|
-
throw
|
|
482
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact stage-1 prefix failed hash verification');
|
|
473
483
|
}
|
|
474
484
|
}
|
|
475
485
|
artifact.residentVerified = true;
|
|
@@ -486,7 +496,7 @@ class PancakeSketchArtifact {
|
|
|
486
496
|
artifact.fullyResident = (async () => {
|
|
487
497
|
const rest = await readChecked(source, sketchesOffset, vectorsOffset - sketchesOffset, 'stage-2 sketches', artifact.maxReadBytes);
|
|
488
498
|
if (rest.byteLength !== vectorsOffset - sketchesOffset) {
|
|
489
|
-
throw
|
|
499
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact stage-2 read is truncated');
|
|
490
500
|
}
|
|
491
501
|
const residentCopy = new Uint8Array((sketchesOffset - SKETCH_HEADER_BYTES) + rest.byteLength);
|
|
492
502
|
residentCopy.set(affineCopy, 0);
|
|
@@ -522,13 +532,13 @@ class PancakeSketchArtifact {
|
|
|
522
532
|
// Verification was requested (callers gate this on verify) but no
|
|
523
533
|
// crypto backend is available. Fail closed rather than admit
|
|
524
534
|
// unverified bytes.
|
|
525
|
-
throw
|
|
535
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
526
536
|
'Sketch artifact verification requested but no crypto backend is available; pass verify:false to skip');
|
|
527
537
|
}
|
|
528
538
|
const expected = header.subarray(56, 88);
|
|
529
539
|
for (let b = 0; b < 32; b++) {
|
|
530
540
|
if (digest[b] !== expected[b]) {
|
|
531
|
-
throw
|
|
541
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact resident prefix failed hash verification');
|
|
532
542
|
}
|
|
533
543
|
}
|
|
534
544
|
this.residentVerified = true;
|
|
@@ -539,7 +549,7 @@ class PancakeSketchArtifact {
|
|
|
539
549
|
// open must release it, or every corrupt artifact leaks an fd.
|
|
540
550
|
const source = new NodeFileRangeSource(filePath);
|
|
541
551
|
try {
|
|
542
|
-
return await
|
|
552
|
+
return await PikeletSketchArtifact.open(source, options);
|
|
543
553
|
} catch (err) {
|
|
544
554
|
await source.close().catch(() => {});
|
|
545
555
|
throw err;
|
|
@@ -607,7 +617,7 @@ class PancakeSketchArtifact {
|
|
|
607
617
|
for (const id of ids) {
|
|
608
618
|
if (rows.has(id)) continue;
|
|
609
619
|
if (!Number.isInteger(id) || id < 0 || id >= this.count) {
|
|
610
|
-
throw
|
|
620
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'row id out of range', { id });
|
|
611
621
|
}
|
|
612
622
|
const cached = this.cachedRow(id);
|
|
613
623
|
if (cached !== undefined) rows.set(id, cached);
|
|
@@ -647,7 +657,7 @@ class PancakeSketchArtifact {
|
|
|
647
657
|
const length = (endId - startId) * dim;
|
|
648
658
|
const bytes = await readChecked(this.source, offset, length, 'row');
|
|
649
659
|
if (bytes.byteLength !== length) {
|
|
650
|
-
throw
|
|
660
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact row read returned a truncated range', { offset, length });
|
|
651
661
|
}
|
|
652
662
|
this.rangeRequests++;
|
|
653
663
|
this.rangeBytes += length;
|
|
@@ -687,7 +697,7 @@ class PancakeSketchArtifact {
|
|
|
687
697
|
for (const id of ids) {
|
|
688
698
|
if (rows.has(id)) continue;
|
|
689
699
|
if (!Number.isInteger(id) || id < 0 || id >= this.count) {
|
|
690
|
-
throw
|
|
700
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'row id out of range', { id });
|
|
691
701
|
}
|
|
692
702
|
const cached = this.cachedRow(id);
|
|
693
703
|
if (cached !== undefined) rows.set(id, cached);
|
|
@@ -730,7 +740,7 @@ class PancakeSketchArtifact {
|
|
|
730
740
|
const length = r.end - r.start;
|
|
731
741
|
const bytes = await readChecked(this.source, r.start, length, 'row block');
|
|
732
742
|
if (bytes.byteLength !== length) {
|
|
733
|
-
throw
|
|
743
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact row read returned a truncated range', { offset: r.start, length });
|
|
734
744
|
}
|
|
735
745
|
this.rangeRequests++;
|
|
736
746
|
this.rangeBytes += length;
|
|
@@ -743,13 +753,13 @@ class PancakeSketchArtifact {
|
|
|
743
753
|
if (this.verifyRows) {
|
|
744
754
|
const pageDigest = await sha256BytesAsync(page);
|
|
745
755
|
if (!pageDigest) {
|
|
746
|
-
throw
|
|
756
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
747
757
|
'Sketch artifact row verification requested but no crypto backend is available; pass verify:false to skip');
|
|
748
758
|
}
|
|
749
759
|
const expected = this.pageTable.subarray(b * 32, b * 32 + 32);
|
|
750
760
|
for (let i = 0; i < 32; i++) {
|
|
751
761
|
if (pageDigest[i] !== expected[i]) {
|
|
752
|
-
throw
|
|
762
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact row digest page failed hash verification', { block: b });
|
|
753
763
|
}
|
|
754
764
|
}
|
|
755
765
|
}
|
|
@@ -761,7 +771,7 @@ class PancakeSketchArtifact {
|
|
|
761
771
|
const slot = page.subarray(idx * rowDigestBytes, (idx + 1) * rowDigestBytes);
|
|
762
772
|
for (let i = 0; i < rowDigestBytes; i++) {
|
|
763
773
|
if (digest[i] !== slot[i]) {
|
|
764
|
-
throw
|
|
774
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact row failed digest verification', { id });
|
|
765
775
|
}
|
|
766
776
|
}
|
|
767
777
|
}
|
|
@@ -802,7 +812,7 @@ class PancakeSketchArtifact {
|
|
|
802
812
|
for (let d = 0; d < dim; d++) norm += query[d] * query[d];
|
|
803
813
|
norm = Math.sqrt(norm);
|
|
804
814
|
if (!(norm > 0) || !Number.isFinite(norm)) {
|
|
805
|
-
throw
|
|
815
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'cosine query must have a nonzero finite norm');
|
|
806
816
|
}
|
|
807
817
|
q = new Float32Array(dim);
|
|
808
818
|
for (let d = 0; d < dim; d++) q[d] = query[d] / norm;
|
|
@@ -828,10 +838,10 @@ class PancakeSketchArtifact {
|
|
|
828
838
|
// declaration: a metric-blind scanner silently loses recall there.
|
|
829
839
|
const scannerMetric = scanner.metric;
|
|
830
840
|
if (scannerMetric !== undefined && scannerMetric !== this.metric) {
|
|
831
|
-
throw
|
|
841
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'scanner metric does not match artifact metric', { scannerMetric, metric: this.metric });
|
|
832
842
|
}
|
|
833
843
|
if (this.metric === 1 && scannerMetric !== 1) {
|
|
834
|
-
throw
|
|
844
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'cosine sketch artifacts require a metric-aware scanner (scanner.metric === 1)', { metric: this.metric });
|
|
835
845
|
}
|
|
836
846
|
ids = scanner.scan(qPool, C);
|
|
837
847
|
} else if (C >= count) {
|
|
@@ -879,13 +889,13 @@ class PancakeSketchArtifact {
|
|
|
879
889
|
// even when the resident scan's top-C missed it.
|
|
880
890
|
if (options.extraCandidates !== undefined) {
|
|
881
891
|
if (!Array.isArray(options.extraCandidates)) {
|
|
882
|
-
throw
|
|
892
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'extraCandidates must be an array of row ids');
|
|
883
893
|
}
|
|
884
894
|
if (!Array.isArray(ids)) ids = Array.from(ids); // a custom scanner may return a typed array
|
|
885
895
|
const have = new Set(ids);
|
|
886
896
|
for (const id of options.extraCandidates) {
|
|
887
897
|
if (!Number.isInteger(id) || id < 0 || id >= count) {
|
|
888
|
-
throw
|
|
898
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'extraCandidates ids must be integers in [0, count)', { id, count });
|
|
889
899
|
}
|
|
890
900
|
if (!have.has(id)) {
|
|
891
901
|
have.add(id);
|
|
@@ -915,7 +925,7 @@ class PancakeSketchArtifact {
|
|
|
915
925
|
}
|
|
916
926
|
exact.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
917
927
|
// Rerank accumulates squared L2; the API contract (README "Distance
|
|
918
|
-
// values") reports Euclidean, matching
|
|
928
|
+
// values") reports Euclidean, matching PikeletIndex.search.
|
|
919
929
|
const sqrtL2 = this.metric !== 1;
|
|
920
930
|
// fullRerankOutput returns every reranked candidate in distance
|
|
921
931
|
// order instead of the top k — the candidates are already fetched
|
|
@@ -936,26 +946,26 @@ class PancakeSketchArtifact {
|
|
|
936
946
|
}
|
|
937
947
|
|
|
938
948
|
// Build a WASM-backed scanner for a sketch artifact's resident tier, usable
|
|
939
|
-
// as the `scanner` option to
|
|
940
|
-
//
|
|
949
|
+
// as the `scanner` option to PikeletSketchArtifact.search(). The engine's
|
|
950
|
+
// pikelet_sketch_scan SIMD kernel runs the O(count*sketchDims) resident scan
|
|
941
951
|
// that is otherwise the browser query bottleneck. `loadEngine` is the
|
|
942
952
|
// entrypoint's own async engine loader (Node/web/workerd all supply one);
|
|
943
953
|
// the sketch tier is staged in the engine heap once at creation.
|
|
944
954
|
async function createSketchScanner(loadEngine, artifact, options = {}) {
|
|
945
955
|
if (typeof loadEngine !== 'function') {
|
|
946
|
-
throw
|
|
956
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'createSketchScanner() requires an engine loader');
|
|
947
957
|
}
|
|
948
|
-
if (!artifact || !(artifact instanceof
|
|
949
|
-
throw
|
|
958
|
+
if (!artifact || !(artifact instanceof PikeletSketchArtifact)) {
|
|
959
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'createSketchScanner() requires a sketch artifact');
|
|
950
960
|
}
|
|
951
961
|
const engine = await loadEngine();
|
|
952
962
|
const { count, metric } = artifact;
|
|
953
963
|
const tierName = options.tier === 'micro' ? 'micro' : 'full';
|
|
954
964
|
if (tierName === 'micro' && !artifact.microDims) {
|
|
955
|
-
throw
|
|
965
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'artifact has no micro tier');
|
|
956
966
|
}
|
|
957
967
|
if (tierName === 'full' && !artifact.sketches) {
|
|
958
|
-
throw
|
|
968
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'full sketches are not resident yet (staged open still in stage 1; await artifact.fullyResident)');
|
|
959
969
|
}
|
|
960
970
|
const sketchDims = tierName === 'micro' ? artifact.microDims : artifact.sketchDims;
|
|
961
971
|
const tierBits = tierName === 'micro' ? artifact.microBits : artifact.sketchBits;
|
|
@@ -984,7 +994,7 @@ async function createSketchScanner(loadEngine, artifact, options = {}) {
|
|
|
984
994
|
for (const ptr of [sketchesPtr, scalesPtr, offsetsPtr, queryPtr, outIdsPtr, outDistsPtr]) {
|
|
985
995
|
if (ptr) engine._emsc_free(ptr);
|
|
986
996
|
}
|
|
987
|
-
throw
|
|
997
|
+
throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED, 'sketch scanner heap allocation failed');
|
|
988
998
|
}
|
|
989
999
|
engine.HEAPU8.set(expanded, sketchesPtr);
|
|
990
1000
|
engine.HEAPF32.set(artifact.scales, scalesPtr >> 2);
|
|
@@ -997,25 +1007,25 @@ async function createSketchScanner(loadEngine, artifact, options = {}) {
|
|
|
997
1007
|
sketchDims,
|
|
998
1008
|
tier: tierName,
|
|
999
1009
|
scan(pooledQuery, c) {
|
|
1000
|
-
if (disposed) throw
|
|
1010
|
+
if (disposed) throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'sketch scanner disposed');
|
|
1001
1011
|
const query = pooledQuery instanceof Float32Array ? pooledQuery : Float32Array.from(pooledQuery);
|
|
1002
1012
|
// The query is copied into a queryPtr sized for exactly sketchDims
|
|
1003
1013
|
// floats: an over- or under-sized input would read/write outside
|
|
1004
1014
|
// that buffer in the WASM heap. Validate before the copy.
|
|
1005
1015
|
if (query.length !== sketchDims) {
|
|
1006
|
-
throw
|
|
1016
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
1007
1017
|
`scan() pooled query has ${query.length} values, expected ${sketchDims}`,
|
|
1008
1018
|
{ expected: sketchDims, actual: query.length });
|
|
1009
1019
|
}
|
|
1010
1020
|
for (let i = 0; i < query.length; i++) {
|
|
1011
1021
|
if (!Number.isFinite(query[i])) {
|
|
1012
|
-
throw
|
|
1022
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR,
|
|
1013
1023
|
'scan() pooled query contains a non-finite value', { index: i });
|
|
1014
1024
|
}
|
|
1015
1025
|
}
|
|
1016
1026
|
engine.HEAPF32.set(query, queryPtr >> 2);
|
|
1017
1027
|
const topC = Math.min(Math.max(1, Math.trunc(c)), maxC);
|
|
1018
|
-
const n = engine.
|
|
1028
|
+
const n = engine._pikelet_sketch_scan(
|
|
1019
1029
|
sketchesPtr, scalesPtr, offsetsPtr, count, sketchDims, queryPtr, metric, topC, outIdsPtr, outDistsPtr
|
|
1020
1030
|
);
|
|
1021
1031
|
return Array.from(engine.HEAPU32.subarray(outIdsPtr >> 2, (outIdsPtr >> 2) + n));
|
|
@@ -1031,7 +1041,7 @@ async function createSketchScanner(loadEngine, artifact, options = {}) {
|
|
|
1031
1041
|
}
|
|
1032
1042
|
|
|
1033
1043
|
module.exports = {
|
|
1034
|
-
|
|
1044
|
+
PikeletSketchArtifact,
|
|
1035
1045
|
createSketchScanner,
|
|
1036
1046
|
buildSketchArtifact,
|
|
1037
1047
|
buildSketchArtifactBytes,
|
|
@@ -4,16 +4,16 @@
|
|
|
4
4
|
|
|
5
5
|
import type {
|
|
6
6
|
NodeFileRangeSourceConstructor,
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
PikeletRangeArtifactConstructor,
|
|
8
|
+
PikeletSketchArtifact,
|
|
9
|
+
PikeletSketchArtifactConstructor,
|
|
10
10
|
RangeArtifactBuildManifest,
|
|
11
11
|
RangeArtifactBuildOptions,
|
|
12
12
|
SketchArtifactBuildManifest,
|
|
13
13
|
SketchArtifactBuildOptions,
|
|
14
14
|
SketchScanner,
|
|
15
15
|
SketchScannerOptions,
|
|
16
|
-
} from './
|
|
16
|
+
} from './pikelet.js';
|
|
17
17
|
|
|
18
18
|
/** A parsed uint8 engine snapshot, as returned by {@link parseUint8Snapshot}. */
|
|
19
19
|
export interface Uint8SnapshotGraph {
|
|
@@ -50,8 +50,8 @@ export interface SketchRowSource {
|
|
|
50
50
|
readonly offsets: Float32Array;
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
-
export declare const
|
|
54
|
-
export declare const
|
|
53
|
+
export declare const PikeletRangeArtifact: PikeletRangeArtifactConstructor;
|
|
54
|
+
export declare const PikeletSketchArtifact: PikeletSketchArtifactConstructor;
|
|
55
55
|
export declare const NodeFileRangeSource: NodeFileRangeSourceConstructor;
|
|
56
56
|
|
|
57
57
|
/**
|
|
@@ -60,13 +60,13 @@ export declare const NodeFileRangeSource: NodeFileRangeSourceConstructor;
|
|
|
60
60
|
*/
|
|
61
61
|
export declare function createSketchScanner(
|
|
62
62
|
loadEngine: () => Promise<unknown>,
|
|
63
|
-
artifact:
|
|
63
|
+
artifact: PikeletSketchArtifact,
|
|
64
64
|
options?: SketchScannerOptions
|
|
65
65
|
): Promise<SketchScanner>;
|
|
66
66
|
|
|
67
67
|
/**
|
|
68
|
-
* @deprecated The `.
|
|
69
|
-
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.
|
|
68
|
+
* @deprecated The `.pikelet-range` profile is deprecated
|
|
69
|
+
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.pikelet-sketch`
|
|
70
70
|
* artifact instead. Readers stay supported for existing artifacts.
|
|
71
71
|
*/
|
|
72
72
|
export declare function buildRangeArtifact(
|
|
@@ -76,8 +76,8 @@ export declare function buildRangeArtifact(
|
|
|
76
76
|
): RangeArtifactBuildManifest;
|
|
77
77
|
|
|
78
78
|
/**
|
|
79
|
-
* @deprecated The `.
|
|
80
|
-
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.
|
|
79
|
+
* @deprecated The `.pikelet-range` profile is deprecated
|
|
80
|
+
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.pikelet-sketch`
|
|
81
81
|
* artifact instead. Readers stay supported for existing artifacts.
|
|
82
82
|
*/
|
|
83
83
|
export declare function buildRangeArtifactFile(
|
|
@@ -1,24 +1,24 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
// pikelet-wasm/artifact — the Search Artifact layer (spec/SEARCH_ARTIFACT_CONTRACT.md).
|
|
3
3
|
// Public entry point; the implementation is split by profile:
|
|
4
|
-
//
|
|
4
|
+
// pikelet-artifact-common.js read budgets + range validation, range sources
|
|
5
5
|
// (NodeFileRangeSource), result heap, snapshot
|
|
6
6
|
// parsing (parseUint8Snapshot), SHA-256 helpers
|
|
7
|
-
//
|
|
8
|
-
//
|
|
7
|
+
// pikelet-artifact-range.js .pikelet-range reader (PikeletRangeArtifact) + builder
|
|
8
|
+
// pikelet-artifact-sketch.js .pikelet-sketch reader (PikeletSketchArtifact),
|
|
9
9
|
// builders, and createSketchScanner
|
|
10
|
-
// The export list below is the package's contract (
|
|
10
|
+
// The export list below is the package's contract (pikelet-artifact.d.ts).
|
|
11
11
|
|
|
12
|
-
const { NodeFileRangeSource, parseUint8Snapshot } = require('./
|
|
13
|
-
const {
|
|
12
|
+
const { NodeFileRangeSource, parseUint8Snapshot } = require('./pikelet-artifact-common.js');
|
|
13
|
+
const { PikeletRangeArtifact, buildRangeArtifact, buildRangeArtifactFile } = require('./pikelet-artifact-range.js');
|
|
14
14
|
const {
|
|
15
|
-
|
|
15
|
+
PikeletSketchArtifact, createSketchScanner,
|
|
16
16
|
buildSketchArtifact, buildSketchArtifactBytes, buildSketchArtifactFile, exportSketchArtifact,
|
|
17
|
-
} = require('./
|
|
17
|
+
} = require('./pikelet-artifact-sketch.js');
|
|
18
18
|
|
|
19
19
|
module.exports = {
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
PikeletRangeArtifact,
|
|
21
|
+
PikeletSketchArtifact,
|
|
22
22
|
createSketchScanner,
|
|
23
23
|
NodeFileRangeSource,
|
|
24
24
|
buildRangeArtifact,
|