pikelet-wasm 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +340 -4
- package/NOTICE +5 -0
- package/README.md +226 -475
- package/complete/builder.d.ts +4 -4
- package/complete/builder.mjs +26 -9
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/fusion.mjs +72 -0
- package/complete/index.d.ts +34 -7
- package/complete/index.mjs +258 -129
- package/complete/inline-transformer.mjs +119 -10
- package/complete/lexical.mjs +11 -11
- package/complete/retrieval-abstention.mjs +136 -31
- package/complete/sources.mjs +22 -1
- package/complete/wordpiece.mjs +22 -0
- package/dist/engine.js +1 -1
- package/dist/engine.scalar.js +1 -1
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +47 -82
- package/{pancake-artifact-common.js → src/artifact/pikelet-artifact-common.js} +42 -42
- package/{pancake-artifact-range.js → src/artifact/pikelet-artifact-range.js} +43 -36
- package/{pancake-artifact-sketch.js → src/artifact/pikelet-artifact-sketch.js} +105 -68
- package/{pancake-artifact.d.ts → src/artifact/pikelet-artifact.d.ts} +11 -11
- package/{pancake-artifact.js → src/artifact/pikelet-artifact.js} +10 -10
- package/{pancake-core.js → src/core/pikelet-core.js} +141 -140
- package/{pancake-errors.js → src/errors/pikelet-errors.js} +9 -8
- package/{pancake.d.ts → src/pikelet.d.ts} +55 -55
- package/{pancake.js → src/pikelet.js} +47 -47
- package/{pancake.node.mjs → src/pikelet.node.mjs} +49 -49
- package/{pancake.web.d.ts → src/pikelet.web.d.ts} +13 -13
- package/{pancake.web.mjs → src/pikelet.web.mjs} +24 -24
- package/{pancake.workerd.mjs → src/pikelet.workerd.mjs} +22 -22
- /package/{pancake-loader.js → src/loader/pikelet-loader.js} +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
// Shared by the range and sketch artifact readers/builders: read budgets and
|
|
3
3
|
// range validation, range sources, the result heap, snapshot parsing, and
|
|
4
|
-
// SHA-256 helpers. Not a public entry;
|
|
5
|
-
// Split out of
|
|
4
|
+
// SHA-256 helpers. Not a public entry; pikelet-artifact.js re-exports what is public.
|
|
5
|
+
// Split out of pikelet-artifact.js (the public entry, which re-exports the
|
|
6
6
|
// three parts); see that file for the module map.
|
|
7
7
|
|
|
8
|
-
const {
|
|
8
|
+
const { pikeletError, PIKELET_ERROR_CODES } = require('../errors/pikelet-errors.js');
|
|
9
9
|
|
|
10
|
-
const
|
|
10
|
+
const PIKELET_MAGIC = 0x504E434B;
|
|
11
11
|
const V1_ENVELOPE_HEADER_SIZE = 24;
|
|
12
12
|
const V2_ENVELOPE_HEADER_SIZE = 20;
|
|
13
13
|
const V3_ENVELOPE_HEADER_SIZE = 32;
|
|
@@ -54,16 +54,16 @@ async function mapLimit(items, limit, fn) {
|
|
|
54
54
|
function checkArtifactRange(source, offset, length, label, limit = MAX_ARTIFACT_READ_BYTES) {
|
|
55
55
|
if (!Number.isSafeInteger(offset) || !Number.isSafeInteger(length)
|
|
56
56
|
|| offset < 0 || length < 0 || !Number.isSafeInteger(offset + length)) {
|
|
57
|
-
throw
|
|
57
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
58
58
|
`Artifact ${label} range is out of bounds`, { offset, length });
|
|
59
59
|
}
|
|
60
60
|
const max = Math.min(limit, MAX_ARTIFACT_READ_BYTES);
|
|
61
61
|
if (length > max) {
|
|
62
|
-
throw
|
|
62
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
63
63
|
`Artifact ${label} range exceeds the maximum read size`, { length, max });
|
|
64
64
|
}
|
|
65
65
|
if (source && Number.isSafeInteger(source.size) && offset + length > source.size) {
|
|
66
|
-
throw
|
|
66
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
67
67
|
`Artifact ${label} range extends past the source size`, { offset, length, size: source.size });
|
|
68
68
|
}
|
|
69
69
|
return length;
|
|
@@ -84,7 +84,7 @@ function resolveMaxReadBytes(value) {
|
|
|
84
84
|
if (value === undefined) return DEFAULT_OPEN_READ_BYTES;
|
|
85
85
|
if (value === Infinity) return MAX_ARTIFACT_READ_BYTES;
|
|
86
86
|
if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
|
|
87
|
-
throw
|
|
87
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
88
88
|
'maxReadBytes must be a positive number or Infinity', { maxReadBytes: value });
|
|
89
89
|
}
|
|
90
90
|
return value;
|
|
@@ -95,7 +95,7 @@ function asUint8Array(bytes) {
|
|
|
95
95
|
if (ArrayBuffer.isView(bytes)) {
|
|
96
96
|
return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
97
97
|
}
|
|
98
|
-
throw
|
|
98
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'Range source returned a non-binary value');
|
|
99
99
|
}
|
|
100
100
|
|
|
101
101
|
// Both readers accept a maxCacheBytes option: undefined keeps the default,
|
|
@@ -106,7 +106,7 @@ function resolveMaxCacheBytes(value, floorBytes, defaultBytes) {
|
|
|
106
106
|
if (value === undefined) return defaultBytes;
|
|
107
107
|
if (value === Infinity) return Infinity;
|
|
108
108
|
if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
|
|
109
|
-
throw
|
|
109
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'maxCacheBytes must be a positive number or Infinity', { maxCacheBytes: value });
|
|
110
110
|
}
|
|
111
111
|
return Math.max(value, floorBytes);
|
|
112
112
|
}
|
|
@@ -213,7 +213,7 @@ class NodeFileRangeSource {
|
|
|
213
213
|
|
|
214
214
|
async read(offset, length) {
|
|
215
215
|
if (this.fd === null) {
|
|
216
|
-
throw
|
|
216
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'NodeFileRangeSource is closed', { filePath: this.filePath });
|
|
217
217
|
}
|
|
218
218
|
// Defense in depth: the callers validate artifact-derived ranges, but
|
|
219
219
|
// a future caller must not be able to drive a giant Buffer.alloc or a
|
|
@@ -221,7 +221,7 @@ class NodeFileRangeSource {
|
|
|
221
221
|
if (!Number.isSafeInteger(offset) || !Number.isSafeInteger(length)
|
|
222
222
|
|| offset < 0 || length < 0 || length > MAX_ARTIFACT_READ_BYTES
|
|
223
223
|
|| offset + length > this.size) {
|
|
224
|
-
throw
|
|
224
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
225
225
|
'NodeFileRangeSource.read() range is out of bounds', { offset, length, size: this.size });
|
|
226
226
|
}
|
|
227
227
|
const buffer = Buffer.alloc(length);
|
|
@@ -245,7 +245,7 @@ class NodeFileRangeSource {
|
|
|
245
245
|
}
|
|
246
246
|
|
|
247
247
|
// The artifact readers apply the same query-vector contract as the core
|
|
248
|
-
// engine (
|
|
248
|
+
// engine (pikelet-core.js assertNumericVector / validateVectorValues): a
|
|
249
249
|
// plain array must hold real numbers (Float32Array.from would coerce '1',
|
|
250
250
|
// '', true silently), every component must be finite for every metric (an
|
|
251
251
|
// L2 search over NaN would otherwise return distance: NaN results), and a
|
|
@@ -253,31 +253,31 @@ class NodeFileRangeSource {
|
|
|
253
253
|
function normalizeQuery(query, dim, metric = 0) {
|
|
254
254
|
if (!(query instanceof Float32Array)) {
|
|
255
255
|
if (!Array.isArray(query)) {
|
|
256
|
-
throw
|
|
256
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR, 'search() query must be a Float32Array or number[]');
|
|
257
257
|
}
|
|
258
258
|
for (let i = 0; i < query.length; i++) {
|
|
259
259
|
if (typeof query[i] !== 'number') {
|
|
260
|
-
throw
|
|
260
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR,
|
|
261
261
|
`search() query must contain only numbers; found ${typeof query[i]} at index ${i}`, { index: i, actualType: typeof query[i] });
|
|
262
262
|
}
|
|
263
263
|
}
|
|
264
264
|
query = Float32Array.from(query);
|
|
265
265
|
}
|
|
266
266
|
if (query.length !== dim) {
|
|
267
|
-
throw
|
|
267
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH, `search() query dimension ${query.length} does not match artifact dimension ${dim}`, { queryDim: query.length, dim });
|
|
268
268
|
}
|
|
269
269
|
let norm = 0;
|
|
270
270
|
for (let i = 0; i < query.length; i++) {
|
|
271
271
|
const value = query[i];
|
|
272
272
|
if (!Number.isFinite(value)) {
|
|
273
|
-
throw
|
|
273
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR, 'search() query contains non-finite value (NaN or Infinity)', { index: i, reason: 'non_finite' });
|
|
274
274
|
}
|
|
275
275
|
norm += value * value;
|
|
276
276
|
}
|
|
277
277
|
if (metric === 1) {
|
|
278
278
|
norm = Math.sqrt(norm);
|
|
279
279
|
if (!Number.isFinite(norm) || norm <= 1e-30) {
|
|
280
|
-
throw
|
|
280
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR, 'search() query has invalid cosine norm', { reason: 'invalid_cosine_norm' });
|
|
281
281
|
}
|
|
282
282
|
const normalized = new Float32Array(query.length);
|
|
283
283
|
for (let i = 0; i < query.length; i++) normalized[i] = query[i] / norm;
|
|
@@ -292,7 +292,7 @@ function normalizeQuery(query, dim, metric = 0) {
|
|
|
292
292
|
// count results, so a k of 1e9 over 300 rows must not allocate 1e9 slots.
|
|
293
293
|
function resolveSearchK(k, count) {
|
|
294
294
|
if (!Number.isSafeInteger(k) || k < 1) {
|
|
295
|
-
throw
|
|
295
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'search() k must be a positive integer', { k });
|
|
296
296
|
}
|
|
297
297
|
return Math.min(k, count);
|
|
298
298
|
}
|
|
@@ -302,41 +302,41 @@ function resolveSearchK(k, count) {
|
|
|
302
302
|
function resolveOptionalPositiveInt(value, name, fallback) {
|
|
303
303
|
if (value === undefined || value === null || value === 0) return fallback;
|
|
304
304
|
if (!Number.isSafeInteger(value) || value < 1) {
|
|
305
|
-
throw
|
|
305
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, `search() ${name} must be a positive integer (0 or undefined selects the default)`, { [name]: value });
|
|
306
306
|
}
|
|
307
307
|
return value;
|
|
308
308
|
}
|
|
309
309
|
|
|
310
310
|
function unwrapSnapshot(bytes) {
|
|
311
311
|
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
312
|
-
if (bytes.byteLength < 4 || view.getUint32(0, true) !==
|
|
312
|
+
if (bytes.byteLength < 4 || view.getUint32(0, true) !== PIKELET_MAGIC) return bytes;
|
|
313
313
|
// Envelope fields are untrusted: every size read from them must be
|
|
314
314
|
// checked against the actual buffer before slicing, so a corrupt
|
|
315
315
|
// envelope fails closed with a coded error instead of a raw RangeError.
|
|
316
316
|
if (bytes.byteLength < 8) {
|
|
317
|
-
throw
|
|
317
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Pikelet envelope is truncated', { byteLength: bytes.byteLength });
|
|
318
318
|
}
|
|
319
319
|
const version = view.getUint32(4, true);
|
|
320
320
|
if (version === 1 || version === 2) {
|
|
321
321
|
const headerSize = version === 1 ? V1_ENVELOPE_HEADER_SIZE : V2_ENVELOPE_HEADER_SIZE;
|
|
322
322
|
if (bytes.byteLength < headerSize) {
|
|
323
|
-
throw
|
|
323
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Pikelet envelope is truncated', { version, byteLength: bytes.byteLength });
|
|
324
324
|
}
|
|
325
325
|
return bytes.subarray(headerSize);
|
|
326
326
|
}
|
|
327
327
|
if (version === 3) {
|
|
328
328
|
if (bytes.byteLength < V3_ENVELOPE_HEADER_SIZE) {
|
|
329
|
-
throw
|
|
329
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Pikelet envelope is truncated', { version, byteLength: bytes.byteLength });
|
|
330
330
|
}
|
|
331
331
|
const mappingCount = view.getUint32(24, true);
|
|
332
332
|
const rawSize = view.getUint32(28, true);
|
|
333
333
|
const rawOffset = V3_ENVELOPE_HEADER_SIZE + mappingCount * MAPPING_ENTRY_SIZE;
|
|
334
334
|
if (rawOffset + rawSize > bytes.byteLength) {
|
|
335
|
-
throw
|
|
335
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Pikelet envelope declares more data than the snapshot contains', { mappingCount, rawSize, byteLength: bytes.byteLength });
|
|
336
336
|
}
|
|
337
337
|
return bytes.subarray(rawOffset, rawOffset + rawSize);
|
|
338
338
|
}
|
|
339
|
-
throw
|
|
339
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, `Unsupported Pikelet envelope version ${version}`, { version });
|
|
340
340
|
}
|
|
341
341
|
|
|
342
342
|
function parseUint8Snapshot(bytes) {
|
|
@@ -346,7 +346,7 @@ function parseUint8Snapshot(bytes) {
|
|
|
346
346
|
// Snapshot bytes are untrusted: sizes read from the header drive every
|
|
347
347
|
// subsequent read, so each read checks the remaining buffer and fails
|
|
348
348
|
// closed with a coded error instead of a raw RangeError.
|
|
349
|
-
const truncated = () =>
|
|
349
|
+
const truncated = () => pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'uint8 snapshot is truncated', { offset, byteLength: raw.byteLength });
|
|
350
350
|
const u32 = () => {
|
|
351
351
|
if (raw.byteLength - offset < 4) throw truncated();
|
|
352
352
|
const value = view.getUint32(offset, true);
|
|
@@ -371,22 +371,22 @@ function parseUint8Snapshot(bytes) {
|
|
|
371
371
|
const metric = u32();
|
|
372
372
|
const efConstruction = u32();
|
|
373
373
|
if (magic !== UINT8_HNSW_MAGIC_V1) {
|
|
374
|
-
throw
|
|
374
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Search Artifact export currently supports uint8 Pikelet snapshots only');
|
|
375
375
|
}
|
|
376
376
|
// Contract §6: reject unknown future versions instead of parsing them
|
|
377
377
|
// as v2 — a changed layout must fail closed, not misparse.
|
|
378
378
|
if (version > 2) {
|
|
379
|
-
throw
|
|
379
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported uint8 snapshot format version', { version });
|
|
380
380
|
}
|
|
381
381
|
if (metric !== 0 && metric !== 1) {
|
|
382
|
-
throw
|
|
382
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported uint8 snapshot metric', { metric });
|
|
383
383
|
}
|
|
384
384
|
// Same structural sanity the engine's own deserialize enforces; the
|
|
385
385
|
// level cap mirrors MAX_DESERIALIZE_LEVEL in src/uint8_float_hnsw.hpp.
|
|
386
386
|
if (dim < 1 || count < 1 || entryPoint >= count || maxLevel > 64
|
|
387
387
|
|| !Number.isSafeInteger(count * dim)
|
|
388
388
|
|| raw.byteLength - offset < count * 8 + count * dim) {
|
|
389
|
-
throw
|
|
389
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'uint8 snapshot header is inconsistent', { dim, count, entryPoint, maxLevel, byteLength: raw.byteLength });
|
|
390
390
|
}
|
|
391
391
|
|
|
392
392
|
const scales = new Float32Array(count);
|
|
@@ -405,7 +405,7 @@ function parseUint8Snapshot(bytes) {
|
|
|
405
405
|
// Same envelope as the engine: create() accepts M in [2, 128] and fixes
|
|
406
406
|
// M0 = 2*M, and its own import rejects M outside that range.
|
|
407
407
|
if (!(M >= 2 && M <= 128) || !(M0 >= M && M0 <= 256)) {
|
|
408
|
-
throw
|
|
408
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'uint8 snapshot graph parameters are implausible', { M, M0 });
|
|
409
409
|
}
|
|
410
410
|
// Edge layout follows the engine's deserializer (uint8_float_hnsw.hpp):
|
|
411
411
|
// version >= 2 stores {u32 id, f32 distance} per edge, version 1 stores
|
|
@@ -418,21 +418,21 @@ function parseUint8Snapshot(bytes) {
|
|
|
418
418
|
for (let id = 0; id < count; id++) {
|
|
419
419
|
const level = u32();
|
|
420
420
|
if (level > maxLevel) {
|
|
421
|
-
throw
|
|
421
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'uint8 snapshot node level exceeds header maxLevel', { id, level, maxLevel });
|
|
422
422
|
}
|
|
423
423
|
levels[id] = level;
|
|
424
424
|
for (let l = 0; l <= level; l++) {
|
|
425
425
|
const size = u32();
|
|
426
426
|
const cap = l === 0 ? M0 : M;
|
|
427
427
|
if (size > cap) {
|
|
428
|
-
throw
|
|
428
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'uint8 snapshot adjacency exceeds the graph parameter bound', { id, level: l, size, cap });
|
|
429
429
|
}
|
|
430
430
|
if (raw.byteLength - offset < size * edgeBytes) throw truncated();
|
|
431
431
|
const edges = new Uint32Array(size);
|
|
432
432
|
for (let e = 0; e < size; e++) {
|
|
433
433
|
const neighbor = u32();
|
|
434
434
|
if (neighbor >= count) {
|
|
435
|
-
throw
|
|
435
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'uint8 snapshot neighbor id is out of range', { id, level: l, neighbor, count });
|
|
436
436
|
}
|
|
437
437
|
edges[e] = neighbor;
|
|
438
438
|
if (version >= 2) offset += 4; // the serialized float edge distance, unused here
|
|
@@ -468,12 +468,12 @@ async function sha256BytesAsync(bytes) {
|
|
|
468
468
|
async function verifySha256(bytes, expected, label) {
|
|
469
469
|
const digest = await sha256BytesAsync(bytes);
|
|
470
470
|
if (!digest) {
|
|
471
|
-
throw
|
|
471
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
472
472
|
'Artifact verification requested but no crypto backend is available; pass verify:false to skip');
|
|
473
473
|
}
|
|
474
474
|
for (let b = 0; b < 32; b++) {
|
|
475
475
|
if (digest[b] !== expected[b]) {
|
|
476
|
-
throw
|
|
476
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, `${label} failed hash verification`);
|
|
477
477
|
}
|
|
478
478
|
}
|
|
479
479
|
}
|
|
@@ -498,7 +498,7 @@ async function verifySegmentSha256(source, offset, totalBytes, expected, label,
|
|
|
498
498
|
const len = Math.min(chunkBytes, totalBytes - done);
|
|
499
499
|
const bytes = await readChecked(source, offset + done, len, label);
|
|
500
500
|
if (bytes.byteLength !== len) {
|
|
501
|
-
throw
|
|
501
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
502
502
|
`Artifact ${label} is truncated`, { offset: offset + done, expected: len, actual: bytes.byteLength });
|
|
503
503
|
}
|
|
504
504
|
hash.update(bytes);
|
|
@@ -507,20 +507,20 @@ async function verifySegmentSha256(source, offset, totalBytes, expected, label,
|
|
|
507
507
|
const digest = new Uint8Array(hash.digest());
|
|
508
508
|
for (let b = 0; b < 32; b++) {
|
|
509
509
|
if (digest[b] !== expected[b]) {
|
|
510
|
-
throw
|
|
510
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, `${label} failed hash verification`);
|
|
511
511
|
}
|
|
512
512
|
}
|
|
513
513
|
return;
|
|
514
514
|
}
|
|
515
515
|
const oneShotLimit = options.oneShotLimit || DEFAULT_OPEN_READ_BYTES;
|
|
516
516
|
if (totalBytes > oneShotLimit) {
|
|
517
|
-
throw
|
|
517
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
518
518
|
`Artifact ${label} is too large to verify without a streaming crypto backend`,
|
|
519
519
|
{ totalBytes, oneShotLimit });
|
|
520
520
|
}
|
|
521
521
|
const bytes = await readChecked(source, offset, totalBytes, label, oneShotLimit);
|
|
522
522
|
if (bytes.byteLength !== totalBytes) {
|
|
523
|
-
throw
|
|
523
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
524
524
|
`Artifact ${label} is truncated`, { offset, expected: totalBytes, actual: bytes.byteLength });
|
|
525
525
|
}
|
|
526
526
|
await verifySha256(bytes, expected, label);
|
|
@@ -529,7 +529,7 @@ async function verifySegmentSha256(source, offset, totalBytes, expected, label,
|
|
|
529
529
|
module.exports = {
|
|
530
530
|
resolveSearchK,
|
|
531
531
|
resolveOptionalPositiveInt,
|
|
532
|
-
|
|
532
|
+
PIKELET_MAGIC,
|
|
533
533
|
V1_ENVELOPE_HEADER_SIZE,
|
|
534
534
|
V2_ENVELOPE_HEADER_SIZE,
|
|
535
535
|
V3_ENVELOPE_HEADER_SIZE,
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
'use strict';
|
|
2
|
-
// Range artifact profile (.
|
|
3
|
-
//
|
|
4
|
-
// Split out of
|
|
2
|
+
// Range artifact profile (.pikelet-range) — spec/SEARCH_ARTIFACT_CONTRACT.md 9.2:
|
|
3
|
+
// PikeletRangeArtifact reader and the range-artifact builder.
|
|
4
|
+
// Split out of pikelet-artifact.js (the public entry, which re-exports the
|
|
5
5
|
// three parts); see that file for the module map.
|
|
6
6
|
|
|
7
|
-
const {
|
|
7
|
+
const { pikeletError, PIKELET_ERROR_CODES } = require('../errors/pikelet-errors.js');
|
|
8
8
|
const {
|
|
9
9
|
MAX_COALESCED_RANGE_BYTES,
|
|
10
10
|
readChecked,
|
|
@@ -22,7 +22,7 @@ const {
|
|
|
22
22
|
verifySegmentSha256,
|
|
23
23
|
resolveSearchK,
|
|
24
24
|
resolveOptionalPositiveInt,
|
|
25
|
-
} = require('./
|
|
25
|
+
} = require('./pikelet-artifact-common.js');
|
|
26
26
|
|
|
27
27
|
const RANGE_MAGIC = 0x31415250; // PRA1
|
|
28
28
|
const HEADER_BYTES = 128;
|
|
@@ -35,7 +35,7 @@ const RANGE_DIGESTS_OFFSET = 128;
|
|
|
35
35
|
const RANGE_DIGEST_BYTES = 32;
|
|
36
36
|
const ROUTER_LOCATION_MASK = 0x80000000;
|
|
37
37
|
const LOCATION_ORDINAL_MASK = 0x7fffffff;
|
|
38
|
-
class
|
|
38
|
+
class PikeletRangeArtifact {
|
|
39
39
|
constructor(source, header, idMap, options = {}) {
|
|
40
40
|
this.source = source;
|
|
41
41
|
this.version = header.version;
|
|
@@ -77,11 +77,11 @@ class PancakeRangeArtifact {
|
|
|
77
77
|
|
|
78
78
|
static async open(source, options = {}) {
|
|
79
79
|
if (!source || typeof source.read !== 'function') {
|
|
80
|
-
throw
|
|
80
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'PikeletRangeArtifact.open() requires a range source with read(offset, length)');
|
|
81
81
|
}
|
|
82
82
|
const headerBytes = await readChecked(source, 0, HEADER_BYTES, 'header');
|
|
83
83
|
if (headerBytes.byteLength < HEADER_BYTES) {
|
|
84
|
-
throw
|
|
84
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact header is truncated');
|
|
85
85
|
}
|
|
86
86
|
const header = parseHeader(headerBytes);
|
|
87
87
|
// Resolved before construction: the id-map read below is driven by an
|
|
@@ -91,7 +91,7 @@ class PancakeRangeArtifact {
|
|
|
91
91
|
if (header.version >= 3) {
|
|
92
92
|
const digestBytes = await readChecked(source, RANGE_DIGESTS_OFFSET, RANGE_DIGEST_BYTES * 3, 'header digests');
|
|
93
93
|
if (digestBytes.byteLength !== RANGE_DIGEST_BYTES * 3) {
|
|
94
|
-
throw
|
|
94
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact header digests are truncated');
|
|
95
95
|
}
|
|
96
96
|
digests = {
|
|
97
97
|
idMap: new Uint8Array(digestBytes.subarray(0, RANGE_DIGEST_BYTES)),
|
|
@@ -101,12 +101,12 @@ class PancakeRangeArtifact {
|
|
|
101
101
|
}
|
|
102
102
|
const idMapBytes = await readChecked(source, header.idMapOffset, header.count * 4, 'id map', maxReadBytes);
|
|
103
103
|
if (idMapBytes.byteLength !== header.count * 4) {
|
|
104
|
-
throw
|
|
104
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact id map is truncated');
|
|
105
105
|
}
|
|
106
106
|
const copied = new Uint8Array(idMapBytes.byteLength);
|
|
107
107
|
copied.set(idMapBytes);
|
|
108
108
|
const idMap = new Uint32Array(copied.buffer);
|
|
109
|
-
const artifact = new
|
|
109
|
+
const artifact = new PikeletRangeArtifact(source, header, idMap, options);
|
|
110
110
|
artifact.digests = digests;
|
|
111
111
|
if (digests && artifact.verify) {
|
|
112
112
|
await verifySha256(copied, digests.idMap, 'Range artifact id map');
|
|
@@ -124,7 +124,7 @@ class PancakeRangeArtifact {
|
|
|
124
124
|
// open must release it, or every corrupt artifact leaks an fd.
|
|
125
125
|
const source = new NodeFileRangeSource(filePath);
|
|
126
126
|
try {
|
|
127
|
-
return await
|
|
127
|
+
return await PikeletRangeArtifact.open(source, options);
|
|
128
128
|
} catch (err) {
|
|
129
129
|
await source.close().catch(() => {});
|
|
130
130
|
throw err;
|
|
@@ -179,7 +179,7 @@ class PancakeRangeArtifact {
|
|
|
179
179
|
// complete-profile manifest.
|
|
180
180
|
async verifyBaseSegment(options = {}) {
|
|
181
181
|
if (!this.digests) {
|
|
182
|
-
throw
|
|
182
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
183
183
|
'Range artifact predates segment digests (format v3); nothing to verify against', { version: this.version });
|
|
184
184
|
}
|
|
185
185
|
const bytes = this.baseCount * this.recordBytes;
|
|
@@ -199,19 +199,19 @@ class PancakeRangeArtifact {
|
|
|
199
199
|
|
|
200
200
|
recordAddressForId(id) {
|
|
201
201
|
if (!Number.isInteger(id) || id < 0 || id >= this.count) {
|
|
202
|
-
throw
|
|
202
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, `Node id ${id} is outside artifact bounds`, { id, count: this.count });
|
|
203
203
|
}
|
|
204
204
|
const location = this.originalToLocation[id];
|
|
205
205
|
if (this.version >= 2) {
|
|
206
206
|
const ordinal = location & LOCATION_ORDINAL_MASK;
|
|
207
207
|
if ((location & ROUTER_LOCATION_MASK) !== 0) {
|
|
208
|
-
if (ordinal >= this.routerCount) throw
|
|
208
|
+
if (ordinal >= this.routerCount) throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, `Router ordinal ${ordinal} is outside artifact`);
|
|
209
209
|
return this.routerRecordsOffset + ordinal * this.recordBytes;
|
|
210
210
|
}
|
|
211
|
-
if (ordinal >= this.baseCount) throw
|
|
211
|
+
if (ordinal >= this.baseCount) throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, `Base ordinal ${ordinal} is outside artifact`);
|
|
212
212
|
return this.baseRecordsOffset + ordinal * this.recordBytes;
|
|
213
213
|
}
|
|
214
|
-
if (location >= this.count) throw
|
|
214
|
+
if (location >= this.count) throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, `Node id ${id} is not addressable in artifact`);
|
|
215
215
|
return this.recordsOffset + location * this.recordBytes;
|
|
216
216
|
}
|
|
217
217
|
|
|
@@ -245,7 +245,7 @@ class PancakeRangeArtifact {
|
|
|
245
245
|
await this.prefetch([id]);
|
|
246
246
|
const node = this.cachedNode(id);
|
|
247
247
|
if (node === undefined) {
|
|
248
|
-
throw
|
|
248
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
249
249
|
'Range artifact record for id was not resolved by its address', { id });
|
|
250
250
|
}
|
|
251
251
|
return node;
|
|
@@ -281,7 +281,7 @@ class PancakeRangeArtifact {
|
|
|
281
281
|
const bytes = end - start;
|
|
282
282
|
const buffer = await readChecked(this.source, start, bytes, 'record');
|
|
283
283
|
if (buffer.byteLength !== bytes) {
|
|
284
|
-
throw
|
|
284
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact record read returned a truncated range', { offset: start, bytes, actual: buffer.byteLength });
|
|
285
285
|
}
|
|
286
286
|
return { start, end, bytes, buffer };
|
|
287
287
|
};
|
|
@@ -297,7 +297,7 @@ class PancakeRangeArtifact {
|
|
|
297
297
|
// poison the cache under an attacker-chosen key and searches
|
|
298
298
|
// for the real id would die uncoded.
|
|
299
299
|
if (originalId >= this.count || this.recordAddressForId(originalId) !== start + off) {
|
|
300
|
-
throw
|
|
300
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
301
301
|
'Range artifact record id does not match its address',
|
|
302
302
|
{ originalId, address: start + off });
|
|
303
303
|
}
|
|
@@ -331,7 +331,7 @@ class PancakeRangeArtifact {
|
|
|
331
331
|
const bytes = this.routerCount * this.recordBytes;
|
|
332
332
|
const buffer = await readChecked(this.source, this.routerRecordsOffset, bytes, 'router segment', this.maxReadBytes);
|
|
333
333
|
if (buffer.byteLength !== bytes) {
|
|
334
|
-
throw
|
|
334
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact router segment is truncated');
|
|
335
335
|
}
|
|
336
336
|
if (this.digests && this.verify) {
|
|
337
337
|
await verifySha256(buffer, this.digests.router, 'Range artifact router segment');
|
|
@@ -344,7 +344,7 @@ class PancakeRangeArtifact {
|
|
|
344
344
|
// Same untrusted-id check as decodeRange: the embedded id must map
|
|
345
345
|
// back to this router slot's address.
|
|
346
346
|
if (originalId >= this.count || this.recordAddressForId(originalId) !== address) {
|
|
347
|
-
throw
|
|
347
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
348
348
|
'Range artifact router record id does not match its address',
|
|
349
349
|
{ originalId, address });
|
|
350
350
|
}
|
|
@@ -362,13 +362,13 @@ class PancakeRangeArtifact {
|
|
|
362
362
|
// Records are untrusted bytes. Counts beyond the header geometry would
|
|
363
363
|
// read filler (or zeros) as edges; fail closed instead.
|
|
364
364
|
if (level > this.maxLevel || baseCount > this.M0) {
|
|
365
|
-
throw
|
|
365
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact record structure is inconsistent', { id, level, baseCount });
|
|
366
366
|
}
|
|
367
367
|
const upperCounts = new Uint16Array(this.maxLevel);
|
|
368
368
|
for (let i = 0; i < this.maxLevel; i++) {
|
|
369
369
|
upperCounts[i] = readU16(view, state);
|
|
370
370
|
if (upperCounts[i] > this.M) {
|
|
371
|
-
throw
|
|
371
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact record structure is inconsistent', { id, level: i + 1, edges: upperCounts[i] });
|
|
372
372
|
}
|
|
373
373
|
}
|
|
374
374
|
const qdata = new Uint8Array(this.dim);
|
|
@@ -376,12 +376,19 @@ class PancakeRangeArtifact {
|
|
|
376
376
|
state.offset += this.dim;
|
|
377
377
|
const scale = readF32(view, state);
|
|
378
378
|
const offset = readF32(view, state);
|
|
379
|
+
// A NaN/Infinity affine pair makes every distance to this node
|
|
380
|
+
// non-finite; NaN compares false everywhere, so the node can sit in
|
|
381
|
+
// the result heap without ever being displaced. Digests do not catch
|
|
382
|
+
// this (the producer signs whatever bytes it wrote), so decode does.
|
|
383
|
+
if (!Number.isFinite(scale) || !Number.isFinite(offset)) {
|
|
384
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact record has a non-finite scale or offset', { id, scale, offset });
|
|
385
|
+
}
|
|
379
386
|
const base = new Uint32Array(baseCount);
|
|
380
387
|
for (let i = 0; i < this.M0; i++) {
|
|
381
388
|
const neighbor = readU32(view, state);
|
|
382
389
|
if (i < baseCount) {
|
|
383
390
|
if (neighbor >= this.count) {
|
|
384
|
-
throw
|
|
391
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact record has an out-of-bounds neighbor', { id, neighbor, count: this.count });
|
|
385
392
|
}
|
|
386
393
|
base[i] = neighbor;
|
|
387
394
|
}
|
|
@@ -393,7 +400,7 @@ class PancakeRangeArtifact {
|
|
|
393
400
|
const neighbor = readU32(view, state);
|
|
394
401
|
if (i < edges.length) {
|
|
395
402
|
if (neighbor >= this.count) {
|
|
396
|
-
throw
|
|
403
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact record has an out-of-bounds neighbor', { id, neighbor, count: this.count });
|
|
397
404
|
}
|
|
398
405
|
edges[i] = neighbor;
|
|
399
406
|
}
|
|
@@ -511,7 +518,7 @@ class PancakeRangeArtifact {
|
|
|
511
518
|
|
|
512
519
|
const top = results.items.sort(compareDistancesAsc).slice(0, k);
|
|
513
520
|
// Traversal orders by squared L2; the API contract (README "Distance
|
|
514
|
-
// values") reports Euclidean, matching
|
|
521
|
+
// values") reports Euclidean, matching PikeletIndex.search.
|
|
515
522
|
if (this.metric !== 1) {
|
|
516
523
|
for (const hit of top) hit.distance = Math.sqrt(hit.distance);
|
|
517
524
|
}
|
|
@@ -528,12 +535,12 @@ function parseHeader(headerBytes) {
|
|
|
528
535
|
const state = { offset: 0 };
|
|
529
536
|
const magic = readU32(view, state);
|
|
530
537
|
if (magic !== RANGE_MAGIC) {
|
|
531
|
-
throw
|
|
538
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Not a Pikelet range artifact', { magic });
|
|
532
539
|
}
|
|
533
540
|
const version = readU32(view, state);
|
|
534
541
|
const kind = readU32(view, state);
|
|
535
542
|
if (kind !== RANGE_KIND_U8) {
|
|
536
|
-
throw
|
|
543
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported Pikelet range artifact kind', { kind });
|
|
537
544
|
}
|
|
538
545
|
const header = {
|
|
539
546
|
version,
|
|
@@ -556,7 +563,7 @@ function parseHeader(headerBytes) {
|
|
|
556
563
|
};
|
|
557
564
|
if (header.metric !== 0) {
|
|
558
565
|
if (header.metric !== 1) {
|
|
559
|
-
throw
|
|
566
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported Pikelet range artifact metric', { metric: header.metric });
|
|
560
567
|
}
|
|
561
568
|
}
|
|
562
569
|
if (version >= 2) {
|
|
@@ -569,7 +576,7 @@ function parseHeader(headerBytes) {
|
|
|
569
576
|
header.baseRecordsOffset = header.recordsOffset;
|
|
570
577
|
}
|
|
571
578
|
if (version < 1 || version > RANGE_VERSION) {
|
|
572
|
-
throw
|
|
579
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported Pikelet range artifact version', { version });
|
|
573
580
|
}
|
|
574
581
|
// recordBytes must agree with the layout the geometry implies, or
|
|
575
582
|
// decodeNode would read past record boundaries on a corrupt header.
|
|
@@ -580,7 +587,7 @@ function parseHeader(headerBytes) {
|
|
|
580
587
|
|| header.entryPoint >= header.count
|
|
581
588
|
|| header.recordBytes !== expectedRecordBytes
|
|
582
589
|
|| (version >= 2 && header.routerCount + header.baseCount !== header.count)) {
|
|
583
|
-
throw
|
|
590
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact header geometry is inconsistent', {
|
|
584
591
|
dim: header.dim, count: header.count, maxLevel: header.maxLevel,
|
|
585
592
|
M: header.M, M0: header.M0, recordBytes: header.recordBytes, expectedRecordBytes,
|
|
586
593
|
});
|
|
@@ -662,7 +669,7 @@ function identityPermutation(count) {
|
|
|
662
669
|
function buildOrdinalToOriginal(index, layout) {
|
|
663
670
|
if (layout === 'identity') return identityPermutation(index.count);
|
|
664
671
|
if (layout === 'rcm') return reverseCuthillMckeePermutation(index);
|
|
665
|
-
throw
|
|
672
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, `Unsupported Search Artifact layout '${layout}'`, { layout });
|
|
666
673
|
}
|
|
667
674
|
|
|
668
675
|
function exportSplitArtifact(index, outPath, options = {}) {
|
|
@@ -742,7 +749,7 @@ function exportSplitArtifact(index, outPath, options = {}) {
|
|
|
742
749
|
fs.closeSync(fd);
|
|
743
750
|
}
|
|
744
751
|
return {
|
|
745
|
-
format: '
|
|
752
|
+
format: 'pikelet-range-artifact',
|
|
746
753
|
formatVersion: RANGE_VERSION,
|
|
747
754
|
integrity: {
|
|
748
755
|
idMapSha256: idMapDigest.toString('hex'),
|
|
@@ -778,7 +785,7 @@ function exportSplitArtifact(index, outPath, options = {}) {
|
|
|
778
785
|
|
|
779
786
|
function buildRangeArtifact(snapshotBytes, outPath, options = {}) {
|
|
780
787
|
if (typeof outPath !== 'string' || outPath.length === 0) {
|
|
781
|
-
throw
|
|
788
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'buildRangeArtifact() requires an output path');
|
|
782
789
|
}
|
|
783
790
|
const index = parseUint8Snapshot(snapshotBytes);
|
|
784
791
|
return exportSplitArtifact(index, outPath, options);
|
|
@@ -791,7 +798,7 @@ function buildRangeArtifactFile(snapshotPath, outPath, options = {}) {
|
|
|
791
798
|
}
|
|
792
799
|
|
|
793
800
|
module.exports = {
|
|
794
|
-
|
|
801
|
+
PikeletRangeArtifact,
|
|
795
802
|
buildRangeArtifact,
|
|
796
803
|
buildRangeArtifactFile,
|
|
797
804
|
exportSplitArtifact,
|