pikelet-wasm 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -3
- package/NOTICE +5 -0
- package/README.md +397 -454
- package/complete/builder.d.ts +4 -4
- package/complete/builder.mjs +8 -8
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/index.d.ts +11 -7
- package/complete/index.mjs +161 -112
- package/complete/inline-transformer.mjs +22 -9
- package/complete/lexical.mjs +11 -11
- package/complete/retrieval-abstention.mjs +18 -6
- package/complete/sources.mjs +22 -1
- package/dist/engine.js +1 -1
- package/dist/engine.scalar.js +1 -1
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +58 -51
- package/{pancake-artifact-common.js → pikelet-artifact-common.js} +42 -42
- package/{pancake-artifact-range.js → pikelet-artifact-range.js} +36 -36
- package/{pancake-artifact-sketch.js → pikelet-artifact-sketch.js} +76 -66
- package/{pancake-artifact.d.ts → pikelet-artifact.d.ts} +11 -11
- package/{pancake-artifact.js → pikelet-artifact.js} +10 -10
- package/{pancake-core.js → pikelet-core.js} +136 -136
- package/{pancake-errors.js → pikelet-errors.js} +9 -8
- package/{pancake.d.ts → pikelet.d.ts} +55 -55
- package/{pancake.js → pikelet.js} +44 -44
- package/{pancake.node.mjs → pikelet.node.mjs} +46 -46
- package/{pancake.web.d.ts → pikelet.web.d.ts} +13 -13
- package/{pancake.web.mjs → pikelet.web.mjs} +20 -20
- package/{pancake.workerd.mjs → pikelet.workerd.mjs} +18 -18
- /package/{pancake-loader.js → pikelet-loader.js} +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
'use strict';
|
|
2
|
-
// Range artifact profile (.
|
|
3
|
-
//
|
|
4
|
-
// Split out of
|
|
2
|
+
// Range artifact profile (.pikelet-range) — spec/SEARCH_ARTIFACT_CONTRACT.md 9.2:
|
|
3
|
+
// PikeletRangeArtifact reader and the range-artifact builder.
|
|
4
|
+
// Split out of pikelet-artifact.js (the public entry, which re-exports the
|
|
5
5
|
// three parts); see that file for the module map.
|
|
6
6
|
|
|
7
|
-
const {
|
|
7
|
+
const { pikeletError, PIKELET_ERROR_CODES } = require('./pikelet-errors.js');
|
|
8
8
|
const {
|
|
9
9
|
MAX_COALESCED_RANGE_BYTES,
|
|
10
10
|
readChecked,
|
|
@@ -22,7 +22,7 @@ const {
|
|
|
22
22
|
verifySegmentSha256,
|
|
23
23
|
resolveSearchK,
|
|
24
24
|
resolveOptionalPositiveInt,
|
|
25
|
-
} = require('./
|
|
25
|
+
} = require('./pikelet-artifact-common.js');
|
|
26
26
|
|
|
27
27
|
const RANGE_MAGIC = 0x31415250; // PRA1
|
|
28
28
|
const HEADER_BYTES = 128;
|
|
@@ -35,7 +35,7 @@ const RANGE_DIGESTS_OFFSET = 128;
|
|
|
35
35
|
const RANGE_DIGEST_BYTES = 32;
|
|
36
36
|
const ROUTER_LOCATION_MASK = 0x80000000;
|
|
37
37
|
const LOCATION_ORDINAL_MASK = 0x7fffffff;
|
|
38
|
-
class
|
|
38
|
+
class PikeletRangeArtifact {
|
|
39
39
|
constructor(source, header, idMap, options = {}) {
|
|
40
40
|
this.source = source;
|
|
41
41
|
this.version = header.version;
|
|
@@ -77,11 +77,11 @@ class PancakeRangeArtifact {
|
|
|
77
77
|
|
|
78
78
|
static async open(source, options = {}) {
|
|
79
79
|
if (!source || typeof source.read !== 'function') {
|
|
80
|
-
throw
|
|
80
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'PikeletRangeArtifact.open() requires a range source with read(offset, length)');
|
|
81
81
|
}
|
|
82
82
|
const headerBytes = await readChecked(source, 0, HEADER_BYTES, 'header');
|
|
83
83
|
if (headerBytes.byteLength < HEADER_BYTES) {
|
|
84
|
-
throw
|
|
84
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact header is truncated');
|
|
85
85
|
}
|
|
86
86
|
const header = parseHeader(headerBytes);
|
|
87
87
|
// Resolved before construction: the id-map read below is driven by an
|
|
@@ -91,7 +91,7 @@ class PancakeRangeArtifact {
|
|
|
91
91
|
if (header.version >= 3) {
|
|
92
92
|
const digestBytes = await readChecked(source, RANGE_DIGESTS_OFFSET, RANGE_DIGEST_BYTES * 3, 'header digests');
|
|
93
93
|
if (digestBytes.byteLength !== RANGE_DIGEST_BYTES * 3) {
|
|
94
|
-
throw
|
|
94
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact header digests are truncated');
|
|
95
95
|
}
|
|
96
96
|
digests = {
|
|
97
97
|
idMap: new Uint8Array(digestBytes.subarray(0, RANGE_DIGEST_BYTES)),
|
|
@@ -101,12 +101,12 @@ class PancakeRangeArtifact {
|
|
|
101
101
|
}
|
|
102
102
|
const idMapBytes = await readChecked(source, header.idMapOffset, header.count * 4, 'id map', maxReadBytes);
|
|
103
103
|
if (idMapBytes.byteLength !== header.count * 4) {
|
|
104
|
-
throw
|
|
104
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact id map is truncated');
|
|
105
105
|
}
|
|
106
106
|
const copied = new Uint8Array(idMapBytes.byteLength);
|
|
107
107
|
copied.set(idMapBytes);
|
|
108
108
|
const idMap = new Uint32Array(copied.buffer);
|
|
109
|
-
const artifact = new
|
|
109
|
+
const artifact = new PikeletRangeArtifact(source, header, idMap, options);
|
|
110
110
|
artifact.digests = digests;
|
|
111
111
|
if (digests && artifact.verify) {
|
|
112
112
|
await verifySha256(copied, digests.idMap, 'Range artifact id map');
|
|
@@ -124,7 +124,7 @@ class PancakeRangeArtifact {
|
|
|
124
124
|
// open must release it, or every corrupt artifact leaks an fd.
|
|
125
125
|
const source = new NodeFileRangeSource(filePath);
|
|
126
126
|
try {
|
|
127
|
-
return await
|
|
127
|
+
return await PikeletRangeArtifact.open(source, options);
|
|
128
128
|
} catch (err) {
|
|
129
129
|
await source.close().catch(() => {});
|
|
130
130
|
throw err;
|
|
@@ -179,7 +179,7 @@ class PancakeRangeArtifact {
|
|
|
179
179
|
// complete-profile manifest.
|
|
180
180
|
async verifyBaseSegment(options = {}) {
|
|
181
181
|
if (!this.digests) {
|
|
182
|
-
throw
|
|
182
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
183
183
|
'Range artifact predates segment digests (format v3); nothing to verify against', { version: this.version });
|
|
184
184
|
}
|
|
185
185
|
const bytes = this.baseCount * this.recordBytes;
|
|
@@ -199,19 +199,19 @@ class PancakeRangeArtifact {
|
|
|
199
199
|
|
|
200
200
|
recordAddressForId(id) {
|
|
201
201
|
if (!Number.isInteger(id) || id < 0 || id >= this.count) {
|
|
202
|
-
throw
|
|
202
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, `Node id ${id} is outside artifact bounds`, { id, count: this.count });
|
|
203
203
|
}
|
|
204
204
|
const location = this.originalToLocation[id];
|
|
205
205
|
if (this.version >= 2) {
|
|
206
206
|
const ordinal = location & LOCATION_ORDINAL_MASK;
|
|
207
207
|
if ((location & ROUTER_LOCATION_MASK) !== 0) {
|
|
208
|
-
if (ordinal >= this.routerCount) throw
|
|
208
|
+
if (ordinal >= this.routerCount) throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, `Router ordinal ${ordinal} is outside artifact`);
|
|
209
209
|
return this.routerRecordsOffset + ordinal * this.recordBytes;
|
|
210
210
|
}
|
|
211
|
-
if (ordinal >= this.baseCount) throw
|
|
211
|
+
if (ordinal >= this.baseCount) throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, `Base ordinal ${ordinal} is outside artifact`);
|
|
212
212
|
return this.baseRecordsOffset + ordinal * this.recordBytes;
|
|
213
213
|
}
|
|
214
|
-
if (location >= this.count) throw
|
|
214
|
+
if (location >= this.count) throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, `Node id ${id} is not addressable in artifact`);
|
|
215
215
|
return this.recordsOffset + location * this.recordBytes;
|
|
216
216
|
}
|
|
217
217
|
|
|
@@ -245,7 +245,7 @@ class PancakeRangeArtifact {
|
|
|
245
245
|
await this.prefetch([id]);
|
|
246
246
|
const node = this.cachedNode(id);
|
|
247
247
|
if (node === undefined) {
|
|
248
|
-
throw
|
|
248
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
249
249
|
'Range artifact record for id was not resolved by its address', { id });
|
|
250
250
|
}
|
|
251
251
|
return node;
|
|
@@ -281,7 +281,7 @@ class PancakeRangeArtifact {
|
|
|
281
281
|
const bytes = end - start;
|
|
282
282
|
const buffer = await readChecked(this.source, start, bytes, 'record');
|
|
283
283
|
if (buffer.byteLength !== bytes) {
|
|
284
|
-
throw
|
|
284
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact record read returned a truncated range', { offset: start, bytes, actual: buffer.byteLength });
|
|
285
285
|
}
|
|
286
286
|
return { start, end, bytes, buffer };
|
|
287
287
|
};
|
|
@@ -297,7 +297,7 @@ class PancakeRangeArtifact {
|
|
|
297
297
|
// poison the cache under an attacker-chosen key and searches
|
|
298
298
|
// for the real id would die uncoded.
|
|
299
299
|
if (originalId >= this.count || this.recordAddressForId(originalId) !== start + off) {
|
|
300
|
-
throw
|
|
300
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
301
301
|
'Range artifact record id does not match its address',
|
|
302
302
|
{ originalId, address: start + off });
|
|
303
303
|
}
|
|
@@ -331,7 +331,7 @@ class PancakeRangeArtifact {
|
|
|
331
331
|
const bytes = this.routerCount * this.recordBytes;
|
|
332
332
|
const buffer = await readChecked(this.source, this.routerRecordsOffset, bytes, 'router segment', this.maxReadBytes);
|
|
333
333
|
if (buffer.byteLength !== bytes) {
|
|
334
|
-
throw
|
|
334
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact router segment is truncated');
|
|
335
335
|
}
|
|
336
336
|
if (this.digests && this.verify) {
|
|
337
337
|
await verifySha256(buffer, this.digests.router, 'Range artifact router segment');
|
|
@@ -344,7 +344,7 @@ class PancakeRangeArtifact {
|
|
|
344
344
|
// Same untrusted-id check as decodeRange: the embedded id must map
|
|
345
345
|
// back to this router slot's address.
|
|
346
346
|
if (originalId >= this.count || this.recordAddressForId(originalId) !== address) {
|
|
347
|
-
throw
|
|
347
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
348
348
|
'Range artifact router record id does not match its address',
|
|
349
349
|
{ originalId, address });
|
|
350
350
|
}
|
|
@@ -362,13 +362,13 @@ class PancakeRangeArtifact {
|
|
|
362
362
|
// Records are untrusted bytes. Counts beyond the header geometry would
|
|
363
363
|
// read filler (or zeros) as edges; fail closed instead.
|
|
364
364
|
if (level > this.maxLevel || baseCount > this.M0) {
|
|
365
|
-
throw
|
|
365
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact record structure is inconsistent', { id, level, baseCount });
|
|
366
366
|
}
|
|
367
367
|
const upperCounts = new Uint16Array(this.maxLevel);
|
|
368
368
|
for (let i = 0; i < this.maxLevel; i++) {
|
|
369
369
|
upperCounts[i] = readU16(view, state);
|
|
370
370
|
if (upperCounts[i] > this.M) {
|
|
371
|
-
throw
|
|
371
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact record structure is inconsistent', { id, level: i + 1, edges: upperCounts[i] });
|
|
372
372
|
}
|
|
373
373
|
}
|
|
374
374
|
const qdata = new Uint8Array(this.dim);
|
|
@@ -381,7 +381,7 @@ class PancakeRangeArtifact {
|
|
|
381
381
|
const neighbor = readU32(view, state);
|
|
382
382
|
if (i < baseCount) {
|
|
383
383
|
if (neighbor >= this.count) {
|
|
384
|
-
throw
|
|
384
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact record has an out-of-bounds neighbor', { id, neighbor, count: this.count });
|
|
385
385
|
}
|
|
386
386
|
base[i] = neighbor;
|
|
387
387
|
}
|
|
@@ -393,7 +393,7 @@ class PancakeRangeArtifact {
|
|
|
393
393
|
const neighbor = readU32(view, state);
|
|
394
394
|
if (i < edges.length) {
|
|
395
395
|
if (neighbor >= this.count) {
|
|
396
|
-
throw
|
|
396
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact record has an out-of-bounds neighbor', { id, neighbor, count: this.count });
|
|
397
397
|
}
|
|
398
398
|
edges[i] = neighbor;
|
|
399
399
|
}
|
|
@@ -511,7 +511,7 @@ class PancakeRangeArtifact {
|
|
|
511
511
|
|
|
512
512
|
const top = results.items.sort(compareDistancesAsc).slice(0, k);
|
|
513
513
|
// Traversal orders by squared L2; the API contract (README "Distance
|
|
514
|
-
// values") reports Euclidean, matching
|
|
514
|
+
// values") reports Euclidean, matching PikeletIndex.search.
|
|
515
515
|
if (this.metric !== 1) {
|
|
516
516
|
for (const hit of top) hit.distance = Math.sqrt(hit.distance);
|
|
517
517
|
}
|
|
@@ -528,12 +528,12 @@ function parseHeader(headerBytes) {
|
|
|
528
528
|
const state = { offset: 0 };
|
|
529
529
|
const magic = readU32(view, state);
|
|
530
530
|
if (magic !== RANGE_MAGIC) {
|
|
531
|
-
throw
|
|
531
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Not a Pikelet range artifact', { magic });
|
|
532
532
|
}
|
|
533
533
|
const version = readU32(view, state);
|
|
534
534
|
const kind = readU32(view, state);
|
|
535
535
|
if (kind !== RANGE_KIND_U8) {
|
|
536
|
-
throw
|
|
536
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported Pikelet range artifact kind', { kind });
|
|
537
537
|
}
|
|
538
538
|
const header = {
|
|
539
539
|
version,
|
|
@@ -556,7 +556,7 @@ function parseHeader(headerBytes) {
|
|
|
556
556
|
};
|
|
557
557
|
if (header.metric !== 0) {
|
|
558
558
|
if (header.metric !== 1) {
|
|
559
|
-
throw
|
|
559
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported Pikelet range artifact metric', { metric: header.metric });
|
|
560
560
|
}
|
|
561
561
|
}
|
|
562
562
|
if (version >= 2) {
|
|
@@ -569,7 +569,7 @@ function parseHeader(headerBytes) {
|
|
|
569
569
|
header.baseRecordsOffset = header.recordsOffset;
|
|
570
570
|
}
|
|
571
571
|
if (version < 1 || version > RANGE_VERSION) {
|
|
572
|
-
throw
|
|
572
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Unsupported Pikelet range artifact version', { version });
|
|
573
573
|
}
|
|
574
574
|
// recordBytes must agree with the layout the geometry implies, or
|
|
575
575
|
// decodeNode would read past record boundaries on a corrupt header.
|
|
@@ -580,7 +580,7 @@ function parseHeader(headerBytes) {
|
|
|
580
580
|
|| header.entryPoint >= header.count
|
|
581
581
|
|| header.recordBytes !== expectedRecordBytes
|
|
582
582
|
|| (version >= 2 && header.routerCount + header.baseCount !== header.count)) {
|
|
583
|
-
throw
|
|
583
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact header geometry is inconsistent', {
|
|
584
584
|
dim: header.dim, count: header.count, maxLevel: header.maxLevel,
|
|
585
585
|
M: header.M, M0: header.M0, recordBytes: header.recordBytes, expectedRecordBytes,
|
|
586
586
|
});
|
|
@@ -662,7 +662,7 @@ function identityPermutation(count) {
|
|
|
662
662
|
function buildOrdinalToOriginal(index, layout) {
|
|
663
663
|
if (layout === 'identity') return identityPermutation(index.count);
|
|
664
664
|
if (layout === 'rcm') return reverseCuthillMckeePermutation(index);
|
|
665
|
-
throw
|
|
665
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, `Unsupported Search Artifact layout '${layout}'`, { layout });
|
|
666
666
|
}
|
|
667
667
|
|
|
668
668
|
function exportSplitArtifact(index, outPath, options = {}) {
|
|
@@ -742,7 +742,7 @@ function exportSplitArtifact(index, outPath, options = {}) {
|
|
|
742
742
|
fs.closeSync(fd);
|
|
743
743
|
}
|
|
744
744
|
return {
|
|
745
|
-
format: '
|
|
745
|
+
format: 'pikelet-range-artifact',
|
|
746
746
|
formatVersion: RANGE_VERSION,
|
|
747
747
|
integrity: {
|
|
748
748
|
idMapSha256: idMapDigest.toString('hex'),
|
|
@@ -778,7 +778,7 @@ function exportSplitArtifact(index, outPath, options = {}) {
|
|
|
778
778
|
|
|
779
779
|
function buildRangeArtifact(snapshotBytes, outPath, options = {}) {
|
|
780
780
|
if (typeof outPath !== 'string' || outPath.length === 0) {
|
|
781
|
-
throw
|
|
781
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'buildRangeArtifact() requires an output path');
|
|
782
782
|
}
|
|
783
783
|
const index = parseUint8Snapshot(snapshotBytes);
|
|
784
784
|
return exportSplitArtifact(index, outPath, options);
|
|
@@ -791,7 +791,7 @@ function buildRangeArtifactFile(snapshotPath, outPath, options = {}) {
|
|
|
791
791
|
}
|
|
792
792
|
|
|
793
793
|
module.exports = {
|
|
794
|
-
|
|
794
|
+
PikeletRangeArtifact,
|
|
795
795
|
buildRangeArtifact,
|
|
796
796
|
buildRangeArtifactFile,
|
|
797
797
|
exportSplitArtifact,
|