pikelet-wasm 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +340 -4
- package/NOTICE +5 -0
- package/README.md +226 -475
- package/complete/builder.d.ts +4 -4
- package/complete/builder.mjs +26 -9
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/fusion.mjs +72 -0
- package/complete/index.d.ts +34 -7
- package/complete/index.mjs +258 -129
- package/complete/inline-transformer.mjs +119 -10
- package/complete/lexical.mjs +11 -11
- package/complete/retrieval-abstention.mjs +136 -31
- package/complete/sources.mjs +22 -1
- package/complete/wordpiece.mjs +22 -0
- package/dist/engine.js +1 -1
- package/dist/engine.scalar.js +1 -1
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +47 -82
- package/{pancake-artifact-common.js → src/artifact/pikelet-artifact-common.js} +42 -42
- package/{pancake-artifact-range.js → src/artifact/pikelet-artifact-range.js} +43 -36
- package/{pancake-artifact-sketch.js → src/artifact/pikelet-artifact-sketch.js} +105 -68
- package/{pancake-artifact.d.ts → src/artifact/pikelet-artifact.d.ts} +11 -11
- package/{pancake-artifact.js → src/artifact/pikelet-artifact.js} +10 -10
- package/{pancake-core.js → src/core/pikelet-core.js} +141 -140
- package/{pancake-errors.js → src/errors/pikelet-errors.js} +9 -8
- package/{pancake.d.ts → src/pikelet.d.ts} +55 -55
- package/{pancake.js → src/pikelet.js} +47 -47
- package/{pancake.node.mjs → src/pikelet.node.mjs} +49 -49
- package/{pancake.web.d.ts → src/pikelet.web.d.ts} +13 -13
- package/{pancake.web.mjs → src/pikelet.web.mjs} +24 -24
- package/{pancake.workerd.mjs → src/pikelet.workerd.mjs} +22 -22
- /package/{pancake-loader.js → src/loader/pikelet-loader.js} +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
const {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
} = require('
|
|
4
|
+
PikeletError,
|
|
5
|
+
PIKELET_ERROR_CODES,
|
|
6
|
+
pikeletError,
|
|
7
|
+
} = require('../errors/pikelet-errors.js');
|
|
8
8
|
|
|
9
9
|
// Envelope header for validated export/import
|
|
10
|
-
const
|
|
10
|
+
const PIKELET_MAGIC = 0x504E434B; // "PNCK"
|
|
11
11
|
const ENVELOPE_VERSION = 3;
|
|
12
12
|
const V1_ENVELOPE_HEADER_SIZE = 24; // magic(4) + version(4) + dim(4) + compressed(4) + metric(4) + quantized(4)
|
|
13
13
|
const V2_ENVELOPE_HEADER_SIZE = 20; // magic(4) + version(4) + dim(4) + metric(4) + quantized(4)
|
|
@@ -31,7 +31,7 @@ const DEFAULT_SEED = 108;
|
|
|
31
31
|
|
|
32
32
|
function validateEfSearch(value, label = 'efSearch') {
|
|
33
33
|
if (!Number.isInteger(value) || value < 1 || value > MAX_EF) {
|
|
34
|
-
throw
|
|
34
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
35
35
|
`${label} must be an integer between 1 and ${MAX_EF}`,
|
|
36
36
|
{ argument: label, value, min: 1, max: MAX_EF });
|
|
37
37
|
}
|
|
@@ -41,7 +41,7 @@ function validateEfSearch(value, label = 'efSearch') {
|
|
|
41
41
|
function resolveSearchEf(options, fallback, methodName) {
|
|
42
42
|
if (options === undefined) return fallback;
|
|
43
43
|
if (!options || typeof options !== 'object' || Array.isArray(options)) {
|
|
44
|
-
throw
|
|
44
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
45
45
|
`${methodName}() options must be an object`, { argument: 'options' });
|
|
46
46
|
}
|
|
47
47
|
return options.efSearch === undefined
|
|
@@ -61,7 +61,7 @@ function assertNumericVector(vec, label) {
|
|
|
61
61
|
if (vec == null || typeof vec.length !== 'number') return; // length check handles these
|
|
62
62
|
for (let i = 0; i < vec.length; i++) {
|
|
63
63
|
if (typeof vec[i] !== 'number') {
|
|
64
|
-
throw
|
|
64
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR,
|
|
65
65
|
`${label} must contain only numbers; found ${typeof vec[i]} at index ${i}`,
|
|
66
66
|
{ index: i, actualType: typeof vec[i] });
|
|
67
67
|
}
|
|
@@ -73,18 +73,18 @@ function validateVectorValues(f32, label, validateCosineNorm) {
|
|
|
73
73
|
for (let i = 0; i < f32.length; i++) {
|
|
74
74
|
const value = f32[i];
|
|
75
75
|
if (!Number.isFinite(value)) {
|
|
76
|
-
throw
|
|
76
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR,
|
|
77
77
|
`${label} contains non-finite value (NaN or Infinity)`, { reason: 'non_finite' });
|
|
78
78
|
}
|
|
79
79
|
if (validateCosineNorm) normSq += value * value;
|
|
80
80
|
}
|
|
81
81
|
if (validateCosineNorm && (!(normSq > 0) || !Number.isFinite(normSq))) {
|
|
82
|
-
throw
|
|
82
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR,
|
|
83
83
|
`${label} has invalid cosine norm`, { reason: 'invalid_cosine_norm' });
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
-
class
|
|
87
|
+
class PikeletIndex {
|
|
88
88
|
constructor(engine, opts, handle, vecPtr, idPtr, distPtr, bufferCapacity) {
|
|
89
89
|
this._e = engine;
|
|
90
90
|
this._dim = opts.dim;
|
|
@@ -116,7 +116,7 @@ class PancakeIndex {
|
|
|
116
116
|
assertNumericVector(vec, 'Vector');
|
|
117
117
|
const f32 = vec instanceof Float32Array ? vec : new Float32Array(vec);
|
|
118
118
|
if (f32.length !== this._dim) {
|
|
119
|
-
throw
|
|
119
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
120
120
|
`Expected vector of length ${this._dim}, got ${f32.length}`,
|
|
121
121
|
{ expected: this._dim, actual: f32.length });
|
|
122
122
|
}
|
|
@@ -128,14 +128,14 @@ class PancakeIndex {
|
|
|
128
128
|
// unmutated. Ids survive deletes, so a long-lived high-churn index
|
|
129
129
|
// can reach this without ever holding 2^32 vectors at once.
|
|
130
130
|
if (this._nextExtId >= 0xFFFFFFFF) {
|
|
131
|
-
throw
|
|
131
|
+
throw pikeletError(PIKELET_ERROR_CODES.INDEX_LIMIT,
|
|
132
132
|
'Stable id space exhausted: ids are serialized as uint32 and the next id would not fit',
|
|
133
133
|
{ nextExtId: this._nextExtId });
|
|
134
134
|
}
|
|
135
135
|
this._e.HEAPF32.set(f32, this._vecPtr >> 2);
|
|
136
|
-
const intId = this._e.
|
|
136
|
+
const intId = this._e._pikelet_add(this._handle, this._vecPtr);
|
|
137
137
|
if (intId === 0xFFFFFFFF || intId < 0) {
|
|
138
|
-
throw
|
|
138
|
+
throw pikeletError(PIKELET_ERROR_CODES.INDEX_FULL,
|
|
139
139
|
'Insert failed (index full or not initialized)');
|
|
140
140
|
}
|
|
141
141
|
|
|
@@ -148,18 +148,18 @@ class PancakeIndex {
|
|
|
148
148
|
addBatch(vectors) {
|
|
149
149
|
this._checkDisposed();
|
|
150
150
|
if (!Array.isArray(vectors)) {
|
|
151
|
-
throw
|
|
151
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
152
152
|
'addBatch() requires an array of vectors');
|
|
153
153
|
}
|
|
154
154
|
if (vectors.length === 0) return [];
|
|
155
155
|
if (this.count + vectors.length > this._maxElements) {
|
|
156
|
-
throw
|
|
156
|
+
throw pikeletError(PIKELET_ERROR_CODES.INDEX_FULL,
|
|
157
157
|
'Insert failed (index full or not initialized)');
|
|
158
158
|
}
|
|
159
159
|
// Same u32 stable-id bound as add(), preflighted for the whole batch
|
|
160
160
|
// so the failure happens before any row is inserted.
|
|
161
161
|
if (this._nextExtId + vectors.length > 0xFFFFFFFF) {
|
|
162
|
-
throw
|
|
162
|
+
throw pikeletError(PIKELET_ERROR_CODES.INDEX_LIMIT,
|
|
163
163
|
'Stable id space exhausted: ids are serialized as uint32 and this batch would run past the last representable id',
|
|
164
164
|
{ nextExtId: this._nextExtId, batch: vectors.length });
|
|
165
165
|
}
|
|
@@ -173,7 +173,7 @@ class PancakeIndex {
|
|
|
173
173
|
const v = vectors[i];
|
|
174
174
|
const len = (v instanceof Float32Array) ? v.length : (v && v.length);
|
|
175
175
|
if (len !== this._dim) {
|
|
176
|
-
throw
|
|
176
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
177
177
|
`Expected vector of length ${this._dim}, got ${len} at index ${i}`,
|
|
178
178
|
{ expected: this._dim, actual: len, index: i });
|
|
179
179
|
}
|
|
@@ -187,7 +187,7 @@ class PancakeIndex {
|
|
|
187
187
|
// intermediate JS Float32Array), call bulk_insert once.
|
|
188
188
|
const totalFloats = vectors.length * this._dim;
|
|
189
189
|
const dataPtr = this._e._emsc_malloc(totalFloats * 4);
|
|
190
|
-
if (!dataPtr) throw
|
|
190
|
+
if (!dataPtr) throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
191
191
|
'WASM malloc failed for bulk insert');
|
|
192
192
|
|
|
193
193
|
try {
|
|
@@ -196,10 +196,10 @@ class PancakeIndex {
|
|
|
196
196
|
this._e.HEAPF32.set(converted[i], heapOffset + i * this._dim);
|
|
197
197
|
}
|
|
198
198
|
const countBefore = this.count;
|
|
199
|
-
const inserted = this._e.
|
|
199
|
+
const inserted = this._e._pikelet_bulk_insert(this._handle, dataPtr, vectors.length);
|
|
200
200
|
const ids = this._recordInsertedRange(countBefore, inserted);
|
|
201
201
|
if (inserted !== vectors.length) {
|
|
202
|
-
throw
|
|
202
|
+
throw pikeletError(PIKELET_ERROR_CODES.INTERNAL_INVARIANT,
|
|
203
203
|
'bulk_insert inserted fewer vectors than the prevalidated batch',
|
|
204
204
|
{ requested: vectors.length, inserted, ids });
|
|
205
205
|
}
|
|
@@ -212,13 +212,13 @@ class PancakeIndex {
|
|
|
212
212
|
search(query, k, options) {
|
|
213
213
|
this._checkDisposed();
|
|
214
214
|
if (!Number.isSafeInteger(k) || k < 0) {
|
|
215
|
-
throw
|
|
215
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
216
216
|
'search() requires a non-negative integer k', { argument: 'k', value: k });
|
|
217
217
|
}
|
|
218
218
|
assertNumericVector(query, 'Query vector');
|
|
219
219
|
const f32 = query instanceof Float32Array ? query : new Float32Array(query);
|
|
220
220
|
if (f32.length !== this._dim) {
|
|
221
|
-
throw
|
|
221
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
222
222
|
`Expected query of length ${this._dim}, got ${f32.length}`,
|
|
223
223
|
{ expected: this._dim, actual: f32.length });
|
|
224
224
|
}
|
|
@@ -233,7 +233,7 @@ class PancakeIndex {
|
|
|
233
233
|
if (boundedK === 0) return [];
|
|
234
234
|
this._ensureSearchCapacity(boundedK);
|
|
235
235
|
this._e.HEAPF32.set(f32, this._vecPtr >> 2);
|
|
236
|
-
const found = this._e.
|
|
236
|
+
const found = this._e._pikelet_query(
|
|
237
237
|
this._handle, this._vecPtr, boundedK, efSearch, this._idPtr, this._distPtr
|
|
238
238
|
);
|
|
239
239
|
return this._readResults(found);
|
|
@@ -242,20 +242,20 @@ class PancakeIndex {
|
|
|
242
242
|
searchFiltered(query, k, allowedIds, options) {
|
|
243
243
|
this._checkDisposed();
|
|
244
244
|
if (!Number.isSafeInteger(k) || k < 0) {
|
|
245
|
-
throw
|
|
245
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
246
246
|
'searchFiltered() requires a non-negative integer k', { argument: 'k', value: k });
|
|
247
247
|
}
|
|
248
248
|
assertNumericVector(query, 'Query vector');
|
|
249
249
|
const f32 = query instanceof Float32Array ? query : new Float32Array(query);
|
|
250
250
|
if (f32.length !== this._dim) {
|
|
251
|
-
throw
|
|
251
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
252
252
|
`Expected query of length ${this._dim}, got ${f32.length}`,
|
|
253
253
|
{ expected: this._dim, actual: f32.length });
|
|
254
254
|
}
|
|
255
255
|
validateVectorValues(f32, 'Query vector', !this._isL2);
|
|
256
256
|
const efSearch = resolveSearchEf(options, this._efSearch, 'searchFiltered');
|
|
257
257
|
if (!(allowedIds instanceof Set)) {
|
|
258
|
-
throw
|
|
258
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
259
259
|
'searchFiltered() requires allowedIds to be a Set<number>', { argument: 'allowedIds' });
|
|
260
260
|
}
|
|
261
261
|
const boundedK = Math.min(k, this.count);
|
|
@@ -263,10 +263,10 @@ class PancakeIndex {
|
|
|
263
263
|
this._ensureSearchCapacity(boundedK);
|
|
264
264
|
|
|
265
265
|
// Build bitset over internal IDs
|
|
266
|
-
const count = this._e.
|
|
266
|
+
const count = this._e._pikelet_count(this._handle);
|
|
267
267
|
const bitsetLen = (count + 7) >> 3;
|
|
268
268
|
const bitsetPtr = this._e._emsc_malloc(bitsetLen);
|
|
269
|
-
if (!bitsetPtr) throw
|
|
269
|
+
if (!bitsetPtr) throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
270
270
|
'WASM malloc failed for filter bitset');
|
|
271
271
|
|
|
272
272
|
try {
|
|
@@ -282,7 +282,7 @@ class PancakeIndex {
|
|
|
282
282
|
}
|
|
283
283
|
|
|
284
284
|
this._e.HEAPF32.set(f32, this._vecPtr >> 2);
|
|
285
|
-
const found = this._e.
|
|
285
|
+
const found = this._e._pikelet_query_filtered(
|
|
286
286
|
this._handle, this._vecPtr, boundedK, efSearch,
|
|
287
287
|
this._idPtr, this._distPtr,
|
|
288
288
|
bitsetPtr, bitsetLen
|
|
@@ -297,7 +297,7 @@ class PancakeIndex {
|
|
|
297
297
|
this._checkDisposed();
|
|
298
298
|
const intId = this._extToInt.get(id);
|
|
299
299
|
if (intId === undefined || this._deletedExt.has(id)) return false;
|
|
300
|
-
this._e.
|
|
300
|
+
this._e._pikelet_delete(this._handle, intId);
|
|
301
301
|
this._deletedExt.add(id);
|
|
302
302
|
return true;
|
|
303
303
|
}
|
|
@@ -319,18 +319,18 @@ class PancakeIndex {
|
|
|
319
319
|
// JS: how compaction assigns new IDs is the engine's contract, and
|
|
320
320
|
// re-implementing it here would silently corrupt the external-ID
|
|
321
321
|
// translation if the engine's assignment order ever changed.
|
|
322
|
-
const countBefore = this._e.
|
|
322
|
+
const countBefore = this._e._pikelet_count(this._handle);
|
|
323
323
|
if (countBefore === 0) {
|
|
324
|
-
this._e.
|
|
324
|
+
this._e._pikelet_compact(this._handle);
|
|
325
325
|
return;
|
|
326
326
|
}
|
|
327
327
|
|
|
328
328
|
const mapPtr = this._e._emsc_malloc(countBefore * 4);
|
|
329
|
-
if (!mapPtr) throw
|
|
329
|
+
if (!mapPtr) throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
330
330
|
'WASM malloc failed for compact remap');
|
|
331
331
|
|
|
332
332
|
try {
|
|
333
|
-
const written = this._e.
|
|
333
|
+
const written = this._e._pikelet_compact_remap(this._handle, mapPtr, countBefore);
|
|
334
334
|
const base = mapPtr >> 2;
|
|
335
335
|
const remapped = [];
|
|
336
336
|
for (let oldInt = 0; oldInt < written; oldInt++) {
|
|
@@ -349,10 +349,10 @@ class PancakeIndex {
|
|
|
349
349
|
this._extToInt.set(extId, newInt);
|
|
350
350
|
}
|
|
351
351
|
|
|
352
|
-
const liveCount = this._e.
|
|
352
|
+
const liveCount = this._e._pikelet_count(this._handle);
|
|
353
353
|
if (this._intToExt.size !== liveCount) {
|
|
354
354
|
this._clearMappings();
|
|
355
|
-
throw
|
|
355
|
+
throw pikeletError(PIKELET_ERROR_CODES.INTERNAL_INVARIANT,
|
|
356
356
|
`compact() remap mismatch: engine reports ${liveCount} live vectors, remap yielded ${this._intToExt.size}`,
|
|
357
357
|
{ liveCount, mappingCount: this._intToExt.size });
|
|
358
358
|
}
|
|
@@ -364,18 +364,18 @@ class PancakeIndex {
|
|
|
364
364
|
export() {
|
|
365
365
|
this._checkDisposed();
|
|
366
366
|
if (this.ghostCount > 0) {
|
|
367
|
-
throw
|
|
367
|
+
throw pikeletError(PIKELET_ERROR_CODES.COMPACTION_REQUIRED,
|
|
368
368
|
'Export failed: compact() required before export when ghostCount > 0',
|
|
369
369
|
{ deletedCount: this.ghostCount });
|
|
370
370
|
}
|
|
371
371
|
|
|
372
372
|
const sizePtr = this._e._emsc_malloc(4);
|
|
373
|
-
if (!sizePtr) throw
|
|
373
|
+
if (!sizePtr) throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
374
374
|
'WASM malloc failed for export');
|
|
375
375
|
try {
|
|
376
|
-
const dataPtr = this._e.
|
|
376
|
+
const dataPtr = this._e._pikelet_export(this._handle, sizePtr);
|
|
377
377
|
if (!dataPtr) {
|
|
378
|
-
throw
|
|
378
|
+
throw pikeletError(PIKELET_ERROR_CODES.INTERNAL_INVARIANT, 'Export failed');
|
|
379
379
|
}
|
|
380
380
|
|
|
381
381
|
const wasmSize = this._e.HEAPU32[sizePtr >> 2];
|
|
@@ -384,7 +384,7 @@ class PancakeIndex {
|
|
|
384
384
|
const mappingBytes = liveMappings.length * MAPPING_ENTRY_SIZE;
|
|
385
385
|
const result = new Uint8Array(V3_ENVELOPE_HEADER_SIZE + mappingBytes + wasmSize);
|
|
386
386
|
const view = new DataView(result.buffer);
|
|
387
|
-
view.setUint32(0,
|
|
387
|
+
view.setUint32(0, PIKELET_MAGIC, true);
|
|
388
388
|
view.setUint32(4, ENVELOPE_VERSION, true);
|
|
389
389
|
view.setUint32(8, this._dim, true);
|
|
390
390
|
view.setUint32(12, this._isL2 ? 0 : 1, true);
|
|
@@ -418,7 +418,7 @@ class PancakeIndex {
|
|
|
418
418
|
if (bytes.length >= V2_ENVELOPE_HEADER_SIZE) {
|
|
419
419
|
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
420
420
|
|
|
421
|
-
if (view.getUint32(0, true) ===
|
|
421
|
+
if (view.getUint32(0, true) === PIKELET_MAGIC) {
|
|
422
422
|
const version = view.getUint32(4, true);
|
|
423
423
|
|
|
424
424
|
let dim;
|
|
@@ -432,13 +432,13 @@ class PancakeIndex {
|
|
|
432
432
|
|
|
433
433
|
if (version === 1) {
|
|
434
434
|
if (bytes.length < V1_ENVELOPE_HEADER_SIZE) {
|
|
435
|
-
throw
|
|
435
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Import failed: truncated v1 envelope');
|
|
436
436
|
}
|
|
437
437
|
|
|
438
438
|
dim = view.getUint32(8, true);
|
|
439
439
|
const compressed = view.getUint32(12, true);
|
|
440
440
|
if (compressed !== dim) {
|
|
441
|
-
throw
|
|
441
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
442
442
|
'Import failed: DCT/PCA indexes are no longer supported');
|
|
443
443
|
}
|
|
444
444
|
|
|
@@ -446,7 +446,7 @@ class PancakeIndex {
|
|
|
446
446
|
quantizedVal = view.getUint32(20, true);
|
|
447
447
|
} else if (version === ENVELOPE_VERSION) {
|
|
448
448
|
if (bytes.length < V3_ENVELOPE_HEADER_SIZE) {
|
|
449
|
-
throw
|
|
449
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Import failed: truncated v3 envelope');
|
|
450
450
|
}
|
|
451
451
|
|
|
452
452
|
dim = view.getUint32(8, true);
|
|
@@ -459,12 +459,12 @@ class PancakeIndex {
|
|
|
459
459
|
wasmOffset = mappingOffset + mappingCount * MAPPING_ENTRY_SIZE;
|
|
460
460
|
|
|
461
461
|
if (wasmOffset > bytes.length || wasmOffset + wasmSize > bytes.length) {
|
|
462
|
-
throw
|
|
462
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
463
463
|
'Import failed: truncated v3 envelope payload');
|
|
464
464
|
}
|
|
465
465
|
} else {
|
|
466
466
|
if (version !== 2) {
|
|
467
|
-
throw
|
|
467
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
468
468
|
`Import failed: unsupported envelope version ${version}`, { version });
|
|
469
469
|
}
|
|
470
470
|
|
|
@@ -474,7 +474,7 @@ class PancakeIndex {
|
|
|
474
474
|
}
|
|
475
475
|
|
|
476
476
|
if (dim !== this._dim) {
|
|
477
|
-
throw
|
|
477
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
478
478
|
`Import failed: dim mismatch (exported ${dim}, expected ${this._dim})`,
|
|
479
479
|
{ field: 'dim', exported: dim, expected: this._dim });
|
|
480
480
|
}
|
|
@@ -483,13 +483,13 @@ class PancakeIndex {
|
|
|
483
483
|
if (exportedL2 !== this._isL2) {
|
|
484
484
|
const got = exportedL2 ? 'l2' : 'cosine';
|
|
485
485
|
const want = this._isL2 ? 'l2' : 'cosine';
|
|
486
|
-
throw
|
|
486
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
487
487
|
`Import failed: metric mismatch (exported ${got}, expected ${want})`,
|
|
488
488
|
{ field: 'metric', exported: got, expected: want });
|
|
489
489
|
}
|
|
490
490
|
|
|
491
491
|
if (!!quantizedVal !== this._quantized) {
|
|
492
|
-
throw
|
|
492
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
493
493
|
'Import failed: quantized mismatch',
|
|
494
494
|
{ field: 'quantized', exported: !!quantizedVal, expected: this._quantized });
|
|
495
495
|
}
|
|
@@ -498,12 +498,12 @@ class PancakeIndex {
|
|
|
498
498
|
wasmBytes = bytes.subarray(wasmOffset, wasmOffset + wasmSize);
|
|
499
499
|
const metadata = parseRawSnapshotMetadata(wasmBytes);
|
|
500
500
|
if (metadata === null) {
|
|
501
|
-
throw
|
|
501
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
502
502
|
'Import failed: unsupported raw snapshot format');
|
|
503
503
|
}
|
|
504
504
|
this._validateRawSnapshotMetadata(metadata);
|
|
505
505
|
if (mappingCount !== metadata.count) {
|
|
506
|
-
throw
|
|
506
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
507
507
|
'Import failed: envelope mapping count mismatch');
|
|
508
508
|
}
|
|
509
509
|
pendingMappings = new Map();
|
|
@@ -532,28 +532,28 @@ class PancakeIndex {
|
|
|
532
532
|
// about both query behavior and distance interpretation.
|
|
533
533
|
const rawMetadata = parseRawSnapshotMetadata(wasmBytes);
|
|
534
534
|
if (rawMetadata === null) {
|
|
535
|
-
throw
|
|
535
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
536
536
|
'Import failed: unsupported raw snapshot format');
|
|
537
537
|
}
|
|
538
538
|
this._validateRawSnapshotMetadata(rawMetadata);
|
|
539
539
|
|
|
540
540
|
const dataPtr = this._e._emsc_malloc(wasmBytes.length);
|
|
541
|
-
if (!dataPtr) throw
|
|
541
|
+
if (!dataPtr) throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
542
542
|
'WASM malloc failed for import');
|
|
543
543
|
|
|
544
544
|
let status;
|
|
545
545
|
try {
|
|
546
546
|
this._e.HEAPU8.set(wasmBytes, dataPtr);
|
|
547
|
-
status = this._e.
|
|
547
|
+
status = this._e._pikelet_import(this._handle, dataPtr, wasmBytes.length);
|
|
548
548
|
} finally {
|
|
549
549
|
this._e._emsc_free(dataPtr);
|
|
550
550
|
}
|
|
551
551
|
|
|
552
552
|
if (status !== 0) {
|
|
553
|
-
throw
|
|
553
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Import failed');
|
|
554
554
|
}
|
|
555
555
|
|
|
556
|
-
const count = this._e.
|
|
556
|
+
const count = this._e._pikelet_count(this._handle);
|
|
557
557
|
if (pendingMappings === null) {
|
|
558
558
|
this._setIdentityMappings(count);
|
|
559
559
|
return;
|
|
@@ -569,22 +569,22 @@ class PancakeIndex {
|
|
|
569
569
|
|
|
570
570
|
get count() {
|
|
571
571
|
this._checkDisposed();
|
|
572
|
-
return this._e.
|
|
572
|
+
return this._e._pikelet_count(this._handle);
|
|
573
573
|
}
|
|
574
574
|
|
|
575
575
|
get ghostCount() {
|
|
576
576
|
this._checkDisposed();
|
|
577
|
-
return this._e.
|
|
577
|
+
return this._e._pikelet_ghost_count(this._handle);
|
|
578
578
|
}
|
|
579
579
|
|
|
580
580
|
get ghostRatio() {
|
|
581
581
|
this._checkDisposed();
|
|
582
|
-
return this._e.
|
|
582
|
+
return this._e._pikelet_ghost_ratio(this._handle);
|
|
583
583
|
}
|
|
584
584
|
|
|
585
585
|
get memory() {
|
|
586
586
|
this._checkDisposed();
|
|
587
|
-
return this._e.
|
|
587
|
+
return this._e._pikelet_memory(this._handle);
|
|
588
588
|
}
|
|
589
589
|
|
|
590
590
|
get dim() { return this._dim; }
|
|
@@ -627,7 +627,7 @@ class PancakeIndex {
|
|
|
627
627
|
get memoryUsage() {
|
|
628
628
|
this._checkDisposed();
|
|
629
629
|
return Object.freeze({
|
|
630
|
-
logicalIndexBytes: this._e.
|
|
630
|
+
logicalIndexBytes: this._e._pikelet_memory(this._handle),
|
|
631
631
|
wasmHeapBytes: this._e.HEAPU8.buffer.byteLength,
|
|
632
632
|
snapshotBufferBytes: this._snapshotBufferBytes,
|
|
633
633
|
});
|
|
@@ -637,7 +637,7 @@ class PancakeIndex {
|
|
|
637
637
|
if (this._disposed) return;
|
|
638
638
|
let thrown = null;
|
|
639
639
|
try {
|
|
640
|
-
this._e.
|
|
640
|
+
this._e._pikelet_dispose(this._handle);
|
|
641
641
|
} catch (err) {
|
|
642
642
|
thrown = err;
|
|
643
643
|
} finally {
|
|
@@ -660,9 +660,9 @@ class PancakeIndex {
|
|
|
660
660
|
|
|
661
661
|
_checkDisposed() {
|
|
662
662
|
if (this._disposed) {
|
|
663
|
-
throw
|
|
664
|
-
|
|
665
|
-
'
|
|
663
|
+
throw pikeletError(
|
|
664
|
+
PIKELET_ERROR_CODES.INDEX_DISPOSED,
|
|
665
|
+
'PikeletIndex has been disposed'
|
|
666
666
|
);
|
|
667
667
|
}
|
|
668
668
|
}
|
|
@@ -675,7 +675,7 @@ class PancakeIndex {
|
|
|
675
675
|
if (!newIdPtr || !newDistPtr) {
|
|
676
676
|
if (newIdPtr) this._e._emsc_free(newIdPtr);
|
|
677
677
|
if (newDistPtr) this._e._emsc_free(newDistPtr);
|
|
678
|
-
throw
|
|
678
|
+
throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
679
679
|
`WASM malloc failed while growing search buffers to k=${k}`, { k });
|
|
680
680
|
}
|
|
681
681
|
|
|
@@ -716,11 +716,11 @@ class PancakeIndex {
|
|
|
716
716
|
|
|
717
717
|
_validateV3Mappings(intToExt, nextExtId, count) {
|
|
718
718
|
if (intToExt.size !== count) {
|
|
719
|
-
throw
|
|
719
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
720
720
|
'Import failed: envelope mapping count mismatch');
|
|
721
721
|
}
|
|
722
722
|
if (!Number.isInteger(nextExtId) || nextExtId < count) {
|
|
723
|
-
throw
|
|
723
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
724
724
|
'Import failed: envelope nextExtId is invalid');
|
|
725
725
|
}
|
|
726
726
|
|
|
@@ -728,22 +728,22 @@ class PancakeIndex {
|
|
|
728
728
|
let maxExtId = -1;
|
|
729
729
|
for (const [intId, extId] of intToExt) {
|
|
730
730
|
if (!Number.isInteger(intId) || intId < 0 || intId >= count) {
|
|
731
|
-
throw
|
|
731
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
732
732
|
'Import failed: envelope mapping contains invalid internal ID');
|
|
733
733
|
}
|
|
734
734
|
if (!Number.isInteger(extId) || extId < 0) {
|
|
735
|
-
throw
|
|
735
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
736
736
|
'Import failed: envelope mapping contains invalid external ID');
|
|
737
737
|
}
|
|
738
738
|
if (extIds.has(extId)) {
|
|
739
|
-
throw
|
|
739
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
740
740
|
'Import failed: envelope mapping contains duplicates');
|
|
741
741
|
}
|
|
742
742
|
extIds.add(extId);
|
|
743
743
|
if (extId > maxExtId) maxExtId = extId;
|
|
744
744
|
}
|
|
745
745
|
if (nextExtId <= maxExtId) {
|
|
746
|
-
throw
|
|
746
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
747
747
|
'Import failed: envelope nextExtId is invalid');
|
|
748
748
|
}
|
|
749
749
|
}
|
|
@@ -765,23 +765,23 @@ class PancakeIndex {
|
|
|
765
765
|
// reads in the native deserializers.
|
|
766
766
|
const maxVersion = metadata.quantized ? 2 : 1;
|
|
767
767
|
if (!Number.isInteger(metadata.version) || metadata.version > maxVersion) {
|
|
768
|
-
throw
|
|
768
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
769
769
|
`Import failed: unsupported snapshot format version ${metadata.version}`,
|
|
770
770
|
{ version: metadata.version, maxSupported: maxVersion });
|
|
771
771
|
}
|
|
772
772
|
if (!Number.isInteger(metadata.count) || metadata.count < 0) {
|
|
773
|
-
throw
|
|
773
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
774
774
|
'Import failed: snapshot count is invalid');
|
|
775
775
|
}
|
|
776
776
|
if (metadata.count > this._maxElements) {
|
|
777
|
-
throw
|
|
778
|
-
|
|
777
|
+
throw pikeletError(
|
|
778
|
+
PIKELET_ERROR_CODES.SNAPSHOT_CAPACITY_EXCEEDED,
|
|
779
779
|
`Import failed: snapshot count ${metadata.count} exceeds maxElements ${this._maxElements}`,
|
|
780
780
|
{ count: metadata.count, maxElements: this._maxElements }
|
|
781
781
|
);
|
|
782
782
|
}
|
|
783
783
|
if (metadata.dim !== this._dim) {
|
|
784
|
-
throw
|
|
784
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
785
785
|
`Import failed: dim mismatch (exported ${metadata.dim}, expected ${this._dim})`,
|
|
786
786
|
{ field: 'dim', exported: metadata.dim, expected: this._dim });
|
|
787
787
|
}
|
|
@@ -790,23 +790,23 @@ class PancakeIndex {
|
|
|
790
790
|
if (exportedL2 !== this._isL2) {
|
|
791
791
|
const got = exportedL2 ? 'l2' : 'cosine';
|
|
792
792
|
const want = this._isL2 ? 'l2' : 'cosine';
|
|
793
|
-
throw
|
|
793
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
794
794
|
`Import failed: metric mismatch (exported ${got}, expected ${want})`,
|
|
795
795
|
{ field: 'metric', exported: got, expected: want });
|
|
796
796
|
}
|
|
797
797
|
|
|
798
798
|
if (metadata.quantized !== this._quantized) {
|
|
799
|
-
throw
|
|
799
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
800
800
|
'Import failed: quantized mismatch',
|
|
801
801
|
{ field: 'quantized', exported: metadata.quantized, expected: this._quantized });
|
|
802
802
|
}
|
|
803
803
|
if (metadata.M !== this._M) {
|
|
804
|
-
throw
|
|
804
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
805
805
|
`Import failed: M mismatch (exported ${metadata.M}, expected ${this._M})`,
|
|
806
806
|
{ field: 'M', exported: metadata.M, expected: this._M });
|
|
807
807
|
}
|
|
808
808
|
if (metadata.efConstruction !== this._efConstruction) {
|
|
809
|
-
throw
|
|
809
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
810
810
|
`Import failed: efConstruction mismatch (exported ${metadata.efConstruction}, expected ${this._efConstruction})`,
|
|
811
811
|
{ field: 'efConstruction', exported: metadata.efConstruction, expected: this._efConstruction });
|
|
812
812
|
}
|
|
@@ -823,8 +823,8 @@ class PancakeIndex {
|
|
|
823
823
|
if (this._isL2) distance = Math.sqrt(distance);
|
|
824
824
|
const extId = this._intToExt.get(intId);
|
|
825
825
|
if (extId === undefined) {
|
|
826
|
-
throw
|
|
827
|
-
|
|
826
|
+
throw pikeletError(
|
|
827
|
+
PIKELET_ERROR_CODES.INTERNAL_INVARIANT,
|
|
828
828
|
`Search invariant failed: missing external ID mapping for internal ID ${intId}`,
|
|
829
829
|
{ internalId: intId }
|
|
830
830
|
);
|
|
@@ -835,7 +835,7 @@ class PancakeIndex {
|
|
|
835
835
|
}
|
|
836
836
|
}
|
|
837
837
|
|
|
838
|
-
function
|
|
838
|
+
function createPikeletApi(loadEngineImpl) {
|
|
839
839
|
function isVectorInput(value) {
|
|
840
840
|
return value instanceof Float32Array || Array.isArray(value);
|
|
841
841
|
}
|
|
@@ -845,7 +845,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
845
845
|
return { vector: item, hasSourceId: false, sourceId: undefined };
|
|
846
846
|
}
|
|
847
847
|
if (!item || typeof item !== 'object' || !('vector' in item)) {
|
|
848
|
-
throw
|
|
848
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
849
849
|
'fromVectors() requires vectors or { vector, id? } records');
|
|
850
850
|
}
|
|
851
851
|
const sourceId = item.id;
|
|
@@ -854,40 +854,41 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
854
854
|
}
|
|
855
855
|
|
|
856
856
|
async function create(opts) {
|
|
857
|
-
if (!opts || !opts.dim) throw
|
|
857
|
+
if (!opts || !opts.dim) throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
858
858
|
'opts.dim is required', { argument: 'dim' });
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
859
|
+
// Upper bound matches the engine constructors' MAX_DIMS.
|
|
860
|
+
if (!Number.isInteger(opts.dim) || opts.dim <= 0 || opts.dim > 65536) {
|
|
861
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
862
|
+
'opts.dim must be an integer between 1 and 65536', { argument: 'dim', value: opts.dim });
|
|
862
863
|
}
|
|
863
864
|
if ('compressed' in opts) {
|
|
864
|
-
throw
|
|
865
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'opts.compressed has been removed');
|
|
865
866
|
}
|
|
866
867
|
if ('varianceSample' in opts) {
|
|
867
|
-
throw
|
|
868
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'opts.varianceSample has been removed');
|
|
868
869
|
}
|
|
869
870
|
if (opts.metric !== undefined && opts.metric !== 'cosine' && opts.metric !== 'l2') {
|
|
870
|
-
throw
|
|
871
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
871
872
|
"opts.metric must be 'cosine' or 'l2'", { argument: 'metric', value: opts.metric });
|
|
872
873
|
}
|
|
873
|
-
// Upper bound matches the engine ABI:
|
|
874
|
+
// Upper bound matches the engine ABI: pikelet_init takes max_elem as a
|
|
874
875
|
// C int, so anything above 2^31-1 would truncate or go negative.
|
|
875
876
|
if (opts.maxElements !== undefined && (!Number.isInteger(opts.maxElements) || opts.maxElements <= 0 || opts.maxElements > 0x7fffffff)) {
|
|
876
|
-
throw
|
|
877
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
877
878
|
'opts.maxElements must be an integer between 1 and 2147483647', { argument: 'maxElements', value: opts.maxElements });
|
|
878
879
|
}
|
|
879
880
|
if (opts.M !== undefined && (!Number.isInteger(opts.M) || opts.M <= 1 || opts.M > 128)) {
|
|
880
|
-
throw
|
|
881
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
881
882
|
'opts.M must be an integer between 2 and 128', { argument: 'M', value: opts.M });
|
|
882
883
|
}
|
|
883
884
|
if (opts.efConstruction !== undefined && (!Number.isInteger(opts.efConstruction) || opts.efConstruction <= 0 || opts.efConstruction > 4096)) {
|
|
884
|
-
throw
|
|
885
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
885
886
|
'opts.efConstruction must be an integer between 1 and 4096',
|
|
886
887
|
{ argument: 'efConstruction', value: opts.efConstruction });
|
|
887
888
|
}
|
|
888
889
|
if (opts.efSearch !== undefined) validateEfSearch(opts.efSearch, 'opts.efSearch');
|
|
889
890
|
if (opts.seed !== undefined && (!Number.isInteger(opts.seed) || opts.seed <= 0 || opts.seed > 0x7fffffff)) {
|
|
890
|
-
throw
|
|
891
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
891
892
|
'opts.seed must be an integer between 1 and 2147483647',
|
|
892
893
|
{ argument: 'seed', value: opts.seed });
|
|
893
894
|
}
|
|
@@ -904,7 +905,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
904
905
|
|
|
905
906
|
// Capacity guard: the backend allocates its arena eagerly at
|
|
906
907
|
// construction, and a request the wasm32 heap cannot satisfy throws
|
|
907
|
-
// std::bad_alloc inside
|
|
908
|
+
// std::bad_alloc inside pikelet_init — reject it here with the
|
|
908
909
|
// estimate instead. Per-element cost mirrors the constructor
|
|
909
910
|
// allocations (uint8_float_hnsw.hpp / float_hnsw.hpp): quantized rows
|
|
910
911
|
// cost dim bytes plus 16*M edge bytes plus ~39 bytes of bookkeeping;
|
|
@@ -912,7 +913,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
912
913
|
const bytesPerElement = isQuantized ? dim + 16 * M + 39 : 4 * dim + 8 * M + 23;
|
|
913
914
|
const estimatedBytes = maxElements * bytesPerElement;
|
|
914
915
|
if (estimatedBytes > MAX_INDEX_ARENA_BYTES) {
|
|
915
|
-
throw
|
|
916
|
+
throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
916
917
|
`Index configuration needs ~${Math.ceil(estimatedBytes / (1024 * 1024))} MiB `
|
|
917
918
|
+ `(${isQuantized ? 'quantized' : 'float32'} backend: maxElements ${maxElements} × ~${bytesPerElement} B/element), `
|
|
918
919
|
+ `above the ${Math.floor(MAX_INDEX_ARENA_BYTES / (1024 * 1024))} MiB wasm32 index budget; `
|
|
@@ -943,10 +944,10 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
943
944
|
if (vecPtr) e._emsc_free(vecPtr);
|
|
944
945
|
if (idPtr) e._emsc_free(idPtr);
|
|
945
946
|
if (distPtr) e._emsc_free(distPtr);
|
|
946
|
-
throw
|
|
947
|
+
throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED, 'WASM malloc failed');
|
|
947
948
|
}
|
|
948
949
|
|
|
949
|
-
const handle = e.
|
|
950
|
+
const handle = e._pikelet_init(dim, maxElements, quantized, metric, M, efConstruction, efSearch, seed);
|
|
950
951
|
|
|
951
952
|
// The engine returns uint32_t INVALID_HANDLE (0xFFFFFFFF), which the
|
|
952
953
|
// Emscripten i32 ABI delivers to JS as -1.
|
|
@@ -954,18 +955,18 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
954
955
|
e._emsc_free(vecPtr);
|
|
955
956
|
e._emsc_free(idPtr);
|
|
956
957
|
e._emsc_free(distPtr);
|
|
957
|
-
throw
|
|
958
|
+
throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED, 'Backend init failed');
|
|
958
959
|
}
|
|
959
960
|
|
|
960
|
-
return new
|
|
961
|
+
return new PikeletIndex(e, resolvedOpts, handle, vecPtr, idPtr, distPtr, initialBufferCapacity);
|
|
961
962
|
}
|
|
962
963
|
|
|
963
964
|
async function fromVectors(items, opts = {}) {
|
|
964
965
|
if (!Array.isArray(items)) {
|
|
965
|
-
throw
|
|
966
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'fromVectors() requires an array');
|
|
966
967
|
}
|
|
967
968
|
if (items.length === 0) {
|
|
968
|
-
throw
|
|
969
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
969
970
|
'fromVectors() requires at least one vector');
|
|
970
971
|
}
|
|
971
972
|
|
|
@@ -977,18 +978,18 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
977
978
|
for (let i = 0; i < items.length; i++) {
|
|
978
979
|
const { vector, hasSourceId, sourceId } = extractVectorRecord(items[i]);
|
|
979
980
|
if (!isVectorInput(vector)) {
|
|
980
|
-
throw
|
|
981
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR,
|
|
981
982
|
`fromVectors() expected a vector at index ${i}`, { index: i });
|
|
982
983
|
}
|
|
983
984
|
const dim = vector.length;
|
|
984
985
|
if (!Number.isInteger(dim) || dim <= 0) {
|
|
985
|
-
throw
|
|
986
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR,
|
|
986
987
|
`fromVectors() expected a non-empty vector at index ${i}`, { index: i });
|
|
987
988
|
}
|
|
988
989
|
if (inferredDim === null) {
|
|
989
990
|
inferredDim = dim;
|
|
990
991
|
} else if (dim !== inferredDim) {
|
|
991
|
-
throw
|
|
992
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
992
993
|
`fromVectors() found mixed vector dimensions (${inferredDim} and ${dim})`,
|
|
993
994
|
{ expected: inferredDim, actual: dim, index: i });
|
|
994
995
|
}
|
|
@@ -998,7 +999,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
998
999
|
}
|
|
999
1000
|
|
|
1000
1001
|
if (opts.dim !== undefined && opts.dim !== inferredDim) {
|
|
1001
|
-
throw
|
|
1002
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
1002
1003
|
`fromVectors() dim mismatch (inferred ${inferredDim}, got opts.dim=${opts.dim})`,
|
|
1003
1004
|
{ expected: inferredDim, actual: opts.dim });
|
|
1004
1005
|
}
|
|
@@ -1029,7 +1030,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
1029
1030
|
|
|
1030
1031
|
async function withIndex(options, fn) {
|
|
1031
1032
|
if (typeof fn !== 'function') {
|
|
1032
|
-
throw
|
|
1033
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
1033
1034
|
'withIndex() requires a callback function', { argument: 'fn' });
|
|
1034
1035
|
}
|
|
1035
1036
|
const index = await create(options);
|
|
@@ -1046,7 +1047,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
1046
1047
|
|
|
1047
1048
|
async function restore(snapshot, overrides = {}) {
|
|
1048
1049
|
if (!overrides || typeof overrides !== 'object' || Array.isArray(overrides)) {
|
|
1049
|
-
throw
|
|
1050
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
1050
1051
|
'restore() overrides must be an object', { argument: 'overrides' });
|
|
1051
1052
|
}
|
|
1052
1053
|
|
|
@@ -1062,7 +1063,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
1062
1063
|
if (metadata.format === 'raw') {
|
|
1063
1064
|
for (const field of Object.keys(fixedConfig)) {
|
|
1064
1065
|
if (overrides[field] === undefined) {
|
|
1065
|
-
throw
|
|
1066
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
1066
1067
|
`restore() requires overrides.${field} for legacy raw snapshots`,
|
|
1067
1068
|
{ argument: field, format: 'raw' });
|
|
1068
1069
|
}
|
|
@@ -1071,7 +1072,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
1071
1072
|
|
|
1072
1073
|
for (const [field, expected] of Object.entries(fixedConfig)) {
|
|
1073
1074
|
if (overrides[field] !== undefined && overrides[field] !== expected) {
|
|
1074
|
-
throw
|
|
1075
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
1075
1076
|
`restore() ${field} override does not match the snapshot`,
|
|
1076
1077
|
{ field, exported: expected, override: overrides[field] });
|
|
1077
1078
|
}
|
|
@@ -1102,8 +1103,8 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
1102
1103
|
withIndex,
|
|
1103
1104
|
restore,
|
|
1104
1105
|
inspectSnapshot,
|
|
1105
|
-
|
|
1106
|
-
|
|
1106
|
+
PikeletError,
|
|
1107
|
+
PIKELET_ERROR_CODES,
|
|
1107
1108
|
};
|
|
1108
1109
|
}
|
|
1109
1110
|
|
|
@@ -1153,14 +1154,14 @@ function snapshotBytes(data, methodName) {
|
|
|
1153
1154
|
if (ArrayBuffer.isView(data)) {
|
|
1154
1155
|
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
1155
1156
|
}
|
|
1156
|
-
throw
|
|
1157
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
1157
1158
|
`${methodName}() requires snapshot bytes`, { argument: 'snapshot' });
|
|
1158
1159
|
}
|
|
1159
1160
|
|
|
1160
1161
|
function inspectSnapshotMetadata(data) {
|
|
1161
1162
|
const bytes = snapshotBytes(data, 'inspectSnapshot');
|
|
1162
1163
|
if (bytes.byteLength < 4) {
|
|
1163
|
-
throw
|
|
1164
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot is too small to inspect');
|
|
1164
1165
|
}
|
|
1165
1166
|
|
|
1166
1167
|
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
@@ -1172,20 +1173,20 @@ function inspectSnapshotMetadata(data) {
|
|
|
1172
1173
|
let envelopeQuantized = null;
|
|
1173
1174
|
let nextId = null;
|
|
1174
1175
|
|
|
1175
|
-
if (view.getUint32(0, true) ===
|
|
1176
|
-
format = '
|
|
1176
|
+
if (view.getUint32(0, true) === PIKELET_MAGIC) {
|
|
1177
|
+
format = 'pikelet';
|
|
1177
1178
|
if (bytes.byteLength < 8) {
|
|
1178
|
-
throw
|
|
1179
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot envelope is truncated');
|
|
1179
1180
|
}
|
|
1180
1181
|
envelopeVersion = view.getUint32(4, true);
|
|
1181
1182
|
let rawOffset;
|
|
1182
1183
|
if (envelopeVersion === 1) {
|
|
1183
1184
|
if (bytes.byteLength < V1_ENVELOPE_HEADER_SIZE) {
|
|
1184
|
-
throw
|
|
1185
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot v1 envelope is truncated');
|
|
1185
1186
|
}
|
|
1186
1187
|
envelopeDim = view.getUint32(8, true);
|
|
1187
1188
|
if (view.getUint32(12, true) !== envelopeDim) {
|
|
1188
|
-
throw
|
|
1189
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1189
1190
|
'DCT/PCA snapshots are no longer supported');
|
|
1190
1191
|
}
|
|
1191
1192
|
envelopeMetric = view.getUint32(16, true);
|
|
@@ -1193,7 +1194,7 @@ function inspectSnapshotMetadata(data) {
|
|
|
1193
1194
|
rawOffset = V1_ENVELOPE_HEADER_SIZE;
|
|
1194
1195
|
} else if (envelopeVersion === 2) {
|
|
1195
1196
|
if (bytes.byteLength < V2_ENVELOPE_HEADER_SIZE) {
|
|
1196
|
-
throw
|
|
1197
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot v2 envelope is truncated');
|
|
1197
1198
|
}
|
|
1198
1199
|
envelopeDim = view.getUint32(8, true);
|
|
1199
1200
|
envelopeMetric = view.getUint32(12, true);
|
|
@@ -1201,7 +1202,7 @@ function inspectSnapshotMetadata(data) {
|
|
|
1201
1202
|
rawOffset = V2_ENVELOPE_HEADER_SIZE;
|
|
1202
1203
|
} else if (envelopeVersion === ENVELOPE_VERSION) {
|
|
1203
1204
|
if (bytes.byteLength < V3_ENVELOPE_HEADER_SIZE) {
|
|
1204
|
-
throw
|
|
1205
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot v3 envelope is truncated');
|
|
1205
1206
|
}
|
|
1206
1207
|
envelopeDim = view.getUint32(8, true);
|
|
1207
1208
|
envelopeMetric = view.getUint32(12, true);
|
|
@@ -1212,12 +1213,12 @@ function inspectSnapshotMetadata(data) {
|
|
|
1212
1213
|
rawOffset = V3_ENVELOPE_HEADER_SIZE + mappingCount * MAPPING_ENTRY_SIZE;
|
|
1213
1214
|
if (!Number.isSafeInteger(rawOffset) || rawOffset > bytes.byteLength ||
|
|
1214
1215
|
rawSize > bytes.byteLength - rawOffset) {
|
|
1215
|
-
throw
|
|
1216
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1216
1217
|
'Snapshot v3 envelope payload is truncated');
|
|
1217
1218
|
}
|
|
1218
1219
|
rawBytes = bytes.subarray(rawOffset, rawOffset + rawSize);
|
|
1219
1220
|
} else {
|
|
1220
|
-
throw
|
|
1221
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1221
1222
|
`Unsupported snapshot envelope version ${envelopeVersion}`, { version: envelopeVersion });
|
|
1222
1223
|
}
|
|
1223
1224
|
if (envelopeVersion !== ENVELOPE_VERSION) rawBytes = bytes.subarray(rawOffset);
|
|
@@ -1225,27 +1226,27 @@ function inspectSnapshotMetadata(data) {
|
|
|
1225
1226
|
|
|
1226
1227
|
const raw = parseRawSnapshotMetadata(rawBytes);
|
|
1227
1228
|
if (raw === null) {
|
|
1228
|
-
throw
|
|
1229
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1229
1230
|
'Unsupported or truncated raw snapshot format');
|
|
1230
1231
|
}
|
|
1231
1232
|
if (raw.dim <= 0 || raw.metric > 1 || raw.M < 2 || raw.M > 128 ||
|
|
1232
1233
|
raw.M0 !== raw.M * 2 || raw.efConstruction < 1 || raw.efConstruction > 4096) {
|
|
1233
|
-
throw
|
|
1234
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1234
1235
|
'Snapshot header contains invalid index configuration');
|
|
1235
1236
|
}
|
|
1236
|
-
if (format === '
|
|
1237
|
+
if (format === 'pikelet' && (envelopeDim !== raw.dim || envelopeMetric !== raw.metric ||
|
|
1237
1238
|
!!envelopeQuantized !== raw.quantized)) {
|
|
1238
|
-
throw
|
|
1239
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1239
1240
|
'Snapshot envelope does not match its raw index header');
|
|
1240
1241
|
}
|
|
1241
1242
|
if (nextId === null) nextId = raw.count;
|
|
1242
1243
|
if (nextId < raw.count) {
|
|
1243
|
-
throw
|
|
1244
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot nextId is invalid');
|
|
1244
1245
|
}
|
|
1245
1246
|
|
|
1246
1247
|
return Object.freeze({
|
|
1247
1248
|
format,
|
|
1248
|
-
version: format === '
|
|
1249
|
+
version: format === 'pikelet' ? envelopeVersion : raw.version,
|
|
1249
1250
|
dim: raw.dim,
|
|
1250
1251
|
count: raw.count,
|
|
1251
1252
|
metric: raw.metric === 0 ? 'l2' : 'cosine',
|
|
@@ -1256,13 +1257,13 @@ function inspectSnapshotMetadata(data) {
|
|
|
1256
1257
|
});
|
|
1257
1258
|
}
|
|
1258
1259
|
|
|
1259
|
-
module.exports =
|
|
1260
|
-
module.exports.default =
|
|
1261
|
-
module.exports.
|
|
1262
|
-
module.exports.
|
|
1260
|
+
module.exports = createPikeletApi;
|
|
1261
|
+
module.exports.default = createPikeletApi;
|
|
1262
|
+
module.exports.PikeletError = PikeletError;
|
|
1263
|
+
module.exports.PIKELET_ERROR_CODES = PIKELET_ERROR_CODES;
|
|
1263
1264
|
|
|
1264
1265
|
if (typeof Symbol.dispose === 'symbol') {
|
|
1265
|
-
|
|
1266
|
+
PikeletIndex.prototype[Symbol.dispose] = function disposeWithSymbol() {
|
|
1266
1267
|
this.dispose();
|
|
1267
1268
|
};
|
|
1268
1269
|
}
|