pikelet-wasm 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -3
- package/NOTICE +5 -0
- package/README.md +397 -454
- package/complete/builder.d.ts +4 -4
- package/complete/builder.mjs +8 -8
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/index.d.ts +11 -7
- package/complete/index.mjs +161 -112
- package/complete/inline-transformer.mjs +22 -9
- package/complete/lexical.mjs +11 -11
- package/complete/retrieval-abstention.mjs +18 -6
- package/complete/sources.mjs +22 -1
- package/dist/engine.js +1 -1
- package/dist/engine.scalar.js +1 -1
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +58 -51
- package/{pancake-artifact-common.js → pikelet-artifact-common.js} +42 -42
- package/{pancake-artifact-range.js → pikelet-artifact-range.js} +36 -36
- package/{pancake-artifact-sketch.js → pikelet-artifact-sketch.js} +76 -66
- package/{pancake-artifact.d.ts → pikelet-artifact.d.ts} +11 -11
- package/{pancake-artifact.js → pikelet-artifact.js} +10 -10
- package/{pancake-core.js → pikelet-core.js} +136 -136
- package/{pancake-errors.js → pikelet-errors.js} +9 -8
- package/{pancake.d.ts → pikelet.d.ts} +55 -55
- package/{pancake.js → pikelet.js} +44 -44
- package/{pancake.node.mjs → pikelet.node.mjs} +46 -46
- package/{pancake.web.d.ts → pikelet.web.d.ts} +13 -13
- package/{pancake.web.mjs → pikelet.web.mjs} +20 -20
- package/{pancake.workerd.mjs → pikelet.workerd.mjs} +18 -18
- /package/{pancake-loader.js → pikelet-loader.js} +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
const {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
} = require('./
|
|
4
|
+
PikeletError,
|
|
5
|
+
PIKELET_ERROR_CODES,
|
|
6
|
+
pikeletError,
|
|
7
|
+
} = require('./pikelet-errors.js');
|
|
8
8
|
|
|
9
9
|
// Envelope header for validated export/import
|
|
10
|
-
const
|
|
10
|
+
const PIKELET_MAGIC = 0x504E434B; // "PNCK"
|
|
11
11
|
const ENVELOPE_VERSION = 3;
|
|
12
12
|
const V1_ENVELOPE_HEADER_SIZE = 24; // magic(4) + version(4) + dim(4) + compressed(4) + metric(4) + quantized(4)
|
|
13
13
|
const V2_ENVELOPE_HEADER_SIZE = 20; // magic(4) + version(4) + dim(4) + metric(4) + quantized(4)
|
|
@@ -31,7 +31,7 @@ const DEFAULT_SEED = 108;
|
|
|
31
31
|
|
|
32
32
|
function validateEfSearch(value, label = 'efSearch') {
|
|
33
33
|
if (!Number.isInteger(value) || value < 1 || value > MAX_EF) {
|
|
34
|
-
throw
|
|
34
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
35
35
|
`${label} must be an integer between 1 and ${MAX_EF}`,
|
|
36
36
|
{ argument: label, value, min: 1, max: MAX_EF });
|
|
37
37
|
}
|
|
@@ -41,7 +41,7 @@ function validateEfSearch(value, label = 'efSearch') {
|
|
|
41
41
|
function resolveSearchEf(options, fallback, methodName) {
|
|
42
42
|
if (options === undefined) return fallback;
|
|
43
43
|
if (!options || typeof options !== 'object' || Array.isArray(options)) {
|
|
44
|
-
throw
|
|
44
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
45
45
|
`${methodName}() options must be an object`, { argument: 'options' });
|
|
46
46
|
}
|
|
47
47
|
return options.efSearch === undefined
|
|
@@ -61,7 +61,7 @@ function assertNumericVector(vec, label) {
|
|
|
61
61
|
if (vec == null || typeof vec.length !== 'number') return; // length check handles these
|
|
62
62
|
for (let i = 0; i < vec.length; i++) {
|
|
63
63
|
if (typeof vec[i] !== 'number') {
|
|
64
|
-
throw
|
|
64
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR,
|
|
65
65
|
`${label} must contain only numbers; found ${typeof vec[i]} at index ${i}`,
|
|
66
66
|
{ index: i, actualType: typeof vec[i] });
|
|
67
67
|
}
|
|
@@ -73,18 +73,18 @@ function validateVectorValues(f32, label, validateCosineNorm) {
|
|
|
73
73
|
for (let i = 0; i < f32.length; i++) {
|
|
74
74
|
const value = f32[i];
|
|
75
75
|
if (!Number.isFinite(value)) {
|
|
76
|
-
throw
|
|
76
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR,
|
|
77
77
|
`${label} contains non-finite value (NaN or Infinity)`, { reason: 'non_finite' });
|
|
78
78
|
}
|
|
79
79
|
if (validateCosineNorm) normSq += value * value;
|
|
80
80
|
}
|
|
81
81
|
if (validateCosineNorm && (!(normSq > 0) || !Number.isFinite(normSq))) {
|
|
82
|
-
throw
|
|
82
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR,
|
|
83
83
|
`${label} has invalid cosine norm`, { reason: 'invalid_cosine_norm' });
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
-
class
|
|
87
|
+
class PikeletIndex {
|
|
88
88
|
constructor(engine, opts, handle, vecPtr, idPtr, distPtr, bufferCapacity) {
|
|
89
89
|
this._e = engine;
|
|
90
90
|
this._dim = opts.dim;
|
|
@@ -116,7 +116,7 @@ class PancakeIndex {
|
|
|
116
116
|
assertNumericVector(vec, 'Vector');
|
|
117
117
|
const f32 = vec instanceof Float32Array ? vec : new Float32Array(vec);
|
|
118
118
|
if (f32.length !== this._dim) {
|
|
119
|
-
throw
|
|
119
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
120
120
|
`Expected vector of length ${this._dim}, got ${f32.length}`,
|
|
121
121
|
{ expected: this._dim, actual: f32.length });
|
|
122
122
|
}
|
|
@@ -128,14 +128,14 @@ class PancakeIndex {
|
|
|
128
128
|
// unmutated. Ids survive deletes, so a long-lived high-churn index
|
|
129
129
|
// can reach this without ever holding 2^32 vectors at once.
|
|
130
130
|
if (this._nextExtId >= 0xFFFFFFFF) {
|
|
131
|
-
throw
|
|
131
|
+
throw pikeletError(PIKELET_ERROR_CODES.INDEX_LIMIT,
|
|
132
132
|
'Stable id space exhausted: ids are serialized as uint32 and the next id would not fit',
|
|
133
133
|
{ nextExtId: this._nextExtId });
|
|
134
134
|
}
|
|
135
135
|
this._e.HEAPF32.set(f32, this._vecPtr >> 2);
|
|
136
|
-
const intId = this._e.
|
|
136
|
+
const intId = this._e._pikelet_add(this._handle, this._vecPtr);
|
|
137
137
|
if (intId === 0xFFFFFFFF || intId < 0) {
|
|
138
|
-
throw
|
|
138
|
+
throw pikeletError(PIKELET_ERROR_CODES.INDEX_FULL,
|
|
139
139
|
'Insert failed (index full or not initialized)');
|
|
140
140
|
}
|
|
141
141
|
|
|
@@ -148,18 +148,18 @@ class PancakeIndex {
|
|
|
148
148
|
addBatch(vectors) {
|
|
149
149
|
this._checkDisposed();
|
|
150
150
|
if (!Array.isArray(vectors)) {
|
|
151
|
-
throw
|
|
151
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
152
152
|
'addBatch() requires an array of vectors');
|
|
153
153
|
}
|
|
154
154
|
if (vectors.length === 0) return [];
|
|
155
155
|
if (this.count + vectors.length > this._maxElements) {
|
|
156
|
-
throw
|
|
156
|
+
throw pikeletError(PIKELET_ERROR_CODES.INDEX_FULL,
|
|
157
157
|
'Insert failed (index full or not initialized)');
|
|
158
158
|
}
|
|
159
159
|
// Same u32 stable-id bound as add(), preflighted for the whole batch
|
|
160
160
|
// so the failure happens before any row is inserted.
|
|
161
161
|
if (this._nextExtId + vectors.length > 0xFFFFFFFF) {
|
|
162
|
-
throw
|
|
162
|
+
throw pikeletError(PIKELET_ERROR_CODES.INDEX_LIMIT,
|
|
163
163
|
'Stable id space exhausted: ids are serialized as uint32 and this batch would run past the last representable id',
|
|
164
164
|
{ nextExtId: this._nextExtId, batch: vectors.length });
|
|
165
165
|
}
|
|
@@ -173,7 +173,7 @@ class PancakeIndex {
|
|
|
173
173
|
const v = vectors[i];
|
|
174
174
|
const len = (v instanceof Float32Array) ? v.length : (v && v.length);
|
|
175
175
|
if (len !== this._dim) {
|
|
176
|
-
throw
|
|
176
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
177
177
|
`Expected vector of length ${this._dim}, got ${len} at index ${i}`,
|
|
178
178
|
{ expected: this._dim, actual: len, index: i });
|
|
179
179
|
}
|
|
@@ -187,7 +187,7 @@ class PancakeIndex {
|
|
|
187
187
|
// intermediate JS Float32Array), call bulk_insert once.
|
|
188
188
|
const totalFloats = vectors.length * this._dim;
|
|
189
189
|
const dataPtr = this._e._emsc_malloc(totalFloats * 4);
|
|
190
|
-
if (!dataPtr) throw
|
|
190
|
+
if (!dataPtr) throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
191
191
|
'WASM malloc failed for bulk insert');
|
|
192
192
|
|
|
193
193
|
try {
|
|
@@ -196,10 +196,10 @@ class PancakeIndex {
|
|
|
196
196
|
this._e.HEAPF32.set(converted[i], heapOffset + i * this._dim);
|
|
197
197
|
}
|
|
198
198
|
const countBefore = this.count;
|
|
199
|
-
const inserted = this._e.
|
|
199
|
+
const inserted = this._e._pikelet_bulk_insert(this._handle, dataPtr, vectors.length);
|
|
200
200
|
const ids = this._recordInsertedRange(countBefore, inserted);
|
|
201
201
|
if (inserted !== vectors.length) {
|
|
202
|
-
throw
|
|
202
|
+
throw pikeletError(PIKELET_ERROR_CODES.INTERNAL_INVARIANT,
|
|
203
203
|
'bulk_insert inserted fewer vectors than the prevalidated batch',
|
|
204
204
|
{ requested: vectors.length, inserted, ids });
|
|
205
205
|
}
|
|
@@ -212,13 +212,13 @@ class PancakeIndex {
|
|
|
212
212
|
search(query, k, options) {
|
|
213
213
|
this._checkDisposed();
|
|
214
214
|
if (!Number.isSafeInteger(k) || k < 0) {
|
|
215
|
-
throw
|
|
215
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
216
216
|
'search() requires a non-negative integer k', { argument: 'k', value: k });
|
|
217
217
|
}
|
|
218
218
|
assertNumericVector(query, 'Query vector');
|
|
219
219
|
const f32 = query instanceof Float32Array ? query : new Float32Array(query);
|
|
220
220
|
if (f32.length !== this._dim) {
|
|
221
|
-
throw
|
|
221
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
222
222
|
`Expected query of length ${this._dim}, got ${f32.length}`,
|
|
223
223
|
{ expected: this._dim, actual: f32.length });
|
|
224
224
|
}
|
|
@@ -233,7 +233,7 @@ class PancakeIndex {
|
|
|
233
233
|
if (boundedK === 0) return [];
|
|
234
234
|
this._ensureSearchCapacity(boundedK);
|
|
235
235
|
this._e.HEAPF32.set(f32, this._vecPtr >> 2);
|
|
236
|
-
const found = this._e.
|
|
236
|
+
const found = this._e._pikelet_query(
|
|
237
237
|
this._handle, this._vecPtr, boundedK, efSearch, this._idPtr, this._distPtr
|
|
238
238
|
);
|
|
239
239
|
return this._readResults(found);
|
|
@@ -242,20 +242,20 @@ class PancakeIndex {
|
|
|
242
242
|
searchFiltered(query, k, allowedIds, options) {
|
|
243
243
|
this._checkDisposed();
|
|
244
244
|
if (!Number.isSafeInteger(k) || k < 0) {
|
|
245
|
-
throw
|
|
245
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
246
246
|
'searchFiltered() requires a non-negative integer k', { argument: 'k', value: k });
|
|
247
247
|
}
|
|
248
248
|
assertNumericVector(query, 'Query vector');
|
|
249
249
|
const f32 = query instanceof Float32Array ? query : new Float32Array(query);
|
|
250
250
|
if (f32.length !== this._dim) {
|
|
251
|
-
throw
|
|
251
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
252
252
|
`Expected query of length ${this._dim}, got ${f32.length}`,
|
|
253
253
|
{ expected: this._dim, actual: f32.length });
|
|
254
254
|
}
|
|
255
255
|
validateVectorValues(f32, 'Query vector', !this._isL2);
|
|
256
256
|
const efSearch = resolveSearchEf(options, this._efSearch, 'searchFiltered');
|
|
257
257
|
if (!(allowedIds instanceof Set)) {
|
|
258
|
-
throw
|
|
258
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
259
259
|
'searchFiltered() requires allowedIds to be a Set<number>', { argument: 'allowedIds' });
|
|
260
260
|
}
|
|
261
261
|
const boundedK = Math.min(k, this.count);
|
|
@@ -263,10 +263,10 @@ class PancakeIndex {
|
|
|
263
263
|
this._ensureSearchCapacity(boundedK);
|
|
264
264
|
|
|
265
265
|
// Build bitset over internal IDs
|
|
266
|
-
const count = this._e.
|
|
266
|
+
const count = this._e._pikelet_count(this._handle);
|
|
267
267
|
const bitsetLen = (count + 7) >> 3;
|
|
268
268
|
const bitsetPtr = this._e._emsc_malloc(bitsetLen);
|
|
269
|
-
if (!bitsetPtr) throw
|
|
269
|
+
if (!bitsetPtr) throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
270
270
|
'WASM malloc failed for filter bitset');
|
|
271
271
|
|
|
272
272
|
try {
|
|
@@ -282,7 +282,7 @@ class PancakeIndex {
|
|
|
282
282
|
}
|
|
283
283
|
|
|
284
284
|
this._e.HEAPF32.set(f32, this._vecPtr >> 2);
|
|
285
|
-
const found = this._e.
|
|
285
|
+
const found = this._e._pikelet_query_filtered(
|
|
286
286
|
this._handle, this._vecPtr, boundedK, efSearch,
|
|
287
287
|
this._idPtr, this._distPtr,
|
|
288
288
|
bitsetPtr, bitsetLen
|
|
@@ -297,7 +297,7 @@ class PancakeIndex {
|
|
|
297
297
|
this._checkDisposed();
|
|
298
298
|
const intId = this._extToInt.get(id);
|
|
299
299
|
if (intId === undefined || this._deletedExt.has(id)) return false;
|
|
300
|
-
this._e.
|
|
300
|
+
this._e._pikelet_delete(this._handle, intId);
|
|
301
301
|
this._deletedExt.add(id);
|
|
302
302
|
return true;
|
|
303
303
|
}
|
|
@@ -319,18 +319,18 @@ class PancakeIndex {
|
|
|
319
319
|
// JS: how compaction assigns new IDs is the engine's contract, and
|
|
320
320
|
// re-implementing it here would silently corrupt the external-ID
|
|
321
321
|
// translation if the engine's assignment order ever changed.
|
|
322
|
-
const countBefore = this._e.
|
|
322
|
+
const countBefore = this._e._pikelet_count(this._handle);
|
|
323
323
|
if (countBefore === 0) {
|
|
324
|
-
this._e.
|
|
324
|
+
this._e._pikelet_compact(this._handle);
|
|
325
325
|
return;
|
|
326
326
|
}
|
|
327
327
|
|
|
328
328
|
const mapPtr = this._e._emsc_malloc(countBefore * 4);
|
|
329
|
-
if (!mapPtr) throw
|
|
329
|
+
if (!mapPtr) throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
330
330
|
'WASM malloc failed for compact remap');
|
|
331
331
|
|
|
332
332
|
try {
|
|
333
|
-
const written = this._e.
|
|
333
|
+
const written = this._e._pikelet_compact_remap(this._handle, mapPtr, countBefore);
|
|
334
334
|
const base = mapPtr >> 2;
|
|
335
335
|
const remapped = [];
|
|
336
336
|
for (let oldInt = 0; oldInt < written; oldInt++) {
|
|
@@ -349,10 +349,10 @@ class PancakeIndex {
|
|
|
349
349
|
this._extToInt.set(extId, newInt);
|
|
350
350
|
}
|
|
351
351
|
|
|
352
|
-
const liveCount = this._e.
|
|
352
|
+
const liveCount = this._e._pikelet_count(this._handle);
|
|
353
353
|
if (this._intToExt.size !== liveCount) {
|
|
354
354
|
this._clearMappings();
|
|
355
|
-
throw
|
|
355
|
+
throw pikeletError(PIKELET_ERROR_CODES.INTERNAL_INVARIANT,
|
|
356
356
|
`compact() remap mismatch: engine reports ${liveCount} live vectors, remap yielded ${this._intToExt.size}`,
|
|
357
357
|
{ liveCount, mappingCount: this._intToExt.size });
|
|
358
358
|
}
|
|
@@ -364,18 +364,18 @@ class PancakeIndex {
|
|
|
364
364
|
export() {
|
|
365
365
|
this._checkDisposed();
|
|
366
366
|
if (this.ghostCount > 0) {
|
|
367
|
-
throw
|
|
367
|
+
throw pikeletError(PIKELET_ERROR_CODES.COMPACTION_REQUIRED,
|
|
368
368
|
'Export failed: compact() required before export when ghostCount > 0',
|
|
369
369
|
{ deletedCount: this.ghostCount });
|
|
370
370
|
}
|
|
371
371
|
|
|
372
372
|
const sizePtr = this._e._emsc_malloc(4);
|
|
373
|
-
if (!sizePtr) throw
|
|
373
|
+
if (!sizePtr) throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
374
374
|
'WASM malloc failed for export');
|
|
375
375
|
try {
|
|
376
|
-
const dataPtr = this._e.
|
|
376
|
+
const dataPtr = this._e._pikelet_export(this._handle, sizePtr);
|
|
377
377
|
if (!dataPtr) {
|
|
378
|
-
throw
|
|
378
|
+
throw pikeletError(PIKELET_ERROR_CODES.INTERNAL_INVARIANT, 'Export failed');
|
|
379
379
|
}
|
|
380
380
|
|
|
381
381
|
const wasmSize = this._e.HEAPU32[sizePtr >> 2];
|
|
@@ -384,7 +384,7 @@ class PancakeIndex {
|
|
|
384
384
|
const mappingBytes = liveMappings.length * MAPPING_ENTRY_SIZE;
|
|
385
385
|
const result = new Uint8Array(V3_ENVELOPE_HEADER_SIZE + mappingBytes + wasmSize);
|
|
386
386
|
const view = new DataView(result.buffer);
|
|
387
|
-
view.setUint32(0,
|
|
387
|
+
view.setUint32(0, PIKELET_MAGIC, true);
|
|
388
388
|
view.setUint32(4, ENVELOPE_VERSION, true);
|
|
389
389
|
view.setUint32(8, this._dim, true);
|
|
390
390
|
view.setUint32(12, this._isL2 ? 0 : 1, true);
|
|
@@ -418,7 +418,7 @@ class PancakeIndex {
|
|
|
418
418
|
if (bytes.length >= V2_ENVELOPE_HEADER_SIZE) {
|
|
419
419
|
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
420
420
|
|
|
421
|
-
if (view.getUint32(0, true) ===
|
|
421
|
+
if (view.getUint32(0, true) === PIKELET_MAGIC) {
|
|
422
422
|
const version = view.getUint32(4, true);
|
|
423
423
|
|
|
424
424
|
let dim;
|
|
@@ -432,13 +432,13 @@ class PancakeIndex {
|
|
|
432
432
|
|
|
433
433
|
if (version === 1) {
|
|
434
434
|
if (bytes.length < V1_ENVELOPE_HEADER_SIZE) {
|
|
435
|
-
throw
|
|
435
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Import failed: truncated v1 envelope');
|
|
436
436
|
}
|
|
437
437
|
|
|
438
438
|
dim = view.getUint32(8, true);
|
|
439
439
|
const compressed = view.getUint32(12, true);
|
|
440
440
|
if (compressed !== dim) {
|
|
441
|
-
throw
|
|
441
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
442
442
|
'Import failed: DCT/PCA indexes are no longer supported');
|
|
443
443
|
}
|
|
444
444
|
|
|
@@ -446,7 +446,7 @@ class PancakeIndex {
|
|
|
446
446
|
quantizedVal = view.getUint32(20, true);
|
|
447
447
|
} else if (version === ENVELOPE_VERSION) {
|
|
448
448
|
if (bytes.length < V3_ENVELOPE_HEADER_SIZE) {
|
|
449
|
-
throw
|
|
449
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Import failed: truncated v3 envelope');
|
|
450
450
|
}
|
|
451
451
|
|
|
452
452
|
dim = view.getUint32(8, true);
|
|
@@ -459,12 +459,12 @@ class PancakeIndex {
|
|
|
459
459
|
wasmOffset = mappingOffset + mappingCount * MAPPING_ENTRY_SIZE;
|
|
460
460
|
|
|
461
461
|
if (wasmOffset > bytes.length || wasmOffset + wasmSize > bytes.length) {
|
|
462
|
-
throw
|
|
462
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
463
463
|
'Import failed: truncated v3 envelope payload');
|
|
464
464
|
}
|
|
465
465
|
} else {
|
|
466
466
|
if (version !== 2) {
|
|
467
|
-
throw
|
|
467
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
468
468
|
`Import failed: unsupported envelope version ${version}`, { version });
|
|
469
469
|
}
|
|
470
470
|
|
|
@@ -474,7 +474,7 @@ class PancakeIndex {
|
|
|
474
474
|
}
|
|
475
475
|
|
|
476
476
|
if (dim !== this._dim) {
|
|
477
|
-
throw
|
|
477
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
478
478
|
`Import failed: dim mismatch (exported ${dim}, expected ${this._dim})`,
|
|
479
479
|
{ field: 'dim', exported: dim, expected: this._dim });
|
|
480
480
|
}
|
|
@@ -483,13 +483,13 @@ class PancakeIndex {
|
|
|
483
483
|
if (exportedL2 !== this._isL2) {
|
|
484
484
|
const got = exportedL2 ? 'l2' : 'cosine';
|
|
485
485
|
const want = this._isL2 ? 'l2' : 'cosine';
|
|
486
|
-
throw
|
|
486
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
487
487
|
`Import failed: metric mismatch (exported ${got}, expected ${want})`,
|
|
488
488
|
{ field: 'metric', exported: got, expected: want });
|
|
489
489
|
}
|
|
490
490
|
|
|
491
491
|
if (!!quantizedVal !== this._quantized) {
|
|
492
|
-
throw
|
|
492
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
493
493
|
'Import failed: quantized mismatch',
|
|
494
494
|
{ field: 'quantized', exported: !!quantizedVal, expected: this._quantized });
|
|
495
495
|
}
|
|
@@ -498,12 +498,12 @@ class PancakeIndex {
|
|
|
498
498
|
wasmBytes = bytes.subarray(wasmOffset, wasmOffset + wasmSize);
|
|
499
499
|
const metadata = parseRawSnapshotMetadata(wasmBytes);
|
|
500
500
|
if (metadata === null) {
|
|
501
|
-
throw
|
|
501
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
502
502
|
'Import failed: unsupported raw snapshot format');
|
|
503
503
|
}
|
|
504
504
|
this._validateRawSnapshotMetadata(metadata);
|
|
505
505
|
if (mappingCount !== metadata.count) {
|
|
506
|
-
throw
|
|
506
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
507
507
|
'Import failed: envelope mapping count mismatch');
|
|
508
508
|
}
|
|
509
509
|
pendingMappings = new Map();
|
|
@@ -532,28 +532,28 @@ class PancakeIndex {
|
|
|
532
532
|
// about both query behavior and distance interpretation.
|
|
533
533
|
const rawMetadata = parseRawSnapshotMetadata(wasmBytes);
|
|
534
534
|
if (rawMetadata === null) {
|
|
535
|
-
throw
|
|
535
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
536
536
|
'Import failed: unsupported raw snapshot format');
|
|
537
537
|
}
|
|
538
538
|
this._validateRawSnapshotMetadata(rawMetadata);
|
|
539
539
|
|
|
540
540
|
const dataPtr = this._e._emsc_malloc(wasmBytes.length);
|
|
541
|
-
if (!dataPtr) throw
|
|
541
|
+
if (!dataPtr) throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
542
542
|
'WASM malloc failed for import');
|
|
543
543
|
|
|
544
544
|
let status;
|
|
545
545
|
try {
|
|
546
546
|
this._e.HEAPU8.set(wasmBytes, dataPtr);
|
|
547
|
-
status = this._e.
|
|
547
|
+
status = this._e._pikelet_import(this._handle, dataPtr, wasmBytes.length);
|
|
548
548
|
} finally {
|
|
549
549
|
this._e._emsc_free(dataPtr);
|
|
550
550
|
}
|
|
551
551
|
|
|
552
552
|
if (status !== 0) {
|
|
553
|
-
throw
|
|
553
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Import failed');
|
|
554
554
|
}
|
|
555
555
|
|
|
556
|
-
const count = this._e.
|
|
556
|
+
const count = this._e._pikelet_count(this._handle);
|
|
557
557
|
if (pendingMappings === null) {
|
|
558
558
|
this._setIdentityMappings(count);
|
|
559
559
|
return;
|
|
@@ -569,22 +569,22 @@ class PancakeIndex {
|
|
|
569
569
|
|
|
570
570
|
get count() {
|
|
571
571
|
this._checkDisposed();
|
|
572
|
-
return this._e.
|
|
572
|
+
return this._e._pikelet_count(this._handle);
|
|
573
573
|
}
|
|
574
574
|
|
|
575
575
|
get ghostCount() {
|
|
576
576
|
this._checkDisposed();
|
|
577
|
-
return this._e.
|
|
577
|
+
return this._e._pikelet_ghost_count(this._handle);
|
|
578
578
|
}
|
|
579
579
|
|
|
580
580
|
get ghostRatio() {
|
|
581
581
|
this._checkDisposed();
|
|
582
|
-
return this._e.
|
|
582
|
+
return this._e._pikelet_ghost_ratio(this._handle);
|
|
583
583
|
}
|
|
584
584
|
|
|
585
585
|
get memory() {
|
|
586
586
|
this._checkDisposed();
|
|
587
|
-
return this._e.
|
|
587
|
+
return this._e._pikelet_memory(this._handle);
|
|
588
588
|
}
|
|
589
589
|
|
|
590
590
|
get dim() { return this._dim; }
|
|
@@ -627,7 +627,7 @@ class PancakeIndex {
|
|
|
627
627
|
get memoryUsage() {
|
|
628
628
|
this._checkDisposed();
|
|
629
629
|
return Object.freeze({
|
|
630
|
-
logicalIndexBytes: this._e.
|
|
630
|
+
logicalIndexBytes: this._e._pikelet_memory(this._handle),
|
|
631
631
|
wasmHeapBytes: this._e.HEAPU8.buffer.byteLength,
|
|
632
632
|
snapshotBufferBytes: this._snapshotBufferBytes,
|
|
633
633
|
});
|
|
@@ -637,7 +637,7 @@ class PancakeIndex {
|
|
|
637
637
|
if (this._disposed) return;
|
|
638
638
|
let thrown = null;
|
|
639
639
|
try {
|
|
640
|
-
this._e.
|
|
640
|
+
this._e._pikelet_dispose(this._handle);
|
|
641
641
|
} catch (err) {
|
|
642
642
|
thrown = err;
|
|
643
643
|
} finally {
|
|
@@ -660,9 +660,9 @@ class PancakeIndex {
|
|
|
660
660
|
|
|
661
661
|
_checkDisposed() {
|
|
662
662
|
if (this._disposed) {
|
|
663
|
-
throw
|
|
664
|
-
|
|
665
|
-
'
|
|
663
|
+
throw pikeletError(
|
|
664
|
+
PIKELET_ERROR_CODES.INDEX_DISPOSED,
|
|
665
|
+
'PikeletIndex has been disposed'
|
|
666
666
|
);
|
|
667
667
|
}
|
|
668
668
|
}
|
|
@@ -675,7 +675,7 @@ class PancakeIndex {
|
|
|
675
675
|
if (!newIdPtr || !newDistPtr) {
|
|
676
676
|
if (newIdPtr) this._e._emsc_free(newIdPtr);
|
|
677
677
|
if (newDistPtr) this._e._emsc_free(newDistPtr);
|
|
678
|
-
throw
|
|
678
|
+
throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
679
679
|
`WASM malloc failed while growing search buffers to k=${k}`, { k });
|
|
680
680
|
}
|
|
681
681
|
|
|
@@ -716,11 +716,11 @@ class PancakeIndex {
|
|
|
716
716
|
|
|
717
717
|
_validateV3Mappings(intToExt, nextExtId, count) {
|
|
718
718
|
if (intToExt.size !== count) {
|
|
719
|
-
throw
|
|
719
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
720
720
|
'Import failed: envelope mapping count mismatch');
|
|
721
721
|
}
|
|
722
722
|
if (!Number.isInteger(nextExtId) || nextExtId < count) {
|
|
723
|
-
throw
|
|
723
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
724
724
|
'Import failed: envelope nextExtId is invalid');
|
|
725
725
|
}
|
|
726
726
|
|
|
@@ -728,22 +728,22 @@ class PancakeIndex {
|
|
|
728
728
|
let maxExtId = -1;
|
|
729
729
|
for (const [intId, extId] of intToExt) {
|
|
730
730
|
if (!Number.isInteger(intId) || intId < 0 || intId >= count) {
|
|
731
|
-
throw
|
|
731
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
732
732
|
'Import failed: envelope mapping contains invalid internal ID');
|
|
733
733
|
}
|
|
734
734
|
if (!Number.isInteger(extId) || extId < 0) {
|
|
735
|
-
throw
|
|
735
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
736
736
|
'Import failed: envelope mapping contains invalid external ID');
|
|
737
737
|
}
|
|
738
738
|
if (extIds.has(extId)) {
|
|
739
|
-
throw
|
|
739
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
740
740
|
'Import failed: envelope mapping contains duplicates');
|
|
741
741
|
}
|
|
742
742
|
extIds.add(extId);
|
|
743
743
|
if (extId > maxExtId) maxExtId = extId;
|
|
744
744
|
}
|
|
745
745
|
if (nextExtId <= maxExtId) {
|
|
746
|
-
throw
|
|
746
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
747
747
|
'Import failed: envelope nextExtId is invalid');
|
|
748
748
|
}
|
|
749
749
|
}
|
|
@@ -765,23 +765,23 @@ class PancakeIndex {
|
|
|
765
765
|
// reads in the native deserializers.
|
|
766
766
|
const maxVersion = metadata.quantized ? 2 : 1;
|
|
767
767
|
if (!Number.isInteger(metadata.version) || metadata.version > maxVersion) {
|
|
768
|
-
throw
|
|
768
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
769
769
|
`Import failed: unsupported snapshot format version ${metadata.version}`,
|
|
770
770
|
{ version: metadata.version, maxSupported: maxVersion });
|
|
771
771
|
}
|
|
772
772
|
if (!Number.isInteger(metadata.count) || metadata.count < 0) {
|
|
773
|
-
throw
|
|
773
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
774
774
|
'Import failed: snapshot count is invalid');
|
|
775
775
|
}
|
|
776
776
|
if (metadata.count > this._maxElements) {
|
|
777
|
-
throw
|
|
778
|
-
|
|
777
|
+
throw pikeletError(
|
|
778
|
+
PIKELET_ERROR_CODES.SNAPSHOT_CAPACITY_EXCEEDED,
|
|
779
779
|
`Import failed: snapshot count ${metadata.count} exceeds maxElements ${this._maxElements}`,
|
|
780
780
|
{ count: metadata.count, maxElements: this._maxElements }
|
|
781
781
|
);
|
|
782
782
|
}
|
|
783
783
|
if (metadata.dim !== this._dim) {
|
|
784
|
-
throw
|
|
784
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
785
785
|
`Import failed: dim mismatch (exported ${metadata.dim}, expected ${this._dim})`,
|
|
786
786
|
{ field: 'dim', exported: metadata.dim, expected: this._dim });
|
|
787
787
|
}
|
|
@@ -790,23 +790,23 @@ class PancakeIndex {
|
|
|
790
790
|
if (exportedL2 !== this._isL2) {
|
|
791
791
|
const got = exportedL2 ? 'l2' : 'cosine';
|
|
792
792
|
const want = this._isL2 ? 'l2' : 'cosine';
|
|
793
|
-
throw
|
|
793
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
794
794
|
`Import failed: metric mismatch (exported ${got}, expected ${want})`,
|
|
795
795
|
{ field: 'metric', exported: got, expected: want });
|
|
796
796
|
}
|
|
797
797
|
|
|
798
798
|
if (metadata.quantized !== this._quantized) {
|
|
799
|
-
throw
|
|
799
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
800
800
|
'Import failed: quantized mismatch',
|
|
801
801
|
{ field: 'quantized', exported: metadata.quantized, expected: this._quantized });
|
|
802
802
|
}
|
|
803
803
|
if (metadata.M !== this._M) {
|
|
804
|
-
throw
|
|
804
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
805
805
|
`Import failed: M mismatch (exported ${metadata.M}, expected ${this._M})`,
|
|
806
806
|
{ field: 'M', exported: metadata.M, expected: this._M });
|
|
807
807
|
}
|
|
808
808
|
if (metadata.efConstruction !== this._efConstruction) {
|
|
809
|
-
throw
|
|
809
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
810
810
|
`Import failed: efConstruction mismatch (exported ${metadata.efConstruction}, expected ${this._efConstruction})`,
|
|
811
811
|
{ field: 'efConstruction', exported: metadata.efConstruction, expected: this._efConstruction });
|
|
812
812
|
}
|
|
@@ -823,8 +823,8 @@ class PancakeIndex {
|
|
|
823
823
|
if (this._isL2) distance = Math.sqrt(distance);
|
|
824
824
|
const extId = this._intToExt.get(intId);
|
|
825
825
|
if (extId === undefined) {
|
|
826
|
-
throw
|
|
827
|
-
|
|
826
|
+
throw pikeletError(
|
|
827
|
+
PIKELET_ERROR_CODES.INTERNAL_INVARIANT,
|
|
828
828
|
`Search invariant failed: missing external ID mapping for internal ID ${intId}`,
|
|
829
829
|
{ internalId: intId }
|
|
830
830
|
);
|
|
@@ -835,7 +835,7 @@ class PancakeIndex {
|
|
|
835
835
|
}
|
|
836
836
|
}
|
|
837
837
|
|
|
838
|
-
function
|
|
838
|
+
function createPikeletApi(loadEngineImpl) {
|
|
839
839
|
function isVectorInput(value) {
|
|
840
840
|
return value instanceof Float32Array || Array.isArray(value);
|
|
841
841
|
}
|
|
@@ -845,7 +845,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
845
845
|
return { vector: item, hasSourceId: false, sourceId: undefined };
|
|
846
846
|
}
|
|
847
847
|
if (!item || typeof item !== 'object' || !('vector' in item)) {
|
|
848
|
-
throw
|
|
848
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
849
849
|
'fromVectors() requires vectors or { vector, id? } records');
|
|
850
850
|
}
|
|
851
851
|
const sourceId = item.id;
|
|
@@ -854,40 +854,40 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
854
854
|
}
|
|
855
855
|
|
|
856
856
|
async function create(opts) {
|
|
857
|
-
if (!opts || !opts.dim) throw
|
|
857
|
+
if (!opts || !opts.dim) throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
858
858
|
'opts.dim is required', { argument: 'dim' });
|
|
859
859
|
if (!Number.isInteger(opts.dim) || opts.dim <= 0) {
|
|
860
|
-
throw
|
|
860
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
861
861
|
'opts.dim must be a positive integer', { argument: 'dim', value: opts.dim });
|
|
862
862
|
}
|
|
863
863
|
if ('compressed' in opts) {
|
|
864
|
-
throw
|
|
864
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'opts.compressed has been removed');
|
|
865
865
|
}
|
|
866
866
|
if ('varianceSample' in opts) {
|
|
867
|
-
throw
|
|
867
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'opts.varianceSample has been removed');
|
|
868
868
|
}
|
|
869
869
|
if (opts.metric !== undefined && opts.metric !== 'cosine' && opts.metric !== 'l2') {
|
|
870
|
-
throw
|
|
870
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
871
871
|
"opts.metric must be 'cosine' or 'l2'", { argument: 'metric', value: opts.metric });
|
|
872
872
|
}
|
|
873
873
|
// Upper bound matches the engine ABI: pancake_init takes max_elem as a
|
|
874
874
|
// C int, so anything above 2^31-1 would truncate or go negative.
|
|
875
875
|
if (opts.maxElements !== undefined && (!Number.isInteger(opts.maxElements) || opts.maxElements <= 0 || opts.maxElements > 0x7fffffff)) {
|
|
876
|
-
throw
|
|
876
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
877
877
|
'opts.maxElements must be an integer between 1 and 2147483647', { argument: 'maxElements', value: opts.maxElements });
|
|
878
878
|
}
|
|
879
879
|
if (opts.M !== undefined && (!Number.isInteger(opts.M) || opts.M <= 1 || opts.M > 128)) {
|
|
880
|
-
throw
|
|
880
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
881
881
|
'opts.M must be an integer between 2 and 128', { argument: 'M', value: opts.M });
|
|
882
882
|
}
|
|
883
883
|
if (opts.efConstruction !== undefined && (!Number.isInteger(opts.efConstruction) || opts.efConstruction <= 0 || opts.efConstruction > 4096)) {
|
|
884
|
-
throw
|
|
884
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
885
885
|
'opts.efConstruction must be an integer between 1 and 4096',
|
|
886
886
|
{ argument: 'efConstruction', value: opts.efConstruction });
|
|
887
887
|
}
|
|
888
888
|
if (opts.efSearch !== undefined) validateEfSearch(opts.efSearch, 'opts.efSearch');
|
|
889
889
|
if (opts.seed !== undefined && (!Number.isInteger(opts.seed) || opts.seed <= 0 || opts.seed > 0x7fffffff)) {
|
|
890
|
-
throw
|
|
890
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
891
891
|
'opts.seed must be an integer between 1 and 2147483647',
|
|
892
892
|
{ argument: 'seed', value: opts.seed });
|
|
893
893
|
}
|
|
@@ -912,7 +912,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
912
912
|
const bytesPerElement = isQuantized ? dim + 16 * M + 39 : 4 * dim + 8 * M + 23;
|
|
913
913
|
const estimatedBytes = maxElements * bytesPerElement;
|
|
914
914
|
if (estimatedBytes > MAX_INDEX_ARENA_BYTES) {
|
|
915
|
-
throw
|
|
915
|
+
throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
916
916
|
`Index configuration needs ~${Math.ceil(estimatedBytes / (1024 * 1024))} MiB `
|
|
917
917
|
+ `(${isQuantized ? 'quantized' : 'float32'} backend: maxElements ${maxElements} × ~${bytesPerElement} B/element), `
|
|
918
918
|
+ `above the ${Math.floor(MAX_INDEX_ARENA_BYTES / (1024 * 1024))} MiB wasm32 index budget; `
|
|
@@ -943,10 +943,10 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
943
943
|
if (vecPtr) e._emsc_free(vecPtr);
|
|
944
944
|
if (idPtr) e._emsc_free(idPtr);
|
|
945
945
|
if (distPtr) e._emsc_free(distPtr);
|
|
946
|
-
throw
|
|
946
|
+
throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED, 'WASM malloc failed');
|
|
947
947
|
}
|
|
948
948
|
|
|
949
|
-
const handle = e.
|
|
949
|
+
const handle = e._pikelet_init(dim, maxElements, quantized, metric, M, efConstruction, efSearch, seed);
|
|
950
950
|
|
|
951
951
|
// The engine returns uint32_t INVALID_HANDLE (0xFFFFFFFF), which the
|
|
952
952
|
// Emscripten i32 ABI delivers to JS as -1.
|
|
@@ -954,18 +954,18 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
954
954
|
e._emsc_free(vecPtr);
|
|
955
955
|
e._emsc_free(idPtr);
|
|
956
956
|
e._emsc_free(distPtr);
|
|
957
|
-
throw
|
|
957
|
+
throw pikeletError(PIKELET_ERROR_CODES.WASM_ALLOCATION_FAILED, 'Backend init failed');
|
|
958
958
|
}
|
|
959
959
|
|
|
960
|
-
return new
|
|
960
|
+
return new PikeletIndex(e, resolvedOpts, handle, vecPtr, idPtr, distPtr, initialBufferCapacity);
|
|
961
961
|
}
|
|
962
962
|
|
|
963
963
|
async function fromVectors(items, opts = {}) {
|
|
964
964
|
if (!Array.isArray(items)) {
|
|
965
|
-
throw
|
|
965
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'fromVectors() requires an array');
|
|
966
966
|
}
|
|
967
967
|
if (items.length === 0) {
|
|
968
|
-
throw
|
|
968
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
969
969
|
'fromVectors() requires at least one vector');
|
|
970
970
|
}
|
|
971
971
|
|
|
@@ -977,18 +977,18 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
977
977
|
for (let i = 0; i < items.length; i++) {
|
|
978
978
|
const { vector, hasSourceId, sourceId } = extractVectorRecord(items[i]);
|
|
979
979
|
if (!isVectorInput(vector)) {
|
|
980
|
-
throw
|
|
980
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR,
|
|
981
981
|
`fromVectors() expected a vector at index ${i}`, { index: i });
|
|
982
982
|
}
|
|
983
983
|
const dim = vector.length;
|
|
984
984
|
if (!Number.isInteger(dim) || dim <= 0) {
|
|
985
|
-
throw
|
|
985
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_VECTOR,
|
|
986
986
|
`fromVectors() expected a non-empty vector at index ${i}`, { index: i });
|
|
987
987
|
}
|
|
988
988
|
if (inferredDim === null) {
|
|
989
989
|
inferredDim = dim;
|
|
990
990
|
} else if (dim !== inferredDim) {
|
|
991
|
-
throw
|
|
991
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
992
992
|
`fromVectors() found mixed vector dimensions (${inferredDim} and ${dim})`,
|
|
993
993
|
{ expected: inferredDim, actual: dim, index: i });
|
|
994
994
|
}
|
|
@@ -998,7 +998,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
998
998
|
}
|
|
999
999
|
|
|
1000
1000
|
if (opts.dim !== undefined && opts.dim !== inferredDim) {
|
|
1001
|
-
throw
|
|
1001
|
+
throw pikeletError(PIKELET_ERROR_CODES.DIMENSION_MISMATCH,
|
|
1002
1002
|
`fromVectors() dim mismatch (inferred ${inferredDim}, got opts.dim=${opts.dim})`,
|
|
1003
1003
|
{ expected: inferredDim, actual: opts.dim });
|
|
1004
1004
|
}
|
|
@@ -1029,7 +1029,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
1029
1029
|
|
|
1030
1030
|
async function withIndex(options, fn) {
|
|
1031
1031
|
if (typeof fn !== 'function') {
|
|
1032
|
-
throw
|
|
1032
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
1033
1033
|
'withIndex() requires a callback function', { argument: 'fn' });
|
|
1034
1034
|
}
|
|
1035
1035
|
const index = await create(options);
|
|
@@ -1046,7 +1046,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
1046
1046
|
|
|
1047
1047
|
async function restore(snapshot, overrides = {}) {
|
|
1048
1048
|
if (!overrides || typeof overrides !== 'object' || Array.isArray(overrides)) {
|
|
1049
|
-
throw
|
|
1049
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
1050
1050
|
'restore() overrides must be an object', { argument: 'overrides' });
|
|
1051
1051
|
}
|
|
1052
1052
|
|
|
@@ -1062,7 +1062,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
1062
1062
|
if (metadata.format === 'raw') {
|
|
1063
1063
|
for (const field of Object.keys(fixedConfig)) {
|
|
1064
1064
|
if (overrides[field] === undefined) {
|
|
1065
|
-
throw
|
|
1065
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
1066
1066
|
`restore() requires overrides.${field} for legacy raw snapshots`,
|
|
1067
1067
|
{ argument: field, format: 'raw' });
|
|
1068
1068
|
}
|
|
@@ -1071,7 +1071,7 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
1071
1071
|
|
|
1072
1072
|
for (const [field, expected] of Object.entries(fixedConfig)) {
|
|
1073
1073
|
if (overrides[field] !== undefined && overrides[field] !== expected) {
|
|
1074
|
-
throw
|
|
1074
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
1075
1075
|
`restore() ${field} override does not match the snapshot`,
|
|
1076
1076
|
{ field, exported: expected, override: overrides[field] });
|
|
1077
1077
|
}
|
|
@@ -1102,8 +1102,8 @@ function createPancakeApi(loadEngineImpl) {
|
|
|
1102
1102
|
withIndex,
|
|
1103
1103
|
restore,
|
|
1104
1104
|
inspectSnapshot,
|
|
1105
|
-
|
|
1106
|
-
|
|
1105
|
+
PikeletError,
|
|
1106
|
+
PIKELET_ERROR_CODES,
|
|
1107
1107
|
};
|
|
1108
1108
|
}
|
|
1109
1109
|
|
|
@@ -1153,14 +1153,14 @@ function snapshotBytes(data, methodName) {
|
|
|
1153
1153
|
if (ArrayBuffer.isView(data)) {
|
|
1154
1154
|
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
1155
1155
|
}
|
|
1156
|
-
throw
|
|
1156
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
1157
1157
|
`${methodName}() requires snapshot bytes`, { argument: 'snapshot' });
|
|
1158
1158
|
}
|
|
1159
1159
|
|
|
1160
1160
|
function inspectSnapshotMetadata(data) {
|
|
1161
1161
|
const bytes = snapshotBytes(data, 'inspectSnapshot');
|
|
1162
1162
|
if (bytes.byteLength < 4) {
|
|
1163
|
-
throw
|
|
1163
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot is too small to inspect');
|
|
1164
1164
|
}
|
|
1165
1165
|
|
|
1166
1166
|
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
@@ -1172,20 +1172,20 @@ function inspectSnapshotMetadata(data) {
|
|
|
1172
1172
|
let envelopeQuantized = null;
|
|
1173
1173
|
let nextId = null;
|
|
1174
1174
|
|
|
1175
|
-
if (view.getUint32(0, true) ===
|
|
1176
|
-
format = '
|
|
1175
|
+
if (view.getUint32(0, true) === PIKELET_MAGIC) {
|
|
1176
|
+
format = 'pikelet';
|
|
1177
1177
|
if (bytes.byteLength < 8) {
|
|
1178
|
-
throw
|
|
1178
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot envelope is truncated');
|
|
1179
1179
|
}
|
|
1180
1180
|
envelopeVersion = view.getUint32(4, true);
|
|
1181
1181
|
let rawOffset;
|
|
1182
1182
|
if (envelopeVersion === 1) {
|
|
1183
1183
|
if (bytes.byteLength < V1_ENVELOPE_HEADER_SIZE) {
|
|
1184
|
-
throw
|
|
1184
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot v1 envelope is truncated');
|
|
1185
1185
|
}
|
|
1186
1186
|
envelopeDim = view.getUint32(8, true);
|
|
1187
1187
|
if (view.getUint32(12, true) !== envelopeDim) {
|
|
1188
|
-
throw
|
|
1188
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1189
1189
|
'DCT/PCA snapshots are no longer supported');
|
|
1190
1190
|
}
|
|
1191
1191
|
envelopeMetric = view.getUint32(16, true);
|
|
@@ -1193,7 +1193,7 @@ function inspectSnapshotMetadata(data) {
|
|
|
1193
1193
|
rawOffset = V1_ENVELOPE_HEADER_SIZE;
|
|
1194
1194
|
} else if (envelopeVersion === 2) {
|
|
1195
1195
|
if (bytes.byteLength < V2_ENVELOPE_HEADER_SIZE) {
|
|
1196
|
-
throw
|
|
1196
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot v2 envelope is truncated');
|
|
1197
1197
|
}
|
|
1198
1198
|
envelopeDim = view.getUint32(8, true);
|
|
1199
1199
|
envelopeMetric = view.getUint32(12, true);
|
|
@@ -1201,7 +1201,7 @@ function inspectSnapshotMetadata(data) {
|
|
|
1201
1201
|
rawOffset = V2_ENVELOPE_HEADER_SIZE;
|
|
1202
1202
|
} else if (envelopeVersion === ENVELOPE_VERSION) {
|
|
1203
1203
|
if (bytes.byteLength < V3_ENVELOPE_HEADER_SIZE) {
|
|
1204
|
-
throw
|
|
1204
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot v3 envelope is truncated');
|
|
1205
1205
|
}
|
|
1206
1206
|
envelopeDim = view.getUint32(8, true);
|
|
1207
1207
|
envelopeMetric = view.getUint32(12, true);
|
|
@@ -1212,12 +1212,12 @@ function inspectSnapshotMetadata(data) {
|
|
|
1212
1212
|
rawOffset = V3_ENVELOPE_HEADER_SIZE + mappingCount * MAPPING_ENTRY_SIZE;
|
|
1213
1213
|
if (!Number.isSafeInteger(rawOffset) || rawOffset > bytes.byteLength ||
|
|
1214
1214
|
rawSize > bytes.byteLength - rawOffset) {
|
|
1215
|
-
throw
|
|
1215
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1216
1216
|
'Snapshot v3 envelope payload is truncated');
|
|
1217
1217
|
}
|
|
1218
1218
|
rawBytes = bytes.subarray(rawOffset, rawOffset + rawSize);
|
|
1219
1219
|
} else {
|
|
1220
|
-
throw
|
|
1220
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1221
1221
|
`Unsupported snapshot envelope version ${envelopeVersion}`, { version: envelopeVersion });
|
|
1222
1222
|
}
|
|
1223
1223
|
if (envelopeVersion !== ENVELOPE_VERSION) rawBytes = bytes.subarray(rawOffset);
|
|
@@ -1225,27 +1225,27 @@ function inspectSnapshotMetadata(data) {
|
|
|
1225
1225
|
|
|
1226
1226
|
const raw = parseRawSnapshotMetadata(rawBytes);
|
|
1227
1227
|
if (raw === null) {
|
|
1228
|
-
throw
|
|
1228
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1229
1229
|
'Unsupported or truncated raw snapshot format');
|
|
1230
1230
|
}
|
|
1231
1231
|
if (raw.dim <= 0 || raw.metric > 1 || raw.M < 2 || raw.M > 128 ||
|
|
1232
1232
|
raw.M0 !== raw.M * 2 || raw.efConstruction < 1 || raw.efConstruction > 4096) {
|
|
1233
|
-
throw
|
|
1233
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1234
1234
|
'Snapshot header contains invalid index configuration');
|
|
1235
1235
|
}
|
|
1236
|
-
if (format === '
|
|
1236
|
+
if (format === 'pikelet' && (envelopeDim !== raw.dim || envelopeMetric !== raw.metric ||
|
|
1237
1237
|
!!envelopeQuantized !== raw.quantized)) {
|
|
1238
|
-
throw
|
|
1238
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1239
1239
|
'Snapshot envelope does not match its raw index header');
|
|
1240
1240
|
}
|
|
1241
1241
|
if (nextId === null) nextId = raw.count;
|
|
1242
1242
|
if (nextId < raw.count) {
|
|
1243
|
-
throw
|
|
1243
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot nextId is invalid');
|
|
1244
1244
|
}
|
|
1245
1245
|
|
|
1246
1246
|
return Object.freeze({
|
|
1247
1247
|
format,
|
|
1248
|
-
version: format === '
|
|
1248
|
+
version: format === 'pikelet' ? envelopeVersion : raw.version,
|
|
1249
1249
|
dim: raw.dim,
|
|
1250
1250
|
count: raw.count,
|
|
1251
1251
|
metric: raw.metric === 0 ? 'l2' : 'cosine',
|
|
@@ -1256,13 +1256,13 @@ function inspectSnapshotMetadata(data) {
|
|
|
1256
1256
|
});
|
|
1257
1257
|
}
|
|
1258
1258
|
|
|
1259
|
-
module.exports =
|
|
1260
|
-
module.exports.default =
|
|
1261
|
-
module.exports.
|
|
1262
|
-
module.exports.
|
|
1259
|
+
module.exports = createPikeletApi;
|
|
1260
|
+
module.exports.default = createPikeletApi;
|
|
1261
|
+
module.exports.PikeletError = PikeletError;
|
|
1262
|
+
module.exports.PIKELET_ERROR_CODES = PIKELET_ERROR_CODES;
|
|
1263
1263
|
|
|
1264
1264
|
if (typeof Symbol.dispose === 'symbol') {
|
|
1265
|
-
|
|
1265
|
+
PikeletIndex.prototype[Symbol.dispose] = function disposeWithSymbol() {
|
|
1266
1266
|
this.dispose();
|
|
1267
1267
|
};
|
|
1268
1268
|
}
|