pikelet-wasm 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +868 -0
- package/LICENSE +191 -0
- package/README.md +590 -0
- package/complete/builder.d.ts +82 -0
- package/complete/builder.mjs +527 -0
- package/complete/encoder-kernels/encoder.mjs +2 -0
- package/complete/encoder-kernels/encoder.node.mjs +2 -0
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/format.mjs +8 -0
- package/complete/index.d.ts +289 -0
- package/complete/index.mjs +1124 -0
- package/complete/inline-transformer.mjs +205 -0
- package/complete/lexical.mjs +264 -0
- package/complete/retrieval-abstention.mjs +119 -0
- package/complete/sources.mjs +186 -0
- package/complete/student-abstention.mjs +102 -0
- package/complete/student-embedder.mjs +231 -0
- package/complete/wordpiece.mjs +68 -0
- package/dist/engine.js +2 -0
- package/dist/engine.scalar.js +2 -0
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +168 -0
- package/pancake-artifact-common.js +561 -0
- package/pancake-artifact-range.js +799 -0
- package/pancake-artifact-sketch.js +1040 -0
- package/pancake-artifact.d.ts +109 -0
- package/pancake-artifact.js +31 -0
- package/pancake-core.js +1268 -0
- package/pancake-errors.js +40 -0
- package/pancake-loader.js +66 -0
- package/pancake.d.ts +691 -0
- package/pancake.js +254 -0
- package/pancake.node.mjs +259 -0
- package/pancake.web.d.ts +50 -0
- package/pancake.web.mjs +88 -0
- package/pancake.workerd.mjs +72 -0
package/pancake-core.js
ADDED
|
@@ -0,0 +1,1268 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const {
|
|
4
|
+
PancakeError,
|
|
5
|
+
PANCAKE_ERROR_CODES,
|
|
6
|
+
pancakeError,
|
|
7
|
+
} = require('./pancake-errors.js');
|
|
8
|
+
|
|
9
|
+
// Envelope header for validated export/import
|
|
10
|
+
const PANCAKE_MAGIC = 0x504E434B; // "PNCK"
|
|
11
|
+
const ENVELOPE_VERSION = 3;
|
|
12
|
+
const V1_ENVELOPE_HEADER_SIZE = 24; // magic(4) + version(4) + dim(4) + compressed(4) + metric(4) + quantized(4)
|
|
13
|
+
const V2_ENVELOPE_HEADER_SIZE = 20; // magic(4) + version(4) + dim(4) + metric(4) + quantized(4)
|
|
14
|
+
const V3_ENVELOPE_HEADER_SIZE = 32; // v2 + nextExtId(4) + mappingCount(4) + wasmSize(4)
|
|
15
|
+
const MAPPING_ENTRY_SIZE = 8; // intId(4) + extId(4)
|
|
16
|
+
const FLOAT_HNSW_MAGIC_V0 = 0x464C4857; // "FLHW"
|
|
17
|
+
const FLOAT_HNSW_MAGIC_V1 = 0x464C4831; // "FLH1"
|
|
18
|
+
const UINT8_HNSW_MAGIC_V0 = 0x49384857; // "I8HW"
|
|
19
|
+
const UINT8_HNSW_MAGIC_V1 = 0x49384831; // "I8H1"
|
|
20
|
+
const MAX_EF = 4096;
|
|
21
|
+
// The wasm32 heap tops out at 2 GiB (no MAXIMUM_MEMORY override, no
|
|
22
|
+
// MEMORY64). The index arena gets 1.5 GiB of that; the rest is headroom for
|
|
23
|
+
// the 16 MiB initial heap, query scratch, export buffers, and envelope
|
|
24
|
+
// copies during import.
|
|
25
|
+
const MAX_INDEX_ARENA_BYTES = 1536 * 1024 * 1024;
|
|
26
|
+
const DEFAULT_MAX_ELEMENTS = 100000;
|
|
27
|
+
const DEFAULT_M = 12;
|
|
28
|
+
const DEFAULT_EF_CONSTRUCTION = 75;
|
|
29
|
+
const DEFAULT_EF_SEARCH = 100;
|
|
30
|
+
const DEFAULT_SEED = 108;
|
|
31
|
+
|
|
32
|
+
function validateEfSearch(value, label = 'efSearch') {
|
|
33
|
+
if (!Number.isInteger(value) || value < 1 || value > MAX_EF) {
|
|
34
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
35
|
+
`${label} must be an integer between 1 and ${MAX_EF}`,
|
|
36
|
+
{ argument: label, value, min: 1, max: MAX_EF });
|
|
37
|
+
}
|
|
38
|
+
return value;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function resolveSearchEf(options, fallback, methodName) {
|
|
42
|
+
if (options === undefined) return fallback;
|
|
43
|
+
if (!options || typeof options !== 'object' || Array.isArray(options)) {
|
|
44
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
45
|
+
`${methodName}() options must be an object`, { argument: 'options' });
|
|
46
|
+
}
|
|
47
|
+
return options.efSearch === undefined
|
|
48
|
+
? fallback
|
|
49
|
+
: validateEfSearch(options.efSearch, `${methodName}() options.efSearch`);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Reject non-numeric elements in a plain-array vector input. A Float32Array
|
|
53
|
+
// already guarantees numeric storage, but `new Float32Array([...])` silently
|
|
54
|
+
// coerces a plain array's elements via Number() — so '1'->1, ''->0, true->1,
|
|
55
|
+
// [2]->2 would be accepted as valid coordinates. That turns malformed input
|
|
56
|
+
// (e.g. an empty CSV field) into a silently-wrong vector instead of a loud
|
|
57
|
+
// error. Require real numbers so callers fail closed on bad data. Float32Array
|
|
58
|
+
// inputs skip this (already numeric). NaN/Infinity are still caught downstream.
|
|
59
|
+
function assertNumericVector(vec, label) {
|
|
60
|
+
if (vec instanceof Float32Array) return;
|
|
61
|
+
if (vec == null || typeof vec.length !== 'number') return; // length check handles these
|
|
62
|
+
for (let i = 0; i < vec.length; i++) {
|
|
63
|
+
if (typeof vec[i] !== 'number') {
|
|
64
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_VECTOR,
|
|
65
|
+
`${label} must contain only numbers; found ${typeof vec[i]} at index ${i}`,
|
|
66
|
+
{ index: i, actualType: typeof vec[i] });
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function validateVectorValues(f32, label, validateCosineNorm) {
|
|
72
|
+
let normSq = 0;
|
|
73
|
+
for (let i = 0; i < f32.length; i++) {
|
|
74
|
+
const value = f32[i];
|
|
75
|
+
if (!Number.isFinite(value)) {
|
|
76
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_VECTOR,
|
|
77
|
+
`${label} contains non-finite value (NaN or Infinity)`, { reason: 'non_finite' });
|
|
78
|
+
}
|
|
79
|
+
if (validateCosineNorm) normSq += value * value;
|
|
80
|
+
}
|
|
81
|
+
if (validateCosineNorm && (!(normSq > 0) || !Number.isFinite(normSq))) {
|
|
82
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_VECTOR,
|
|
83
|
+
`${label} has invalid cosine norm`, { reason: 'invalid_cosine_norm' });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
class PancakeIndex {
|
|
88
|
+
constructor(engine, opts, handle, vecPtr, idPtr, distPtr, bufferCapacity) {
|
|
89
|
+
this._e = engine;
|
|
90
|
+
this._dim = opts.dim;
|
|
91
|
+
this._maxElements = opts.maxElements;
|
|
92
|
+
this._quantized = !!opts.quantized;
|
|
93
|
+
this._isL2 = (opts.metric === 'l2');
|
|
94
|
+
this._M = opts.M;
|
|
95
|
+
this._efConstruction = opts.efConstruction;
|
|
96
|
+
this._efSearch = opts.efSearch;
|
|
97
|
+
this._seed = opts.seed;
|
|
98
|
+
this._handle = handle;
|
|
99
|
+
this._vecPtr = vecPtr;
|
|
100
|
+
this._idPtr = idPtr;
|
|
101
|
+
this._distPtr = distPtr;
|
|
102
|
+
this._bufferCapacity = bufferCapacity;
|
|
103
|
+
this._snapshotBufferBytes = 0;
|
|
104
|
+
this._disposed = false;
|
|
105
|
+
|
|
106
|
+
// ID translation layer -- WASM compact() reassigns sequential IDs,
|
|
107
|
+
// so we maintain a stable external ID space for the consumer.
|
|
108
|
+
this._nextExtId = 0;
|
|
109
|
+
this._intToExt = new Map(); // internal (WASM) id -> external (user) id
|
|
110
|
+
this._extToInt = new Map(); // external (user) id -> internal (WASM) id
|
|
111
|
+
this._deletedExt = new Set(); // external ids that have been soft-deleted
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
add(vec) {
|
|
115
|
+
this._checkDisposed();
|
|
116
|
+
assertNumericVector(vec, 'Vector');
|
|
117
|
+
const f32 = vec instanceof Float32Array ? vec : new Float32Array(vec);
|
|
118
|
+
if (f32.length !== this._dim) {
|
|
119
|
+
throw pancakeError(PANCAKE_ERROR_CODES.DIMENSION_MISMATCH,
|
|
120
|
+
`Expected vector of length ${this._dim}, got ${f32.length}`,
|
|
121
|
+
{ expected: this._dim, actual: f32.length });
|
|
122
|
+
}
|
|
123
|
+
validateVectorValues(f32, 'Vector', !this._isL2);
|
|
124
|
+
// The v3 snapshot stores every external id and the next-id counter as
|
|
125
|
+
// u32. Assigning past that boundary would produce ids the snapshot
|
|
126
|
+
// cannot represent (export would truncate, restore would reject), so
|
|
127
|
+
// the limit is enforced before the engine insert, leaving the index
|
|
128
|
+
// unmutated. Ids survive deletes, so a long-lived high-churn index
|
|
129
|
+
// can reach this without ever holding 2^32 vectors at once.
|
|
130
|
+
if (this._nextExtId >= 0xFFFFFFFF) {
|
|
131
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INDEX_LIMIT,
|
|
132
|
+
'Stable id space exhausted: ids are serialized as uint32 and the next id would not fit',
|
|
133
|
+
{ nextExtId: this._nextExtId });
|
|
134
|
+
}
|
|
135
|
+
this._e.HEAPF32.set(f32, this._vecPtr >> 2);
|
|
136
|
+
const intId = this._e._pancake_add(this._handle, this._vecPtr);
|
|
137
|
+
if (intId === 0xFFFFFFFF || intId < 0) {
|
|
138
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INDEX_FULL,
|
|
139
|
+
'Insert failed (index full or not initialized)');
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const extId = this._nextExtId++;
|
|
143
|
+
this._intToExt.set(intId, extId);
|
|
144
|
+
this._extToInt.set(extId, intId);
|
|
145
|
+
return extId;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
addBatch(vectors) {
|
|
149
|
+
this._checkDisposed();
|
|
150
|
+
if (!Array.isArray(vectors)) {
|
|
151
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
152
|
+
'addBatch() requires an array of vectors');
|
|
153
|
+
}
|
|
154
|
+
if (vectors.length === 0) return [];
|
|
155
|
+
if (this.count + vectors.length > this._maxElements) {
|
|
156
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INDEX_FULL,
|
|
157
|
+
'Insert failed (index full or not initialized)');
|
|
158
|
+
}
|
|
159
|
+
// Same u32 stable-id bound as add(), preflighted for the whole batch
|
|
160
|
+
// so the failure happens before any row is inserted.
|
|
161
|
+
if (this._nextExtId + vectors.length > 0xFFFFFFFF) {
|
|
162
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INDEX_LIMIT,
|
|
163
|
+
'Stable id space exhausted: ids are serialized as uint32 and this batch would run past the last representable id',
|
|
164
|
+
{ nextExtId: this._nextExtId, batch: vectors.length });
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Validate ALL vectors first before mutating the index, so a bad
|
|
168
|
+
// input mid-batch can't leave the index in a partially-inserted state.
|
|
169
|
+
// Keep the converted Float32Arrays so the write pass below doesn't
|
|
170
|
+
// convert plain arrays a second time.
|
|
171
|
+
const converted = new Array(vectors.length);
|
|
172
|
+
for (let i = 0; i < vectors.length; i++) {
|
|
173
|
+
const v = vectors[i];
|
|
174
|
+
const len = (v instanceof Float32Array) ? v.length : (v && v.length);
|
|
175
|
+
if (len !== this._dim) {
|
|
176
|
+
throw pancakeError(PANCAKE_ERROR_CODES.DIMENSION_MISMATCH,
|
|
177
|
+
`Expected vector of length ${this._dim}, got ${len} at index ${i}`,
|
|
178
|
+
{ expected: this._dim, actual: len, index: i });
|
|
179
|
+
}
|
|
180
|
+
assertNumericVector(v, `Vector at index ${i}`);
|
|
181
|
+
const f32 = v instanceof Float32Array ? v : new Float32Array(v);
|
|
182
|
+
validateVectorValues(f32, `Vector at index ${i}`, !this._isL2);
|
|
183
|
+
converted[i] = f32;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// Allocate the WASM buffer once, write directly into the heap (no
|
|
187
|
+
// intermediate JS Float32Array), call bulk_insert once.
|
|
188
|
+
const totalFloats = vectors.length * this._dim;
|
|
189
|
+
const dataPtr = this._e._emsc_malloc(totalFloats * 4);
|
|
190
|
+
if (!dataPtr) throw pancakeError(PANCAKE_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
191
|
+
'WASM malloc failed for bulk insert');
|
|
192
|
+
|
|
193
|
+
try {
|
|
194
|
+
const heapOffset = dataPtr >> 2;
|
|
195
|
+
for (let i = 0; i < vectors.length; i++) {
|
|
196
|
+
this._e.HEAPF32.set(converted[i], heapOffset + i * this._dim);
|
|
197
|
+
}
|
|
198
|
+
const countBefore = this.count;
|
|
199
|
+
const inserted = this._e._pancake_bulk_insert(this._handle, dataPtr, vectors.length);
|
|
200
|
+
const ids = this._recordInsertedRange(countBefore, inserted);
|
|
201
|
+
if (inserted !== vectors.length) {
|
|
202
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INTERNAL_INVARIANT,
|
|
203
|
+
'bulk_insert inserted fewer vectors than the prevalidated batch',
|
|
204
|
+
{ requested: vectors.length, inserted, ids });
|
|
205
|
+
}
|
|
206
|
+
return ids;
|
|
207
|
+
} finally {
|
|
208
|
+
this._e._emsc_free(dataPtr);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
search(query, k, options) {
|
|
213
|
+
this._checkDisposed();
|
|
214
|
+
if (!Number.isSafeInteger(k) || k < 0) {
|
|
215
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
216
|
+
'search() requires a non-negative integer k', { argument: 'k', value: k });
|
|
217
|
+
}
|
|
218
|
+
assertNumericVector(query, 'Query vector');
|
|
219
|
+
const f32 = query instanceof Float32Array ? query : new Float32Array(query);
|
|
220
|
+
if (f32.length !== this._dim) {
|
|
221
|
+
throw pancakeError(PANCAKE_ERROR_CODES.DIMENSION_MISMATCH,
|
|
222
|
+
`Expected query of length ${this._dim}, got ${f32.length}`,
|
|
223
|
+
{ expected: this._dim, actual: f32.length });
|
|
224
|
+
}
|
|
225
|
+
validateVectorValues(f32, 'Query vector', !this._isL2);
|
|
226
|
+
const efSearch = resolveSearchEf(options, this._efSearch, 'search');
|
|
227
|
+
// The C ABI takes a signed 32-bit k and the WASM allocations take
|
|
228
|
+
// 32-bit byte sizes. Passing a larger JS integer through either boundary
|
|
229
|
+
// can wrap and allocate undersized output buffers. An index cannot
|
|
230
|
+
// return more than count results, so cap k before allocating or calling
|
|
231
|
+
// into WASM while preserving the documented k > count behavior.
|
|
232
|
+
const boundedK = Math.min(k, this.count);
|
|
233
|
+
if (boundedK === 0) return [];
|
|
234
|
+
this._ensureSearchCapacity(boundedK);
|
|
235
|
+
this._e.HEAPF32.set(f32, this._vecPtr >> 2);
|
|
236
|
+
const found = this._e._pancake_query(
|
|
237
|
+
this._handle, this._vecPtr, boundedK, efSearch, this._idPtr, this._distPtr
|
|
238
|
+
);
|
|
239
|
+
return this._readResults(found);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
searchFiltered(query, k, allowedIds, options) {
|
|
243
|
+
this._checkDisposed();
|
|
244
|
+
if (!Number.isSafeInteger(k) || k < 0) {
|
|
245
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
246
|
+
'searchFiltered() requires a non-negative integer k', { argument: 'k', value: k });
|
|
247
|
+
}
|
|
248
|
+
assertNumericVector(query, 'Query vector');
|
|
249
|
+
const f32 = query instanceof Float32Array ? query : new Float32Array(query);
|
|
250
|
+
if (f32.length !== this._dim) {
|
|
251
|
+
throw pancakeError(PANCAKE_ERROR_CODES.DIMENSION_MISMATCH,
|
|
252
|
+
`Expected query of length ${this._dim}, got ${f32.length}`,
|
|
253
|
+
{ expected: this._dim, actual: f32.length });
|
|
254
|
+
}
|
|
255
|
+
validateVectorValues(f32, 'Query vector', !this._isL2);
|
|
256
|
+
const efSearch = resolveSearchEf(options, this._efSearch, 'searchFiltered');
|
|
257
|
+
if (!(allowedIds instanceof Set)) {
|
|
258
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
259
|
+
'searchFiltered() requires allowedIds to be a Set<number>', { argument: 'allowedIds' });
|
|
260
|
+
}
|
|
261
|
+
const boundedK = Math.min(k, this.count);
|
|
262
|
+
if (boundedK === 0 || allowedIds.size === 0) return [];
|
|
263
|
+
this._ensureSearchCapacity(boundedK);
|
|
264
|
+
|
|
265
|
+
// Build bitset over internal IDs
|
|
266
|
+
const count = this._e._pancake_count(this._handle);
|
|
267
|
+
const bitsetLen = (count + 7) >> 3;
|
|
268
|
+
const bitsetPtr = this._e._emsc_malloc(bitsetLen);
|
|
269
|
+
if (!bitsetPtr) throw pancakeError(PANCAKE_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
270
|
+
'WASM malloc failed for filter bitset');
|
|
271
|
+
|
|
272
|
+
try {
|
|
273
|
+
// Zero the bitset
|
|
274
|
+
this._e.HEAPU8.fill(0, bitsetPtr, bitsetPtr + bitsetLen);
|
|
275
|
+
|
|
276
|
+
// Set bits for allowed internal IDs
|
|
277
|
+
for (const extId of allowedIds) {
|
|
278
|
+
const intId = this._extToInt.get(extId);
|
|
279
|
+
if (intId !== undefined && !this._deletedExt.has(extId)) {
|
|
280
|
+
this._e.HEAPU8[bitsetPtr + (intId >> 3)] |= (1 << (intId & 7));
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
this._e.HEAPF32.set(f32, this._vecPtr >> 2);
|
|
285
|
+
const found = this._e._pancake_query_filtered(
|
|
286
|
+
this._handle, this._vecPtr, boundedK, efSearch,
|
|
287
|
+
this._idPtr, this._distPtr,
|
|
288
|
+
bitsetPtr, bitsetLen
|
|
289
|
+
);
|
|
290
|
+
return this._readResults(found);
|
|
291
|
+
} finally {
|
|
292
|
+
this._e._emsc_free(bitsetPtr);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
delete(id) {
|
|
297
|
+
this._checkDisposed();
|
|
298
|
+
const intId = this._extToInt.get(id);
|
|
299
|
+
if (intId === undefined || this._deletedExt.has(id)) return false;
|
|
300
|
+
this._e._pancake_delete(this._handle, intId);
|
|
301
|
+
this._deletedExt.add(id);
|
|
302
|
+
return true;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
has(id) {
|
|
306
|
+
this._checkDisposed();
|
|
307
|
+
return this._extToInt.has(id) && !this._deletedExt.has(id);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
isDeleted(id) {
|
|
311
|
+
this._checkDisposed();
|
|
312
|
+
return this._deletedExt.has(id);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
compact() {
|
|
316
|
+
this._checkDisposed();
|
|
317
|
+
|
|
318
|
+
// Use the engine's own old->new remap rather than re-deriving it in
|
|
319
|
+
// JS: how compaction assigns new IDs is the engine's contract, and
|
|
320
|
+
// re-implementing it here would silently corrupt the external-ID
|
|
321
|
+
// translation if the engine's assignment order ever changed.
|
|
322
|
+
const countBefore = this._e._pancake_count(this._handle);
|
|
323
|
+
if (countBefore === 0) {
|
|
324
|
+
this._e._pancake_compact(this._handle);
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
const mapPtr = this._e._emsc_malloc(countBefore * 4);
|
|
329
|
+
if (!mapPtr) throw pancakeError(PANCAKE_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
330
|
+
'WASM malloc failed for compact remap');
|
|
331
|
+
|
|
332
|
+
try {
|
|
333
|
+
const written = this._e._pancake_compact_remap(this._handle, mapPtr, countBefore);
|
|
334
|
+
const base = mapPtr >> 2;
|
|
335
|
+
const remapped = [];
|
|
336
|
+
for (let oldInt = 0; oldInt < written; oldInt++) {
|
|
337
|
+
const newInt = this._e.HEAPU32[base + oldInt];
|
|
338
|
+
if (newInt === 0xFFFFFFFF) continue; // deleted by compaction
|
|
339
|
+
const extId = this._intToExt.get(oldInt);
|
|
340
|
+
if (extId === undefined) continue;
|
|
341
|
+
remapped.push([newInt, extId]);
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
this._intToExt.clear();
|
|
345
|
+
this._extToInt.clear();
|
|
346
|
+
this._deletedExt.clear();
|
|
347
|
+
for (const [newInt, extId] of remapped) {
|
|
348
|
+
this._intToExt.set(newInt, extId);
|
|
349
|
+
this._extToInt.set(extId, newInt);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
const liveCount = this._e._pancake_count(this._handle);
|
|
353
|
+
if (this._intToExt.size !== liveCount) {
|
|
354
|
+
this._clearMappings();
|
|
355
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INTERNAL_INVARIANT,
|
|
356
|
+
`compact() remap mismatch: engine reports ${liveCount} live vectors, remap yielded ${this._intToExt.size}`,
|
|
357
|
+
{ liveCount, mappingCount: this._intToExt.size });
|
|
358
|
+
}
|
|
359
|
+
} finally {
|
|
360
|
+
this._e._emsc_free(mapPtr);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
export() {
|
|
365
|
+
this._checkDisposed();
|
|
366
|
+
if (this.ghostCount > 0) {
|
|
367
|
+
throw pancakeError(PANCAKE_ERROR_CODES.COMPACTION_REQUIRED,
|
|
368
|
+
'Export failed: compact() required before export when ghostCount > 0',
|
|
369
|
+
{ deletedCount: this.ghostCount });
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
const sizePtr = this._e._emsc_malloc(4);
|
|
373
|
+
if (!sizePtr) throw pancakeError(PANCAKE_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
374
|
+
'WASM malloc failed for export');
|
|
375
|
+
try {
|
|
376
|
+
const dataPtr = this._e._pancake_export(this._handle, sizePtr);
|
|
377
|
+
if (!dataPtr) {
|
|
378
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INTERNAL_INVARIANT, 'Export failed');
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
const wasmSize = this._e.HEAPU32[sizePtr >> 2];
|
|
382
|
+
this._snapshotBufferBytes = wasmSize;
|
|
383
|
+
const liveMappings = Array.from(this._intToExt.entries()).sort((a, b) => a[0] - b[0]);
|
|
384
|
+
const mappingBytes = liveMappings.length * MAPPING_ENTRY_SIZE;
|
|
385
|
+
const result = new Uint8Array(V3_ENVELOPE_HEADER_SIZE + mappingBytes + wasmSize);
|
|
386
|
+
const view = new DataView(result.buffer);
|
|
387
|
+
view.setUint32(0, PANCAKE_MAGIC, true);
|
|
388
|
+
view.setUint32(4, ENVELOPE_VERSION, true);
|
|
389
|
+
view.setUint32(8, this._dim, true);
|
|
390
|
+
view.setUint32(12, this._isL2 ? 0 : 1, true);
|
|
391
|
+
view.setUint32(16, this._quantized ? 1 : 0, true);
|
|
392
|
+
view.setUint32(20, this._nextExtId, true);
|
|
393
|
+
view.setUint32(24, liveMappings.length, true);
|
|
394
|
+
view.setUint32(28, wasmSize, true);
|
|
395
|
+
|
|
396
|
+
let offset = V3_ENVELOPE_HEADER_SIZE;
|
|
397
|
+
for (const [intId, extId] of liveMappings) {
|
|
398
|
+
view.setUint32(offset, intId, true);
|
|
399
|
+
view.setUint32(offset + 4, extId, true);
|
|
400
|
+
offset += MAPPING_ENTRY_SIZE;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
result.set(this._e.HEAPU8.subarray(dataPtr, dataPtr + wasmSize), offset);
|
|
404
|
+
return result;
|
|
405
|
+
} finally {
|
|
406
|
+
this._e._emsc_free(sizePtr);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
import(data) {
|
|
411
|
+
this._checkDisposed();
|
|
412
|
+
const bytes = data instanceof Uint8Array ? data : new Uint8Array(data);
|
|
413
|
+
|
|
414
|
+
let wasmBytes = bytes;
|
|
415
|
+
let pendingMappings = null;
|
|
416
|
+
let pendingNextExtId = null;
|
|
417
|
+
|
|
418
|
+
if (bytes.length >= V2_ENVELOPE_HEADER_SIZE) {
|
|
419
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
420
|
+
|
|
421
|
+
if (view.getUint32(0, true) === PANCAKE_MAGIC) {
|
|
422
|
+
const version = view.getUint32(4, true);
|
|
423
|
+
|
|
424
|
+
let dim;
|
|
425
|
+
let metricVal;
|
|
426
|
+
let quantizedVal;
|
|
427
|
+
let nextExtId = null;
|
|
428
|
+
let mappingCount = null;
|
|
429
|
+
let mappingOffset = null;
|
|
430
|
+
let wasmOffset = null;
|
|
431
|
+
let wasmSize = null;
|
|
432
|
+
|
|
433
|
+
if (version === 1) {
|
|
434
|
+
if (bytes.length < V1_ENVELOPE_HEADER_SIZE) {
|
|
435
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID, 'Import failed: truncated v1 envelope');
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
dim = view.getUint32(8, true);
|
|
439
|
+
const compressed = view.getUint32(12, true);
|
|
440
|
+
if (compressed !== dim) {
|
|
441
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
442
|
+
'Import failed: DCT/PCA indexes are no longer supported');
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
metricVal = view.getUint32(16, true);
|
|
446
|
+
quantizedVal = view.getUint32(20, true);
|
|
447
|
+
} else if (version === ENVELOPE_VERSION) {
|
|
448
|
+
if (bytes.length < V3_ENVELOPE_HEADER_SIZE) {
|
|
449
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID, 'Import failed: truncated v3 envelope');
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
dim = view.getUint32(8, true);
|
|
453
|
+
metricVal = view.getUint32(12, true);
|
|
454
|
+
quantizedVal = view.getUint32(16, true);
|
|
455
|
+
nextExtId = view.getUint32(20, true);
|
|
456
|
+
mappingCount = view.getUint32(24, true);
|
|
457
|
+
wasmSize = view.getUint32(28, true);
|
|
458
|
+
mappingOffset = V3_ENVELOPE_HEADER_SIZE;
|
|
459
|
+
wasmOffset = mappingOffset + mappingCount * MAPPING_ENTRY_SIZE;
|
|
460
|
+
|
|
461
|
+
if (wasmOffset > bytes.length || wasmOffset + wasmSize > bytes.length) {
|
|
462
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
463
|
+
'Import failed: truncated v3 envelope payload');
|
|
464
|
+
}
|
|
465
|
+
} else {
|
|
466
|
+
if (version !== 2) {
|
|
467
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
468
|
+
`Import failed: unsupported envelope version ${version}`, { version });
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
dim = view.getUint32(8, true);
|
|
472
|
+
metricVal = view.getUint32(12, true);
|
|
473
|
+
quantizedVal = view.getUint32(16, true);
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
if (dim !== this._dim) {
|
|
477
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
478
|
+
`Import failed: dim mismatch (exported ${dim}, expected ${this._dim})`,
|
|
479
|
+
{ field: 'dim', exported: dim, expected: this._dim });
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
const exportedL2 = metricVal === 0;
|
|
483
|
+
if (exportedL2 !== this._isL2) {
|
|
484
|
+
const got = exportedL2 ? 'l2' : 'cosine';
|
|
485
|
+
const want = this._isL2 ? 'l2' : 'cosine';
|
|
486
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
487
|
+
`Import failed: metric mismatch (exported ${got}, expected ${want})`,
|
|
488
|
+
{ field: 'metric', exported: got, expected: want });
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
if (!!quantizedVal !== this._quantized) {
|
|
492
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
493
|
+
'Import failed: quantized mismatch',
|
|
494
|
+
{ field: 'quantized', exported: !!quantizedVal, expected: this._quantized });
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
if (version === ENVELOPE_VERSION) {
|
|
498
|
+
wasmBytes = bytes.subarray(wasmOffset, wasmOffset + wasmSize);
|
|
499
|
+
const metadata = parseRawSnapshotMetadata(wasmBytes);
|
|
500
|
+
if (metadata === null) {
|
|
501
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
502
|
+
'Import failed: unsupported raw snapshot format');
|
|
503
|
+
}
|
|
504
|
+
this._validateRawSnapshotMetadata(metadata);
|
|
505
|
+
if (mappingCount !== metadata.count) {
|
|
506
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
507
|
+
'Import failed: envelope mapping count mismatch');
|
|
508
|
+
}
|
|
509
|
+
pendingMappings = new Map();
|
|
510
|
+
let offset = mappingOffset;
|
|
511
|
+
for (let i = 0; i < mappingCount; i++) {
|
|
512
|
+
const intId = view.getUint32(offset, true);
|
|
513
|
+
const extId = view.getUint32(offset + 4, true);
|
|
514
|
+
pendingMappings.set(intId, extId);
|
|
515
|
+
offset += MAPPING_ENTRY_SIZE;
|
|
516
|
+
}
|
|
517
|
+
this._validateV3Mappings(pendingMappings, nextExtId, metadata.count);
|
|
518
|
+
pendingNextExtId = nextExtId;
|
|
519
|
+
} else {
|
|
520
|
+
// The v1 header carries an extra `compressed` field, so its
|
|
521
|
+
// payload starts 4 bytes later than v2's.
|
|
522
|
+
wasmBytes = bytes.subarray(
|
|
523
|
+
version === 1 ? V1_ENVELOPE_HEADER_SIZE : V2_ENVELOPE_HEADER_SIZE
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
// Validate the embedded raw snapshot even when an envelope is present.
|
|
530
|
+
// The engine metric is restored from the raw header, so trusting only
|
|
531
|
+
// the envelope would allow the public wrapper and backend to disagree
|
|
532
|
+
// about both query behavior and distance interpretation.
|
|
533
|
+
const rawMetadata = parseRawSnapshotMetadata(wasmBytes);
|
|
534
|
+
if (rawMetadata === null) {
|
|
535
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
536
|
+
'Import failed: unsupported raw snapshot format');
|
|
537
|
+
}
|
|
538
|
+
this._validateRawSnapshotMetadata(rawMetadata);
|
|
539
|
+
|
|
540
|
+
const dataPtr = this._e._emsc_malloc(wasmBytes.length);
|
|
541
|
+
if (!dataPtr) throw pancakeError(PANCAKE_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
542
|
+
'WASM malloc failed for import');
|
|
543
|
+
|
|
544
|
+
let status;
|
|
545
|
+
try {
|
|
546
|
+
this._e.HEAPU8.set(wasmBytes, dataPtr);
|
|
547
|
+
status = this._e._pancake_import(this._handle, dataPtr, wasmBytes.length);
|
|
548
|
+
} finally {
|
|
549
|
+
this._e._emsc_free(dataPtr);
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
if (status !== 0) {
|
|
553
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID, 'Import failed');
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
const count = this._e._pancake_count(this._handle);
|
|
557
|
+
if (pendingMappings === null) {
|
|
558
|
+
this._setIdentityMappings(count);
|
|
559
|
+
return;
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
try {
|
|
563
|
+
this._commitMappings(pendingMappings, pendingNextExtId);
|
|
564
|
+
} catch (err) {
|
|
565
|
+
this._clearMappings();
|
|
566
|
+
throw err;
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
get count() {
|
|
571
|
+
this._checkDisposed();
|
|
572
|
+
return this._e._pancake_count(this._handle);
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
get ghostCount() {
|
|
576
|
+
this._checkDisposed();
|
|
577
|
+
return this._e._pancake_ghost_count(this._handle);
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
get ghostRatio() {
|
|
581
|
+
this._checkDisposed();
|
|
582
|
+
return this._e._pancake_ghost_ratio(this._handle);
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
get memory() {
|
|
586
|
+
this._checkDisposed();
|
|
587
|
+
return this._e._pancake_memory(this._handle);
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
get dim() { return this._dim; }
|
|
591
|
+
|
|
592
|
+
get liveCount() {
|
|
593
|
+
return this.count - this.deletedCount;
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
get deletedCount() {
|
|
597
|
+
return this.ghostCount;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
get deletedRatio() {
|
|
601
|
+
return this.ghostRatio;
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
get capacity() {
|
|
605
|
+
this._checkDisposed();
|
|
606
|
+
return this._maxElements;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
get remainingCapacity() {
|
|
610
|
+
return this.capacity - this.count;
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
get config() {
|
|
614
|
+
this._checkDisposed();
|
|
615
|
+
return Object.freeze({
|
|
616
|
+
dim: this._dim,
|
|
617
|
+
maxElements: this._maxElements,
|
|
618
|
+
metric: this._isL2 ? 'l2' : 'cosine',
|
|
619
|
+
quantized: this._quantized,
|
|
620
|
+
M: this._M,
|
|
621
|
+
efConstruction: this._efConstruction,
|
|
622
|
+
efSearch: this._efSearch,
|
|
623
|
+
seed: this._seed,
|
|
624
|
+
});
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
get memoryUsage() {
|
|
628
|
+
this._checkDisposed();
|
|
629
|
+
return Object.freeze({
|
|
630
|
+
logicalIndexBytes: this._e._pancake_memory(this._handle),
|
|
631
|
+
wasmHeapBytes: this._e.HEAPU8.buffer.byteLength,
|
|
632
|
+
snapshotBufferBytes: this._snapshotBufferBytes,
|
|
633
|
+
});
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
dispose() {
|
|
637
|
+
if (this._disposed) return;
|
|
638
|
+
let thrown = null;
|
|
639
|
+
try {
|
|
640
|
+
this._e._pancake_dispose(this._handle);
|
|
641
|
+
} catch (err) {
|
|
642
|
+
thrown = err;
|
|
643
|
+
} finally {
|
|
644
|
+
for (const ptr of [this._vecPtr, this._idPtr, this._distPtr]) {
|
|
645
|
+
try {
|
|
646
|
+
this._e._emsc_free(ptr);
|
|
647
|
+
} catch (err) {
|
|
648
|
+
if (thrown === null) thrown = err;
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
this._disposed = true;
|
|
652
|
+
}
|
|
653
|
+
if (thrown !== null) throw thrown;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
setEfSearch(ef) {
|
|
657
|
+
this._checkDisposed();
|
|
658
|
+
this._efSearch = validateEfSearch(ef, 'setEfSearch() efSearch');
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
_checkDisposed() {
|
|
662
|
+
if (this._disposed) {
|
|
663
|
+
throw pancakeError(
|
|
664
|
+
PANCAKE_ERROR_CODES.INDEX_DISPOSED,
|
|
665
|
+
'PancakeIndex has been disposed'
|
|
666
|
+
);
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
_ensureSearchCapacity(k) {
|
|
671
|
+
if (k <= this._bufferCapacity) return;
|
|
672
|
+
|
|
673
|
+
const newIdPtr = this._e._emsc_malloc(k * 8);
|
|
674
|
+
const newDistPtr = this._e._emsc_malloc(k * 4);
|
|
675
|
+
if (!newIdPtr || !newDistPtr) {
|
|
676
|
+
if (newIdPtr) this._e._emsc_free(newIdPtr);
|
|
677
|
+
if (newDistPtr) this._e._emsc_free(newDistPtr);
|
|
678
|
+
throw pancakeError(PANCAKE_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
679
|
+
`WASM malloc failed while growing search buffers to k=${k}`, { k });
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
this._e._emsc_free(this._idPtr);
|
|
683
|
+
this._e._emsc_free(this._distPtr);
|
|
684
|
+
this._idPtr = newIdPtr;
|
|
685
|
+
this._distPtr = newDistPtr;
|
|
686
|
+
this._bufferCapacity = k;
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
_recordInsertedRange(firstIntId, count) {
|
|
690
|
+
const ids = new Array(count);
|
|
691
|
+
for (let i = 0; i < count; i++) {
|
|
692
|
+
const intId = firstIntId + i;
|
|
693
|
+
const extId = this._nextExtId++;
|
|
694
|
+
this._intToExt.set(intId, extId);
|
|
695
|
+
this._extToInt.set(extId, intId);
|
|
696
|
+
ids[i] = extId;
|
|
697
|
+
}
|
|
698
|
+
return ids;
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
_clearMappings() {
|
|
702
|
+
this._intToExt.clear();
|
|
703
|
+
this._extToInt.clear();
|
|
704
|
+
this._deletedExt.clear();
|
|
705
|
+
this._nextExtId = 0;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
_commitMappings(intToExt, nextExtId) {
|
|
709
|
+
this._clearMappings();
|
|
710
|
+
for (const [intId, extId] of intToExt) {
|
|
711
|
+
this._intToExt.set(intId, extId);
|
|
712
|
+
this._extToInt.set(extId, intId);
|
|
713
|
+
}
|
|
714
|
+
this._nextExtId = nextExtId;
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
_validateV3Mappings(intToExt, nextExtId, count) {
|
|
718
|
+
if (intToExt.size !== count) {
|
|
719
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
720
|
+
'Import failed: envelope mapping count mismatch');
|
|
721
|
+
}
|
|
722
|
+
if (!Number.isInteger(nextExtId) || nextExtId < count) {
|
|
723
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
724
|
+
'Import failed: envelope nextExtId is invalid');
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
const extIds = new Set();
|
|
728
|
+
let maxExtId = -1;
|
|
729
|
+
for (const [intId, extId] of intToExt) {
|
|
730
|
+
if (!Number.isInteger(intId) || intId < 0 || intId >= count) {
|
|
731
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
732
|
+
'Import failed: envelope mapping contains invalid internal ID');
|
|
733
|
+
}
|
|
734
|
+
if (!Number.isInteger(extId) || extId < 0) {
|
|
735
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
736
|
+
'Import failed: envelope mapping contains invalid external ID');
|
|
737
|
+
}
|
|
738
|
+
if (extIds.has(extId)) {
|
|
739
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
740
|
+
'Import failed: envelope mapping contains duplicates');
|
|
741
|
+
}
|
|
742
|
+
extIds.add(extId);
|
|
743
|
+
if (extId > maxExtId) maxExtId = extId;
|
|
744
|
+
}
|
|
745
|
+
if (nextExtId <= maxExtId) {
|
|
746
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
747
|
+
'Import failed: envelope nextExtId is invalid');
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
_setIdentityMappings(count) {
|
|
752
|
+
this._clearMappings();
|
|
753
|
+
for (let i = 0; i < count; i++) {
|
|
754
|
+
this._intToExt.set(i, i);
|
|
755
|
+
this._extToInt.set(i, i);
|
|
756
|
+
}
|
|
757
|
+
this._nextExtId = count;
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
_validateRawSnapshotMetadata(metadata) {
|
|
761
|
+
// Unsupported major format versions must be rejected, not parsed as
|
|
762
|
+
// the newest known layout (Search Artifact Contract §6). Current
|
|
763
|
+
// writers emit v2 (quantized) / v1 (float); a future v3 would
|
|
764
|
+
// otherwise be silently misparsed as v2 by the open-ended version
|
|
765
|
+
// reads in the native deserializers.
|
|
766
|
+
const maxVersion = metadata.quantized ? 2 : 1;
|
|
767
|
+
if (!Number.isInteger(metadata.version) || metadata.version > maxVersion) {
|
|
768
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
769
|
+
`Import failed: unsupported snapshot format version ${metadata.version}`,
|
|
770
|
+
{ version: metadata.version, maxSupported: maxVersion });
|
|
771
|
+
}
|
|
772
|
+
if (!Number.isInteger(metadata.count) || metadata.count < 0) {
|
|
773
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
774
|
+
'Import failed: snapshot count is invalid');
|
|
775
|
+
}
|
|
776
|
+
if (metadata.count > this._maxElements) {
|
|
777
|
+
throw pancakeError(
|
|
778
|
+
PANCAKE_ERROR_CODES.SNAPSHOT_CAPACITY_EXCEEDED,
|
|
779
|
+
`Import failed: snapshot count ${metadata.count} exceeds maxElements ${this._maxElements}`,
|
|
780
|
+
{ count: metadata.count, maxElements: this._maxElements }
|
|
781
|
+
);
|
|
782
|
+
}
|
|
783
|
+
if (metadata.dim !== this._dim) {
|
|
784
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
785
|
+
`Import failed: dim mismatch (exported ${metadata.dim}, expected ${this._dim})`,
|
|
786
|
+
{ field: 'dim', exported: metadata.dim, expected: this._dim });
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
const exportedL2 = metadata.metric === 0;
|
|
790
|
+
if (exportedL2 !== this._isL2) {
|
|
791
|
+
const got = exportedL2 ? 'l2' : 'cosine';
|
|
792
|
+
const want = this._isL2 ? 'l2' : 'cosine';
|
|
793
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
794
|
+
`Import failed: metric mismatch (exported ${got}, expected ${want})`,
|
|
795
|
+
{ field: 'metric', exported: got, expected: want });
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
if (metadata.quantized !== this._quantized) {
|
|
799
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
800
|
+
'Import failed: quantized mismatch',
|
|
801
|
+
{ field: 'quantized', exported: metadata.quantized, expected: this._quantized });
|
|
802
|
+
}
|
|
803
|
+
if (metadata.M !== this._M) {
|
|
804
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
805
|
+
`Import failed: M mismatch (exported ${metadata.M}, expected ${this._M})`,
|
|
806
|
+
{ field: 'M', exported: metadata.M, expected: this._M });
|
|
807
|
+
}
|
|
808
|
+
if (metadata.efConstruction !== this._efConstruction) {
|
|
809
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
810
|
+
`Import failed: efConstruction mismatch (exported ${metadata.efConstruction}, expected ${this._efConstruction})`,
|
|
811
|
+
{ field: 'efConstruction', exported: metadata.efConstruction, expected: this._efConstruction });
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
_readResults(n) {
|
|
816
|
+
const dv = new DataView(this._e.HEAPU8.buffer);
|
|
817
|
+
const results = new Array(n);
|
|
818
|
+
for (let i = 0; i < n; i++) {
|
|
819
|
+
const lo = dv.getUint32(this._idPtr + i * 8, true);
|
|
820
|
+
const hi = dv.getUint32(this._idPtr + i * 8 + 4, true);
|
|
821
|
+
const intId = hi * 0x100000000 + lo;
|
|
822
|
+
let distance = this._e.HEAPF32[(this._distPtr >> 2) + i];
|
|
823
|
+
if (this._isL2) distance = Math.sqrt(distance);
|
|
824
|
+
const extId = this._intToExt.get(intId);
|
|
825
|
+
if (extId === undefined) {
|
|
826
|
+
throw pancakeError(
|
|
827
|
+
PANCAKE_ERROR_CODES.INTERNAL_INVARIANT,
|
|
828
|
+
`Search invariant failed: missing external ID mapping for internal ID ${intId}`,
|
|
829
|
+
{ internalId: intId }
|
|
830
|
+
);
|
|
831
|
+
}
|
|
832
|
+
results[i] = { id: extId, distance };
|
|
833
|
+
}
|
|
834
|
+
return results;
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
function createPancakeApi(loadEngineImpl) {
|
|
839
|
+
function isVectorInput(value) {
|
|
840
|
+
return value instanceof Float32Array || Array.isArray(value);
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
function extractVectorRecord(item) {
|
|
844
|
+
if (isVectorInput(item)) {
|
|
845
|
+
return { vector: item, hasSourceId: false, sourceId: undefined };
|
|
846
|
+
}
|
|
847
|
+
if (!item || typeof item !== 'object' || !('vector' in item)) {
|
|
848
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
849
|
+
'fromVectors() requires vectors or { vector, id? } records');
|
|
850
|
+
}
|
|
851
|
+
const sourceId = item.id;
|
|
852
|
+
const hasSourceId = Object.prototype.hasOwnProperty.call(item, 'id');
|
|
853
|
+
return { vector: item.vector, hasSourceId, sourceId };
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
async function create(opts) {
|
|
857
|
+
if (!opts || !opts.dim) throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
858
|
+
'opts.dim is required', { argument: 'dim' });
|
|
859
|
+
if (!Number.isInteger(opts.dim) || opts.dim <= 0) {
|
|
860
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
861
|
+
'opts.dim must be a positive integer', { argument: 'dim', value: opts.dim });
|
|
862
|
+
}
|
|
863
|
+
if ('compressed' in opts) {
|
|
864
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, 'opts.compressed has been removed');
|
|
865
|
+
}
|
|
866
|
+
if ('varianceSample' in opts) {
|
|
867
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, 'opts.varianceSample has been removed');
|
|
868
|
+
}
|
|
869
|
+
if (opts.metric !== undefined && opts.metric !== 'cosine' && opts.metric !== 'l2') {
|
|
870
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
871
|
+
"opts.metric must be 'cosine' or 'l2'", { argument: 'metric', value: opts.metric });
|
|
872
|
+
}
|
|
873
|
+
// Upper bound matches the engine ABI: pancake_init takes max_elem as a
|
|
874
|
+
// C int, so anything above 2^31-1 would truncate or go negative.
|
|
875
|
+
if (opts.maxElements !== undefined && (!Number.isInteger(opts.maxElements) || opts.maxElements <= 0 || opts.maxElements > 0x7fffffff)) {
|
|
876
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
877
|
+
'opts.maxElements must be an integer between 1 and 2147483647', { argument: 'maxElements', value: opts.maxElements });
|
|
878
|
+
}
|
|
879
|
+
if (opts.M !== undefined && (!Number.isInteger(opts.M) || opts.M <= 1 || opts.M > 128)) {
|
|
880
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
881
|
+
'opts.M must be an integer between 2 and 128', { argument: 'M', value: opts.M });
|
|
882
|
+
}
|
|
883
|
+
if (opts.efConstruction !== undefined && (!Number.isInteger(opts.efConstruction) || opts.efConstruction <= 0 || opts.efConstruction > 4096)) {
|
|
884
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
885
|
+
'opts.efConstruction must be an integer between 1 and 4096',
|
|
886
|
+
{ argument: 'efConstruction', value: opts.efConstruction });
|
|
887
|
+
}
|
|
888
|
+
if (opts.efSearch !== undefined) validateEfSearch(opts.efSearch, 'opts.efSearch');
|
|
889
|
+
if (opts.seed !== undefined && (!Number.isInteger(opts.seed) || opts.seed <= 0 || opts.seed > 0x7fffffff)) {
|
|
890
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
891
|
+
'opts.seed must be an integer between 1 and 2147483647',
|
|
892
|
+
{ argument: 'seed', value: opts.seed });
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
const dim = opts.dim;
|
|
896
|
+
const metric = (opts.metric === 'l2') ? 0 : 1;
|
|
897
|
+
const maxElements = opts.maxElements ?? DEFAULT_MAX_ELEMENTS;
|
|
898
|
+
const isQuantized = opts.quantized !== undefined ? !!opts.quantized : true;
|
|
899
|
+
const quantized = isQuantized ? 1 : 0;
|
|
900
|
+
const M = opts.M ?? DEFAULT_M;
|
|
901
|
+
const efConstruction = opts.efConstruction ?? DEFAULT_EF_CONSTRUCTION;
|
|
902
|
+
const efSearch = opts.efSearch ?? DEFAULT_EF_SEARCH;
|
|
903
|
+
const seed = opts.seed ?? DEFAULT_SEED;
|
|
904
|
+
|
|
905
|
+
// Capacity guard: the backend allocates its arena eagerly at
|
|
906
|
+
// construction, and a request the wasm32 heap cannot satisfy throws
|
|
907
|
+
// std::bad_alloc inside pancake_init — reject it here with the
|
|
908
|
+
// estimate instead. Per-element cost mirrors the constructor
|
|
909
|
+
// allocations (uint8_float_hnsw.hpp / float_hnsw.hpp): quantized rows
|
|
910
|
+
// cost dim bytes plus 16*M edge bytes plus ~39 bytes of bookkeeping;
|
|
911
|
+
// float rows cost 4*dim plus 8*M plus ~23.
|
|
912
|
+
const bytesPerElement = isQuantized ? dim + 16 * M + 39 : 4 * dim + 8 * M + 23;
|
|
913
|
+
const estimatedBytes = maxElements * bytesPerElement;
|
|
914
|
+
if (estimatedBytes > MAX_INDEX_ARENA_BYTES) {
|
|
915
|
+
throw pancakeError(PANCAKE_ERROR_CODES.WASM_ALLOCATION_FAILED,
|
|
916
|
+
`Index configuration needs ~${Math.ceil(estimatedBytes / (1024 * 1024))} MiB `
|
|
917
|
+
+ `(${isQuantized ? 'quantized' : 'float32'} backend: maxElements ${maxElements} × ~${bytesPerElement} B/element), `
|
|
918
|
+
+ `above the ${Math.floor(MAX_INDEX_ARENA_BYTES / (1024 * 1024))} MiB wasm32 index budget; `
|
|
919
|
+
+ 'lower maxElements or dim, or shard across indexes',
|
|
920
|
+
{ estimatedBytes, budgetBytes: MAX_INDEX_ARENA_BYTES, dim, maxElements, M, quantized: isQuantized });
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
const resolvedOpts = {
|
|
924
|
+
...opts,
|
|
925
|
+
dim,
|
|
926
|
+
maxElements,
|
|
927
|
+
metric: metric === 0 ? 'l2' : 'cosine',
|
|
928
|
+
quantized: isQuantized,
|
|
929
|
+
M,
|
|
930
|
+
efConstruction,
|
|
931
|
+
efSearch,
|
|
932
|
+
seed,
|
|
933
|
+
};
|
|
934
|
+
|
|
935
|
+
const e = await loadEngineImpl();
|
|
936
|
+
|
|
937
|
+
const vecPtr = e._emsc_malloc(dim * 4);
|
|
938
|
+
const initialBufferCapacity = 16;
|
|
939
|
+
const idPtr = e._emsc_malloc(initialBufferCapacity * 8);
|
|
940
|
+
const distPtr = e._emsc_malloc(initialBufferCapacity * 4);
|
|
941
|
+
|
|
942
|
+
if (!vecPtr || !idPtr || !distPtr) {
|
|
943
|
+
if (vecPtr) e._emsc_free(vecPtr);
|
|
944
|
+
if (idPtr) e._emsc_free(idPtr);
|
|
945
|
+
if (distPtr) e._emsc_free(distPtr);
|
|
946
|
+
throw pancakeError(PANCAKE_ERROR_CODES.WASM_ALLOCATION_FAILED, 'WASM malloc failed');
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
const handle = e._pancake_init(dim, maxElements, quantized, metric, M, efConstruction, efSearch, seed);
|
|
950
|
+
|
|
951
|
+
// The engine returns uint32_t INVALID_HANDLE (0xFFFFFFFF), which the
|
|
952
|
+
// Emscripten i32 ABI delivers to JS as -1.
|
|
953
|
+
if ((handle >>> 0) === 0xFFFFFFFF) {
|
|
954
|
+
e._emsc_free(vecPtr);
|
|
955
|
+
e._emsc_free(idPtr);
|
|
956
|
+
e._emsc_free(distPtr);
|
|
957
|
+
throw pancakeError(PANCAKE_ERROR_CODES.WASM_ALLOCATION_FAILED, 'Backend init failed');
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
return new PancakeIndex(e, resolvedOpts, handle, vecPtr, idPtr, distPtr, initialBufferCapacity);
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
async function fromVectors(items, opts = {}) {
|
|
964
|
+
if (!Array.isArray(items)) {
|
|
965
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, 'fromVectors() requires an array');
|
|
966
|
+
}
|
|
967
|
+
if (items.length === 0) {
|
|
968
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
969
|
+
'fromVectors() requires at least one vector');
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
const vectors = new Array(items.length);
|
|
973
|
+
const sourceIds = new Array(items.length);
|
|
974
|
+
const hasSourceIds = new Array(items.length);
|
|
975
|
+
|
|
976
|
+
let inferredDim = null;
|
|
977
|
+
for (let i = 0; i < items.length; i++) {
|
|
978
|
+
const { vector, hasSourceId, sourceId } = extractVectorRecord(items[i]);
|
|
979
|
+
if (!isVectorInput(vector)) {
|
|
980
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_VECTOR,
|
|
981
|
+
`fromVectors() expected a vector at index ${i}`, { index: i });
|
|
982
|
+
}
|
|
983
|
+
const dim = vector.length;
|
|
984
|
+
if (!Number.isInteger(dim) || dim <= 0) {
|
|
985
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_VECTOR,
|
|
986
|
+
`fromVectors() expected a non-empty vector at index ${i}`, { index: i });
|
|
987
|
+
}
|
|
988
|
+
if (inferredDim === null) {
|
|
989
|
+
inferredDim = dim;
|
|
990
|
+
} else if (dim !== inferredDim) {
|
|
991
|
+
throw pancakeError(PANCAKE_ERROR_CODES.DIMENSION_MISMATCH,
|
|
992
|
+
`fromVectors() found mixed vector dimensions (${inferredDim} and ${dim})`,
|
|
993
|
+
{ expected: inferredDim, actual: dim, index: i });
|
|
994
|
+
}
|
|
995
|
+
vectors[i] = vector;
|
|
996
|
+
sourceIds[i] = sourceId;
|
|
997
|
+
hasSourceIds[i] = hasSourceId;
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
if (opts.dim !== undefined && opts.dim !== inferredDim) {
|
|
1001
|
+
throw pancakeError(PANCAKE_ERROR_CODES.DIMENSION_MISMATCH,
|
|
1002
|
+
`fromVectors() dim mismatch (inferred ${inferredDim}, got opts.dim=${opts.dim})`,
|
|
1003
|
+
{ expected: inferredDim, actual: opts.dim });
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
const createOpts = {
|
|
1007
|
+
...opts,
|
|
1008
|
+
dim: opts.dim ?? inferredDim,
|
|
1009
|
+
maxElements: opts.maxElements ?? items.length,
|
|
1010
|
+
};
|
|
1011
|
+
|
|
1012
|
+
const index = await create(createOpts);
|
|
1013
|
+
try {
|
|
1014
|
+
const ids = index.addBatch(vectors);
|
|
1015
|
+
const idMap = new Map();
|
|
1016
|
+
for (let i = 0; i < ids.length; i++) {
|
|
1017
|
+
if (hasSourceIds[i]) {
|
|
1018
|
+
idMap.set(ids[i], sourceIds[i]);
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
return { index, ids, idMap };
|
|
1022
|
+
} catch (err) {
|
|
1023
|
+
try {
|
|
1024
|
+
index.dispose();
|
|
1025
|
+
} catch {}
|
|
1026
|
+
throw err;
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
async function withIndex(options, fn) {
|
|
1031
|
+
if (typeof fn !== 'function') {
|
|
1032
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
1033
|
+
'withIndex() requires a callback function', { argument: 'fn' });
|
|
1034
|
+
}
|
|
1035
|
+
const index = await create(options);
|
|
1036
|
+
try {
|
|
1037
|
+
return await fn(index);
|
|
1038
|
+
} finally {
|
|
1039
|
+
index.dispose();
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
function inspectSnapshot(data) {
|
|
1044
|
+
return inspectSnapshotMetadata(data);
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
async function restore(snapshot, overrides = {}) {
|
|
1048
|
+
if (!overrides || typeof overrides !== 'object' || Array.isArray(overrides)) {
|
|
1049
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
1050
|
+
'restore() overrides must be an object', { argument: 'overrides' });
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
const metadata = inspectSnapshotMetadata(snapshot);
|
|
1054
|
+
const fixedConfig = {
|
|
1055
|
+
dim: metadata.dim,
|
|
1056
|
+
metric: metadata.metric,
|
|
1057
|
+
quantized: metadata.quantized,
|
|
1058
|
+
M: metadata.M,
|
|
1059
|
+
efConstruction: metadata.efConstruction,
|
|
1060
|
+
};
|
|
1061
|
+
|
|
1062
|
+
if (metadata.format === 'raw') {
|
|
1063
|
+
for (const field of Object.keys(fixedConfig)) {
|
|
1064
|
+
if (overrides[field] === undefined) {
|
|
1065
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
1066
|
+
`restore() requires overrides.${field} for legacy raw snapshots`,
|
|
1067
|
+
{ argument: field, format: 'raw' });
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
for (const [field, expected] of Object.entries(fixedConfig)) {
|
|
1073
|
+
if (overrides[field] !== undefined && overrides[field] !== expected) {
|
|
1074
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_CONFIG_MISMATCH,
|
|
1075
|
+
`restore() ${field} override does not match the snapshot`,
|
|
1076
|
+
{ field, exported: expected, override: overrides[field] });
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
const options = {
|
|
1081
|
+
...overrides,
|
|
1082
|
+
...fixedConfig,
|
|
1083
|
+
maxElements: overrides.maxElements ?? Math.max(1, metadata.count),
|
|
1084
|
+
efSearch: overrides.efSearch ?? DEFAULT_EF_SEARCH,
|
|
1085
|
+
seed: overrides.seed ?? DEFAULT_SEED,
|
|
1086
|
+
};
|
|
1087
|
+
const index = await create(options);
|
|
1088
|
+
try {
|
|
1089
|
+
index.import(snapshot);
|
|
1090
|
+
return index;
|
|
1091
|
+
} catch (error) {
|
|
1092
|
+
try {
|
|
1093
|
+
index.dispose();
|
|
1094
|
+
} catch {}
|
|
1095
|
+
throw error;
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
return {
|
|
1100
|
+
create,
|
|
1101
|
+
fromVectors,
|
|
1102
|
+
withIndex,
|
|
1103
|
+
restore,
|
|
1104
|
+
inspectSnapshot,
|
|
1105
|
+
PancakeError,
|
|
1106
|
+
PANCAKE_ERROR_CODES,
|
|
1107
|
+
};
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1110
|
+
function parseRawSnapshotMetadata(bytes) {
|
|
1111
|
+
if (!bytes || bytes.length < 12) return null;
|
|
1112
|
+
|
|
1113
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
1114
|
+
const magic = view.getUint32(0, true);
|
|
1115
|
+
|
|
1116
|
+
if (magic === FLOAT_HNSW_MAGIC_V1 || magic === UINT8_HNSW_MAGIC_V1) {
|
|
1117
|
+
if (bytes.length < 40) return null;
|
|
1118
|
+
return {
|
|
1119
|
+
version: view.getUint32(8, true),
|
|
1120
|
+
dim: view.getUint32(4, true),
|
|
1121
|
+
count: view.getUint32(12, true),
|
|
1122
|
+
M: view.getUint32(24, true),
|
|
1123
|
+
M0: view.getUint32(28, true),
|
|
1124
|
+
metric: view.getUint32(32, true),
|
|
1125
|
+
efConstruction: view.getUint32(36, true),
|
|
1126
|
+
quantized: magic === UINT8_HNSW_MAGIC_V1,
|
|
1127
|
+
};
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
if (magic === FLOAT_HNSW_MAGIC_V0 || magic === UINT8_HNSW_MAGIC_V0) {
|
|
1131
|
+
if (bytes.length < 36) return null;
|
|
1132
|
+
return {
|
|
1133
|
+
version: 0,
|
|
1134
|
+
dim: view.getUint32(4, true),
|
|
1135
|
+
count: view.getUint32(8, true),
|
|
1136
|
+
M: view.getUint32(20, true),
|
|
1137
|
+
M0: view.getUint32(24, true),
|
|
1138
|
+
metric: view.getUint32(28, true),
|
|
1139
|
+
efConstruction: 200,
|
|
1140
|
+
quantized: magic === UINT8_HNSW_MAGIC_V0,
|
|
1141
|
+
};
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
return null;
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
function snapshotBytes(data, methodName) {
|
|
1148
|
+
if (data instanceof Uint8Array) return data;
|
|
1149
|
+
if (data instanceof ArrayBuffer) return new Uint8Array(data);
|
|
1150
|
+
if (typeof SharedArrayBuffer !== 'undefined' && data instanceof SharedArrayBuffer) {
|
|
1151
|
+
return new Uint8Array(data);
|
|
1152
|
+
}
|
|
1153
|
+
if (ArrayBuffer.isView(data)) {
|
|
1154
|
+
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
1155
|
+
}
|
|
1156
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT,
|
|
1157
|
+
`${methodName}() requires snapshot bytes`, { argument: 'snapshot' });
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
function inspectSnapshotMetadata(data) {
|
|
1161
|
+
const bytes = snapshotBytes(data, 'inspectSnapshot');
|
|
1162
|
+
if (bytes.byteLength < 4) {
|
|
1163
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot is too small to inspect');
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
1167
|
+
let rawBytes = bytes;
|
|
1168
|
+
let format = 'raw';
|
|
1169
|
+
let envelopeVersion = null;
|
|
1170
|
+
let envelopeDim = null;
|
|
1171
|
+
let envelopeMetric = null;
|
|
1172
|
+
let envelopeQuantized = null;
|
|
1173
|
+
let nextId = null;
|
|
1174
|
+
|
|
1175
|
+
if (view.getUint32(0, true) === PANCAKE_MAGIC) {
|
|
1176
|
+
format = 'pancake';
|
|
1177
|
+
if (bytes.byteLength < 8) {
|
|
1178
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot envelope is truncated');
|
|
1179
|
+
}
|
|
1180
|
+
envelopeVersion = view.getUint32(4, true);
|
|
1181
|
+
let rawOffset;
|
|
1182
|
+
if (envelopeVersion === 1) {
|
|
1183
|
+
if (bytes.byteLength < V1_ENVELOPE_HEADER_SIZE) {
|
|
1184
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot v1 envelope is truncated');
|
|
1185
|
+
}
|
|
1186
|
+
envelopeDim = view.getUint32(8, true);
|
|
1187
|
+
if (view.getUint32(12, true) !== envelopeDim) {
|
|
1188
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1189
|
+
'DCT/PCA snapshots are no longer supported');
|
|
1190
|
+
}
|
|
1191
|
+
envelopeMetric = view.getUint32(16, true);
|
|
1192
|
+
envelopeQuantized = view.getUint32(20, true);
|
|
1193
|
+
rawOffset = V1_ENVELOPE_HEADER_SIZE;
|
|
1194
|
+
} else if (envelopeVersion === 2) {
|
|
1195
|
+
if (bytes.byteLength < V2_ENVELOPE_HEADER_SIZE) {
|
|
1196
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot v2 envelope is truncated');
|
|
1197
|
+
}
|
|
1198
|
+
envelopeDim = view.getUint32(8, true);
|
|
1199
|
+
envelopeMetric = view.getUint32(12, true);
|
|
1200
|
+
envelopeQuantized = view.getUint32(16, true);
|
|
1201
|
+
rawOffset = V2_ENVELOPE_HEADER_SIZE;
|
|
1202
|
+
} else if (envelopeVersion === ENVELOPE_VERSION) {
|
|
1203
|
+
if (bytes.byteLength < V3_ENVELOPE_HEADER_SIZE) {
|
|
1204
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot v3 envelope is truncated');
|
|
1205
|
+
}
|
|
1206
|
+
envelopeDim = view.getUint32(8, true);
|
|
1207
|
+
envelopeMetric = view.getUint32(12, true);
|
|
1208
|
+
envelopeQuantized = view.getUint32(16, true);
|
|
1209
|
+
nextId = view.getUint32(20, true);
|
|
1210
|
+
const mappingCount = view.getUint32(24, true);
|
|
1211
|
+
const rawSize = view.getUint32(28, true);
|
|
1212
|
+
rawOffset = V3_ENVELOPE_HEADER_SIZE + mappingCount * MAPPING_ENTRY_SIZE;
|
|
1213
|
+
if (!Number.isSafeInteger(rawOffset) || rawOffset > bytes.byteLength ||
|
|
1214
|
+
rawSize > bytes.byteLength - rawOffset) {
|
|
1215
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1216
|
+
'Snapshot v3 envelope payload is truncated');
|
|
1217
|
+
}
|
|
1218
|
+
rawBytes = bytes.subarray(rawOffset, rawOffset + rawSize);
|
|
1219
|
+
} else {
|
|
1220
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1221
|
+
`Unsupported snapshot envelope version ${envelopeVersion}`, { version: envelopeVersion });
|
|
1222
|
+
}
|
|
1223
|
+
if (envelopeVersion !== ENVELOPE_VERSION) rawBytes = bytes.subarray(rawOffset);
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
const raw = parseRawSnapshotMetadata(rawBytes);
|
|
1227
|
+
if (raw === null) {
|
|
1228
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1229
|
+
'Unsupported or truncated raw snapshot format');
|
|
1230
|
+
}
|
|
1231
|
+
if (raw.dim <= 0 || raw.metric > 1 || raw.M < 2 || raw.M > 128 ||
|
|
1232
|
+
raw.M0 !== raw.M * 2 || raw.efConstruction < 1 || raw.efConstruction > 4096) {
|
|
1233
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1234
|
+
'Snapshot header contains invalid index configuration');
|
|
1235
|
+
}
|
|
1236
|
+
if (format === 'pancake' && (envelopeDim !== raw.dim || envelopeMetric !== raw.metric ||
|
|
1237
|
+
!!envelopeQuantized !== raw.quantized)) {
|
|
1238
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID,
|
|
1239
|
+
'Snapshot envelope does not match its raw index header');
|
|
1240
|
+
}
|
|
1241
|
+
if (nextId === null) nextId = raw.count;
|
|
1242
|
+
if (nextId < raw.count) {
|
|
1243
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID, 'Snapshot nextId is invalid');
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
return Object.freeze({
|
|
1247
|
+
format,
|
|
1248
|
+
version: format === 'pancake' ? envelopeVersion : raw.version,
|
|
1249
|
+
dim: raw.dim,
|
|
1250
|
+
count: raw.count,
|
|
1251
|
+
metric: raw.metric === 0 ? 'l2' : 'cosine',
|
|
1252
|
+
quantized: raw.quantized,
|
|
1253
|
+
M: raw.M,
|
|
1254
|
+
efConstruction: raw.efConstruction,
|
|
1255
|
+
nextId,
|
|
1256
|
+
});
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
module.exports = createPancakeApi;
|
|
1260
|
+
module.exports.default = createPancakeApi;
|
|
1261
|
+
module.exports.PancakeError = PancakeError;
|
|
1262
|
+
module.exports.PANCAKE_ERROR_CODES = PANCAKE_ERROR_CODES;
|
|
1263
|
+
|
|
1264
|
+
if (typeof Symbol.dispose === 'symbol') {
|
|
1265
|
+
PancakeIndex.prototype[Symbol.dispose] = function disposeWithSymbol() {
|
|
1266
|
+
this.dispose();
|
|
1267
|
+
};
|
|
1268
|
+
}
|