pikelet-wasm 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +868 -0
- package/LICENSE +191 -0
- package/README.md +590 -0
- package/complete/builder.d.ts +82 -0
- package/complete/builder.mjs +527 -0
- package/complete/encoder-kernels/encoder.mjs +2 -0
- package/complete/encoder-kernels/encoder.node.mjs +2 -0
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/format.mjs +8 -0
- package/complete/index.d.ts +289 -0
- package/complete/index.mjs +1124 -0
- package/complete/inline-transformer.mjs +205 -0
- package/complete/lexical.mjs +264 -0
- package/complete/retrieval-abstention.mjs +119 -0
- package/complete/sources.mjs +186 -0
- package/complete/student-abstention.mjs +102 -0
- package/complete/student-embedder.mjs +231 -0
- package/complete/wordpiece.mjs +68 -0
- package/dist/engine.js +2 -0
- package/dist/engine.scalar.js +2 -0
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +168 -0
- package/pancake-artifact-common.js +561 -0
- package/pancake-artifact-range.js +799 -0
- package/pancake-artifact-sketch.js +1040 -0
- package/pancake-artifact.d.ts +109 -0
- package/pancake-artifact.js +31 -0
- package/pancake-core.js +1268 -0
- package/pancake-errors.js +40 -0
- package/pancake-loader.js +66 -0
- package/pancake.d.ts +691 -0
- package/pancake.js +254 -0
- package/pancake.node.mjs +259 -0
- package/pancake.web.d.ts +50 -0
- package/pancake.web.mjs +88 -0
- package/pancake.workerd.mjs +72 -0
package/pancake.d.ts
ADDED
|
@@ -0,0 +1,691 @@
|
|
|
1
|
+
export type Metric = 'cosine' | 'l2';
|
|
2
|
+
|
|
3
|
+
export const PANCAKE_ERROR_CODES: Readonly<{
|
|
4
|
+
INVALID_ARGUMENT: 'INVALID_ARGUMENT';
|
|
5
|
+
DIMENSION_MISMATCH: 'DIMENSION_MISMATCH';
|
|
6
|
+
INVALID_VECTOR: 'INVALID_VECTOR';
|
|
7
|
+
INDEX_FULL: 'INDEX_FULL';
|
|
8
|
+
INDEX_DISPOSED: 'INDEX_DISPOSED';
|
|
9
|
+
COMPACTION_REQUIRED: 'COMPACTION_REQUIRED';
|
|
10
|
+
SNAPSHOT_INVALID: 'SNAPSHOT_INVALID';
|
|
11
|
+
SNAPSHOT_CONFIG_MISMATCH: 'SNAPSHOT_CONFIG_MISMATCH';
|
|
12
|
+
SNAPSHOT_CAPACITY_EXCEEDED: 'SNAPSHOT_CAPACITY_EXCEEDED';
|
|
13
|
+
WASM_LOAD_FAILED: 'WASM_LOAD_FAILED';
|
|
14
|
+
WASM_ALLOCATION_FAILED: 'WASM_ALLOCATION_FAILED';
|
|
15
|
+
FILE_IO_FAILED: 'FILE_IO_FAILED';
|
|
16
|
+
PARSE_FAILED: 'PARSE_FAILED';
|
|
17
|
+
INTERNAL_INVARIANT: 'INTERNAL_INVARIANT';
|
|
18
|
+
INDEX_LIMIT: 'INDEX_LIMIT';
|
|
19
|
+
}>;
|
|
20
|
+
|
|
21
|
+
export type PancakeErrorCode = typeof PANCAKE_ERROR_CODES[keyof typeof PANCAKE_ERROR_CODES];
|
|
22
|
+
|
|
23
|
+
export class PancakeError extends Error {
|
|
24
|
+
readonly code: PancakeErrorCode;
|
|
25
|
+
readonly details?: Readonly<Record<string, unknown>>;
|
|
26
|
+
readonly cause?: unknown;
|
|
27
|
+
constructor(code: PancakeErrorCode, message: string, details?: Readonly<Record<string, unknown>>, cause?: unknown);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export type VectorInput = Float32Array | readonly number[];
|
|
31
|
+
|
|
32
|
+
export interface VectorRecord<Id = unknown> {
|
|
33
|
+
id?: Id;
|
|
34
|
+
vector: VectorInput;
|
|
35
|
+
[key: string]: unknown;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface CreateOptions {
|
|
39
|
+
/** Input vector dimension (required). */
|
|
40
|
+
dim: number;
|
|
41
|
+
/**
|
|
42
|
+
* Maximum index capacity (default: 100000). The backend allocates its
|
|
43
|
+
* arena eagerly at ~(dim + 16*M + 39) bytes per element quantized, or
|
|
44
|
+
* ~(4*dim + 8*M + 23) float32; configurations estimated above the 1.5 GiB
|
|
45
|
+
* wasm32 index budget are rejected with WASM_ALLOCATION_FAILED at
|
|
46
|
+
* create() rather than aborting inside the engine. Hard ABI ceiling:
|
|
47
|
+
* 2147483647.
|
|
48
|
+
*/
|
|
49
|
+
maxElements?: number;
|
|
50
|
+
/** Distance metric (default: 'cosine'). */
|
|
51
|
+
metric?: Metric;
|
|
52
|
+
/** Use uint8 quantized storage (default: true). */
|
|
53
|
+
quantized?: boolean;
|
|
54
|
+
/** HNSW graph connectivity (default: 12). Valid range: 2-128. */
|
|
55
|
+
M?: number;
|
|
56
|
+
/** Build-time search breadth (default: 75). Valid range: 1-4096. */
|
|
57
|
+
efConstruction?: number;
|
|
58
|
+
/** Query-time search breadth (default: 100). Valid range: 1-4096. */
|
|
59
|
+
efSearch?: number;
|
|
60
|
+
/** RNG seed for deterministic HNSW level assignment (default: 108). */
|
|
61
|
+
seed?: number;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export interface SearchOptions {
|
|
65
|
+
/** Override query-time search breadth for this call. Valid range: 1-4096. */
|
|
66
|
+
efSearch?: number;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export interface FromVectorsResult<Id = unknown> {
|
|
70
|
+
/** Created and populated index. Caller owns dispose(). */
|
|
71
|
+
index: PancakeIndex;
|
|
72
|
+
/** Stable Pancake IDs returned in insertion order. */
|
|
73
|
+
ids: number[];
|
|
74
|
+
/** Mapping from Pancake IDs back to caller-provided source IDs. */
|
|
75
|
+
idMap: Map<number, Id>;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export interface JsonFileOptions extends Omit<CreateOptions, 'dim'> {
|
|
79
|
+
/** Input vector dimension. Default: inferred from the first vector in the file. */
|
|
80
|
+
dim?: number;
|
|
81
|
+
/** Force JSON or JSONL parsing. Default: inferred from file extension. */
|
|
82
|
+
format?: 'json' | 'jsonl';
|
|
83
|
+
/** Record field holding the vector. Default: 'vector'. */
|
|
84
|
+
vectorKey?: string;
|
|
85
|
+
/** Record field holding the caller's source ID. Default: 'id'. */
|
|
86
|
+
idKey?: string;
|
|
87
|
+
/** Maximum JSON/JSONL file size in bytes. Default: 64 MiB. */
|
|
88
|
+
maxFileBytes?: number;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export interface RestoreOptions extends Partial<CreateOptions> {}
|
|
92
|
+
|
|
93
|
+
export type SnapshotFormat = 'pancake' | 'raw';
|
|
94
|
+
|
|
95
|
+
export interface SnapshotInspection {
|
|
96
|
+
readonly format: SnapshotFormat;
|
|
97
|
+
readonly version: number;
|
|
98
|
+
readonly dim: number;
|
|
99
|
+
readonly count: number;
|
|
100
|
+
readonly metric: Metric;
|
|
101
|
+
readonly quantized: boolean;
|
|
102
|
+
readonly M: number;
|
|
103
|
+
readonly efConstruction: number;
|
|
104
|
+
readonly nextId: number;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export interface SnapshotFileOptions extends RestoreOptions {
|
|
108
|
+
/** Maximum snapshot file size in bytes. Default: 512 MiB. */
|
|
109
|
+
maxFileBytes?: number;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export interface SearchResult {
|
|
113
|
+
/** Vector ID as returned by add(). */
|
|
114
|
+
id: number;
|
|
115
|
+
/** Distance from query (lower is more similar). */
|
|
116
|
+
distance: number;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export interface RangeReadSource {
|
|
120
|
+
/** Return exactly length bytes starting at offset. */
|
|
121
|
+
read(offset: number, length: number): Uint8Array | ArrayBuffer | Promise<Uint8Array | ArrayBuffer>;
|
|
122
|
+
/** Optional close hook for file-backed sources. */
|
|
123
|
+
close?(): void | Promise<void>;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export interface RangeArtifactSearchOptions {
|
|
127
|
+
/** Query-time search breadth. Default: 100. */
|
|
128
|
+
efSearch?: number;
|
|
129
|
+
/** Merge adjacent v1 record runs separated by this many skipped records. Default: 0. */
|
|
130
|
+
gap?: number;
|
|
131
|
+
/** Candidates expanded per fetch round (deeper rounds trade requests for latency). Default: 1. */
|
|
132
|
+
expansionBatch?: number;
|
|
133
|
+
/** Concurrent range reads per fetch round. Default: 1. */
|
|
134
|
+
parallelism?: number;
|
|
135
|
+
/** Overrides `parallelism` for range reads when both are set. */
|
|
136
|
+
rangeParallelism?: number;
|
|
137
|
+
/**
|
|
138
|
+
* Split coalesced record runs at this many bytes (record-aligned, floor
|
|
139
|
+
* one record) so no single range read grows unbounded. Default: 16 MiB.
|
|
140
|
+
*/
|
|
141
|
+
maxRangeBytes?: number;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** A decoded graph record, as returned by {@link PancakeRangeArtifact.readNode}. */
|
|
145
|
+
export interface RangeArtifactNode {
|
|
146
|
+
readonly id: number;
|
|
147
|
+
readonly level: number;
|
|
148
|
+
readonly base: Uint32Array;
|
|
149
|
+
readonly upper: readonly Uint32Array[];
|
|
150
|
+
readonly qdata: Uint8Array;
|
|
151
|
+
readonly scale: number;
|
|
152
|
+
readonly offset: number;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export interface RangeArtifactStats {
|
|
156
|
+
readonly rangeRequests: number;
|
|
157
|
+
readonly rangeBytes: number;
|
|
158
|
+
readonly rangeNodesDecoded: number;
|
|
159
|
+
readonly cachedNodes: number;
|
|
160
|
+
readonly lazyCacheBytes: number;
|
|
161
|
+
readonly routerResident: {
|
|
162
|
+
readonly records: number;
|
|
163
|
+
readonly bytes: number;
|
|
164
|
+
};
|
|
165
|
+
/**
|
|
166
|
+
* Which segments have passed their v3 whole-segment digests. Always false
|
|
167
|
+
* for pre-v3 artifacts (which carry no digests) and under verify:false;
|
|
168
|
+
* base becomes true only after a successful verifyBaseSegment().
|
|
169
|
+
*/
|
|
170
|
+
readonly segmentVerified: {
|
|
171
|
+
readonly idMap: boolean;
|
|
172
|
+
readonly router: boolean;
|
|
173
|
+
readonly base: boolean;
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export interface RangeArtifactRound {
|
|
178
|
+
readonly ids: number;
|
|
179
|
+
readonly requests: number;
|
|
180
|
+
readonly bytes: number;
|
|
181
|
+
readonly rangeBytes: readonly number[];
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
export interface RangeArtifactSearchResult {
|
|
185
|
+
readonly results: SearchResult[];
|
|
186
|
+
readonly rounds: RangeArtifactRound[];
|
|
187
|
+
readonly stats: RangeArtifactStats;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
export interface RangeArtifactOpenOptions {
|
|
191
|
+
/** Eagerly load the v2 router segment. Default: true. */
|
|
192
|
+
loadRouter?: boolean;
|
|
193
|
+
/**
|
|
194
|
+
* Verify the id map and router segments against the header digests at open
|
|
195
|
+
* (v3 artifacts only; pre-v3 artifacts carry no digests and open
|
|
196
|
+
* unverified either way). Default: true.
|
|
197
|
+
*/
|
|
198
|
+
verify?: boolean;
|
|
199
|
+
/**
|
|
200
|
+
* Budget for any single open-path read (id map, router segment), honored
|
|
201
|
+
* strictly: a budget below what the artifact's resident segments need
|
|
202
|
+
* fails the open with SNAPSHOT_INVALID. Default: 256 MiB; Infinity defers
|
|
203
|
+
* to the 2 GiB absolute backstop. Other non-positive or non-numeric
|
|
204
|
+
* values throw INVALID_ARGUMENT.
|
|
205
|
+
*/
|
|
206
|
+
maxReadBytes?: number;
|
|
207
|
+
/**
|
|
208
|
+
* Byte budget for the lazily-fetched record cache (LRU eviction; the
|
|
209
|
+
* resident router segment is not counted). Default: 64 MiB. Pass Infinity
|
|
210
|
+
* for the pre-0.3 unbounded behavior. Budgets below 64 records are raised
|
|
211
|
+
* to that floor; other non-positive or non-numeric values throw
|
|
212
|
+
* INVALID_ARGUMENT.
|
|
213
|
+
*/
|
|
214
|
+
maxCacheBytes?: number;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
export interface RangeArtifactBuildOptions {
|
|
218
|
+
/** Deterministic base-layer layout. Default: 'rcm'. */
|
|
219
|
+
layout?: 'rcm' | 'identity';
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
export interface RangeArtifactBuildManifest {
|
|
223
|
+
readonly format: 'pancake-range-artifact';
|
|
224
|
+
readonly formatVersion: number;
|
|
225
|
+
/** Hex SHA-256 whole-segment digests stamped into the v3 header. */
|
|
226
|
+
readonly integrity: Readonly<{
|
|
227
|
+
idMapSha256: string;
|
|
228
|
+
routerSha256: string;
|
|
229
|
+
baseSha256: string;
|
|
230
|
+
}>;
|
|
231
|
+
readonly file: string;
|
|
232
|
+
readonly sizeBytes: number;
|
|
233
|
+
readonly kind: string;
|
|
234
|
+
readonly metric: Metric;
|
|
235
|
+
readonly layout: Readonly<Record<string, unknown>>;
|
|
236
|
+
readonly graph: Readonly<Record<string, number>>;
|
|
237
|
+
readonly addressing: Readonly<Record<string, number>>;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* Node-only file-backed range source. This is NOT a named runtime export of
|
|
242
|
+
* any entrypoint: reach the constructor via the Node API object
|
|
243
|
+
* (`Pancake.NodeFileRangeSource`) or the `pikelet-wasm/artifact` subpath.
|
|
244
|
+
*/
|
|
245
|
+
export interface NodeFileRangeSource extends RangeReadSource {
|
|
246
|
+
readonly filePath: string;
|
|
247
|
+
readonly size: number;
|
|
248
|
+
read(offset: number, length: number): Promise<Uint8Array>;
|
|
249
|
+
close(): Promise<void>;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
export interface NodeFileRangeSourceConstructor {
|
|
253
|
+
new (filePath: string): NodeFileRangeSource;
|
|
254
|
+
readonly prototype: NodeFileRangeSource;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* A range-readable Search Artifact reader. The class is NOT a named runtime
|
|
259
|
+
* export of any entrypoint: reach it via `Pancake.RangeArtifact` or the
|
|
260
|
+
* `pikelet-wasm/artifact` subpath.
|
|
261
|
+
*/
|
|
262
|
+
export interface PancakeRangeArtifact {
|
|
263
|
+
readonly version: number;
|
|
264
|
+
readonly kind: number;
|
|
265
|
+
readonly dim: number;
|
|
266
|
+
readonly count: number;
|
|
267
|
+
readonly entryPoint: number;
|
|
268
|
+
readonly maxLevel: number;
|
|
269
|
+
readonly M: number;
|
|
270
|
+
readonly M0: number;
|
|
271
|
+
readonly metric: number;
|
|
272
|
+
readonly recordBytes: number;
|
|
273
|
+
readonly routerCount: number;
|
|
274
|
+
readonly baseCount: number;
|
|
275
|
+
readonly routerResident: { readonly records: number; readonly bytes: number };
|
|
276
|
+
search(query: VectorInput, k: number, options?: RangeArtifactSearchOptions): Promise<RangeArtifactSearchResult>;
|
|
277
|
+
stats(): RangeArtifactStats;
|
|
278
|
+
/**
|
|
279
|
+
* Verify the lazily-read base segment against the v3 header digest (not a
|
|
280
|
+
* query-path operation). Streams the segment in bounded chunks where a
|
|
281
|
+
* streaming hash exists (Node); other runtimes fall back to one read
|
|
282
|
+
* bounded by maxReadBytes. Resolves true on match; throws
|
|
283
|
+
* SNAPSHOT_INVALID on mismatch, INVALID_ARGUMENT for pre-v3 artifacts,
|
|
284
|
+
* which carry no digests.
|
|
285
|
+
*/
|
|
286
|
+
verifyBaseSegment(options?: { chunkBytes?: number }): Promise<true>;
|
|
287
|
+
resetStats(): void;
|
|
288
|
+
clearCache(options?: { reloadRouter?: boolean }): Promise<{ records: number; bytes: number }>;
|
|
289
|
+
/** Fetch (and cache) the given node ids in coalesced ranges. Returns the number of range reads issued. */
|
|
290
|
+
prefetch(ids: readonly number[], options?: { gap?: number; parallelism?: number; maxRangeBytes?: number }): Promise<number>;
|
|
291
|
+
/** Read a single node, fetching it if not cached. */
|
|
292
|
+
readNode(id: number): Promise<RangeArtifactNode>;
|
|
293
|
+
/** Byte offset of a node's record within the artifact. */
|
|
294
|
+
recordAddressForId(id: number): number;
|
|
295
|
+
/** Mark the current position in the range-read log (see rangesSince). */
|
|
296
|
+
markRanges(): number;
|
|
297
|
+
/** Ranges read since a markRanges() mark, as [start, end) byte offsets. */
|
|
298
|
+
rangesSince(mark: number): Array<[number, number]>;
|
|
299
|
+
close(): Promise<void>;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
export interface PancakeRangeArtifactConstructor {
|
|
303
|
+
open(source: RangeReadSource, options?: RangeArtifactOpenOptions): Promise<PancakeRangeArtifact>;
|
|
304
|
+
/** Node entrypoints only. */
|
|
305
|
+
openFile(filePath: string, options?: RangeArtifactOpenOptions): Promise<PancakeRangeArtifact>;
|
|
306
|
+
readonly prototype: PancakeRangeArtifact;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/** Resident sketch tier: the coarse staged-boot tier or the full tier. */
|
|
310
|
+
export type SketchTier = 'micro' | 'full';
|
|
311
|
+
|
|
312
|
+
export interface SketchStageEvent {
|
|
313
|
+
readonly tier: SketchTier;
|
|
314
|
+
readonly residentBytes: number;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
export interface SketchArtifactOpenOptions {
|
|
318
|
+
/** Verify the resident prefix hash when a crypto backend exists. Default: true. */
|
|
319
|
+
verify?: boolean;
|
|
320
|
+
/**
|
|
321
|
+
* Budget for any single open-path read (resident prefix, staged tiers),
|
|
322
|
+
* honored strictly: a budget below what the artifact's resident tier
|
|
323
|
+
* needs fails the open with SNAPSHOT_INVALID. Default: 256 MiB; Infinity
|
|
324
|
+
* defers to the 2 GiB absolute backstop. Other non-positive or
|
|
325
|
+
* non-numeric values throw INVALID_ARGUMENT.
|
|
326
|
+
*/
|
|
327
|
+
maxReadBytes?: number;
|
|
328
|
+
/**
|
|
329
|
+
* Byte budget for the fetched-row cache (LRU eviction; the resident sketch
|
|
330
|
+
* tier is not counted). Default: 64 MiB. Pass Infinity to disable eviction.
|
|
331
|
+
* Budgets below 256 rows are raised to that floor; other non-positive or
|
|
332
|
+
* non-numeric values throw INVALID_ARGUMENT.
|
|
333
|
+
*/
|
|
334
|
+
maxCacheBytes?: number;
|
|
335
|
+
/**
|
|
336
|
+
* Staged boot: when the artifact carries a micro tier, become searchable
|
|
337
|
+
* after loading only the stage-1 prefix and swap to the full tier in the
|
|
338
|
+
* background (see {@link PancakeSketchArtifact.fullyResident}). Ignored for
|
|
339
|
+
* artifacts without a micro tier. Default: false.
|
|
340
|
+
*/
|
|
341
|
+
staged?: boolean;
|
|
342
|
+
/** Called on each residency transition of a staged open (micro, then full). */
|
|
343
|
+
onStage?: (event: SketchStageEvent) => void;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
export interface SketchScanner {
|
|
347
|
+
/**
|
|
348
|
+
* Return the top-C row ids for a pooled query, ascending by sketch
|
|
349
|
+
* distance. `c` is silently clamped to {@link maxRerank}.
|
|
350
|
+
*/
|
|
351
|
+
scan(pooledQuery: Float32Array | readonly number[], c: number): number[];
|
|
352
|
+
/** Free the scanner's WASM heap allocations. */
|
|
353
|
+
dispose(): void;
|
|
354
|
+
readonly maxRerank: number;
|
|
355
|
+
/** The artifact metric this scanner scores (0 = l2, 1 = cosine). */
|
|
356
|
+
readonly metric: number;
|
|
357
|
+
/** Pooled dimensionality of the tier this scanner was built for. */
|
|
358
|
+
readonly sketchDims: number;
|
|
359
|
+
/** The resident tier this scanner scores. */
|
|
360
|
+
readonly tier: SketchTier;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
export interface SketchScannerOptions {
|
|
364
|
+
/** Upper bound on rerank depth the scanner will return. Default: 1024. */
|
|
365
|
+
maxRerank?: number;
|
|
366
|
+
/**
|
|
367
|
+
* Which resident tier to build the scanner over. 'micro' requires an
|
|
368
|
+
* artifact with a micro tier. Default: 'full'.
|
|
369
|
+
*/
|
|
370
|
+
tier?: SketchTier;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
export interface SketchArtifactSearchOptions {
|
|
374
|
+
/**
|
|
375
|
+
* Rerank depth (top-C sketch candidates fetched for exact scoring). A
|
|
376
|
+
* scanner-backed search additionally clamps this to the scanner's
|
|
377
|
+
* maxRerank.
|
|
378
|
+
*/
|
|
379
|
+
rerank?: number;
|
|
380
|
+
/** Byte gap for coalescing row fetches. Default: 2048. */
|
|
381
|
+
gap?: number;
|
|
382
|
+
/** Concurrent range reads per fetch round. Default: 8. */
|
|
383
|
+
parallelism?: number;
|
|
384
|
+
/**
|
|
385
|
+
* Row ids that join the exact rerank alongside the sketch scan's
|
|
386
|
+
* selection (e.g. a lexical index's matches in a hybrid query). Scored
|
|
387
|
+
* by true distance like any candidate. Ids must be integers in
|
|
388
|
+
* [0, count).
|
|
389
|
+
*/
|
|
390
|
+
extraCandidates?: readonly number[];
|
|
391
|
+
/**
|
|
392
|
+
* Return every reranked candidate in distance order instead of the top
|
|
393
|
+
* k. The candidates are already fetched and exactly scored; this
|
|
394
|
+
* changes only the output slice. Hybrid callers use it so rank fusion
|
|
395
|
+
* sees the true distance of every lexical candidate.
|
|
396
|
+
*/
|
|
397
|
+
fullRerankOutput?: boolean;
|
|
398
|
+
/**
|
|
399
|
+
* Split coalesced row runs at this many bytes (row-aligned, floor one
|
|
400
|
+
* row) so no single range read grows unbounded. Default: 16 MiB.
|
|
401
|
+
*/
|
|
402
|
+
maxRangeBytes?: number;
|
|
403
|
+
/**
|
|
404
|
+
* External top-C scanner (e.g. from createSketchScanner). Must implement
|
|
405
|
+
* the artifact's metric: cosine artifacts require `metric === 1`, and a
|
|
406
|
+
* declared metric that disagrees with the artifact throws
|
|
407
|
+
* INVALID_ARGUMENT. A scanner whose tier does not match the artifact's
|
|
408
|
+
* current resident tier is bypassed, not misused.
|
|
409
|
+
*/
|
|
410
|
+
scanner?: SketchScanner | { metric?: number; scan(pooledQuery: Float32Array, c: number): number[] };
|
|
411
|
+
/**
|
|
412
|
+
* While serving from the micro tier, multiply the candidate pool by this
|
|
413
|
+
* factor to compensate for the coarser sketches. Default: 4.
|
|
414
|
+
*/
|
|
415
|
+
microBoost?: number;
|
|
416
|
+
/**
|
|
417
|
+
* Scanner built over the micro tier (createSketchScanner with
|
|
418
|
+
* `tier: 'micro'`); engaged only while the micro tier is the active
|
|
419
|
+
* resident tier and `sketchDims` matches it.
|
|
420
|
+
*/
|
|
421
|
+
microScanner?: SketchScanner | { sketchDims: number; metric?: number; scan(pooledQuery: Float32Array, c: number): number[] };
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
export interface SketchArtifactSearchResult {
|
|
425
|
+
readonly results: SearchResult[];
|
|
426
|
+
/** Number of candidates fetched for exact rerank. */
|
|
427
|
+
readonly rerank: number;
|
|
428
|
+
/** The resident tier that served this search. */
|
|
429
|
+
readonly tier: SketchTier;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
export interface SketchArtifactStats {
|
|
433
|
+
readonly rangeRequests: number;
|
|
434
|
+
readonly rangeBytes: number;
|
|
435
|
+
readonly cachedRows: number;
|
|
436
|
+
readonly cacheBytes: number;
|
|
437
|
+
readonly residentBytes: number;
|
|
438
|
+
readonly residentVerified: boolean;
|
|
439
|
+
/** True after a successful verifyVectors(). */
|
|
440
|
+
readonly vectorsVerified: boolean;
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
export interface SketchArtifactBuildOptions {
|
|
444
|
+
/** Pooled sketch dimensionality; must divide dim. Default: dim / 2. */
|
|
445
|
+
sketchDims?: number;
|
|
446
|
+
/** Bits per sketch dimension. Default: 4. */
|
|
447
|
+
sketchBits?: 4 | 8;
|
|
448
|
+
/** Producer-recommended rerank depth recorded in the header. */
|
|
449
|
+
recommendedRerank?: number;
|
|
450
|
+
/**
|
|
451
|
+
* Add a staged-boot micro tier at this pooled dimensionality; must divide
|
|
452
|
+
* sketchDims and be smaller. Default: no micro tier.
|
|
453
|
+
*/
|
|
454
|
+
microDims?: number;
|
|
455
|
+
/** Bits per micro-tier dimension. Default: 4. */
|
|
456
|
+
microBits?: 4 | 8;
|
|
457
|
+
/**
|
|
458
|
+
* Per-row integrity (format version 2, the default): the vectors region is
|
|
459
|
+
* written as blocks of rowsPerBlock rows, each preceded by a digest page
|
|
460
|
+
* of truncated per-row SHA-256s, anchored by a page-hash table in the
|
|
461
|
+
* resident prefix. Pass false to emit a version-1 artifact.
|
|
462
|
+
*/
|
|
463
|
+
rowIntegrity?: boolean;
|
|
464
|
+
/** Rows per digest block (format 2). Default: 16. */
|
|
465
|
+
rowsPerBlock?: number;
|
|
466
|
+
/** Truncated per-row digest size in bytes, 8..32 (format 2). Default: 16. */
|
|
467
|
+
rowDigestBytes?: number;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
export interface SketchArtifactBuildManifest {
|
|
471
|
+
readonly format: 'pancake-sketch-artifact';
|
|
472
|
+
readonly formatVersion: number;
|
|
473
|
+
/** Output path; absent on the bytes-in/bytes-out builder. */
|
|
474
|
+
readonly file?: string;
|
|
475
|
+
readonly sizeBytes: number;
|
|
476
|
+
readonly metric: Metric;
|
|
477
|
+
readonly graph: Readonly<Record<string, number>>;
|
|
478
|
+
readonly sketch: Readonly<Record<string, number>>;
|
|
479
|
+
/** Micro-tier geometry, or null when built without one. */
|
|
480
|
+
readonly micro: Readonly<{
|
|
481
|
+
microDims: number;
|
|
482
|
+
microBits: number;
|
|
483
|
+
microPool: number;
|
|
484
|
+
stage1Bytes: number;
|
|
485
|
+
}> | null;
|
|
486
|
+
readonly addressing: Readonly<Record<string, number>>;
|
|
487
|
+
readonly recommendedRerank: number;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/**
|
|
491
|
+
* A sketch Search Artifact reader. The class is NOT a named runtime export of
|
|
492
|
+
* any entrypoint: reach it via `Pancake.SketchArtifact` or the
|
|
493
|
+
* `pikelet-wasm/artifact` subpath.
|
|
494
|
+
*/
|
|
495
|
+
export interface PancakeSketchArtifact {
|
|
496
|
+
readonly metric: number;
|
|
497
|
+
readonly dim: number;
|
|
498
|
+
readonly count: number;
|
|
499
|
+
readonly sketchDims: number;
|
|
500
|
+
readonly sketchBits: number;
|
|
501
|
+
/** Micro-tier pooled dimensionality; 0 when the artifact has none. */
|
|
502
|
+
readonly microDims: number;
|
|
503
|
+
/** Micro-tier bits per dimension; 0 when the artifact has none. */
|
|
504
|
+
readonly microBits: number;
|
|
505
|
+
/** The currently active resident tier ('micro' only during a staged boot). */
|
|
506
|
+
readonly tier: SketchTier;
|
|
507
|
+
/**
|
|
508
|
+
* Resolves with this artifact once the full tier is resident and verified.
|
|
509
|
+
* Already resolved for non-staged opens; rejects if the background stage-2
|
|
510
|
+
* load fails.
|
|
511
|
+
*/
|
|
512
|
+
readonly fullyResident: Promise<PancakeSketchArtifact>;
|
|
513
|
+
readonly recommendedRerank: number;
|
|
514
|
+
readonly residentBytes: number;
|
|
515
|
+
readonly residentVerified: boolean;
|
|
516
|
+
/** True after a successful verifyVectors(). */
|
|
517
|
+
readonly vectorsVerified: boolean;
|
|
518
|
+
search(query: VectorInput, k: number, options?: SketchArtifactSearchOptions): Promise<SketchArtifactSearchResult>;
|
|
519
|
+
stats(): SketchArtifactStats;
|
|
520
|
+
/** Drop all cached rows; subsequent searches fetch cold. Never affects result semantics. */
|
|
521
|
+
clearCache(): void;
|
|
522
|
+
/**
|
|
523
|
+
* Verify the lazy vectors segment against the header's whole-segment hash
|
|
524
|
+
* (not a query-path operation). Streams the segment in bounded chunks
|
|
525
|
+
* where a streaming hash exists (Node); other runtimes fall back to one
|
|
526
|
+
* read bounded by maxReadBytes. Resolves true on match; throws
|
|
527
|
+
* SNAPSHOT_INVALID on mismatch or when verification is impossible (no
|
|
528
|
+
* crypto backend).
|
|
529
|
+
*/
|
|
530
|
+
verifyVectors(options?: { chunkBytes?: number }): Promise<true>;
|
|
531
|
+
close(): Promise<void>;
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
export interface PancakeSketchArtifactConstructor {
|
|
535
|
+
open(source: RangeReadSource, options?: SketchArtifactOpenOptions): Promise<PancakeSketchArtifact>;
|
|
536
|
+
/** Node entrypoints only. */
|
|
537
|
+
openFile(filePath: string, options?: SketchArtifactOpenOptions): Promise<PancakeSketchArtifact>;
|
|
538
|
+
readonly prototype: PancakeSketchArtifact;
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
export interface MemoryUsage {
|
|
542
|
+
/** Backend-owned graph and vector storage estimate. */
|
|
543
|
+
readonly logicalIndexBytes: number;
|
|
544
|
+
/** Current byte length of this index's isolated WebAssembly heap. */
|
|
545
|
+
readonly wasmHeapBytes: number;
|
|
546
|
+
/** Raw backend snapshot buffer retained by the most recent export(). */
|
|
547
|
+
readonly snapshotBufferBytes: number;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
export interface ResolvedConfig {
|
|
551
|
+
readonly dim: number;
|
|
552
|
+
readonly maxElements: number;
|
|
553
|
+
readonly metric: Metric;
|
|
554
|
+
readonly quantized: boolean;
|
|
555
|
+
readonly M: number;
|
|
556
|
+
readonly efConstruction: number;
|
|
557
|
+
readonly efSearch: number;
|
|
558
|
+
readonly seed: number;
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
export interface PancakeIndex {
|
|
562
|
+
/** Insert a single vector. Returns its stable external ID. */
|
|
563
|
+
add(vector: VectorInput): number;
|
|
564
|
+
/** Insert multiple vectors. Returns stable external IDs in insertion order. */
|
|
565
|
+
addBatch(vectors: readonly VectorInput[]): number[];
|
|
566
|
+
/** Find the k nearest neighbors of a query vector. */
|
|
567
|
+
search(query: VectorInput, k: number, options?: SearchOptions): SearchResult[];
|
|
568
|
+
/** Find the k nearest neighbors restricted to IDs in allowedIds. */
|
|
569
|
+
searchFiltered(query: VectorInput, k: number, allowedIds: Set<number>, options?: SearchOptions): SearchResult[];
|
|
570
|
+
/** Change the default query-time search breadth for future searches. */
|
|
571
|
+
setEfSearch(efSearch: number): void;
|
|
572
|
+
/** Soft-delete a live vector. Returns false for unknown or already-deleted IDs. */
|
|
573
|
+
delete(id: number): boolean;
|
|
574
|
+
/** Whether an ID currently refers to a live vector. */
|
|
575
|
+
has(id: number): boolean;
|
|
576
|
+
/** Whether an ID is awaiting reclamation by compact(). */
|
|
577
|
+
isDeleted(id: number): boolean;
|
|
578
|
+
/** Rebuild the graph without soft-deleted entries. */
|
|
579
|
+
compact(): void;
|
|
580
|
+
/** Serialize index state. Throws if ghostCount > 0; call compact() first after deletions. */
|
|
581
|
+
export(): Uint8Array;
|
|
582
|
+
/**
|
|
583
|
+
* Restore index state from a previous export().
|
|
584
|
+
* Validation failures leave the target index unchanged.
|
|
585
|
+
*/
|
|
586
|
+
import(data: Uint8Array | ArrayBufferLike): void;
|
|
587
|
+
/** Free WASM buffers. The instance is unusable after this. */
|
|
588
|
+
dispose(): void;
|
|
589
|
+
/** Dispose support for JavaScript `using` declarations. */
|
|
590
|
+
[Symbol.dispose](): void;
|
|
591
|
+
/** Vectors stored by the backend, including soft-deleted entries until compact(). */
|
|
592
|
+
readonly count: number;
|
|
593
|
+
/** Live vectors available to search. */
|
|
594
|
+
readonly liveCount: number;
|
|
595
|
+
/** Preferred name for soft-deleted vectors awaiting compaction. */
|
|
596
|
+
readonly deletedCount: number;
|
|
597
|
+
/** Preferred name for the soft-deleted fraction. */
|
|
598
|
+
readonly deletedRatio: number;
|
|
599
|
+
/** Fixed insertion capacity. */
|
|
600
|
+
readonly capacity: number;
|
|
601
|
+
/** Remaining insertion slots; soft deletion does not increase this value. */
|
|
602
|
+
readonly remainingCapacity: number;
|
|
603
|
+
/** Soft-deleted vectors awaiting compaction. */
|
|
604
|
+
readonly ghostCount: number;
|
|
605
|
+
/** Ratio reported by the backend; after soft deletes this behaves as ghostCount / count. */
|
|
606
|
+
readonly ghostRatio: number;
|
|
607
|
+
/** Estimated index memory in bytes. */
|
|
608
|
+
readonly memory: number;
|
|
609
|
+
/** Logical index, isolated WASM heap, and retained snapshot-buffer memory. */
|
|
610
|
+
readonly memoryUsage: MemoryUsage;
|
|
611
|
+
/** Fully resolved construction and current query-policy values. */
|
|
612
|
+
readonly config: ResolvedConfig;
|
|
613
|
+
/** Input vector dimension. */
|
|
614
|
+
readonly dim: number;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* The portable API exposed by every entrypoint (Node, browser, Workers).
|
|
619
|
+
* The browser/Workers entrypoints expose exactly this surface; the Node
|
|
620
|
+
* entrypoints add file helpers (see {@link NodePancakeApi}).
|
|
621
|
+
*/
|
|
622
|
+
export interface PancakeApi {
|
|
623
|
+
readonly PancakeError: typeof PancakeError;
|
|
624
|
+
readonly PANCAKE_ERROR_CODES: typeof PANCAKE_ERROR_CODES;
|
|
625
|
+
readonly RangeArtifact: PancakeRangeArtifactConstructor;
|
|
626
|
+
readonly SketchArtifact: PancakeSketchArtifactConstructor;
|
|
627
|
+
/** Build a WASM-backed SIMD scanner for a sketch artifact's resident tier. */
|
|
628
|
+
createSketchScanner(artifact: PancakeSketchArtifact, options?: SketchScannerOptions): Promise<SketchScanner>;
|
|
629
|
+
/** Create a new Pancake index using the runtime-specific packaged entrypoint. */
|
|
630
|
+
create(opts: CreateOptions): Promise<PancakeIndex>;
|
|
631
|
+
/** Restore an envelope snapshot, inferring its construction config. */
|
|
632
|
+
restore(snapshot: Uint8Array | ArrayBufferLike, overrides?: RestoreOptions): Promise<PancakeIndex>;
|
|
633
|
+
/** Validate and inspect snapshot headers without creating a WASM index. */
|
|
634
|
+
inspectSnapshot(snapshot: Uint8Array | ArrayBufferLike): SnapshotInspection;
|
|
635
|
+
/** Create an index, run a callback, and always dispose the index afterward. */
|
|
636
|
+
withIndex<T>(opts: CreateOptions, fn: (index: PancakeIndex) => T | Promise<T>): Promise<T>;
|
|
637
|
+
/** Create and populate an index from raw vectors. Infers dim and maxElements by default. */
|
|
638
|
+
fromVectors(vectors: readonly VectorInput[], opts?: Omit<CreateOptions, 'dim'> & Partial<Pick<CreateOptions, 'dim'>>): Promise<FromVectorsResult<never>>;
|
|
639
|
+
/** Create and populate an index from { id, vector } records. */
|
|
640
|
+
fromVectors<Id = unknown>(records: readonly VectorRecord<Id>[], opts?: Omit<CreateOptions, 'dim'> & Partial<Pick<CreateOptions, 'dim'>>): Promise<FromVectorsResult<Id>>;
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
/**
|
|
644
|
+
* The Node.js API: the portable surface plus filesystem helpers. These helpers
|
|
645
|
+
* exist only on the Node entrypoints (`pikelet-wasm`, `pikelet-wasm/node`); the
|
|
646
|
+
* browser/Workers entrypoints do not expose them, and importing those returns
|
|
647
|
+
* the narrower {@link PancakeApi}.
|
|
648
|
+
*/
|
|
649
|
+
export interface NodePancakeApi extends PancakeApi {
|
|
650
|
+
readonly NodeFileRangeSource: NodeFileRangeSourceConstructor;
|
|
651
|
+
/**
|
|
652
|
+
* Build a range-readable Search Artifact from a uint8 Pancake snapshot.
|
|
653
|
+
* @deprecated The `.pancake-range` profile is deprecated
|
|
654
|
+
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.pancake-sketch`
|
|
655
|
+
* artifact instead. Readers stay supported for existing artifacts.
|
|
656
|
+
*/
|
|
657
|
+
buildRangeArtifact(snapshot: Uint8Array | ArrayBufferLike, outPath: string, opts?: RangeArtifactBuildOptions): RangeArtifactBuildManifest;
|
|
658
|
+
/**
|
|
659
|
+
* Build a range-readable Search Artifact from a uint8 Pancake snapshot file.
|
|
660
|
+
* @deprecated The `.pancake-range` profile is deprecated
|
|
661
|
+
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.pancake-sketch`
|
|
662
|
+
* artifact instead. Readers stay supported for existing artifacts.
|
|
663
|
+
*/
|
|
664
|
+
buildRangeArtifactFile(snapshotPath: string, outPath: string, opts?: RangeArtifactBuildOptions): RangeArtifactBuildManifest;
|
|
665
|
+
/** Open a range-readable Search Artifact from a local file. */
|
|
666
|
+
openRangeArtifactFile(filePath: string, opts?: RangeArtifactOpenOptions): Promise<PancakeRangeArtifact>;
|
|
667
|
+
/** Build a sketch Search Artifact from a uint8 Pancake snapshot. */
|
|
668
|
+
buildSketchArtifact(snapshot: Uint8Array | ArrayBufferLike, outPath: string, opts?: SketchArtifactBuildOptions): SketchArtifactBuildManifest;
|
|
669
|
+
/** Build a sketch Search Artifact from a uint8 Pancake snapshot file. */
|
|
670
|
+
buildSketchArtifactFile(snapshotPath: string, outPath: string, opts?: SketchArtifactBuildOptions): SketchArtifactBuildManifest;
|
|
671
|
+
/**
|
|
672
|
+
* Bytes-in/bytes-out sketch build for producers assembling segments in
|
|
673
|
+
* memory (e.g. a complete-artifact compiler); output identical to
|
|
674
|
+
* buildSketchArtifact, no filesystem involved.
|
|
675
|
+
*/
|
|
676
|
+
buildSketchArtifactBytes(snapshot: Uint8Array | ArrayBufferLike, opts?: SketchArtifactBuildOptions): { bytes: Uint8Array; manifest: SketchArtifactBuildManifest };
|
|
677
|
+
/** Open a sketch Search Artifact from a local file. */
|
|
678
|
+
openSketchArtifactFile(filePath: string, opts?: SketchArtifactOpenOptions): Promise<PancakeSketchArtifact>;
|
|
679
|
+
/** Load vectors from a JSON/JSONL file and build an index. */
|
|
680
|
+
loadJsonFile<Id = unknown>(filePath: string, opts?: JsonFileOptions): Promise<FromVectorsResult<Id>>;
|
|
681
|
+
/**
|
|
682
|
+
* Load a previously exported Pancake snapshot from disk. Envelope snapshots
|
|
683
|
+
* carry their own config, so `opts` is only required for raw engine
|
|
684
|
+
* snapshots (which need the full create config).
|
|
685
|
+
*/
|
|
686
|
+
loadSnapshotFile(filePath: string, opts?: SnapshotFileOptions): Promise<PancakeIndex>;
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
declare const Pancake: NodePancakeApi;
|
|
690
|
+
|
|
691
|
+
export default Pancake;
|