pikelet-wasm 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -3
- package/NOTICE +5 -0
- package/README.md +397 -454
- package/complete/builder.d.ts +4 -4
- package/complete/builder.mjs +8 -8
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/index.d.ts +11 -7
- package/complete/index.mjs +161 -112
- package/complete/inline-transformer.mjs +22 -9
- package/complete/lexical.mjs +11 -11
- package/complete/retrieval-abstention.mjs +18 -6
- package/complete/sources.mjs +22 -1
- package/dist/engine.js +1 -1
- package/dist/engine.scalar.js +1 -1
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +58 -51
- package/{pancake-artifact-common.js → pikelet-artifact-common.js} +42 -42
- package/{pancake-artifact-range.js → pikelet-artifact-range.js} +36 -36
- package/{pancake-artifact-sketch.js → pikelet-artifact-sketch.js} +76 -66
- package/{pancake-artifact.d.ts → pikelet-artifact.d.ts} +11 -11
- package/{pancake-artifact.js → pikelet-artifact.js} +10 -10
- package/{pancake-core.js → pikelet-core.js} +136 -136
- package/{pancake-errors.js → pikelet-errors.js} +9 -8
- package/{pancake.d.ts → pikelet.d.ts} +55 -55
- package/{pancake.js → pikelet.js} +44 -44
- package/{pancake.node.mjs → pikelet.node.mjs} +46 -46
- package/{pancake.web.d.ts → pikelet.web.d.ts} +13 -13
- package/{pancake.web.mjs → pikelet.web.mjs} +20 -20
- package/{pancake.workerd.mjs → pikelet.workerd.mjs} +18 -18
- /package/{pancake-loader.js → pikelet-loader.js} +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
const
|
|
3
|
+
const PIKELET_ERROR_CODES = Object.freeze({
|
|
4
4
|
INVALID_ARGUMENT: 'INVALID_ARGUMENT',
|
|
5
5
|
DIMENSION_MISMATCH: 'DIMENSION_MISMATCH',
|
|
6
6
|
INVALID_VECTOR: 'INVALID_VECTOR',
|
|
@@ -17,24 +17,25 @@ const PANCAKE_ERROR_CODES = Object.freeze({
|
|
|
17
17
|
INTERNAL_INVARIANT: 'INTERNAL_INVARIANT',
|
|
18
18
|
// Reserved for the deferred shared-runtime API.
|
|
19
19
|
INDEX_LIMIT: 'INDEX_LIMIT',
|
|
20
|
+
ARTIFACT_CHANGED: 'ARTIFACT_CHANGED',
|
|
20
21
|
});
|
|
21
22
|
|
|
22
|
-
class
|
|
23
|
+
class PikeletError extends Error {
|
|
23
24
|
constructor(code, message, details, cause) {
|
|
24
25
|
super(message);
|
|
25
|
-
this.name = '
|
|
26
|
+
this.name = 'PikeletError';
|
|
26
27
|
this.code = code;
|
|
27
28
|
if (details !== undefined) this.details = details;
|
|
28
29
|
if (cause !== undefined) this.cause = cause;
|
|
29
30
|
}
|
|
30
31
|
}
|
|
31
32
|
|
|
32
|
-
function
|
|
33
|
-
return new
|
|
33
|
+
function pikeletError(code, message, details, cause) {
|
|
34
|
+
return new PikeletError(code, message, details, cause);
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
module.exports = {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
38
|
+
PikeletError,
|
|
39
|
+
PIKELET_ERROR_CODES,
|
|
40
|
+
pikeletError,
|
|
40
41
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export type Metric = 'cosine' | 'l2';
|
|
2
2
|
|
|
3
|
-
export const
|
|
3
|
+
export const PIKELET_ERROR_CODES: Readonly<{
|
|
4
4
|
INVALID_ARGUMENT: 'INVALID_ARGUMENT';
|
|
5
5
|
DIMENSION_MISMATCH: 'DIMENSION_MISMATCH';
|
|
6
6
|
INVALID_VECTOR: 'INVALID_VECTOR';
|
|
@@ -18,13 +18,13 @@ export const PANCAKE_ERROR_CODES: Readonly<{
|
|
|
18
18
|
INDEX_LIMIT: 'INDEX_LIMIT';
|
|
19
19
|
}>;
|
|
20
20
|
|
|
21
|
-
export type
|
|
21
|
+
export type PikeletErrorCode = typeof PIKELET_ERROR_CODES[keyof typeof PIKELET_ERROR_CODES];
|
|
22
22
|
|
|
23
|
-
export class
|
|
24
|
-
readonly code:
|
|
23
|
+
export class PikeletError extends Error {
|
|
24
|
+
readonly code: PikeletErrorCode;
|
|
25
25
|
readonly details?: Readonly<Record<string, unknown>>;
|
|
26
26
|
readonly cause?: unknown;
|
|
27
|
-
constructor(code:
|
|
27
|
+
constructor(code: PikeletErrorCode, message: string, details?: Readonly<Record<string, unknown>>, cause?: unknown);
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
export type VectorInput = Float32Array | readonly number[];
|
|
@@ -68,10 +68,10 @@ export interface SearchOptions {
|
|
|
68
68
|
|
|
69
69
|
export interface FromVectorsResult<Id = unknown> {
|
|
70
70
|
/** Created and populated index. Caller owns dispose(). */
|
|
71
|
-
index:
|
|
72
|
-
/** Stable
|
|
71
|
+
index: PikeletIndex;
|
|
72
|
+
/** Stable Pikelet IDs returned in insertion order. */
|
|
73
73
|
ids: number[];
|
|
74
|
-
/** Mapping from
|
|
74
|
+
/** Mapping from Pikelet IDs back to caller-provided source IDs. */
|
|
75
75
|
idMap: Map<number, Id>;
|
|
76
76
|
}
|
|
77
77
|
|
|
@@ -90,7 +90,7 @@ export interface JsonFileOptions extends Omit<CreateOptions, 'dim'> {
|
|
|
90
90
|
|
|
91
91
|
export interface RestoreOptions extends Partial<CreateOptions> {}
|
|
92
92
|
|
|
93
|
-
export type SnapshotFormat = '
|
|
93
|
+
export type SnapshotFormat = 'pikelet' | 'raw';
|
|
94
94
|
|
|
95
95
|
export interface SnapshotInspection {
|
|
96
96
|
readonly format: SnapshotFormat;
|
|
@@ -141,7 +141,7 @@ export interface RangeArtifactSearchOptions {
|
|
|
141
141
|
maxRangeBytes?: number;
|
|
142
142
|
}
|
|
143
143
|
|
|
144
|
-
/** A decoded graph record, as returned by {@link
|
|
144
|
+
/** A decoded graph record, as returned by {@link PikeletRangeArtifact.readNode}. */
|
|
145
145
|
export interface RangeArtifactNode {
|
|
146
146
|
readonly id: number;
|
|
147
147
|
readonly level: number;
|
|
@@ -220,7 +220,7 @@ export interface RangeArtifactBuildOptions {
|
|
|
220
220
|
}
|
|
221
221
|
|
|
222
222
|
export interface RangeArtifactBuildManifest {
|
|
223
|
-
readonly format: '
|
|
223
|
+
readonly format: 'pikelet-range-artifact';
|
|
224
224
|
readonly formatVersion: number;
|
|
225
225
|
/** Hex SHA-256 whole-segment digests stamped into the v3 header. */
|
|
226
226
|
readonly integrity: Readonly<{
|
|
@@ -240,7 +240,7 @@ export interface RangeArtifactBuildManifest {
|
|
|
240
240
|
/**
|
|
241
241
|
* Node-only file-backed range source. This is NOT a named runtime export of
|
|
242
242
|
* any entrypoint: reach the constructor via the Node API object
|
|
243
|
-
* (`
|
|
243
|
+
* (`Pikelet.NodeFileRangeSource`) or the `pikelet-wasm/artifact` subpath.
|
|
244
244
|
*/
|
|
245
245
|
export interface NodeFileRangeSource extends RangeReadSource {
|
|
246
246
|
readonly filePath: string;
|
|
@@ -256,10 +256,10 @@ export interface NodeFileRangeSourceConstructor {
|
|
|
256
256
|
|
|
257
257
|
/**
|
|
258
258
|
* A range-readable Search Artifact reader. The class is NOT a named runtime
|
|
259
|
-
* export of any entrypoint: reach it via `
|
|
259
|
+
* export of any entrypoint: reach it via `Pikelet.RangeArtifact` or the
|
|
260
260
|
* `pikelet-wasm/artifact` subpath.
|
|
261
261
|
*/
|
|
262
|
-
export interface
|
|
262
|
+
export interface PikeletRangeArtifact {
|
|
263
263
|
readonly version: number;
|
|
264
264
|
readonly kind: number;
|
|
265
265
|
readonly dim: number;
|
|
@@ -299,11 +299,11 @@ export interface PancakeRangeArtifact {
|
|
|
299
299
|
close(): Promise<void>;
|
|
300
300
|
}
|
|
301
301
|
|
|
302
|
-
export interface
|
|
303
|
-
open(source: RangeReadSource, options?: RangeArtifactOpenOptions): Promise<
|
|
302
|
+
export interface PikeletRangeArtifactConstructor {
|
|
303
|
+
open(source: RangeReadSource, options?: RangeArtifactOpenOptions): Promise<PikeletRangeArtifact>;
|
|
304
304
|
/** Node entrypoints only. */
|
|
305
|
-
openFile(filePath: string, options?: RangeArtifactOpenOptions): Promise<
|
|
306
|
-
readonly prototype:
|
|
305
|
+
openFile(filePath: string, options?: RangeArtifactOpenOptions): Promise<PikeletRangeArtifact>;
|
|
306
|
+
readonly prototype: PikeletRangeArtifact;
|
|
307
307
|
}
|
|
308
308
|
|
|
309
309
|
/** Resident sketch tier: the coarse staged-boot tier or the full tier. */
|
|
@@ -335,7 +335,7 @@ export interface SketchArtifactOpenOptions {
|
|
|
335
335
|
/**
|
|
336
336
|
* Staged boot: when the artifact carries a micro tier, become searchable
|
|
337
337
|
* after loading only the stage-1 prefix and swap to the full tier in the
|
|
338
|
-
* background (see {@link
|
|
338
|
+
* background (see {@link PikeletSketchArtifact.fullyResident}). Ignored for
|
|
339
339
|
* artifacts without a micro tier. Default: false.
|
|
340
340
|
*/
|
|
341
341
|
staged?: boolean;
|
|
@@ -468,7 +468,7 @@ export interface SketchArtifactBuildOptions {
|
|
|
468
468
|
}
|
|
469
469
|
|
|
470
470
|
export interface SketchArtifactBuildManifest {
|
|
471
|
-
readonly format: '
|
|
471
|
+
readonly format: 'pikelet-sketch-artifact';
|
|
472
472
|
readonly formatVersion: number;
|
|
473
473
|
/** Output path; absent on the bytes-in/bytes-out builder. */
|
|
474
474
|
readonly file?: string;
|
|
@@ -489,10 +489,10 @@ export interface SketchArtifactBuildManifest {
|
|
|
489
489
|
|
|
490
490
|
/**
|
|
491
491
|
* A sketch Search Artifact reader. The class is NOT a named runtime export of
|
|
492
|
-
* any entrypoint: reach it via `
|
|
492
|
+
* any entrypoint: reach it via `Pikelet.SketchArtifact` or the
|
|
493
493
|
* `pikelet-wasm/artifact` subpath.
|
|
494
494
|
*/
|
|
495
|
-
export interface
|
|
495
|
+
export interface PikeletSketchArtifact {
|
|
496
496
|
readonly metric: number;
|
|
497
497
|
readonly dim: number;
|
|
498
498
|
readonly count: number;
|
|
@@ -509,7 +509,7 @@ export interface PancakeSketchArtifact {
|
|
|
509
509
|
* Already resolved for non-staged opens; rejects if the background stage-2
|
|
510
510
|
* load fails.
|
|
511
511
|
*/
|
|
512
|
-
readonly fullyResident: Promise<
|
|
512
|
+
readonly fullyResident: Promise<PikeletSketchArtifact>;
|
|
513
513
|
readonly recommendedRerank: number;
|
|
514
514
|
readonly residentBytes: number;
|
|
515
515
|
readonly residentVerified: boolean;
|
|
@@ -531,11 +531,11 @@ export interface PancakeSketchArtifact {
|
|
|
531
531
|
close(): Promise<void>;
|
|
532
532
|
}
|
|
533
533
|
|
|
534
|
-
export interface
|
|
535
|
-
open(source: RangeReadSource, options?: SketchArtifactOpenOptions): Promise<
|
|
534
|
+
export interface PikeletSketchArtifactConstructor {
|
|
535
|
+
open(source: RangeReadSource, options?: SketchArtifactOpenOptions): Promise<PikeletSketchArtifact>;
|
|
536
536
|
/** Node entrypoints only. */
|
|
537
|
-
openFile(filePath: string, options?: SketchArtifactOpenOptions): Promise<
|
|
538
|
-
readonly prototype:
|
|
537
|
+
openFile(filePath: string, options?: SketchArtifactOpenOptions): Promise<PikeletSketchArtifact>;
|
|
538
|
+
readonly prototype: PikeletSketchArtifact;
|
|
539
539
|
}
|
|
540
540
|
|
|
541
541
|
export interface MemoryUsage {
|
|
@@ -558,7 +558,7 @@ export interface ResolvedConfig {
|
|
|
558
558
|
readonly seed: number;
|
|
559
559
|
}
|
|
560
560
|
|
|
561
|
-
export interface
|
|
561
|
+
export interface PikeletIndex {
|
|
562
562
|
/** Insert a single vector. Returns its stable external ID. */
|
|
563
563
|
add(vector: VectorInput): number;
|
|
564
564
|
/** Insert multiple vectors. Returns stable external IDs in insertion order. */
|
|
@@ -617,23 +617,23 @@ export interface PancakeIndex {
|
|
|
617
617
|
/**
|
|
618
618
|
* The portable API exposed by every entrypoint (Node, browser, Workers).
|
|
619
619
|
* The browser/Workers entrypoints expose exactly this surface; the Node
|
|
620
|
-
* entrypoints add file helpers (see {@link
|
|
620
|
+
* entrypoints add file helpers (see {@link NodePikeletApi}).
|
|
621
621
|
*/
|
|
622
|
-
export interface
|
|
623
|
-
readonly
|
|
624
|
-
readonly
|
|
625
|
-
readonly RangeArtifact:
|
|
626
|
-
readonly SketchArtifact:
|
|
622
|
+
export interface PikeletApi {
|
|
623
|
+
readonly PikeletError: typeof PikeletError;
|
|
624
|
+
readonly PIKELET_ERROR_CODES: typeof PIKELET_ERROR_CODES;
|
|
625
|
+
readonly RangeArtifact: PikeletRangeArtifactConstructor;
|
|
626
|
+
readonly SketchArtifact: PikeletSketchArtifactConstructor;
|
|
627
627
|
/** Build a WASM-backed SIMD scanner for a sketch artifact's resident tier. */
|
|
628
|
-
createSketchScanner(artifact:
|
|
629
|
-
/** Create a new
|
|
630
|
-
create(opts: CreateOptions): Promise<
|
|
628
|
+
createSketchScanner(artifact: PikeletSketchArtifact, options?: SketchScannerOptions): Promise<SketchScanner>;
|
|
629
|
+
/** Create a new Pikelet index using the runtime-specific packaged entrypoint. */
|
|
630
|
+
create(opts: CreateOptions): Promise<PikeletIndex>;
|
|
631
631
|
/** Restore an envelope snapshot, inferring its construction config. */
|
|
632
|
-
restore(snapshot: Uint8Array | ArrayBufferLike, overrides?: RestoreOptions): Promise<
|
|
632
|
+
restore(snapshot: Uint8Array | ArrayBufferLike, overrides?: RestoreOptions): Promise<PikeletIndex>;
|
|
633
633
|
/** Validate and inspect snapshot headers without creating a WASM index. */
|
|
634
634
|
inspectSnapshot(snapshot: Uint8Array | ArrayBufferLike): SnapshotInspection;
|
|
635
635
|
/** Create an index, run a callback, and always dispose the index afterward. */
|
|
636
|
-
withIndex<T>(opts: CreateOptions, fn: (index:
|
|
636
|
+
withIndex<T>(opts: CreateOptions, fn: (index: PikeletIndex) => T | Promise<T>): Promise<T>;
|
|
637
637
|
/** Create and populate an index from raw vectors. Infers dim and maxElements by default. */
|
|
638
638
|
fromVectors(vectors: readonly VectorInput[], opts?: Omit<CreateOptions, 'dim'> & Partial<Pick<CreateOptions, 'dim'>>): Promise<FromVectorsResult<never>>;
|
|
639
639
|
/** Create and populate an index from { id, vector } records. */
|
|
@@ -644,29 +644,29 @@ export interface PancakeApi {
|
|
|
644
644
|
* The Node.js API: the portable surface plus filesystem helpers. These helpers
|
|
645
645
|
* exist only on the Node entrypoints (`pikelet-wasm`, `pikelet-wasm/node`); the
|
|
646
646
|
* browser/Workers entrypoints do not expose them, and importing those returns
|
|
647
|
-
* the narrower {@link
|
|
647
|
+
* the narrower {@link PikeletApi}.
|
|
648
648
|
*/
|
|
649
|
-
export interface
|
|
649
|
+
export interface NodePikeletApi extends PikeletApi {
|
|
650
650
|
readonly NodeFileRangeSource: NodeFileRangeSourceConstructor;
|
|
651
651
|
/**
|
|
652
|
-
* Build a range-readable Search Artifact from a uint8
|
|
653
|
-
* @deprecated The `.
|
|
654
|
-
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.
|
|
652
|
+
* Build a range-readable Search Artifact from a uint8 Pikelet snapshot.
|
|
653
|
+
* @deprecated The `.pikelet-range` profile is deprecated
|
|
654
|
+
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.pikelet-sketch`
|
|
655
655
|
* artifact instead. Readers stay supported for existing artifacts.
|
|
656
656
|
*/
|
|
657
657
|
buildRangeArtifact(snapshot: Uint8Array | ArrayBufferLike, outPath: string, opts?: RangeArtifactBuildOptions): RangeArtifactBuildManifest;
|
|
658
658
|
/**
|
|
659
|
-
* Build a range-readable Search Artifact from a uint8
|
|
660
|
-
* @deprecated The `.
|
|
661
|
-
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.
|
|
659
|
+
* Build a range-readable Search Artifact from a uint8 Pikelet snapshot file.
|
|
660
|
+
* @deprecated The `.pikelet-range` profile is deprecated
|
|
661
|
+
* (spec/SEARCH_ARTIFACT_CONTRACT.md 9.2); build a `.pikelet-sketch`
|
|
662
662
|
* artifact instead. Readers stay supported for existing artifacts.
|
|
663
663
|
*/
|
|
664
664
|
buildRangeArtifactFile(snapshotPath: string, outPath: string, opts?: RangeArtifactBuildOptions): RangeArtifactBuildManifest;
|
|
665
665
|
/** Open a range-readable Search Artifact from a local file. */
|
|
666
|
-
openRangeArtifactFile(filePath: string, opts?: RangeArtifactOpenOptions): Promise<
|
|
667
|
-
/** Build a sketch Search Artifact from a uint8
|
|
666
|
+
openRangeArtifactFile(filePath: string, opts?: RangeArtifactOpenOptions): Promise<PikeletRangeArtifact>;
|
|
667
|
+
/** Build a sketch Search Artifact from a uint8 Pikelet snapshot. */
|
|
668
668
|
buildSketchArtifact(snapshot: Uint8Array | ArrayBufferLike, outPath: string, opts?: SketchArtifactBuildOptions): SketchArtifactBuildManifest;
|
|
669
|
-
/** Build a sketch Search Artifact from a uint8
|
|
669
|
+
/** Build a sketch Search Artifact from a uint8 Pikelet snapshot file. */
|
|
670
670
|
buildSketchArtifactFile(snapshotPath: string, outPath: string, opts?: SketchArtifactBuildOptions): SketchArtifactBuildManifest;
|
|
671
671
|
/**
|
|
672
672
|
* Bytes-in/bytes-out sketch build for producers assembling segments in
|
|
@@ -675,17 +675,17 @@ export interface NodePancakeApi extends PancakeApi {
|
|
|
675
675
|
*/
|
|
676
676
|
buildSketchArtifactBytes(snapshot: Uint8Array | ArrayBufferLike, opts?: SketchArtifactBuildOptions): { bytes: Uint8Array; manifest: SketchArtifactBuildManifest };
|
|
677
677
|
/** Open a sketch Search Artifact from a local file. */
|
|
678
|
-
openSketchArtifactFile(filePath: string, opts?: SketchArtifactOpenOptions): Promise<
|
|
678
|
+
openSketchArtifactFile(filePath: string, opts?: SketchArtifactOpenOptions): Promise<PikeletSketchArtifact>;
|
|
679
679
|
/** Load vectors from a JSON/JSONL file and build an index. */
|
|
680
680
|
loadJsonFile<Id = unknown>(filePath: string, opts?: JsonFileOptions): Promise<FromVectorsResult<Id>>;
|
|
681
681
|
/**
|
|
682
|
-
* Load a previously exported
|
|
682
|
+
* Load a previously exported Pikelet snapshot from disk. Envelope snapshots
|
|
683
683
|
* carry their own config, so `opts` is only required for raw engine
|
|
684
684
|
* snapshots (which need the full create config).
|
|
685
685
|
*/
|
|
686
|
-
loadSnapshotFile(filePath: string, opts?: SnapshotFileOptions): Promise<
|
|
686
|
+
loadSnapshotFile(filePath: string, opts?: SnapshotFileOptions): Promise<PikeletIndex>;
|
|
687
687
|
}
|
|
688
688
|
|
|
689
|
-
declare const
|
|
689
|
+
declare const Pikelet: NodePikeletApi;
|
|
690
690
|
|
|
691
|
-
export default
|
|
691
|
+
export default Pikelet;
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
const loadEngine = require('./dist/engine.js');
|
|
4
4
|
const loadScalarEngine = require('./dist/engine.scalar.js');
|
|
5
|
-
const
|
|
6
|
-
const {
|
|
7
|
-
const {
|
|
8
|
-
const { createCachedModuleLoader } = require('./
|
|
5
|
+
const createPikeletApi = require('./pikelet-core.js');
|
|
6
|
+
const { PikeletError, PIKELET_ERROR_CODES, pikeletError } = require('./pikelet-errors.js');
|
|
7
|
+
const { PikeletRangeArtifact, PikeletSketchArtifact, createSketchScanner, NodeFileRangeSource, buildRangeArtifact, buildRangeArtifactFile, buildSketchArtifact, buildSketchArtifactBytes, buildSketchArtifactFile } = require('./pikelet-artifact.js');
|
|
8
|
+
const { createCachedModuleLoader } = require('./pikelet-loader.js');
|
|
9
9
|
const _path = require('path');
|
|
10
10
|
const _fs = require('fs');
|
|
11
11
|
let _engineVariantPromise = null;
|
|
@@ -29,13 +29,13 @@ function readWasmBinary(fileName) {
|
|
|
29
29
|
);
|
|
30
30
|
} catch (error) {
|
|
31
31
|
const message = error && error.message ? error.message : String(error);
|
|
32
|
-
throw
|
|
32
|
+
throw pikeletError(PIKELET_ERROR_CODES.WASM_LOAD_FAILED, `Failed to load Pikelet WASM binary (${fileName}): ${message}`, { fileName }, error);
|
|
33
33
|
}
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
function makeLoadError(message, error) {
|
|
37
37
|
const detail = error && error.message ? error.message : String(error);
|
|
38
|
-
return
|
|
38
|
+
return pikeletError(PIKELET_ERROR_CODES.WASM_LOAD_FAILED, `${message}: ${detail}`, undefined, error);
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
const _moduleLoader = createCachedModuleLoader((variant) =>
|
|
@@ -58,7 +58,7 @@ function parseJsonLines(text, filePath) {
|
|
|
58
58
|
rows.push(JSON.parse(line));
|
|
59
59
|
} catch (error) {
|
|
60
60
|
const message = error && error.message ? error.message : String(error);
|
|
61
|
-
throw
|
|
61
|
+
throw pikeletError(PIKELET_ERROR_CODES.PARSE_FAILED, `Failed to parse JSONL in ${filePath} at line ${i + 1}: ${message}`, { filePath, line: i + 1 }, error);
|
|
62
62
|
}
|
|
63
63
|
}
|
|
64
64
|
return rows;
|
|
@@ -67,22 +67,22 @@ function parseJsonLines(text, filePath) {
|
|
|
67
67
|
function inferJsonFormat(filePath) {
|
|
68
68
|
if (/\.json$/i.test(filePath)) return 'json';
|
|
69
69
|
if (/\.(jsonl|ndjson)$/i.test(filePath)) return 'jsonl';
|
|
70
|
-
throw
|
|
70
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, `loadJsonFile() could not infer format from '${filePath}'. Use a .json/.jsonl/.ndjson extension or pass opts.format.`, { filePath });
|
|
71
71
|
}
|
|
72
72
|
|
|
73
73
|
function remapJsonRows(rows, vectorKey, idKey) {
|
|
74
74
|
if (!Array.isArray(rows)) {
|
|
75
|
-
throw
|
|
75
|
+
throw pikeletError(PIKELET_ERROR_CODES.PARSE_FAILED, 'loadJsonFile() expects a JSON array or JSONL sequence of vectors/records');
|
|
76
76
|
}
|
|
77
77
|
return rows.map((row, i) => {
|
|
78
78
|
if (row instanceof Float32Array || Array.isArray(row)) {
|
|
79
79
|
return row;
|
|
80
80
|
}
|
|
81
81
|
if (!row || typeof row !== 'object') {
|
|
82
|
-
throw
|
|
82
|
+
throw pikeletError(PIKELET_ERROR_CODES.PARSE_FAILED, `loadJsonFile() expected an object or vector at index ${i}`, { index: i });
|
|
83
83
|
}
|
|
84
84
|
if (!(vectorKey in row)) {
|
|
85
|
-
throw
|
|
85
|
+
throw pikeletError(PIKELET_ERROR_CODES.PARSE_FAILED, `loadJsonFile() missing vectorKey '${vectorKey}' at index ${i}`, { index: i, vectorKey });
|
|
86
86
|
}
|
|
87
87
|
const mapped = { vector: row[vectorKey] };
|
|
88
88
|
if (Object.prototype.hasOwnProperty.call(row, idKey)) {
|
|
@@ -94,13 +94,13 @@ function remapJsonRows(rows, vectorKey, idKey) {
|
|
|
94
94
|
|
|
95
95
|
function validateFilePath(filePath, helperName) {
|
|
96
96
|
if (typeof filePath !== 'string' || filePath.length === 0) {
|
|
97
|
-
throw
|
|
97
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, `${helperName}() requires a non-empty file path`);
|
|
98
98
|
}
|
|
99
99
|
}
|
|
100
100
|
|
|
101
101
|
function validateMaxFileBytes(maxFileBytes, helperName) {
|
|
102
102
|
if (!Number.isInteger(maxFileBytes) || maxFileBytes <= 0) {
|
|
103
|
-
throw
|
|
103
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, `${helperName}() maxFileBytes must be a positive integer`, { maxFileBytes });
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
|
|
@@ -110,17 +110,17 @@ function statRegularFile(filePath, helperName) {
|
|
|
110
110
|
stat = _fs.statSync(filePath);
|
|
111
111
|
} catch (error) {
|
|
112
112
|
const message = error && error.message ? error.message : String(error);
|
|
113
|
-
throw
|
|
113
|
+
throw pikeletError(PIKELET_ERROR_CODES.FILE_IO_FAILED, `${helperName}() could not stat ${filePath}: ${message}`, { filePath }, error);
|
|
114
114
|
}
|
|
115
115
|
if (!stat.isFile()) {
|
|
116
|
-
throw
|
|
116
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, `${helperName}() expected a regular file: ${filePath}`, { filePath });
|
|
117
117
|
}
|
|
118
118
|
return stat;
|
|
119
119
|
}
|
|
120
120
|
|
|
121
121
|
function enforceFileSize(stat, maxFileBytes, helperName, filePath) {
|
|
122
122
|
if (stat.size > maxFileBytes) {
|
|
123
|
-
throw
|
|
123
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, `${helperName}() file exceeds maxFileBytes (${stat.size} > ${maxFileBytes}): ${filePath}`, { filePath, fileBytes: stat.size, maxFileBytes });
|
|
124
124
|
}
|
|
125
125
|
}
|
|
126
126
|
|
|
@@ -133,7 +133,7 @@ function readUtf8FileWithLimit(filePath, helperName, maxFileBytes) {
|
|
|
133
133
|
return _fs.readFileSync(filePath, 'utf8');
|
|
134
134
|
} catch (error) {
|
|
135
135
|
const message = error && error.message ? error.message : String(error);
|
|
136
|
-
throw
|
|
136
|
+
throw pikeletError(PIKELET_ERROR_CODES.FILE_IO_FAILED, `${helperName}() could not read ${filePath}: ${message}`, { filePath }, error);
|
|
137
137
|
}
|
|
138
138
|
}
|
|
139
139
|
|
|
@@ -146,18 +146,18 @@ function readBinaryFileWithLimit(filePath, helperName, maxFileBytes) {
|
|
|
146
146
|
return _fs.readFileSync(filePath);
|
|
147
147
|
} catch (error) {
|
|
148
148
|
const message = error && error.message ? error.message : String(error);
|
|
149
|
-
throw
|
|
149
|
+
throw pikeletError(PIKELET_ERROR_CODES.FILE_IO_FAILED, `${helperName}() could not read ${filePath}: ${message}`, { filePath }, error);
|
|
150
150
|
}
|
|
151
151
|
}
|
|
152
152
|
|
|
153
153
|
function validateSnapshotBytes(snapshot, filePath) {
|
|
154
154
|
if (!snapshot || snapshot.byteLength < 4) {
|
|
155
|
-
throw
|
|
155
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, `loadSnapshotFile() snapshot is too small to be valid: ${filePath}`, { filePath });
|
|
156
156
|
}
|
|
157
157
|
const view = new DataView(snapshot.buffer, snapshot.byteOffset, snapshot.byteLength);
|
|
158
158
|
const magic = view.getUint32(0, true);
|
|
159
159
|
if (!SUPPORTED_SNAPSHOT_MAGICS.has(magic)) {
|
|
160
|
-
throw
|
|
160
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, `loadSnapshotFile() unsupported snapshot file type: ${filePath}`, { filePath, magic });
|
|
161
161
|
}
|
|
162
162
|
}
|
|
163
163
|
|
|
@@ -168,7 +168,7 @@ async function loadNodeEngine() {
|
|
|
168
168
|
try {
|
|
169
169
|
return await _moduleLoader.instantiate(loadScalarEngine, 'scalar');
|
|
170
170
|
} catch (error) {
|
|
171
|
-
throw makeLoadError('
|
|
171
|
+
throw makeLoadError('Pikelet failed to load the scalar WASM engine', error);
|
|
172
172
|
}
|
|
173
173
|
}
|
|
174
174
|
|
|
@@ -181,37 +181,37 @@ async function loadNodeEngine() {
|
|
|
181
181
|
return engine;
|
|
182
182
|
} catch (scalarError) {
|
|
183
183
|
throw makeLoadError(
|
|
184
|
-
`
|
|
184
|
+
`Pikelet failed to load either the SIMD or scalar WASM engine (SIMD error: ${simdError && simdError.message ? simdError.message : String(simdError)})`,
|
|
185
185
|
scalarError
|
|
186
186
|
);
|
|
187
187
|
}
|
|
188
188
|
}
|
|
189
189
|
}
|
|
190
190
|
|
|
191
|
-
const
|
|
191
|
+
const Pikelet = createPikeletApi(loadNodeEngine);
|
|
192
192
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
193
|
+
Pikelet.RangeArtifact = PikeletRangeArtifact;
|
|
194
|
+
Pikelet.NodeFileRangeSource = NodeFileRangeSource;
|
|
195
|
+
Pikelet.buildRangeArtifact = buildRangeArtifact;
|
|
196
|
+
Pikelet.buildRangeArtifactFile = buildRangeArtifactFile;
|
|
197
197
|
|
|
198
|
-
|
|
198
|
+
Pikelet.openRangeArtifactFile = async function openRangeArtifactFile(filePath, opts) {
|
|
199
199
|
validateFilePath(filePath, 'openRangeArtifactFile');
|
|
200
|
-
return
|
|
200
|
+
return PikeletRangeArtifact.openFile(filePath, opts);
|
|
201
201
|
};
|
|
202
202
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
203
|
+
Pikelet.SketchArtifact = PikeletSketchArtifact;
|
|
204
|
+
Pikelet.createSketchScanner = (artifact, options) => createSketchScanner(loadNodeEngine, artifact, options);
|
|
205
|
+
Pikelet.buildSketchArtifact = buildSketchArtifact;
|
|
206
|
+
Pikelet.buildSketchArtifactBytes = buildSketchArtifactBytes;
|
|
207
|
+
Pikelet.buildSketchArtifactFile = buildSketchArtifactFile;
|
|
208
208
|
|
|
209
|
-
|
|
209
|
+
Pikelet.openSketchArtifactFile = async function openSketchArtifactFile(filePath, opts) {
|
|
210
210
|
validateFilePath(filePath, 'openSketchArtifactFile');
|
|
211
|
-
return
|
|
211
|
+
return PikeletSketchArtifact.openFile(filePath, opts);
|
|
212
212
|
};
|
|
213
213
|
|
|
214
|
-
|
|
214
|
+
Pikelet.loadSnapshotFile = async function loadSnapshotFile(filePath, opts) {
|
|
215
215
|
const {
|
|
216
216
|
maxFileBytes = DEFAULT_SNAPSHOT_FILE_BYTES,
|
|
217
217
|
...createOpts
|
|
@@ -219,10 +219,10 @@ Pancake.loadSnapshotFile = async function loadSnapshotFile(filePath, opts) {
|
|
|
219
219
|
|
|
220
220
|
const snapshot = readBinaryFileWithLimit(filePath, 'loadSnapshotFile', maxFileBytes);
|
|
221
221
|
validateSnapshotBytes(snapshot, filePath);
|
|
222
|
-
return
|
|
222
|
+
return Pikelet.restore(snapshot, createOpts);
|
|
223
223
|
};
|
|
224
224
|
|
|
225
|
-
|
|
225
|
+
Pikelet.loadJsonFile = async function loadJsonFile(filePath, opts = {}) {
|
|
226
226
|
const {
|
|
227
227
|
format = inferJsonFormat(filePath),
|
|
228
228
|
vectorKey = 'vector',
|
|
@@ -232,7 +232,7 @@ Pancake.loadJsonFile = async function loadJsonFile(filePath, opts = {}) {
|
|
|
232
232
|
} = opts;
|
|
233
233
|
|
|
234
234
|
if (format !== 'json' && format !== 'jsonl') {
|
|
235
|
-
throw
|
|
235
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, `loadJsonFile() format must be 'json' or 'jsonl', got '${format}'`, { format });
|
|
236
236
|
}
|
|
237
237
|
|
|
238
238
|
const text = readUtf8FileWithLimit(filePath, 'loadJsonFile', maxFileBytes);
|
|
@@ -244,11 +244,11 @@ Pancake.loadJsonFile = async function loadJsonFile(filePath, opts = {}) {
|
|
|
244
244
|
rows = JSON.parse(text);
|
|
245
245
|
} catch (error) {
|
|
246
246
|
const message = error && error.message ? error.message : String(error);
|
|
247
|
-
throw
|
|
247
|
+
throw pikeletError(PIKELET_ERROR_CODES.PARSE_FAILED, `Failed to parse JSON in ${filePath}: ${message}`, { filePath }, error);
|
|
248
248
|
}
|
|
249
249
|
}
|
|
250
|
-
return
|
|
250
|
+
return Pikelet.fromVectors(remapJsonRows(rows, vectorKey, idKey), createOpts);
|
|
251
251
|
};
|
|
252
252
|
|
|
253
|
-
module.exports =
|
|
254
|
-
module.exports.default =
|
|
253
|
+
module.exports = Pikelet;
|
|
254
|
+
module.exports.default = Pikelet;
|