pikelet-wasm 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +868 -0
- package/LICENSE +191 -0
- package/README.md +590 -0
- package/complete/builder.d.ts +82 -0
- package/complete/builder.mjs +527 -0
- package/complete/encoder-kernels/encoder.mjs +2 -0
- package/complete/encoder-kernels/encoder.node.mjs +2 -0
- package/complete/encoder-kernels/encoder.node.wasm +0 -0
- package/complete/encoder-kernels/encoder.wasm +0 -0
- package/complete/format.mjs +8 -0
- package/complete/index.d.ts +289 -0
- package/complete/index.mjs +1124 -0
- package/complete/inline-transformer.mjs +205 -0
- package/complete/lexical.mjs +264 -0
- package/complete/retrieval-abstention.mjs +119 -0
- package/complete/sources.mjs +186 -0
- package/complete/student-abstention.mjs +102 -0
- package/complete/student-embedder.mjs +231 -0
- package/complete/wordpiece.mjs +68 -0
- package/dist/engine.js +2 -0
- package/dist/engine.scalar.js +2 -0
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +168 -0
- package/pancake-artifact-common.js +561 -0
- package/pancake-artifact-range.js +799 -0
- package/pancake-artifact-sketch.js +1040 -0
- package/pancake-artifact.d.ts +109 -0
- package/pancake-artifact.js +31 -0
- package/pancake-core.js +1268 -0
- package/pancake-errors.js +40 -0
- package/pancake-loader.js +66 -0
- package/pancake.d.ts +691 -0
- package/pancake.js +254 -0
- package/pancake.node.mjs +259 -0
- package/pancake.web.d.ts +50 -0
- package/pancake.web.mjs +88 -0
- package/pancake.workerd.mjs +72 -0
package/pancake.js
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const loadEngine = require('./dist/engine.js');
|
|
4
|
+
const loadScalarEngine = require('./dist/engine.scalar.js');
|
|
5
|
+
const createPancakeApi = require('./pancake-core.js');
|
|
6
|
+
const { PancakeError, PANCAKE_ERROR_CODES, pancakeError } = require('./pancake-errors.js');
|
|
7
|
+
const { PancakeRangeArtifact, PancakeSketchArtifact, createSketchScanner, NodeFileRangeSource, buildRangeArtifact, buildRangeArtifactFile, buildSketchArtifact, buildSketchArtifactBytes, buildSketchArtifactFile } = require('./pancake-artifact.js');
|
|
8
|
+
const { createCachedModuleLoader } = require('./pancake-loader.js');
|
|
9
|
+
const _path = require('path');
|
|
10
|
+
const _fs = require('fs');
|
|
11
|
+
let _engineVariantPromise = null;
|
|
12
|
+
|
|
13
|
+
const DEFAULT_JSON_FILE_BYTES = 64 * 1024 * 1024;
|
|
14
|
+
const DEFAULT_SNAPSHOT_FILE_BYTES = 512 * 1024 * 1024;
|
|
15
|
+
const SUPPORTED_SNAPSHOT_MAGICS = new Set([
|
|
16
|
+
0x504E434B, // PNCK envelope
|
|
17
|
+
0x464C4857, // FLHW raw float v0
|
|
18
|
+
0x464C4831, // FLH1 raw float v1
|
|
19
|
+
0x49384857, // I8HW raw uint8 v0
|
|
20
|
+
0x49384831, // I8H1 raw uint8 v1
|
|
21
|
+
]);
|
|
22
|
+
|
|
23
|
+
function readWasmBinary(fileName) {
|
|
24
|
+
try {
|
|
25
|
+
const wasmBinary = _fs.readFileSync(_path.join(__dirname, 'dist', fileName));
|
|
26
|
+
return wasmBinary.buffer.slice(
|
|
27
|
+
wasmBinary.byteOffset,
|
|
28
|
+
wasmBinary.byteOffset + wasmBinary.byteLength
|
|
29
|
+
);
|
|
30
|
+
} catch (error) {
|
|
31
|
+
const message = error && error.message ? error.message : String(error);
|
|
32
|
+
throw pancakeError(PANCAKE_ERROR_CODES.WASM_LOAD_FAILED, `Failed to load Pancake WASM binary (${fileName}): ${message}`, { fileName }, error);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function makeLoadError(message, error) {
|
|
37
|
+
const detail = error && error.message ? error.message : String(error);
|
|
38
|
+
return pancakeError(PANCAKE_ERROR_CODES.WASM_LOAD_FAILED, `${message}: ${detail}`, undefined, error);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const _moduleLoader = createCachedModuleLoader((variant) =>
|
|
42
|
+
readWasmBinary(variant === 'simd' ? 'engine.wasm' : 'engine.scalar.wasm')
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
function selectEngineVariant() {
|
|
46
|
+
_engineVariantPromise ??= _moduleLoader.supports('simd')
|
|
47
|
+
.then((supported) => supported ? 'simd' : 'scalar');
|
|
48
|
+
return _engineVariantPromise;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function parseJsonLines(text, filePath) {
|
|
52
|
+
const rows = [];
|
|
53
|
+
const lines = text.split(/\r?\n/);
|
|
54
|
+
for (let i = 0; i < lines.length; i++) {
|
|
55
|
+
const line = lines[i].trim();
|
|
56
|
+
if (line.length === 0) continue;
|
|
57
|
+
try {
|
|
58
|
+
rows.push(JSON.parse(line));
|
|
59
|
+
} catch (error) {
|
|
60
|
+
const message = error && error.message ? error.message : String(error);
|
|
61
|
+
throw pancakeError(PANCAKE_ERROR_CODES.PARSE_FAILED, `Failed to parse JSONL in ${filePath} at line ${i + 1}: ${message}`, { filePath, line: i + 1 }, error);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return rows;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function inferJsonFormat(filePath) {
|
|
68
|
+
if (/\.json$/i.test(filePath)) return 'json';
|
|
69
|
+
if (/\.(jsonl|ndjson)$/i.test(filePath)) return 'jsonl';
|
|
70
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, `loadJsonFile() could not infer format from '${filePath}'. Use a .json/.jsonl/.ndjson extension or pass opts.format.`, { filePath });
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function remapJsonRows(rows, vectorKey, idKey) {
|
|
74
|
+
if (!Array.isArray(rows)) {
|
|
75
|
+
throw pancakeError(PANCAKE_ERROR_CODES.PARSE_FAILED, 'loadJsonFile() expects a JSON array or JSONL sequence of vectors/records');
|
|
76
|
+
}
|
|
77
|
+
return rows.map((row, i) => {
|
|
78
|
+
if (row instanceof Float32Array || Array.isArray(row)) {
|
|
79
|
+
return row;
|
|
80
|
+
}
|
|
81
|
+
if (!row || typeof row !== 'object') {
|
|
82
|
+
throw pancakeError(PANCAKE_ERROR_CODES.PARSE_FAILED, `loadJsonFile() expected an object or vector at index ${i}`, { index: i });
|
|
83
|
+
}
|
|
84
|
+
if (!(vectorKey in row)) {
|
|
85
|
+
throw pancakeError(PANCAKE_ERROR_CODES.PARSE_FAILED, `loadJsonFile() missing vectorKey '${vectorKey}' at index ${i}`, { index: i, vectorKey });
|
|
86
|
+
}
|
|
87
|
+
const mapped = { vector: row[vectorKey] };
|
|
88
|
+
if (Object.prototype.hasOwnProperty.call(row, idKey)) {
|
|
89
|
+
mapped.id = row[idKey];
|
|
90
|
+
}
|
|
91
|
+
return mapped;
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function validateFilePath(filePath, helperName) {
|
|
96
|
+
if (typeof filePath !== 'string' || filePath.length === 0) {
|
|
97
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, `${helperName}() requires a non-empty file path`);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function validateMaxFileBytes(maxFileBytes, helperName) {
|
|
102
|
+
if (!Number.isInteger(maxFileBytes) || maxFileBytes <= 0) {
|
|
103
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, `${helperName}() maxFileBytes must be a positive integer`, { maxFileBytes });
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function statRegularFile(filePath, helperName) {
|
|
108
|
+
let stat;
|
|
109
|
+
try {
|
|
110
|
+
stat = _fs.statSync(filePath);
|
|
111
|
+
} catch (error) {
|
|
112
|
+
const message = error && error.message ? error.message : String(error);
|
|
113
|
+
throw pancakeError(PANCAKE_ERROR_CODES.FILE_IO_FAILED, `${helperName}() could not stat ${filePath}: ${message}`, { filePath }, error);
|
|
114
|
+
}
|
|
115
|
+
if (!stat.isFile()) {
|
|
116
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, `${helperName}() expected a regular file: ${filePath}`, { filePath });
|
|
117
|
+
}
|
|
118
|
+
return stat;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function enforceFileSize(stat, maxFileBytes, helperName, filePath) {
|
|
122
|
+
if (stat.size > maxFileBytes) {
|
|
123
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, `${helperName}() file exceeds maxFileBytes (${stat.size} > ${maxFileBytes}): ${filePath}`, { filePath, fileBytes: stat.size, maxFileBytes });
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function readUtf8FileWithLimit(filePath, helperName, maxFileBytes) {
|
|
128
|
+
validateFilePath(filePath, helperName);
|
|
129
|
+
validateMaxFileBytes(maxFileBytes, helperName);
|
|
130
|
+
const stat = statRegularFile(filePath, helperName);
|
|
131
|
+
enforceFileSize(stat, maxFileBytes, helperName, filePath);
|
|
132
|
+
try {
|
|
133
|
+
return _fs.readFileSync(filePath, 'utf8');
|
|
134
|
+
} catch (error) {
|
|
135
|
+
const message = error && error.message ? error.message : String(error);
|
|
136
|
+
throw pancakeError(PANCAKE_ERROR_CODES.FILE_IO_FAILED, `${helperName}() could not read ${filePath}: ${message}`, { filePath }, error);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function readBinaryFileWithLimit(filePath, helperName, maxFileBytes) {
|
|
141
|
+
validateFilePath(filePath, helperName);
|
|
142
|
+
validateMaxFileBytes(maxFileBytes, helperName);
|
|
143
|
+
const stat = statRegularFile(filePath, helperName);
|
|
144
|
+
enforceFileSize(stat, maxFileBytes, helperName, filePath);
|
|
145
|
+
try {
|
|
146
|
+
return _fs.readFileSync(filePath);
|
|
147
|
+
} catch (error) {
|
|
148
|
+
const message = error && error.message ? error.message : String(error);
|
|
149
|
+
throw pancakeError(PANCAKE_ERROR_CODES.FILE_IO_FAILED, `${helperName}() could not read ${filePath}: ${message}`, { filePath }, error);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function validateSnapshotBytes(snapshot, filePath) {
|
|
154
|
+
if (!snapshot || snapshot.byteLength < 4) {
|
|
155
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID, `loadSnapshotFile() snapshot is too small to be valid: ${filePath}`, { filePath });
|
|
156
|
+
}
|
|
157
|
+
const view = new DataView(snapshot.buffer, snapshot.byteOffset, snapshot.byteLength);
|
|
158
|
+
const magic = view.getUint32(0, true);
|
|
159
|
+
if (!SUPPORTED_SNAPSHOT_MAGICS.has(magic)) {
|
|
160
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID, `loadSnapshotFile() unsupported snapshot file type: ${filePath}`, { filePath, magic });
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
async function loadNodeEngine() {
|
|
165
|
+
const engineVariant = await selectEngineVariant();
|
|
166
|
+
|
|
167
|
+
if (engineVariant === 'scalar') {
|
|
168
|
+
try {
|
|
169
|
+
return await _moduleLoader.instantiate(loadScalarEngine, 'scalar');
|
|
170
|
+
} catch (error) {
|
|
171
|
+
throw makeLoadError('Pancake failed to load the scalar WASM engine', error);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
try {
|
|
176
|
+
return await _moduleLoader.instantiate(loadEngine, 'simd');
|
|
177
|
+
} catch (simdError) {
|
|
178
|
+
try {
|
|
179
|
+
const engine = await _moduleLoader.instantiate(loadScalarEngine, 'scalar');
|
|
180
|
+
_engineVariantPromise = Promise.resolve('scalar');
|
|
181
|
+
return engine;
|
|
182
|
+
} catch (scalarError) {
|
|
183
|
+
throw makeLoadError(
|
|
184
|
+
`Pancake failed to load either the SIMD or scalar WASM engine (SIMD error: ${simdError && simdError.message ? simdError.message : String(simdError)})`,
|
|
185
|
+
scalarError
|
|
186
|
+
);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const Pancake = createPancakeApi(loadNodeEngine);
|
|
192
|
+
|
|
193
|
+
Pancake.RangeArtifact = PancakeRangeArtifact;
|
|
194
|
+
Pancake.NodeFileRangeSource = NodeFileRangeSource;
|
|
195
|
+
Pancake.buildRangeArtifact = buildRangeArtifact;
|
|
196
|
+
Pancake.buildRangeArtifactFile = buildRangeArtifactFile;
|
|
197
|
+
|
|
198
|
+
Pancake.openRangeArtifactFile = async function openRangeArtifactFile(filePath, opts) {
|
|
199
|
+
validateFilePath(filePath, 'openRangeArtifactFile');
|
|
200
|
+
return PancakeRangeArtifact.openFile(filePath, opts);
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
Pancake.SketchArtifact = PancakeSketchArtifact;
|
|
204
|
+
Pancake.createSketchScanner = (artifact, options) => createSketchScanner(loadNodeEngine, artifact, options);
|
|
205
|
+
Pancake.buildSketchArtifact = buildSketchArtifact;
|
|
206
|
+
Pancake.buildSketchArtifactBytes = buildSketchArtifactBytes;
|
|
207
|
+
Pancake.buildSketchArtifactFile = buildSketchArtifactFile;
|
|
208
|
+
|
|
209
|
+
Pancake.openSketchArtifactFile = async function openSketchArtifactFile(filePath, opts) {
|
|
210
|
+
validateFilePath(filePath, 'openSketchArtifactFile');
|
|
211
|
+
return PancakeSketchArtifact.openFile(filePath, opts);
|
|
212
|
+
};
|
|
213
|
+
|
|
214
|
+
Pancake.loadSnapshotFile = async function loadSnapshotFile(filePath, opts) {
|
|
215
|
+
const {
|
|
216
|
+
maxFileBytes = DEFAULT_SNAPSHOT_FILE_BYTES,
|
|
217
|
+
...createOpts
|
|
218
|
+
} = opts || {};
|
|
219
|
+
|
|
220
|
+
const snapshot = readBinaryFileWithLimit(filePath, 'loadSnapshotFile', maxFileBytes);
|
|
221
|
+
validateSnapshotBytes(snapshot, filePath);
|
|
222
|
+
return Pancake.restore(snapshot, createOpts);
|
|
223
|
+
};
|
|
224
|
+
|
|
225
|
+
Pancake.loadJsonFile = async function loadJsonFile(filePath, opts = {}) {
|
|
226
|
+
const {
|
|
227
|
+
format = inferJsonFormat(filePath),
|
|
228
|
+
vectorKey = 'vector',
|
|
229
|
+
idKey = 'id',
|
|
230
|
+
maxFileBytes = DEFAULT_JSON_FILE_BYTES,
|
|
231
|
+
...createOpts
|
|
232
|
+
} = opts;
|
|
233
|
+
|
|
234
|
+
if (format !== 'json' && format !== 'jsonl') {
|
|
235
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, `loadJsonFile() format must be 'json' or 'jsonl', got '${format}'`, { format });
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const text = readUtf8FileWithLimit(filePath, 'loadJsonFile', maxFileBytes);
|
|
239
|
+
let rows;
|
|
240
|
+
if (format === 'jsonl') {
|
|
241
|
+
rows = parseJsonLines(text, filePath);
|
|
242
|
+
} else {
|
|
243
|
+
try {
|
|
244
|
+
rows = JSON.parse(text);
|
|
245
|
+
} catch (error) {
|
|
246
|
+
const message = error && error.message ? error.message : String(error);
|
|
247
|
+
throw pancakeError(PANCAKE_ERROR_CODES.PARSE_FAILED, `Failed to parse JSON in ${filePath}: ${message}`, { filePath }, error);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
return Pancake.fromVectors(remapJsonRows(rows, vectorKey, idKey), createOpts);
|
|
251
|
+
};
|
|
252
|
+
|
|
253
|
+
module.exports = Pancake;
|
|
254
|
+
module.exports.default = Pancake;
|
package/pancake.node.mjs
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
import { readFileSync, statSync } from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import loadEngine from './dist/engine.js';
|
|
5
|
+
import loadScalarEngine from './dist/engine.scalar.js';
|
|
6
|
+
import createPancakeApi from './pancake-core.js';
|
|
7
|
+
import errorContract from './pancake-errors.js';
|
|
8
|
+
import loaderContract from './pancake-loader.js';
|
|
9
|
+
import artifactContract from './pancake-artifact.js';
|
|
10
|
+
const { PancakeError, PANCAKE_ERROR_CODES, pancakeError } = errorContract;
|
|
11
|
+
const { createCachedModuleLoader } = loaderContract;
|
|
12
|
+
const { PancakeRangeArtifact, PancakeSketchArtifact, createSketchScanner, NodeFileRangeSource, buildRangeArtifact, buildRangeArtifactFile, buildSketchArtifact, buildSketchArtifactBytes, buildSketchArtifactFile } = artifactContract;
|
|
13
|
+
|
|
14
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
15
|
+
const __dirname = path.dirname(__filename);
|
|
16
|
+
let engineVariantPromise = null;
|
|
17
|
+
|
|
18
|
+
const DEFAULT_JSON_FILE_BYTES = 64 * 1024 * 1024;
|
|
19
|
+
const DEFAULT_SNAPSHOT_FILE_BYTES = 512 * 1024 * 1024;
|
|
20
|
+
const SUPPORTED_SNAPSHOT_MAGICS = new Set([
|
|
21
|
+
0x504E434B, // PNCK envelope
|
|
22
|
+
0x464C4857, // FLHW raw float v0
|
|
23
|
+
0x464C4831, // FLH1 raw float v1
|
|
24
|
+
0x49384857, // I8HW raw uint8 v0
|
|
25
|
+
0x49384831, // I8H1 raw uint8 v1
|
|
26
|
+
]);
|
|
27
|
+
|
|
28
|
+
function readWasmBinary(fileName) {
|
|
29
|
+
try {
|
|
30
|
+
const wasmBinary = readFileSync(path.join(__dirname, 'dist', fileName));
|
|
31
|
+
return wasmBinary.buffer.slice(
|
|
32
|
+
wasmBinary.byteOffset,
|
|
33
|
+
wasmBinary.byteOffset + wasmBinary.byteLength
|
|
34
|
+
);
|
|
35
|
+
} catch (error) {
|
|
36
|
+
const message = error && error.message ? error.message : String(error);
|
|
37
|
+
throw pancakeError(PANCAKE_ERROR_CODES.WASM_LOAD_FAILED, `Failed to load Pancake WASM binary (${fileName}): ${message}`, { fileName }, error);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function makeLoadError(message, error) {
|
|
42
|
+
const detail = error && error.message ? error.message : String(error);
|
|
43
|
+
return pancakeError(PANCAKE_ERROR_CODES.WASM_LOAD_FAILED, `${message}: ${detail}`, undefined, error);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const moduleLoader = createCachedModuleLoader((variant) =>
|
|
47
|
+
readWasmBinary(variant === 'simd' ? 'engine.wasm' : 'engine.scalar.wasm')
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
function selectEngineVariant() {
|
|
51
|
+
engineVariantPromise ??= moduleLoader.supports('simd')
|
|
52
|
+
.then((supported) => supported ? 'simd' : 'scalar');
|
|
53
|
+
return engineVariantPromise;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function parseJsonLines(text, filePath) {
|
|
57
|
+
const rows = [];
|
|
58
|
+
const lines = text.split(/\r?\n/);
|
|
59
|
+
for (let i = 0; i < lines.length; i++) {
|
|
60
|
+
const line = lines[i].trim();
|
|
61
|
+
if (line.length === 0) continue;
|
|
62
|
+
try {
|
|
63
|
+
rows.push(JSON.parse(line));
|
|
64
|
+
} catch (error) {
|
|
65
|
+
const message = error && error.message ? error.message : String(error);
|
|
66
|
+
throw pancakeError(PANCAKE_ERROR_CODES.PARSE_FAILED, `Failed to parse JSONL in ${filePath} at line ${i + 1}: ${message}`, { filePath, line: i + 1 }, error);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return rows;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function inferJsonFormat(filePath) {
|
|
73
|
+
if (/\.json$/i.test(filePath)) return 'json';
|
|
74
|
+
if (/\.(jsonl|ndjson)$/i.test(filePath)) return 'jsonl';
|
|
75
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, `loadJsonFile() could not infer format from '${filePath}'. Use a .json/.jsonl/.ndjson extension or pass opts.format.`, { filePath });
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function remapJsonRows(rows, vectorKey, idKey) {
|
|
79
|
+
if (!Array.isArray(rows)) {
|
|
80
|
+
throw pancakeError(PANCAKE_ERROR_CODES.PARSE_FAILED, 'loadJsonFile() expects a JSON array or JSONL sequence of vectors/records');
|
|
81
|
+
}
|
|
82
|
+
return rows.map((row, i) => {
|
|
83
|
+
if (row instanceof Float32Array || Array.isArray(row)) {
|
|
84
|
+
return row;
|
|
85
|
+
}
|
|
86
|
+
if (!row || typeof row !== 'object') {
|
|
87
|
+
throw pancakeError(PANCAKE_ERROR_CODES.PARSE_FAILED, `loadJsonFile() expected an object or vector at index ${i}`, { index: i });
|
|
88
|
+
}
|
|
89
|
+
if (!(vectorKey in row)) {
|
|
90
|
+
throw pancakeError(PANCAKE_ERROR_CODES.PARSE_FAILED, `loadJsonFile() missing vectorKey '${vectorKey}' at index ${i}`, { index: i, vectorKey });
|
|
91
|
+
}
|
|
92
|
+
const mapped = { vector: row[vectorKey] };
|
|
93
|
+
if (Object.prototype.hasOwnProperty.call(row, idKey)) {
|
|
94
|
+
mapped.id = row[idKey];
|
|
95
|
+
}
|
|
96
|
+
return mapped;
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function validateFilePath(filePath, helperName) {
|
|
101
|
+
if (typeof filePath !== 'string' || filePath.length === 0) {
|
|
102
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, `${helperName}() requires a non-empty file path`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function validateMaxFileBytes(maxFileBytes, helperName) {
|
|
107
|
+
if (!Number.isInteger(maxFileBytes) || maxFileBytes <= 0) {
|
|
108
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, `${helperName}() maxFileBytes must be a positive integer`, { maxFileBytes });
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function statRegularFile(filePath, helperName) {
|
|
113
|
+
let stat;
|
|
114
|
+
try {
|
|
115
|
+
stat = statSync(filePath);
|
|
116
|
+
} catch (error) {
|
|
117
|
+
const message = error && error.message ? error.message : String(error);
|
|
118
|
+
throw pancakeError(PANCAKE_ERROR_CODES.FILE_IO_FAILED, `${helperName}() could not stat ${filePath}: ${message}`, { filePath }, error);
|
|
119
|
+
}
|
|
120
|
+
if (!stat.isFile()) {
|
|
121
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, `${helperName}() expected a regular file: ${filePath}`, { filePath });
|
|
122
|
+
}
|
|
123
|
+
return stat;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function enforceFileSize(stat, maxFileBytes, helperName, filePath) {
|
|
127
|
+
if (stat.size > maxFileBytes) {
|
|
128
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, `${helperName}() file exceeds maxFileBytes (${stat.size} > ${maxFileBytes}): ${filePath}`, { filePath, fileBytes: stat.size, maxFileBytes });
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function readUtf8FileWithLimit(filePath, helperName, maxFileBytes) {
|
|
133
|
+
validateFilePath(filePath, helperName);
|
|
134
|
+
validateMaxFileBytes(maxFileBytes, helperName);
|
|
135
|
+
const stat = statRegularFile(filePath, helperName);
|
|
136
|
+
enforceFileSize(stat, maxFileBytes, helperName, filePath);
|
|
137
|
+
try {
|
|
138
|
+
return readFileSync(filePath, 'utf8');
|
|
139
|
+
} catch (error) {
|
|
140
|
+
const message = error && error.message ? error.message : String(error);
|
|
141
|
+
throw pancakeError(PANCAKE_ERROR_CODES.FILE_IO_FAILED, `${helperName}() could not read ${filePath}: ${message}`, { filePath }, error);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function readBinaryFileWithLimit(filePath, helperName, maxFileBytes) {
|
|
146
|
+
validateFilePath(filePath, helperName);
|
|
147
|
+
validateMaxFileBytes(maxFileBytes, helperName);
|
|
148
|
+
const stat = statRegularFile(filePath, helperName);
|
|
149
|
+
enforceFileSize(stat, maxFileBytes, helperName, filePath);
|
|
150
|
+
try {
|
|
151
|
+
return readFileSync(filePath);
|
|
152
|
+
} catch (error) {
|
|
153
|
+
const message = error && error.message ? error.message : String(error);
|
|
154
|
+
throw pancakeError(PANCAKE_ERROR_CODES.FILE_IO_FAILED, `${helperName}() could not read ${filePath}: ${message}`, { filePath }, error);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function validateSnapshotBytes(snapshot, filePath) {
|
|
159
|
+
if (!snapshot || snapshot.byteLength < 4) {
|
|
160
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID, `loadSnapshotFile() snapshot is too small to be valid: ${filePath}`, { filePath });
|
|
161
|
+
}
|
|
162
|
+
const view = new DataView(snapshot.buffer, snapshot.byteOffset, snapshot.byteLength);
|
|
163
|
+
const magic = view.getUint32(0, true);
|
|
164
|
+
if (!SUPPORTED_SNAPSHOT_MAGICS.has(magic)) {
|
|
165
|
+
throw pancakeError(PANCAKE_ERROR_CODES.SNAPSHOT_INVALID, `loadSnapshotFile() unsupported snapshot file type: ${filePath}`, { filePath, magic });
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
async function loadNodeEngine() {
|
|
170
|
+
const engineVariant = await selectEngineVariant();
|
|
171
|
+
|
|
172
|
+
if (engineVariant === 'scalar') {
|
|
173
|
+
try {
|
|
174
|
+
return await moduleLoader.instantiate(loadScalarEngine, 'scalar');
|
|
175
|
+
} catch (error) {
|
|
176
|
+
throw makeLoadError('Pancake failed to load the scalar WASM engine', error);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
try {
|
|
181
|
+
return await moduleLoader.instantiate(loadEngine, 'simd');
|
|
182
|
+
} catch (simdError) {
|
|
183
|
+
try {
|
|
184
|
+
const engine = await moduleLoader.instantiate(loadScalarEngine, 'scalar');
|
|
185
|
+
engineVariantPromise = Promise.resolve('scalar');
|
|
186
|
+
return engine;
|
|
187
|
+
} catch (scalarError) {
|
|
188
|
+
throw makeLoadError(
|
|
189
|
+
`Pancake failed to load either the SIMD or scalar WASM engine (SIMD error: ${simdError && simdError.message ? simdError.message : String(simdError)})`,
|
|
190
|
+
scalarError
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
const Pancake = createPancakeApi(loadNodeEngine);
|
|
197
|
+
export { PancakeError, PANCAKE_ERROR_CODES };
|
|
198
|
+
|
|
199
|
+
Pancake.RangeArtifact = PancakeRangeArtifact;
|
|
200
|
+
Pancake.NodeFileRangeSource = NodeFileRangeSource;
|
|
201
|
+
Pancake.buildRangeArtifact = buildRangeArtifact;
|
|
202
|
+
Pancake.buildRangeArtifactFile = buildRangeArtifactFile;
|
|
203
|
+
|
|
204
|
+
Pancake.openRangeArtifactFile = async function openRangeArtifactFile(filePath, opts) {
|
|
205
|
+
validateFilePath(filePath, 'openRangeArtifactFile');
|
|
206
|
+
return PancakeRangeArtifact.openFile(filePath, opts);
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
Pancake.SketchArtifact = PancakeSketchArtifact;
|
|
210
|
+
Pancake.createSketchScanner = (artifact, options) => createSketchScanner(loadNodeEngine, artifact, options);
|
|
211
|
+
Pancake.buildSketchArtifact = buildSketchArtifact;
|
|
212
|
+
Pancake.buildSketchArtifactBytes = buildSketchArtifactBytes;
|
|
213
|
+
Pancake.buildSketchArtifactFile = buildSketchArtifactFile;
|
|
214
|
+
|
|
215
|
+
Pancake.openSketchArtifactFile = async function openSketchArtifactFile(filePath, opts) {
|
|
216
|
+
validateFilePath(filePath, 'openSketchArtifactFile');
|
|
217
|
+
return PancakeSketchArtifact.openFile(filePath, opts);
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
Pancake.loadSnapshotFile = async function loadSnapshotFile(filePath, opts) {
|
|
221
|
+
const {
|
|
222
|
+
maxFileBytes = DEFAULT_SNAPSHOT_FILE_BYTES,
|
|
223
|
+
...createOpts
|
|
224
|
+
} = opts || {};
|
|
225
|
+
|
|
226
|
+
const snapshot = readBinaryFileWithLimit(filePath, 'loadSnapshotFile', maxFileBytes);
|
|
227
|
+
validateSnapshotBytes(snapshot, filePath);
|
|
228
|
+
return Pancake.restore(snapshot, createOpts);
|
|
229
|
+
};
|
|
230
|
+
|
|
231
|
+
Pancake.loadJsonFile = async function loadJsonFile(filePath, opts = {}) {
|
|
232
|
+
const {
|
|
233
|
+
format = inferJsonFormat(filePath),
|
|
234
|
+
vectorKey = 'vector',
|
|
235
|
+
idKey = 'id',
|
|
236
|
+
maxFileBytes = DEFAULT_JSON_FILE_BYTES,
|
|
237
|
+
...createOpts
|
|
238
|
+
} = opts;
|
|
239
|
+
|
|
240
|
+
if (format !== 'json' && format !== 'jsonl') {
|
|
241
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, `loadJsonFile() format must be 'json' or 'jsonl', got '${format}'`, { format });
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const text = readUtf8FileWithLimit(filePath, 'loadJsonFile', maxFileBytes);
|
|
245
|
+
let rows;
|
|
246
|
+
if (format === 'jsonl') {
|
|
247
|
+
rows = parseJsonLines(text, filePath);
|
|
248
|
+
} else {
|
|
249
|
+
try {
|
|
250
|
+
rows = JSON.parse(text);
|
|
251
|
+
} catch (error) {
|
|
252
|
+
const message = error && error.message ? error.message : String(error);
|
|
253
|
+
throw pancakeError(PANCAKE_ERROR_CODES.PARSE_FAILED, `Failed to parse JSON in ${filePath}: ${message}`, { filePath }, error);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
return Pancake.fromVectors(remapJsonRows(rows, vectorKey, idKey), createOpts);
|
|
257
|
+
};
|
|
258
|
+
|
|
259
|
+
export default Pancake;
|
package/pancake.web.d.ts
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// Types for the browser / Cloudflare Workers entrypoints (`pikelet-wasm/web`).
|
|
2
|
+
// These runtimes expose the portable API, including range-readable Search
|
|
3
|
+
// Artifacts. Only PancakeError and PANCAKE_ERROR_CODES exist as named runtime
|
|
4
|
+
// exports; every other re-export below is type-only. Node-only helpers
|
|
5
|
+
// (NodeFileRangeSource, the build*/open*File functions, loadJsonFile /
|
|
6
|
+
// loadSnapshotFile) are absent from this surface so using them is a compile
|
|
7
|
+
// error rather than a runtime throw.
|
|
8
|
+
export { PancakeError, PANCAKE_ERROR_CODES } from './pancake.js';
|
|
9
|
+
export type {
|
|
10
|
+
Metric,
|
|
11
|
+
PancakeErrorCode,
|
|
12
|
+
VectorInput,
|
|
13
|
+
VectorRecord,
|
|
14
|
+
CreateOptions,
|
|
15
|
+
SearchOptions,
|
|
16
|
+
FromVectorsResult,
|
|
17
|
+
RestoreOptions,
|
|
18
|
+
SnapshotFormat,
|
|
19
|
+
SnapshotInspection,
|
|
20
|
+
SearchResult,
|
|
21
|
+
MemoryUsage,
|
|
22
|
+
ResolvedConfig,
|
|
23
|
+
PancakeIndex,
|
|
24
|
+
RangeReadSource,
|
|
25
|
+
RangeArtifactSearchOptions,
|
|
26
|
+
RangeArtifactNode,
|
|
27
|
+
RangeArtifactStats,
|
|
28
|
+
RangeArtifactRound,
|
|
29
|
+
RangeArtifactSearchResult,
|
|
30
|
+
RangeArtifactOpenOptions,
|
|
31
|
+
PancakeRangeArtifact,
|
|
32
|
+
PancakeRangeArtifactConstructor,
|
|
33
|
+
SketchTier,
|
|
34
|
+
SketchStageEvent,
|
|
35
|
+
SketchArtifactOpenOptions,
|
|
36
|
+
SketchScanner,
|
|
37
|
+
SketchScannerOptions,
|
|
38
|
+
SketchArtifactSearchOptions,
|
|
39
|
+
SketchArtifactSearchResult,
|
|
40
|
+
SketchArtifactStats,
|
|
41
|
+
PancakeSketchArtifact,
|
|
42
|
+
PancakeSketchArtifactConstructor,
|
|
43
|
+
PancakeApi,
|
|
44
|
+
} from './pancake.js';
|
|
45
|
+
|
|
46
|
+
import type { PancakeApi } from './pancake.js';
|
|
47
|
+
|
|
48
|
+
declare const Pancake: PancakeApi;
|
|
49
|
+
|
|
50
|
+
export default Pancake;
|
package/pancake.web.mjs
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import loadEngine from './dist/engine.js';
|
|
2
|
+
import loadScalarEngine from './dist/engine.scalar.js';
|
|
3
|
+
import simdWasmAsset from './dist/engine.wasm?url';
|
|
4
|
+
import scalarWasmAsset from './dist/engine.scalar.wasm?url';
|
|
5
|
+
import createPancakeApi from './pancake-core.js';
|
|
6
|
+
import errorContract from './pancake-errors.js';
|
|
7
|
+
import loaderContract from './pancake-loader.js';
|
|
8
|
+
import artifactContract from './pancake-artifact.js';
|
|
9
|
+
const { PancakeError, PANCAKE_ERROR_CODES, pancakeError } = errorContract;
|
|
10
|
+
const { createCachedModuleLoader } = loaderContract;
|
|
11
|
+
const { PancakeRangeArtifact, PancakeSketchArtifact, createSketchScanner } = artifactContract;
|
|
12
|
+
let engineVariantPromise = null;
|
|
13
|
+
|
|
14
|
+
function makeLoadError(message, error) {
|
|
15
|
+
const detail = error && error.message ? error.message : String(error);
|
|
16
|
+
return pancakeError(PANCAKE_ERROR_CODES.WASM_LOAD_FAILED, `${message}: ${detail}`, undefined, error);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function isUrlLikeWasmAsset(asset) {
|
|
20
|
+
return typeof asset === 'string' || asset instanceof URL;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
async function loadWasmAsset(asset) {
|
|
24
|
+
if (!isUrlLikeWasmAsset(asset)) return asset;
|
|
25
|
+
const href = asset instanceof URL ? asset.href : asset;
|
|
26
|
+
const response = await fetch(href, { credentials: 'same-origin' });
|
|
27
|
+
if (!response.ok) {
|
|
28
|
+
throw pancakeError(PANCAKE_ERROR_CODES.WASM_LOAD_FAILED,
|
|
29
|
+
`${response.status} ${response.statusText}`.trim(), { status: response.status });
|
|
30
|
+
}
|
|
31
|
+
return response.arrayBuffer();
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const moduleLoader = createCachedModuleLoader((variant) =>
|
|
35
|
+
loadWasmAsset(variant === 'simd' ? simdWasmAsset : scalarWasmAsset)
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
function selectEngineVariant() {
|
|
39
|
+
engineVariantPromise ??= moduleLoader.supports('simd')
|
|
40
|
+
.then((supported) => supported ? 'simd' : 'scalar')
|
|
41
|
+
.catch((error) => { throw makeLoadError('Failed to probe Pancake SIMD WASM support', error); });
|
|
42
|
+
return engineVariantPromise;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async function loadWebEngine() {
|
|
46
|
+
const engineVariant = await selectEngineVariant();
|
|
47
|
+
|
|
48
|
+
if (engineVariant === 'scalar') {
|
|
49
|
+
try {
|
|
50
|
+
return await moduleLoader.instantiate(loadScalarEngine, 'scalar');
|
|
51
|
+
} catch (error) {
|
|
52
|
+
throw makeLoadError('Pancake failed to load the scalar WASM engine', error);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
try {
|
|
57
|
+
return await moduleLoader.instantiate(loadEngine, 'simd');
|
|
58
|
+
} catch (simdError) {
|
|
59
|
+
try {
|
|
60
|
+
const engine = await moduleLoader.instantiate(loadScalarEngine, 'scalar');
|
|
61
|
+
engineVariantPromise = Promise.resolve('scalar');
|
|
62
|
+
return engine;
|
|
63
|
+
} catch (scalarError) {
|
|
64
|
+
throw makeLoadError(
|
|
65
|
+
`Pancake failed to load either the SIMD or scalar WASM engine (SIMD error: ${simdError && simdError.message ? simdError.message : String(simdError)})`,
|
|
66
|
+
scalarError
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const Pancake = createPancakeApi(loadWebEngine);
|
|
73
|
+
export { PancakeError, PANCAKE_ERROR_CODES };
|
|
74
|
+
|
|
75
|
+
Pancake.RangeArtifact = PancakeRangeArtifact;
|
|
76
|
+
Pancake.SketchArtifact = PancakeSketchArtifact;
|
|
77
|
+
Pancake.createSketchScanner = (artifact, options) => createSketchScanner(loadWebEngine, artifact, options);
|
|
78
|
+
|
|
79
|
+
function unsupportedNodeFileHelper(name) {
|
|
80
|
+
return async function unsupported() {
|
|
81
|
+
throw pancakeError(PANCAKE_ERROR_CODES.INVALID_ARGUMENT, `${name}() is only available in the Node.js entrypoints`);
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
Pancake.loadSnapshotFile = unsupportedNodeFileHelper('loadSnapshotFile');
|
|
86
|
+
Pancake.loadJsonFile = unsupportedNodeFileHelper('loadJsonFile');
|
|
87
|
+
|
|
88
|
+
export default Pancake;
|