pikelet-wasm 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +399 -2
- package/README.md +81 -257
- package/complete/builder.mjs +35 -5
- package/complete/format.mjs +6 -1
- package/complete/fusion.mjs +72 -0
- package/complete/index.d.ts +23 -0
- package/complete/index.mjs +580 -76
- package/complete/inline-transformer.mjs +97 -1
- package/complete/layer-append.mjs +246 -0
- package/complete/layer-chain.mjs +284 -0
- package/complete/layer-compact.mjs +319 -0
- package/complete/layer-encoder.mjs +223 -0
- package/complete/layer-locator.mjs +228 -0
- package/complete/layer-manifest.mjs +271 -0
- package/complete/layer-reader.mjs +759 -0
- package/complete/layer-rebase.mjs +218 -0
- package/complete/lexical.mjs +78 -8
- package/complete/retrieval-abstention.mjs +132 -28
- package/complete/sources.mjs +1 -1
- package/complete/tombstones.mjs +281 -0
- package/complete/wordpiece.mjs +22 -0
- package/dist/engine.js +1 -1
- package/dist/engine.scalar.js +1 -1
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +65 -77
- package/{pikelet-artifact-common.js → src/artifact/pikelet-artifact-common.js} +1 -1
- package/{pikelet-artifact-range.js → src/artifact/pikelet-artifact-range.js} +8 -1
- package/{pikelet-artifact-sketch.js → src/artifact/pikelet-artifact-sketch.js} +97 -8
- package/{pikelet-artifact.d.ts → src/artifact/pikelet-artifact.d.ts} +1 -1
- package/{pikelet-core.js → src/core/pikelet-core.js} +6 -5
- package/{pikelet.js → src/pikelet.js} +7 -7
- package/{pikelet.node.mjs → src/pikelet.node.mjs} +7 -7
- package/{pikelet.web.mjs → src/pikelet.web.mjs} +8 -8
- package/{pikelet.workerd.mjs → src/pikelet.workerd.mjs} +8 -8
- /package/{pikelet-artifact.js → src/artifact/pikelet-artifact.js} +0 -0
- /package/{pikelet-errors.js → src/errors/pikelet-errors.js} +0 -0
- /package/{pikelet-loader.js → src/loader/pikelet-loader.js} +0 -0
- /package/{pikelet.d.ts → src/pikelet.d.ts} +0 -0
- /package/{pikelet.web.d.ts → src/pikelet.web.d.ts} +0 -0
package/package.json
CHANGED
|
@@ -1,67 +1,67 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pikelet-wasm",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Vector search engine and .pikelet artifact readers/builders, in WebAssembly. Compile a corpus into one self-contained, hash-verified search file, or use the underlying HNSW engine directly. Runs in Node.js, browsers, and Cloudflare Workers with no native dependencies.",
|
|
5
|
-
"main": "pikelet.js",
|
|
6
|
-
"types": "pikelet.d.ts",
|
|
5
|
+
"main": "src/pikelet.js",
|
|
6
|
+
"types": "src/pikelet.d.ts",
|
|
7
7
|
"exports": {
|
|
8
8
|
".": {
|
|
9
9
|
"workerd": {
|
|
10
|
-
"types": "./pikelet.web.d.ts",
|
|
11
|
-
"default": "./pikelet.workerd.mjs"
|
|
10
|
+
"types": "./src/pikelet.web.d.ts",
|
|
11
|
+
"default": "./src/pikelet.workerd.mjs"
|
|
12
12
|
},
|
|
13
13
|
"browser": {
|
|
14
|
-
"types": "./pikelet.web.d.ts",
|
|
15
|
-
"default": "./pikelet.web.mjs"
|
|
14
|
+
"types": "./src/pikelet.web.d.ts",
|
|
15
|
+
"default": "./src/pikelet.web.mjs"
|
|
16
16
|
},
|
|
17
17
|
"import": {
|
|
18
|
-
"types": "./pikelet.d.ts",
|
|
19
|
-
"default": "./pikelet.node.mjs"
|
|
18
|
+
"types": "./src/pikelet.d.ts",
|
|
19
|
+
"default": "./src/pikelet.node.mjs"
|
|
20
20
|
},
|
|
21
21
|
"require": {
|
|
22
|
-
"types": "./pikelet.d.ts",
|
|
23
|
-
"default": "./pikelet.js"
|
|
22
|
+
"types": "./src/pikelet.d.ts",
|
|
23
|
+
"default": "./src/pikelet.js"
|
|
24
24
|
},
|
|
25
25
|
"default": {
|
|
26
|
-
"types": "./pikelet.d.ts",
|
|
27
|
-
"default": "./pikelet.js"
|
|
26
|
+
"types": "./src/pikelet.d.ts",
|
|
27
|
+
"default": "./src/pikelet.js"
|
|
28
28
|
}
|
|
29
29
|
},
|
|
30
30
|
"./web": {
|
|
31
31
|
"workerd": {
|
|
32
|
-
"types": "./pikelet.web.d.ts",
|
|
33
|
-
"default": "./pikelet.workerd.mjs"
|
|
32
|
+
"types": "./src/pikelet.web.d.ts",
|
|
33
|
+
"default": "./src/pikelet.workerd.mjs"
|
|
34
34
|
},
|
|
35
35
|
"browser": {
|
|
36
|
-
"types": "./pikelet.web.d.ts",
|
|
37
|
-
"default": "./pikelet.web.mjs"
|
|
36
|
+
"types": "./src/pikelet.web.d.ts",
|
|
37
|
+
"default": "./src/pikelet.web.mjs"
|
|
38
38
|
},
|
|
39
39
|
"import": {
|
|
40
|
-
"types": "./pikelet.web.d.ts",
|
|
41
|
-
"default": "./pikelet.web.mjs"
|
|
40
|
+
"types": "./src/pikelet.web.d.ts",
|
|
41
|
+
"default": "./src/pikelet.web.mjs"
|
|
42
42
|
},
|
|
43
43
|
"default": {
|
|
44
|
-
"types": "./pikelet.web.d.ts",
|
|
45
|
-
"default": "./pikelet.web.mjs"
|
|
44
|
+
"types": "./src/pikelet.web.d.ts",
|
|
45
|
+
"default": "./src/pikelet.web.mjs"
|
|
46
46
|
}
|
|
47
47
|
},
|
|
48
48
|
"./node": {
|
|
49
49
|
"import": {
|
|
50
|
-
"types": "./pikelet.d.ts",
|
|
51
|
-
"default": "./pikelet.node.mjs"
|
|
50
|
+
"types": "./src/pikelet.d.ts",
|
|
51
|
+
"default": "./src/pikelet.node.mjs"
|
|
52
52
|
},
|
|
53
53
|
"require": {
|
|
54
|
-
"types": "./pikelet.d.ts",
|
|
55
|
-
"default": "./pikelet.js"
|
|
54
|
+
"types": "./src/pikelet.d.ts",
|
|
55
|
+
"default": "./src/pikelet.js"
|
|
56
56
|
},
|
|
57
57
|
"default": {
|
|
58
|
-
"types": "./pikelet.d.ts",
|
|
59
|
-
"default": "./pikelet.js"
|
|
58
|
+
"types": "./src/pikelet.d.ts",
|
|
59
|
+
"default": "./src/pikelet.js"
|
|
60
60
|
}
|
|
61
61
|
},
|
|
62
62
|
"./artifact": {
|
|
63
|
-
"types": "./pikelet-artifact.d.ts",
|
|
64
|
-
"default": "./pikelet-artifact.js"
|
|
63
|
+
"types": "./src/artifact/pikelet-artifact.d.ts",
|
|
64
|
+
"default": "./src/artifact/pikelet-artifact.js"
|
|
65
65
|
},
|
|
66
66
|
"./complete": {
|
|
67
67
|
"types": "./complete/index.d.ts",
|
|
@@ -71,24 +71,41 @@
|
|
|
71
71
|
"types": "./complete/builder.d.ts",
|
|
72
72
|
"default": "./complete/builder.mjs"
|
|
73
73
|
},
|
|
74
|
+
"./complete/tombstones.mjs": {
|
|
75
|
+
"default": "./complete/tombstones.mjs"
|
|
76
|
+
},
|
|
77
|
+
"./complete/format.mjs": {
|
|
78
|
+
"default": "./complete/format.mjs"
|
|
79
|
+
},
|
|
80
|
+
"./complete/layer-manifest.mjs": {
|
|
81
|
+
"default": "./complete/layer-manifest.mjs"
|
|
82
|
+
},
|
|
83
|
+
"./complete/layer-encoder.mjs": {
|
|
84
|
+
"default": "./complete/layer-encoder.mjs"
|
|
85
|
+
},
|
|
86
|
+
"./complete/layer-locator.mjs": {
|
|
87
|
+
"default": "./complete/layer-locator.mjs"
|
|
88
|
+
},
|
|
89
|
+
"./complete/layer-chain.mjs": {
|
|
90
|
+
"default": "./complete/layer-chain.mjs"
|
|
91
|
+
},
|
|
92
|
+
"./complete/layer-reader.mjs": {
|
|
93
|
+
"default": "./complete/layer-reader.mjs"
|
|
94
|
+
},
|
|
95
|
+
"./complete/layer-append.mjs": {
|
|
96
|
+
"default": "./complete/layer-append.mjs"
|
|
97
|
+
},
|
|
98
|
+
"./complete/layer-rebase.mjs": {
|
|
99
|
+
"default": "./complete/layer-rebase.mjs"
|
|
100
|
+
},
|
|
101
|
+
"./complete/layer-compact.mjs": {
|
|
102
|
+
"default": "./complete/layer-compact.mjs"
|
|
103
|
+
},
|
|
74
104
|
"./engine.wasm": "./dist/engine.wasm"
|
|
75
105
|
},
|
|
76
106
|
"files": [
|
|
107
|
+
"src",
|
|
77
108
|
"complete",
|
|
78
|
-
"pikelet.js",
|
|
79
|
-
"pikelet-artifact.js",
|
|
80
|
-
"pikelet-artifact-common.js",
|
|
81
|
-
"pikelet-artifact-range.js",
|
|
82
|
-
"pikelet-artifact-sketch.js",
|
|
83
|
-
"pikelet.node.mjs",
|
|
84
|
-
"pikelet.web.mjs",
|
|
85
|
-
"pikelet.workerd.mjs",
|
|
86
|
-
"pikelet-core.js",
|
|
87
|
-
"pikelet-errors.js",
|
|
88
|
-
"pikelet-loader.js",
|
|
89
|
-
"pikelet.d.ts",
|
|
90
|
-
"pikelet.web.d.ts",
|
|
91
|
-
"pikelet-artifact.d.ts",
|
|
92
109
|
"dist/engine.js",
|
|
93
110
|
"dist/engine.wasm",
|
|
94
111
|
"dist/engine.scalar.js",
|
|
@@ -99,28 +116,9 @@
|
|
|
99
116
|
"NOTICE"
|
|
100
117
|
],
|
|
101
118
|
"scripts": {
|
|
102
|
-
"
|
|
103
|
-
"
|
|
104
|
-
"
|
|
105
|
-
"bench": "node benchmarks/run.js",
|
|
106
|
-
"demo:data": "node scripts/make-demo-vectors.mjs",
|
|
107
|
-
"demo:veyra": "node examples/05-one-file-search/web/veyra-corpus/fetch-veyra.mjs",
|
|
108
|
-
"demo:artifact": "node examples/legacy/range-artifact-demo/demo.js --compact",
|
|
109
|
-
"demo:sketch": "node examples/legacy/range-artifact-demo/sketch_demo.js --compact",
|
|
110
|
-
"demo:artifact:browser:build": "vite build examples/legacy/range-artifact-demo/static",
|
|
111
|
-
"demo:artifact:browser": "vite preview examples/legacy/range-artifact-demo/static --host 127.0.0.1",
|
|
112
|
-
"build:pages-demo": "node examples/legacy/03-edge-docs-search/pages-ui/build.mjs",
|
|
113
|
-
"test": "node run_tests.js && node test/model_c_loader.js && node test/sketch_profile.js && node test/sketch_staged.js && node test/sketch_row_integrity.js && node test/artifact_hardening.js && node test/complete_profile.mjs && node test/mcp_conformance.mjs && node test/ingestion/run.mjs && npm run test:types",
|
|
114
|
-
"test:types": "tsc -p test/types/tsconfig.json",
|
|
115
|
-
"test:browser": "node test/browser_smoke.js",
|
|
116
|
-
"test:worker-web": "node test/test_worker_web_entry.js",
|
|
117
|
-
"test:simd": "node test/simd_parity.js",
|
|
118
|
-
"test:fuzz": "node test/fuzz_import.js",
|
|
119
|
-
"test:worker-example": "cd examples/legacy/03-edge-docs-search && npx wrangler deploy --dry-run --outdir ../../../.tmp-test-work/student-worker-distilled && cd ../../.. && node examples/legacy/03-edge-docs-search/test_worker.mjs",
|
|
120
|
-
"test:scaffold": "node scripts/test-scaffold-e2e.mjs",
|
|
121
|
-
"bench:range-storage": "node scripts/benchmark-range-storage.mjs",
|
|
122
|
-
"pack:dry-run": "node scripts/pack-dry-run.mjs",
|
|
123
|
-
"prepublishOnly": "npm run build:all && npm test && npm run test:fuzz && npm run test:simd && npm run test:browser && npm run test:worker-web && npm run test:worker-example && npm run pack:dry-run"
|
|
119
|
+
"prepack": "node ../../scripts/stage-package-docs.mjs pikelet-wasm",
|
|
120
|
+
"postpack": "node ../../scripts/stage-package-docs.mjs pikelet-wasm --clean",
|
|
121
|
+
"prepublishOnly": "npm --prefix ../.. run prepublish-checks"
|
|
124
122
|
},
|
|
125
123
|
"keywords": [
|
|
126
124
|
"vector-search",
|
|
@@ -159,17 +157,7 @@
|
|
|
159
157
|
"access": "public"
|
|
160
158
|
},
|
|
161
159
|
"engines": {
|
|
162
|
-
"node": ">=
|
|
163
|
-
},
|
|
164
|
-
"sideEffects": false,
|
|
165
|
-
"devDependencies": {
|
|
166
|
-
"miniflare": "^4.20260409.0",
|
|
167
|
-
"playwright": "^1.60.0",
|
|
168
|
-
"typescript": "^7.0.2",
|
|
169
|
-
"vite": "^8.0.16",
|
|
170
|
-
"wrangler": "^4.81.1"
|
|
160
|
+
"node": ">=18"
|
|
171
161
|
},
|
|
172
|
-
"
|
|
173
|
-
"esbuild": "^0.28.1"
|
|
174
|
-
}
|
|
162
|
+
"sideEffects": false
|
|
175
163
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// Split out of pikelet-artifact.js (the public entry, which re-exports the
|
|
6
6
|
// three parts); see that file for the module map.
|
|
7
7
|
|
|
8
|
-
const { pikeletError, PIKELET_ERROR_CODES } = require('
|
|
8
|
+
const { pikeletError, PIKELET_ERROR_CODES } = require('../errors/pikelet-errors.js');
|
|
9
9
|
|
|
10
10
|
const PIKELET_MAGIC = 0x504E434B;
|
|
11
11
|
const V1_ENVELOPE_HEADER_SIZE = 24;
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// Split out of pikelet-artifact.js (the public entry, which re-exports the
|
|
5
5
|
// three parts); see that file for the module map.
|
|
6
6
|
|
|
7
|
-
const { pikeletError, PIKELET_ERROR_CODES } = require('
|
|
7
|
+
const { pikeletError, PIKELET_ERROR_CODES } = require('../errors/pikelet-errors.js');
|
|
8
8
|
const {
|
|
9
9
|
MAX_COALESCED_RANGE_BYTES,
|
|
10
10
|
readChecked,
|
|
@@ -376,6 +376,13 @@ class PikeletRangeArtifact {
|
|
|
376
376
|
state.offset += this.dim;
|
|
377
377
|
const scale = readF32(view, state);
|
|
378
378
|
const offset = readF32(view, state);
|
|
379
|
+
// A NaN/Infinity affine pair makes every distance to this node
|
|
380
|
+
// non-finite; NaN compares false everywhere, so the node can sit in
|
|
381
|
+
// the result heap without ever being displaced. Digests do not catch
|
|
382
|
+
// this (the producer signs whatever bytes it wrote), so decode does.
|
|
383
|
+
if (!Number.isFinite(scale) || !Number.isFinite(offset)) {
|
|
384
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Range artifact record has a non-finite scale or offset', { id, scale, offset });
|
|
385
|
+
}
|
|
379
386
|
const base = new Uint32Array(baseCount);
|
|
380
387
|
for (let i = 0; i < this.M0; i++) {
|
|
381
388
|
const neighbor = readU32(view, state);
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// Split out of pikelet-artifact.js (the public entry, which re-exports the
|
|
6
6
|
// three parts); see that file for the module map.
|
|
7
7
|
|
|
8
|
-
const { pikeletError, PIKELET_ERROR_CODES } = require('
|
|
8
|
+
const { pikeletError, PIKELET_ERROR_CODES } = require('../errors/pikelet-errors.js');
|
|
9
9
|
const {
|
|
10
10
|
DEFAULT_OPEN_READ_BYTES,
|
|
11
11
|
MAX_COALESCED_RANGE_BYTES,
|
|
@@ -38,6 +38,19 @@ const SKETCH_MAGIC = 0x31415350; // PSA1
|
|
|
38
38
|
const SKETCH_HEADER_BYTES = 256;
|
|
39
39
|
const SKETCH_KIND_U8 = 1;
|
|
40
40
|
|
|
41
|
+
// The affine table feeds every rerank distance; a NaN/Infinity entry makes
|
|
42
|
+
// that row's distance non-finite and lets it sit in the result set
|
|
43
|
+
// unchallenged (NaN never loses a comparison). The digests do not catch
|
|
44
|
+
// this, since the producer signs whatever bytes it wrote, so every path
|
|
45
|
+
// that adopts the table checks it.
|
|
46
|
+
function assertFiniteAffine(scales, offsets) {
|
|
47
|
+
for (let i = 0; i < scales.length; i++) {
|
|
48
|
+
if (!Number.isFinite(scales[i]) || !Number.isFinite(offsets[i])) {
|
|
49
|
+
throw pikeletError(PIKELET_ERROR_CODES.SNAPSHOT_INVALID, 'Sketch artifact affine table has a non-finite scale or offset', { id: i });
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
41
54
|
function buildSketchArtifact(snapshotBytes, outPath, options = {}) {
|
|
42
55
|
if (typeof outPath !== 'string' || outPath.length === 0) {
|
|
43
56
|
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'buildSketchArtifact() requires an output path');
|
|
@@ -486,6 +499,7 @@ class PikeletSketchArtifact {
|
|
|
486
499
|
}
|
|
487
500
|
artifact.scales = new Float32Array(affineCopy.buffer, 0, count);
|
|
488
501
|
artifact.offsets = new Float32Array(affineCopy.buffer, count * 4, count);
|
|
502
|
+
assertFiniteAffine(artifact.scales, artifact.offsets);
|
|
489
503
|
artifact.sketches = null;
|
|
490
504
|
artifact.microSketches = microCopy;
|
|
491
505
|
if (pageTableCopy) artifact.pageTable = pageTableCopy;
|
|
@@ -514,8 +528,21 @@ class PikeletSketchArtifact {
|
|
|
514
528
|
}
|
|
515
529
|
|
|
516
530
|
_adoptResident(residentCopy, count, sketchRowBytes, microRowBytes, microOffset) {
|
|
517
|
-
|
|
518
|
-
this.offsets
|
|
531
|
+
// Validate into locals BEFORE touching a single member. Assigning
|
|
532
|
+
// this.scales/this.offsets first and validating after left a rejected
|
|
533
|
+
// stage-2 adopt half-applied: this.sketches still null, this.tier
|
|
534
|
+
// still 'micro', this.microSketches still pointing at the STAGE-1
|
|
535
|
+
// buffer -- while scales/offsets had already moved to stage-2. The
|
|
536
|
+
// staged path swallows the rejection (see fullyResident.catch above),
|
|
537
|
+
// so the artifact stayed open and kept serving micro-tier queries
|
|
538
|
+
// that paired stage-1 sketches against stage-2 affine parameters:
|
|
539
|
+
// silently wrong ranking, which is worse than the non-finite
|
|
540
|
+
// distances this check exists to reject.
|
|
541
|
+
const scales = new Float32Array(residentCopy.buffer, 0, count);
|
|
542
|
+
const offsets = new Float32Array(residentCopy.buffer, count * 4, count);
|
|
543
|
+
assertFiniteAffine(scales, offsets);
|
|
544
|
+
this.scales = scales;
|
|
545
|
+
this.offsets = offsets;
|
|
519
546
|
this.sketches = new Uint8Array(residentCopy.buffer, count * 8, count * sketchRowBytes);
|
|
520
547
|
if (this.microDims) {
|
|
521
548
|
this.microSketches = new Uint8Array(residentCopy.buffer, microOffset - SKETCH_HEADER_BYTES, count * microRowBytes);
|
|
@@ -801,7 +828,34 @@ class PikeletSketchArtifact {
|
|
|
801
828
|
const base = resolveOptionalPositiveInt(options.rerank, 'rerank', 0);
|
|
802
829
|
const boost = Math.max(1, Math.trunc(options.microBoost || 4));
|
|
803
830
|
const rec = this.recommendedRerank || Math.max(100, k * 10);
|
|
804
|
-
|
|
831
|
+
// options.exclude: a LSB-first bitset over local rows, one bit per row,
|
|
832
|
+
// supplied by a layered chain (LAYERED_PROFILE.md 5.3). A masked row
|
|
833
|
+
// must not enter the candidate heap, must not count toward C, and must
|
|
834
|
+
// not be fetched for reranking. Masking AFTER truncation is a defect,
|
|
835
|
+
// not a simplification: if the C nearest rows are all masked and the
|
|
836
|
+
// nearest live row is the C+1-th, a post-hoc mask returns nothing while
|
|
837
|
+
// the correct answer exists.
|
|
838
|
+
const exclude = options.exclude || null;
|
|
839
|
+
if (exclude !== null) {
|
|
840
|
+
if (!(exclude instanceof Uint8Array)) {
|
|
841
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'search() exclude must be a Uint8Array bitset');
|
|
842
|
+
}
|
|
843
|
+
if (exclude.length < Math.ceil(this.count / 8)) {
|
|
844
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'search() exclude bitset is too short for this artifact', { bytes: exclude.length, needed: Math.ceil(this.count / 8) });
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
const isExcluded = exclude === null
|
|
848
|
+
? () => false
|
|
849
|
+
: (row) => ((exclude[row >> 3] >> (row & 7)) & 1) === 1;
|
|
850
|
+
// C counts unmasked rows, so a heavily tombstoned tier still offers a
|
|
851
|
+
// full candidate pool over what is live.
|
|
852
|
+
let liveCountForC = this.count;
|
|
853
|
+
if (exclude !== null) {
|
|
854
|
+
liveCountForC = 0;
|
|
855
|
+
for (let i = 0; i < this.count; i++) if (!isExcluded(i)) liveCountForC++;
|
|
856
|
+
}
|
|
857
|
+
if (liveCountForC === 0) return { results: [], stats: this.stats() };
|
|
858
|
+
const C = Math.min(liveCountForC, Math.max(k, base > 0 ? base : tier.name === 'micro' ? rec * boost : rec));
|
|
805
859
|
const { dim, count } = this;
|
|
806
860
|
const tierDims = tier.dims;
|
|
807
861
|
const pool = dim / tierDims;
|
|
@@ -832,6 +886,27 @@ class PikeletSketchArtifact {
|
|
|
832
886
|
// C beyond them would silently shrink the candidate pool — recall
|
|
833
887
|
// loss, not an error. Fall back to the JS scan for that query.
|
|
834
888
|
if (scanner && scanner.maxRerank !== undefined && C > scanner.maxRerank) scanner = null;
|
|
889
|
+
// A scan kernel scores every resident row and knows nothing about an
|
|
890
|
+
// exclusion mask. Masking its top-C output would be the post-truncation
|
|
891
|
+
// defect 5.3 forbids — if the C nearest rows are all masked, a post-hoc
|
|
892
|
+
// filter returns nothing while the correct answer exists.
|
|
893
|
+
//
|
|
894
|
+
// But the kernel CAN be used safely by over-fetching: ask it for
|
|
895
|
+
// C + (number of masked rows) candidates, then drop the masked ones.
|
|
896
|
+
// At least C live rows survive by construction, because at most
|
|
897
|
+
// maskedCount of the returned rows can be masked. That is not a
|
|
898
|
+
// post-hoc mask on a truncated list; it is an exact selection over a
|
|
899
|
+
// provably sufficient superset.
|
|
900
|
+
//
|
|
901
|
+
// When the over-fetch would exceed the kernel's buffers or the row
|
|
902
|
+
// count, fall back to the masked JS scan, which is always correct.
|
|
903
|
+
let scannerOverfetch = 0;
|
|
904
|
+
if (scanner && exclude !== null) {
|
|
905
|
+
const maskedCount = this.count - liveCountForC;
|
|
906
|
+
scannerOverfetch = C + maskedCount;
|
|
907
|
+
const limit = Math.min(count, scanner.maxRerank ?? count);
|
|
908
|
+
if (scannerOverfetch > limit) { scanner = null; scannerOverfetch = 0; }
|
|
909
|
+
}
|
|
835
910
|
if (scanner) {
|
|
836
911
|
// A scanner scores the resident sketches itself, so it must
|
|
837
912
|
// implement this artifact's metric. Cosine requires an explicit
|
|
@@ -843,18 +918,32 @@ class PikeletSketchArtifact {
|
|
|
843
918
|
if (this.metric === 1 && scannerMetric !== 1) {
|
|
844
919
|
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'cosine sketch artifacts require a metric-aware scanner (scanner.metric === 1)', { metric: this.metric });
|
|
845
920
|
}
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
921
|
+
// Over-fetch when a mask is in play (see above), then drop the
|
|
922
|
+
// masked rows. The survivors are the exact top-C live rows.
|
|
923
|
+
ids = scanner.scan(qPool, scannerOverfetch || C);
|
|
924
|
+
if (exclude !== null) {
|
|
925
|
+
const live = [];
|
|
926
|
+
for (const id of ids) {
|
|
927
|
+
if (!isExcluded(id)) live.push(id);
|
|
928
|
+
if (live.length === C) break;
|
|
929
|
+
}
|
|
930
|
+
ids = live;
|
|
931
|
+
}
|
|
932
|
+
} else if (C >= liveCountForC) {
|
|
933
|
+
// Every live row is a candidate: the exact rerank below scores them
|
|
849
934
|
// all, so the resident scan has nothing to select. Without this
|
|
850
935
|
// the selection loop degenerates to 2*count^2 slot operations
|
|
851
936
|
// (candMax stays Infinity until the last slot fills).
|
|
852
|
-
ids =
|
|
937
|
+
ids = [];
|
|
938
|
+
for (let i = 0; i < count; i++) if (!isExcluded(i)) ids.push(i);
|
|
853
939
|
} else {
|
|
854
940
|
const candDist = new Float64Array(C).fill(Infinity);
|
|
855
941
|
const candId = new Int32Array(C).fill(-1);
|
|
856
942
|
let candMax = Infinity;
|
|
857
943
|
for (let i = 0; i < count; i++) {
|
|
944
|
+
// Skipped before scoring, so a masked row never occupies a
|
|
945
|
+
// candidate slot (5.3).
|
|
946
|
+
if (isExcluded(i)) continue;
|
|
858
947
|
const s = this.scales[i];
|
|
859
948
|
const o = this.offsets[i];
|
|
860
949
|
let metricAcc = 0;
|
|
@@ -13,7 +13,7 @@ import type {
|
|
|
13
13
|
SketchArtifactBuildOptions,
|
|
14
14
|
SketchScanner,
|
|
15
15
|
SketchScannerOptions,
|
|
16
|
-
} from '
|
|
16
|
+
} from '../pikelet.js';
|
|
17
17
|
|
|
18
18
|
/** A parsed uint8 engine snapshot, as returned by {@link parseUint8Snapshot}. */
|
|
19
19
|
export interface Uint8SnapshotGraph {
|
|
@@ -4,7 +4,7 @@ const {
|
|
|
4
4
|
PikeletError,
|
|
5
5
|
PIKELET_ERROR_CODES,
|
|
6
6
|
pikeletError,
|
|
7
|
-
} = require('
|
|
7
|
+
} = require('../errors/pikelet-errors.js');
|
|
8
8
|
|
|
9
9
|
// Envelope header for validated export/import
|
|
10
10
|
const PIKELET_MAGIC = 0x504E434B; // "PNCK"
|
|
@@ -856,9 +856,10 @@ function createPikeletApi(loadEngineImpl) {
|
|
|
856
856
|
async function create(opts) {
|
|
857
857
|
if (!opts || !opts.dim) throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
858
858
|
'opts.dim is required', { argument: 'dim' });
|
|
859
|
-
|
|
859
|
+
// Upper bound matches the engine constructors' MAX_DIMS.
|
|
860
|
+
if (!Number.isInteger(opts.dim) || opts.dim <= 0 || opts.dim > 65536) {
|
|
860
861
|
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
861
|
-
'opts.dim must be
|
|
862
|
+
'opts.dim must be an integer between 1 and 65536', { argument: 'dim', value: opts.dim });
|
|
862
863
|
}
|
|
863
864
|
if ('compressed' in opts) {
|
|
864
865
|
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'opts.compressed has been removed');
|
|
@@ -870,7 +871,7 @@ function createPikeletApi(loadEngineImpl) {
|
|
|
870
871
|
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
871
872
|
"opts.metric must be 'cosine' or 'l2'", { argument: 'metric', value: opts.metric });
|
|
872
873
|
}
|
|
873
|
-
// Upper bound matches the engine ABI:
|
|
874
|
+
// Upper bound matches the engine ABI: pikelet_init takes max_elem as a
|
|
874
875
|
// C int, so anything above 2^31-1 would truncate or go negative.
|
|
875
876
|
if (opts.maxElements !== undefined && (!Number.isInteger(opts.maxElements) || opts.maxElements <= 0 || opts.maxElements > 0x7fffffff)) {
|
|
876
877
|
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT,
|
|
@@ -904,7 +905,7 @@ function createPikeletApi(loadEngineImpl) {
|
|
|
904
905
|
|
|
905
906
|
// Capacity guard: the backend allocates its arena eagerly at
|
|
906
907
|
// construction, and a request the wasm32 heap cannot satisfy throws
|
|
907
|
-
// std::bad_alloc inside
|
|
908
|
+
// std::bad_alloc inside pikelet_init — reject it here with the
|
|
908
909
|
// estimate instead. Per-element cost mirrors the constructor
|
|
909
910
|
// allocations (uint8_float_hnsw.hpp / float_hnsw.hpp): quantized rows
|
|
910
911
|
// cost dim bytes plus 16*M edge bytes plus ~39 bytes of bookkeeping;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
const loadEngine = require('
|
|
4
|
-
const loadScalarEngine = require('
|
|
5
|
-
const createPikeletApi = require('./pikelet-core.js');
|
|
6
|
-
const { PikeletError, PIKELET_ERROR_CODES, pikeletError } = require('./pikelet-errors.js');
|
|
7
|
-
const { PikeletRangeArtifact, PikeletSketchArtifact, createSketchScanner, NodeFileRangeSource, buildRangeArtifact, buildRangeArtifactFile, buildSketchArtifact, buildSketchArtifactBytes, buildSketchArtifactFile } = require('./pikelet-artifact.js');
|
|
8
|
-
const { createCachedModuleLoader } = require('./pikelet-loader.js');
|
|
3
|
+
const loadEngine = require('../dist/engine.js');
|
|
4
|
+
const loadScalarEngine = require('../dist/engine.scalar.js');
|
|
5
|
+
const createPikeletApi = require('./core/pikelet-core.js');
|
|
6
|
+
const { PikeletError, PIKELET_ERROR_CODES, pikeletError } = require('./errors/pikelet-errors.js');
|
|
7
|
+
const { PikeletRangeArtifact, PikeletSketchArtifact, createSketchScanner, NodeFileRangeSource, buildRangeArtifact, buildRangeArtifactFile, buildSketchArtifact, buildSketchArtifactBytes, buildSketchArtifactFile } = require('./artifact/pikelet-artifact.js');
|
|
8
|
+
const { createCachedModuleLoader } = require('./loader/pikelet-loader.js');
|
|
9
9
|
const _path = require('path');
|
|
10
10
|
const _fs = require('fs');
|
|
11
11
|
let _engineVariantPromise = null;
|
|
@@ -22,7 +22,7 @@ const SUPPORTED_SNAPSHOT_MAGICS = new Set([
|
|
|
22
22
|
|
|
23
23
|
function readWasmBinary(fileName) {
|
|
24
24
|
try {
|
|
25
|
-
const wasmBinary = _fs.readFileSync(_path.join(__dirname, 'dist', fileName));
|
|
25
|
+
const wasmBinary = _fs.readFileSync(_path.join(__dirname, '..', 'dist', fileName));
|
|
26
26
|
return wasmBinary.buffer.slice(
|
|
27
27
|
wasmBinary.byteOffset,
|
|
28
28
|
wasmBinary.byteOffset + wasmBinary.byteLength
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { readFileSync, statSync } from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
|
-
import loadEngine from '
|
|
5
|
-
import loadScalarEngine from '
|
|
6
|
-
import createPikeletApi from './pikelet-core.js';
|
|
7
|
-
import errorContract from './pikelet-errors.js';
|
|
8
|
-
import loaderContract from './pikelet-loader.js';
|
|
9
|
-
import artifactContract from './pikelet-artifact.js';
|
|
4
|
+
import loadEngine from '../dist/engine.js';
|
|
5
|
+
import loadScalarEngine from '../dist/engine.scalar.js';
|
|
6
|
+
import createPikeletApi from './core/pikelet-core.js';
|
|
7
|
+
import errorContract from './errors/pikelet-errors.js';
|
|
8
|
+
import loaderContract from './loader/pikelet-loader.js';
|
|
9
|
+
import artifactContract from './artifact/pikelet-artifact.js';
|
|
10
10
|
const { PikeletError, PIKELET_ERROR_CODES, pikeletError } = errorContract;
|
|
11
11
|
const { createCachedModuleLoader } = loaderContract;
|
|
12
12
|
const { PikeletRangeArtifact, PikeletSketchArtifact, createSketchScanner, NodeFileRangeSource, buildRangeArtifact, buildRangeArtifactFile, buildSketchArtifact, buildSketchArtifactBytes, buildSketchArtifactFile } = artifactContract;
|
|
@@ -27,7 +27,7 @@ const SUPPORTED_SNAPSHOT_MAGICS = new Set([
|
|
|
27
27
|
|
|
28
28
|
function readWasmBinary(fileName) {
|
|
29
29
|
try {
|
|
30
|
-
const wasmBinary = readFileSync(path.join(__dirname, 'dist', fileName));
|
|
30
|
+
const wasmBinary = readFileSync(path.join(__dirname, '..', 'dist', fileName));
|
|
31
31
|
return wasmBinary.buffer.slice(
|
|
32
32
|
wasmBinary.byteOffset,
|
|
33
33
|
wasmBinary.byteOffset + wasmBinary.byteLength
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import loadEngine from '
|
|
2
|
-
import loadScalarEngine from '
|
|
3
|
-
import simdWasmAsset from '
|
|
4
|
-
import scalarWasmAsset from '
|
|
5
|
-
import createPikeletApi from './pikelet-core.js';
|
|
6
|
-
import errorContract from './pikelet-errors.js';
|
|
7
|
-
import loaderContract from './pikelet-loader.js';
|
|
8
|
-
import artifactContract from './pikelet-artifact.js';
|
|
1
|
+
import loadEngine from '../dist/engine.js';
|
|
2
|
+
import loadScalarEngine from '../dist/engine.scalar.js';
|
|
3
|
+
import simdWasmAsset from '../dist/engine.wasm?url';
|
|
4
|
+
import scalarWasmAsset from '../dist/engine.scalar.wasm?url';
|
|
5
|
+
import createPikeletApi from './core/pikelet-core.js';
|
|
6
|
+
import errorContract from './errors/pikelet-errors.js';
|
|
7
|
+
import loaderContract from './loader/pikelet-loader.js';
|
|
8
|
+
import artifactContract from './artifact/pikelet-artifact.js';
|
|
9
9
|
const { PikeletError, PIKELET_ERROR_CODES, pikeletError } = errorContract;
|
|
10
10
|
const { createCachedModuleLoader } = loaderContract;
|
|
11
11
|
const { PikeletRangeArtifact, PikeletSketchArtifact, createSketchScanner } = artifactContract;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import loadEngine from '
|
|
2
|
-
import loadScalarEngine from '
|
|
3
|
-
import simdWasmAsset from '
|
|
4
|
-
import scalarWasmAsset from '
|
|
5
|
-
import createPikeletApi from './pikelet-core.js';
|
|
6
|
-
import errorContract from './pikelet-errors.js';
|
|
7
|
-
import loaderContract from './pikelet-loader.js';
|
|
8
|
-
import artifactContract from './pikelet-artifact.js';
|
|
1
|
+
import loadEngine from '../dist/engine.js';
|
|
2
|
+
import loadScalarEngine from '../dist/engine.scalar.js';
|
|
3
|
+
import simdWasmAsset from '../dist/engine.wasm';
|
|
4
|
+
import scalarWasmAsset from '../dist/engine.scalar.wasm';
|
|
5
|
+
import createPikeletApi from './core/pikelet-core.js';
|
|
6
|
+
import errorContract from './errors/pikelet-errors.js';
|
|
7
|
+
import loaderContract from './loader/pikelet-loader.js';
|
|
8
|
+
import artifactContract from './artifact/pikelet-artifact.js';
|
|
9
9
|
const { PikeletError, PIKELET_ERROR_CODES, pikeletError } = errorContract;
|
|
10
10
|
const { createCachedModuleLoader } = loaderContract;
|
|
11
11
|
const { PikeletRangeArtifact, PikeletSketchArtifact, createSketchScanner } = artifactContract;
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|