pikelet-wasm 0.8.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +110 -33
- package/README.md +17 -1
- package/complete/builder.mjs +17 -4
- package/complete/format.mjs +6 -1
- package/complete/index.mjs +482 -58
- package/complete/layer-append.mjs +246 -0
- package/complete/layer-chain.mjs +284 -0
- package/complete/layer-compact.mjs +319 -0
- package/complete/layer-encoder.mjs +223 -0
- package/complete/layer-locator.mjs +228 -0
- package/complete/layer-manifest.mjs +271 -0
- package/complete/layer-reader.mjs +759 -0
- package/complete/layer-rebase.mjs +218 -0
- package/complete/lexical.mjs +78 -8
- package/complete/retrieval-abstention.mjs +15 -4
- package/complete/tombstones.mjs +281 -0
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +31 -1
- package/src/artifact/pikelet-artifact-sketch.js +67 -5
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
// Tombstone segment (kind 6), layout `tombstones-v1` — LAYERED_PROFILE.md 4.4.
|
|
2
|
+
//
|
|
3
|
+
// A layer hides records by setting bits in a cumulative bitset over
|
|
4
|
+
// [0, rowBase), one bit per global id, LSB-first within each byte. The set is
|
|
5
|
+
// cumulative, not a delta: every bit set in the parent's mask (zero-extended
|
|
6
|
+
// over the ids the parent did not yet address) is set in the child's. A layer
|
|
7
|
+
// may also record that a new record supersedes a tombstoned one, as
|
|
8
|
+
// (oldId, newId) pairs.
|
|
9
|
+
//
|
|
10
|
+
// The segment is read eagerly at mount and verified against its manifest
|
|
11
|
+
// digest, but a digest only proves the publisher wrote these bytes — the
|
|
12
|
+
// publisher is also the adversary for a distributed artifact, so every
|
|
13
|
+
// structural rule below is enforced on parse, not assumed. Section 9's stance:
|
|
14
|
+
// "the tombstone and lineage layouts carry normative structural validation
|
|
15
|
+
// precisely because a publisher can sign anything."
|
|
16
|
+
//
|
|
17
|
+
// [0,4) u32 version = 1
|
|
18
|
+
// [4,8) u32 flags (0; readers MUST reject non-zero)
|
|
19
|
+
// [8,16) u64 rowBase MUST equal manifest layer.rowBase
|
|
20
|
+
// [16,24) u64 tombstoneCount popcount; MUST equal manifest layer.tombstones
|
|
21
|
+
// [24,28) u32 supersessionCount MUST equal manifest layer.supersessions
|
|
22
|
+
// [28,32) reserved, zero
|
|
23
|
+
// [32, 32 + ceil(rowBase/8)) bitset, LSB-first; pad bits above rowBase MUST be zero
|
|
24
|
+
// [...] supersessionCount x (u32 oldId, u32 newId), sorted by oldId, oldId unique
|
|
25
|
+
|
|
26
|
+
export const TOMBSTONE_HEADER_BYTES = 32;
|
|
27
|
+
export const TOMBSTONES_VERSION = 1;
|
|
28
|
+
// The reader's id ceiling (3.3): a chain holds at most 2^31 - 1 records, so
|
|
29
|
+
// every rowBase and rowTotal is bounded by it and every id by 2^31 - 2.
|
|
30
|
+
export const MAX_RECORDS = 2 ** 31 - 1;
|
|
31
|
+
|
|
32
|
+
const POPCOUNT = new Uint8Array(256);
|
|
33
|
+
for (let i = 0; i < 256; i++) POPCOUNT[i] = (i & 1) + POPCOUNT[i >> 1];
|
|
34
|
+
|
|
35
|
+
export function popcountBytes(bytes) {
|
|
36
|
+
let n = 0;
|
|
37
|
+
for (let i = 0; i < bytes.length; i++) n += POPCOUNT[bytes[i]];
|
|
38
|
+
return n;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function bitsetBytesFor(rowBase) {
|
|
42
|
+
return Math.ceil(rowBase / 8);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function testBit(bitset, id) {
|
|
46
|
+
const byte = id >> 3;
|
|
47
|
+
if (byte >= bitset.length) return false;
|
|
48
|
+
return ((bitset[byte] >> (id & 7)) & 1) === 1;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function setBit(bitset, id) {
|
|
52
|
+
bitset[id >> 3] |= 1 << (id & 7);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Build a tombstones-v1 segment.
|
|
57
|
+
*
|
|
58
|
+
* @param {{rowBase: number, tombstonedIds: Iterable<number>, supersessions?: Array<[number, number]>,
|
|
59
|
+
* records?: number}} spec
|
|
60
|
+
* `records` is the layer's own record count, used only to validate that each
|
|
61
|
+
* `newId` lies in this layer's own id range — a layer cannot supersede into
|
|
62
|
+
* another layer's ids (3.5).
|
|
63
|
+
*/
|
|
64
|
+
export function buildTombstoneSegment({ rowBase, tombstonedIds, supersessions = [], records = 0 }) {
|
|
65
|
+
if (!Number.isInteger(rowBase) || rowBase < 0 || rowBase > MAX_RECORDS) {
|
|
66
|
+
throw new Error(`tombstones: rowBase must be an integer in [0, ${MAX_RECORDS}], got ${rowBase}`);
|
|
67
|
+
}
|
|
68
|
+
if (!Number.isInteger(records) || records < 0 || rowBase + records > MAX_RECORDS) {
|
|
69
|
+
throw new Error(`tombstones: rowBase + records must not exceed ${MAX_RECORDS}`);
|
|
70
|
+
}
|
|
71
|
+
const bitsetBytes = bitsetBytesFor(rowBase);
|
|
72
|
+
const bitset = new Uint8Array(bitsetBytes);
|
|
73
|
+
for (const id of tombstonedIds) {
|
|
74
|
+
if (!Number.isInteger(id) || id < 0 || id >= rowBase) {
|
|
75
|
+
// A layer cannot retract its own records (3.4): ids at or above
|
|
76
|
+
// rowBase are out of range by construction.
|
|
77
|
+
throw new Error(`tombstones: id ${id} is outside [0, ${rowBase})`);
|
|
78
|
+
}
|
|
79
|
+
setBit(bitset, id);
|
|
80
|
+
}
|
|
81
|
+
const tombstoneCount = popcountBytes(bitset);
|
|
82
|
+
|
|
83
|
+
const edges = [...supersessions].map(([oldId, newId]) => {
|
|
84
|
+
if (!Number.isInteger(oldId) || !Number.isInteger(newId)) {
|
|
85
|
+
throw new Error('tombstones: supersession ids must be integers');
|
|
86
|
+
}
|
|
87
|
+
if (!testBit(bitset, oldId)) {
|
|
88
|
+
throw new Error(`tombstones: supersession oldId ${oldId} is not tombstoned in this layer's bitset`);
|
|
89
|
+
}
|
|
90
|
+
if (newId < rowBase || newId >= rowBase + records) {
|
|
91
|
+
throw new Error(`tombstones: supersession newId ${newId} is outside this layer's range [${rowBase}, ${rowBase + records})`);
|
|
92
|
+
}
|
|
93
|
+
return [oldId, newId];
|
|
94
|
+
});
|
|
95
|
+
edges.sort((a, b) => a[0] - b[0]);
|
|
96
|
+
for (let i = 1; i < edges.length; i++) {
|
|
97
|
+
if (edges[i][0] === edges[i - 1][0]) {
|
|
98
|
+
throw new Error(`tombstones: oldId ${edges[i][0]} appears in more than one supersession in this layer`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const bytes = new Uint8Array(TOMBSTONE_HEADER_BYTES + bitsetBytes + edges.length * 8);
|
|
103
|
+
const view = new DataView(bytes.buffer);
|
|
104
|
+
view.setUint32(0, TOMBSTONES_VERSION, true);
|
|
105
|
+
view.setUint32(4, 0, true);
|
|
106
|
+
view.setBigUint64(8, BigInt(rowBase), true);
|
|
107
|
+
view.setBigUint64(16, BigInt(tombstoneCount), true);
|
|
108
|
+
view.setUint32(24, edges.length, true);
|
|
109
|
+
view.setUint32(28, 0, true);
|
|
110
|
+
bytes.set(bitset, TOMBSTONE_HEADER_BYTES);
|
|
111
|
+
let at = TOMBSTONE_HEADER_BYTES + bitsetBytes;
|
|
112
|
+
for (const [oldId, newId] of edges) {
|
|
113
|
+
view.setUint32(at, oldId, true);
|
|
114
|
+
view.setUint32(at + 4, newId, true);
|
|
115
|
+
at += 8;
|
|
116
|
+
}
|
|
117
|
+
return bytes;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Parse and fully validate a tombstones-v1 segment.
|
|
122
|
+
*
|
|
123
|
+
* @param {Uint8Array} bytes
|
|
124
|
+
* @param {{rowBase: number, tombstones: number, supersessions: number, records?: number}} [expect]
|
|
125
|
+
* The manifest's `layer` fields. When given, every one is checked against the
|
|
126
|
+
* segment: a segment that disagrees with the manifest committing to it is a
|
|
127
|
+
* defect, and the reader refuses rather than picking a winner.
|
|
128
|
+
*/
|
|
129
|
+
export function parseTombstoneSegment(bytes, expect = null) {
|
|
130
|
+
if (!(bytes instanceof Uint8Array)) throw new Error('tombstones: segment must be a Uint8Array');
|
|
131
|
+
if (bytes.length < TOMBSTONE_HEADER_BYTES) {
|
|
132
|
+
throw new Error(`tombstones: segment is ${bytes.length} bytes, shorter than the ${TOMBSTONE_HEADER_BYTES}-byte header`);
|
|
133
|
+
}
|
|
134
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
135
|
+
const version = view.getUint32(0, true);
|
|
136
|
+
if (version !== TOMBSTONES_VERSION) {
|
|
137
|
+
throw new Error(`tombstones: version ${version} is not ${TOMBSTONES_VERSION}`);
|
|
138
|
+
}
|
|
139
|
+
const flags = view.getUint32(4, true);
|
|
140
|
+
if (flags !== 0) throw new Error(`tombstones: flags must be 0, got ${flags}`);
|
|
141
|
+
|
|
142
|
+
// u64 fields are read as BigInt and refused above MAX_SAFE_INTEGER rather
|
|
143
|
+
// than silently losing precision, matching the reader's u64() rule.
|
|
144
|
+
const rowBaseBig = view.getBigUint64(8, true);
|
|
145
|
+
const countBig = view.getBigUint64(16, true);
|
|
146
|
+
if (rowBaseBig > BigInt(Number.MAX_SAFE_INTEGER) || countBig > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
147
|
+
throw new Error('tombstones: a u64 field exceeds MAX_SAFE_INTEGER');
|
|
148
|
+
}
|
|
149
|
+
const rowBase = Number(rowBaseBig);
|
|
150
|
+
const tombstoneCount = Number(countBig);
|
|
151
|
+
const supersessionCount = view.getUint32(24, true);
|
|
152
|
+
const reserved = view.getUint32(28, true);
|
|
153
|
+
if (reserved !== 0) throw new Error('tombstones: reserved bytes must be zero');
|
|
154
|
+
if (rowBase > MAX_RECORDS) throw new Error(`tombstones: rowBase ${rowBase} exceeds ${MAX_RECORDS}`);
|
|
155
|
+
if (tombstoneCount > rowBase) {
|
|
156
|
+
throw new Error(`tombstones: tombstoneCount ${tombstoneCount} exceeds rowBase ${rowBase}`);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// The regions MUST tile the segment exactly (4.4). Computed with
|
|
160
|
+
// subtraction so a hostile count cannot overflow the comparison.
|
|
161
|
+
const bitsetBytes = bitsetBytesFor(rowBase);
|
|
162
|
+
if (bytes.length - TOMBSTONE_HEADER_BYTES < bitsetBytes) {
|
|
163
|
+
throw new Error(`tombstones: segment holds ${bytes.length - TOMBSTONE_HEADER_BYTES} bytes after the header, needs ${bitsetBytes} for the bitset`);
|
|
164
|
+
}
|
|
165
|
+
const afterBitset = bytes.length - TOMBSTONE_HEADER_BYTES - bitsetBytes;
|
|
166
|
+
if (afterBitset !== supersessionCount * 8) {
|
|
167
|
+
throw new Error(`tombstones: ${afterBitset} bytes after the bitset but supersessionCount ${supersessionCount} needs ${supersessionCount * 8}`);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const bitset = bytes.subarray(TOMBSTONE_HEADER_BYTES, TOMBSTONE_HEADER_BYTES + bitsetBytes);
|
|
171
|
+
// Pad bits above rowBase MUST be zero: otherwise two segments with the
|
|
172
|
+
// same meaning could differ in bytes, and popcount would disagree.
|
|
173
|
+
const padBits = bitsetBytes * 8 - rowBase;
|
|
174
|
+
if (padBits > 0 && bitsetBytes > 0) {
|
|
175
|
+
const lastByte = bitset[bitsetBytes - 1];
|
|
176
|
+
const validMask = (1 << (8 - padBits)) - 1;
|
|
177
|
+
if ((lastByte & ~validMask & 0xff) !== 0) {
|
|
178
|
+
throw new Error(`tombstones: ${padBits} pad bit(s) above rowBase ${rowBase} are non-zero`);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
const actual = popcountBytes(bitset);
|
|
182
|
+
if (actual !== tombstoneCount) {
|
|
183
|
+
throw new Error(`tombstones: header says ${tombstoneCount} tombstones, bitset popcount is ${actual}`);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const supersessions = [];
|
|
187
|
+
let at = TOMBSTONE_HEADER_BYTES + bitsetBytes;
|
|
188
|
+
let prevOld = -1;
|
|
189
|
+
for (let i = 0; i < supersessionCount; i++) {
|
|
190
|
+
const oldId = view.getUint32(at, true);
|
|
191
|
+
const newId = view.getUint32(at + 4, true);
|
|
192
|
+
at += 8;
|
|
193
|
+
if (oldId >= rowBase) {
|
|
194
|
+
throw new Error(`tombstones: supersession oldId ${oldId} is not below rowBase ${rowBase}`);
|
|
195
|
+
}
|
|
196
|
+
if (!testBit(bitset, oldId)) {
|
|
197
|
+
throw new Error(`tombstones: supersession oldId ${oldId} is not tombstoned`);
|
|
198
|
+
}
|
|
199
|
+
if (oldId <= prevOld) {
|
|
200
|
+
throw new Error(`tombstones: supersessions must be sorted by oldId ascending and unique (${oldId} after ${prevOld})`);
|
|
201
|
+
}
|
|
202
|
+
prevOld = oldId;
|
|
203
|
+
if (expect && Number.isInteger(expect.records)) {
|
|
204
|
+
const lo = rowBase;
|
|
205
|
+
const hi = rowBase + expect.records;
|
|
206
|
+
if (newId < lo || newId >= hi) {
|
|
207
|
+
throw new Error(`tombstones: supersession newId ${newId} is outside this layer's range [${lo}, ${hi})`);
|
|
208
|
+
}
|
|
209
|
+
} else if (newId < rowBase) {
|
|
210
|
+
// Without the manifest's record count only the lower bound is
|
|
211
|
+
// checkable; 3.5 requires newId >= rowBase unconditionally.
|
|
212
|
+
throw new Error(`tombstones: supersession newId ${newId} is below rowBase ${rowBase}`);
|
|
213
|
+
}
|
|
214
|
+
supersessions.push([oldId, newId]);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
if (expect) {
|
|
218
|
+
if (Number.isInteger(expect.rowBase) && expect.rowBase !== rowBase) {
|
|
219
|
+
throw new Error(`tombstones: segment rowBase ${rowBase} disagrees with manifest layer.rowBase ${expect.rowBase}`);
|
|
220
|
+
}
|
|
221
|
+
if (Number.isInteger(expect.tombstones) && expect.tombstones !== tombstoneCount) {
|
|
222
|
+
throw new Error(`tombstones: segment tombstoneCount ${tombstoneCount} disagrees with manifest layer.tombstones ${expect.tombstones}`);
|
|
223
|
+
}
|
|
224
|
+
if (Number.isInteger(expect.supersessions) && expect.supersessions !== supersessionCount) {
|
|
225
|
+
throw new Error(`tombstones: segment supersessionCount ${supersessionCount} disagrees with manifest layer.supersessions ${expect.supersessions}`);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
return { version, rowBase, tombstoneCount, supersessions, bitset };
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* `zeroExtend(T_parent, rowBase_child)` — 3.4. The parent's mask followed by
|
|
234
|
+
* zero bits over [rowBase_parent, rowBase_child): the parent's own records
|
|
235
|
+
* become addressable by the child, and start clear.
|
|
236
|
+
*/
|
|
237
|
+
export function zeroExtend(parentBitset, childRowBase) {
|
|
238
|
+
const out = new Uint8Array(bitsetBytesFor(childRowBase));
|
|
239
|
+
if (parentBitset && parentBitset.length) {
|
|
240
|
+
out.set(parentBitset.subarray(0, Math.min(parentBitset.length, out.length)));
|
|
241
|
+
}
|
|
242
|
+
return out;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* The superset (cumulative) rule of 3.4: every bit set in the zero-extended
|
|
247
|
+
* parent mask MUST be set in the child's. Returns the first offending id, or
|
|
248
|
+
* -1 when the rule holds.
|
|
249
|
+
*/
|
|
250
|
+
export function firstSupersetViolation(childBitset, parentBitset, childRowBase) {
|
|
251
|
+
const extended = zeroExtend(parentBitset, childRowBase);
|
|
252
|
+
for (let byte = 0; byte < extended.length; byte++) {
|
|
253
|
+
const missing = extended[byte] & ~(childBitset[byte] || 0) & 0xff;
|
|
254
|
+
if (missing !== 0) {
|
|
255
|
+
for (let bit = 0; bit < 8; bit++) {
|
|
256
|
+
if ((missing >> bit) & 1) return byte * 8 + bit;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
return -1;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* `ΔT_child` — the deletions a child introduced over its parent (3.4). This is
|
|
265
|
+
* what a rebase replays; it is derived, never stored.
|
|
266
|
+
*/
|
|
267
|
+
export function tombstoneDelta(childBitset, parentBitset, childRowBase) {
|
|
268
|
+
const extended = zeroExtend(parentBitset, childRowBase);
|
|
269
|
+
const ids = [];
|
|
270
|
+
for (let byte = 0; byte < childBitset.length; byte++) {
|
|
271
|
+
const added = childBitset[byte] & ~(extended[byte] || 0) & 0xff;
|
|
272
|
+
if (added === 0) continue;
|
|
273
|
+
for (let bit = 0; bit < 8; bit++) {
|
|
274
|
+
if ((added >> bit) & 1) {
|
|
275
|
+
const id = byte * 8 + bit;
|
|
276
|
+
if (id < childRowBase) ids.push(id);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
return ids;
|
|
281
|
+
}
|
package/dist/engine.scalar.wasm
CHANGED
|
Binary file
|
package/dist/engine.wasm
CHANGED
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pikelet-wasm",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Vector search engine and .pikelet artifact readers/builders, in WebAssembly. Compile a corpus into one self-contained, hash-verified search file, or use the underlying HNSW engine directly. Runs in Node.js, browsers, and Cloudflare Workers with no native dependencies.",
|
|
5
5
|
"main": "src/pikelet.js",
|
|
6
6
|
"types": "src/pikelet.d.ts",
|
|
@@ -71,6 +71,36 @@
|
|
|
71
71
|
"types": "./complete/builder.d.ts",
|
|
72
72
|
"default": "./complete/builder.mjs"
|
|
73
73
|
},
|
|
74
|
+
"./complete/tombstones.mjs": {
|
|
75
|
+
"default": "./complete/tombstones.mjs"
|
|
76
|
+
},
|
|
77
|
+
"./complete/format.mjs": {
|
|
78
|
+
"default": "./complete/format.mjs"
|
|
79
|
+
},
|
|
80
|
+
"./complete/layer-manifest.mjs": {
|
|
81
|
+
"default": "./complete/layer-manifest.mjs"
|
|
82
|
+
},
|
|
83
|
+
"./complete/layer-encoder.mjs": {
|
|
84
|
+
"default": "./complete/layer-encoder.mjs"
|
|
85
|
+
},
|
|
86
|
+
"./complete/layer-locator.mjs": {
|
|
87
|
+
"default": "./complete/layer-locator.mjs"
|
|
88
|
+
},
|
|
89
|
+
"./complete/layer-chain.mjs": {
|
|
90
|
+
"default": "./complete/layer-chain.mjs"
|
|
91
|
+
},
|
|
92
|
+
"./complete/layer-reader.mjs": {
|
|
93
|
+
"default": "./complete/layer-reader.mjs"
|
|
94
|
+
},
|
|
95
|
+
"./complete/layer-append.mjs": {
|
|
96
|
+
"default": "./complete/layer-append.mjs"
|
|
97
|
+
},
|
|
98
|
+
"./complete/layer-rebase.mjs": {
|
|
99
|
+
"default": "./complete/layer-rebase.mjs"
|
|
100
|
+
},
|
|
101
|
+
"./complete/layer-compact.mjs": {
|
|
102
|
+
"default": "./complete/layer-compact.mjs"
|
|
103
|
+
},
|
|
74
104
|
"./engine.wasm": "./dist/engine.wasm"
|
|
75
105
|
},
|
|
76
106
|
"files": [
|
|
@@ -828,7 +828,34 @@ class PikeletSketchArtifact {
|
|
|
828
828
|
const base = resolveOptionalPositiveInt(options.rerank, 'rerank', 0);
|
|
829
829
|
const boost = Math.max(1, Math.trunc(options.microBoost || 4));
|
|
830
830
|
const rec = this.recommendedRerank || Math.max(100, k * 10);
|
|
831
|
-
|
|
831
|
+
// options.exclude: a LSB-first bitset over local rows, one bit per row,
|
|
832
|
+
// supplied by a layered chain (LAYERED_PROFILE.md 5.3). A masked row
|
|
833
|
+
// must not enter the candidate heap, must not count toward C, and must
|
|
834
|
+
// not be fetched for reranking. Masking AFTER truncation is a defect,
|
|
835
|
+
// not a simplification: if the C nearest rows are all masked and the
|
|
836
|
+
// nearest live row is the C+1-th, a post-hoc mask returns nothing while
|
|
837
|
+
// the correct answer exists.
|
|
838
|
+
const exclude = options.exclude || null;
|
|
839
|
+
if (exclude !== null) {
|
|
840
|
+
if (!(exclude instanceof Uint8Array)) {
|
|
841
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'search() exclude must be a Uint8Array bitset');
|
|
842
|
+
}
|
|
843
|
+
if (exclude.length < Math.ceil(this.count / 8)) {
|
|
844
|
+
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'search() exclude bitset is too short for this artifact', { bytes: exclude.length, needed: Math.ceil(this.count / 8) });
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
const isExcluded = exclude === null
|
|
848
|
+
? () => false
|
|
849
|
+
: (row) => ((exclude[row >> 3] >> (row & 7)) & 1) === 1;
|
|
850
|
+
// C counts unmasked rows, so a heavily tombstoned tier still offers a
|
|
851
|
+
// full candidate pool over what is live.
|
|
852
|
+
let liveCountForC = this.count;
|
|
853
|
+
if (exclude !== null) {
|
|
854
|
+
liveCountForC = 0;
|
|
855
|
+
for (let i = 0; i < this.count; i++) if (!isExcluded(i)) liveCountForC++;
|
|
856
|
+
}
|
|
857
|
+
if (liveCountForC === 0) return { results: [], stats: this.stats() };
|
|
858
|
+
const C = Math.min(liveCountForC, Math.max(k, base > 0 ? base : tier.name === 'micro' ? rec * boost : rec));
|
|
832
859
|
const { dim, count } = this;
|
|
833
860
|
const tierDims = tier.dims;
|
|
834
861
|
const pool = dim / tierDims;
|
|
@@ -859,6 +886,27 @@ class PikeletSketchArtifact {
|
|
|
859
886
|
// C beyond them would silently shrink the candidate pool — recall
|
|
860
887
|
// loss, not an error. Fall back to the JS scan for that query.
|
|
861
888
|
if (scanner && scanner.maxRerank !== undefined && C > scanner.maxRerank) scanner = null;
|
|
889
|
+
// A scan kernel scores every resident row and knows nothing about an
|
|
890
|
+
// exclusion mask. Masking its top-C output would be the post-truncation
|
|
891
|
+
// defect 5.3 forbids — if the C nearest rows are all masked, a post-hoc
|
|
892
|
+
// filter returns nothing while the correct answer exists.
|
|
893
|
+
//
|
|
894
|
+
// But the kernel CAN be used safely by over-fetching: ask it for
|
|
895
|
+
// C + (number of masked rows) candidates, then drop the masked ones.
|
|
896
|
+
// At least C live rows survive by construction, because at most
|
|
897
|
+
// maskedCount of the returned rows can be masked. That is not a
|
|
898
|
+
// post-hoc mask on a truncated list; it is an exact selection over a
|
|
899
|
+
// provably sufficient superset.
|
|
900
|
+
//
|
|
901
|
+
// When the over-fetch would exceed the kernel's buffers or the row
|
|
902
|
+
// count, fall back to the masked JS scan, which is always correct.
|
|
903
|
+
let scannerOverfetch = 0;
|
|
904
|
+
if (scanner && exclude !== null) {
|
|
905
|
+
const maskedCount = this.count - liveCountForC;
|
|
906
|
+
scannerOverfetch = C + maskedCount;
|
|
907
|
+
const limit = Math.min(count, scanner.maxRerank ?? count);
|
|
908
|
+
if (scannerOverfetch > limit) { scanner = null; scannerOverfetch = 0; }
|
|
909
|
+
}
|
|
862
910
|
if (scanner) {
|
|
863
911
|
// A scanner scores the resident sketches itself, so it must
|
|
864
912
|
// implement this artifact's metric. Cosine requires an explicit
|
|
@@ -870,18 +918,32 @@ class PikeletSketchArtifact {
|
|
|
870
918
|
if (this.metric === 1 && scannerMetric !== 1) {
|
|
871
919
|
throw pikeletError(PIKELET_ERROR_CODES.INVALID_ARGUMENT, 'cosine sketch artifacts require a metric-aware scanner (scanner.metric === 1)', { metric: this.metric });
|
|
872
920
|
}
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
921
|
+
// Over-fetch when a mask is in play (see above), then drop the
|
|
922
|
+
// masked rows. The survivors are the exact top-C live rows.
|
|
923
|
+
ids = scanner.scan(qPool, scannerOverfetch || C);
|
|
924
|
+
if (exclude !== null) {
|
|
925
|
+
const live = [];
|
|
926
|
+
for (const id of ids) {
|
|
927
|
+
if (!isExcluded(id)) live.push(id);
|
|
928
|
+
if (live.length === C) break;
|
|
929
|
+
}
|
|
930
|
+
ids = live;
|
|
931
|
+
}
|
|
932
|
+
} else if (C >= liveCountForC) {
|
|
933
|
+
// Every live row is a candidate: the exact rerank below scores them
|
|
876
934
|
// all, so the resident scan has nothing to select. Without this
|
|
877
935
|
// the selection loop degenerates to 2*count^2 slot operations
|
|
878
936
|
// (candMax stays Infinity until the last slot fills).
|
|
879
|
-
ids =
|
|
937
|
+
ids = [];
|
|
938
|
+
for (let i = 0; i < count; i++) if (!isExcluded(i)) ids.push(i);
|
|
880
939
|
} else {
|
|
881
940
|
const candDist = new Float64Array(C).fill(Infinity);
|
|
882
941
|
const candId = new Int32Array(C).fill(-1);
|
|
883
942
|
let candMax = Infinity;
|
|
884
943
|
for (let i = 0; i < count; i++) {
|
|
944
|
+
// Skipped before scoring, so a masked row never occupies a
|
|
945
|
+
// candidate slot (5.3).
|
|
946
|
+
if (isExcluded(i)) continue;
|
|
885
947
|
const s = this.scales[i];
|
|
886
948
|
const o = this.offsets[i];
|
|
887
949
|
let metricAcc = 0;
|