probe-filters 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +298 -0
- package/package.json +50 -0
- package/src/index.js +5 -0
- package/src/mergeOperators.js +16 -0
- package/src/multiFilter.js +307 -0
- package/src/packedBitset.js +74 -0
- package/src/packedRsqf.js +302 -0
- package/src/pointFilter.js +871 -0
- package/src/rangeFilter.js +298 -0
- package/src/serialization.js +214 -0
- package/src/spatialFilter.js +317 -0
- package/src/temporalFilter.js +259 -0
- package/src/zeno.js +85 -0
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
import { RangeFilter } from './rangeFilter.js';
|
|
2
|
+
import { BinaryWriter, BinaryReader, MAGIC, wrapCRC32, unwrapCRC32 } from './serialization.js';
|
|
3
|
+
|
|
4
|
+
function assertGridCoordinate(value, bitsPerCoordinate, label) {
|
|
5
|
+
const max = 2 ** bitsPerCoordinate;
|
|
6
|
+
if (!Number.isInteger(value) || value < 0 || value >= max) {
|
|
7
|
+
throw new Error(`${label} must be an integer in [0, ${max}).`);
|
|
8
|
+
}
|
|
9
|
+
return value >>> 0;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function parseFiniteCoordinate(value, label) {
|
|
13
|
+
const parsed = typeof value === 'string' ? Number(value) : value;
|
|
14
|
+
if (!Number.isFinite(parsed)) {
|
|
15
|
+
throw new Error(`${label} must be a finite numeric coordinate.`);
|
|
16
|
+
}
|
|
17
|
+
return parsed;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function clamp(value, min, max) {
|
|
21
|
+
return Math.max(min, Math.min(max, value));
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function scaleToGrid(value, min, max, bitsPerCoordinate, label) {
|
|
25
|
+
const parsed = parseFiniteCoordinate(value, label);
|
|
26
|
+
if (parsed < min || parsed > max) {
|
|
27
|
+
throw new Error(`${label} must be in [${min}, ${max}].`);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const gridMax = (2 ** bitsPerCoordinate) - 1;
|
|
31
|
+
const unit = (parsed - min) / (max - min);
|
|
32
|
+
return clamp(Math.floor(unit * gridMax), 0, gridMax) >>> 0;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function makeCoordinateCodec(options) {
|
|
36
|
+
const bitsPerCoordinate = options.bitsPerCoordinate;
|
|
37
|
+
const system = options.coordinateSystem ?? 'uint';
|
|
38
|
+
|
|
39
|
+
if (options.coordinateCodec) {
|
|
40
|
+
return options.coordinateCodec;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const base = {
|
|
44
|
+
gridMax: (2 ** bitsPerCoordinate) - 1,
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
if (system === 'latlon') {
|
|
48
|
+
return {
|
|
49
|
+
...base,
|
|
50
|
+
normalizePoint(point) {
|
|
51
|
+
if (!Array.isArray(point) || point.length !== 2) {
|
|
52
|
+
throw new Error('Lat/lon points must be [lat, lon] pairs.');
|
|
53
|
+
}
|
|
54
|
+
return {
|
|
55
|
+
x: scaleToGrid(point[1], -180, 180, bitsPerCoordinate, 'lon'),
|
|
56
|
+
y: scaleToGrid(point[0], -90, 90, bitsPerCoordinate, 'lat'),
|
|
57
|
+
raw: point,
|
|
58
|
+
};
|
|
59
|
+
},
|
|
60
|
+
normalizeBox(min, max) {
|
|
61
|
+
if (!Array.isArray(min) || !Array.isArray(max) || min.length !== 2 || max.length !== 2) {
|
|
62
|
+
throw new Error('Lat/lon boxes must use [lat, lon] min/max pairs.');
|
|
63
|
+
}
|
|
64
|
+
const minLat = parseFiniteCoordinate(min[0], 'min lat');
|
|
65
|
+
const minLon = parseFiniteCoordinate(min[1], 'min lon');
|
|
66
|
+
const maxLat = parseFiniteCoordinate(max[0], 'max lat');
|
|
67
|
+
const maxLon = parseFiniteCoordinate(max[1], 'max lon');
|
|
68
|
+
if (minLat > maxLat || minLon > maxLon) {
|
|
69
|
+
throw new Error('Lat/lon box min coordinates must be <= max coordinates.');
|
|
70
|
+
}
|
|
71
|
+
return {
|
|
72
|
+
minX: scaleToGrid(minLon, -180, 180, bitsPerCoordinate, 'min lon'),
|
|
73
|
+
minY: scaleToGrid(minLat, -90, 90, bitsPerCoordinate, 'min lat'),
|
|
74
|
+
maxX: scaleToGrid(maxLon, -180, 180, bitsPerCoordinate, 'max lon'),
|
|
75
|
+
maxY: scaleToGrid(maxLat, -90, 90, bitsPerCoordinate, 'max lat'),
|
|
76
|
+
};
|
|
77
|
+
},
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (system === 'float') {
|
|
82
|
+
const bounds = options.bounds ?? [[0, 0], [1, 1]];
|
|
83
|
+
const minB = bounds[0], maxB = bounds[1];
|
|
84
|
+
return {
|
|
85
|
+
...base,
|
|
86
|
+
normalizePoint(point) {
|
|
87
|
+
if (!Array.isArray(point) || point.length !== 2) {
|
|
88
|
+
throw new Error('Float spatial points must be [x, y] pairs.');
|
|
89
|
+
}
|
|
90
|
+
return {
|
|
91
|
+
x: scaleToGrid(point[0], minB[0], maxB[0], bitsPerCoordinate, 'x'),
|
|
92
|
+
y: scaleToGrid(point[1], minB[1], maxB[1], bitsPerCoordinate, 'y'),
|
|
93
|
+
raw: point,
|
|
94
|
+
};
|
|
95
|
+
},
|
|
96
|
+
normalizeBox(min, max) {
|
|
97
|
+
if (!Array.isArray(min) || !Array.isArray(max) || min.length !== 2 || max.length !== 2) {
|
|
98
|
+
throw new Error('Float spatial boxes must use [x, y] min/max pairs.');
|
|
99
|
+
}
|
|
100
|
+
const mx = parseFiniteCoordinate(min[0], 'min x');
|
|
101
|
+
const my = parseFiniteCoordinate(min[1], 'min y');
|
|
102
|
+
const Mx = parseFiniteCoordinate(max[0], 'max x');
|
|
103
|
+
const My = parseFiniteCoordinate(max[1], 'max y');
|
|
104
|
+
if (mx > Mx || my > My) {
|
|
105
|
+
throw new Error('Float spatial box min coordinates must be <= max coordinates.');
|
|
106
|
+
}
|
|
107
|
+
return {
|
|
108
|
+
minX: scaleToGrid(mx, minB[0], maxB[0], bitsPerCoordinate, 'min x'),
|
|
109
|
+
minY: scaleToGrid(my, minB[1], maxB[1], bitsPerCoordinate, 'min y'),
|
|
110
|
+
maxX: scaleToGrid(Mx, minB[0], maxB[0], bitsPerCoordinate, 'max x'),
|
|
111
|
+
maxY: scaleToGrid(My, minB[1], maxB[1], bitsPerCoordinate, 'max y'),
|
|
112
|
+
};
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
return {
|
|
118
|
+
...base,
|
|
119
|
+
normalizePoint(point) {
|
|
120
|
+
if (!Array.isArray(point) || point.length !== 2) {
|
|
121
|
+
throw new Error('Spatial points must be [x, y] pairs.');
|
|
122
|
+
}
|
|
123
|
+
const x = assertGridCoordinate(point[0], bitsPerCoordinate, 'x');
|
|
124
|
+
const y = assertGridCoordinate(point[1], bitsPerCoordinate, 'y');
|
|
125
|
+
return { x, y, raw: point };
|
|
126
|
+
},
|
|
127
|
+
normalizeBox(min, max) {
|
|
128
|
+
const a = this.normalizePoint(min);
|
|
129
|
+
const b = this.normalizePoint(max);
|
|
130
|
+
if (a.x > b.x || a.y > b.y) {
|
|
131
|
+
throw new Error('Spatial box min coordinates must be <= max coordinates.');
|
|
132
|
+
}
|
|
133
|
+
return { minX: a.x, minY: a.y, maxX: b.x, maxY: b.y };
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export function morton2D(x, y, bitsPerCoordinate = 16) {
|
|
139
|
+
const nx = assertGridCoordinate(x, bitsPerCoordinate, 'x');
|
|
140
|
+
const ny = assertGridCoordinate(y, bitsPerCoordinate, 'y');
|
|
141
|
+
let code = 0;
|
|
142
|
+
for (let bit = 0; bit < bitsPerCoordinate; bit++) {
|
|
143
|
+
code |= ((nx >>> bit) & 1) << (2 * bit);
|
|
144
|
+
code |= ((ny >>> bit) & 1) << (2 * bit + 1);
|
|
145
|
+
}
|
|
146
|
+
return code >>> 0;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export function unmorton2D(code, bitsPerCoordinate = 16) {
|
|
150
|
+
const nc = code >>> 0;
|
|
151
|
+
let x = 0, y = 0;
|
|
152
|
+
for (let bit = 0; bit < bitsPerCoordinate; bit++) {
|
|
153
|
+
x |= ((nc >>> (2 * bit)) & 1) << bit;
|
|
154
|
+
y |= ((nc >>> (2 * bit + 1)) & 1) << bit;
|
|
155
|
+
}
|
|
156
|
+
return [x >>> 0, y >>> 0];
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export function boxToMortonRanges(minX, maxX, minY, maxY, bitsPerCoordinate) {
|
|
160
|
+
const ranges = [];
|
|
161
|
+
const maxCoord = (1 << bitsPerCoordinate) - 1;
|
|
162
|
+
|
|
163
|
+
function walk(prefix, prefixBits, xLo, xHi, yLo, yHi) {
|
|
164
|
+
if (xLo > maxX || xHi < minX || yLo > maxY || yHi < minY) return;
|
|
165
|
+
|
|
166
|
+
if (xLo >= minX && xHi <= maxX && yLo >= minY && yHi <= maxY) {
|
|
167
|
+
const remaining = 2 * bitsPerCoordinate - prefixBits;
|
|
168
|
+
const lo = prefix << remaining;
|
|
169
|
+
const hi = lo | ((1 << remaining) - 1);
|
|
170
|
+
ranges.push([lo >>> 0, hi >>> 0]);
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
if (prefixBits >= 2 * bitsPerCoordinate) return;
|
|
175
|
+
|
|
176
|
+
const xMid = (xLo + xHi) >> 1;
|
|
177
|
+
const yMid = (yLo + yHi) >> 1;
|
|
178
|
+
const np = prefixBits + 2;
|
|
179
|
+
|
|
180
|
+
walk((prefix << 2) | 0, np, xLo, xMid, yLo, yMid);
|
|
181
|
+
walk((prefix << 2) | 1, np, xMid + 1, xHi, yLo, yMid);
|
|
182
|
+
walk((prefix << 2) | 2, np, xLo, xMid, yMid + 1, yHi);
|
|
183
|
+
walk((prefix << 2) | 3, np, xMid + 1, xHi, yMid + 1, yHi);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
walk(0, 0, 0, maxCoord, 0, maxCoord);
|
|
187
|
+
return ranges;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
export class SpatialFilter {
|
|
191
|
+
constructor(options = {}) {
|
|
192
|
+
this.bitsPerCoordinate = options.bitsPerCoordinate ?? 16;
|
|
193
|
+
if (this.bitsPerCoordinate <= 0 || this.bitsPerCoordinate > 16) {
|
|
194
|
+
throw new Error('bitsPerCoordinate must be in [1, 16] for 32-bit Morton codes.');
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
this.maxShapeCells = options.maxShapeCells ?? 64;
|
|
198
|
+
this.codec = makeCoordinateCodec({ ...options, bitsPerCoordinate: this.bitsPerCoordinate });
|
|
199
|
+
|
|
200
|
+
this.rangeFilter = new RangeFilter(
|
|
201
|
+
options.rangeOptions ?? options.filterOptions ?? {
|
|
202
|
+
partitionSize: 16,
|
|
203
|
+
fingerprintBits: 8,
|
|
204
|
+
maxFingerprintBits: 16,
|
|
205
|
+
}
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
_mortonOfPoint(x, y) {
|
|
210
|
+
return morton2D(x, y, this.bitsPerCoordinate);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
_mortonOfBox(box) {
|
|
214
|
+
return boxToMortonRanges(box.minX, box.maxX, box.minY, box.maxY, this.bitsPerCoordinate);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
insert(point) {
|
|
218
|
+
const entry = this.codec.normalizePoint(point);
|
|
219
|
+
const code = this._mortonOfPoint(entry.x, entry.y);
|
|
220
|
+
this.rangeFilter.insert(code);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
insertBox(min, max, _id = null) {
|
|
224
|
+
const box = this.codec.normalizeBox(min, max);
|
|
225
|
+
const ranges = this._mortonOfBox(box);
|
|
226
|
+
if (ranges.length > this.maxShapeCells) {
|
|
227
|
+
throw new Error(`Box covers ${ranges.length} Morton ranges; maxShapeCells is ${this.maxShapeCells}.`);
|
|
228
|
+
}
|
|
229
|
+
for (const [lo, hi] of ranges) {
|
|
230
|
+
for (let c = lo; c <= hi; c++) {
|
|
231
|
+
this.rangeFilter.insert(c);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
insertCircle(center, radius, _id = null) {
|
|
237
|
+
const entry = this.codec.normalizePoint(center);
|
|
238
|
+
const r = parseFiniteCoordinate(radius, 'radius');
|
|
239
|
+
if (r < 0) throw new Error('radius must be non-negative.');
|
|
240
|
+
const gm = (1 << this.bitsPerCoordinate) - 1;
|
|
241
|
+
const box = {
|
|
242
|
+
minX: clamp(Math.floor(entry.x - r), 0, gm),
|
|
243
|
+
minY: clamp(Math.floor(entry.y - r), 0, gm),
|
|
244
|
+
maxX: clamp(Math.ceil(entry.x + r), 0, gm),
|
|
245
|
+
maxY: clamp(Math.ceil(entry.y + r), 0, gm),
|
|
246
|
+
};
|
|
247
|
+
this.insertBox([box.minX, box.minY], [box.maxX, box.maxY]);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
query(point) {
|
|
251
|
+
const entry = this.codec.normalizePoint(point);
|
|
252
|
+
const code = this._mortonOfPoint(entry.x, entry.y);
|
|
253
|
+
return this.rangeFilter.queryRange(code, code);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
delete(point) {
|
|
257
|
+
const entry = this.codec.normalizePoint(point);
|
|
258
|
+
const code = this._mortonOfPoint(entry.x, entry.y);
|
|
259
|
+
return this.rangeFilter.delete(code);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
queryBox(min, max) {
|
|
263
|
+
const box = this.codec.normalizeBox(min, max);
|
|
264
|
+
const ranges = this._mortonOfBox(box);
|
|
265
|
+
for (const [lo, hi] of ranges) {
|
|
266
|
+
if (this.rangeFilter.queryRange(lo, hi)) return true;
|
|
267
|
+
}
|
|
268
|
+
return false;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
adaptFalsePositiveBox(min, max) {
|
|
272
|
+
const box = this.codec.normalizeBox(min, max);
|
|
273
|
+
const ranges = this._mortonOfBox(box);
|
|
274
|
+
let adapted = false;
|
|
275
|
+
for (const [lo, hi] of ranges) {
|
|
276
|
+
if (this.rangeFilter.adaptFalsePositive(lo, hi)) adapted = true;
|
|
277
|
+
}
|
|
278
|
+
return adapted;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
getStats() {
|
|
282
|
+
return {
|
|
283
|
+
...this.rangeFilter.getStats(),
|
|
284
|
+
bitsPerCoordinate: this.bitsPerCoordinate,
|
|
285
|
+
gridSize: 2 ** this.bitsPerCoordinate,
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
serialize() {
|
|
290
|
+
const writer = new BinaryWriter();
|
|
291
|
+
writer.uint32(MAGIC.SPATIAL);
|
|
292
|
+
writer.uint32(this.bitsPerCoordinate);
|
|
293
|
+
|
|
294
|
+
const filterData = this.rangeFilter.serialize();
|
|
295
|
+
writer.bytes(new Uint8Array(filterData));
|
|
296
|
+
|
|
297
|
+
return wrapCRC32(writer.toArrayBuffer());
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
static deserialize(buffer, options = {}) {
|
|
301
|
+
buffer = unwrapCRC32(buffer);
|
|
302
|
+
const reader = new BinaryReader(buffer);
|
|
303
|
+
if (reader.uint32() !== MAGIC.SPATIAL) throw new Error('Not a SpatialFilter binary.');
|
|
304
|
+
|
|
305
|
+
const bitsPerCoordinate = reader.uint32();
|
|
306
|
+
|
|
307
|
+
const filter = new SpatialFilter({
|
|
308
|
+
bitsPerCoordinate,
|
|
309
|
+
...options,
|
|
310
|
+
});
|
|
311
|
+
|
|
312
|
+
const filterData = reader.bytes();
|
|
313
|
+
filter.rangeFilter = RangeFilter.deserialize(filterData.buffer);
|
|
314
|
+
|
|
315
|
+
return filter;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
import { PointFilter } from './pointFilter.js';
|
|
2
|
+
import { BinaryWriter, BinaryReader, MAGIC, wrapCRC32, unwrapCRC32 } from './serialization.js';
|
|
3
|
+
|
|
4
|
+
function assertPositiveFinite(value, label) {
|
|
5
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
6
|
+
throw new Error(`${label} must be a positive finite number.`);
|
|
7
|
+
}
|
|
8
|
+
return value;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function assertNonNegativeFinite(value, label) {
|
|
12
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
13
|
+
throw new Error(`${label} must be a non-negative finite number.`);
|
|
14
|
+
}
|
|
15
|
+
return value;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function normalizeTimestamp(value) {
|
|
19
|
+
if (!Number.isFinite(value)) {
|
|
20
|
+
throw new Error('timestampMs must be a finite number.');
|
|
21
|
+
}
|
|
22
|
+
return Math.floor(value);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export class TemporalFilter {
|
|
26
|
+
constructor(options = {}) {
|
|
27
|
+
this.bucketDurationMs = assertPositiveFinite(options.bucketDurationMs ?? 60000, 'bucketDurationMs');
|
|
28
|
+
this.retentionDurationMs = assertPositiveFinite(options.retentionDurationMs ?? (24 * 60 * 60 * 1000), 'retentionDurationMs');
|
|
29
|
+
this.maxBuckets = Math.max(1, Math.ceil(this.retentionDurationMs / this.bucketDurationMs));
|
|
30
|
+
this.filterOptions = options.filterOptions ?? {};
|
|
31
|
+
this.buckets = new Array(this.maxBuckets).fill(null).map(() => ({ bucketId: Number.NaN, filter: null }));
|
|
32
|
+
this.adaptedFalsePositiveQueries = new Set();
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
_bucketId(timestampMs) {
|
|
36
|
+
return Math.floor(timestampMs / this.bucketDurationMs);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
_bucketIndex(bucketId) {
|
|
40
|
+
const mod = bucketId % this.maxBuckets;
|
|
41
|
+
return mod < 0 ? mod + this.maxBuckets : mod;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
_latestBucketInHorizon(nowBucketId) {
|
|
45
|
+
return nowBucketId;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
_oldestBucketInHorizon(nowBucketId) {
|
|
49
|
+
return nowBucketId - this.maxBuckets + 1;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
_createBucket(bucketId) {
|
|
53
|
+
return {
|
|
54
|
+
bucketId,
|
|
55
|
+
filter: new PointFilter({ initialCapacity: 32, fingerprintSize: 8, ...this.filterOptions }),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
_getOrCreateBucket(bucketId) {
|
|
60
|
+
const index = this._bucketIndex(bucketId);
|
|
61
|
+
const slot = this.buckets[index];
|
|
62
|
+
|
|
63
|
+
if (slot.filter && slot.bucketId === bucketId) {
|
|
64
|
+
return slot;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const fresh = this._createBucket(bucketId);
|
|
68
|
+
this.buckets[index] = fresh;
|
|
69
|
+
return fresh;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
_queryBucket(key, bucketId) {
|
|
73
|
+
const slot = this.buckets[this._bucketIndex(bucketId)];
|
|
74
|
+
if (!slot.filter || slot.bucketId !== bucketId) return false;
|
|
75
|
+
return slot.filter.query(key);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
_queryBucketSpan(key, startBucketId, endBucketId) {
|
|
79
|
+
for (let bucketId = startBucketId; bucketId <= endBucketId; bucketId++) {
|
|
80
|
+
if (this._queryBucket(key, bucketId)) {
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
_queryRangeSignature(key, startBucketId, endBucketId) {
|
|
88
|
+
return `${String(key)}:${startBucketId}:${endBucketId}`;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
_bucketRangeForWithinLast(durationMs, nowMs) {
|
|
92
|
+
const normalizedNow = normalizeTimestamp(nowMs);
|
|
93
|
+
const boundedDuration = assertNonNegativeFinite(durationMs, 'durationMs');
|
|
94
|
+
const nowBucketId = this._bucketId(normalizedNow);
|
|
95
|
+
const latest = this._latestBucketInHorizon(nowBucketId);
|
|
96
|
+
const oldest = this._oldestBucketInHorizon(nowBucketId);
|
|
97
|
+
const requestedStart = nowBucketId - Math.ceil(boundedDuration / this.bucketDurationMs);
|
|
98
|
+
const start = Math.max(oldest, requestedStart);
|
|
99
|
+
return { start, end: latest };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
_bucketRangeForBetweenAges(minAgeMs, maxAgeMs, nowMs) {
|
|
103
|
+
const normalizedNow = normalizeTimestamp(nowMs);
|
|
104
|
+
const minAge = assertNonNegativeFinite(minAgeMs, 'minAgeMs');
|
|
105
|
+
const maxAge = assertNonNegativeFinite(maxAgeMs, 'maxAgeMs');
|
|
106
|
+
if (minAge > maxAge) {
|
|
107
|
+
throw new Error('minAgeMs must be <= maxAgeMs.');
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const nowBucketId = this._bucketId(normalizedNow);
|
|
111
|
+
const oldest = this._oldestBucketInHorizon(nowBucketId);
|
|
112
|
+
const youngest = this._latestBucketInHorizon(nowBucketId);
|
|
113
|
+
const start = Math.max(oldest, nowBucketId - Math.ceil(maxAge / this.bucketDurationMs));
|
|
114
|
+
const end = Math.min(youngest, nowBucketId - Math.floor(minAge / this.bucketDurationMs));
|
|
115
|
+
return { start, end };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
_adaptFalsePositiveOverRange(key, startBucketId, endBucketId) {
|
|
119
|
+
if (startBucketId > endBucketId) return false;
|
|
120
|
+
|
|
121
|
+
const signature = this._queryRangeSignature(key, startBucketId, endBucketId);
|
|
122
|
+
if (this.adaptedFalsePositiveQueries.has(signature) && !this._queryBucketSpan(key, startBucketId, endBucketId)) {
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
let adapted = false;
|
|
127
|
+
for (let bucketId = startBucketId; bucketId <= endBucketId; bucketId++) {
|
|
128
|
+
const slot = this.buckets[this._bucketIndex(bucketId)];
|
|
129
|
+
if (!slot.filter || slot.bucketId !== bucketId) continue;
|
|
130
|
+
adapted = slot.filter.rejuvenate(key) || adapted;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
if (adapted && !this._queryBucketSpan(key, startBucketId, endBucketId)) {
|
|
134
|
+
this.adaptedFalsePositiveQueries.add(signature);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
return adapted;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
insertAt(key, timestampMs = Date.now()) {
|
|
141
|
+
const normalizedTimestamp = normalizeTimestamp(timestampMs);
|
|
142
|
+
const bucketId = this._bucketId(normalizedTimestamp);
|
|
143
|
+
this._getOrCreateBucket(bucketId).filter.insert(key);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
queryWithinLast(key, durationMs, nowMs = Date.now()) {
|
|
147
|
+
const range = this._bucketRangeForWithinLast(durationMs, nowMs);
|
|
148
|
+
return this._queryBucketSpan(key, range.start, range.end);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
queryAgo(key, ageMs, toleranceMs = this.bucketDurationMs, nowMs = Date.now()) {
|
|
152
|
+
const normalizedNow = normalizeTimestamp(nowMs);
|
|
153
|
+
const normalizedAge = assertNonNegativeFinite(ageMs, 'ageMs');
|
|
154
|
+
const normalizedTolerance = assertNonNegativeFinite(toleranceMs, 'toleranceMs');
|
|
155
|
+
const nowBucketId = this._bucketId(normalizedNow);
|
|
156
|
+
const centerBucket = nowBucketId - Math.ceil(normalizedAge / this.bucketDurationMs);
|
|
157
|
+
const toleranceBuckets = Math.floor(normalizedTolerance / this.bucketDurationMs);
|
|
158
|
+
const start = centerBucket - toleranceBuckets;
|
|
159
|
+
const end = centerBucket + toleranceBuckets;
|
|
160
|
+
|
|
161
|
+
return this.queryBetweenAges(
|
|
162
|
+
key,
|
|
163
|
+
Math.max(0, (nowBucketId - end) * this.bucketDurationMs),
|
|
164
|
+
Math.max(0, (nowBucketId - start) * this.bucketDurationMs),
|
|
165
|
+
nowMs
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
queryBetweenAges(key, minAgeMs, maxAgeMs, nowMs = Date.now()) {
|
|
170
|
+
const range = this._bucketRangeForBetweenAges(minAgeMs, maxAgeMs, nowMs);
|
|
171
|
+
if (range.start > range.end) return false;
|
|
172
|
+
return this._queryBucketSpan(key, range.start, range.end);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
adaptFalsePositiveWithinLast(key, durationMs, nowMs = Date.now()) {
|
|
176
|
+
const range = this._bucketRangeForWithinLast(durationMs, nowMs);
|
|
177
|
+
return this._adaptFalsePositiveOverRange(key, range.start, range.end);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
adaptFalsePositiveBetweenAges(key, minAgeMs, maxAgeMs, nowMs = Date.now()) {
|
|
181
|
+
const range = this._bucketRangeForBetweenAges(minAgeMs, maxAgeMs, nowMs);
|
|
182
|
+
if (range.start > range.end) return false;
|
|
183
|
+
return this._adaptFalsePositiveOverRange(key, range.start, range.end);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
adaptFalsePositiveAgo(key, ageMs, toleranceMs = this.bucketDurationMs, nowMs = Date.now()) {
|
|
187
|
+
const normalizedNow = normalizeTimestamp(nowMs);
|
|
188
|
+
const normalizedAge = assertNonNegativeFinite(ageMs, 'ageMs');
|
|
189
|
+
const normalizedTolerance = assertNonNegativeFinite(toleranceMs, 'toleranceMs');
|
|
190
|
+
const nowBucketId = this._bucketId(normalizedNow);
|
|
191
|
+
const centerBucket = nowBucketId - Math.ceil(normalizedAge / this.bucketDurationMs);
|
|
192
|
+
const toleranceBuckets = Math.floor(normalizedTolerance / this.bucketDurationMs);
|
|
193
|
+
const minAge = Math.max(0, (nowBucketId - (centerBucket + toleranceBuckets)) * this.bucketDurationMs);
|
|
194
|
+
const maxAge = Math.max(0, (nowBucketId - (centerBucket - toleranceBuckets)) * this.bucketDurationMs);
|
|
195
|
+
return this.adaptFalsePositiveBetweenAges(key, minAge, maxAge, nowMs);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
getStats(nowMs = Date.now()) {
|
|
199
|
+
const nowBucketId = this._bucketId(normalizeTimestamp(nowMs));
|
|
200
|
+
const oldest = this._oldestBucketInHorizon(nowBucketId);
|
|
201
|
+
let activeBuckets = 0;
|
|
202
|
+
|
|
203
|
+
for (const bucket of this.buckets) {
|
|
204
|
+
if (!bucket.filter) continue;
|
|
205
|
+
if (bucket.bucketId < oldest || bucket.bucketId > nowBucketId) continue;
|
|
206
|
+
activeBuckets++;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
return {
|
|
210
|
+
bucketDurationMs: this.bucketDurationMs,
|
|
211
|
+
retentionDurationMs: this.retentionDurationMs,
|
|
212
|
+
maxBuckets: this.maxBuckets,
|
|
213
|
+
activeBuckets,
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
serialize() {
|
|
218
|
+
const writer = new BinaryWriter();
|
|
219
|
+
writer.uint32(MAGIC.TEMPORAL);
|
|
220
|
+
writer.float64(this.bucketDurationMs);
|
|
221
|
+
writer.float64(this.retentionDurationMs);
|
|
222
|
+
writer.uint32(this.maxBuckets);
|
|
223
|
+
|
|
224
|
+
const populated = this.buckets.filter(b => b.filter);
|
|
225
|
+
writer.uint32(populated.length);
|
|
226
|
+
for (const bucket of populated) {
|
|
227
|
+
writer.uint32(bucket.bucketId >>> 0);
|
|
228
|
+
const filterData = bucket.filter.serialize();
|
|
229
|
+
writer.bytes(new Uint8Array(filterData));
|
|
230
|
+
}
|
|
231
|
+
return wrapCRC32(writer.toArrayBuffer());
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
static deserialize(buffer, options = {}) {
|
|
235
|
+
buffer = unwrapCRC32(buffer);
|
|
236
|
+
const reader = new BinaryReader(buffer);
|
|
237
|
+
if (reader.uint32() !== MAGIC.TEMPORAL) throw new Error('Not a TemporalFilter binary.');
|
|
238
|
+
|
|
239
|
+
const filter = new TemporalFilter({
|
|
240
|
+
bucketDurationMs: reader.float64(),
|
|
241
|
+
retentionDurationMs: reader.float64(),
|
|
242
|
+
...options,
|
|
243
|
+
});
|
|
244
|
+
filter.maxBuckets = reader.uint32();
|
|
245
|
+
if (filter.buckets.length !== filter.maxBuckets) {
|
|
246
|
+
filter.buckets = new Array(filter.maxBuckets).fill(null)
|
|
247
|
+
.map(() => ({ bucketId: Number.NaN, filter: null }));
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
const count = reader.uint32();
|
|
251
|
+
for (let i = 0; i < count; i++) {
|
|
252
|
+
const bucketId = reader.uint32();
|
|
253
|
+
const filterData = reader.bytes();
|
|
254
|
+
const f = PointFilter.deserialize(filterData.buffer);
|
|
255
|
+
filter.buckets[filter._bucketIndex(bucketId)] = { bucketId, filter: f };
|
|
256
|
+
}
|
|
257
|
+
return filter;
|
|
258
|
+
}
|
|
259
|
+
}
|
package/src/zeno.js
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
export function zenoGrowthFactor(growthCoefficient = 2) {
|
|
2
|
+
if (!Number.isInteger(growthCoefficient) || growthCoefficient <= 0) {
|
|
3
|
+
throw new Error('growthCoefficient must be a positive integer.');
|
|
4
|
+
}
|
|
5
|
+
return 2 ** (1 / growthCoefficient);
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export function zenoStretchIndex(index, epoch, growthCoefficient = 2) {
|
|
9
|
+
if (!Number.isInteger(index) || index < 0) {
|
|
10
|
+
throw new Error('index must be a non-negative integer.');
|
|
11
|
+
}
|
|
12
|
+
if (!Number.isInteger(epoch) || epoch < 0 || epoch > growthCoefficient) {
|
|
13
|
+
throw new Error('epoch must be an integer in [0, growthCoefficient].');
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
return Math.floor(index * (2 ** (epoch / growthCoefficient)));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function zenoShouldSacrifice(epoch, growthCoefficient = 2) {
|
|
20
|
+
return epoch > 0 && epoch % growthCoefficient === 0;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function zenoDirectoryAddress(slot) {
|
|
24
|
+
if (!Number.isInteger(slot) || slot < 0) {
|
|
25
|
+
throw new Error('slot must be a non-negative integer.');
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const j = slot + 1;
|
|
29
|
+
const level = Math.floor(Math.log2(j));
|
|
30
|
+
const floorHalf = level >>> 1;
|
|
31
|
+
const ceilHalf = (level + 1) >>> 1;
|
|
32
|
+
const blocksBeforeLevel = (2 ** floorHalf) * (2 + (level & 1)) - 2;
|
|
33
|
+
const blockInLevel = (j >>> ceilHalf) & ((2 ** floorHalf) - 1);
|
|
34
|
+
const offsetInBlock = j & ((2 ** ceilHalf) - 1);
|
|
35
|
+
|
|
36
|
+
return {
|
|
37
|
+
level,
|
|
38
|
+
blockIndex: blocksBeforeLevel + blockInLevel,
|
|
39
|
+
offset: offsetInBlock,
|
|
40
|
+
blocksBeforeLevel,
|
|
41
|
+
blockInLevel,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export class ZenoExpansionPolicy {
|
|
46
|
+
constructor(options = {}) {
|
|
47
|
+
this.growthCoefficient = options.growthCoefficient ?? 2;
|
|
48
|
+
this.expansionThreshold = options.expansionThreshold ?? 0.8;
|
|
49
|
+
this.epoch = 0;
|
|
50
|
+
this.period = 0;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
nextCapacity(currentCapacity) {
|
|
54
|
+
if (!Number.isInteger(currentCapacity) || currentCapacity <= 0) {
|
|
55
|
+
throw new Error('currentCapacity must be a positive integer.');
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const nextEpoch = this.epoch + 1;
|
|
59
|
+
if (zenoShouldSacrifice(nextEpoch, this.growthCoefficient)) {
|
|
60
|
+
return currentCapacity * 2;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return Math.ceil(currentCapacity * zenoGrowthFactor(this.growthCoefficient));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
mapIndex(index) {
|
|
67
|
+
const nextEpoch = this.epoch + 1;
|
|
68
|
+
if (zenoShouldSacrifice(nextEpoch, this.growthCoefficient)) {
|
|
69
|
+
return { index: index * 2, sacrifice: true };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return {
|
|
73
|
+
index: zenoStretchIndex(index, nextEpoch % this.growthCoefficient, this.growthCoefficient),
|
|
74
|
+
sacrifice: false,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
advance() {
|
|
79
|
+
this.epoch++;
|
|
80
|
+
if (this.epoch % this.growthCoefficient === 0) {
|
|
81
|
+
this.period++;
|
|
82
|
+
this.epoch = 0;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|