probe-filters 1.0.0

package/src/multiFilter.js

@@ -0,0 +1,307 @@
+import { PointFilter } from './pointFilter.js';
+import { RangeFilter } from './rangeFilter.js';
+import { SpatialFilter } from './spatialFilter.js';
+import { TemporalFilter } from './temporalFilter.js';
+import { FILTER_MERGE_OPERATORS, normalizeFilterMergeOperator } from './mergeOperators.js';
+import { BinaryWriter, BinaryReader, MAGIC, wrapCRC32, unwrapCRC32 } from './serialization.js';
+
+
+function isLiveAlephState(state) {
+    return state !== 0 && state !== 3;
+}
+
+function insertMotherHash(targetFilter, motherHash, state = 1) {
+    const fingerprintBits = Math.min(targetFilter.currentFingerprintLengthForNewEntries, 32 - targetFilter.bitsForSlotAddress);
+    if (fingerprintBits <= 0) {
+        throw new Error('Cannot merge into target filter with zero fingerprint bits.');
+    }
+
+    const addressSpace = 2 ** targetFilter.bitsForSlotAddress;
+    const canonicalSlot = motherHash % addressSpace;
+    const shiftedHash = Math.floor(motherHash / addressSpace);
+    const fingerprint = shiftedHash % (2 ** fingerprintBits);
+    targetFilter._performInsertion(canonicalSlot, fingerprint, state, fingerprintBits, motherHash);
+}
+
+function mergeAlephFilters(targetFilter, sourceFilter) {
+    for (let i = 0; i < sourceFilter.currentCapacity; i++) {
+        const state = sourceFilter.states[i];
+        if (!isLiveAlephState(state)) continue;
+        insertMotherHash(targetFilter, sourceFilter.motherHashes[i], state);
+    }
+}
+
+function mergeRangeFilters(targetFilter, sourceFilter) {
+    for (const partition of sourceFilter.partitions.keys()) {
+        targetFilter.insert(partition * sourceFilter.partitionSize);
+    }
+}
+
+function mergeSpatialFilters(targetFilter, sourceFilter) {
+    const source = sourceFilter.rangeFilter;
+    const target = targetFilter.rangeFilter;
+
+    for (const partition of source.partitions.keys()) {
+        target.insert(partition * source.partitionSize);
+    }
+}
+
+function mergeTemporalFilters(targetFilter, sourceFilter) {
+    for (const bucket of sourceFilter.buckets) {
+        if (!bucket.filter || Number.isNaN(bucket.bucketId)) continue;
+        const targetBucket = targetFilter._getOrCreateBucket(bucket.bucketId);
+        mergeAlephFilters(targetBucket.filter, bucket.filter);
+    }
+}
+
+export class MultiFilter {
+    constructor(options = {}) {
+        this.point = options.point === false ? null : new PointFilter(options.pointOptions ?? options);
+        this.range = options.range === false ? null : new RangeFilter(options.rangeOptions ?? {});
+        this.spatial = options.spatial === false ? null : new SpatialFilter(options.spatialOptions ?? {});
+        this.temporal = options.temporal === false ? null : new TemporalFilter(options.temporalOptions ?? {});
+    }
+
+    insert(key) {
+        if (!this.point) throw new Error('Point filtering is disabled.');
+        this.point.insert(key);
+    }
+
+    set(key) {
+        return this.insert(key);
+    }
+
+    delete(key) {
+        if (!this.point) throw new Error('Point filtering is disabled.');
+        return this.point.delete(key);
+    }
+
+    remove(key) {
+        return this.delete(key);
+    }
+
+    query(key) {
+        if (!this.point) throw new Error('Point filtering is disabled.');
+        return this.point.query(key);
+    }
+
+    has(key) {
+        return this.query(key);
+    }
+
+    insertRangeKey(key) {
+        if (!this.range) throw new Error('Range filtering is disabled.');
+        this.range.insert(key);
+    }
+
+    setRangeKey(key) {
+        return this.insertRangeKey(key);
+    }
+
+    deleteRangeKey(key) {
+        if (!this.range) throw new Error('Range filtering is disabled.');
+        return this.range.delete(key);
+    }
+
+    removeRangeKey(key) {
+        return this.deleteRangeKey(key);
+    }
+
+    queryRange(start, end) {
+        if (!this.range) throw new Error('Range filtering is disabled.');
+        return this.range.queryRange(start, end);
+    }
+
+    hasRange(start, end) {
+        return this.queryRange(start, end);
+    }
+
+    insertSpatialPoint(point) {
+        if (!this.spatial) throw new Error('Spatial filtering is disabled.');
+        this.spatial.insert(point);
+    }
+
+    setSpatialPoint(point) {
+        return this.insertSpatialPoint(point);
+    }
+
+    deleteSpatialPoint(point) {
+        if (!this.spatial) throw new Error('Spatial filtering is disabled.');
+        return this.spatial.delete(point);
+    }
+
+    removeSpatialPoint(point) {
+        return this.deleteSpatialPoint(point);
+    }
+
+    queryBox(min, max) {
+        if (!this.spatial) throw new Error('Spatial filtering is disabled.');
+        return this.spatial.queryBox(min, max);
+    }
+
+    hasBox(min, max) {
+        return this.queryBox(min, max);
+    }
+
+    insertSpatialBox(min, max, id = null) {
+        if (!this.spatial) throw new Error('Spatial filtering is disabled.');
+        this.spatial.insertBox(min, max, id);
+    }
+
+    setSpatialBox(min, max, id = null) {
+        return this.insertSpatialBox(min, max, id);
+    }
+
+    insertSpatialCircle(center, radius, id = null) {
+        if (!this.spatial) throw new Error('Spatial filtering is disabled.');
+        this.spatial.insertCircle(center, radius, id);
+    }
+
+    setSpatialCircle(center, radius, id = null) {
+        return this.insertSpatialCircle(center, radius, id);
+    }
+
+    insertTemporalKey(key, timestampMs = Date.now()) {
+        if (!this.temporal) throw new Error('Temporal filtering is disabled.');
+        this.temporal.insertAt(key, timestampMs);
+    }
+
+    setTemporalKey(key, timestampMs = Date.now()) {
+        return this.insertTemporalKey(key, timestampMs);
+    }
+
+    queryWithinLast(key, durationMs, nowMs = Date.now()) {
+        if (!this.temporal) throw new Error('Temporal filtering is disabled.');
+        return this.temporal.queryWithinLast(key, durationMs, nowMs);
+    }
+
+    queryAgo(key, ageMs, toleranceMs, nowMs = Date.now()) {
+        if (!this.temporal) throw new Error('Temporal filtering is disabled.');
+        return this.temporal.queryAgo(key, ageMs, toleranceMs, nowMs);
+    }
+
+    queryBetweenAges(key, minAgeMs, maxAgeMs, nowMs = Date.now()) {
+        if (!this.temporal) throw new Error('Temporal filtering is disabled.');
+        return this.temporal.queryBetweenAges(key, minAgeMs, maxAgeMs, nowMs);
+    }
+
+    adaptTemporalFalsePositiveWithinLast(key, durationMs, nowMs = Date.now()) {
+        if (!this.temporal) throw new Error('Temporal filtering is disabled.');
+        return this.temporal.adaptFalsePositiveWithinLast(key, durationMs, nowMs);
+    }
+
+    adaptTemporalFalsePositiveAgo(key, ageMs, toleranceMs, nowMs = Date.now()) {
+        if (!this.temporal) throw new Error('Temporal filtering is disabled.');
+        return this.temporal.adaptFalsePositiveAgo(key, ageMs, toleranceMs, nowMs);
+    }
+
+    adaptTemporalFalsePositiveBetweenAges(key, minAgeMs, maxAgeMs, nowMs = Date.now()) {
+        if (!this.temporal) throw new Error('Temporal filtering is disabled.');
+        return this.temporal.adaptFalsePositiveBetweenAges(key, minAgeMs, maxAgeMs, nowMs);
+    }
+
+    adaptFalsePositive(query, kind = 'point') {
+        if (kind === 'point') {
+            if (!this.point) throw new Error('Point filtering is disabled.');
+            return this.point.rejuvenate(query);
+        }
+
+        if (kind === 'range') {
+            if (!this.range) throw new Error('Range filtering is disabled.');
+            if (Array.isArray(query) && query.length === 2) {
+                return this.range.adaptFalsePositive(query[0], query[1]);
+            }
+            return this.range.adaptFalsePositive(query, query);
+        }
+
+        if (kind === 'spatial') {
+            if (!this.spatial) throw new Error('Spatial filtering is disabled.');
+            return this.spatial.adaptFalsePositiveBox(query.min, query.max);
+        }
+
+        if (kind === 'temporal') {
+            if (!this.temporal) throw new Error('Temporal filtering is disabled.');
+            if (query.durationMs !== undefined) {
+                return this.temporal.adaptFalsePositiveWithinLast(query.key, query.durationMs, query.nowMs ?? Date.now());
+            }
+            if (query.ageMs !== undefined) {
+                return this.temporal.adaptFalsePositiveAgo(query.key, query.ageMs, query.toleranceMs, query.nowMs ?? Date.now());
+            }
+            if (query.minAgeMs !== undefined && query.maxAgeMs !== undefined) {
+                return this.temporal.adaptFalsePositiveBetweenAges(query.key, query.minAgeMs, query.maxAgeMs, query.nowMs ?? Date.now());
+            }
+            throw new Error('Temporal adaptation requires durationMs, ageMs, or minAgeMs/maxAgeMs.');
+        }
+
+        throw new Error(`Unknown filter kind: ${kind}`);
+    }
+
+    mergeFrom(other, operator = FILTER_MERGE_OPERATORS.UNION) {
+        const normalizedOperator = normalizeFilterMergeOperator(operator);
+        if (!(other instanceof MultiFilter)) {
+            throw new Error('mergeFrom expects another MultiFilter instance.');
+        }
+
+        if (normalizedOperator === FILTER_MERGE_OPERATORS.UNION) {
+            if (this.point && other.point) mergeAlephFilters(this.point, other.point);
+            if (this.range && other.range) mergeRangeFilters(this.range, other.range);
+            if (this.spatial && other.spatial) mergeSpatialFilters(this.spatial, other.spatial);
+            if (this.temporal && other.temporal) mergeTemporalFilters(this.temporal, other.temporal);
+            return this;
+        }
+
+        throw new Error(`Unsupported merge operator: ${normalizedOperator}`);
+    }
+
+    getStats() {
+        return {
+            point: this.point?.getStats() ?? null,
+            range: this.range?.getStats() ?? null,
+            spatial: this.spatial?.getStats() ?? null,
+            temporal: this.temporal?.getStats?.() ?? null,
+        };
+    }
+
+    serialize() {
+        const parts = [];
+        if (this.point) parts.push({ type: 'point', filter: this.point });
+        if (this.range) parts.push({ type: 'range', filter: this.range });
+        if (this.spatial) parts.push({ type: 'spatial', filter: this.spatial });
+        if (this.temporal) parts.push({ type: 'temporal', filter: this.temporal });
+
+        const writer = new BinaryWriter();
+        writer.uint32(MAGIC.MULTI);
+        writer.uint32(parts.length);
+        for (const { type, filter } of parts) {
+            writer.uint32(type.length);
+            for (let i = 0; i < type.length; i++) writer.uint8(type.charCodeAt(i));
+            const data = filter.serialize();
+            writer.bytes(new Uint8Array(data));
+        }
+        return wrapCRC32(writer.toArrayBuffer());
+    }
+
+    static deserialize(buffer, options = {}) {
+        buffer = unwrapCRC32(buffer);
+        const reader = new BinaryReader(buffer);
+        if (reader.uint32() !== MAGIC.MULTI) throw new Error('Not a MultiFilter binary.');
+
+        const filter = new MultiFilter(options);
+        const partCount = reader.uint32();
+        for (let i = 0; i < partCount; i++) {
+            const typeLen = reader.uint32();
+            let type = '';
+            for (let j = 0; j < typeLen; j++) type += String.fromCharCode(reader.uint8());
+            const data = reader.bytes();
+
+            switch (type) {
+                case 'point': filter.point = PointFilter.deserialize(data.buffer); break;
+                case 'range': filter.range = RangeFilter.deserialize(data.buffer); break;
+                case 'spatial': filter.spatial = SpatialFilter.deserialize(data.buffer); break;
+                case 'temporal': filter.temporal = TemporalFilter.deserialize(data.buffer); break;
+                default: throw new Error(`Unknown serialized filter type: ${type}`);
+            }
+        }
+        return filter;
+    }
+}

package/src/packedBitset.js

@@ -0,0 +1,74 @@
+const WORD_BITS = 32;
+
+export class PackedBitset {
+    constructor(size) {
+        this.size = size;
+        this.words = new Uint32Array(Math.ceil(size / WORD_BITS));
+    }
+
+    get(index) {
+        return (this.words[index >>> 5] >>> (index & 31)) & 1;
+    }
+
+    set(index, value) {
+        const wordIndex = index >>> 5;
+        const mask = 1 << (index & 31);
+
+        if (value) {
+            this.words[wordIndex] |= mask;
+        } else {
+            this.words[wordIndex] &= ~mask;
+        }
+    }
+
+    clear() {
+        this.words.fill(0);
+    }
+
+    rank1(index) {
+        if (index < 0) return 0;
+        const capped = Math.min(index, this.size - 1);
+        const fullWords = capped >>> 5;
+        let count = 0;
+
+        for (let word = 0; word < fullWords; word++) {
+            count += popcount32(this.words[word]);
+        }
+
+        const bitsInLastWord = (capped & 31) + 1;
+        const mask = bitsInLastWord === WORD_BITS ? 0xffffffff : (2 ** bitsInLastWord) - 1;
+        return count + popcount32(this.words[fullWords] & mask);
+    }
+
+    select1(rank) {
+        if (rank <= 0) return -1;
+
+        let remaining = rank;
+        for (let wordIndex = 0; wordIndex < this.words.length; wordIndex++) {
+            const word = this.words[wordIndex];
+            const count = popcount32(word);
+            if (remaining > count) {
+                remaining -= count;
+                continue;
+            }
+
+            for (let bit = 0; bit < WORD_BITS; bit++) {
+                const index = (wordIndex * WORD_BITS) + bit;
+                if (index >= this.size) return -1;
+                if (((word >>> bit) & 1) === 1) {
+                    remaining--;
+                    if (remaining === 0) return index;
+                }
+            }
+        }
+
+        return -1;
+    }
+}
+
+export function popcount32(value) {
+    let v = value >>> 0;
+    v -= (v >>> 1) & 0x55555555;
+    v = (v & 0x33333333) + ((v >>> 2) & 0x33333333);
+    return (((v + (v >>> 4)) & 0x0f0f0f0f) * 0x01010101) >>> 24;
+}

package/src/packedRsqf.js

@@ -0,0 +1,302 @@
+import { djb2 } from './pointFilter.js';
+import { PackedBitset } from './packedBitset.js';
+
+const EMPTY = 0xffffffff;
+
+export class PackedRSQF {
+    constructor(options = {}) {
+        this.capacity = options.capacity ?? 256;
+        this.remainderBits = options.remainderBits ?? 12;
+        this.hashFunction = options.hashFunction ?? djb2;
+        this.maxLoadFactor = options.maxLoadFactor ?? 0.9;
+
+        if (Math.log2(this.capacity) % 1 !== 0) {
+            throw new Error('capacity must be a power of 2.');
+        }
+        if (this.remainderBits <= 0 || this.remainderBits > 31) {
+            throw new Error('remainderBits must be in [1, 31].');
+        }
+
+        this.addressBits = Math.log2(this.capacity);
+        this.remainderMask = (2 ** this.remainderBits) - 1;
+        this.remainders = new Uint32Array(this.capacity);
+        this.remainders.fill(EMPTY);
+        this.quotients = new Uint32Array(this.capacity);
+        this.quotients.fill(EMPTY);
+        this.occupied = new PackedBitset(this.capacity);
+        this.shifted = new PackedBitset(this.capacity);
+        this.continuation = new PackedBitset(this.capacity);
+        this.runends = new PackedBitset(this.capacity);
+        this.blockOffsets = new Uint8Array(Math.ceil(this.capacity / 64));
+        this.count = 0;
+    }
+
+    _hashKey(key) {
+        return this.hashFunction(String(key), 0) >>> 0;
+    }
+
+    _quotientAndRemainder(key) {
+        const hash = this._hashKey(key);
+        const quotient = hash & (this.capacity - 1);
+        const remainder = Math.floor(hash / this.capacity) & this.remainderMask;
+        return { quotient, remainder };
+    }
+
+    _isEmpty(index) {
+        return this.remainders[index] === EMPTY;
+    }
+
+    _isClusterStart(index) {
+        return !this._isEmpty(index) && this.shifted.get(index) === 0;
+    }
+
+    _findClusterStart(index) {
+        let current = index;
+        for (let i = 0; i < this.capacity; i++) {
+            if (this._isEmpty(current) || this.shifted.get(current) === 0) {
+                return current;
+            }
+            current = (current - 1 + this.capacity) % this.capacity;
+        }
+        return index;
+    }
+
+    _runStart(quotient) {
+        if (this.occupied.get(quotient) === 0) return -1;
+
+        const clusterStart = this._findClusterStart(quotient);
+        const runRankBeforeQuotient = this.occupied.rank1(quotient) - this.occupied.get(quotient);
+        const runRankBeforeCluster = clusterStart === 0 ? 0 : this.occupied.rank1(clusterStart - 1);
+        let runsToSkip = runRankBeforeQuotient - runRankBeforeCluster;
+        let current = clusterStart;
+
+        while (runsToSkip > 0) {
+            current = (current + 1) % this.capacity;
+            while (this.continuation.get(current) === 1) {
+                current = (current + 1) % this.capacity;
+            }
+            runsToSkip--;
+        }
+
+        return current;
+    }
+
+    _runEnd(runStart) {
+        if (runStart === -1) return -1;
+        let current = runStart;
+        for (let i = 1; i < this.capacity; i++) {
+            const next = (current + 1) % this.capacity;
+            if (this._isEmpty(next) || this.continuation.get(next) === 0) {
+                return current;
+            }
+            current = next;
+        }
+        return current;
+    }
+
+    _findInsertionPoint(quotient, remainder) {
+        const runStart = this._runStart(quotient);
+        if (runStart === -1) {
+            if (this._isEmpty(quotient) || this.shifted.get(quotient) === 0) return quotient;
+
+            const clusterStart = this._findClusterStart(quotient);
+            let precedingRuns = this.occupied.rank1(quotient) - (clusterStart === 0 ? 0 : this.occupied.rank1(clusterStart - 1));
+            let current = clusterStart;
+
+            while (precedingRuns > 0) {
+                current = (this._runEnd(current) + 1) % this.capacity;
+                precedingRuns--;
+            }
+
+            return current;
+        }
+
+        let current = runStart;
+        for (let i = 0; i < this.capacity; i++) {
+            if (this.remainders[current] >= remainder) return current;
+
+            const next = (current + 1) % this.capacity;
+            if (this._isEmpty(next) || this.continuation.get(next) === 0) {
+                return next;
+            }
+            current = next;
+        }
+
+        return runStart;
+    }
+
+    _findEmptyFrom(start) {
+        for (let distance = 0; distance < this.capacity; distance++) {
+            const index = (start + distance) % this.capacity;
+            if (this._isEmpty(index)) return index;
+        }
+        return -1;
+    }
+
+    _copySlot(dest, src) {
+        this.remainders[dest] = this.remainders[src];
+        this.quotients[dest] = this.quotients[src];
+        this.shifted.set(dest, this.shifted.get(src));
+        this.continuation.set(dest, this.continuation.get(src));
+    }
+
+    _clearSlot(index) {
+        this.remainders[index] = EMPTY;
+        this.quotients[index] = EMPTY;
+        this.shifted.set(index, 0);
+        this.continuation.set(index, 0);
+    }
+
+    _rebuildMetadata() {
+        this.occupied.clear();
+        this.shifted.clear();
+        this.continuation.clear();
+        this.runends.clear();
+        this.blockOffsets.fill(0);
+
+        for (let index = 0; index < this.capacity; index++) {
+            if (this._isEmpty(index)) continue;
+            const quotient = this.quotients[index];
+            this.occupied.set(quotient, 1);
+            this.shifted.set(index, quotient !== index);
+
+            const previous = (index - 1 + this.capacity) % this.capacity;
+            if (!this._isEmpty(previous) && this.quotients[previous] === quotient) {
+                this.continuation.set(index, 1);
+            }
+
+            const next = (index + 1) % this.capacity;
+            if (this._isEmpty(next) || this.quotients[next] !== quotient) {
+                this.runends.set(index, 1);
+            }
+        }
+
+        for (let block = 0; block < this.blockOffsets.length; block++) {
+            const blockStart = block * 64;
+            let offset = 0;
+            while (blockStart + offset < this.capacity && offset < 255 && this.shifted.get(blockStart + offset) === 1) {
+                offset++;
+            }
+            this.blockOffsets[block] = offset;
+        }
+    }
+
+    _runEndByRank(quotient) {
+        if (this.occupied.get(quotient) === 0) return -1;
+        return this.runends.select1(this.occupied.rank1(quotient));
+    }
+
+    _shiftRight(start, empty) {
+        let current = empty;
+        while (current !== start) {
+            const previous = (current - 1 + this.capacity) % this.capacity;
+            this._copySlot(current, previous);
+            this.shifted.set(current, 1);
+            current = previous;
+        }
+        this._clearSlot(start);
+    }
+
+    _writeInsertedSlot(index, quotient, remainder, runStartBeforeInsert) {
+        this.remainders[index] = remainder >>> 0;
+        this.quotients[index] = quotient >>> 0;
+        this.shifted.set(index, index !== quotient);
+
+        if (runStartBeforeInsert === -1 || index === quotient || index === runStartBeforeInsert) {
+            this.continuation.set(index, 0);
+        } else {
+            this.continuation.set(index, 1);
+        }
+
+        const next = (index + 1) % this.capacity;
+        if (runStartBeforeInsert !== -1 && index === runStartBeforeInsert && !this._isEmpty(next)) {
+            this.continuation.set(next, 1);
+        }
+    }
+
+    insert(key) {
+        if ((this.count + 1) / this.capacity > this.maxLoadFactor) {
+            throw new Error('PackedRSQF is above max load factor; expand before inserting.');
+        }
+
+        const { quotient, remainder } = this._quotientAndRemainder(key);
+        if (this.query(key)) return false;
+
+        const runStartBeforeInsert = this._runStart(quotient);
+        const insertionPoint = this._findInsertionPoint(quotient, remainder);
+        const empty = this._findEmptyFrom(insertionPoint);
+        if (empty === -1) throw new Error('PackedRSQF is full.');
+
+        if (empty !== insertionPoint) {
+            this._shiftRight(insertionPoint, empty);
+        }
+
+        this.occupied.set(quotient, 1);
+        this._writeInsertedSlot(insertionPoint, quotient, remainder, runStartBeforeInsert);
+        this._rebuildMetadata();
+        this.count++;
+        return true;
+    }
+
+    query(key) {
+        const { quotient, remainder } = this._quotientAndRemainder(key);
+        for (let index = 0; index < this.capacity; index++) {
+            if (this.quotients[index] === quotient && this.remainders[index] === remainder) return true;
+        }
+        return false;
+    }
+
+    _slotFor(key) {
+        const { quotient, remainder } = this._quotientAndRemainder(key);
+        for (let index = 0; index < this.capacity; index++) {
+            if (this.quotients[index] === quotient && this.remainders[index] === remainder) return { slot: index, quotient };
+        }
+        return { slot: -1, quotient };
+    }
+
+    delete(key) {
+        const { slot, quotient } = this._slotFor(key);
+        if (slot === -1) return false;
+
+        const runStart = this._runStart(quotient);
+        const runEnd = this._runEnd(runStart);
+        const runLength = runEnd >= runStart ? (runEnd - runStart + 1) : (this.capacity - runStart + runEnd + 1);
+
+        this._deleteSlot(slot, quotient);
+        if (runLength === 1) {
+            this.occupied.set(quotient, 0);
+        }
+        this._rebuildMetadata();
+        this.count--;
+        return true;
+    }
+
+    _deleteSlot(slot, quotient) {
+        let current = slot;
+        while (true) {
+            const next = (current + 1) % this.capacity;
+            if (this._isEmpty(next) || this.quotients[next] === next) {
+                this._clearSlot(current);
+                break;
+            }
+
+            this._copySlot(current, next);
+            current = next;
+        }
+
+    }
+
+    metadataBitsPerSlot() {
+        return 2.125;
+    }
+
+    debugMetadata(index) {
+        return {
+            occupied: this.occupied.get(index),
+            shifted: this.shifted.get(index),
+            continuation: this.continuation.get(index),
+            runend: this.runends.get(index),
+            blockOffset: this.blockOffsets[index >>> 6],
+        };
+    }
+}