probe-filters 1.0.0

package/src/rangeFilter.js

@@ -0,0 +1,298 @@
+import { djb2 } from './pointFilter.js';
+import { BinaryWriter, BinaryReader, MAGIC, wrapCRC32, unwrapCRC32 } from './serialization.js';
+
+function nextPowerOfTwo(value) {
+    if (value <= 1) return 1;
+    return 2 ** Math.ceil(Math.log2(value));
+}
+
+function normalizeKey(key) {
+    if (!Number.isInteger(key) || key < 0) {
+        throw new Error('RangeFilter keys must be non-negative integers.');
+    }
+    return key >>> 0;
+}
+
+export class RangeFilter {
+    constructor(options = {}) {
+        const maxRangeLength = options.maxRangeLength ?? 1024;
+        this.partitionSize = options.partitionSize ?? nextPowerOfTwo(maxRangeLength);
+        this.partitionBits = Math.log2(this.partitionSize);
+        this.fingerprintBits = options.fingerprintBits ?? 12;
+        this.maxFingerprintBits = options.maxFingerprintBits ?? 24;
+        this.hashFunction = options.hashFunction ?? djb2;
+        this.expansionLevel = 0;
+        this.partitions = new Map();
+        this.boxes = new Map();
+        this.partitionToBoxKeys = new Map();
+    }
+
+    _partitionOf(key) {
+        return Math.floor(key / this.partitionSize) >>> 0;
+    }
+
+    _partitionHash(partition) {
+        return this.hashFunction(String(partition), 0) >>> 0;
+    }
+
+    _fingerprint(partition, length) {
+        if (length <= 0) return 0;
+        const hash = this._partitionHash(partition);
+        return hash % (2 ** length);
+    }
+
+    _boxKey(length, fingerprint) {
+        return `${length}:${fingerprint}`;
+    }
+
+    _getOrCreateBox(length, fingerprint) {
+        const key = this._boxKey(length, fingerprint);
+        let box = this.boxes.get(key);
+        if (!box) {
+            box = { length, fingerprint, partitions: new Map(), extensionBits: Math.max(0, length - this.fingerprintBits), age: 0 };
+            this.boxes.set(key, box);
+        }
+        return box;
+    }
+
+    _removePartitionFromBox(entry) {
+        const key = this._boxKey(entry.length, entry.fingerprint);
+        const box = this.boxes.get(key);
+        if (!box) return;
+
+        box.partitions.delete(entry.partition);
+        if (box.partitions.size === 0) {
+            this.boxes.delete(key);
+        }
+
+        const boxKeys = this.partitionToBoxKeys.get(entry.partition);
+        if (boxKeys) {
+            boxKeys.delete(key);
+            if (boxKeys.size === 0) this.partitionToBoxKeys.delete(entry.partition);
+        }
+    }
+
+    _placePartition(entry) {
+        entry.fingerprint = this._fingerprint(entry.partition, entry.length);
+        const box = this._getOrCreateBox(entry.length, entry.fingerprint);
+        box.partitions.set(entry.partition, entry);
+        const boxKey = this._boxKey(entry.length, entry.fingerprint);
+        let boxKeys = this.partitionToBoxKeys.get(entry.partition);
+        if (!boxKeys) {
+            boxKeys = new Set();
+            this.partitionToBoxKeys.set(entry.partition, boxKeys);
+        }
+        boxKeys.add(boxKey);
+    }
+
+    insert(key) {
+        const normalized = normalizeKey(key);
+        const partition = this._partitionOf(normalized);
+
+        let entry = this.partitions.get(partition);
+        if (!entry) {
+            entry = {
+                partition,
+                length: this.fingerprintBits,
+                fingerprint: this._fingerprint(partition, this.fingerprintBits),
+                falsePositiveCount: 0,
+                age: 0,
+                count: 1,
+            };
+            this.partitions.set(partition, entry);
+            this._placePartition(entry);
+        } else {
+            entry.count++;
+        }
+    }
+
+    delete(key) {
+        const normalized = normalizeKey(key);
+        const partition = this._partitionOf(normalized);
+
+        const entry = this.partitions.get(partition);
+        if (!entry) return false;
+
+        entry.count--;
+        if (entry.count <= 0) {
+            this._removePartitionFromBox(entry);
+            this.partitions.delete(partition);
+        }
+        return true;
+    }
+
+    _subqueries(start, end) {
+        const normalizedStart = normalizeKey(start);
+        const normalizedEnd = normalizeKey(end);
+        if (normalizedStart > normalizedEnd) {
+            throw new Error('Range start must be <= range end.');
+        }
+
+        const startPartition = this._partitionOf(normalizedStart);
+        const endPartition = this._partitionOf(normalizedEnd);
+        const subqueries = [];
+
+        for (let partition = startPartition; partition <= endPartition; partition++) {
+            subqueries.push({ partition });
+        }
+
+        return subqueries;
+    }
+
+    _matchingBoxes(partition) {
+        const matches = [];
+        const hash = this._partitionHash(partition);
+        for (let length = 1; length <= this.maxFingerprintBits; length++) {
+            const fingerprint = hash % (2 ** length);
+            const box = this.boxes.get(this._boxKey(length, fingerprint));
+            if (box) matches.push(box);
+        }
+        return matches;
+    }
+
+    queryRange(start, end) {
+        for (const subquery of this._subqueries(start, end)) {
+            if (this._matchingBoxes(subquery.partition).length > 0) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    _splitBoxAwayFromQuery(box, queryPartition) {
+        let newLength = box.length + 1;
+        const entries = Array.from(box.partitions.values());
+
+        while (newLength <= this.maxFingerprintBits) {
+            const queryFingerprint = this._fingerprint(queryPartition, newLength);
+            const stillCollides = entries.some(entry => entry.partition !== queryPartition && this._fingerprint(entry.partition, newLength) === queryFingerprint);
+            if (!stillCollides) break;
+            newLength++;
+        }
+
+        if (newLength > this.maxFingerprintBits) {
+            return false;
+        }
+
+        const oldKey = this._boxKey(box.length, box.fingerprint);
+        this.boxes.delete(oldKey);
+        for (const entry of entries) {
+            const boxKeys = this.partitionToBoxKeys.get(entry.partition);
+            if (boxKeys) {
+                boxKeys.delete(oldKey);
+                if (boxKeys.size === 0) this.partitionToBoxKeys.delete(entry.partition);
+            }
+        }
+
+        for (const entry of entries) {
+            entry.length = Math.max(entry.length, newLength);
+            entry.falsePositiveCount++;
+            entry.age = 0;
+            this._placePartition(entry);
+        }
+
+        return true;
+    }
+
+    adaptFalsePositive(start, end) {
+        let adapted = false;
+
+        for (const subquery of this._subqueries(start, end)) {
+            for (const box of this._matchingBoxes(subquery.partition)) {
+                if (box.partitions.has(subquery.partition)) continue;
+                adapted = this._splitBoxAwayFromQuery(box, subquery.partition) || adapted;
+            }
+        }
+
+        return adapted;
+    }
+
+    expand() {
+        this.expansionLevel++;
+        this.boxes.clear();
+        this.partitionToBoxKeys.clear();
+
+        for (const entry of this.partitions.values()) {
+            entry.age++;
+
+            if (entry.length > 1) {
+                entry.length--;
+            } else {
+                entry.length = this.fingerprintBits;
+                entry.age = 0;
+            }
+
+            this._placePartition(entry);
+        }
+    }
+
+    getStats() {
+        let maxAge = 0;
+        let maxLength = 0;
+        for (const entry of this.partitions.values()) {
+            maxAge = Math.max(maxAge, entry.age);
+            maxLength = Math.max(maxLength, entry.length);
+        }
+
+        return {
+            partitions: this.partitions.size,
+            keepsakeBoxes: this.boxes.size,
+            expansionLevel: this.expansionLevel,
+            maxAge,
+            maxLength,
+        };
+    }
+
+    serialize() {
+        const writer = new BinaryWriter();
+        writer.uint32(MAGIC.RANGE);
+        writer.uint32(this.partitionSize);
+        writer.uint32(this.fingerprintBits);
+        writer.uint32(this.maxFingerprintBits);
+        writer.uint32(this.expansionLevel);
+
+        const partitionEntries = Array.from(this.partitions.entries());
+        writer.uint32(partitionEntries.length);
+        for (const [key, entry] of partitionEntries) {
+            writer.uint32(key);
+            writer.uint8(entry.length ?? 0);
+            writer.uint32(entry.count ?? 1);
+            writer.uint32(entry.falsePositiveCount ?? 0);
+            writer.uint32(entry.age ?? 0);
+        }
+
+        return wrapCRC32(writer.toArrayBuffer());
+    }
+
+    static deserialize(buffer, options = {}) {
+        buffer = unwrapCRC32(buffer);
+        const reader = new BinaryReader(buffer);
+        if (reader.uint32() !== MAGIC.RANGE) throw new Error('Not a RangeFilter binary.');
+
+        const filter = new RangeFilter({
+            partitionSize: reader.uint32(),
+            fingerprintBits: reader.uint32(),
+            maxFingerprintBits: reader.uint32(),
+            ...options,
+        });
+        filter.expansionLevel = reader.uint32();
+
+        const partitionCount = reader.uint32();
+        for (let i = 0; i < partitionCount; i++) {
+            const key = reader.uint32();
+            const entry = {
+                partition: key,
+                fingerprint: 0,
+                length: reader.uint8(),
+                count: reader.uint32(),
+                falsePositiveCount: reader.uint32(),
+                age: 0,
+            };
+            entry.age = reader.uint32();
+            filter.partitions.set(key, entry);
+            filter._placePartition(entry);
+        }
+
+        return filter;
+    }
+}

package/src/serialization.js

@@ -0,0 +1,214 @@
+/**
+ * Serialization primitives for probe-filters and probe-maplets.
+ *
+ * Wire format for every filter/maplet:
+ *   [4-byte MAGIC] [payload ...] [4-byte IEEE 802.3 CRC32]
+ *
+ * PointFilter payload:
+ *   capacity(u32) elements(u32) slotAddrBits(u32) fpSize(u32) curFpLen(u32)
+ *   deletionQ: count(u32) [motherHash(u32)...]
+ *   rejuvenQ: count(u32) [motherHash(u32)...]
+ *   liveSlots: count(u32) indexWidth(u8)  // indexWidth = 2 if capacity≤64K else 4
+ *     [index(u16|u32) meta(u8) motherHash(u32)...]
+ *   meta = (state << 6) | fingerprintLength  // state = 2 bits, fpLen = 6 bits
+ *   fingerprints[] and canonicalSlots[] are reconstructed from motherHash on load:
+ *     canonicalSlot = motherHash & (capacity - 1)
+ *     fingerprint = (motherHash >>> slotAddrBits) & mask[fpLen]
+ *   RSQF packed metadata (occupied, shifted, continuation, runends, runCounts,
+ *   blockOffsets) is rebuilt via _rebuildPackedMetadata() on deserialize.
+ *
+ * RangeFilter payload:
+ *   partitionSize(u32) fpBits(u32) maxFpBits(u32) expansionLevel(u32)
+ *   partitions: count(u32) [key(u32) length(u8) count(u32) age(u32)...]
+ *   fingerprint is recomputed from partition hash; keepsake boxes and
+ *   partitionToBoxKeys reverse map are rebuilt via _placePartition() on load.
+ *
+ * SpatialFilter payload:
+ *   bitsPerCoordinate(u32) [RangeFilter binary (wrapped bytes)...]
+ *
+ * TemporalFilter payload:
+ *   bucketDurationMs(f64) retentionDurationMs(f64) maxBuckets(u32)
+ *   buckets: count(u32) [bucketId(u32) [PointFilter binary (wrapped bytes)]...]
+ *
+ * MultiFilter payload:
+ *   parts: count(u32) [typeLen(u32) type(utf8) [sub-filter binary (bytes)]...]
+ *
+ * Maplet wire formats follow the same pattern with [MAGIC] [filter binary (bytes)]
+ * [domainType(string)] [domain state] [values: key → encoded state].
+ */
+
+const MAGIC = {
+    POINT: 0x41505446,   // 'APTF' — Aleph Point Filter (PointFilter)
+    RANGE: 0x52414E47,   // 'RANG' — RangeFilter
+    SPATIAL: 0x5350544C, // 'SPTL' — SpatialFilter
+    TEMPORAL: 0x54454D50, // 'TEMP' — TemporalFilter
+    MULTI: 0x4D554C54,   // 'MULT' — MultiFilter
+    MAPLET: 0x4D41504C,  // 'MAPL' — MultiMaplet
+    MAPLET_POINT: 0x4D504E54,    // 'MPNT' — PointMaplet
+    MAPLET_RANGE: 0x4D524E47,    // 'MRNG' — RangeMaplet
+    MAPLET_SPATIAL: 0x4D53504C,  // 'MSPL' — SpatialMaplet
+    MAPLET_TEMPORAL: 0x4D544D50, // 'MTMP' — TemporalMaplet
+    MAPLET_MULTI: 0x4D4D4C54,    // 'MMLT' — MultiMaplet
+    LIST: 0x4C495354,    // 'LIST' — type-tagged list
+};
+
+export { MAGIC };
+
+export class BinaryWriter {
+    constructor(initialCapacity = 1024) {
+        this.buffer = new ArrayBuffer(initialCapacity);
+        this.dataView = new DataView(this.buffer);
+        this.offset = 0;
+    }
+
+    _grow(needed) {
+        const required = this.offset + needed;
+        if (required <= this.buffer.byteLength) return;
+        let newSize = this.buffer.byteLength;
+        while (newSize < required) newSize *= 2;
+        const newBuffer = new ArrayBuffer(newSize);
+        new Uint8Array(newBuffer).set(new Uint8Array(this.buffer, 0, this.offset));
+        this.buffer = newBuffer;
+        this.dataView = new DataView(this.buffer);
+    }
+
+    uint8(value) {
+        this._grow(1);
+        this.dataView.setUint8(this.offset, value >>> 0);
+        this.offset += 1;
+        return this;
+    }
+
+    uint16(value) {
+        this._grow(2);
+        this.dataView.setUint16(this.offset, value >>> 0, true);
+        this.offset += 2;
+        return this;
+    }
+
+    uint32(value) {
+        this._grow(4);
+        this.dataView.setUint32(this.offset, value >>> 0, true);
+        this.offset += 4;
+        return this;
+    }
+
+    float64(value) {
+        this._grow(8);
+        this.dataView.setFloat64(this.offset, value, true);
+        this.offset += 8;
+        return this;
+    }
+
+    bytes(array) {
+        const len = Array.isArray(array) ? array.length : array.byteLength ?? array.length;
+        this.uint32(len);
+        this._grow(len);
+        if (array instanceof Uint8Array) {
+            new Uint8Array(this.buffer, this.offset, len).set(array);
+        } else {
+            const view = new Uint8Array(array.buffer ?? array, array.byteOffset ?? 0, len);
+            new Uint8Array(this.buffer, this.offset, len).set(view);
+        }
+        this.offset += len;
+        return this;
+    }
+
+    string(str) {
+        const encoded = new TextEncoder().encode(str);
+        this.uint32(encoded.length);
+        this._grow(encoded.length);
+        new Uint8Array(this.buffer, this.offset, encoded.length).set(encoded);
+        this.offset += encoded.length;
+        return this;
+    }
+
+    toArrayBuffer() {
+        return this.buffer.slice(0, this.offset);
+    }
+}
+
+export class BinaryReader {
+    constructor(buffer) {
+        this.buffer = buffer instanceof ArrayBuffer ? buffer : buffer.buffer;
+        this.dataView = new DataView(this.buffer);
+        this.offset = 0;
+    }
+
+    uint8() {
+        const value = this.dataView.getUint8(this.offset);
+        this.offset += 1;
+        return value;
+    }
+
+    uint16() {
+        const value = this.dataView.getUint16(this.offset, true);
+        this.offset += 2;
+        return value;
+    }
+
+    uint32() {
+        const value = this.dataView.getUint32(this.offset, true);
+        this.offset += 4;
+        return value;
+    }
+
+    float64() {
+        const value = this.dataView.getFloat64(this.offset, true);
+        this.offset += 8;
+        return value;
+    }
+
+    string() {
+        const len = this.uint32();
+        const bytes = new Uint8Array(this.buffer, this.offset, len);
+        this.offset += len;
+        return new TextDecoder().decode(bytes);
+    }
+
+    bytes() {
+        const len = this.uint32();
+        const aligned = new ArrayBuffer(len);
+        new Uint8Array(aligned).set(new Uint8Array(this.buffer, this.offset, len));
+        this.offset += len;
+        return new Uint8Array(aligned);
+    }
+
+    done() {
+        return this.offset >= this.buffer.byteLength;
+    }
+}
+
+const CRC32_TABLE = (() => {
+    const table = new Int32Array(256);
+    for (let i = 0; i < 256; i++) {
+        let crc = i;
+        for (let j = 0; j < 8; j++) crc = crc & 1 ? (crc >>> 1) ^ 0xEDB88320 : crc >>> 1;
+        table[i] = crc;
+    }
+    return table;
+})();
+
+function crc32(data) {
+    let crc = 0xFFFFFFFF;
+    for (let i = 0; i < data.length; i++) crc = (crc >>> 8) ^ CRC32_TABLE[(crc ^ data[i]) & 0xFF];
+    return (crc ^ 0xFFFFFFFF) >>> 0;
+}
+
+export function wrapCRC32(buffer) {
+    const data = new Uint8Array(buffer);
+    const checksum = crc32(data);
+    const out = new Uint8Array(data.length + 4);
+    out.set(data);
+    new DataView(out.buffer).setUint32(data.length, checksum, true);
+    return out.buffer;
+}
+
+export function unwrapCRC32(buffer) {
+    const bytes = buffer instanceof Uint8Array ? new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength) : new Uint8Array(buffer);
+    if (bytes.length < 4) throw new Error('Buffer too small for CRC32');
+    const payload = bytes.subarray(0, bytes.length - 4);
+    const expected = new DataView(bytes.buffer, bytes.byteOffset + bytes.length - 4, 4).getUint32(0, true);
+    if (crc32(payload) !== expected) throw new Error('CRC32 mismatch: data may be corrupted');
+    return payload.buffer.slice(payload.byteOffset, payload.byteOffset + payload.byteLength);
+}