querysub 0.356.0 → 0.358.0

package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts

@@ -0,0 +1,146 @@
+import { formatPercent, formatNumber } from "socket-function/src/formatting/format";
+import { red } from "socket-function/src/formatting/logColors";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { Unit } from "./BufferIndexHelpers";
+
+export class UnitSet {
+    // Hash-based set for checking unit membership (no positions stored)
+    // Similar to the hash table approach in UnitRefList.encode, but only stores presence
+
+    @measureFnc
+    static encode(blocks: Buffer[][]): Buffer {
+        const MAX_FILL_RATIO = 0.35;
+
+        // First pass: count total units
+        let totalUnits = 0;
+        for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
+            const block = blocks[blockIndex];
+            for (let bufferIndex = 0; bufferIndex < block.length; bufferIndex++) {
+                const buffer = block[bufferIndex];
+                if (buffer.length >= 4) {
+                    totalUnits += buffer.length - 3;
+                }
+            }
+        }
+
+        if (totalUnits === 0) {
+            return Buffer.from(new Uint32Array([0]).buffer);
+        }
+
+        function tryEncodeWithRatio(estimatedUniqueRatio: number): { hashTable: Uint32Array; collisions: number; totalInserts: number; } | undefined {
+            const HASH_FACTOR = 2;
+            const estimatedUnique = Math.max(Math.ceil(totalUnits / estimatedUniqueRatio), 16);
+            const desiredCapacity = estimatedUnique * HASH_FACTOR;
+            const hashTableCount = Math.pow(2, Math.ceil(Math.log2(desiredCapacity)));
+
+            const maxUniqueCount = Math.floor(hashTableCount * MAX_FILL_RATIO);
+
+            // Each entry is just a single uint32 (the unit value, 0 means empty)
+            const hashTable = new Uint32Array(hashTableCount);
+
+            function getHashIndex(unit: number): number {
+                const hash = Math.imul(unit, 2654435761);
+                return (hash >>> 16) & (hashTableCount - 1);
+            }
+
+            function getNextIndex(index: number): number {
+                return (index + 1) & (hashTableCount - 1);
+            }
+
+            let uniqueCount = 0;
+            let collisions = 0;
+            let totalInserts = 0;
+
+            // Iterate and add to hash table directly (no first pass)
+            for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
+                const block = blocks[blockIndex];
+                for (let bufferIndex = 0; bufferIndex < block.length; bufferIndex++) {
+                    const buffer = block[bufferIndex];
+                    // Extract units directly
+                    for (let i = 0; i <= buffer.length - 4; i++) {
+                        const unit = buffer.readUint32LE(i);
+                        if (!unit) continue;
+
+                        totalInserts++;
+                        let index = getHashIndex(unit);
+                        // Linear probing
+                        while (true) {
+                            if (hashTable[index] === 0) {
+                                // Empty slot - insert
+                                hashTable[index] = unit;
+                                uniqueCount++;
+                                if (uniqueCount > maxUniqueCount) {
+                                    return undefined; // Failed, exceeded fill ratio
+                                }
+                                break;
+                            }
+                            if (hashTable[index] === unit) {
+                                // Already present
+                                break;
+                            }
+                            // Collision, probe next
+                            collisions++;
+                            index = getNextIndex(index);
+                        }
+                    }
+                }
+            }
+
+            return { hashTable, collisions, totalInserts };
+        }
+
+        // Try with increasingly conservative ratios until we succeed
+        let ratio = 1000;
+        let result: ReturnType<typeof tryEncodeWithRatio>;
+        while (true) {
+            result = tryEncodeWithRatio(ratio);
+            if (result) break;
+            ratio /= 2;
+        }
+
+        const collisionRate = result.totalInserts > 0 ? result.collisions / result.totalInserts : 0;
+
+        if (collisionRate > 0.5) {
+            console.warn(red(`WARNING: UnitSet collision rate is over 50%: ${formatPercent(collisionRate)}`));
+        }
+
+        // Store capacity as first uint32, followed by the hash table
+        const output = new Uint32Array(1 + result.hashTable.length);
+        output[0] = result.hashTable.length;
+        output.set(result.hashTable, 1);
+
+        return Buffer.from(output.buffer);
+    }
+
+    @measureFnc
+    static has(data: Buffer, unit: Unit): boolean {
+        const hashTableCount = data.readUInt32LE(0);
+
+        if (hashTableCount === 0) return false;
+
+        function getHashIndex(unit: number): number {
+            const hash = Math.imul(unit, 2654435761);
+            return (hash >>> 16) & (hashTableCount - 1);
+        }
+
+        function getNextIndex(index: number): number {
+            return (index + 1) & (hashTableCount - 1);
+        }
+
+        let index = getHashIndex(unit);
+        const maxProbes = hashTableCount;
+
+        for (let probes = 0; probes < maxProbes; probes++) {
+            const value = data.readUInt32LE((1 + index) * 4);
+            if (value === 0) return false; // Empty slot, unit not present
+            if (value === unit) return true; // Found it
+            index = getNextIndex(index);
+        }
+
+        return false; // Shouldn't reach here if hash table isn't full
+    }
+}
+
+
+
+