querysub 0.355.0 → 0.357.0

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -0,0 +1,719 @@
+
+
+import { LZ4 } from "../../../storage/LZ4";
+import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
+import { Zip } from "../../../zip";
+import { BufferReader, Reader, WILD_CARD_BYTE, createMatchesPattern, SearchParams } from "./BufferIndexHelpers";
+import { formatNumber, formatPercent } from "socket-function/src/formatting/format";
+import { lazy } from "socket-function/src/caching";
+import { list, sort } from "socket-function/src/misc";
+import { testDisableCache } from "../../../-a-archives/archivesMemoryCache";
+import { devDebugbreak } from "../../../config";
+import { BufferUnitIndexParallelSearchCount, DEFAULT_BLOCK_SIZE, DEFAULT_TARGET_UNITS_PER_BUCKET } from "./BufferIndexLogsOptimizationConstants";
+import { runInParallel } from "socket-function/src/batching";
+
+const USE_COMPRESSION = true;
+
+
+// Lazy-load the C++ addon (only loaded when actually needed, not in client-side code)
+let getCppAddon = lazy(() => {
+    try {
+        // Check if we're in Node.js environment
+        if (typeof process === "undefined" || !process.versions || !process.versions.node) {
+            return undefined;
+        }
+        // Try to require the C++ addon
+        const addon = require("./BufferIndexCPP") as typeof import("./BufferIndexCPP");
+        return addon;
+    } catch (e) {
+        console.warn("C++ addon not available, falling back to TypeScript implementation", e);
+        return undefined;
+    }
+});
+
+function compressBuffer(buffer: Buffer): Buffer {
+    if (!USE_COMPRESSION) {
+        return buffer;
+    }
+    return LZ4.compress(buffer);
+}
+
+function decompressBuffer(buffer: Buffer): Buffer {
+    if (!USE_COMPRESSION) {
+        return buffer;
+    }
+    return LZ4.decompress(buffer);
+}
+
+function getMaskedHash(unit: number, mask: number): number {
+    // Optimized: Simple multiply and shift
+    let hash = unit;
+    hash = Math.imul(hash, 0x27d4eb2d);
+    hash = hash ^ (hash >>> 15);
+
+    let maskedHash = hash & mask;
+    // Avoid collision with empty marker (0)
+    if (maskedHash === 0) {
+        maskedHash = 1;
+    }
+    return maskedHash;
+}
+
+
+/*
+    - If you store every single position, the index is going to really struggle to be significantly smaller than your input data. Because you'll be storing a unit value per byte, which is just a lot of data. 
+    - SO, we hash each unit, only taking some bits. By only storing some of the units, we massively decrease the storage size, making this significantly smaller than the input data, or even the compressed data. 
+        - This doesn't affect most searches, because most searches have multiple unique units, and the intersection of them likely only contains the actual blocks.
+    - We also store the list of blocks associated with each hash unit outside the hash table in a list, and each one of those lists are compressed.
+    - As a result, our compression ratio is usually amazing, although we do have some false positives.
+        - The index is often easily 100X smaller than the input data. And then this can increase multiple times if you compress the index (you can't compress it and still use it, however, this will reduce your network usage and disk usage).
+
+    PERFORMANCE OPTIMIZATION:
+    - Uses Uint32Arrays for hash table and linked lists for fast, cache-friendly memory access
+    - Linked lists handle collisions more efficiently than linear probing, with better cache locality
+    - Dynamic memory allocation (starting at totalUnits/100, growing 4x when needed) keeps memory usage optimal
+
+
+    BlocksBuffer {
+        byte type
+        byte[3] padding
+        uint32 blockCount
+        uint32[blockCount + 1] blockOffsets
+        compress(Block)[]
+    }
+
+    Block {
+        uint32 bufferCount
+        uint32[bufferCount + 1] bufferOffsets (offsets from the start of the uncompressed block)
+        byte[][] bufferData
+    }
+
+    Index {
+        byte type
+        byte[3] padding
+        uint32 mask
+        uint32 dataOffset
+        uint32[] hashTable (data offsets for each bucket, 0 = empty)
+        DataEntry[] dataSection
+    }
+
+    DataEntry {
+        uint32 compressedSize
+        compress(uint16[] deltaEncodedBlockIndices)
+    }
+*/
+export class BufferUnitIndex {
+
+    // IMPORTANT! The input data values should be sorted from newest to oldest. 
+    @measureFnc
+    public static encode(allData: Buffer[], options: {
+        type: number;
+        blockSize?: number;
+        targetUnitsPerBucket?: number;
+    }): {
+        blocks: Buffer;
+        index: Buffer;
+    } {
+        const type = options.type;
+        const BLOCK_SIZE = options.blockSize ?? DEFAULT_BLOCK_SIZE;
+        const TARGET_UNITS_PER_BUCKET = options.targetUnitsPerBucket ?? DEFAULT_TARGET_UNITS_PER_BUCKET;
+
+        // Step 1: Group buffers into blocks and collect unique units
+        const blocks: Buffer[] = [];
+        const blocksUncompressed: Buffer[] = [];
+        measureBlock(() => {
+            function createBlockWithMetadata(buffers: Buffer[]): { block: Buffer; uncompressed: Buffer } {
+                const bufferCount = buffers.length;
+                const headerBuffer = Buffer.from(new Uint32Array([
+                    bufferCount,
+                    ...list(bufferCount + 1).map(i => 0),
+                ]).buffer);
+                let currentOffset = headerBuffer.length;
+                let headerPos = 4;
+                for (const buffer of buffers) {
+                    headerBuffer.writeUInt32LE(currentOffset, headerPos);
+                    headerPos += 4;
+                    currentOffset += buffer.length;
+                }
+                headerBuffer.writeUInt32LE(currentOffset, headerPos);
+
+                const concatenatedData = Buffer.concat(buffers);
+
+                const block = Buffer.concat([headerBuffer, concatenatedData]);
+
+                return {
+                    block: block,
+                    uncompressed: concatenatedData
+                };
+            }
+
+            let currentBlockBuffers: Buffer[] = [];
+            let currentBlockSize = 0;
+
+            for (const buffer of allData) {
+                let nextSize = currentBlockSize + buffer.length;
+                if (currentBlockBuffers.length > 0 && nextSize > BLOCK_SIZE) {
+                    const block = createBlockWithMetadata(currentBlockBuffers);
+                    blocks.push(block.block);
+                    blocksUncompressed.push(block.uncompressed);
+                    currentBlockBuffers = [];
+                    currentBlockSize = 0;
+                }
+
+                currentBlockBuffers.push(buffer);
+                currentBlockSize += buffer.length;
+            }
+
+            if (currentBlockBuffers.length > 0) {
+                const block = createBlockWithMetadata(currentBlockBuffers);
+                blocks.push(block.block);
+                blocksUncompressed.push(block.uncompressed);
+            }
+        }, `createBlocks`);
+
+        // Step 2: Create final blocks buffer with compressed blocks
+        const finalBlocksBuffer = measureBlock(() => {
+            // Compress each entire block (metadata + data)
+            const compressedBlocks: Buffer[] = [];
+            for (const block of blocks) {
+                const compressed = compressBuffer(block);
+                compressedBlocks.push(compressed);
+            }
+
+            const blocksHeaderBuffer = Buffer.allocUnsafe(4 + 4 + ((compressedBlocks.length + 1) * 4));
+
+            blocksHeaderBuffer.writeUInt8(type, 0);
+            blocksHeaderBuffer.writeUInt8(0, 1);
+            blocksHeaderBuffer.writeUInt8(0, 2);
+            blocksHeaderBuffer.writeUInt8(0, 3);
+            blocksHeaderBuffer.writeUInt32LE(compressedBlocks.length, 4);
+
+            let currentBlockOffset = blocksHeaderBuffer.length;
+            for (let i = 0; i < compressedBlocks.length; i++) {
+                blocksHeaderBuffer.writeUInt32LE(currentBlockOffset, 8 + (i * 4));
+                currentBlockOffset += compressedBlocks[i].length;
+            }
+            blocksHeaderBuffer.writeUInt32LE(currentBlockOffset, 8 + (compressedBlocks.length * 4));
+
+            return Buffer.concat([blocksHeaderBuffer, ...compressedBlocks]);
+        }, `createFinalBlocksBuffer`);
+
+        const estimatedUniqueUnits = measureBlock(() => {
+            const cppAddon = getCppAddon();
+            if (cppAddon) {
+                // Use C++ addon for performance
+                return cppAddon.estimateUniqueUnits(allData);
+            } else {
+                // TypeScript fallback implementation
+                // Use a 64KB bitmap to track which 16-bit masked hashes we've seen
+                const seenMaskedHashes = Buffer.allocUnsafe(65536);
+                seenMaskedHashes.fill(0);
+                let uniqueCount = 0;
+
+                for (const buffer of allData) {
+                    for (let i = 0; i <= buffer.length - 4; i++) {
+                        const unit = buffer.readUint32LE(i);
+                        const maskedHash = getMaskedHash(unit, 0xFFFF);
+
+                        if (seenMaskedHashes[maskedHash] === 0) {
+                            seenMaskedHashes[maskedHash] = 1;
+                            uniqueCount++;
+                        }
+                    }
+                }
+
+                // Estimate actual unique units by multiplying by 2
+                // This accounts for collisions in the 16-bit space
+                // Only inaccurate if unique units approaches or exceeds 65k
+                return uniqueCount * 2;
+            }
+        }, `collectUniqueUnits`);
+
+        // Step 3: Calculate hash table size and mask
+        const { hashTableCapacity, mask } = measureBlock(() => {
+            const uniqueUnitCount = estimatedUniqueUnits;
+            const targetBuckets = Math.max(1, Math.floor(uniqueUnitCount / TARGET_UNITS_PER_BUCKET));
+            let hashTableCapacity = Math.pow(2, Math.floor(Math.log2(targetBuckets * 2)) + 1);
+            let mask = hashTableCapacity - 1;
+
+            // Cap mask at 16 bits (2 bytes)
+            if (mask > 0xFFFF) {
+                mask = 0xFFFF;
+                hashTableCapacity = (mask + 1);
+            }
+
+            console.log(`UnitBlockLookup: ${blocks.length} blocks, ${uniqueUnitCount} unique units, ${hashTableCapacity} hash table capacity (~${(uniqueUnitCount / hashTableCapacity).toFixed(1)} units/bucket), mask: ${mask}`);
+
+            return { hashTableCapacity, mask };
+        }, `calculateHashTableSize`);
+
+
+
+        // Step 4: Build hash table with linked lists using Uint32Arrays for performance
+        const { hashTable, linkedListData, nodeCount, filledSlots } = measureBlock(() => {
+            const cppAddon = getCppAddon();
+            if (cppAddon) {
+                // Use C++ addon for performance
+                return cppAddon.buildHashTable(blocksUncompressed, hashTableCapacity, mask);
+            } else {
+                // TypeScript fallback implementation
+                // Calculate initial linked list capacity based on total units
+                let totalUnits = 0;
+                for (const buffer of blocksUncompressed) {
+                    totalUnits += Math.max(0, buffer.length - 3); // number of 4-byte units
+                }
+                let linkedListCapacity = Math.max(1000, Math.floor(totalUnits / 100));
+
+                // Hash table: stores head node index for each bucket (0 means empty)
+                const hashTable = new Uint32Array(hashTableCapacity);
+
+                // Linked list storage: each node is 3 uint32s: [maskedHash, blockIndex, nextNodeIndex]
+                let linkedListData = new Uint32Array(linkedListCapacity * 3);
+                let linkedListSize = 0;
+                let filledSlots = 0;
+
+                function allocateNode(maskedHash: number, blockIndex: number, nextNodeIndex: number): number {
+                    if (linkedListSize >= linkedListCapacity) {
+                        console.log(`Growing linked list buffer`);
+                        // Reallocate with 4x capacity
+                        const newCapacity = linkedListCapacity * 4;
+                        const newData = new Uint32Array(newCapacity * 3);
+                        newData.set(linkedListData);
+                        linkedListData = newData;
+                        linkedListCapacity = newCapacity;
+                    }
+                    const nodeIndex = linkedListSize + 1; // +1 because 0 means empty
+                    const nodeOffset = linkedListSize * 3;
+                    linkedListData[nodeOffset] = maskedHash;
+                    linkedListData[nodeOffset + 1] = blockIndex;
+                    linkedListData[nodeOffset + 2] = nextNodeIndex;
+                    linkedListSize++;
+                    return nodeIndex;
+                }
+
+                // Build the hash table with linked lists (iterate backwards so linked lists are in forward order)
+                for (let blockIndex = blocksUncompressed.length - 1; blockIndex >= 0; blockIndex--) {
+                    const blockUncompressed = blocksUncompressed[blockIndex];
+
+                    // Iterate through all units and add them directly
+                    for (let i = 0; i <= blockUncompressed.length - 4; i++) {
+                        const unit = blockUncompressed.readUint32LE(i);
+                        const maskedHash = getMaskedHash(unit, mask);
+
+                        const currentHead = hashTable[maskedHash];
+
+                        // Skip if we already added this masked hash for the current block
+                        // (check if head node has the same blockIndex)
+                        if (currentHead !== 0) {
+                            const headNodeOffset = (currentHead - 1) * 3;
+                            const headBlockIndex = linkedListData[headNodeOffset + 1];
+                            if (headBlockIndex === blockIndex) {
+                                continue;
+                            }
+                        }
+
+                        // Track when we're filling a previously empty slot
+                        if (currentHead === 0) {
+                            filledSlots++;
+                        }
+
+                        // Create new node and prepend to the chain
+                        const newNodeIndex = allocateNode(maskedHash, blockIndex, currentHead);
+                        hashTable[maskedHash] = newNodeIndex;
+                    }
+                }
+
+                return {
+                    hashTable,
+                    linkedListData: linkedListData.slice(0, linkedListSize * 3),
+                    nodeCount: linkedListSize,
+                    filledSlots
+                };
+            }
+        }, `buildHashTableWithLinkedLists`);
+
+        console.log(`Hash table filled slots: ${filledSlots} / ${hashTableCapacity} (${formatPercent(filledSlots / hashTableCapacity)} utilization)`);
+
+        // Step 5: Flatten linked lists and encode to binary
+        // Build data section and hash table by traversing linked lists
+        const { dataSection, hashTableBuffer } = measureBlock(() => {
+            const dataBuffers: Buffer[] = [];
+            let currentDataOffset = 0;
+
+            // Hash table stores data offset for each bucket (0 means empty)
+            const hashTableBuffer = Buffer.allocUnsafe(hashTableCapacity * 4);
+            hashTableBuffer.fill(0);
+
+            let totalBlocks = 0;
+            let nonEmptyBuckets = 0;
+
+            // Traverse each bucket and flatten its linked list
+            for (let bucketIndex = 0; bucketIndex < hashTableCapacity; bucketIndex++) {
+                let nodeIndex = hashTable[bucketIndex];
+
+                if (nodeIndex === 0) {
+                    // Empty bucket
+                    continue;
+                }
+
+                nonEmptyBuckets++;
+
+                // First pass: count blocks in this chain
+                let blockCount = 0;
+                let tempNodeIndex = nodeIndex;
+                while (tempNodeIndex !== 0) {
+                    blockCount++;
+                    const nodeOffset = (tempNodeIndex - 1) * 3;
+                    tempNodeIndex = linkedListData[nodeOffset + 2];
+                }
+
+                totalBlocks += blockCount;
+
+                // Allocate buffer for delta-encoded blocks
+                const deltaEncoded = new Uint16Array(blockCount);
+
+                // Second pass: write delta-encoded values directly
+                let writeIndex = 0;
+                let prevBlock = 0;
+                nodeIndex = hashTable[bucketIndex];
+                while (nodeIndex !== 0) {
+                    const nodeOffset = (nodeIndex - 1) * 3;
+                    const blockIndex = linkedListData[nodeOffset + 1];
+
+                    if (writeIndex === 0) {
+                        deltaEncoded[writeIndex] = blockIndex;
+                        prevBlock = blockIndex;
+                    } else {
+                        deltaEncoded[writeIndex] = blockIndex - prevBlock;
+                        prevBlock = blockIndex;
+                    }
+
+                    writeIndex++;
+                    nodeIndex = linkedListData[nodeOffset + 2];
+                }
+
+                // Compress the delta-encoded blocks
+                const uncompressedBuffer = Buffer.from(deltaEncoded.buffer, deltaEncoded.byteOffset, deltaEncoded.byteLength);
+                const dataBuffer = compressBuffer(uncompressedBuffer);
+
+                // Store: [dataSize, data]
+                const sizeBuffer = Buffer.allocUnsafe(4);
+                sizeBuffer.writeUInt32LE(dataBuffer.length, 0);
+
+                const entryBuffer = Buffer.concat([sizeBuffer, dataBuffer]);
+                dataBuffers.push(entryBuffer);
+
+                // Store data offset in hash table
+                hashTableBuffer.writeUInt32LE(currentDataOffset, bucketIndex * 4);
+                currentDataOffset += entryBuffer.length;
+            }
+
+            const dataSection = Buffer.concat(dataBuffers);
+
+            const avgBlocks = totalBlocks / nonEmptyBuckets;
+            console.log(`Block data buffers: ${dataBuffers.length}, avg blocks per hash: ${formatNumber(avgBlocks)}`);
+
+            return { dataSection, hashTableBuffer };
+        }, `flattenLinkedListsAndBuildDataSection`);
+
+        // Build header: [type (byte), padding (byte[3]), mask (uint32), dataOffset (uint32)]
+        const index = measureBlock(() => {
+            const header = Buffer.allocUnsafe(4 + 8);
+            header.writeUInt8(type, 0);
+            header.writeUInt8(0, 1); // padding
+            header.writeUInt8(0, 2); // padding
+            header.writeUInt8(0, 3); // padding
+            header.writeUInt32LE(mask, 4);
+            header.writeUInt32LE(12 + hashTableBuffer.length, 8); // dataOffset: after type+padding+mask+dataOffset+hashTable
+
+            return Buffer.concat([header, hashTableBuffer, dataSection]);
+        }, `buildFinalIndex`);
+
+        return { blocks: finalBlocksBuffer, index };
+    }
+
+    @measureFnc
+    public static async decodeAll(blocks: Buffer): Promise<Buffer[]> {
+        let reader = new BufferReader(blocks);
+        const blockCount = await this.getBlockCount(reader);
+        const allBuffers: Buffer[] = [];
+
+        for (let blockIndex = 0; blockIndex < blockCount; blockIndex++) {
+            let obj = await this.getBlock(reader, blockIndex);
+            let blockReader = new BufferReader(obj.block);
+            let bufferCount = await this.getBufferCountFromBlock(blockReader);
+            for (let i = 0; i < bufferCount; i++) {
+                let buffer = await this.getBufferFromBlock(blockReader, i);
+                allBuffers.push(buffer);
+            }
+        }
+
+        return allBuffers;
+    }
+
+    @measureFnc
+    public static async find(config: {
+        params: SearchParams;
+        keepIterating: () => boolean;
+        onResult: (match: Buffer) => void;
+        index: Buffer;
+        reader: Reader;
+    }): Promise<{
+        blocksChecked: number;
+        blocksCheckedCompressedSize: number;
+        blocksCheckedDecompressedSize: number;
+        totalBlockCount: number;
+        blockSearchTime: number;
+        blocksWithErrors: string[];
+    }> {
+        const { params, index, reader, keepIterating } = config;
+
+        // Split on wildcards if present
+        function splitOnWildcard(buffer: Buffer): Buffer[] {
+            const segments: Buffer[] = [];
+            let start = 0;
+            for (let i = 0; i <= buffer.length; i++) {
+                if (i === buffer.length || buffer[i] === WILD_CARD_BYTE) {
+                    segments.push(buffer.slice(start, i));
+                    start = i + 1;
+                }
+            }
+            return segments;
+        }
+
+        const segments = params.disableWildCards && [params.findBuffer] || splitOnWildcard(params.findBuffer).filter(s => s.length > 0);
+
+        // Find blocks for each segment >= 4 bytes
+        const candidateBlocks = measureBlock(() => {
+            const candidateBlocksPerSegment: number[][] = [];
+            for (const segment of segments) {
+                if (segment.length < 4) continue;
+                const blockIndices = this.findBlocks({ findBuffer: segment, index });
+                candidateBlocksPerSegment.push(blockIndices);
+            }
+
+            if (candidateBlocksPerSegment.length === 0) {
+                throw new Error("Search pattern too short: all segments are fewer than 4 bytes, cannot use index");
+            }
+
+            let intersectionSet = new Set<number>(candidateBlocksPerSegment[0]);
+            for (let i = 1; i < candidateBlocksPerSegment.length; i++) {
+                const currentSet = new Set(candidateBlocksPerSegment[i]);
+                intersectionSet = new Set([...intersectionSet].filter(x => currentSet.has(x)));
+            }
+
+            // Search first first, as moveLogsToPublic should have made it so this is the newest.
+            return Array.from(intersectionSet).sort((a, b) => a - b);
+        }, `findCandidateBlocks`);
+
+
+        const matchesPattern = createMatchesPattern(params.findBuffer, !!params.disableWildCards);
+
+        const headerBuffer = await reader.read(0, 8);
+        const blockCount = headerBuffer.readUInt32LE(4);
+        let blocksCheckedCompressedSize = 0;
+        let blocksCheckedDecompressedSize = 0;
+
+        // Read blocks and search for matches
+        let blocksChecked = 0;
+        let blocksWithErrors: string[] = [];
+        let blockSearchTimeStart = Date.now();
+        await measureBlock(async () => {
+            let matchCount = 0;
+            let matchCounts = list(blockCount).fill(0);
+
+            const searchBlock = async (blockIndex: number) => {
+                // This is kind of a weird thing. Basically, because we search in parallel, we might search out of order. So we can only look at the counts before or at us, as if we match a whole bunch after us, but we should still keep going as our matches are going to take precedence.
+                let stopIterating = () => {
+                    let countBefore = 0;
+                    for (let i = 0; i <= blockIndex; i++) {
+                        countBefore += matchCounts[i];
+                    }
+                    return countBefore >= params.limit || !keepIterating();
+                };
+                if (stopIterating()) return;
+
+                let debugOffsets = {
+                    startOffset: 0,
+                    endOffset: 0,
+                };
+                try {
+                    if (blockIndex >= blockCount) {
+                        throw new Error(`Block index ${blockIndex} out of range, blockCount is ${blockCount}`);
+                    }
+                    let obj = await this.getBlock(reader, blockIndex, debugOffsets);
+
+                    blocksCheckedDecompressedSize += obj.block.length;
+                    blocksCheckedCompressedSize += obj.compressedSize;
+                    blocksChecked++;
+
+                    let blockReader = new BufferReader(obj.block);
+                    let bufferCount = await this.getBufferCountFromBlock(blockReader);
+
+                    // Check each buffer for a match
+                    for (let i = 0; i < bufferCount; i++) {
+                        if (stopIterating()) break;
+
+                        const buffer = await this.getBufferFromBlock(blockReader, i);
+                        if (matchesPattern(buffer)) {
+                            config.onResult(buffer);
+                            matchCount++;
+                            matchCounts[blockIndex]++;
+                        }
+                    }
+                } catch (e: any) {
+                    console.warn(`Error decompressing/searching block: ${e}`);
+                    blocksWithErrors.push(`(for block ${blockIndex + 1} / ${blockCount}, ${debugOffsets.startOffset}-${debugOffsets.endOffset}) ${String(e?.stack || e)}`);
+                }
+            };
+
+            const runSearchBlock = runInParallel(
+                { parallelCount: BufferUnitIndexParallelSearchCount },
+                searchBlock
+            );
+            await Promise.all(list(blockCount).map(runSearchBlock));
+
+        }, `searchBlocks`);
+        let blockSearchTime = Date.now() - blockSearchTimeStart;
+
+        return { blocksChecked, blocksCheckedCompressedSize, blocksCheckedDecompressedSize, totalBlockCount: blockCount, blockSearchTime, blocksWithErrors };
+    }
+
+    private static findBlocks(config: {
+        findBuffer: Buffer;
+        index: Buffer;
+    }): number[] {
+        const { findBuffer, index } = config;
+
+        // Extract all unique units from findBuffer
+        const units = measureBlock(() => {
+            const units: number[] = [];
+            for (let i = 0; i <= findBuffer.length - 4; i++) {
+                const unit = findBuffer.readUint32LE(i);
+                if (unit !== 0 && !units.includes(unit)) {
+                    units.push(unit);
+                }
+            }
+            return units;
+        }, `extractUnits`);
+
+        if (units.length === 0) {
+            return [];
+        }
+
+        // Get blocks for each unit and intersect
+        return measureBlock(() => {
+            const candidateBlocksPerUnit: number[][] = [];
+            for (const unit of units) {
+                const blockIndices = this.getBlocksForUnit(index, unit);
+                candidateBlocksPerUnit.push(blockIndices);
+            }
+
+            // Intersect all block sets
+            let intersectionSet = new Set<number>(candidateBlocksPerUnit[0]);
+            for (let i = 1; i < candidateBlocksPerUnit.length; i++) {
+                const currentSet = new Set(candidateBlocksPerUnit[i]);
+                intersectionSet = new Set([...intersectionSet].filter(x => currentSet.has(x)));
+            }
+
+            let allCounts = candidateBlocksPerUnit.map(b => b.length);
+            sort(allCounts, x => x);
+
+            //console.log(`Candidate blocks ${intersectionSet.size}, minimum: ${Math.min(...candidateBlocksPerUnit.map(b => b.length))}, best 4 counts: ${allCounts.slice(0, 4).join(", ")}`);
+
+            return Array.from(intersectionSet);
+        }, `intersectBlocks`);
+    }
+
+    private static getBlocksForUnit(index: Buffer, unit: number): number[] {
+        // Read header (skip type + padding at offset 0-3)
+        const mask = index.readUInt32LE(4);
+        const dataOffset = index.readUInt32LE(8);
+
+        const maskedHash = getMaskedHash(unit, mask);
+
+        // Hash table starts after header (type + padding + mask + dataOffset = 12 bytes)
+        const hashTableOffset = 12 + maskedHash * 4;
+        const bucketDataOffset = index.readUInt32LE(hashTableOffset);
+
+        if (bucketDataOffset === 0) {
+            // Empty bucket
+            return [];
+        }
+
+        // Read from data section
+        const dataSection = index.slice(dataOffset);
+        const dataSize = dataSection.readUInt32LE(bucketDataOffset);
+
+        if (dataSize === 0) {
+            return [];
+        }
+
+        const data = dataSection.slice(bucketDataOffset + 4, bucketDataOffset + 4 + dataSize);
+        const decompressedData = decompressBuffer(data);
+
+        const blocks: number[] = [];
+        const uint16View = new Uint16Array(decompressedData.buffer, decompressedData.byteOffset, decompressedData.length / 2);
+
+        if (uint16View.length === 0) {
+            return [];
+        }
+
+        // Decode delta encoding
+        let currentBlock = uint16View[0];
+        blocks.push(currentBlock);
+
+        for (let i = 1; i < uint16View.length; i++) {
+            currentBlock += uint16View[i];
+            blocks.push(currentBlock);
+        }
+
+        return blocks;
+    }
+
+
+    private static async getBlockCount(reader: Reader): Promise<number> {
+        const headerBuffer = await reader.read(4, 8);
+        return headerBuffer.readUInt32LE(0);
+    }
+
+    private static async getBlock(reader: Reader, blockIndex: number, debugOffsets?: {
+        startOffset: number;
+        endOffset: number;
+    }): Promise<{
+        compressedSize: number;
+        block: Buffer;
+    }> {
+        const startOffset = (await reader.read(8 + blockIndex * 4, 4)).readUInt32LE(0);
+        const endOffset = (await reader.read(8 + (blockIndex + 1) * 4, 4)).readUInt32LE(0);
+        if (debugOffsets) {
+            debugOffsets.startOffset = startOffset;
+            debugOffsets.endOffset = endOffset;
+        }
+        let size = endOffset - startOffset;
+        let compressedBlock = await reader.read(startOffset, size);
+        if (compressedBlock.length !== size) {
+            devDebugbreak();
+            let test = await reader.read(startOffset, size);
+            throw new Error(`Reading block data at index ${blockIndex} we were given an incorrect amount of data, expected ${size}, got ${compressedBlock.length}. ${size > compressedBlock.length ? "It is likely the file is missing trailing data?" : "It's likely that the reader function is incorrect."}`);
+        }
+        let block = decompressBuffer(compressedBlock);
+        return {
+            compressedSize: endOffset - startOffset,
+            block: block,
+        };
+    }
+
+    public static async getBufferCountFromBlock(reader: Reader): Promise<number> {
+        return (await reader.read(0, 4)).readUInt32LE(0);
+    }
+
+    private static async getBufferFromBlock(reader: Reader, bufferIndex: number): Promise<Buffer> {
+        let startOffset = (await reader.read(4 + bufferIndex * 4, 4)).readUInt32LE(0);
+        let endOffset = (await reader.read(4 + (bufferIndex + 1) * 4, 4)).readUInt32LE(0);
+        let buffer = await reader.read(startOffset, endOffset - startOffset);
+        return buffer;
+    }
+
+}
+