querysub 0.356.0 → 0.358.0

package/src/diagnostics/logs/IndexedLogs/BufferIndexWAT.wat.d.ts

@@ -0,0 +1,178 @@
+import { WatModuleExports } from "../../../wat/watHandler";
+
+export interface UnitRefWatModule extends WatModuleExports {
+    /**
+     * Populate Units From Block: Extract units from concatenated block of buffers
+     * Block format: [count, length1, length2, ..., lengthN, buffer1_bytes, buffer2_bytes, ..., bufferN_bytes]
+     * @param blockDataPtr Pointer to concatenated block data
+     * @param blockDataSize Total size of block data in bytes
+     * @param blockIndex Block index value to store
+     * @param unitsOutputPtr Pointer to units output array
+     * @param bufferIndicesOutputPtr Pointer to buffer indices output array
+     * @param blocksOutputPtr Pointer to blocks output array
+     * @param startOffset Starting write position in output arrays
+     * @returns Number of units written
+     */
+    populate_units_from_block(
+        blockDataPtr: bigint,
+        blockDataSize: number,
+        blockIndex: number,
+        unitsOutputPtr: bigint,
+        bufferIndicesOutputPtr: bigint,
+        blocksOutputPtr: bigint,
+        startOffset: number
+    ): number;
+
+    /**
+     * Populate Units From Block (Loop Unrolled): Extract units from concatenated block of buffers
+     * Block format: [count, length1, length2, ..., lengthN, buffer1_bytes, buffer2_bytes, ..., bufferN_bytes]
+     * Uses loop unrolling to process 4 units at a time with optimized fast path for non-zero units
+     * @param blockDataPtr Pointer to concatenated block data
+     * @param blockDataSize Total size of block data in bytes
+     * @param blockIndex Block index value to store
+     * @param unitsOutputPtr Pointer to units output array
+     * @param bufferIndicesOutputPtr Pointer to buffer indices output array
+     * @param blocksOutputPtr Pointer to blocks output array
+     * @param startOffset Starting write position in output arrays
+     * @returns Number of units written
+     */
+    populate_units_from_block_simd(
+        blockDataPtr: bigint,
+        blockDataSize: number,
+        blockIndex: number,
+        unitsOutputPtr: bigint,
+        bufferIndicesOutputPtr: bigint,
+        blocksOutputPtr: bigint,
+        startOffset: number
+    ): number;
+
+    /**
+     * Populate Units From Buffer: Extract units from a buffer into three parallel arrays
+     * @param bufferPtr Pointer to source buffer bytes
+     * @param bufferLength Length of source buffer in bytes
+     * @param bufferIndex Buffer index value to store
+     * @param block Block value to store
+     * @param unitsOutputPtr Pointer to units output array
+     * @param bufferIndicesOutputPtr Pointer to buffer indices output array
+     * @param blocksOutputPtr Pointer to blocks output array
+     * @param startOffset Starting write position in output arrays
+     * @returns Number of units written
+     */
+    populate_units_from_buffer(
+        bufferPtr: bigint,
+        bufferLength: number,
+        bufferIndex: number,
+        block: number,
+        unitsOutputPtr: bigint,
+        bufferIndicesOutputPtr: bigint,
+        blocksOutputPtr: bigint,
+        startOffset: number
+    ): number;
+
+    /**
+     * Phase 1: Count & Deduplicate using hash table
+     * @param unitsPtr Pointer to units array
+     * @param totalUnits Number of units
+     * @param hashTablePtr Pointer to hash table
+     * @param hashTableCapacity Capacity of hash table
+     * @param uniqueUnitListPtr Pointer to store unique units
+     * @param maxUniqueCount Maximum fill threshold
+     * @returns uniqueCount (or -1 if exceeded threshold)
+     */
+    phase1_count_deduplicate(
+        unitsPtr: bigint,
+        totalUnits: number,
+        hashTablePtr: bigint,
+        hashTableCapacity: number,
+        uniqueUnitListPtr: bigint,
+        maxUniqueCount: number
+    ): number;
+
+    /**
+     * Phase 3: Calculate offsets for filtered arrays
+     * @param uniqueUnitListPtr Pointer to sorted unique units
+     * @param uniqueCount Number of unique units
+     * @param hashTablePtr Pointer to hash table
+     * @param hashTableCapacity Capacity of hash table
+     * @param maxPositionsPerUnit Max positions per unit (typically 1000)
+     * @returns Total offset (size of output arrays)
+     */
+    phase3_calc_offsets(
+        uniqueUnitListPtr: bigint,
+        uniqueCount: number,
+        hashTablePtr: bigint,
+        hashTableCapacity: number,
+        maxPositionsPerUnit: number
+    ): number;
+
+    /**
+     * Phase 4: Fill & Filter output arrays
+     * @param unitsPtr Pointer to units array
+     * @param bufferIndicesPtr Pointer to buffer indices array
+     * @param blocksPtr Pointer to blocks array
+     * @param totalUnits Number of units
+     * @param hashTablePtr Pointer to hash table
+     * @param hashTableCapacity Capacity of hash table
+     * @param maxPositionsPerUnit Max positions per unit
+     * @param filteredUnitsPtr Output array for units
+     * @param filteredBufferIndicesPtr Output array for buffer indices
+     * @param filteredBlocksPtr Output array for blocks
+     */
+    phase4_fill_filter(
+        unitsPtr: bigint,
+        bufferIndicesPtr: bigint,
+        blocksPtr: bigint,
+        totalUnits: number,
+        hashTablePtr: bigint,
+        hashTableCapacity: number,
+        maxPositionsPerUnit: number,
+        filteredUnitsPtr: bigint,
+        filteredBufferIndicesPtr: bigint,
+        filteredBlocksPtr: bigint
+    ): void;
+
+    /**
+     * Phase 4 Fill Filter Vector: Fill with collision detection (SIMD vectorized)
+     * @param unitsPtr Pointer to units array
+     * @param bufferIndicesPtr Pointer to buffer indices array
+     * @param blocksPtr Pointer to blocks array
+     * @param totalUnits Number of units
+     * @param hashTablePtr Pointer to hash table
+     * @param hashTableCapacity Capacity of hash table
+     * @param maxPositionsPerUnit Max positions per unit
+     * @param filteredUnitsPtr Output array for units
+     * @param filteredBufferIndicesPtr Output array for buffer indices
+     * @param filteredBlocksPtr Output array for blocks
+     * @returns Number of times slow path was taken (as f64)
+     */
+    phase4_fill_filter_vector(
+        unitsPtr: bigint,
+        bufferIndicesPtr: bigint,
+        blocksPtr: bigint,
+        totalUnits: number,
+        hashTablePtr: bigint,
+        hashTableCapacity: number,
+        maxPositionsPerUnit: number,
+        filteredUnitsPtr: bigint,
+        filteredBufferIndicesPtr: bigint,
+        filteredBlocksPtr: bigint
+    ): number;
+
+    /**
+     * Count Collisions: Calculate collision count for hash table
+     * @param unitsPtr Pointer to units array
+     * @param totalUnits Number of units
+     * @param hashTablePtr Pointer to hash table
+     * @param hashTableCapacity Capacity of hash table
+     * @returns Total collision count
+     */
+    count_collisions(
+        unitsPtr: bigint,
+        totalUnits: number,
+        hashTablePtr: bigint,
+        hashTableCapacity: number
+    ): number;
+}
+
+declare const module: UnitRefWatModule;
+export default module;

package/src/diagnostics/logs/IndexedLogs/BufferListStreamer.ts

@@ -0,0 +1,208 @@
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { BufferReader, Reader } from "./BufferIndexHelpers";
+
+// Suffix layout (12 bytes), appended after each block's data:
+//   Int32LE  dataLength  — byte count of the block's data
+//   Int32LE  blockIndex  — 0-based ordinal of this block
+//   UInt32LE magic       — configurable per instance
+const BUFFER_LIST_STREAMER_SUFFIX_SIZE = 12;
+
+interface BlockInfo {
+    suffixPos: number;
+    blockDataStart: number;
+    blockDataEnd: number;
+    dataLength: number;
+    blockIndex: number;
+    isIncomplete: boolean;
+}
+
+// NOTE: This is fully interruptable. You don't need to call finishBlock on your last block.
+export class BufferListStreamer {
+    private magic: number;
+    private includeLastBlock: boolean;
+
+    constructor(magic: number, includeLastBlock: boolean = false) {
+        this.magic = magic;
+        this.includeLastBlock = includeLastBlock;
+    }
+
+    // Iterator that yields blocks in reverse order (from last to first).
+    // If includeLastBlock is true, the incomplete block at the end is yielded first.
+    private async *iterateBlocksReverse(buffer: Buffer): AsyncGenerator<BlockInfo> {
+        let reader = new BufferReader(buffer);
+        for await (let blockInfo of this.iterateBlocksReverseAsync(reader)) {
+            yield blockInfo;
+        }
+    }
+
+    // Async iterator for Reader interface - yields blocks in reverse order
+    private async *iterateBlocksReverseAsync(reader: Reader): AsyncGenerator<BlockInfo> {
+        let length = await reader.getLength();
+
+        const isValidHeader = async (suffixPos: number): Promise<number> => {
+            if (suffixPos < 0 || suffixPos + BUFFER_LIST_STREAMER_SUFFIX_SIZE > length) return -1;
+
+            let suffix = await reader.read(suffixPos, BUFFER_LIST_STREAMER_SUFFIX_SIZE);
+            if (suffix.length < BUFFER_LIST_STREAMER_SUFFIX_SIZE) return -1;
+            if (suffix.readUInt32LE(8) !== this.magic) return -1;
+
+            let dataLength = suffix.readInt32LE(0);
+            if (dataLength < 0) return -1;
+
+            let blockDataStart = suffixPos - dataLength;
+            if (blockDataStart < 0) return -1;
+
+            if (blockDataStart === 0) return blockDataStart;
+
+            let prevSuffixStart = blockDataStart - BUFFER_LIST_STREAMER_SUFFIX_SIZE;
+            if (prevSuffixStart < 0) return -1;
+
+            let prevSuffix = await reader.read(prevSuffixStart, BUFFER_LIST_STREAMER_SUFFIX_SIZE);
+            if (prevSuffix.length < BUFFER_LIST_STREAMER_SUFFIX_SIZE) return -1;
+            if (prevSuffix.readUInt32LE(8) !== this.magic) return -1;
+
+            return blockDataStart;
+        };
+
+        let firstValidHeaderFound = false;
+        let suffixPos = length - BUFFER_LIST_STREAMER_SUFFIX_SIZE;
+
+        while (suffixPos >= 0) {
+            let blockDataStart = await isValidHeader(suffixPos);
+            if (blockDataStart < 0) {
+                suffixPos--;
+                continue;
+            }
+
+            if (!firstValidHeaderFound) {
+                firstValidHeaderFound = true;
+
+                if (this.includeLastBlock) {
+                    let lastBlockEnd = suffixPos + BUFFER_LIST_STREAMER_SUFFIX_SIZE;
+                    if (lastBlockEnd < length) {
+                        let suffix = await reader.read(suffixPos, BUFFER_LIST_STREAMER_SUFFIX_SIZE);
+                        let blockIndex = suffix.readInt32LE(4);
+                        yield {
+                            suffixPos: -1,
+                            blockDataStart: lastBlockEnd,
+                            blockDataEnd: length,
+                            dataLength: length - lastBlockEnd,
+                            blockIndex: blockIndex + 1,
+                            isIncomplete: true,
+                        };
+                    }
+                }
+            }
+
+            let suffix = await reader.read(suffixPos, BUFFER_LIST_STREAMER_SUFFIX_SIZE);
+            let dataLength = suffixPos - blockDataStart;
+            let blockIndex = suffix.readInt32LE(4);
+            yield {
+                suffixPos,
+                blockDataStart,
+                blockDataEnd: suffixPos,
+                dataLength,
+                blockIndex,
+                isIncomplete: false,
+            };
+
+            suffixPos = blockDataStart - BUFFER_LIST_STREAMER_SUFFIX_SIZE;
+        }
+
+        if (!firstValidHeaderFound && this.includeLastBlock && length > 0) {
+            yield {
+                suffixPos: -1,
+                blockDataStart: 0,
+                blockDataEnd: length,
+                dataLength: length,
+                blockIndex: 0,
+                isIncomplete: true,
+            };
+        }
+    }
+
+    @measureFnc
+    public async getAllBlocks(buffer: Buffer): Promise<Buffer[]> {
+        let blocks: Buffer[] = [];
+        for await (let blockInfo of this.iterateBlocksReverse(buffer)) {
+            blocks.push(buffer.slice(blockInfo.blockDataStart, blockInfo.blockDataEnd));
+        }
+        blocks.reverse();
+        return blocks;
+    }
+
+    public async getBlockRange(config: {
+        reader: Reader;
+        startIndex: number;
+        endIndex: number;
+    }): Promise<Buffer[]> {
+        let { reader, startIndex, endIndex } = config;
+        let blocks: Buffer[] = [];
+
+        for await (let blockInfo of this.iterateBlocksReverseAsync(reader)) {
+            if (blockInfo.blockIndex >= startIndex && blockInfo.blockIndex < endIndex) {
+                let blockData = await reader.read(blockInfo.blockDataStart, blockInfo.dataLength);
+                blocks.push(blockData);
+            }
+            if (blockInfo.blockIndex < startIndex) break;
+        }
+
+        blocks.reverse();
+        return blocks;
+    }
+
+    // Efficiently gets the block count by finding the last valid suffix and reading its blockIndex.
+    public async getBlockCount(reader: Reader): Promise<number> {
+        let maxBlockIndex = -1;
+        for await (let blockInfo of this.iterateBlocksReverseAsync(reader)) {
+            if (blockInfo.blockIndex > maxBlockIndex) {
+                maxBlockIndex = blockInfo.blockIndex;
+            }
+        }
+        return maxBlockIndex + 1;
+    }
+
+    // Returns true if the last block ends exactly at the end of the reader (not corrupted).
+    // If false, the reader can still be read but cannot be appended to efficiently.
+    // If includeLastBlock is true, this always returns true.
+    public async isClean(reader: Reader): Promise<boolean> {
+        if (this.includeLastBlock) return true;
+
+        let length = await reader.getLength();
+        if (length < BUFFER_LIST_STREAMER_SUFFIX_SIZE) return false;
+
+        // Get the first block info (which is the last block in the file)
+        for await (let blockInfo of this.iterateBlocksReverseAsync(reader)) {
+            // The first block we encounter should be a complete block ending at the file end
+            if (blockInfo.isIncomplete) return false;
+            return blockInfo.blockDataEnd + BUFFER_LIST_STREAMER_SUFFIX_SIZE === length;
+        }
+
+        return false;
+    }
+
+    public createStreamer(): {
+        add(data: Buffer): Buffer;
+        // NOTE: Finish block doesn't finish the BufferListStreamer, it just stops the current block. The BufferListStreamer has multiple blocks.
+        finishBlock(): Buffer;
+    } {
+        let currentBlockSize = 0;
+        let currentBlockIndex = 0;
+        let magic = this.magic;
+        return {
+            add(data: Buffer) {
+                currentBlockSize += data.length;
+                return data;
+            },
+            finishBlock() {
+                let suffix = Buffer.alloc(BUFFER_LIST_STREAMER_SUFFIX_SIZE);
+                suffix.writeInt32LE(currentBlockSize, 0);
+                suffix.writeInt32LE(currentBlockIndex, 4);
+                suffix.writeUInt32LE(magic, 8);
+                currentBlockIndex++;
+                currentBlockSize = 0;
+                return suffix;
+            },
+        };
+    }
+}