querysub 0.356.0 → 0.358.0

package/src/diagnostics/logs/IndexedLogs/BufferIndexCPP.cpp

@@ -0,0 +1,327 @@
+#include <node_api.h>
+#include <stdlib.h>
+#include <string.h>
+#include <algorithm>
+
+static napi_value PopulateUnits(napi_env env, napi_callback_info info) {
+    size_t argc = 2;
+    napi_value args[2];
+    napi_get_cb_info(env, info, &argc, args, NULL, NULL);
+    
+    // args[0] = blocks (Buffer[][])
+    // args[1] = blockStartIndex (number)
+    
+    uint32_t blockStartIndex = 0;
+    napi_get_value_uint32(env, args[1], &blockStartIndex);
+    
+    // Get length of blocks array
+    uint32_t blocksLength;
+    napi_get_array_length(env, args[0], &blocksLength);
+    
+    // First pass: calculate total units needed
+    uint32_t totalUnits = 0;
+    
+    for (uint32_t blockIndex = 0; blockIndex < blocksLength; blockIndex++) {
+        napi_value block;
+        napi_get_element(env, args[0], blockIndex, &block);
+        
+        uint32_t blockLength;
+        napi_get_array_length(env, block, &blockLength);
+        
+        for (uint32_t bufferIndex = 0; bufferIndex < blockLength; bufferIndex++) {
+            napi_value bufferVal;
+            napi_get_element(env, block, bufferIndex, &bufferVal);
+            
+            uint8_t* bufferData;
+            size_t bufferLength;
+            napi_get_buffer_info(env, bufferVal, (void**)&bufferData, &bufferLength);
+            
+            if (bufferLength >= 4) {
+                totalUnits += (uint32_t)(bufferLength - 3);
+            }
+        }
+    }
+    
+    // Allocate the three output arrays
+    uint32_t* unitsArray = (uint32_t*)malloc(totalUnits * sizeof(uint32_t));
+    uint32_t* bufferIndicesArray = (uint32_t*)malloc(totalUnits * sizeof(uint32_t));
+    uint32_t* blocksArray = (uint32_t*)malloc(totalUnits * sizeof(uint32_t));
+    
+    uint32_t currentOffset = 0;
+    
+    // Second pass: populate the arrays
+    for (uint32_t blockIndex = 0; blockIndex < blocksLength; blockIndex++) {
+        napi_value block;
+        napi_get_element(env, args[0], blockIndex, &block);
+        
+        uint32_t blockLength;
+        napi_get_array_length(env, block, &blockLength);
+        
+        for (uint32_t bufferIndex = 0; bufferIndex < blockLength; bufferIndex++) {
+            napi_value bufferVal;
+            napi_get_element(env, block, bufferIndex, &bufferVal);
+            
+            uint8_t* bufferData;
+            size_t bufferLength;
+            napi_get_buffer_info(env, bufferVal, (void**)&bufferData, &bufferLength);
+            
+            if (bufferLength < 4) continue;
+            
+            // For each byte position, create a unit from that position and the next 3 bytes
+            for (size_t i = 0; i <= bufferLength - 4; i++) {
+                // Read 4 bytes as little-endian uint32
+                uint32_t unit = bufferData[i] | 
+                               (bufferData[i+1] << 8) | 
+                               (bufferData[i+2] << 16) | 
+                               (bufferData[i+3] << 24);
+                
+                if (unit == 0) continue;
+                
+                unitsArray[currentOffset] = unit;
+                bufferIndicesArray[currentOffset] = bufferIndex;
+                blocksArray[currentOffset] = blockIndex + blockStartIndex;
+                currentOffset++;
+            }
+        }
+    }
+    
+    // Create TypedArrays from the raw buffers
+    napi_value unitsResult, bufferIndicesResult, blocksResult;
+    
+    napi_create_external_arraybuffer(env, unitsArray, currentOffset * sizeof(uint32_t),
+        [](napi_env env, void* data, void* hint) { free(data); }, NULL, &unitsResult);
+    napi_create_external_arraybuffer(env, bufferIndicesArray, currentOffset * sizeof(uint32_t),
+        [](napi_env env, void* data, void* hint) { free(data); }, NULL, &bufferIndicesResult);
+    napi_create_external_arraybuffer(env, blocksArray, currentOffset * sizeof(uint32_t),
+        [](napi_env env, void* data, void* hint) { free(data); }, NULL, &blocksResult);
+    
+    napi_value units, bufferIndices, blocks;
+    napi_create_typedarray(env, napi_uint32_array, currentOffset, unitsResult, 0, &units);
+    napi_create_typedarray(env, napi_uint32_array, currentOffset, bufferIndicesResult, 0, &bufferIndices);
+    napi_create_typedarray(env, napi_uint32_array, currentOffset, blocksResult, 0, &blocks);
+    
+    // Return an object with three properties
+    napi_value result;
+    napi_create_object(env, &result);
+    napi_set_named_property(env, result, "units", units);
+    napi_set_named_property(env, result, "bufferIndices", bufferIndices);
+    napi_set_named_property(env, result, "blocks", blocks);
+    
+    return result;
+}
+
+// C++ addon for Node: Hash computation matching TypeScript getMaskedHash
+static uint32_t GetMaskedHash(uint32_t unit, uint32_t mask) {
+    uint32_t hash = (unit * 0x27d4eb2du);
+    hash = hash ^ (hash >> 15);
+    uint32_t maskedHash = hash & mask;
+    // Avoid collision with empty marker (0)
+    if (maskedHash == 0) {
+        maskedHash = 1;
+    }
+    return maskedHash;
+}
+
+// C++ addon for Node: Build hash table with linked lists for performance
+static napi_value BuildHashTable(napi_env env, napi_callback_info info) {
+    size_t argc = 3;
+    napi_value args[3];
+    napi_get_cb_info(env, info, &argc, args, NULL, NULL);
+    
+    // args[0] = blocksUncompressed (Buffer[])
+    // args[1] = hashTableCapacity (number)
+    // args[2] = mask (number)
+    
+    uint32_t hashTableCapacity = 0;
+    napi_get_value_uint32(env, args[1], &hashTableCapacity);
+    
+    uint32_t mask = 0;
+    napi_get_value_uint32(env, args[2], &mask);
+    
+    // Get length of blocksUncompressed array
+    uint32_t blocksLength;
+    napi_get_array_length(env, args[0], &blocksLength);
+    
+    // Calculate initial linked list capacity based on total units
+    uint32_t totalUnits = 0;
+    for (uint32_t blockIndex = 0; blockIndex < blocksLength; blockIndex++) {
+        napi_value block;
+        napi_get_element(env, args[0], blockIndex, &block);
+        
+        uint8_t* blockData;
+        size_t blockLength;
+        napi_get_buffer_info(env, block, (void**)&blockData, &blockLength);
+        
+        if (blockLength >= 4) {
+            totalUnits += (uint32_t)(blockLength - 3);
+        }
+    }
+    
+    uint32_t linkedListCapacity = std::max(1000u, totalUnits / 100);
+    
+    // Hash table: stores head node index for each bucket (0 means empty)
+    uint32_t* hashTable = (uint32_t*)calloc(hashTableCapacity, sizeof(uint32_t));
+    
+    // Linked list storage: each node is 3 uint32s: [maskedHash, blockIndex, nextNodeIndex]
+    uint32_t* linkedListData = (uint32_t*)malloc(linkedListCapacity * 3 * sizeof(uint32_t));
+    uint32_t linkedListSize = 0;
+    uint32_t filledSlots = 0;
+    
+    // Build the hash table with linked lists (iterate backwards so linked lists are in forward order)
+    for (int32_t blockIndex = (int32_t)blocksLength - 1; blockIndex >= 0; blockIndex--) {
+        napi_value block;
+        napi_get_element(env, args[0], blockIndex, &block);
+        
+        uint8_t* blockData;
+        size_t blockLength;
+        napi_get_buffer_info(env, block, (void**)&blockData, &blockLength);
+        
+        if (blockLength < 4) continue;
+        
+        // Iterate through all units (overlapping 4-byte windows)
+        for (size_t i = 0; i <= blockLength - 4; i++) {
+            // Read unit directly by casting pointer (little-endian architecture)
+            // This reads 4 bytes starting at position i
+            uint32_t unit = *(uint32_t*)(blockData + i);
+            
+            uint32_t maskedHash = GetMaskedHash(unit, mask);
+            uint32_t currentHead = hashTable[maskedHash];
+            
+            // Skip if we already added this masked hash for the current block
+            // (check if head node has the same blockIndex)
+            if (currentHead != 0) {
+                uint32_t headNodeOffset = (currentHead - 1) * 3;
+                uint32_t headBlockIndex = linkedListData[headNodeOffset + 1];
+                if (headBlockIndex == (uint32_t)blockIndex) {
+                    continue;
+                }
+            }
+            
+            // Track when we're filling a previously empty slot
+            if (currentHead == 0) {
+                filledSlots++;
+            }
+            
+            // Reallocate if needed (grow 4x like TypeScript version)
+            if (linkedListSize >= linkedListCapacity) {
+                uint32_t newCapacity = linkedListCapacity * 4;
+                uint32_t* newData = (uint32_t*)malloc(newCapacity * 3 * sizeof(uint32_t));
+                memcpy(newData, linkedListData, linkedListSize * 3 * sizeof(uint32_t));
+                free(linkedListData);
+                linkedListData = newData;
+                linkedListCapacity = newCapacity;
+            }
+            
+            // Allocate new node
+            uint32_t nodeIndex = linkedListSize + 1; // +1 because 0 means empty
+            uint32_t nodeOffset = linkedListSize * 3;
+            linkedListData[nodeOffset] = maskedHash;
+            linkedListData[nodeOffset + 1] = (uint32_t)blockIndex;
+            linkedListData[nodeOffset + 2] = currentHead;
+            linkedListSize++;
+            
+            // Prepend to the chain
+            hashTable[maskedHash] = nodeIndex;
+        }
+    }
+    
+    // Trim linkedListData to actual size
+    uint32_t* finalLinkedListData = (uint32_t*)malloc(linkedListSize * 3 * sizeof(uint32_t));
+    memcpy(finalLinkedListData, linkedListData, linkedListSize * 3 * sizeof(uint32_t));
+    free(linkedListData);
+    
+    // Create TypedArrays from the data
+    napi_value hashTableResult, linkedListDataResult;
+    
+    napi_create_external_arraybuffer(env, hashTable, hashTableCapacity * sizeof(uint32_t),
+        [](napi_env env, void* data, void* hint) { free(data); }, NULL, &hashTableResult);
+    
+    napi_create_external_arraybuffer(env, finalLinkedListData, linkedListSize * 3 * sizeof(uint32_t),
+        [](napi_env env, void* data, void* hint) { free(data); }, NULL, &linkedListDataResult);
+    
+    napi_value hashTableTyped, linkedListDataTyped;
+    napi_create_typedarray(env, napi_uint32_array, hashTableCapacity, hashTableResult, 0, &hashTableTyped);
+    napi_create_typedarray(env, napi_uint32_array, linkedListSize * 3, linkedListDataResult, 0, &linkedListDataTyped);
+    
+    // Create result object
+    napi_value result;
+    napi_create_object(env, &result);
+    
+    napi_set_named_property(env, result, "hashTable", hashTableTyped);
+    napi_set_named_property(env, result, "linkedListData", linkedListDataTyped);
+    
+    napi_value nodeCountValue, filledSlotsValue;
+    napi_create_uint32(env, linkedListSize, &nodeCountValue);
+    napi_create_uint32(env, filledSlots, &filledSlotsValue);
+    napi_set_named_property(env, result, "nodeCount", nodeCountValue);
+    napi_set_named_property(env, result, "filledSlots", filledSlotsValue);
+    
+    return result;
+}
+
+// C++ addon for Node: Estimate unique units using a 16-bit bitmap
+static napi_value EstimateUniqueUnits(napi_env env, napi_callback_info info) {
+    size_t argc = 1;
+    napi_value args[1];
+    napi_get_cb_info(env, info, &argc, args, NULL, NULL);
+    
+    // args[0] = allData (Buffer[])
+    
+    // Get length of allData array
+    uint32_t dataLength;
+    napi_get_array_length(env, args[0], &dataLength);
+    
+    // Use a 64KB bitmap to track which 16-bit masked hashes we've seen
+    uint8_t* seenMaskedHashes = (uint8_t*)calloc(65536, sizeof(uint8_t));
+    uint32_t uniqueCount = 0;
+    
+    // Process each buffer
+    for (uint32_t bufIndex = 0; bufIndex < dataLength; bufIndex++) {
+        napi_value buffer;
+        napi_get_element(env, args[0], bufIndex, &buffer);
+        
+        uint8_t* bufferData;
+        size_t bufferLength;
+        napi_get_buffer_info(env, buffer, (void**)&bufferData, &bufferLength);
+        
+        if (bufferLength < 4) continue;
+        
+        // Iterate through all units (overlapping 4-byte windows)
+        for (size_t i = 0; i <= bufferLength - 4; i++) {
+            // Read unit directly by casting pointer (little-endian architecture)
+            uint32_t unit = *(uint32_t*)(bufferData + i);
+            
+            // Get masked hash with 0xFFFF mask (16-bit)
+            uint32_t maskedHash = GetMaskedHash(unit, 0xFFFF);
+            
+            // Track if we've seen this hash before
+            if (seenMaskedHashes[maskedHash] == 0) {
+                seenMaskedHashes[maskedHash] = 1;
+                uniqueCount++;
+            }
+        }
+    }
+    
+    free(seenMaskedHashes);
+    
+    // Estimate actual unique units by multiplying by 2
+    // This accounts for collisions in the 16-bit space
+    uint32_t estimate = uniqueCount * 2;
+    
+    napi_value result;
+    napi_create_uint32(env, estimate, &result);
+    return result;
+}
+
+static napi_value Init(napi_env env, napi_value exports) {
+    napi_value populateUnitsFn, buildHashTableFn, estimateUniqueUnitsFn;
+    napi_create_function(env, NULL, 0, PopulateUnits, NULL, &populateUnitsFn);
+    napi_create_function(env, NULL, 0, BuildHashTable, NULL, &buildHashTableFn);
+    napi_create_function(env, NULL, 0, EstimateUniqueUnits, NULL, &estimateUniqueUnitsFn);
+    napi_set_named_property(env, exports, "populateUnits", populateUnitsFn);
+    napi_set_named_property(env, exports, "buildHashTable", buildHashTableFn);
+    napi_set_named_property(env, exports, "estimateUniqueUnits", estimateUniqueUnitsFn);
+    return exports;
+}
+
+NAPI_MODULE(NODE_GYP_MODULE_NAME, Init)

package/src/diagnostics/logs/IndexedLogs/BufferIndexCPP.d.ts

@@ -0,0 +1,18 @@
+export interface PopulateUnitsResult {
+    units: Uint32Array;
+    bufferIndices: Uint32Array;
+    blocks: Uint32Array;
+}
+
+export interface BuildHashTableResult {
+    hashTable: Uint32Array;
+    linkedListData: Uint32Array;
+    nodeCount: number;
+    filledSlots: number;
+}
+
+export function populateUnits(blocks: Buffer[][], blockStartIndex: number): PopulateUnitsResult;
+
+export function buildHashTable(blocksUncompressed: Buffer[], hashTableCapacity: number, mask: number): BuildHashTableResult;
+
+export function estimateUniqueUnits(allData: Buffer[]): number;

package/src/diagnostics/logs/IndexedLogs/BufferIndexCPP.js

@@ -0,0 +1 @@
+module.exports = require("./build/Release/BufferIndexCPP.node");

package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts

@@ -0,0 +1,222 @@
+
+import { measureFnc, measureWrap } from "socket-function/src/profiling/measure";
+import { formatNumber, formatPercent } from "socket-function/src/formatting/format";
+import { red } from "socket-function/src/formatting/logColors";
+import { MaybePromise } from "socket-function/src/types";
+import { TimeFilePathWithSize } from "./IndexedLogs";
+import { LimitGroup } from "../../../functional/limitProcessing";
+
+export const INDEX_EXTENSION = ".index";
+
+export type SearchParams = {
+    startTime: number;
+    endTime: number;
+    limit: number;
+    disableWildCards?: boolean;
+    findBuffer: Buffer;
+    pathOverrides?: TimeFilePathWithSize[];
+    only?: "local" | "public";
+};
+
+export type Unit = number;
+export type UnitRef = {
+    unit: Unit;
+    bufferIndex: number;
+    block: number;
+};
+
+export type Reader = {
+    getLength(): MaybePromise<number>;
+    read(offset: number, length: number): MaybePromise<Buffer>;
+};
+
+
+export class BufferReader implements Reader {
+    constructor(private buffer: Buffer) { }
+
+    async getLength(): Promise<number> {
+        return this.buffer.length;
+    }
+
+    async read(offset: number, length: number): Promise<Buffer> {
+        return this.buffer.slice(offset, offset + length);
+    }
+}
+
+
+
+export function createOffsetReader(reader: Reader, offset: number): Reader {
+    return {
+        async getLength() {
+            return (await reader.getLength()) - offset;
+        },
+        async read(readOffset: number, length: number) {
+            return reader.read(readOffset + offset, length);
+        },
+    };
+}
+
+
+export const getAllUnits = measureWrap(function getAllUnits(config: {
+    buffer: Buffer;
+    bufferIndex: number;
+    block: number;
+}): UnitRef[] {
+    let { buffer, bufferIndex, block } = config;
+
+    // We need at least 4 bytes to create a unit
+    if (buffer.length < 4) return [];
+
+    const result: UnitRef[] = [];
+    // For each byte position, create a unit from that position and the next 3 bytes
+    for (let i = 0; i <= buffer.length - 4; i++) {
+        const unit = buffer.readUint32LE(i);
+        if (!unit) continue;
+
+        result.push({
+            unit,
+            bufferIndex,
+            block
+        });
+    }
+
+    return result;
+});
+
+
+export type IndexedLogResults = {
+    matchCount: number;
+
+    // NOTE: A lot of the metadata won't be accurate if multiple searches happen at the same time. However, for debugging, it should be sufficient. 
+    reads: {
+        cached: boolean;
+        remote: boolean;
+        count: number;
+        size: number;
+
+        totalSize: number;
+        totalCount: number;
+    }[];
+
+    totalLocalFiles: number;
+    totalBackblazeFiles: number;
+    localFilesSearched: number;
+    backblazeFilesSearched: number;
+
+    totalBlockCount: number;
+    blockCheckedCount: number;
+
+    remoteBlockCount: number;
+    localBlockCount: number;
+    remoteBlockCheckedCount: number;
+    localBlockCheckedCount: number;
+
+    blocksCheckedCompressedSize: number;
+    blocksCheckedDecompressedSize: number;
+    blockErrors: string[];
+
+    fileErrors: { error: string; path: string; }[];
+
+    remoteIndexesSearched: number;
+    remoteIndexSize: number;
+    localIndexesSearched: number;
+    localIndexSize: number;
+
+    timeToFirstMatch: number;
+    fileFindTime: number;
+    indexSearchTime: number;
+    blockSearchTime: number;
+
+    totalSearchTime: number;
+
+    cancel?: boolean;
+    limitGroup?: LimitGroup;
+};
+export function createEmptyIndexedLogResults(): IndexedLogResults {
+    return {
+        matchCount: 0, reads: [], totalLocalFiles: 0, totalBackblazeFiles: 0, localFilesSearched: 0, backblazeFilesSearched: 0, totalBlockCount: 0, blockCheckedCount: 0, remoteBlockCount: 0, localBlockCount: 0, remoteBlockCheckedCount: 0, localBlockCheckedCount: 0, blocksCheckedCompressedSize: 0, blocksCheckedDecompressedSize: 0, blockErrors: [], fileErrors: [], remoteIndexesSearched: 0, remoteIndexSize: 0, localIndexesSearched: 0, localIndexSize: 0, timeToFirstMatch: 0, fileFindTime: 0, indexSearchTime: 0, blockSearchTime: 0, totalSearchTime: 0, cancel: undefined, limitGroup: undefined,
+    };
+}
+
+export function addReadToResults(results: IndexedLogResults, read: {
+    cached: boolean;
+    remote: boolean;
+    count: number;
+    size: number;
+}) {
+    let existingRead = results.reads.find(r => r.cached === read.cached && r.remote === read.remote);
+    if (!existingRead) {
+        existingRead = {
+            cached: read.cached,
+            remote: read.remote,
+            count: 0,
+            size: 0,
+            totalSize: 0,
+            totalCount: 0,
+        };
+        results.reads.push(existingRead);
+    }
+    existingRead.count += read.count;
+    existingRead.size += read.size;
+    return existingRead;
+}
+
+
+export const WILD_CARD_BYTE = 42;
+export function splitOnWildcard(buffer: Buffer): Buffer[] {
+    let segments: Buffer[] = [];
+    let start = 0;
+    for (let i = 0; i <= buffer.length; i++) {
+        if (i === buffer.length || buffer[i] === WILD_CARD_BYTE) {
+            segments.push(buffer.slice(start, i));
+            start = i + 1;
+        }
+    }
+    return segments;
+}
+
+// Each WILD_CARD_BYTE in pattern acts as a multi-byte wildcard: the segments on either
+// side must appear in order somewhere within buffer.
+// Returns a function that matches buffers against the pre-processed pattern.
+export function createMatchesPattern(pattern: Buffer, disableWildCards: boolean): (buffer: Buffer) => boolean {
+    let segments = disableWildCards && [pattern] || splitOnWildcard(pattern).filter(s => s.length > 0);
+
+    return measureWrap(function matchesPattern(buffer: Buffer): boolean {
+        // Fast path: check if all segments exist anywhere in the buffer using indexOf
+        for (let seg of segments) {
+            if (buffer.indexOf(seg) === -1) {
+                return false;
+            }
+        }
+
+        // Tries to match all segments in order starting from bufferPos, returning the end
+        // position after the last match, or -1 if not all segments could be found.
+        function matchSegmentsFrom(bufferPos: number): number {
+            for (let seg of segments) {
+                function segMatchesAt(pos: number): boolean {
+                    for (let i = 0; i < seg.length; i++) {
+                        if (buffer[pos + i] !== seg[i]) return false;
+                    }
+                    return true;
+                }
+                let found = false;
+                for (let searchPos = bufferPos; searchPos <= buffer.length - seg.length; searchPos++) {
+                    if (segMatchesAt(searchPos)) {
+                        // NOTE: I think this is safe because every segment has a wildcard after it. So we can never have a case where we didn't skip far enough because the wild card will just skip farther. And we won't have a partial match as we're matching the whole chunk. So we won't match a prefix and then get stuck. I think... it does seem weird though...
+                        bufferPos = searchPos + seg.length;
+                        found = true;
+                        break;
+                    }
+                }
+                if (!found) return -1;
+            }
+            return bufferPos;
+        }
+
+        for (let startPos = 0; startPos <= buffer.length; startPos++) {
+            if (matchSegmentsFrom(startPos) >= 0) return true;
+        }
+        return false;
+    }, "BufferIndex|matchesPattern");
+}
+

package/src/diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants.ts

@@ -0,0 +1,22 @@
+import { timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
+
+// NOTE: This is the block size BEFORE compression. The actual download size is probably going to be around 1/20th of this. 
+export const DEFAULT_BLOCK_SIZE = 1024 * 1024;
+export const DEFAULT_TARGET_UNITS_PER_BUCKET = 16;
+
+// Worst case scenario, the logs the user wants are very spread out, in which case having too high of a threshold will require decoding more data (the threshold amount per result).
+export const STREAMING_BLOCK_THRESHOLD = 1024 * 1024;
+
+
+
+export const PUBLIC_MOVE_THRESHOLD = timeInHour * 3;
+// Max uncompressed data size
+export const MAX_SINGLE_FILE_DATA = 1024 * 1024 * 512;
+
+export const MOVING_TIMEOUT = timeInMinute * 5;
+
+export const DISK_FLUSH_INTERVAL = timeInSecond * 15;
+export const MAX_COUNT_PER_FILE = 1000 * 1000 * 10;
+
+// High parallel factor, as these will often be remote, and so accessing them has high latency.
+export const BufferUnitIndexParallelSearchCount = 8;