querysub 0.357.0 → 0.359.0

package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts

@@ -9,7 +9,7 @@ import { cacheArgsEqual, cacheLimited, cacheWeak, lazy } from "socket-function/s
 import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { magenta, yellow } from "socket-function/src/formatting/logColors";
-import { Unit, getAllUnits, Reader, createMatchesPattern, createOffsetReader, splitOnWildcard, SearchParams } from "./BufferIndexHelpers";
+import { Unit, getAllUnits, Reader, createMatchesPattern, createOffsetReader, splitOnWildcard, SearchParams, IndexedLogResults } from "./BufferIndexHelpers";
 import { UnitSet } from "./BufferUnitSet";
 import { BufferUnitIndex } from "./BufferUnitIndex";
 import { BufferListStreamer } from "./BufferListStreamer";
@@ -192,8 +192,8 @@ export class BufferIndex {
         };
     }
 
-    // Rebuild index completely from data reader
-    private static async rebuildIndexFromData(dataReader: Reader): Promise<Buffer> {
+    @measureFnc
+    private static async rebuildLocalIndexFromData(dataReader: Reader): Promise<Buffer> {
         let data = await dataReader.read(0, await dataReader.getLength());
         let dataBlocks = await BufferIndex.decodeAllBlocked(data);
 
@@ -210,9 +210,11 @@ export class BufferIndex {
         return Buffer.concat(parts);
     }
 
+    @measureFnc
     public static async fixPartialIndex(config: {
         index: Buffer;
         dataReader: Reader;
+        results: IndexedLogResults;
     }): Promise<Buffer> {
         let type = config.index[0];
         // Only streaming indexes can be partial — bulk indexes are written all at once.
@@ -223,7 +225,7 @@ export class BufferIndex {
         let decoded = decodeTypeHeader(index);
         if (!decoded) {
             // Index header is corrupted, regenerate complete index from data
-            return await BufferIndex.rebuildIndexFromData(dataReader);
+            return await BufferIndex.rebuildLocalIndexFromData(dataReader);
         }
         try {
             let { headerContent, data: indexData } = decoded;
@@ -270,6 +272,7 @@ export class BufferIndex {
                 let indexStreamer = indexStreamerType.createStreamer();
                 // Continue from where the index left off
                 for (let block of blocks) {
+                    await config.results.limitGroup?.wait();
                     try {
                         let decompressedBlock = CompressedStream.decode(block);
                         let blockBuffers = await blockStreamerType.getAllBlocks(decompressedBlock);
@@ -287,45 +290,120 @@ export class BufferIndex {
         } catch (e) {
             console.error(`Error fixing partial index. This SHOULDN'T error, but... we will just rebuild from the data and it should work: ${e}`);
         }
-        return await BufferIndex.rebuildIndexFromData(dataReader);
+        return await BufferIndex.rebuildLocalIndexFromData(dataReader);
     }
 
     @measureFnc
-    public static async find(config: {
+    private static async findLocal(config: {
         index: Buffer;
         dataReader: Reader;
-
         params: SearchParams;
-
         keepIterating: () => boolean;
         onResult: (match: Buffer) => void;
-    }): Promise<{
-        blocksChecked: number;
-        blocksCheckedCompressedSize: number;
-        blocksCheckedDecompressedSize: number;
+        results: IndexedLogResults;
+        allSearchUnits: Set<Unit>;
+        matchesPattern: (buffer: Buffer) => boolean;
+    }) {
+        let { index, dataReader, params, keepIterating, onResult, results, allSearchUnits, matchesPattern } = config;
+        // NOTE: Ironically, the stream type is actually the least efficient to read. Because there's no central index, it means that even if we do try to lazily read it, every single read call would likely have to scan through most of the file to find that specific block. However, this is fine. The stream type is mostly just used for pending files, which shouldn't be that large. And we still do only read the blocks when we have at least one match
+
+        let decoded = decodeTypeHeader(index);
+        if (!decoded) {
+            // Index is too corrupted, return empty results
+            return;
+        }
+        let { data: rawIndexData } = decoded;
+        let indexEntries = await indexStreamerType.getAllBlocks(rawIndexData);
+
+        const getDataBlocks = lazy(async (): Promise<Buffer[]> => {
+            return await measureBlock(async () => {
+                // NOTE: While this is somewhat inefficient, the fact that all these blocks are individually compressed makes this reasonable fast.
+                let length = await dataReader.getLength();
+                let dataIn = await dataReader.read(0, length);
+                let decoded = decodeTypeHeader(dataIn);
+                if (!decoded) return [];
+                let { data } = decoded;
+                return await dataStreamerType.getAllBlocks(data);
+            }, `BufferIndex|readLocalBlocks`);
+        });
 
-        blocksWithErrors: string[];
+        let matchCount = 0;
+        let blockSearchTimeStart = Date.now();
 
+        results.totalBlockCount += indexEntries.length;
+        results.localBlockCount += indexEntries.length;
 
-        isPending: boolean;
-        indexCount: number;
-        indexSize: number;
-        totalBlockCount: number;
+        // Iterate newest-first so the caller gets the most recent matches first.
+        for (let i = indexEntries.length - 1; i >= 0; i--) {
+            if (matchCount >= params.limit || !config.keepIterating()) break;
+            await config.results.limitGroup?.wait();
+            const blockIndex = i;
 
-        blockSearchTime: number;
-    }> {
-        let { index, dataReader, params } = config;
-        let emptyResult = { blocksChecked: 0, blocksCheckedCompressedSize: 0, blocksCheckedDecompressedSize: 0, blocksWithErrors: [], isPending: false, indexCount: 0, totalBlockCount: 0, blockSearchTime: 0, indexSize: 0 };
+            // Each index entry is a UnitSet
+            let blockIndexData = indexEntries[i];
 
-        // Handle empty or too small index buffer - rebuild from data
-        if (index.length === 0) {
-            index = await BufferIndex.rebuildIndexFromData(dataReader);
-            if (index.length === 0) {
-                return emptyResult;
+            // Check if this block contains all search units
+            let hasAllUnits = true;
+            for (let unit of allSearchUnits) {
+                if (!UnitSet.has(blockIndexData, unit)) {
+                    hasAllUnits = false;
+                    break;
+                }
+            }
+
+            results.localIndexesSearched += 1;
+            results.localIndexSize += blockIndexData.length;
+
+            if (!hasAllUnits) continue;
+
+            const dataBlocks = await getDataBlocks();
+
+            // Load and scan this block
+            try {
+                let blockCompressed = dataBlocks[blockIndex];
+                if (!blockCompressed) {
+                    throw new Error(`Not enough blocks in data, have ${dataBlocks.length}, expected ${blockIndex + 1}`);
+                }
+                results.blocksCheckedCompressedSize += blockCompressed.length;
+                let blockData = CompressedStream.decode(blockCompressed);
+                results.blocksCheckedDecompressedSize += blockData.length;
+                let buffers = await blockStreamerType.getAllBlocks(blockData);
+                results.blockCheckedCount++;
+                results.localBlockCheckedCount++;
+
+                // Scan all buffers in this block
+                for (let bufferIndex = buffers.length - 1; bufferIndex >= 0; bufferIndex--) {
+                    if (matchCount >= params.limit || !config.keepIterating()) break;
+                    await config.results.limitGroup?.wait();
+
+                    let buffer = buffers[bufferIndex];
+                    if (matchesPattern(buffer)) {
+                        config.onResult(buffer);
+                        matchCount++;
+                    }
+                }
+            } catch (e: any) {
+                // Skip corrupted block
+                results.blockErrors.push(`(for block ${blockIndex + 1} / ${dataBlocks.length}) ${String(e?.stack || e)}`);
+                console.warn(`Error decompressing block for search: ${e.stack || e}`);
+                continue;
             }
         }
+        results.blockSearchTime += Date.now() - blockSearchTimeStart;
+    }
+
+    @measureFnc
+    public static async find(config: {
+        index: Buffer;
+        dataReader: Reader;
+
+        params: SearchParams;
 
-        let candidateCount = 0;
+        keepIterating: () => boolean;
+        onResult: (match: Buffer) => void;
+        results: IndexedLogResults;
+    }): Promise<void> {
+        let { index, dataReader, params, results } = config;
 
         // Create the pattern matcher once with pre-calculated segments
         const matchesPattern = createMatchesPattern(params.findBuffer, !!params.disableWildCards);
@@ -342,118 +420,41 @@ export class BufferIndex {
             }
             if (allSearchUnits.size === 0) {
                 // Search pattern too short to use index, return empty results
-                return emptyResult;
+                return;
             }
         }
 
         let type = index[0];
+        if (!type) {
+            type = await measureBlock(async () => {
+                return (await dataReader.read(0, 1))?.[0];
+            }, `BufferIndex|recover type from data`);
+        }
 
         if (type === STREAM_TYPE) {
-
-            // Fix partial index before processing
-            index = await BufferIndex.fixPartialIndex({ index, dataReader });
-
-            // NOTE: Ironically, the stream type is actually the least efficient to read. Because there's no central index, it means that even if we do try to lazily read it, every single read call would likely have to scan through most of the file to find that specific block. However, this is fine. The stream type is mostly just used for pending files, which shouldn't be that large. And we still do only read the blocks when we have at least one match
-
-            let decoded = decodeTypeHeader(index);
-            if (!decoded) {
-                // Index is too corrupted, return empty results
-                return emptyResult;
-            }
-            let { data: rawIndexData } = decoded;
-            let indexEntries = await indexStreamerType.getAllBlocks(rawIndexData);
-
-            const getDataBlocks = lazy(async (): Promise<Buffer[]> => {
-                // NOTE: While this is somewhat inefficient, the fact that all these blocks are individually compressed makes this reasonable fast.
-                let length = await dataReader.getLength();
-                let dataIn = await dataReader.read(0, length);
-                let decoded = decodeTypeHeader(dataIn);
-                if (!decoded) return [];
-                let { data } = decoded;
-                return await dataStreamerType.getAllBlocks(data);
-            });
-
-            let matchCount = 0;
-            let blocksChecked = 0;
-            let blocksCheckedCompressedSize = 0;
-            let blocksCheckedDecompressedSize = 0;
-            let blockSearchTime = 0;
-            let blocksWithErrors: string[] = [];
-
-            // Iterate newest-first so the caller gets the most recent matches first.
-            for (let i = indexEntries.length - 1; i >= 0; i--) {
-                if (matchCount >= params.limit || !config.keepIterating()) break;
-                const blockIndex = i;
-
-                // Each index entry is a UnitSet
-                let blockIndexData = indexEntries[i];
-
-                // Check if this block contains all search units
-                let hasAllUnits = true;
-                for (let unit of allSearchUnits) {
-                    if (!UnitSet.has(blockIndexData, unit)) {
-                        hasAllUnits = false;
-                        break;
-                    }
+            // Handle empty or too small index buffer - rebuild from data
+            if (index.length === 0) {
+                index = await BufferIndex.rebuildLocalIndexFromData(dataReader);
+                if (index.length === 0) {
+                    return;
                 }
+            }
 
-                if (!hasAllUnits) continue;
-
-                const dataBlocks = await getDataBlocks();
+            // Fix partial index before processing
+            index = await BufferIndex.fixPartialIndex({ index, dataReader, results });
 
-                let blockSearchTimeStart = Date.now();
-                // Load and scan this block
-                try {
-                    let blockCompressed = dataBlocks[blockIndex];
-                    if (!blockCompressed) {
-                        throw new Error(`Not enough blocks in data, have ${dataBlocks.length}, expected ${blockIndex + 1}`);
-                    }
-                    blocksCheckedCompressedSize += blockCompressed.length;
-                    let blockData = CompressedStream.decode(blockCompressed);
-                    blocksCheckedDecompressedSize += blockData.length;
-                    let buffers = await blockStreamerType.getAllBlocks(blockData);
-                    blocksChecked++;
-
-                    // Scan all buffers in this block
-                    for (let bufferIndex = buffers.length - 1; bufferIndex >= 0; bufferIndex--) {
-                        if (matchCount >= params.limit || !config.keepIterating()) break;
-
-                        candidateCount++;
-                        let buffer = buffers[bufferIndex];
-                        if (matchesPattern(buffer)) {
-                            config.onResult(buffer);
-                            matchCount++;
-                        }
-                    }
-                } catch (e: any) {
-                    // Skip corrupted block
-                    blocksWithErrors.push(`(for block ${blockIndex + 1} / ${dataBlocks.length}) ${String(e?.stack || e)}`);
-                    console.warn(`Error decompressing block for search: ${e.stack || e}`);
-                    continue;
-                }
-                blockSearchTime += Date.now() - blockSearchTimeStart;
-            }
-            let indexSize = indexEntries.map(x => x.length).reduce((a, b) => a + b, 0);
-            return { blocksChecked, blocksCheckedCompressedSize, blocksCheckedDecompressedSize, blocksWithErrors, isPending: true, indexCount: indexEntries.length, indexSize, totalBlockCount: indexEntries.length, blockSearchTime };
+            await BufferIndex.findLocal({ index, dataReader, params, keepIterating: config.keepIterating, onResult: config.onResult, results, allSearchUnits, matchesPattern });
         } else if (type === BULK_TYPE) {
-            let results = await BufferUnitIndex.find({
+            await BufferUnitIndex.find({
                 params,
                 index,
                 reader: dataReader,
                 keepIterating: config.keepIterating,
                 onResult: config.onResult,
+                results,
             });
-            return {
-                blocksChecked: results.blocksChecked,
-                blocksCheckedCompressedSize: results.blocksCheckedCompressedSize,
-                blocksCheckedDecompressedSize: results.blocksCheckedDecompressedSize,
-                totalBlockCount: results.totalBlockCount,
-                blockSearchTime: results.blockSearchTime,
-                blocksWithErrors: results.blocksWithErrors,
-                isPending: false,
-                indexCount: 1,
-                indexSize: index.length,
-            };
+            results.remoteIndexesSearched += 1;
+            results.remoteIndexSize += index.length;
         } else {
             throw new Error(`Unknown type in index file: ${type}`);
         }

package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts

@@ -4,6 +4,9 @@ import { formatNumber, formatPercent } from "socket-function/src/formatting/form
 import { red } from "socket-function/src/formatting/logColors";
 import { MaybePromise } from "socket-function/src/types";
 import { TimeFilePathWithSize } from "./IndexedLogs";
+import { LimitGroup } from "../../../functional/limitProcessing";
+
+export const INDEX_EXTENSION = ".index";
 
 export type SearchParams = {
     startTime: number;
@@ -41,7 +44,6 @@ export class BufferReader implements Reader {
 }
 
 
-export const WILD_CARD_BYTE = 42;
 
 export function createOffsetReader(reader: Reader, offset: number): Reader {
     return {
@@ -81,6 +83,86 @@ export const getAllUnits = measureWrap(function getAllUnits(config: {
     return result;
 });
 
+
+export type IndexedLogResults = {
+    matchCount: number;
+
+    // NOTE: A lot of the metadata won't be accurate if multiple searches happen at the same time. However, for debugging, it should be sufficient. 
+    reads: {
+        cached: boolean;
+        remote: boolean;
+        count: number;
+        size: number;
+
+        totalSize: number;
+        totalCount: number;
+    }[];
+
+    totalLocalFiles: number;
+    totalBackblazeFiles: number;
+    localFilesSearched: number;
+    backblazeFilesSearched: number;
+
+    totalBlockCount: number;
+    blockCheckedCount: number;
+
+    remoteBlockCount: number;
+    localBlockCount: number;
+    remoteBlockCheckedCount: number;
+    localBlockCheckedCount: number;
+
+    blocksCheckedCompressedSize: number;
+    blocksCheckedDecompressedSize: number;
+    blockErrors: string[];
+
+    fileErrors: { error: string; path: string; }[];
+
+    remoteIndexesSearched: number;
+    remoteIndexSize: number;
+    localIndexesSearched: number;
+    localIndexSize: number;
+
+    timeToFirstMatch: number;
+    fileFindTime: number;
+    indexSearchTime: number;
+    blockSearchTime: number;
+
+    totalSearchTime: number;
+
+    cancel?: boolean;
+    limitGroup?: LimitGroup;
+};
+export function createEmptyIndexedLogResults(): IndexedLogResults {
+    return {
+        matchCount: 0, reads: [], totalLocalFiles: 0, totalBackblazeFiles: 0, localFilesSearched: 0, backblazeFilesSearched: 0, totalBlockCount: 0, blockCheckedCount: 0, remoteBlockCount: 0, localBlockCount: 0, remoteBlockCheckedCount: 0, localBlockCheckedCount: 0, blocksCheckedCompressedSize: 0, blocksCheckedDecompressedSize: 0, blockErrors: [], fileErrors: [], remoteIndexesSearched: 0, remoteIndexSize: 0, localIndexesSearched: 0, localIndexSize: 0, timeToFirstMatch: 0, fileFindTime: 0, indexSearchTime: 0, blockSearchTime: 0, totalSearchTime: 0, cancel: undefined, limitGroup: undefined,
+    };
+}
+
+export function addReadToResults(results: IndexedLogResults, read: {
+    cached: boolean;
+    remote: boolean;
+    count: number;
+    size: number;
+}) {
+    let existingRead = results.reads.find(r => r.cached === read.cached && r.remote === read.remote);
+    if (!existingRead) {
+        existingRead = {
+            cached: read.cached,
+            remote: read.remote,
+            count: 0,
+            size: 0,
+            totalSize: 0,
+            totalCount: 0,
+        };
+        results.reads.push(existingRead);
+    }
+    existingRead.count += read.count;
+    existingRead.size += read.size;
+    return existingRead;
+}
+
+
+export const WILD_CARD_BYTE = 42;
 export function splitOnWildcard(buffer: Buffer): Buffer[] {
     let segments: Buffer[] = [];
     let start = 0;

package/src/diagnostics/logs/IndexedLogs/BufferListStreamer.ts

@@ -1,3 +1,4 @@
+import { measureFnc } from "socket-function/src/profiling/measure";
 import { BufferReader, Reader } from "./BufferIndexHelpers";
 
 // Suffix layout (12 bytes), appended after each block's data:
@@ -120,6 +121,7 @@ export class BufferListStreamer {
         }
     }
 
+    @measureFnc
     public async getAllBlocks(buffer: Buffer): Promise<Buffer[]> {
         let blocks: Buffer[] = [];
         for await (let blockInfo of this.iterateBlocksReverse(buffer)) {

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -3,7 +3,7 @@
 import { LZ4 } from "../../../storage/LZ4";
 import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
 import { Zip } from "../../../zip";
-import { BufferReader, Reader, WILD_CARD_BYTE, createMatchesPattern, SearchParams } from "./BufferIndexHelpers";
+import { BufferReader, Reader, WILD_CARD_BYTE, createMatchesPattern, SearchParams, IndexedLogResults } from "./BufferIndexHelpers";
 import { formatNumber, formatPercent } from "socket-function/src/formatting/format";
 import { lazy } from "socket-function/src/caching";
 import { list, sort } from "socket-function/src/misc";
@@ -458,15 +458,9 @@ export class BufferUnitIndex {
         onResult: (match: Buffer) => void;
         index: Buffer;
         reader: Reader;
-    }): Promise<{
-        blocksChecked: number;
-        blocksCheckedCompressedSize: number;
-        blocksCheckedDecompressedSize: number;
-        totalBlockCount: number;
-        blockSearchTime: number;
-        blocksWithErrors: string[];
-    }> {
-        const { params, index, reader, keepIterating } = config;
+        results: IndexedLogResults;
+    }): Promise<void> {
+        const { params, index, reader, keepIterating, results } = config;
 
         // Split on wildcards if present
         function splitOnWildcard(buffer: Buffer): Buffer[] {
@@ -502,8 +496,8 @@ export class BufferUnitIndex {
                 intersectionSet = new Set([...intersectionSet].filter(x => currentSet.has(x)));
             }
 
-            // Search first first, as moveLogsToPublic should have made it so this is the newest.
-            return Array.from(intersectionSet).sort((a, b) => a - b);
+
+            return intersectionSet;
         }, `findCandidateBlocks`);
 
 
@@ -511,18 +505,18 @@ export class BufferUnitIndex {
 
         const headerBuffer = await reader.read(0, 8);
         const blockCount = headerBuffer.readUInt32LE(4);
-        let blocksCheckedCompressedSize = 0;
-        let blocksCheckedDecompressedSize = 0;
+
+        results.totalBlockCount += blockCount;
+        results.remoteBlockCount += blockCount;
 
         // Read blocks and search for matches
-        let blocksChecked = 0;
-        let blocksWithErrors: string[] = [];
         let blockSearchTimeStart = Date.now();
         await measureBlock(async () => {
             let matchCount = 0;
             let matchCounts = list(blockCount).fill(0);
 
             const searchBlock = async (blockIndex: number) => {
+                if (!candidateBlocks.has(blockIndex)) return;
                 // This is kind of a weird thing. Basically, because we search in parallel, we might search out of order. So we can only look at the counts before or at us, as if we match a whole bunch after us, but we should still keep going as our matches are going to take precedence.
                 let stopIterating = () => {
                     let countBefore = 0;
@@ -543,9 +537,10 @@ export class BufferUnitIndex {
                     }
                     let obj = await this.getBlock(reader, blockIndex, debugOffsets);
 
-                    blocksCheckedDecompressedSize += obj.block.length;
-                    blocksCheckedCompressedSize += obj.compressedSize;
-                    blocksChecked++;
+                    results.blocksCheckedDecompressedSize += obj.block.length;
+                    results.blocksCheckedCompressedSize += obj.compressedSize;
+                    results.blockCheckedCount++;
+                    results.remoteBlockCheckedCount++;
 
                     let blockReader = new BufferReader(obj.block);
                     let bufferCount = await this.getBufferCountFromBlock(blockReader);
@@ -553,6 +548,7 @@ export class BufferUnitIndex {
                     // Check each buffer for a match
                     for (let i = 0; i < bufferCount; i++) {
                         if (stopIterating()) break;
+                        await results.limitGroup?.wait();
 
                         const buffer = await this.getBufferFromBlock(blockReader, i);
                         if (matchesPattern(buffer)) {
@@ -563,7 +559,7 @@ export class BufferUnitIndex {
                     }
                 } catch (e: any) {
                     console.warn(`Error decompressing/searching block: ${e}`);
-                    blocksWithErrors.push(`(for block ${blockIndex + 1} / ${blockCount}, ${debugOffsets.startOffset}-${debugOffsets.endOffset}) ${String(e?.stack || e)}`);
+                    results.blockErrors.push(`(for block ${blockIndex + 1} / ${blockCount}, ${debugOffsets.startOffset}-${debugOffsets.endOffset}) ${String(e?.stack || e)}`);
                 }
             };
 
@@ -571,12 +567,13 @@ export class BufferUnitIndex {
                 { parallelCount: BufferUnitIndexParallelSearchCount },
                 searchBlock
             );
-            await Promise.all(list(blockCount).map(runSearchBlock));
+            // Search first first, as moveLogsToPublic should have made it so this is the newest.
+            let searchOrder = Array.from(candidateBlocks);
+            sort(searchOrder, x => x);
+            await Promise.all(searchOrder.map(runSearchBlock));
 
         }, `searchBlocks`);
-        let blockSearchTime = Date.now() - blockSearchTimeStart;
-
-        return { blocksChecked, blocksCheckedCompressedSize, blocksCheckedDecompressedSize, totalBlockCount: blockCount, blockSearchTime, blocksWithErrors };
+        results.blockSearchTime += Date.now() - blockSearchTimeStart;
     }
 
     private static findBlocks(config: {

package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts

@@ -9,7 +9,7 @@ export class UnitSet {
 
     @measureFnc
     static encode(blocks: Buffer[][]): Buffer {
-        const MAX_FILL_RATIO = 0.65;
+        const MAX_FILL_RATIO = 0.35;
 
         // First pass: count total units
         let totalUnits = 0;