querysub 0.366.0 → 0.368.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.366.0",
+  "version": "0.368.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/archiveapps/archiveGCEntry.tsx

@@ -1,4 +1,4 @@
-import "querysub/inject";
+import "../inject";
 
 import { Querysub } from "../4-querysub/QuerysubController";
 import { logErrors } from "../errors";

package/src/archiveapps/archiveMergeEntry.tsx

@@ -1,4 +1,4 @@
-import "querysub/inject";
+import "../inject";
 
 import { Querysub } from "../4-querysub/QuerysubController";
 import { logErrors } from "../errors";

package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts

@@ -9,7 +9,8 @@ import { cacheArgsEqual, cacheLimited, cacheWeak, lazy } from "socket-function/s
 import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { magenta, yellow } from "socket-function/src/formatting/logColors";
-import { Unit, getAllUnits, Reader, createMatchesPattern, createOffsetReader, splitOnWildcard, SearchParams, IndexedLogResults } from "./BufferIndexHelpers";
+import { Unit, getAllUnits, Reader, createOffsetReader, SearchParams, IndexedLogResults } from "./BufferIndexHelpers";
+import { createMatchesPattern, getSearchUnits } from "./bufferMatcher";
 import { UnitSet } from "./BufferUnitSet";
 import { BufferUnitIndex } from "./BufferUnitIndex";
 import { BufferListStreamer } from "./BufferListStreamer";
@@ -301,7 +302,7 @@ export class BufferIndex {
         keepIterating: () => boolean;
         onResult: (match: Buffer) => void;
         results: IndexedLogResults;
-        allSearchUnits: Set<Unit>;
+        allSearchUnits: Unit[][];
         matchesPattern: (buffer: Buffer) => boolean;
     }) {
         let { index, dataReader, params, keepIterating, onResult, results, allSearchUnits, matchesPattern } = config;
@@ -343,10 +344,17 @@ export class BufferIndex {
             let blockIndexData = indexEntries[i];
 
             // Check if this block contains all search units
-            let hasAllUnits = true;
-            for (let unit of allSearchUnits) {
-                if (!UnitSet.has(blockIndexData, unit)) {
-                    hasAllUnits = false;
+            let hasAnyOr = false;
+            for (let or of allSearchUnits) {
+                let hasAllUnits = true;
+                for (let unit of or) {
+                    if (!UnitSet.has(blockIndexData, unit)) {
+                        hasAllUnits = false;
+                        break;
+                    }
+                }
+                if (hasAllUnits) {
+                    hasAnyOr = true;
                     break;
                 }
             }
@@ -354,7 +362,7 @@ export class BufferIndex {
             results.localIndexesSearched += 1;
             results.localIndexSize += blockIndexData.length;
 
-            if (!hasAllUnits) continue;
+            if (!hasAnyOr) continue;
 
             const dataBlocks = await getDataBlocks();
 
@@ -405,23 +413,11 @@ export class BufferIndex {
     }): Promise<void> {
         let { index, dataReader, params, results } = config;
 
-        // Create the pattern matcher once with pre-calculated segments
         const matchesPattern = createMatchesPattern(params.findBuffer, !!params.disableWildCards);
 
-        // Compute search units once — shared by both index types
-        let allSearchUnits = new Set<Unit>();
-        {
-            let segments = params.disableWildCards && [params.findBuffer] || splitOnWildcard(params.findBuffer).filter(s => s.length > 0);
-            for (let seg of segments) {
-                if (seg.length < 4) continue;
-                for (let ref of getAllUnits({ buffer: seg, bufferIndex: 0, block: 0 })) {
-                    allSearchUnits.add(ref.unit);
-                }
-            }
-            if (allSearchUnits.size === 0) {
-                // Search pattern too short to use index, return empty results
-                return;
-            }
+        let allSearchUnits = getSearchUnits(params.findBuffer, !!params.disableWildCards);
+        if (allSearchUnits.length === 0) {
+            return;
         }
 
         let type = index[0];
@@ -447,6 +443,7 @@ export class BufferIndex {
         } else if (type === BULK_TYPE) {
             await BufferUnitIndex.find({
                 params,
+                allSearchUnits,
                 index,
                 reader: dataReader,
                 keepIterating: config.keepIterating,

package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts

@@ -16,6 +16,7 @@ export type SearchParams = {
     findBuffer: Buffer;
     pathOverrides?: TimeFilePathWithSize[];
     only?: "local" | "public";
+    forceReadPublic?: boolean;
 };
 
 export type Unit = number;
@@ -134,10 +135,72 @@ export type IndexedLogResults = {
 };
 export function createEmptyIndexedLogResults(): IndexedLogResults {
     return {
-        matchCount: 0, reads: [], totalLocalFiles: 0, totalBackblazeFiles: 0, localFilesSearched: 0, backblazeFilesSearched: 0, totalBlockCount: 0, blockCheckedCount: 0, remoteBlockCount: 0, localBlockCount: 0, remoteBlockCheckedCount: 0, localBlockCheckedCount: 0, blocksCheckedCompressedSize: 0, blocksCheckedDecompressedSize: 0, blockErrors: [], fileErrors: [], remoteIndexesSearched: 0, remoteIndexSize: 0, localIndexesSearched: 0, localIndexSize: 0, timeToFirstMatch: 0, fileFindTime: 0, indexSearchTime: 0, blockSearchTime: 0, totalSearchTime: 0, cancel: undefined, limitGroup: undefined,
+        matchCount: 0, reads: [], totalLocalFiles: 0, totalBackblazeFiles: 0, localFilesSearched: 0, backblazeFilesSearched: 0, totalBlockCount: 0, blockCheckedCount: 0, remoteBlockCount: 0, localBlockCount: 0, remoteBlockCheckedCount: 0, localBlockCheckedCount: 0, blocksCheckedCompressedSize: 0, blocksCheckedDecompressedSize: 0, blockErrors: [], fileErrors: [], remoteIndexesSearched: 0, remoteIndexSize: 0, localIndexesSearched: 0, localIndexSize: 0, timeToFirstMatch: -1, fileFindTime: 0, indexSearchTime: 0, blockSearchTime: 0, totalSearchTime: 0, cancel: undefined, limitGroup: undefined,
     };
 }
 
+export function mergeIndexedLogResults(existing: IndexedLogResults, incoming: IndexedLogResults): IndexedLogResults {
+    let readsByKey = new Map<string, typeof existing.reads[0]>();
+
+    for (let read of existing.reads) {
+        let key = `${read.cached}-${read.remote}`;
+        let existingRead = readsByKey.get(key);
+        if (existingRead) {
+            existingRead.count += read.count;
+            existingRead.size += read.size;
+            existingRead.totalSize = Math.max(existingRead.totalSize, read.totalSize);
+            existingRead.totalCount = Math.max(existingRead.totalCount, read.totalCount);
+        } else {
+            readsByKey.set(key, { ...read });
+        }
+    }
+
+    for (let read of incoming.reads) {
+        let key = `${read.cached}-${read.remote}`;
+        let existingRead = readsByKey.get(key);
+        if (existingRead) {
+            existingRead.count += read.count;
+            existingRead.size += read.size;
+            existingRead.totalSize = Math.max(existingRead.totalSize, read.totalSize);
+            existingRead.totalCount = Math.max(existingRead.totalCount, read.totalCount);
+        } else {
+            readsByKey.set(key, { ...read });
+        }
+    }
+
+    return {
+        matchCount: existing.matchCount + incoming.matchCount,
+        totalLocalFiles: existing.totalLocalFiles + incoming.totalLocalFiles,
+        totalBackblazeFiles: existing.totalBackblazeFiles + incoming.totalBackblazeFiles,
+        reads: Array.from(readsByKey.values()),
+        localFilesSearched: existing.localFilesSearched + incoming.localFilesSearched,
+        backblazeFilesSearched: existing.backblazeFilesSearched + incoming.backblazeFilesSearched,
+        totalBlockCount: existing.totalBlockCount + incoming.totalBlockCount,
+        blockCheckedCount: existing.blockCheckedCount + incoming.blockCheckedCount,
+        blocksCheckedCompressedSize: existing.blocksCheckedCompressedSize + incoming.blocksCheckedCompressedSize,
+        blocksCheckedDecompressedSize: existing.blocksCheckedDecompressedSize + incoming.blocksCheckedDecompressedSize,
+        blockErrors: [...existing.blockErrors, ...incoming.blockErrors],
+        fileErrors: [...existing.fileErrors, ...incoming.fileErrors],
+        remoteIndexesSearched: existing.remoteIndexesSearched + incoming.remoteIndexesSearched,
+        remoteIndexSize: existing.remoteIndexSize + incoming.remoteIndexSize,
+        localIndexesSearched: existing.localIndexesSearched + incoming.localIndexesSearched,
+        localIndexSize: existing.localIndexSize + incoming.localIndexSize,
+        timeToFirstMatch: Math.min(
+            existing.timeToFirstMatch === -1 ? Infinity : existing.timeToFirstMatch,
+            incoming.timeToFirstMatch === -1 ? Infinity : incoming.timeToFirstMatch
+        ),
+        fileFindTime: existing.fileFindTime + incoming.fileFindTime,
+        indexSearchTime: existing.indexSearchTime + incoming.indexSearchTime,
+        blockSearchTime: existing.blockSearchTime + incoming.blockSearchTime,
+        totalSearchTime: Math.max(existing.totalSearchTime, incoming.totalSearchTime),
+        remoteBlockCount: existing.remoteBlockCount + incoming.remoteBlockCount,
+        localBlockCount: existing.localBlockCount + incoming.localBlockCount,
+        remoteBlockCheckedCount: existing.remoteBlockCheckedCount + incoming.remoteBlockCheckedCount,
+        localBlockCheckedCount: existing.localBlockCheckedCount + incoming.localBlockCheckedCount,
+    };
+}
+
+
 export function addReadToResults(results: IndexedLogResults, read: {
     cached: boolean;
     remote: boolean;
@@ -161,62 +224,3 @@ export function addReadToResults(results: IndexedLogResults, read: {
     return existingRead;
 }
 
-
-export const WILD_CARD_BYTE = 42;
-export function splitOnWildcard(buffer: Buffer): Buffer[] {
-    let segments: Buffer[] = [];
-    let start = 0;
-    for (let i = 0; i <= buffer.length; i++) {
-        if (i === buffer.length || buffer[i] === WILD_CARD_BYTE) {
-            segments.push(buffer.slice(start, i));
-            start = i + 1;
-        }
-    }
-    return segments;
-}
-
-// Each WILD_CARD_BYTE in pattern acts as a multi-byte wildcard: the segments on either
-// side must appear in order somewhere within buffer.
-// Returns a function that matches buffers against the pre-processed pattern.
-export function createMatchesPattern(pattern: Buffer, disableWildCards: boolean): (buffer: Buffer) => boolean {
-    let segments = disableWildCards && [pattern] || splitOnWildcard(pattern).filter(s => s.length > 0);
-
-    return measureWrap(function matchesPattern(buffer: Buffer): boolean {
-        // Fast path: check if all segments exist anywhere in the buffer using indexOf
-        for (let seg of segments) {
-            if (buffer.indexOf(seg) === -1) {
-                return false;
-            }
-        }
-
-        // Tries to match all segments in order starting from bufferPos, returning the end
-        // position after the last match, or -1 if not all segments could be found.
-        function matchSegmentsFrom(bufferPos: number): number {
-            for (let seg of segments) {
-                function segMatchesAt(pos: number): boolean {
-                    for (let i = 0; i < seg.length; i++) {
-                        if (buffer[pos + i] !== seg[i]) return false;
-                    }
-                    return true;
-                }
-                let found = false;
-                for (let searchPos = bufferPos; searchPos <= buffer.length - seg.length; searchPos++) {
-                    if (segMatchesAt(searchPos)) {
-                        // NOTE: I think this is safe because every segment has a wildcard after it. So we can never have a case where we didn't skip far enough because the wild card will just skip farther. And we won't have a partial match as we're matching the whole chunk. So we won't match a prefix and then get stuck. I think... it does seem weird though...
-                        bufferPos = searchPos + seg.length;
-                        found = true;
-                        break;
-                    }
-                }
-                if (!found) return -1;
-            }
-            return bufferPos;
-        }
-
-        for (let startPos = 0; startPos <= buffer.length; startPos++) {
-            if (matchSegmentsFrom(startPos) >= 0) return true;
-        }
-        return false;
-    }, "BufferIndex|matchesPattern");
-}
-

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -3,7 +3,7 @@
 import { LZ4 } from "../../../storage/LZ4";
 import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
 import { Zip } from "../../../zip";
-import { BufferReader, Reader, WILD_CARD_BYTE, createMatchesPattern, SearchParams, IndexedLogResults } from "./BufferIndexHelpers";
+import { BufferReader, Reader, SearchParams, IndexedLogResults, Unit } from "./BufferIndexHelpers";
 import { formatNumber, formatPercent } from "socket-function/src/formatting/format";
 import { lazy } from "socket-function/src/caching";
 import { list, sort } from "socket-function/src/misc";
@@ -11,6 +11,7 @@ import { testDisableCache } from "../../../-a-archives/archivesMemoryCache";
 import { devDebugbreak } from "../../../config";
 import { BufferUnitIndexParallelSearchCount, DEFAULT_BLOCK_SIZE, DEFAULT_TARGET_UNITS_PER_BUCKET } from "./BufferIndexLogsOptimizationConstants";
 import { runInParallel } from "socket-function/src/batching";
+import { createMatchesPattern } from "./bufferMatcher";
 
 const USE_COMPRESSION = true;
 
@@ -454,51 +455,17 @@ export class BufferUnitIndex {
     @measureFnc
     public static async find(config: {
         params: SearchParams;
+        allSearchUnits: Unit[][];
         keepIterating: () => boolean;
         onResult: (match: Buffer) => void;
         index: Buffer;
         reader: Reader;
         results: IndexedLogResults;
     }): Promise<void> {
-        const { params, index, reader, keepIterating, results } = config;
-
-        // Split on wildcards if present
-        function splitOnWildcard(buffer: Buffer): Buffer[] {
-            const segments: Buffer[] = [];
-            let start = 0;
-            for (let i = 0; i <= buffer.length; i++) {
-                if (i === buffer.length || buffer[i] === WILD_CARD_BYTE) {
-                    segments.push(buffer.slice(start, i));
-                    start = i + 1;
-                }
-            }
-            return segments;
-        }
-
-        const segments = params.disableWildCards && [params.findBuffer] || splitOnWildcard(params.findBuffer).filter(s => s.length > 0);
-
-        // Find blocks for each segment >= 4 bytes
-        const candidateBlocks = measureBlock(() => {
-            const candidateBlocksPerSegment: number[][] = [];
-            for (const segment of segments) {
-                if (segment.length < 4) continue;
-                const blockIndices = this.findBlocks({ findBuffer: segment, index });
-                candidateBlocksPerSegment.push(blockIndices);
-            }
-
-            if (candidateBlocksPerSegment.length === 0) {
-                throw new Error("Search pattern too short: all segments are fewer than 4 bytes, cannot use index");
-            }
-
-            let intersectionSet = new Set<number>(candidateBlocksPerSegment[0]);
-            for (let i = 1; i < candidateBlocksPerSegment.length; i++) {
-                const currentSet = new Set(candidateBlocksPerSegment[i]);
-                intersectionSet = new Set([...intersectionSet].filter(x => currentSet.has(x)));
-            }
+        const { params, index, reader, keepIterating, results, allSearchUnits } = config;
 
-
-            return intersectionSet;
-        }, `findCandidateBlocks`);
+        let candidateBlocksList = allSearchUnits.map(units => this.findBlocks({ units, index })).flat();
+        let candidateBlocksSet = new Set(candidateBlocksList);
 
 
         const matchesPattern = createMatchesPattern(params.findBuffer, !!params.disableWildCards);
@@ -516,7 +483,7 @@ export class BufferUnitIndex {
             let matchCounts = list(blockCount).fill(0);
 
             const searchBlock = async (blockIndex: number) => {
-                if (!candidateBlocks.has(blockIndex)) return;
+                if (!candidateBlocksSet.has(blockIndex)) return;
                 // This is kind of a weird thing. Basically, because we search in parallel, we might search out of order. So we can only look at the counts before or at us, as if we match a whole bunch after us, but we should still keep going as our matches are going to take precedence.
                 let stopIterating = () => {
                     let countBefore = 0;
@@ -568,7 +535,7 @@ export class BufferUnitIndex {
                 searchBlock
             );
             // Search first first, as moveLogsToPublic should have made it so this is the newest.
-            let searchOrder = Array.from(candidateBlocks);
+            let searchOrder = Array.from(candidateBlocksSet);
             sort(searchOrder, x => x);
             await Promise.all(searchOrder.map(runSearchBlock));
 
@@ -576,23 +543,12 @@ export class BufferUnitIndex {
         results.blockSearchTime += Date.now() - blockSearchTimeStart;
     }
 
+    @measureFnc
     private static findBlocks(config: {
-        findBuffer: Buffer;
+        units: number[];
         index: Buffer;
     }): number[] {
-        const { findBuffer, index } = config;
-
-        // Extract all unique units from findBuffer
-        const units = measureBlock(() => {
-            const units: number[] = [];
-            for (let i = 0; i <= findBuffer.length - 4; i++) {
-                const unit = findBuffer.readUint32LE(i);
-                if (unit !== 0 && !units.includes(unit)) {
-                    units.push(unit);
-                }
-            }
-            return units;
-        }, `extractUnits`);
+        const { units, index } = config;
 
         if (units.length === 0) {
             return [];
@@ -600,25 +556,12 @@ export class BufferUnitIndex {
 
         // Get blocks for each unit and intersect
         return measureBlock(() => {
-            const candidateBlocksPerUnit: number[][] = [];
-            for (const unit of units) {
-                const blockIndices = this.getBlocksForUnit(index, unit);
-                candidateBlocksPerUnit.push(blockIndices);
+            let candidateBlocks = this.getBlocksForUnit(index, units[0]);
+            for (let i = 1; i < units.length; i++) {
+                let nextBlocks = new Set(this.getBlocksForUnit(index, units[i]));
+                candidateBlocks = candidateBlocks.filter(b => nextBlocks.has(b));
             }
-
-            // Intersect all block sets
-            let intersectionSet = new Set<number>(candidateBlocksPerUnit[0]);
-            for (let i = 1; i < candidateBlocksPerUnit.length; i++) {
-                const currentSet = new Set(candidateBlocksPerUnit[i]);
-                intersectionSet = new Set([...intersectionSet].filter(x => currentSet.has(x)));
-            }
-
-            let allCounts = candidateBlocksPerUnit.map(b => b.length);
-            sort(allCounts, x => x);
-
-            //console.log(`Candidate blocks ${intersectionSet.size}, minimum: ${Math.min(...candidateBlocksPerUnit.map(b => b.length))}, best 4 counts: ${allCounts.slice(0, 4).join(", ")}`);
-
-            return Array.from(intersectionSet);
+            return candidateBlocks;
         }, `intersectBlocks`);
     }
 

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -1,13 +1,13 @@
 import { lazy } from "socket-function/src/caching";
 import { Archives, nestArchives } from "../../../-a-archives/archives";
-import { deepCloneJSON, keyByArray, nextId, sort, timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { deepCloneJSON, keyByArray, nextId, sort, timeInHour, timeInMinute, timeInSecond, timeoutToUndefinedSilent } from "socket-function/src/misc";
 import { BufferIndex } from "./BufferIndex";
 import { delay, runInParallel, runInSerial, runInfinitePoll } from "socket-function/src/batching";
-import { IndexedLogResults, Reader, SearchParams, addReadToResults, createEmptyIndexedLogResults, INDEX_EXTENSION } from "./BufferIndexHelpers";
+import { IndexedLogResults, Reader, SearchParams, addReadToResults, createEmptyIndexedLogResults, INDEX_EXTENSION, mergeIndexedLogResults } from "./BufferIndexHelpers";
 import { getDomain, isPublic } from "../../../config";
 import { getArchivesHome, getArchivesLocal } from "../../../-a-archives/archivesDisk";
 import { getArchivesBackblaze, getArchivesBackblazePrivateImmutable } from "../../../-a-archives/archivesBackBlaze";
-import { getOwnThreadId } from "../../../-a-auth/certs";
+import { getMachineId, getOwnThreadId } from "../../../-a-auth/certs";
 import { ArchivesMemoryCacheStats, createArchivesMemoryCache } from "../../../-a-archives/archivesMemoryCache";
 import { registerShutdownHandler } from "../../periodic";
 import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
@@ -23,6 +23,8 @@ import { assertIsManagementUser } from "../../managementPages";
 import { ignoreErrors } from "../../../errors";
 import { blue } from "socket-function/src/formatting/logColors";
 import { LimitGroup } from "../../../functional/limitProcessing";
+import { getAllNodeIds } from "../../../-f-node-discovery/NodeDiscovery";
+import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabilities";
 
 export type TimeFilePathWithSize = TimeFilePath & {
     size: number;
@@ -222,32 +224,102 @@ export class IndexedLogs<T> {
         }
     }
 
+    private machineNodeCache = new Map<string, string>();
+    private async getMachineNodes(): Promise<string[]> {
+        let ownMachineId = getOwnMachineId();
+        let nodeIds = await getAllNodeIds();
+        let byMachineId = keyByArray(nodeIds, x => getMachineId(x));
+        byMachineId.delete(ownMachineId);
+
+        await Promise.all(Array.from(byMachineId.entries()).map(async ([machineId, nodeIds]) => {
+            let added = false;
+            const tryAdd = async (nodeId: string, preferredOnly = false) => {
+                if (added) return;
+                let hasLogger = await timeoutToUndefinedSilent(2500, IndexedLogShimController.nodes[nodeId].hasLogger(this.config.name));
+                if (!hasLogger) return false;
+                // NOTE: Prefer to do the searching on the move logs service. However, if it's not available, any service can do searching. It just might lag that server... 
+                if (preferredOnly) {
+                    let entryPoint = await timeoutToUndefinedSilent(2500, NodeCapabilitiesController.nodes[nodeId].getEntryPoint());
+                    if (!entryPoint?.includes("movelogs")) return false;
+                }
+                added = true;
+
+                this.machineNodeCache.set(machineId, nodeId);
+                return true;
+            };
+            let cached = this.machineNodeCache.get(machineId);
+            if (cached) {
+                if (await tryAdd(cached)) return;
+            }
+            // NOTE: I don't think we need to try a few nodes at a time. How many nodes are going to exist for one machine id anyways? The more machines we add, it won't add more nodes per machine id.
+            await Promise.all(nodeIds.map(nodeId => tryAdd(nodeId, true)));
+            if (!added) {
+                await Promise.all(nodeIds.map(nodeId => tryAdd(nodeId, false)));
+            }
+            if (!added) {
+                console.warn(`No machine node found for machine ${machineId}`, { machineId, nodeIds });
+            }
+        }));
+
+        return Array.from(this.machineNodeCache.values());
+    }
+
     @measureFnc
     public async getPaths(config: {
         startTime: number;
         endTime: number;
         only?: "local" | "public";
+        forceReadPublic?: boolean;
     }): Promise<TimeFilePathWithSize[]> {
+        let finalPaths: TimeFilePathWithSize[] = [];
+
+        if (!config.only && (config.forceReadPublic || isPublic())) {
+            let machineNodes = await this.getMachineNodes();
+            await Promise.all(machineNodes.map(async (machineNode) => {
+                try {
+                    let paths = await IndexedLogShimController.nodes[machineNode].getPaths({
+                        ...config,
+                        only: "local",
+                        indexedLogsName: this.config.name,
+                    });
+                    finalPaths.push(...paths);
+                } catch (e) {
+                    console.error(`Error in getting paths on machine node ${machineNode}: ${e}`);
+                }
+            }));
+        }
+
+        // If we're forcefully reading from the public server, but we're not public, the code above will be the only code which adds to results
+        if (!isPublic() && config.forceReadPublic) {
+            return finalPaths;
+        }
+
         let localLogs = this.getLocalLogs();
         let backblazeLogs = this.getPublicLogs();
-
         let paths: TimeFilePath[] = [];
 
+        let promises: Promise<void>[] = [];
+
         if (config.only !== "public") {
-            let localPaths = await new TimeFileTree(localLogs).findAllPaths({
-                startTime: config.startTime,
-                endTime: config.endTime,
-            });
-            paths.push(...localPaths);
+            promises.push((async () => {
+                let localPaths = await new TimeFileTree(localLogs).findAllPaths({
+                    startTime: config.startTime,
+                    endTime: config.endTime,
+                });
+                paths.push(...localPaths);
+            })());
         }
 
         if (config.only !== "local") {
-            let backblazePaths = await new TimeFileTree(backblazeLogs).findAllPaths({
-                startTime: config.startTime,
-                endTime: config.endTime,
-            });
-            paths.push(...backblazePaths);
+            promises.push((async () => {
+                let backblazePaths = await new TimeFileTree(backblazeLogs).findAllPaths({
+                    startTime: config.startTime,
+                    endTime: config.endTime,
+                });
+                paths.push(...backblazePaths);
+            })());
         }
+        await Promise.all(promises);
 
         // Bake in the size to ensure more cache hits (without this, every single access will always result in a file/remote call).
         let pathsWithSize = await Promise.all(paths.map(async (path): Promise<TimeFilePathWithSize> => {
@@ -262,7 +334,9 @@ export class IndexedLogs<T> {
             return { ...path, size, indexSize, sourceName: this.config.name };
         }));
 
-        return pathsWithSize;
+        finalPaths.push(...pathsWithSize);
+
+        return finalPaths;
     }
 
     @measureFnc
@@ -274,7 +348,51 @@ export class IndexedLogs<T> {
         let startTime = Date.now();
         let interval: NodeJS.Timeout | undefined;
 
+        let getFinalResults = (): IndexedLogResults => {
+            let results = createEmptyIndexedLogResults();
+            for (let result of allResults.values()) {
+                results = mergeIndexedLogResults(results, result);
+            }
+            return results;
+        };
+        let allResults = new Map<string, IndexedLogResults>();
+        let allDone: Promise<void>[] = [];
+
+        if (!config.params.only && (config.params.forceReadPublic || isPublic())) {
+            let machineNodes = await this.getMachineNodes();
+            allDone.push(...machineNodes.map(async (machineNode) => {
+                try {
+                    let resultObj = await this.clientFind({
+                        ...config,
+                        params: {
+                            ...config.params,
+                            only: "local",
+                        },
+                        nodeId: machineNode,
+                        onResult(match) {
+                            config.onResult(match);
+                        },
+                        onResults(results) {
+                            allResults.set(machineNode, results);
+                        },
+                    });
+                    allResults.set(machineNode, resultObj);
+                } catch (e) {
+                    console.error(`Error in finding logs on machine node ${machineNode}: ${e}`);
+                }
+            }));
+        }
+
+        // If we're forcefully reading from the public server, but we're not public, the code above will be the only code which adds to results
+        if (!isPublic() && config.params.forceReadPublic) {
+            return getFinalResults();
+        }
+
+
         let results: IndexedLogResults = createEmptyIndexedLogResults();
+        allResults.set("", results);
+
+
         results.limitGroup = new LimitGroup({
             maxTimePerBeforeWait: 500,
             waitTime: 250,
@@ -316,7 +434,8 @@ export class IndexedLogs<T> {
         if (onResultsCallback) {
             interval = setInterval(async () => {
                 updateResultsStats();
-                let shouldContinue = await onResultsCallback(deepCloneJSON(results));
+                let results = getFinalResults();
+                let shouldContinue = await onResultsCallback(results);
                 if (!shouldContinue) {
                     if (!results.cancel) {
                         console.log(blue(`Cancelled search on ${this.config.name}`));
@@ -327,13 +446,15 @@ export class IndexedLogs<T> {
         }
 
         try {
-            await this.findBase({
+            let promise = this.findBase({
                 params: config.params,
                 onResult: config.onResult,
                 results,
             });
+            allDone.push(promise);
+            await Promise.allSettled(allDone);
             updateResultsStats();
-            return results;
+            return getFinalResults();
         } finally {
             if (interval) {
                 clearInterval(interval);
@@ -482,8 +603,9 @@ export class IndexedLogs<T> {
         params: SearchParams;
         onResult: (match: T) => void;
         onResults?: (results: IndexedLogResults) => void;
+        nodeId?: string;
     }): Promise<IndexedLogResults> {
-        let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
+        let controller = IndexedLogShimController.nodes[config.nodeId || SocketFunction.getBrowserNodeId()];
         let findId = nextId();
         let callback = (match: T) => {
             config.onResult(match);
@@ -514,6 +636,8 @@ export class IndexedLogs<T> {
         startTime: number;
         endTime: number;
         only?: "local" | "public";
+        forceReadPublic?: boolean;
+
     }): Promise<TimeFilePathWithSize[]> {
         let controller = IndexedLogShimController.nodes[SocketFunction.getBrowserNodeId()];
         return await controller.getPaths({
@@ -521,6 +645,7 @@ export class IndexedLogs<T> {
             startTime: config.startTime,
             endTime: config.endTime,
             only: config.only,
+            forceReadPublic: config.forceReadPublic,
         });
     }
     public onFindResult(config: {
@@ -642,6 +767,7 @@ class IndexedLogShim {
         startTime: number;
         endTime: number;
         only?: "local" | "public";
+        forceReadPublic?: boolean;
     }): Promise<TimeFilePathWithSize[]> {
         let indexedLogs = loggerByName.get(config.indexedLogsName);
         if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
@@ -649,6 +775,7 @@ class IndexedLogShim {
             startTime: config.startTime,
             endTime: config.endTime,
             only: config.only,
+            forceReadPublic: config.forceReadPublic,
         });
     }
 
@@ -659,6 +786,10 @@ class IndexedLogShim {
         if (!indexedLogs) throw new Error(`Indexed logs ${config.indexedLogsName} not found`);
         await indexedLogs.moveLogsToPublic(true);
     }
+
+    public async hasLogger(name: string) {
+        return loggerByName.has(name);
+    }
 }
 
 const IndexedLogShimController = SocketFunction.register(
@@ -673,6 +804,9 @@ const IndexedLogShimController = SocketFunction.register(
         },
         forceMoveLogsToPublic: {
             hooks: [assertIsManagementUser]
+        },
+        hasLogger: {
+            hooks: [assertIsManagementUser]
         }
     })
 );

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -21,14 +21,16 @@ import { ObjectDisplay } from "../ObjectDisplay";
 import { formatDateJSX } from "../../../misc/formatJSX";
 import { PUBLIC_MOVE_THRESHOLD } from "./BufferIndexLogsOptimizationConstants";
 import { atomic } from "../../../2-proxy/PathValueProxyWatcher";
-import { errorToUndefined } from "querysub/src/errors";
-import { IndexedLogResults, createEmptyIndexedLogResults } from "./BufferIndexHelpers";
+import { errorToUndefined } from "../../../errors";
+import { IndexedLogResults, createEmptyIndexedLogResults, mergeIndexedLogResults } from "./BufferIndexHelpers";
 import { TimeFilePath } from "./TimeFileTree";
+import { isPublic } from "../../../config";
+import { Zip } from "../../../zip";
 
 let searchText = new URLParam("searchText", "");
-let readLiveData = new URLParam("readLiveData", false);
 let excludePendingResults = new URLParam("excludePendingResults", false);
 let limitURL = new URLParam("limit", 100);
+let readPublicLogs = new URLParam("readPublicLogs", false);
 
 let savedPathsURL = new URLParam("savedPaths", "");
 let selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
@@ -43,63 +45,6 @@ const defaultSelectedFields = {
     __entry: true,
 };
 
-function mergeIndexedLogResults(existing: IndexedLogResults, incoming: IndexedLogResults): IndexedLogResults {
-    let readsByKey = new Map<string, typeof existing.reads[0]>();
-
-    for (let read of existing.reads) {
-        let key = `${read.cached}-${read.remote}`;
-        let existingRead = readsByKey.get(key);
-        if (existingRead) {
-            existingRead.count += read.count;
-            existingRead.size += read.size;
-            existingRead.totalSize = Math.max(existingRead.totalSize, read.totalSize);
-            existingRead.totalCount = Math.max(existingRead.totalCount, read.totalCount);
-        } else {
-            readsByKey.set(key, { ...read });
-        }
-    }
-
-    for (let read of incoming.reads) {
-        let key = `${read.cached}-${read.remote}`;
-        let existingRead = readsByKey.get(key);
-        if (existingRead) {
-            existingRead.count += read.count;
-            existingRead.size += read.size;
-            existingRead.totalSize = Math.max(existingRead.totalSize, read.totalSize);
-            existingRead.totalCount = Math.max(existingRead.totalCount, read.totalCount);
-        } else {
-            readsByKey.set(key, { ...read });
-        }
-    }
-
-    return {
-        matchCount: existing.matchCount + incoming.matchCount,
-        totalLocalFiles: existing.totalLocalFiles + incoming.totalLocalFiles,
-        totalBackblazeFiles: existing.totalBackblazeFiles + incoming.totalBackblazeFiles,
-        reads: Array.from(readsByKey.values()),
-        localFilesSearched: existing.localFilesSearched + incoming.localFilesSearched,
-        backblazeFilesSearched: existing.backblazeFilesSearched + incoming.backblazeFilesSearched,
-        totalBlockCount: existing.totalBlockCount + incoming.totalBlockCount,
-        blockCheckedCount: existing.blockCheckedCount + incoming.blockCheckedCount,
-        blocksCheckedCompressedSize: existing.blocksCheckedCompressedSize + incoming.blocksCheckedCompressedSize,
-        blocksCheckedDecompressedSize: existing.blocksCheckedDecompressedSize + incoming.blocksCheckedDecompressedSize,
-        blockErrors: [...existing.blockErrors, ...incoming.blockErrors],
-        fileErrors: [...existing.fileErrors, ...incoming.fileErrors],
-        remoteIndexesSearched: existing.remoteIndexesSearched + incoming.remoteIndexesSearched,
-        remoteIndexSize: existing.remoteIndexSize + incoming.remoteIndexSize,
-        localIndexesSearched: existing.localIndexesSearched + incoming.localIndexesSearched,
-        localIndexSize: existing.localIndexSize + incoming.localIndexSize,
-        timeToFirstMatch: Math.min(existing.timeToFirstMatch === 0 ? Infinity : existing.timeToFirstMatch, incoming.timeToFirstMatch === 0 ? Infinity : incoming.timeToFirstMatch),
-        fileFindTime: existing.fileFindTime + incoming.fileFindTime,
-        indexSearchTime: existing.indexSearchTime + incoming.indexSearchTime,
-        blockSearchTime: existing.blockSearchTime + incoming.blockSearchTime,
-        totalSearchTime: Math.max(existing.totalSearchTime, incoming.totalSearchTime),
-        remoteBlockCount: existing.remoteBlockCount + incoming.remoteBlockCount,
-        localBlockCount: existing.localBlockCount + incoming.localBlockCount,
-        remoteBlockCheckedCount: existing.remoteBlockCheckedCount + incoming.remoteBlockCheckedCount,
-        localBlockCheckedCount: existing.localBlockCheckedCount + incoming.localBlockCheckedCount,
-    };
-}
 
 export class LogViewer3 extends qreact.Component {
     static renderInProgress = true;
@@ -132,14 +77,11 @@ export class LogViewer3 extends qreact.Component {
         let paths: TimeFilePathWithSize[] = [];
         if (savedPathsURL.value) {
             const compressed = Buffer.from(savedPathsURL.value, "base64");
-            const decompressed = LZ4.decompress(compressed);
+            const decompressed = Zip.gunzipSync(compressed);
             paths = JSON.parse(decompressed.toString("utf8"));
         } else {
             paths = this.state.paths;
         }
-        if (excludePendingResults.value) {
-            paths = paths.filter(x => x.logCount !== undefined);
-        }
         return paths;
     }
 
@@ -154,7 +96,7 @@ export class LogViewer3 extends qreact.Component {
     freezePaths() {
         const json = JSON.stringify(this.state.paths);
         const buffer = Buffer.from(json, "utf8");
-        const compressed = LZ4.compress(buffer);
+        const compressed = Zip.gzipSync(buffer, 9);
         savedPathsURL.value = compressed.toString("base64");
     }
 
@@ -202,8 +144,8 @@ export class LogViewer3 extends qreact.Component {
         const renderStage = (title: string, items: preact.ComponentChild[], hue: number | undefined, isFirst: boolean, isLast: boolean) => {
             const triangleWidth = 20;
 
-            let color = hue ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
-            let colorhsl = hue ? css.hslcolor(hue, 80, 30) : css;
+            let color = hue !== undefined ? css.hsl(hue, 70, 80) : css.hsl(0, 0, 100);
+            let colorhsl = hue !== undefined ? css.hslcolor(hue, 80, 30) : css;
 
             return (
                 <div className={css.vbox(2).relative.minWidth(200).paddingTop(8).paddingBottom(8).paddingRight(10).paddingLeft(isFirst ? 10 : 10 + triangleWidth) + color}>
@@ -211,7 +153,7 @@ export class LogViewer3 extends qreact.Component {
                     {items.map((item, i) => (
                         <div key={i}>{item}</div>
                     ))}
-                    {!isLast && (
+                    {!isLast && hue !== undefined && (
                         <svg width={triangleWidth} height="100" viewBox="0 0 20 100" preserveAspectRatio="none" className={css.absolute.right(-triangleWidth).top(0).fillHeight.zIndex(1)}>
                             <polygon points="0,0 20,50 0,100" fill={`hsl(${hue}, 70%, 80%)`} />
                         </svg>
@@ -262,8 +204,8 @@ export class LogViewer3 extends qreact.Component {
                 {renderStage("Indexes", indexItems, 250, false, false)}
                 {renderStage("Blocks", blockItems, 210, false, false)}
                 {renderStage("Reads", cacheItems, 160, false, false)}
-                {renderStage("Results", resultItems, 120, false, false)}
-                {renderStage("Done", doneItems, undefined, false, !hasErrors)}
+                {renderStage("Results", resultItems, 120, false, this.state.searching && !hasErrors)}
+                {!this.state.searching && renderStage("Done", doneItems, undefined, false, !hasErrors)}
                 {(() => {
                     if (!hasErrors) return undefined;
                     let errorItems: preact.ComponentChild[] = [];
@@ -339,7 +281,7 @@ export class LogViewer3 extends qreact.Component {
                 )}
                 {!isFrozen && (
                     <>
-                        <div className={css.fontWeight(600)}>FIX pending logs not being merged!</div>
+                        <div className={css.fontWeight(600)}>Pending logs not being merged!</div>
                         {warnings.map((warning) => (
                             <div key={warning.machineId} className={css.hbox(10)}>
                                 <MachineThreadInfo machineId={warning.machineId} />
@@ -401,6 +343,8 @@ export class LogViewer3 extends qreact.Component {
 
         Querysub.commitLocal(() => {
             this.state.loadingPaths = true;
+            this.state.results = [];
+            this.state.stats = undefined;
         });
 
         let loggers = await getLoggers2Async();
@@ -423,13 +367,15 @@ export class LogViewer3 extends qreact.Component {
         let allPaths: TimeFilePathWithSize[] = [];
         let range = Querysub.localRead(() => getTimeRange());
 
-        for (let logger of selectedLoggers) {
+        await Promise.all(selectedLoggers.map(async (logger) => {
             let paths = await logger.clientGetPaths({
                 startTime: range.startTime,
                 endTime: range.endTime,
+                only: excludePendingResults.value ? "local" : undefined,
+                forceReadPublic: readPublicLogs.value,
             });
             allPaths.push(...paths);
-        }
+        }));
         sort(allPaths, x => -x.startTime);
 
         Querysub.commitLocal(() => {
@@ -439,7 +385,7 @@ export class LogViewer3 extends qreact.Component {
         return allPaths;
     }
 
-    async updatePaths() {
+    async updateToLatestResults() {
         let prevPaths = this.getPaths();
         let newPaths = await this.loadPaths();
         Querysub.commitLocal(() => {
@@ -449,6 +395,7 @@ export class LogViewer3 extends qreact.Component {
             let keep = new Set(prevPaths.map(getHash));
             this.state.paths = newPaths?.filter(x => keep.has(getHash(x))) || [];
         });
+        await this.search();
     }
 
     cancel = async () => {
@@ -506,7 +453,7 @@ export class LogViewer3 extends qreact.Component {
 
         let hasPaths = Querysub.localRead(() => this.getPaths().length > 0);
         let getFilesTime = 0;
-        if (readLiveData.value || !hasPaths) {
+        if (!hasPaths) {
             let startTime = Date.now();
             await this.loadPaths();
             getFilesTime = Date.now() - startTime;
@@ -564,6 +511,8 @@ export class LogViewer3 extends qreact.Component {
                     limit: limitURL.value,
                     findBuffer: searchBuffer,
                     pathOverrides: paths,
+                    only: excludePendingResults.value ? "local" : undefined,
+                    forceReadPublic: readPublicLogs.value,
                 },
                 onResult: (match: LogDatum) => {
                     console.log("onResult", match);
@@ -734,16 +683,16 @@ export class LogViewer3 extends qreact.Component {
             <div className={css.vbox(20).pad2(20).fillBoth}>
                 <div className={css.hbox(20)}>
                     <div>Log Viewer 3</div>
-                    <InputLabelURL
-                        checkbox
-                        label="Always Read Live Data"
-                        url={readLiveData}
-                    />
                     <InputLabelURL
                         checkbox
                         label="Exclude Pending Results"
                         url={excludePendingResults}
                     />
+                    {!isPublic() && <InputLabelURL
+                        checkbox
+                        label="Read Public Logs"
+                        url={readPublicLogs}
+                    />}
                     <InputLabelURL
                         label="Limit"
                         number
@@ -794,6 +743,9 @@ export class LogViewer3 extends qreact.Component {
                     </Button>
                     <TimeRangeSelector />
                 </div>
+                {!isPublic() && readPublicLogs.value && <div className={css.fontSize(40).pad2(12, 6).hsl(280, 40, 50).colorhsl(0, 0, 100).boldStyle}>
+                    Reading public logs
+                </div>}
 
                 <div className={css.hbox(10).fillWidth}>
                     <InputLabelURL
@@ -814,12 +766,12 @@ export class LogViewer3 extends qreact.Component {
                             onClick={() => void this.loadPaths()}
                             hue={this.state.paths.length ? 200 : 120}
                         >
-                            {this.state.paths.length && "Reset Files" || "Preview Files"}
+                            {this.getPaths().length && "Reset Files" || "Preview Files"}
                         </Button>
                     )}
                     {this.getPaths().length > 0 &&
-                        <Button flavor="large" hue={120} onClick={() => void this.updatePaths()}>
-                            Update Files
+                        <Button flavor="large" hue={120} onClick={() => void this.updateToLatestResults()}>
+                            Search Latest
                         </Button>
                     }
                     {savedPathsURL.value && (

package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts

@@ -156,12 +156,12 @@ export class TimeFileTree {
                 // Find all days in this month
                 const days = await this.archives.find(monthFolder + "/", { shallow: true, type: "folders" });
 
-                for (const dayFolder of days) {
+                await Promise.all(days.map(async (dayFolder) => {
                     // dayFolder is a full path like "2024/01/15"
                     const day = parseInt(dayFolder.split("/").at(-1) || "", 10);
 
                     if (isNaN(day) || day < 1 || day > 31) {
-                        continue;
+                        return;
                     }
 
                     // Check if this day is in range
@@ -169,7 +169,7 @@ export class TimeFileTree {
                     const dayEndDate = new Date(Date.UTC(year, month - 1, day, 23, 59, 59, 999));
 
                     if (dayEndDate.getTime() < config.startTime || dayDate.getTime() > config.endTime) {
-                        continue;
+                        return;
                     }
 
                     // Find all log files in this day
@@ -187,7 +187,7 @@ export class TimeFileTree {
                             results.push(decoded);
                         }
                     }
-                }
+                }));
             }
         }
 

package/src/diagnostics/logs/IndexedLogs/bufferMatcher.ts

@@ -0,0 +1,215 @@
+import { measureWrap } from "socket-function/src/profiling/measure";
+import { Unit, getAllUnits } from "./BufferIndexHelpers";
+
+const WILD_CARD_BYTE = 42;
+const WILD_CARD_CHAR = "*";
+const SEARCH_OR_BYTE = 124;
+const SEARCH_OR_CHAR = "|";
+const SEARCH_AND_BYTE = 38;
+const SEARCH_AND_CHAR = "&";
+
+export type MatchStructure = {
+    type: "or";
+    parts: MatchStructure[];
+} | {
+    type: "and";
+    parts: MatchStructure[];
+} | {
+    // An AND, but the order must be subsequentially, and there can't be overlapping ("cat*ate" matches "cat ate my bird", but not "cate", where as "cat&ate" matches "cate")
+    type: "wildcard";
+    parts: MatchStructure[];
+} | {
+    type: "exact";
+    value: Buffer;
+};
+
+function parseMatchStructure(pattern: Buffer): MatchStructure {
+    let text = pattern.toString("utf-8");
+
+    let orParts = text.split(SEARCH_OR_CHAR);
+    if (orParts.length > 1) {
+        return {
+            type: "or",
+            parts: orParts.map(part => parseMatchStructure(Buffer.from(part)))
+        };
+    }
+
+    let andParts = text.split(SEARCH_AND_CHAR);
+    if (andParts.length > 1) {
+        return {
+            type: "and",
+            parts: andParts.map(part => parseMatchStructure(Buffer.from(part)))
+        };
+    }
+
+    let wildcardParts = text.split(WILD_CARD_CHAR).filter(p => p.length > 0);
+    if (wildcardParts.length > 1) {
+        return {
+            type: "wildcard",
+            parts: wildcardParts.map(part => ({ type: "exact" as const, value: Buffer.from(part) }))
+        };
+    }
+
+    return { type: "exact", value: pattern };
+}
+// inside is and, outside is outer
+export function getSearchUnits(pattern: Buffer, disableSpecialCharacters: boolean): Unit[][] {
+    if (disableSpecialCharacters || !pattern.includes(SEARCH_OR_BYTE) && !pattern.includes(SEARCH_AND_BYTE) && !pattern.includes(WILD_CARD_BYTE)) {
+        return [getAllUnits({ buffer: pattern, bufferIndex: 0, block: 0 }).map(u => u.unit)];
+    }
+    let simpler = getSimplerStructure(parseMatchStructure(pattern));
+    let mapped = simpler.map(inner =>
+        inner.map(buffer =>
+            getAllUnits({ buffer, bufferIndex: 0, block: 0 }).map(u => u.unit)
+        ).flat()
+    ).filter(part => part.length > 0);
+    return mapped;
+}
+
+// inside is and, outside is outer
+function getSimplerStructure(structure: MatchStructure): Buffer[][] {
+    const type = structure.type;
+    if (type === "exact") {
+        return [[structure.value]];
+    }
+
+    if (type === "or") {
+        let result: Buffer[][] = [];
+        for (let part of structure.parts) {
+            result.push(...getSimplerStructure(part));
+        }
+        return result;
+    }
+
+    if (type === "and" || type === "wildcard") {
+        let result: Buffer[][] = [[]];
+
+        for (let part of structure.parts) {
+            let partResult = getSimplerStructure(part);
+            let newResult: Buffer[][] = [];
+
+            for (let existingClause of result) {
+                for (let partClause of partResult) {
+                    newResult.push([...existingClause, ...partClause]);
+                }
+            }
+
+            result = newResult;
+        }
+
+        return result;
+    }
+    let unhandled: never = type;
+
+    throw new Error(`Unknown structure type: ${type}`);
+}
+// Each WILD_CARD_BYTE in pattern acts as a multi-byte wildcard: the segments on either
+// side must appear in order somewhere within buffer.
+// Returns a function that matches buffers against the pre-processed pattern.
+export function createMatchesPattern(pattern: Buffer, disableSpecialCharacters: boolean): (buffer: Buffer) => boolean {
+
+    if (disableSpecialCharacters || !pattern.includes(SEARCH_OR_BYTE) && !pattern.includes(SEARCH_AND_BYTE) && !pattern.includes(WILD_CARD_BYTE)) {
+        return measureWrap(function matchesPattern(buffer: Buffer): boolean {
+            return buffer.indexOf(pattern) !== -1;
+        }, "BufferIndex|matchesPatternExact");
+    }
+
+
+    let structure = parseMatchStructure(pattern);
+    let simplerStructure = getSimplerStructure(structure);
+    let hasAnyWildcard = pattern.includes(WILD_CARD_BYTE);
+
+    return measureWrap(function matchesPattern(buffer: Buffer): boolean {
+        // 1) First, use simplerStructure to exclude any false cases
+        // simplerStructure is Buffer[][] where outer array is OR, inner array is AND
+        let anyOrClausePassed = false;
+        for (let andClause of simplerStructure) {
+            let allAndTermsFound = true;
+            for (let seg of andClause) {
+                if (buffer.indexOf(seg) === -1) {
+                    allAndTermsFound = false;
+                    break;
+                }
+            }
+            if (allAndTermsFound) {
+                anyOrClausePassed = true;
+                break;
+            }
+        }
+
+        if (!anyOrClausePassed) {
+            return false;
+        }
+
+        // 2) If !hasAnyWildcard, return true
+        if (!hasAnyWildcard) {
+            return true;
+        }
+
+        // 3) Do the expensive wildcard check on the full structure.
+        return matchesStructure(buffer, structure);
+
+        function matchesStructure(buf: Buffer, struct: MatchStructure): boolean {
+            const type = struct.type;
+
+            if (type === "exact") {
+                return buf.indexOf(struct.value) !== -1;
+            }
+
+            if (type === "or") {
+                for (let part of struct.parts) {
+                    if (matchesStructure(buf, part)) {
+                        return true;
+                    }
+                }
+                return false;
+            }
+
+            if (type === "and") {
+                for (let part of struct.parts) {
+                    if (!matchesStructure(buf, part)) {
+                        return false;
+                    }
+                }
+                return true;
+            }
+
+            if (type === "wildcard") {
+                let segments = struct.parts.map(p => {
+                    if (p.type !== "exact") throw new Error("Wildcard parts must be exact");
+                    return p.value;
+                });
+
+                function matchSegmentsFrom(bufferPos: number): number {
+                    for (let seg of segments) {
+                        function segMatchesAt(pos: number): boolean {
+                            for (let i = 0; i < seg.length; i++) {
+                                if (buf[pos + i] !== seg[i]) return false;
+                            }
+                            return true;
+                        }
+                        let found = false;
+                        for (let searchPos = bufferPos; searchPos <= buf.length - seg.length; searchPos++) {
+                            if (segMatchesAt(searchPos)) {
+                                bufferPos = searchPos + seg.length;
+                                found = true;
+                                break;
+                            }
+                        }
+                        if (!found) return -1;
+                    }
+                    return bufferPos;
+                }
+
+                for (let startPos = 0; startPos <= buf.length; startPos++) {
+                    if (matchSegmentsFrom(startPos) >= 0) return true;
+                }
+                return false;
+            }
+
+            let unhandled: never = type;
+            throw new Error(`Unknown structure type: ${type}`);
+        }
+    }, "BufferIndex|matchesPattern");
+}
+

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -19,35 +19,18 @@ IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs shou
 
 
 
-Make sure audits are appearing.
-    new non-local WATCH
+0) Deploy again
+1) Verify isPublic searching works (from our local machine)
 
 
-2.0) SUPPORT reading pending from multiple servers
-    - The main controller has to find a node on each other machine, and call it. Only one node per machine though, so it shouldn't be too difficult.
-        - We'll cache the last node per machine that we picked. 
-        - If the cache value doesn't exist, or if it doesn't work, if it throws an error when we try to verify it works, then we'll call a function to get the entry point on all of the nodes for that machine
-            - After we receive the first result, we'll wait at least a second so we get some more results, and then we'll prioritize the one that's the function endpoint, which will end in function.js. 
-
-2) Add a UI toggle to read public logs (only shows up on a non-public server though, as otherwise it wouldn't make sense)
-    - Basically, just changes the code we're reading from multiple servers to select public servers instead, and then, of course, skip ourselves. 
-
-BUG: UGH... live logs for the remote server isn't working...
-    new non-local WATCH
-    - UGH... it being pending logs is annoying, as that's hard to debug locally...
-    AH! Why do we have so few logs?
-
-
-1) Support & / | syntax in find
-    - Add note above search for *, &, | syntax
-
-NOTE: The reason we're not going to add the ability to pin to specific threads is because The files are going to be combined cross-thread anyway, and that should be mostly the files we're searching eventually. 
 
 2) Create lot of remote server logs
     - Via our refresh loop
 
 
-3) Deploy service for movelogs
+3) Create and deploy service for movelogs
+3.0) Update preferred entrypoint to be the movelogs entrypoint
+
 0) Run move logs in function runner, in development, just so we don't get too far behind