querysub 0.365.0 → 0.367.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.365.0",
+  "version": "0.367.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/archiveapps/archiveGCEntry.tsx

@@ -1,4 +1,4 @@
-import "querysub/inject";
+import "../inject";
 
 import { Querysub } from "../4-querysub/QuerysubController";
 import { logErrors } from "../errors";

package/src/archiveapps/archiveMergeEntry.tsx

@@ -1,4 +1,4 @@
-import "querysub/inject";
+import "../inject";
 
 import { Querysub } from "../4-querysub/QuerysubController";
 import { logErrors } from "../errors";

package/src/config.ts

@@ -25,7 +25,6 @@ let yargObj = parseArgsFactory()
     .option("diskaudit", {
         type: "boolean",
         // NOTE: I wanna see how long I can keep this on for. Eventually it's gonna become a problem and we're gonna have to turn it off. But for testing it's certainly useful as we don't know exactly what is gonna cause a problem. But it probably will be synchronization related, and every server does synchronization.
-        default: true,
         desc: "Track all audit logs to disk. This might end up writing A LOT of data."
     })
     .argv
@@ -81,7 +80,7 @@ export function isRecovery() {
 }
 
 export function isDiskAudit() {
-    return !!yargObj.diskaudit;
+    return !!(yargObj.diskaudit ?? true);
 }
 
 export function devDebugbreak() {

package/src/deployManager/machineController.ts

@@ -192,8 +192,8 @@ export const MachineController = getSyncedController(SocketFunction.register(
     }),
 ), {
     writes: {
-        deployMachineFromBrowser: ["MachineInfo"],
-        deployMachine: ["MachineInfo"],
+        deployMachineFromBrowser: ["MachineInfo", "RollingInfo"],
+        deployMachine: ["MachineInfo", "RollingInfo"],
         killRollingServicesFromBrowser: ["RollingInfo"],
     },
     reads: {

package/src/diagnostics/logs/IndexedLogs/BufferIndex.ts

@@ -9,7 +9,8 @@ import { cacheArgsEqual, cacheLimited, cacheWeak, lazy } from "socket-function/s
 import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { magenta, yellow } from "socket-function/src/formatting/logColors";
-import { Unit, getAllUnits, Reader, createMatchesPattern, createOffsetReader, splitOnWildcard, SearchParams, IndexedLogResults } from "./BufferIndexHelpers";
+import { Unit, getAllUnits, Reader, createOffsetReader, SearchParams, IndexedLogResults } from "./BufferIndexHelpers";
+import { createMatchesPattern, getSearchUnits } from "./bufferMatcher";
 import { UnitSet } from "./BufferUnitSet";
 import { BufferUnitIndex } from "./BufferUnitIndex";
 import { BufferListStreamer } from "./BufferListStreamer";
@@ -301,7 +302,7 @@ export class BufferIndex {
         keepIterating: () => boolean;
         onResult: (match: Buffer) => void;
         results: IndexedLogResults;
-        allSearchUnits: Set<Unit>;
+        allSearchUnits: Unit[][];
         matchesPattern: (buffer: Buffer) => boolean;
     }) {
         let { index, dataReader, params, keepIterating, onResult, results, allSearchUnits, matchesPattern } = config;
@@ -343,10 +344,17 @@ export class BufferIndex {
             let blockIndexData = indexEntries[i];
 
             // Check if this block contains all search units
-            let hasAllUnits = true;
-            for (let unit of allSearchUnits) {
-                if (!UnitSet.has(blockIndexData, unit)) {
-                    hasAllUnits = false;
+            let hasAnyOr = false;
+            for (let or of allSearchUnits) {
+                let hasAllUnits = true;
+                for (let unit of or) {
+                    if (!UnitSet.has(blockIndexData, unit)) {
+                        hasAllUnits = false;
+                        break;
+                    }
+                }
+                if (hasAllUnits) {
+                    hasAnyOr = true;
                     break;
                 }
             }
@@ -354,7 +362,7 @@ export class BufferIndex {
             results.localIndexesSearched += 1;
             results.localIndexSize += blockIndexData.length;
 
-            if (!hasAllUnits) continue;
+            if (!hasAnyOr) continue;
 
             const dataBlocks = await getDataBlocks();
 
@@ -405,23 +413,11 @@ export class BufferIndex {
     }): Promise<void> {
         let { index, dataReader, params, results } = config;
 
-        // Create the pattern matcher once with pre-calculated segments
         const matchesPattern = createMatchesPattern(params.findBuffer, !!params.disableWildCards);
 
-        // Compute search units once — shared by both index types
-        let allSearchUnits = new Set<Unit>();
-        {
-            let segments = params.disableWildCards && [params.findBuffer] || splitOnWildcard(params.findBuffer).filter(s => s.length > 0);
-            for (let seg of segments) {
-                if (seg.length < 4) continue;
-                for (let ref of getAllUnits({ buffer: seg, bufferIndex: 0, block: 0 })) {
-                    allSearchUnits.add(ref.unit);
-                }
-            }
-            if (allSearchUnits.size === 0) {
-                // Search pattern too short to use index, return empty results
-                return;
-            }
+        let allSearchUnits = getSearchUnits(params.findBuffer, !!params.disableWildCards);
+        if (allSearchUnits.length === 0) {
+            return;
         }
 
         let type = index[0];
@@ -447,6 +443,7 @@ export class BufferIndex {
         } else if (type === BULK_TYPE) {
             await BufferUnitIndex.find({
                 params,
+                allSearchUnits,
                 index,
                 reader: dataReader,
                 keepIterating: config.keepIterating,

package/src/diagnostics/logs/IndexedLogs/BufferIndexHelpers.ts

@@ -16,6 +16,7 @@ export type SearchParams = {
     findBuffer: Buffer;
     pathOverrides?: TimeFilePathWithSize[];
     only?: "local" | "public";
+    forceReadPublic?: boolean;
 };
 
 export type Unit = number;
@@ -134,10 +135,72 @@ export type IndexedLogResults = {
 };
 export function createEmptyIndexedLogResults(): IndexedLogResults {
     return {
-        matchCount: 0, reads: [], totalLocalFiles: 0, totalBackblazeFiles: 0, localFilesSearched: 0, backblazeFilesSearched: 0, totalBlockCount: 0, blockCheckedCount: 0, remoteBlockCount: 0, localBlockCount: 0, remoteBlockCheckedCount: 0, localBlockCheckedCount: 0, blocksCheckedCompressedSize: 0, blocksCheckedDecompressedSize: 0, blockErrors: [], fileErrors: [], remoteIndexesSearched: 0, remoteIndexSize: 0, localIndexesSearched: 0, localIndexSize: 0, timeToFirstMatch: 0, fileFindTime: 0, indexSearchTime: 0, blockSearchTime: 0, totalSearchTime: 0, cancel: undefined, limitGroup: undefined,
+        matchCount: 0, reads: [], totalLocalFiles: 0, totalBackblazeFiles: 0, localFilesSearched: 0, backblazeFilesSearched: 0, totalBlockCount: 0, blockCheckedCount: 0, remoteBlockCount: 0, localBlockCount: 0, remoteBlockCheckedCount: 0, localBlockCheckedCount: 0, blocksCheckedCompressedSize: 0, blocksCheckedDecompressedSize: 0, blockErrors: [], fileErrors: [], remoteIndexesSearched: 0, remoteIndexSize: 0, localIndexesSearched: 0, localIndexSize: 0, timeToFirstMatch: -1, fileFindTime: 0, indexSearchTime: 0, blockSearchTime: 0, totalSearchTime: 0, cancel: undefined, limitGroup: undefined,
     };
 }
 
+export function mergeIndexedLogResults(existing: IndexedLogResults, incoming: IndexedLogResults): IndexedLogResults {
+    let readsByKey = new Map<string, typeof existing.reads[0]>();
+
+    for (let read of existing.reads) {
+        let key = `${read.cached}-${read.remote}`;
+        let existingRead = readsByKey.get(key);
+        if (existingRead) {
+            existingRead.count += read.count;
+            existingRead.size += read.size;
+            existingRead.totalSize = Math.max(existingRead.totalSize, read.totalSize);
+            existingRead.totalCount = Math.max(existingRead.totalCount, read.totalCount);
+        } else {
+            readsByKey.set(key, { ...read });
+        }
+    }
+
+    for (let read of incoming.reads) {
+        let key = `${read.cached}-${read.remote}`;
+        let existingRead = readsByKey.get(key);
+        if (existingRead) {
+            existingRead.count += read.count;
+            existingRead.size += read.size;
+            existingRead.totalSize = Math.max(existingRead.totalSize, read.totalSize);
+            existingRead.totalCount = Math.max(existingRead.totalCount, read.totalCount);
+        } else {
+            readsByKey.set(key, { ...read });
+        }
+    }
+
+    return {
+        matchCount: existing.matchCount + incoming.matchCount,
+        totalLocalFiles: existing.totalLocalFiles + incoming.totalLocalFiles,
+        totalBackblazeFiles: existing.totalBackblazeFiles + incoming.totalBackblazeFiles,
+        reads: Array.from(readsByKey.values()),
+        localFilesSearched: existing.localFilesSearched + incoming.localFilesSearched,
+        backblazeFilesSearched: existing.backblazeFilesSearched + incoming.backblazeFilesSearched,
+        totalBlockCount: existing.totalBlockCount + incoming.totalBlockCount,
+        blockCheckedCount: existing.blockCheckedCount + incoming.blockCheckedCount,
+        blocksCheckedCompressedSize: existing.blocksCheckedCompressedSize + incoming.blocksCheckedCompressedSize,
+        blocksCheckedDecompressedSize: existing.blocksCheckedDecompressedSize + incoming.blocksCheckedDecompressedSize,
+        blockErrors: [...existing.blockErrors, ...incoming.blockErrors],
+        fileErrors: [...existing.fileErrors, ...incoming.fileErrors],
+        remoteIndexesSearched: existing.remoteIndexesSearched + incoming.remoteIndexesSearched,
+        remoteIndexSize: existing.remoteIndexSize + incoming.remoteIndexSize,
+        localIndexesSearched: existing.localIndexesSearched + incoming.localIndexesSearched,
+        localIndexSize: existing.localIndexSize + incoming.localIndexSize,
+        timeToFirstMatch: Math.min(
+            existing.timeToFirstMatch === -1 ? Infinity : existing.timeToFirstMatch,
+            incoming.timeToFirstMatch === -1 ? Infinity : incoming.timeToFirstMatch
+        ),
+        fileFindTime: existing.fileFindTime + incoming.fileFindTime,
+        indexSearchTime: existing.indexSearchTime + incoming.indexSearchTime,
+        blockSearchTime: existing.blockSearchTime + incoming.blockSearchTime,
+        totalSearchTime: Math.max(existing.totalSearchTime, incoming.totalSearchTime),
+        remoteBlockCount: existing.remoteBlockCount + incoming.remoteBlockCount,
+        localBlockCount: existing.localBlockCount + incoming.localBlockCount,
+        remoteBlockCheckedCount: existing.remoteBlockCheckedCount + incoming.remoteBlockCheckedCount,
+        localBlockCheckedCount: existing.localBlockCheckedCount + incoming.localBlockCheckedCount,
+    };
+}
+
+
 export function addReadToResults(results: IndexedLogResults, read: {
     cached: boolean;
     remote: boolean;
@@ -161,62 +224,3 @@ export function addReadToResults(results: IndexedLogResults, read: {
     return existingRead;
 }
 
-
-export const WILD_CARD_BYTE = 42;
-export function splitOnWildcard(buffer: Buffer): Buffer[] {
-    let segments: Buffer[] = [];
-    let start = 0;
-    for (let i = 0; i <= buffer.length; i++) {
-        if (i === buffer.length || buffer[i] === WILD_CARD_BYTE) {
-            segments.push(buffer.slice(start, i));
-            start = i + 1;
-        }
-    }
-    return segments;
-}
-
-// Each WILD_CARD_BYTE in pattern acts as a multi-byte wildcard: the segments on either
-// side must appear in order somewhere within buffer.
-// Returns a function that matches buffers against the pre-processed pattern.
-export function createMatchesPattern(pattern: Buffer, disableWildCards: boolean): (buffer: Buffer) => boolean {
-    let segments = disableWildCards && [pattern] || splitOnWildcard(pattern).filter(s => s.length > 0);
-
-    return measureWrap(function matchesPattern(buffer: Buffer): boolean {
-        // Fast path: check if all segments exist anywhere in the buffer using indexOf
-        for (let seg of segments) {
-            if (buffer.indexOf(seg) === -1) {
-                return false;
-            }
-        }
-
-        // Tries to match all segments in order starting from bufferPos, returning the end
-        // position after the last match, or -1 if not all segments could be found.
-        function matchSegmentsFrom(bufferPos: number): number {
-            for (let seg of segments) {
-                function segMatchesAt(pos: number): boolean {
-                    for (let i = 0; i < seg.length; i++) {
-                        if (buffer[pos + i] !== seg[i]) return false;
-                    }
-                    return true;
-                }
-                let found = false;
-                for (let searchPos = bufferPos; searchPos <= buffer.length - seg.length; searchPos++) {
-                    if (segMatchesAt(searchPos)) {
-                        // NOTE: I think this is safe because every segment has a wildcard after it. So we can never have a case where we didn't skip far enough because the wild card will just skip farther. And we won't have a partial match as we're matching the whole chunk. So we won't match a prefix and then get stuck. I think... it does seem weird though...
-                        bufferPos = searchPos + seg.length;
-                        found = true;
-                        break;
-                    }
-                }
-                if (!found) return -1;
-            }
-            return bufferPos;
-        }
-
-        for (let startPos = 0; startPos <= buffer.length; startPos++) {
-            if (matchSegmentsFrom(startPos) >= 0) return true;
-        }
-        return false;
-    }, "BufferIndex|matchesPattern");
-}
-

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -3,7 +3,7 @@
 import { LZ4 } from "../../../storage/LZ4";
 import { measureBlock, measureFnc } from "socket-function/src/profiling/measure";
 import { Zip } from "../../../zip";
-import { BufferReader, Reader, WILD_CARD_BYTE, createMatchesPattern, SearchParams, IndexedLogResults } from "./BufferIndexHelpers";
+import { BufferReader, Reader, SearchParams, IndexedLogResults, Unit } from "./BufferIndexHelpers";
 import { formatNumber, formatPercent } from "socket-function/src/formatting/format";
 import { lazy } from "socket-function/src/caching";
 import { list, sort } from "socket-function/src/misc";
@@ -11,6 +11,7 @@ import { testDisableCache } from "../../../-a-archives/archivesMemoryCache";
 import { devDebugbreak } from "../../../config";
 import { BufferUnitIndexParallelSearchCount, DEFAULT_BLOCK_SIZE, DEFAULT_TARGET_UNITS_PER_BUCKET } from "./BufferIndexLogsOptimizationConstants";
 import { runInParallel } from "socket-function/src/batching";
+import { createMatchesPattern } from "./bufferMatcher";
 
 const USE_COMPRESSION = true;
 
@@ -454,51 +455,17 @@ export class BufferUnitIndex {
     @measureFnc
     public static async find(config: {
         params: SearchParams;
+        allSearchUnits: Unit[][];
         keepIterating: () => boolean;
         onResult: (match: Buffer) => void;
         index: Buffer;
         reader: Reader;
         results: IndexedLogResults;
     }): Promise<void> {
-        const { params, index, reader, keepIterating, results } = config;
-
-        // Split on wildcards if present
-        function splitOnWildcard(buffer: Buffer): Buffer[] {
-            const segments: Buffer[] = [];
-            let start = 0;
-            for (let i = 0; i <= buffer.length; i++) {
-                if (i === buffer.length || buffer[i] === WILD_CARD_BYTE) {
-                    segments.push(buffer.slice(start, i));
-                    start = i + 1;
-                }
-            }
-            return segments;
-        }
-
-        const segments = params.disableWildCards && [params.findBuffer] || splitOnWildcard(params.findBuffer).filter(s => s.length > 0);
-
-        // Find blocks for each segment >= 4 bytes
-        const candidateBlocks = measureBlock(() => {
-            const candidateBlocksPerSegment: number[][] = [];
-            for (const segment of segments) {
-                if (segment.length < 4) continue;
-                const blockIndices = this.findBlocks({ findBuffer: segment, index });
-                candidateBlocksPerSegment.push(blockIndices);
-            }
-
-            if (candidateBlocksPerSegment.length === 0) {
-                throw new Error("Search pattern too short: all segments are fewer than 4 bytes, cannot use index");
-            }
-
-            let intersectionSet = new Set<number>(candidateBlocksPerSegment[0]);
-            for (let i = 1; i < candidateBlocksPerSegment.length; i++) {
-                const currentSet = new Set(candidateBlocksPerSegment[i]);
-                intersectionSet = new Set([...intersectionSet].filter(x => currentSet.has(x)));
-            }
+        const { params, index, reader, keepIterating, results, allSearchUnits } = config;
 
-
-            return intersectionSet;
-        }, `findCandidateBlocks`);
+        let candidateBlocksList = allSearchUnits.map(units => this.findBlocks({ units, index })).flat();
+        let candidateBlocksSet = new Set(candidateBlocksList);
 
 
         const matchesPattern = createMatchesPattern(params.findBuffer, !!params.disableWildCards);
@@ -516,7 +483,7 @@ export class BufferUnitIndex {
             let matchCounts = list(blockCount).fill(0);
 
             const searchBlock = async (blockIndex: number) => {
-                if (!candidateBlocks.has(blockIndex)) return;
+                if (!candidateBlocksSet.has(blockIndex)) return;
                 // This is kind of a weird thing. Basically, because we search in parallel, we might search out of order. So we can only look at the counts before or at us, as if we match a whole bunch after us, but we should still keep going as our matches are going to take precedence.
                 let stopIterating = () => {
                     let countBefore = 0;
@@ -568,7 +535,7 @@ export class BufferUnitIndex {
                 searchBlock
             );
             // Search first first, as moveLogsToPublic should have made it so this is the newest.
-            let searchOrder = Array.from(candidateBlocks);
+            let searchOrder = Array.from(candidateBlocksSet);
             sort(searchOrder, x => x);
             await Promise.all(searchOrder.map(runSearchBlock));
 
@@ -576,23 +543,12 @@ export class BufferUnitIndex {
         results.blockSearchTime += Date.now() - blockSearchTimeStart;
     }
 
+    @measureFnc
     private static findBlocks(config: {
-        findBuffer: Buffer;
+        units: number[];
         index: Buffer;
     }): number[] {
-        const { findBuffer, index } = config;
-
-        // Extract all unique units from findBuffer
-        const units = measureBlock(() => {
-            const units: number[] = [];
-            for (let i = 0; i <= findBuffer.length - 4; i++) {
-                const unit = findBuffer.readUint32LE(i);
-                if (unit !== 0 && !units.includes(unit)) {
-                    units.push(unit);
-                }
-            }
-            return units;
-        }, `extractUnits`);
+        const { units, index } = config;
 
         if (units.length === 0) {
             return [];
@@ -600,25 +556,12 @@ export class BufferUnitIndex {
 
         // Get blocks for each unit and intersect
         return measureBlock(() => {
-            const candidateBlocksPerUnit: number[][] = [];
-            for (const unit of units) {
-                const blockIndices = this.getBlocksForUnit(index, unit);
-                candidateBlocksPerUnit.push(blockIndices);
+            let candidateBlocks = this.getBlocksForUnit(index, units[0]);
+            for (let i = 1; i < units.length; i++) {
+                let nextBlocks = new Set(this.getBlocksForUnit(index, units[i]));
+                candidateBlocks = candidateBlocks.filter(b => nextBlocks.has(b));
             }
-
-            // Intersect all block sets
-            let intersectionSet = new Set<number>(candidateBlocksPerUnit[0]);
-            for (let i = 1; i < candidateBlocksPerUnit.length; i++) {
-                const currentSet = new Set(candidateBlocksPerUnit[i]);
-                intersectionSet = new Set([...intersectionSet].filter(x => currentSet.has(x)));
-            }
-
-            let allCounts = candidateBlocksPerUnit.map(b => b.length);
-            sort(allCounts, x => x);
-
-            //console.log(`Candidate blocks ${intersectionSet.size}, minimum: ${Math.min(...candidateBlocksPerUnit.map(b => b.length))}, best 4 counts: ${allCounts.slice(0, 4).join(", ")}`);
-
-            return Array.from(intersectionSet);
+            return candidateBlocks;
         }, `intersectBlocks`);
     }