querysub 0.373.0 → 0.375.0

package/src/diagnostics/logs/FastArchiveAppendable.ts

@@ -1,843 +0,0 @@
-module.hotreload = true;
-module.noserverhotreload = false;
-import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
-import { getMachineId, getOwnMachineId } from "../../-a-auth/certs";
-import { isDefined, parseFileNameKVP, parsePath, partialCopyObject, streamToIteratable, sum, toFileNameKVP } from "../../misc";
-import { registerShutdownHandler } from "../periodic";
-import { batchFunction, delay, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { PromiseObj, isNode, keyByArray, list, nextId, sort, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
-import os from "os";
-import { getOwnThreadId } from "../../-f-node-discovery/NodeDiscovery";
-import fs from "fs";
-import { MaybePromise, canHaveChildren } from "socket-function/src/types";
-import { formatNumber, formatTime } from "socket-function/src/formatting/format";
-import { cache, lazy } from "socket-function/src/caching";
-import { Archives, getArchives, nestArchives } from "../../-a-archives/archives";
-import { Zip } from "../../zip";
-import { SocketFunction } from "socket-function/SocketFunction";
-import { assertIsManagementUser } from "../managementPages";
-import { getControllerNodeIdList } from "../../-g-core-values/NodeCapabilities";
-import { errorToUndefined, ignoreErrors, timeoutToUndefinedSilent } from "../../errors";
-import { getCallObj } from "socket-function/src/nodeProxy";
-import { getSyncedController } from "../../library-components/SyncedController";
-import { getBrowserUrlNode, getOwnNodeId } from "../../-f-node-discovery/NodeDiscovery";
-import { secureRandom, shuffle } from "../../misc/random";
-import { getPathIndex, getPathStr2 } from "../../path";
-import { onNextPaint } from "../../functional/onNextPaint";
-import { getArchivesBackblazePrivateImmutable, getArchivesBackblazePublicImmutable } from "../../-a-archives/archivesBackBlaze";
-import { httpsRequest } from "socket-function/src/https";
-import { getDomain, isPublic } from "../../config";
-import { getIPDomain } from "../../-e-certs/EdgeCertController";
-import { getArchivesPrivateFileSystem } from "../../-a-archives/archivesPrivateFileSystem";
-import { createArchivesLimitedCache } from "../../-a-archives/archivesLimitedCache";
-import { sha256 } from "js-sha256";
-import { assertIsNetworkTrusted } from "../../-d-trust/NetworkTrust2";
-import { blue, magenta } from "socket-function/src/formatting/logColors";
-import { FileMetadata, FastArchiveAppendableControllerBase, FastArchiveAppendableController, getFileMetadataHash } from "./FastArchiveController";
-import { fsExistsAsync } from "../../fs";
-import { ScanFnc } from "./FastArchiveViewer";
-import { getArchivesLocal } from "../../-a-archives/archivesDisk";
-
-// NOTE: In a single command line micro-test it looks like we can write about 40K writes of 500 per once, when using 10X parallel, on a fairly potato server. We should probably batch though, and only do 1X parallel.
-/*
-Append Benchmarks
-    10 processes
-        Windows = 100K/S
-        Linux Digital Ocean = 1M/S
-        Linux PI SD Card = 300K/S
-        Linux PI USB = 300K/S
-    1 process
-        Windows = 40K/S
-        Linux Digital Ocean = 685K/S
-        Linux PI = 200K/S
-
-rm test.txt
-for i in {0..9}; do node -e 'const fs=require("fs");const id='$i';let i=0;const start=Date.now();while(Date.now()-start<5000){fs.appendFileSync("test.txt", `${id},${i++}\n`)}' & done; wait
-node -e 'const fs=require("fs");const seqs=new Map();fs.readFileSync("test.txt","utf8").trim().split("\n").forEach((l,i)=>{const[id,seq]=l.split(",").map(Number);if(!seqs.has(id))seqs.set(id,{last:-1,errs:0});const s=seqs.get(id);if(seq!==s.last+1){console.error(`Error for id ${id} at line ${i}: ${s.last}->${seq}`);s.errs++}s.last=seq});seqs.forEach((v,id)=>console.log(`ID ${id}: final seq ${v.last}, ${v.errs} gaps`))'
-*/
-
-const UNCOMPRESSED_LOG_FILE_WARN_THRESHOLD = 1024 * 1024 * 512;
-const UNCOMPRESSED_LOG_FILE_STOP_THRESHOLD = 1024 * 1024 * 1024 * 2;
-
-// Add a large wait, due to day light saving time, or whatever
-const UPLOAD_THRESHOLD = timeInHour * 1.5;
-const DEAD_TIMEOUT = timeInHour * 6;
-const DELETE_TIMEOUT = timeInHour * 12;
-
-const MAX_WORK_PER_PAINT = 40;
-
-const ON_DATA_BATCH_COUNT = 1024 * 10;
-//const ON_DATA_BATCH_COUNT = 1;
-
-// NOTE: This used to be small, for atomic writes, but it causes lag, and... why do we need atomic writes? I believe we use a file per thread, so... we will be the only writer.
-const WRITE_CHUNK_SIZE = 1024 * 1024 * 16;
-
-
-
-const MAX_LOCAL_CACHED_FILES = 1000 * 10;
-const MAX_LOCAL_CACHED_SIZE = 1024 * 1024 * 1024 * 10;
-
-const getFileCache = cache((rootPath: string) => {
-    let data = getArchivesPrivateFileSystem(rootPath);
-    return createArchivesLimitedCache(data, {
-        maxFiles: MAX_LOCAL_CACHED_FILES,
-        maxSize: MAX_LOCAL_CACHED_SIZE,
-    });
-});
-
-
-// IMPORTANT! Use these like this, with one object per type of log
-// const errorAppendable = new FastArchiveAppendable<LogObject>("logs/error/");
-// const warnAppendable = new FastArchiveAppendable<LogObject>("logs/warn/");
-// const infoAppendable = new FastArchiveAppendable<LogObject>("logs/info/");
-// const logsAppendable = new FastArchiveAppendable<LogObject>("logs/logs/");
-
-
-// NOTE: We don't unescape. Because it massively slows down encoding, and... our delimitter is unlikely to randomly appear. IF it does, then it is likely not structural, and so the data will only look a bit weird (as in, it is likely not a length that cbor uses to decode or anything like that.
-export const objectDelimitterBuffer = Buffer.from([0x253, 0xe5, 0x05, 0x199, 0x5c, 0xbb, 0x63, 0x251]);
-
-export type DatumStats = {
-    matchedSize: number;
-    notMatchedSize: number;
-    errors: number;
-    matchedCount: number;
-    notMatchedCount: number;
-};
-
-
-export function getFileTimeStamp(path: string): {
-    startTime: number;
-    endTime: number;
-} {
-    let file = path.replaceAll("\\", "/").split("/").at(-1)!;
-    // Remove .log extension and parse as ISO date
-    let dateStr = file.replace(/\.log$/, "");
-    // Add missing parts to make it a valid ISO string: "2025-09-06T07" -> "2025-09-06T07:00:00.000Z"
-    if (dateStr.length === 13) { // YYYY-MM-DDTHH format
-        dateStr += ":00:00.000Z";
-    }
-    let startTime = new Date(dateStr).getTime();
-    if (!startTime) {
-        let pathParts = path.split("/");
-        let year = parseInt(pathParts[0], 10);
-        let month = parseInt(pathParts[1], 10) - 1;
-        let day = parseInt(pathParts[2], 10);
-        let hour = parseInt(pathParts[3], 10);
-        let hourStart = Date.UTC(year, month, day, hour);
-        startTime = hourStart;
-    }
-    return {
-        startTime,
-        endTime: startTime + timeInHour,
-    };
-}
-
-
-// NOTE: Global, to prevent hot reloading, or redundant FastArchiveAppendable, from breaking things.
-const appendableSerialLock = (globalThis as any).appendableSerialLock ?? runInSerial(async (fnc: () => Promise<void>) => {
-    await fnc();
-});
-(globalThis as any).appendableSerialLock = appendableSerialLock;
-
-
-export class FastArchiveAppendable<Datum> {
-    private lastSizeWarningTime = 0;
-
-    public constructor(public rootPath: string) {
-        if (!this.rootPath.endsWith("/")) {
-            this.rootPath += "/";
-        }
-        if (isNode()) {
-            registerShutdownHandler(async () => {
-                await this.flushNow();
-            });
-            runInfinitePoll(timeInMinute, async () => {
-                await this.flushNow();
-            });
-            // Random, to try to prevent the dead file detection code from getting in sync. It's fine if it gets in sync, it's just inefficient as we'll have multiple services uploading the same file to the same location at the same time. 
-            void runInfinitePoll(timeInMinute * 20 + Math.random() * timeInMinute * 5, async () => {
-                await this.moveLogsToBackblaze();
-            });
-        }
-    }
-
-    public getArchives = cache((forceGetPublic: boolean) => {
-        let archives: Archives;
-        if (!isPublic() && !forceGetPublic) {
-            archives = getArchivesLocal(getDomain());
-        } else {
-            archives = getArchivesBackblazePrivateImmutable(getDomain());
-        }
-        return nestArchives("fast-logs/" + this.rootPath, archives);
-    });
-
-    public baseGetLocalPathRoot = () => {
-        return (
-            os.homedir()
-            + "/fast-log-cache/"
-            + getDomain() + "/"
-            + this.rootPath
-        );
-    };
-    public getLocalPathRoot = lazy(() => {
-        let path = this.baseGetLocalPathRoot();
-        if (!fs.existsSync(path)) {
-            fs.mkdirSync(path, { recursive: true });
-        }
-        return path;
-    });
-
-    // NOTE: Batching is both faster, and allows a lot of the code to log (which will cause an append) without causing an infinite loop.
-    private pendingWriteQueue: unknown[] = [];
-    @measureFnc
-    private escapeDelimitter(data: Buffer) {
-        if (!data.includes(objectDelimitterBuffer)) return data;
-
-        let startIndex = 0;
-        while (true) {
-            let index = data.indexOf(objectDelimitterBuffer, startIndex);
-            if (index === -1) break;
-            data[index]++;
-            startIndex = index;
-        }
-    }
-
-
-    private insideAppend = false;
-    /** NOTE: If the input data might contain user data, or mutated, use partialCopyObject first, to make a copy and prevent it from being excessively large. */
-    @measureFnc
-    public append(data: Datum) {
-        // Hmm... so... 
-        if (!isNode()) return;
-        // Just from logging, so... ignore it
-        if (this.insideAppend) return;
-        this.insideAppend = true;
-        try {
-            this.pendingWriteQueue.push(data);
-        } finally {
-            this.insideAppend = false;
-        }
-    }
-
-    // NOTE: No timing on this as it's just waiting on the disk. It's not actually slowing anything down. 
-    public async flushNow(now = Date.now()) {
-
-        await appendableSerialLock(async () => {
-            if (this.pendingWriteQueue.length === 0) return;
-
-            // 2025-09-06T07
-            let hourFile = new Date(now).toISOString().slice(0, 13) + ".log";
-            let localCacheFolder = this.getLocalPathRoot() + getOwnThreadId() + "/";
-            let localCachePath = localCacheFolder + hourFile;
-            await fs.promises.mkdir(localCacheFolder, { recursive: true });
-            // Always heartbeat
-            await fs.promises.writeFile(localCacheFolder + "heartbeat", Buffer.from(now + ""));
-
-            try {
-                let beforeSize = await fs.promises.stat(localCachePath);
-                if (beforeSize.size > UNCOMPRESSED_LOG_FILE_STOP_THRESHOLD) {
-                    console.error(`FastArchiveAppendable: ${localCachePath} too large, refusing to add more data to it, current size ${formatNumber(beforeSize.size)}B > ${formatNumber(UNCOMPRESSED_LOG_FILE_STOP_THRESHOLD)}B`);
-                    return;
-                }
-            } catch {
-                // File not existing is fine
-            }
-
-            let chunks: Buffer[][] = [];
-            measureBlock(() => {
-                // NOTE: We can't use anything but JSON, as we need it to be scannable before decoding it (otherwise scanning takes 100X longer)
-                let writeData = this.pendingWriteQueue.map(v => {
-                    let buffer = Buffer.from(JSON.stringify(v));
-                    this.escapeDelimitter(buffer);
-                    return buffer;
-                });
-                this.pendingWriteQueue = [];
-
-                // Group lines into WRITE_ATOMIC_LIMIT byte chunks
-
-                let currentChunk: Buffer[] = [];
-                let currentSize = 0;
-                for (let line of writeData) {
-                    if (currentSize + line.length + objectDelimitterBuffer.length > WRITE_CHUNK_SIZE && currentChunk.length > 0) {
-                        chunks.push(currentChunk);
-                        currentChunk = [];
-                        currentSize = 0;
-                    }
-                    currentChunk.push(line);
-                    currentSize += line.length;
-                    currentChunk.push(objectDelimitterBuffer);
-                    currentSize += objectDelimitterBuffer.length;
-                }
-                if (currentChunk.length > 0) {
-                    chunks.push(currentChunk);
-                }
-            }, `FastArchiveAppendable|serialize log data`);
-
-            for (let chunk of chunks) {
-                await fs.promises.appendFile(localCachePath, Buffer.concat(chunk));
-            }
-
-            let finalSize = await fs.promises.stat(localCachePath);
-            if (finalSize.size > UNCOMPRESSED_LOG_FILE_WARN_THRESHOLD) {
-                const now = Date.now();
-                const timeSinceLastWarning = now - this.lastSizeWarningTime;
-
-                if (timeSinceLastWarning >= timeInMinute * 15) {
-                    console.error(`FastArchiveAppendable: ${localCachePath} is getting very big. This might cause logging to be disabled, to maintain readable of the logs (which will be required to debug why we are writing so much data! Current size ${formatNumber(finalSize.size)}B > ${formatNumber(UNCOMPRESSED_LOG_FILE_WARN_THRESHOLD)}B)`);
-                    this.lastSizeWarningTime = now;
-                }
-            }
-        });
-    }
-
-
-    public static getBackblazePath(config: { fileName: string; threadId: string }): string {
-        // 2025-09-06T07
-        let mainName = config.fileName.replace(/\.log$/, "");
-        let [year, month, day, hour] = mainName.split(/[-T:]/);
-        return `${year}/${month}/${day}/${hour}/${config.threadId}.log`;
-    }
-
-    public async moveLogsToBackblaze() {
-        await appendableSerialLock(async () => {
-            let rootCacheFolder = this.baseGetLocalPathRoot();
-            if (!await fsExistsAsync(rootCacheFolder)) return;
-            console.log(magenta(`Moving old logs to Backblaze from ${rootCacheFolder}`));
-            // Ugh... what is this even? Is it... hot reloading?
-            if (rootCacheFolder.includes("undefined/undefined/undefined/")) {
-                require("debugbreak")(2);
-                debugger;
-            }
-
-            let archives = this.getArchives(false);
-            async function moveLogsForFolder(threadId: string) {
-                let threadDir = rootCacheFolder + threadId + "/";
-                if (!await fsExistsAsync(threadDir)) return;
-                let files = await fs.promises.readdir(threadDir);
-                // Shuffle, so if we do run multiple at the same time, we are less likely to use the same files at the same time.
-                files = shuffle(files, Math.random());
-                for (let file of files) {
-                    if (file === "heartbeat") continue;
-                    let fullPath = threadDir + file;
-                    try {
-                        if (!await fsExistsAsync(fullPath)) continue;
-                        // We could use modified time here? Although, this is nice if we move files around, and then manually have them moved, although even then... this could cause problem be tripping while we are copying the file, so... maybe this is just wrong?
-                        let timeStamp = getFileTimeStamp(fullPath);
-                        if (timeStamp.endTime > Date.now() - UPLOAD_THRESHOLD) continue;
-
-                        // NOTE: Because we use the same target path, if multiple services do this at the same time it's fine. Not great, but... fine.
-                        let backblazePath = FastArchiveAppendable.getBackblazePath({ fileName: file, threadId });
-                        console.log(magenta(`Moving ${fullPath} to Backblaze as ${backblazePath}`));
-
-                        let data = await measureBlock(async () => fs.promises.readFile(fullPath), "FastArchiveAppendable|readBeforeUploading");
-                        let compressed = await measureBlock(async () => Zip.gzip(data), "FastArchiveAppendable|compress");
-                        console.log(`Uploading ${formatNumber(data.length)}B (compressed to ${formatNumber(compressed.length)}B) logs to ${backblazePath} from ${fullPath}`);
-                        await archives.set(backblazePath, compressed);
-                        // Ignore unlink errors to reduce excess logging. This races on startup, so it is likely we'll hit this a fair amount (especially because archives.set is so slow)
-                        try {
-                            await fs.promises.unlink(fullPath);
-                        } catch { }
-                    } catch (e: any) {
-                        // Just skip it, if the first file in the directory is broken we don't want to never move any files
-                        console.error(`Error moving log file ${fullPath}: ${e.stack}`);
-                    }
-                }
-            }
-
-            await moveLogsForFolder(getOwnThreadId());
-            let allFolders = await fs.promises.readdir(rootCacheFolder);
-            for (let threadId of allFolders) {
-                if (threadId === getOwnThreadId()) continue;
-
-                let heartbeat = 0;
-                try {
-                    let heartbeatStr = await fs.promises.readFile(rootCacheFolder + threadId + "/heartbeat", "utf8");
-                    heartbeat = Number(heartbeatStr);
-                } catch { }
-                if (heartbeat < Date.now() - DEAD_TIMEOUT) {
-                    await moveLogsForFolder(threadId);
-                }
-                if (heartbeat < Date.now() - DELETE_TIMEOUT) {
-                    await fs.promises.rmdir(rootCacheFolder + threadId, { recursive: true });
-                }
-            }
-        });
-    }
-
-    private lastSynchronize: {
-        stopSynchronize: () => void;
-        parametersHash: string;
-        parametersResult: PromiseObj<{
-            files: FileMetadata[];
-        }>;
-    } | undefined;
-    public cancelAllSynchronizes() {
-        // We miss some cases here, but... it's probably fine. This should usually cancel.
-        this.lastSynchronize?.stopSynchronize();
-        this.lastSynchronize = undefined;
-    }
-    /** If called cancels the previous outstanding synchronize. */
-    public async synchronizeData(config: {
-        range: {
-            startTime: number;
-            endTime: number;
-        };
-        cacheBust: number;
-        forceGetPublic?: boolean;
-        scanFnc?: ScanFnc;
-        getWantData?: (file: FileMetadata) => Promise<((posStart: number, posEnd: number, data: Buffer) => boolean) | undefined>;
-        onData: (datum: Datum[], file: FileMetadata) => void;
-        // Called after onData
-        onStats?: (stats: DatumStats, file: FileMetadata) => void;
-        onError?: (error: Error, file: FileMetadata) => void;
-        onFinish?: () => void;
-
-        onProgress?: (progress: {
-            section: string;
-            value: number;
-            max: number;
-        }) => void;
-    }): Promise<{
-        metadata: {
-            files: FileMetadata[];
-        };
-        stopSynchronize: () => void;
-    } | "cancelled"> {
-        let { onData, onStats, onError } = config;
-        // Create unique client sync ID upfront
-        let syncId = nextId();
-
-        console.log(`Synchronizing ${this.rootPath} with syncId ${syncId}`, config);
-
-        let isPublicValue = isPublic() || config.forceGetPublic || false;
-
-        // Register progress callback immediately so we can receive progress during setup
-        // - It also helps with cancellation
-        FastArchiveAppendableControllerBase.progressCallbacks.set(syncId, config.onProgress ?? (() => { }));
-
-        let stopped = false;
-        let stoppedPromise = new PromiseObj<void>();
-        const stopSynchronize = () => {
-            stoppedPromise.resolve();
-            // Wait a bit for trailing progress, as progress is batched and delayed
-            setTimeout(() => {
-                stopped = true;
-                FastArchiveAppendableControllerBase.progressCallbacks.delete(syncId);
-            }, 5000);
-        };
-
-        let parametersHash = sha256(JSON.stringify({
-            range: config.range,
-            isPublicValue,
-            cacheBust: config.cacheBust,
-            version: 1,
-        })) + ".parameters";
-
-        let synchronizeObj = {
-            stopSynchronize,
-            parametersHash,
-            parametersResult: new PromiseObj<{
-                files: FileMetadata[];
-            }>(),
-        };
-        let last = this.lastSynchronize;
-        // Wait for the last one to finish getting the parameters, as we'll use the same ones.
-        if (last?.parametersHash === parametersHash) {
-            await last.parametersResult.promise;
-            // Another call happened before we finished
-            if (this.lastSynchronize !== last) return "cancelled";
-        }
-        this.lastSynchronize?.stopSynchronize();
-
-        this.lastSynchronize = synchronizeObj;
-
-        let baseOnProgress = config.onProgress;
-        let timeOfLastPaint = Date.now();
-        let throttlePerPaint = runInSerial(async () => {
-            let now = Date.now();
-            let workDone = now - timeOfLastPaint;
-            if (workDone < MAX_WORK_PER_PAINT) return;
-            await onNextPaint();
-            timeOfLastPaint = now;
-        });
-
-        let createProgress = async (section: string, max: number) => {
-            section = `${this.rootPath}|${section}`;
-            let cancelled: Error | undefined;
-            let lastValue = 0;
-            let baseBatch = batchFunction({ delay: 150, },
-                async (config: { value: number, overrideMax?: number }[]) => {
-                    if (cancelled) return;
-                    if (stopped) {
-                        cancelled = new Error(`Synchronization stopped`);
-                        return;
-                    }
-
-                    let value = config.at(-1)!.value;
-                    lastValue = value;
-                    let usedMax = config.map(c => c.overrideMax).filter(isDefined).at(-1) ?? max;
-                    value = Math.min(value, usedMax);
-                    baseOnProgress?.({ section, value, max: usedMax });
-                }
-            );
-            let deltaBatch = batchFunction({ delay: 150, },
-                async (config: { value: number, max: number }[]) => {
-                    if (cancelled) return;
-                    if (stopped) {
-                        cancelled = new Error(`Synchronization stopped`);
-                        return;
-                    }
-                    for (let delta of config) {
-                        lastValue += delta.value;
-                        max += delta.max;
-                    }
-                    baseOnProgress?.({ section, value: lastValue, max });
-                }
-            );
-            let firstCall = true;
-            let onProgress = async (value: number, overrideMax?: number, delta?: boolean) => {
-                // Call it immediately, so the output order isn't broken by the batching delay varying slightly
-                if (firstCall) {
-                    firstCall = false;
-                    baseOnProgress?.({ section, value: 0, max });
-                }
-                if (cancelled) throw cancelled;
-                if (delta) {
-                    void deltaBatch({ value, max: overrideMax ?? 0 });
-                } else {
-                    void baseBatch({ value, overrideMax });
-                }
-                await throttlePerPaint();
-            };
-            // Ordering is better if we wait until the first progress to send any progress
-            //await onProgress(0);
-            return onProgress;
-        };
-
-        try {
-            const localCache = getFileCache(this.rootPath);
-
-            let syncResult: { files: FileMetadata[]; createTime?: number; } | undefined;
-            let cachedSyncResultBuffer = await localCache.get(parametersHash);
-            let downloadSyncId: string = "";
-            let controller = FastArchiveAppendableController.nodes[getBrowserUrlNode()];
-            if (cachedSyncResultBuffer?.length) {
-                try {
-                    syncResult = JSON.parse(cachedSyncResultBuffer.toString());
-                    downloadSyncId = await controller.createSyncSession();
-                } catch (e: any) {
-                    console.error(`Failed to parsed cached sync result, synchronizing from scratch instead\n${e.stack}`);
-                }
-            }
-            if (!syncResult) {
-                let findingFiles = await createProgress("Finding files", 0);
-                syncResult = await controller.startSynchronize({
-                    syncId,
-                    range: config.range,
-                    rootPath: this.rootPath,
-                    forceGetPublic: config.forceGetPublic,
-                });
-                syncResult.createTime = Date.now();
-                await findingFiles(syncResult.files.length, syncResult.files.length, true);
-                await localCache.set(parametersHash, Buffer.from(JSON.stringify(syncResult)));
-            }
-            synchronizeObj.parametersResult.resolve(syncResult);
-
-
-            let downloadProgress = await createProgress(`Downloading (bytes)`, 0);
-            let decompressProgress = await createProgress("Decompressing (bytes)", 0);
-            let scanProgress = await createProgress("Scanning (datums)", 0);
-            let decodeProgress = await createProgress("Decoding (datums)", 0);
-            let processProgress = await createProgress("Processing (datums)", 0);
-            let corruptDatumsProgress = await createProgress("Corrupt Datums (datums)", 0);
-
-            let scanFnc = config.scanFnc;
-
-
-            async function downloadAndParseFile(file: FileMetadata, runInner: (code: () => Promise<void>) => Promise<void>) {
-                const onFetchedData = runInSerial(async (data: Buffer) => {
-                    await downloadProgress(data.length, data.length, true);
-                    await decompressWriter.write(data);
-                });
-
-                const onDecompressedData = createLogScanner({
-                    debugName: file.path,
-                    onParsedData,
-                });
-                let batchedData: Buffer[] = [];
-                let notMatchedCount = 0;
-                let matchedSize = 0;
-                let notMatchedSize = 0;
-
-                let scanProgressCount = 0;
-
-                const wantData = await config.getWantData?.(file);
-
-                function onParsedData(posStart: number, posEnd: number, buffer: Buffer | "done"): MaybePromise<void> {
-                    if (buffer !== "done") {
-                        scanProgressCount++;
-                    }
-                    if (buffer !== "done") {
-                        if (scanFnc) {
-                            scanFnc(posStart, posEnd, buffer);
-                        }
-                        if (wantData && !wantData(posStart, posEnd, buffer)) {
-                            notMatchedSize += (posEnd - posStart);
-                            notMatchedCount++;
-                            return;
-                        }
-                        batchedData.push(buffer.slice(posStart, posEnd));
-                        matchedSize += (posEnd - posStart);
-                    }
-                    // IMPORTANT! We use scanProgressCount here, so queries that match few searches get results quickly!
-                    if (scanProgressCount >= ON_DATA_BATCH_COUNT || buffer === "done") {
-                        return (async () => {
-                            await scanProgress(scanProgressCount, scanProgressCount, true);
-                            scanProgressCount = 0;
-                            let errors: Error[] = [];
-
-                            let data = await measureBlock(async () => {
-                                let decoded: Datum[] = [];
-                                for (let datum of batchedData) {
-                                    try {
-                                        decoded.push(JSON.parse(datum.toString()) as Datum);
-                                    } catch (e: any) {
-                                        errors.push(e);
-                                    }
-                                    await decodeProgress(1, 1, true);
-                                }
-                                return decoded;
-                            }, "FastArchiveAppendable|deserializeData");
-                            if (errors.length > 0) {
-                                console.error(`${errors.length} errors decoding datums in ${file.path}, first error is:\n${errors[0].stack}`);
-                                await corruptDatumsProgress(errors.length, errors.length, true);
-                            }
-                            batchedData = [];
-                            if (data.length > 0) {
-                                await measureBlock(() => onData(data, file), "FastArchiveAppendable|onData(callback)");
-                            }
-                            if (onStats) {
-                                let stats: DatumStats = { matchedSize, notMatchedSize, errors: errors.length, notMatchedCount, matchedCount: data.length };
-                                matchedSize = 0;
-                                notMatchedSize = 0;
-                                notMatchedCount = 0;
-                                await measureBlock(() => onStats!(stats, file), "FastArchiveAppendable|onStats(callback)");
-                            }
-                            let count = data.length;
-                            await processProgress(count, count, true);
-                        })();
-                    }
-                }
-
-
-                // Create decompression stream
-                const decompressStream = new DecompressionStream("gzip");
-                const decompressWriter = decompressStream.writable.getWriter();
-                const decompressReader = decompressStream.readable.getReader();
-
-                // Decompress pipeline
-                let decompressPromise = (async () => {
-                    for await (let value of streamToIteratable(decompressReader)) {
-                        if (stoppedPromise.resolveCalled) return;
-                        let buffer = Buffer.from(value);
-
-                        await decompressProgress(buffer.length, buffer.length, true);
-                        await onDecompressedData(buffer);
-                    }
-                    await onDecompressedData("done");
-                })();
-
-                // Fetch the file data in a streaming manner
-
-                // TODO: Stream from the local cache instead? It should be possible, we can get the total size, and read chunks.
-                let hash = getFileMetadataHash(file) + ".file";
-                let contents = await localCache.get(hash);
-                if (contents?.length !== file.size) {
-                    contents = undefined;
-                }
-                if (stoppedPromise.resolveCalled) return;
-                await runInner(async () => {
-                    if (contents?.length) {
-                        const CHUNK_SIZE = 1000 * 1000 * 10;
-                        for (let i = 0; i < contents.length; i += CHUNK_SIZE) {
-                            let data = contents.slice(i, i + CHUNK_SIZE);
-                            await onFetchedData(data);
-                        }
-                    } else {
-                        let urlObj = new URL(file.url);
-                        urlObj.searchParams.set("cacheBust", config.cacheBust.toString());
-                        urlObj.searchParams.set("isPublic", isPublicValue.toString());
-                        if (file.nodeId && downloadSyncId) {
-                            let args = JSON.parse(urlObj.searchParams.get("args") || "");
-                            args[0] = downloadSyncId;
-                            urlObj.searchParams.set("args", JSON.stringify(args));
-                        }
-                        let url = urlObj.toString();
-                        const response = await fetch(url);
-                        if (!response.ok) {
-                            throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
-                        }
-
-                        if (!response.body) {
-                            throw new Error(`Response body is undefined for ${url}`);
-                        }
-                        let values: Buffer[] = [];
-                        const reader = response.body.getReader();
-                        void stoppedPromise.promise.finally(() => {
-                            void response.body?.cancel();
-                        });
-                        try {
-                            for await (let value of streamToIteratable(reader)) {
-                                // Cancel entirely
-                                if (stoppedPromise.resolveCalled) return;
-                                if (!value) continue;
-                                let buffer = Buffer.from(value);
-                                values.push(buffer);
-                                await onFetchedData(buffer);
-                            }
-                        } finally {
-                            reader.releaseLock();
-                        }
-
-                        await localCache.set(hash, Buffer.concat(values));
-                    }
-                    await decompressWriter.close();
-                    await decompressPromise;
-                });
-            }
-
-            // Fork off the processing
-            void (async () => {
-                try {
-                    // Iterate over all files and process them
-                    let fileProgress = await createProgress("Files", syncResult.files.length);
-                    let fileInnerProgress = await createProgress("Files Inner", syncResult.files.length);
-                    let failedFiles = await createProgress("Failed Files", 0);
-                    let runSerial = runInSerial(async (fnc: () => Promise<void>) => {
-                        try {
-                            await fnc();
-                        } finally {
-                            await fileInnerProgress(1, 0, true);
-                        }
-                    });
-                    async function downloadFileWrapper(file: FileMetadata) {
-                        if (stoppedPromise.resolveCalled) return;
-                        try {
-                            await downloadAndParseFile(file, runSerial);
-                        } catch (e: any) {
-                            console.warn(`Failed to download and parse file ${file.path}:\n${e.stack}`);
-                            await failedFiles(1, 1, true);
-                            if (onError) {
-                                onError(e, file);
-                            }
-                        }
-                        if (stoppedPromise.resolveCalled) return;
-                        await fileProgress(1, 0, true);
-                    }
-                    let remaining = syncResult.files.slice();
-                    async function runThread() {
-                        while (true) {
-                            let file = remaining.shift();
-                            if (!file) {
-                                return;
-                            }
-                            await downloadFileWrapper(file);
-                        }
-                    }
-                    await Promise.all(list(32).map(() => runThread()));
-
-                    await (await createProgress("Done", 0))(1, 1, true);
-                } catch (e: any) {
-                    baseOnProgress?.({
-                        section: `Error ${e.stack}`,
-                        value: 1,
-                        max: 1,
-                    });
-                } finally {
-                    config.onFinish?.();
-                    stopSynchronize();
-                }
-            })();
-
-            return {
-                metadata: syncResult,
-                stopSynchronize,
-            };
-
-        } catch (e) {
-            await stopSynchronize();
-            throw e;
-        }
-    }
-}
-
-
-export function createLogScanner(config: {
-    debugName: string;
-    onParsedData: (posStart: number, posEnd: number, buffer: Buffer | "done") => MaybePromise<void>;
-}): (data: Buffer | "done") => Promise<void> {
-    const { onParsedData } = config;
-    let pendingData: Buffer[] = [];
-
-    let finished = false;
-
-    let delimitterMatchIndex = 0;
-    return (async (data: Buffer | "done") => {
-        if (data === "done") {
-            finished = true;
-            // Flush any pending data, even though we have no delimitter. It will probably fail to parse, but... maybe it will work?
-            if (pendingData.length > 0) {
-                let combinedBuffer = Buffer.concat(pendingData);
-                pendingData = [];
-                await onParsedData(0, combinedBuffer.length, combinedBuffer);
-            }
-            await onParsedData(0, 0, "done");
-            return;
-        }
-        if (finished) {
-            throw new Error(`Finished scan, but we received more data: ${data.length}, sample is: ${data.slice(0, 100).toString("hex")}, ${config.debugName}`);
-        }
-
-        let lastStart = 0;
-        await measureBlock(async () => {
-            for (let i = 0; i < data.length; i++) {
-                if (data[i] === objectDelimitterBuffer[delimitterMatchIndex]) {
-                    delimitterMatchIndex++;
-                } else {
-                    delimitterMatchIndex = 0;
-                }
-                if (delimitterMatchIndex === objectDelimitterBuffer.length) {
-                    delimitterMatchIndex = 0;
-
-                    let buffer: Buffer;
-                    let posStart = -1;
-                    let posEnd = -1;
-                    if (pendingData.length > 0) {
-                        buffer = Buffer.concat([
-                            ...pendingData,
-                            data.slice(lastStart, i + 1),
-                        ]).slice(0, -objectDelimitterBuffer.length);
-                        pendingData = [];
-                        posStart = 0;
-                        posEnd = buffer.length;
-                    } else {
-                        buffer = data;
-                        posStart = lastStart;
-                        posEnd = i + 1 - objectDelimitterBuffer.length;
-                    }
-                    // Only sometimes awaiting here makes scanning almost 2X faster, in the normal case, somehow?
-                    let maybePromise = onParsedData(posStart, posEnd, buffer);
-                    if (maybePromise) {
-                        await maybePromise;
-                    }
-
-                    lastStart = i + 1;
-                }
-            }
-            if (lastStart < data.length) {
-                let remaining = data.slice(lastStart);
-                pendingData.push(remaining);
-            }
-        }, "FastArchiveAppendable|scanForDatumDelimitters");
-    });
-}