querysub 0.312.0 → 0.313.0

package/src/diagnostics/logs/FastArchiveAppendable.ts

@@ -0,0 +1,748 @@
+module.hotreload = true;
+module.noserverhotreload = false;
+import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
+import { getMachineId, getOwnMachineId } from "../../-a-auth/certs";
+import { isDefined, parseFileNameKVP, parsePath, partialCopyObject, streamToIteratable, sum, toFileNameKVP } from "../../misc";
+import { registerShutdownHandler } from "../periodic";
+import { batchFunction, delay, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { PromiseObj, isNode, keyByArray, nextId, sort, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
+import os from "os";
+import { getOwnThreadId } from "../../-f-node-discovery/NodeDiscovery";
+import fs from "fs";
+import { MaybePromise, canHaveChildren } from "socket-function/src/types";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { cache, lazy } from "socket-function/src/caching";
+import { getArchives, nestArchives } from "../../-a-archives/archives";
+import { Zip } from "../../zip";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { assertIsManagementUser } from "../managementPages";
+import { getControllerNodeIdList } from "../../-g-core-values/NodeCapabilities";
+import { errorToUndefined, ignoreErrors, timeoutToUndefinedSilent } from "../../errors";
+import { getCallObj } from "socket-function/src/nodeProxy";
+import { getSyncedController } from "../../library-components/SyncedController";
+import { getBrowserUrlNode, getOwnNodeId } from "../../-f-node-discovery/NodeDiscovery";
+import { secureRandom } from "../../misc/random";
+import { getPathIndex, getPathStr2 } from "../../path";
+import { onNextPaint } from "../../functional/onNextPaint";
+import { getArchivesBackblazePrivateImmutable, getArchivesBackblazePublicImmutable } from "../../-a-archives/archivesBackBlaze";
+import { httpsRequest } from "socket-function/src/https";
+import { getDomain } from "../../config";
+import { getIPDomain } from "../../-e-certs/EdgeCertController";
+import { getArchivesPrivateFileSystem } from "../../-a-archives/archivesPrivateFileSystem";
+import { createArchivesLimitedCache } from "../../-a-archives/archivesLimitedCache";
+import { sha256 } from "js-sha256";
+import { assertIsNetworkTrusted } from "../../-d-trust/NetworkTrust2";
+import { blue, magenta } from "socket-function/src/formatting/logColors";
+import { FileMetadata, FastArchiveAppendableControllerBase, FastArchiveAppendableController, getFileMetadataHash } from "./FastArchiveController";
+
+// NOTE: In a single command line micro-test it looks like we can write about 40K writes of 500 per once, when using 10X parallel, on a fairly potato server. We should probably batch though, and only do 1X parallel.
+/*
+Append Benchmarks
+    10 processes
+        Windows = 100K/S
+        Linux Digital Ocean = 1M/S
+        Linux PI SD Card = 300K/S
+        Linux PI USB = 300K/S
+    1 process
+        Windows = 40K/S
+        Linux Digital Ocean = 685K/S
+        Linux PI = 200K/S
+
+rm test.txt
+for i in {0..9}; do node -e 'const fs=require("fs");const id='$i';let i=0;const start=Date.now();while(Date.now()-start<5000){fs.appendFileSync("test.txt", `${id},${i++}\n`)}' & done; wait
+node -e 'const fs=require("fs");const seqs=new Map();fs.readFileSync("test.txt","utf8").trim().split("\n").forEach((l,i)=>{const[id,seq]=l.split(",").map(Number);if(!seqs.has(id))seqs.set(id,{last:-1,errs:0});const s=seqs.get(id);if(seq!==s.last+1){console.error(`Error for id ${id} at line ${i}: ${s.last}->${seq}`);s.errs++}s.last=seq});seqs.forEach((v,id)=>console.log(`ID ${id}: final seq ${v.last}, ${v.errs} gaps`))'
+*/
+
+const UNCOMPRESSED_LOG_FILE_WARN_THRESHOLD = 1024 * 1024 * 512;
+const UNCOMPRESSED_LOG_FILE_STOP_THRESHOLD = 1024 * 1024 * 1024 * 2;
+
+// Add a large wait, due to day light saving time, or whatever
+const UPLOAD_THRESHOLD = timeInHour * 3;
+const DEAD_TIMEOUT = timeInHour * 6;
+const DELETE_TIMEOUT = timeInHour * 12;
+
+const MAX_WORK_PER_PAINT = 40;
+
+const ON_DATA_BATCH_COUNT = 1024 * 10;
+//const ON_DATA_BATCH_COUNT = 1;
+
+
+
+const MAX_LOCAL_CACHED_FILES = 1000 * 10;
+const MAX_LOCAL_CACHED_SIZE = 1024 * 1024 * 1024 * 10;
+
+const getFileCache = cache((rootPath: string) => {
+    let data = getArchivesPrivateFileSystem(rootPath);
+    return createArchivesLimitedCache(data, {
+        maxFiles: MAX_LOCAL_CACHED_FILES,
+        maxSize: MAX_LOCAL_CACHED_SIZE,
+    });
+});
+
+
+// IMPORTANT! Use these like this, with one object per type of log
+// const errorAppendable = new FastArchiveAppendable<LogObject>("logs/error/");
+// const warnAppendable = new FastArchiveAppendable<LogObject>("logs/warn/");
+// const infoAppendable = new FastArchiveAppendable<LogObject>("logs/info/");
+// const logsAppendable = new FastArchiveAppendable<LogObject>("logs/logs/");
+
+
+// NOTE: We don't unescape. Because it massively slows down encoding, and... our delimitter is unlikely to randomly appear. IF it does, then it is likely not structural, and so the data will only look a bit weird (as in, it is likely not a length that cbor uses to decode or anything like that.
+export const objectDelimitterBuffer = Buffer.from([0x253, 0xe5, 0x05, 0x199, 0x5c, 0xbb, 0x63, 0x251]);
+
+export type DatumStats = {
+    matchedSize: number;
+    notMatchedSize: number;
+    errors: number;
+    matchedCount: number;
+    notMatchedCount: number;
+};
+
+
+export function getFileTimeStamp(path: string): number {
+    let file = path.replaceAll("\\", "/").split("/").at(-1)!;
+    // Remove .log extension and parse as ISO date
+    let dateStr = file.replace(/\.log$/, "");
+    // Add missing parts to make it a valid ISO string: "2025-09-06T07" -> "2025-09-06T07:00:00.000Z"
+    if (dateStr.length === 13) { // YYYY-MM-DDTHH format
+        dateStr += ":00:00.000Z";
+    }
+    return new Date(dateStr).getTime();
+}
+
+export class FastArchiveAppendable<Datum> {
+    private lastSizeWarningTime = 0;
+
+    public constructor(public rootPath: string) {
+        if (!this.rootPath.endsWith("/")) {
+            this.rootPath += "/";
+        }
+        if (isNode()) {
+            registerShutdownHandler(async () => {
+                await this.flushNow();
+            });
+            runInfinitePoll(timeInMinute, async () => {
+                await this.flushNow();
+            });
+            // Random, to try to prevent the dead file detection code from getting in sync. It's fine if it gets in sync, it's just inefficient as we'll have multiple services uploading the same file to the same location at the same time. 
+            void runInfinitePollCallAtStart(timeInMinute * 20 + Math.random() * timeInMinute * 5, async () => {
+                await this.moveLogsToBackblaze();
+            });
+        }
+    }
+
+    public getArchives = lazy(() => {
+        let archives = getArchivesBackblazePrivateImmutable(getDomain());
+        return nestArchives("fast-logs/" + this.rootPath, archives);
+    });
+
+    public getLocalPathRoot = lazy(() => {
+        let path = (
+            os.homedir()
+            + "/fast-log-cache/"
+            + getDomain() + "/"
+            + this.rootPath
+        );
+        if (!fs.existsSync(path)) {
+            fs.mkdirSync(path, { recursive: true });
+        }
+        return path;
+    });
+
+    // NOTE: Batching is both faster, and allows a lot of the code to log (which will cause an append) without causing an infinite loop.
+    private pendingWriteQueue: unknown[] = [];
+    @measureFnc
+    private escapeDelimitter(data: Buffer) {
+        if (!data.includes(objectDelimitterBuffer)) return data;
+
+        let startIndex = 0;
+        while (true) {
+            let index = data.indexOf(objectDelimitterBuffer, startIndex);
+            if (index === -1) break;
+            data[index]++;
+            startIndex = index;
+        }
+    }
+
+
+    private insideAppend = false;
+    /** NOTE: If the input data might contain user data, or mutated, use partialCopyObject first, to make a copy and prevent it from being excessively large. */
+    @measureFnc
+    public append(data: Datum) {
+        // Hmm... so... 
+        if (!isNode()) return;
+        // Just from logging, so... ignore it
+        if (this.insideAppend) return;
+        this.insideAppend = true;
+        try {
+            this.pendingWriteQueue.push(data);
+        } finally {
+            this.insideAppend = false;
+        }
+    }
+
+    private serialLock = runInSerial(async (fnc: () => Promise<void>) => {
+        await fnc();
+    });
+
+    // NOTE: This is disk writing, which should be fast, but if it's slow we might be able to remove the measureWrap (as technically spending 50% of our time writing to the disk is fine, and won't lag anything).
+    @measureFnc
+    public async flushNow(now = Date.now()) {
+        await this.serialLock(async () => {
+            // 2025-09-06T07
+            let hourFile = new Date(now).toISOString().slice(0, 13) + ".log";
+            let localCacheFolder = this.getLocalPathRoot() + getOwnThreadId() + "/";
+            let localCachePath = localCacheFolder + hourFile;
+            await fs.promises.mkdir(localCacheFolder, { recursive: true });
+            // Always heartbeat
+            await fs.promises.writeFile(localCacheFolder + "heartbeat", Buffer.from(now + ""));
+
+            if (this.pendingWriteQueue.length === 0) return;
+
+            try {
+                let beforeSize = await fs.promises.stat(localCachePath);
+                if (beforeSize.size > UNCOMPRESSED_LOG_FILE_STOP_THRESHOLD) {
+                    console.error(`FastArchiveAppendable: ${localCachePath} too large, refusing to add more data to it, current size ${formatNumber(beforeSize.size)}B > ${formatNumber(UNCOMPRESSED_LOG_FILE_STOP_THRESHOLD)}B`);
+                    return;
+                }
+            } catch {
+                // File not existing is fine
+            }
+
+            // NOTE: We can't use anything but JSON, as we need it to be scannable before decoding it (otherwise scanning takes 100X longer)
+            let writeData = this.pendingWriteQueue.map(v => {
+                let buffer = Buffer.from(JSON.stringify(v));
+                this.escapeDelimitter(buffer);
+                return buffer;
+            });
+            this.pendingWriteQueue = [];
+
+            // Apparently, anything more than this and our writes might not be atomic
+            const WRITE_ATOMIC_LIMIT = 4096;
+
+            // Group lines into WRITE_ATOMIC_LIMIT byte chunks
+            let chunks: Buffer[][] = [];
+            let currentChunk: Buffer[] = [];
+            let currentSize = 0;
+            for (let line of writeData) {
+                if (currentSize + line.length + objectDelimitterBuffer.length > WRITE_ATOMIC_LIMIT && currentChunk.length > 0) {
+                    chunks.push(currentChunk);
+                    currentChunk = [];
+                    currentSize = 0;
+                }
+                currentChunk.push(line);
+                currentSize += line.length;
+                currentChunk.push(objectDelimitterBuffer);
+                currentSize += objectDelimitterBuffer.length;
+            }
+            if (currentChunk.length > 0) {
+                chunks.push(currentChunk);
+            }
+
+            for (let chunk of chunks) {
+                await fs.promises.appendFile(localCachePath, Buffer.concat(chunk));
+            }
+
+            let finalSize = await fs.promises.stat(localCachePath);
+            if (finalSize.size > UNCOMPRESSED_LOG_FILE_WARN_THRESHOLD) {
+                const now = Date.now();
+                const timeSinceLastWarning = now - this.lastSizeWarningTime;
+
+                if (timeSinceLastWarning >= timeInMinute * 15) {
+                    console.error(`FastArchiveAppendable: ${localCachePath} is getting very big. This might cause logging to be disabled, to maintain readable of the logs (which will be required to debug why we are writing so much data! Current size ${formatNumber(finalSize.size)}B > ${formatNumber(UNCOMPRESSED_LOG_FILE_WARN_THRESHOLD)}B)`);
+                    this.lastSizeWarningTime = now;
+                }
+            }
+        });
+    }
+
+
+    public static getBackblazePath(config: { fileName: string; threadId: string }): string {
+        // 2025-09-06T07
+        let [year, month, day, hour] = config.fileName.split(/[-T:]/);
+        return `${year}/${month}/${day}/${hour}/${config.threadId}.log`;
+    }
+
+    public async moveLogsToBackblaze() {
+        await this.serialLock(async () => {
+            let rootCacheFolder = this.getLocalPathRoot();
+
+            let archives = this.getArchives();
+            async function moveLogsForFolder(threadId: string) {
+                let threadDir = rootCacheFolder + threadId + "/";
+                if (!fs.existsSync(threadDir)) return;
+                let files = await fs.promises.readdir(threadDir);
+                for (let file of files) {
+                    if (file === "heartbeat") continue;
+                    // 2025-09-06T07
+                    let fullPath = threadDir + file;
+
+                    // We could use modified time here? Although, this is nice if we move files around, and then manually have them moved, although even then... this could cause problem be tripping while we are copying the file, so... maybe this is just wrong?
+                    let timeStamp = getFileTimeStamp(fullPath);
+                    if (timeStamp > Date.now() - UPLOAD_THRESHOLD) continue;
+
+
+                    // NOTE: Because we use the same target path, if multiple services do this at the same time it's fine. Not great, but... fine.
+                    let backblazePath = FastArchiveAppendable.getBackblazePath({ fileName: file, threadId });
+                    console.log(magenta(`Moving ${fullPath} to Backblaze as ${backblazePath}`));
+                    let data = await measureBlock(async () => fs.promises.readFile(fullPath), "FastArchiveAppendable|readBeforeUploading");
+                    let compressed = await measureBlock(async () => Zip.gzip(data), "FastArchiveAppendable|compress");
+                    console.log(`Uploading ${formatNumber(data.length)}B (compressed to ${formatNumber(compressed.length)}B) logs to ${backblazePath} from ${fullPath}`);
+                    await archives.set(backblazePath, compressed);
+                    await fs.promises.unlink(fullPath);
+                }
+            }
+
+            await moveLogsForFolder(getOwnThreadId());
+            let allFolders = await fs.promises.readdir(rootCacheFolder);
+            for (let threadId of allFolders) {
+                if (threadId === getOwnThreadId()) continue;
+
+                let heartbeat = 0;
+                try {
+                    let heartbeatStr = await fs.promises.readFile(rootCacheFolder + threadId + "/heartbeat", "utf8");
+                    heartbeat = Number(heartbeatStr);
+                } catch { }
+                if (heartbeat < Date.now() - DEAD_TIMEOUT) {
+                    await moveLogsForFolder(threadId);
+                }
+                if (heartbeat < Date.now() - DELETE_TIMEOUT) {
+                    await fs.promises.rmdir(rootCacheFolder + threadId, { recursive: true });
+                }
+            }
+        });
+    }
+
+    private lastSynchronize: {
+        stopSynchronize: () => void;
+        parametersHash: string;
+        parametersResult: PromiseObj<{
+            files: FileMetadata[];
+        }>;
+    } | undefined;
+    public cancelAllSynchronizes() {
+        // We miss some cases here, but... it's probably fine. This should usually cancel.
+        this.lastSynchronize?.stopSynchronize();
+        this.lastSynchronize = undefined;
+    }
+    /** If called cancels the previous outstanding synchronize. */
+    public async synchronizeData(config: {
+        range: {
+            startTime: number;
+            endTime: number;
+        };
+        cacheBust: number;
+        wantData?: (posStart: number, posEnd: number, json: Buffer, file: FileMetadata) => boolean;
+        onData: (datum: Datum[], file: FileMetadata) => void;
+        // Called after onData
+        onStats?: (stats: DatumStats, file: FileMetadata) => void;
+        onFinish?: () => void;
+
+        onProgress?: (progress: {
+            section: string;
+            value: number;
+            max: number;
+        }) => void;
+    }): Promise<{
+        metadata: {
+            files: FileMetadata[];
+        };
+        stopSynchronize: () => void;
+    } | "cancelled"> {
+        let { wantData, onData, onStats } = config;
+        // Create unique client sync ID upfront
+        let syncId = nextId();
+
+        // Register progress callback immediately so we can receive progress during setup
+        // - It also helps with cancellation
+        FastArchiveAppendableControllerBase.progressCallbacks.set(syncId, config.onProgress ?? (() => { }));
+
+        let stopped = false;
+        let stoppedPromise = new PromiseObj<void>();
+        const stopSynchronize = () => {
+            stoppedPromise.resolve();
+            // Wait a bit for trailing progress, as progress is batched and delayed
+            setTimeout(() => {
+                stopped = true;
+                FastArchiveAppendableControllerBase.progressCallbacks.delete(syncId);
+            }, 5000);
+        };
+
+        let parametersHash = sha256(JSON.stringify({
+            range: config.range,
+            cacheBust: config.cacheBust,
+            version: 1,
+        })) + ".parameters";
+
+        let synchronizeObj = {
+            stopSynchronize,
+            parametersHash,
+            parametersResult: new PromiseObj<{
+                files: FileMetadata[];
+            }>(),
+        };
+        let last = this.lastSynchronize;
+        // Wait for the last one to finish getting the parameters, as we'll use the same ones.
+        if (last?.parametersHash === parametersHash) {
+            await last.parametersResult.promise;
+            // Another call happened before we finished
+            if (this.lastSynchronize !== last) return "cancelled";
+        }
+        this.lastSynchronize?.stopSynchronize();
+
+        this.lastSynchronize = synchronizeObj;
+
+        let baseOnProgress = config.onProgress;
+        let timeOfLastPaint = Date.now();
+        let throttlePerPaint = runInSerial(async () => {
+            let now = Date.now();
+            let workDone = now - timeOfLastPaint;
+            if (workDone < MAX_WORK_PER_PAINT) return;
+            await onNextPaint();
+            timeOfLastPaint = now;
+        });
+
+        let createProgress = async (section: string, max: number) => {
+            let cancelled: Error | undefined;
+            let lastValue = 0;
+            let baseBatch = batchFunction({ delay: 150, },
+                async (config: { value: number, overrideMax?: number }[]) => {
+                    if (cancelled) return;
+                    if (stopped) {
+                        cancelled = new Error(`Synchronization stopped`);
+                        return;
+                    }
+
+                    let value = config.at(-1)!.value;
+                    lastValue = value;
+                    let usedMax = config.map(c => c.overrideMax).filter(isDefined).at(-1) ?? max;
+                    value = Math.min(value, usedMax);
+                    baseOnProgress?.({ section, value, max: usedMax });
+                }
+            );
+            let deltaBatch = batchFunction({ delay: 150, },
+                async (config: { value: number, max: number }[]) => {
+                    if (cancelled) return;
+                    if (stopped) {
+                        cancelled = new Error(`Synchronization stopped`);
+                        return;
+                    }
+                    for (let delta of config) {
+                        lastValue += delta.value;
+                        max += delta.max;
+                    }
+                    baseOnProgress?.({ section, value: lastValue, max });
+                }
+            );
+            let firstCall = true;
+            let onProgress = async (value: number, overrideMax?: number, delta?: boolean) => {
+                // Call it immediately, so the output order isn't broken by the batching delay varying slightly
+                if (firstCall) {
+                    firstCall = false;
+                    baseOnProgress?.({ section, value: 0, max });
+                }
+                if (cancelled) throw cancelled;
+                if (delta) {
+                    void deltaBatch({ value, max: overrideMax ?? 0 });
+                } else {
+                    void baseBatch({ value, overrideMax });
+                }
+                await throttlePerPaint();
+            };
+            // Ordering is better if we wait until the first progress to send any progress
+            //await onProgress(0);
+            return onProgress;
+        };
+
+        try {
+            const localCache = getFileCache(this.rootPath);
+
+            let syncResult: { files: FileMetadata[]; createTime?: number; } | undefined;
+            let cachedSyncResultBuffer = await localCache.get(parametersHash);
+            let downloadSyncId: string = "";
+            let controller = FastArchiveAppendableController.nodes[getBrowserUrlNode()];
+            if (cachedSyncResultBuffer?.length) {
+                try {
+                    syncResult = JSON.parse(cachedSyncResultBuffer.toString());
+                    downloadSyncId = await controller.createSyncSession();
+                } catch (e: any) {
+                    console.error(`Failed to parsed cached sync result, synchronizing from scratch instead\n${e.stack}`);
+                }
+            }
+            if (!syncResult) {
+                let findingFiles = await createProgress("Finding files", 0);
+                syncResult = await controller.startSynchronize({
+                    syncId,
+                    range: config.range,
+                    rootPath: this.rootPath,
+                });
+                syncResult.createTime = Date.now();
+                await findingFiles(syncResult.files.length, syncResult.files.length, true);
+                await localCache.set(parametersHash, Buffer.from(JSON.stringify(syncResult)));
+            }
+            synchronizeObj.parametersResult.resolve(syncResult);
+
+
+            let downloadProgress = await createProgress("Downloading (bytes)", 0);
+            let decompressProgress = await createProgress("Decompressing (bytes)", 0);
+            let scanProgress = await createProgress("Scanning (datums)", 0);
+            let decodeProgress = await createProgress("Decoding (datums)", 0);
+            let processProgress = await createProgress("Processing (datums)", 0);
+            let corruptDatumsProgress = await createProgress("Corrupt Datums (datums)", 0);
+
+            const self = this;
+
+
+            async function downloadAndParseFile(file: FileMetadata) {
+                const onFetchedData = runInSerial(async (data: Buffer) => {
+                    await downloadProgress(data.length, data.length, true);
+                    await decompressWriter.write(data);
+                });
+
+                const onDecompressedData = createLogScanner({
+                    onParsedData,
+                });
+                let batchedData: Buffer[] = [];
+                let notMatchedCount = 0;
+                let matchedSize = 0;
+                let notMatchedSize = 0;
+
+                let scanProgressCount = 0;
+
+                function onParsedData(posStart: number, posEnd: number, buffer: Buffer | "done"): MaybePromise<void> {
+                    if (buffer !== "done") {
+                        scanProgressCount++;
+                    }
+                    if (buffer !== "done") {
+                        if (wantData && !wantData(posStart, posEnd, buffer, file)) {
+                            notMatchedSize += (posEnd - posStart);
+                            notMatchedCount++;
+                            return;
+                        }
+                        batchedData.push(buffer.slice(posStart, posEnd));
+                        matchedSize += (posEnd - posStart);
+                    }
+                    // IMPORTANT! We use scanProgressCount here, so queries that match few searches get results quickly!
+                    if (scanProgressCount >= ON_DATA_BATCH_COUNT || buffer === "done") {
+                        return (async () => {
+                            await scanProgress(scanProgressCount, scanProgressCount, true);
+                            scanProgressCount = 0;
+                            let errors: Error[] = [];
+
+                            let data = await measureBlock(async () => {
+                                let decoded: Datum[] = [];
+                                for (let datum of batchedData) {
+                                    try {
+                                        decoded.push(JSON.parse(datum.toString()) as Datum);
+                                    } catch (e: any) {
+                                        errors.push(e);
+                                    }
+                                    await decodeProgress(1, 1, true);
+                                }
+                                return decoded;
+                            }, "FastArchiveAppendable|deserializeData");
+                            if (errors.length > 0) {
+                                console.error(`${errors.length} errors decoding datums in ${file.path}, first error is:\n${errors[0].stack}`);
+                                await corruptDatumsProgress(errors.length, errors.length, true);
+                            }
+                            batchedData = [];
+                            if (data.length > 0) {
+                                await measureBlock(() => onData(data, file), "FastArchiveAppendable|onData(callback)");
+                            }
+                            if (onStats) {
+                                let stats: DatumStats = { matchedSize, notMatchedSize, errors: errors.length, notMatchedCount, matchedCount: data.length };
+                                matchedSize = 0;
+                                notMatchedSize = 0;
+                                notMatchedCount = 0;
+                                await measureBlock(() => onStats!(stats, file), "FastArchiveAppendable|onStats(callback)");
+                            }
+                            let count = data.length;
+                            await processProgress(count, count, true);
+                        })();
+                    }
+                }
+
+
+                // Create decompression stream
+                const decompressStream = new DecompressionStream("gzip");
+                const decompressWriter = decompressStream.writable.getWriter();
+                const decompressReader = decompressStream.readable.getReader();
+
+                // Decompress pipeline
+                let decompressPromise = (async () => {
+                    for await (let value of streamToIteratable(decompressReader)) {
+                        if (stoppedPromise.resolveCalled) return;
+                        let buffer = Buffer.from(value);
+
+                        await decompressProgress(buffer.length, buffer.length, true);
+                        await onDecompressedData(buffer);
+                    }
+                    await onDecompressedData("done");
+                })();
+
+                // Fetch the file data in a streaming manner
+                let urlObj = new URL(file.url);
+                urlObj.searchParams.set("cacheBust", config.cacheBust.toString());
+                if (file.nodeId && downloadSyncId) {
+                    let args = JSON.parse(urlObj.searchParams.get("args") || "");
+                    args[0] = downloadSyncId;
+                    urlObj.searchParams.set("args", JSON.stringify(args));
+                }
+                let url = urlObj.toString();
+
+                // TODO: Stream from the local cache instead? It should be possible, we can get the total size, and read chunks.
+                let hash = getFileMetadataHash(file) + ".file";
+                let info = await localCache.getInfo(hash);
+                if (info?.size) {
+                    const CHUNK_SIZE = 1000 * 1000 * 10;
+                    for (let i = 0; i < info.size; i += CHUNK_SIZE) {
+                        let data = await localCache.get(hash, { range: { start: i, end: i + CHUNK_SIZE } });
+                        if (!data) {
+                            throw new Error(`Cached data disappeared (the info was there?) for ${hash} at ${i}`);
+                        }
+                        await onFetchedData(data);
+                    }
+                } else {
+                    const response = await fetch(url);
+                    if (!response.ok) {
+                        throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+                    }
+
+                    if (!response.body) {
+                        throw new Error(`Response body is undefined for ${url}`);
+                    }
+                    let values: Buffer[] = [];
+                    const reader = response.body.getReader();
+                    void stoppedPromise.promise.finally(() => {
+                        void response.body?.cancel();
+                    });
+                    try {
+                        for await (let value of streamToIteratable(reader)) {
+                            // Cancel entirely
+                            if (stoppedPromise.resolveCalled) return;
+                            if (!value) continue;
+                            let buffer = Buffer.from(value);
+                            values.push(buffer);
+                            await onFetchedData(buffer);
+                        }
+                    } finally {
+                        reader.releaseLock();
+                    }
+
+                    await localCache.set(hash, Buffer.concat(values));
+                }
+                await decompressWriter.close();
+                await decompressPromise;
+            }
+
+            // Fork off the processing
+            void (async () => {
+                try {
+                    // Iterate over all files and process them
+                    let fileProgress = await createProgress("Files", 0);
+                    let failedFiles = await createProgress("Failed Files", 0);
+                    for (let file of syncResult.files) {
+                        if (stoppedPromise.resolveCalled) return;
+                        try {
+                            await downloadAndParseFile(file);
+                        } catch (e: any) {
+                            console.warn(`Failed to download and parse file ${file.path}:\n${e.stack}`);
+                            await failedFiles(1, 1, true);
+                        }
+                        await fileProgress(1, 1, true);
+                    }
+
+                    await (await createProgress("Done", 0))(1, 1, true);
+                } catch (e: any) {
+                    baseOnProgress?.({
+                        section: `Error ${e.stack}`,
+                        value: 1,
+                        max: 1,
+                    });
+                } finally {
+                    config.onFinish?.();
+                    stopSynchronize();
+                }
+            })();
+
+            return {
+                metadata: syncResult,
+                stopSynchronize,
+            };
+
+        } catch (e) {
+            await stopSynchronize();
+            throw e;
+        }
+    }
+}
+
+
+export function createLogScanner(config: {
+    onParsedData: (posStart: number, posEnd: number, buffer: Buffer | "done") => MaybePromise<void>;
+}): (data: Buffer | "done") => Promise<void> {
+    const { onParsedData } = config;
+    let pendingData: Buffer[] = [];
+
+    let delimitterMatchIndex = 0;
+    return runInSerial(async (data: Buffer | "done") => {
+        if (data === "done") {
+            // Flush any pending data, even though we have no delimitter. It will probably fail to parse, but... maybe it will work?
+            if (pendingData.length > 0) {
+                let combinedBuffer = Buffer.concat(pendingData);
+                pendingData = [];
+                await onParsedData(0, combinedBuffer.length, combinedBuffer);
+            }
+            await onParsedData(0, 0, "done");
+            return;
+        }
+
+        let lastStart = 0;
+        await measureBlock(async () => {
+            for (let i = 0; i < data.length; i++) {
+                if (data[i] === objectDelimitterBuffer[delimitterMatchIndex]) {
+                    delimitterMatchIndex++;
+                } else {
+                    delimitterMatchIndex = 0;
+                }
+                if (delimitterMatchIndex === objectDelimitterBuffer.length) {
+                    delimitterMatchIndex = 0;
+
+                    let buffer: Buffer;
+                    let posStart = -1;
+                    let posEnd = -1;
+                    if (pendingData.length > 0) {
+                        buffer = Buffer.concat([
+                            ...pendingData,
+                            data.slice(lastStart, i + 1),
+                        ]).slice(0, -objectDelimitterBuffer.length);
+                        posStart = 0;
+                        posEnd = buffer.length;
+                    } else {
+                        buffer = data;
+                        posStart = lastStart;
+                        posEnd = i + 1 - objectDelimitterBuffer.length;
+                    }
+                    // Delimitter was the start of the chunk, and it's the first chunk. Just skip it.
+                    if (posStart === posEnd && i === 0) {
+                        lastStart = i + 1;
+                        continue;
+                    }
+
+                    // Only sometimes awaiting here makes scanning almost 2X faster, in the normal case, somehow?
+                    let maybePromise = onParsedData(posStart, posEnd, buffer);
+                    if (maybePromise) {
+                        await maybePromise;
+                    }
+
+                    pendingData = [];
+                    lastStart = i + 1;
+                }
+            }
+            if (lastStart < data.length) {
+                let remaining = data.slice(lastStart);
+                pendingData.push(remaining);
+            }
+        }, "FastArchiveAppendable|scanForDatumDelimitters");
+    });
+}