querysub 0.374.0 → 0.376.0

package/src/diagnostics/logs/FastArchiveViewer.tsx

@@ -1,1090 +0,0 @@
-import { measureFnc } from "socket-function/src/profiling/measure";
-import { t } from "../../2-proxy/schema2";
-import { qreact } from "../../4-dom/qreact";
-import { DatumStats, FastArchiveAppendable } from "./FastArchiveAppendable";
-import { Querysub } from "../../4-querysub/Querysub";
-import { URLParam } from "../../library-components/URLParam";
-import { TimeRangeSelector, getTimeRange } from "./TimeRangeSelector";
-import { formatNumber, formatTime, formatVeryNiceDateTime, formatDateTime } from "socket-function/src/formatting/format";
-import { list, sort, throttleFunction, timeInHour } from "socket-function/src/misc";
-import { css } from "typesafecss";
-import { logErrors } from "../../errors";
-import { InputLabelURL } from "../../library-components/InputLabel";
-import { clamp } from "../../misc";
-import { ButtonSelector } from "../../library-components/ButtonSelector";
-import { Button } from "../../library-components/Button";
-import { lazy } from "socket-function/src/caching";
-import { LOG_LIMIT_FLAG } from "./diskLogger";
-import { MaybePromise, canHaveChildren } from "socket-function/src/types";
-import { niceParse } from "../../niceStringify";
-import { FileMetadata } from "./FastArchiveController";
-import { throttleRender } from "../../functional/throttleRender";
-
-const RENDER_INTERVAL = 1000;
-
-const HISTOGRAM_RERENDER_INTERVAL = 10000;
-
-export const filterParam = new URLParam("filter", "");
-// If filter2 is set, we also need it. Not great (why not support arbitrary counts of these), but for now... this should be fine (and support arbitrary counts might slow down our incredibly optimized scan function, where most of our time is spent).
-// NOTE: This supports less than filterParam (no object parsing, it only does text contains)
-export const filterParam2 = new URLParam("filter2", "");
-
-export const cacheBustParam = new URLParam("cacheBust", 0);
-const caseInsensitiveParam = new URLParam("caseInsensitive", false);
-const hideAllDataParam = new URLParam("hideAllData", false);
-export type ScanFnc = (posStart: number, posEnd: number, data: Buffer) => boolean;
-
-export class FastArchiveViewer<T> extends qreact.Component<{
-    fastArchives: () => FastArchiveAppendable<T>[];
-    forceGetPublic?: boolean;
-    runOnLoad?: boolean;
-    onStart: () => MaybePromise<void>;
-    getScanFnc?: () => ScanFnc;
-    getWantData?: (file: FileMetadata) => Promise<ScanFnc | undefined>;
-    onDatums: (source: FastArchiveAppendable<T>, datums: T[], metadata: FileMetadata) => void;
-    // Called after onData
-    onStats?: (source: FastArchiveAppendable<T>, stats: DatumStats, metadata: FileMetadata) => void;
-    onFinish?: () => void;
-}> {
-    state = t.state({
-        runCount: t.atomic<number>(0),
-        // rootPath =>
-        fileMetadata: t.atomic<({
-            files: FileMetadata[];
-            createTime?: number;
-        } | undefined)[]>([]),
-        finished: t.atomic(true),
-        error: t.atomic<string | undefined>(undefined),
-        pendingSyncInitializations: t.atomic<number>(0),
-
-        // Current sync parameters - set before synchronizeData is called
-        currentSyncParams: t.atomic<{
-            filterString: string;
-            startTime: number;
-            endTime: number;
-            fastArchivePaths: string[];
-        } | undefined>(undefined),
-
-        renderSeqNum: t.atomic<number>(0),
-    });
-
-    // Histogram data arrays
-    private histogramAllDataCounts: Float64Array = new Float64Array(0);
-    private histogramSelectedDataCounts: Float64Array = new Float64Array(0);
-    private histogramAllDataSizes: Float64Array = new Float64Array(0);
-    private histogramSelectedDataSizes: Float64Array = new Float64Array(0);
-    private histogramStartTime: number = 0;
-    private histogramEndTime: number = 0;
-    private histogramBucketTime: number = timeInHour;
-
-    private allSize = 0;
-    private matchedSize = 0;
-
-    private limitedScanCount = 0;
-    private limitedMatchCount = 0;
-
-    private fileErrors: Array<{ file: FileMetadata; error: Error }> = [];
-
-    private progressBars: Record<string, {
-        section: string,
-        value: number,
-        max: number,
-        initialTime: number,
-        lastSetTime: number,
-    }> = {};
-
-    private lastRenderTime = 0;
-
-    private getHistogramBucketIndex(startTime: number): number {
-        const index = Math.floor((startTime - this.histogramStartTime) / this.histogramBucketTime);
-        return clamp(index, 0, this.histogramAllDataCounts.length - 1);
-    }
-
-    componentWillMount(): void {
-        qreact.onDispose(() => {
-            this.cancelAllSynchronizes();
-        });
-    }
-
-    cancelAllSynchronizes() {
-        for (let fastArchive of this.props.fastArchives()) {
-            fastArchive.cancelAllSynchronizes();
-        }
-    }
-
-    private currentSequenceNumber = 0;
-    private latestSequenceNumber = 0;
-
-    @measureFnc
-    private async synchronizeData() {
-        console.log("synchronizeData");
-        let onStart = Querysub.fastRead(() => this.props.onStart);
-        let onDatums = Querysub.fastRead(() => this.props.onDatums);
-        let onStats = Querysub.fastRead(() => this.props.onStats);
-        let fastArchives = Querysub.fastRead(() => this.props.fastArchives);
-        let onFinish = Querysub.fastRead(() => this.props.onFinish);
-        let getWantData = Querysub.fastRead(() => this.props.getWantData);
-        // Increment sequence number for this new sync attempt
-        this.currentSequenceNumber++;
-        this.latestSequenceNumber = this.currentSequenceNumber;
-        const mySequenceNumber = this.currentSequenceNumber;
-
-        // Increment run count for each new run
-        Querysub.commit(() => {
-            this.state.runCount++;
-        });
-
-        // Helper function to check if this sequence number is still the latest
-        const isLatestSync = () => mySequenceNumber === this.latestSequenceNumber;
-
-        // Helper function to safely update state only if we're the latest sync
-        const ifLatest = (updater: () => void) => {
-            if (isLatestSync()) {
-                Querysub.commit(updater);
-            } else {
-                console.log(`Ignoring state update from sequence ${mySequenceNumber}, latest is ${this.latestSequenceNumber}`);
-            }
-        };
-
-        // Increment sync count before making the call
-        Querysub.commit(() => {
-            this.state.pendingSyncInitializations++;
-        });
-
-
-        const timeRange = getTimeRange();
-        let filterString = filterParam.value;
-        let filterString2 = filterParam2.value;
-        let scannedValueCount = 0;
-
-        // Store current sync parameters BEFORE calling synchronizeData
-        ifLatest(() => {
-            this.state.currentSyncParams = {
-                filterString,
-                startTime: timeRange.startTime,
-                endTime: timeRange.endTime,
-                fastArchivePaths: fastArchives().map(archive => archive.rootPath),
-            };
-        });
-
-        ifLatest(() => {
-            this.state.error = undefined;
-            this.state.finished = false;
-
-            // Clear existing progress bars
-            for (let key of Object.keys(this.progressBars)) {
-                delete this.progressBars[key];
-            }
-            this.state.fileMetadata = [];
-            this.fileErrors = [];
-            this.limitedScanCount = 0;
-            this.limitedMatchCount = 0;
-            this.allSize = 0;
-            this.matchedSize = 0;
-
-            this.histogramBucketTime = Math.max(timeInHour, (timeRange.endTime - timeRange.startTime) / 100);
-
-            this.histogramStartTime = timeRange.startTime - this.histogramBucketTime * 2;
-            this.histogramEndTime = timeRange.endTime + this.histogramBucketTime * 2;
-            const bucketCount = clamp(Math.ceil((this.histogramEndTime - this.histogramStartTime) / this.histogramBucketTime), 1, 10000);
-
-            this.histogramAllDataCounts = new Float64Array(bucketCount);
-            this.histogramSelectedDataCounts = new Float64Array(bucketCount);
-            this.histogramAllDataSizes = new Float64Array(bucketCount);
-            this.histogramSelectedDataSizes = new Float64Array(bucketCount);
-        });
-
-        const onProgress = (progress: { section: string; value: number; max: number }) => {
-            if (!isLatestSync()) return;
-            const key = progress.section;
-
-            // Add or update progress bar
-            let existingBar = this.progressBars[key];
-            this.progressBars[key] = {
-                section: progress.section,
-                value: progress.value,
-                max: progress.max,
-                initialTime: existingBar?.initialTime || Date.now(),
-                lastSetTime: Date.now(),
-            };
-
-            let now = Date.now();
-            if (now - this.lastRenderTime > RENDER_INTERVAL) {
-                this.lastRenderTime = now;
-                ifLatest(() => {
-                    this.state.renderSeqNum++;
-                });
-            }
-        };
-        try {
-            await onStart();
-
-            const caseInsensitive = Querysub.fastRead(() => caseInsensitiveParam.value);
-            let caseInsensitiveMapping = new Uint8Array(256);
-            for (let i = 0; i < 256; i++) {
-                caseInsensitiveMapping[i] = String.fromCharCode(i).toLowerCase().charCodeAt(0);
-            }
-
-            type ScanMatch = (posStart: number, posEnd: number, buffer: Buffer) => boolean;
-            type DatumMatch = (datum: unknown) => boolean;
-            type MatchObj = {
-                scanMatch: ScanMatch,
-                datumMatch: DatumMatch,
-            };
-
-            function parsePart(filterString: string): MatchObj {
-                if (!filterString.trim()) {
-                    return {
-                        scanMatch: () => true,
-                        datumMatch: () => true,
-                    };
-                }
-                let datumMatch = (datum: unknown) => true;
-                if (filterString.includes("=")) {
-                    let equalIndex = filterString.indexOf("=");
-                    let key = filterString.slice(0, equalIndex).trim();
-                    let value = niceParse(filterString.slice(equalIndex + 1).trim());
-                    filterString = `${JSON.stringify(key)}:${JSON.stringify(value)}`;
-                    datumMatch = (datum: unknown) => {
-                        if (!canHaveChildren(datum)) return false;
-                        let curKey = key;
-                        if (!(curKey in datum)) {
-                            curKey = curKey.toLowerCase();
-                            let keys = Object.keys(datum);
-                            curKey = keys.find(x => x.toLowerCase() === curKey) || "";
-                            if (!curKey) return false;
-                        }
-                        return datum[curKey] === value;
-                    };
-                }
-                if (caseInsensitive) {
-                    filterString = filterString.toLowerCase();
-                }
-                const filterBuffer = Buffer.from(filterString);
-                const char0 = filterBuffer[0];
-                return {
-                    scanMatch: (posStart: number, posEnd: number, buffer: Buffer) => {
-                        for (let i = posStart; i < posEnd; i++) {
-                            // NOTE: This is slow for repeated prefixes, but... in practice, even if we have prefixes, they won't repeat, so... this should be quite fast.
-                            let ch = buffer[i];
-                            if (caseInsensitive) {
-                                ch = caseInsensitiveMapping[ch];
-                            }
-                            if (ch === char0) {
-                                if (filterBuffer.length === 1) return true;
-                                for (let j = 1; j < filterBuffer.length; j++) {
-                                    let ch2 = buffer[i + j];
-                                    if (caseInsensitive) {
-                                        ch2 = caseInsensitiveMapping[ch2];
-                                    }
-                                    if (ch2 !== filterBuffer[j]) {
-                                        break;
-                                    }
-                                    if (j === filterBuffer.length - 1) {
-                                        return true;
-                                    }
-                                }
-                            }
-                        }
-                        return false;
-                    },
-                    datumMatch,
-                };
-            }
-
-            function joinMatches(matches: MatchObj[], condition: "||" | "&&"): MatchObj {
-                if (condition === "||") {
-                    return {
-                        scanMatch: (posStart: number, posEnd: number, buffer: Buffer) => {
-                            return matches.some(match => match.scanMatch(posStart, posEnd, buffer));
-                        },
-                        datumMatch: (datum: unknown) => {
-                            return matches.some(match => match.datumMatch(datum));
-                        },
-                    };
-                }
-                return {
-                    scanMatch: (posStart: number, posEnd: number, buffer: Buffer) => {
-                        return matches.every(match => match.scanMatch(posStart, posEnd, buffer));
-                    },
-                    datumMatch: (datum: unknown) => {
-                        return matches.every(match => match.datumMatch(datum));
-                    },
-                };
-            }
-
-
-            let andMatches = filterString.split("|").map(orParts => {
-                let andParts = orParts.split("&");
-                let andMatches = andParts.map(part => parsePart(part.trim()));
-                return joinMatches(andMatches, "&&");
-            });
-            let { scanMatch, datumMatch } = joinMatches(andMatches, "||");
-
-            let andMatches2 = filterString2.split("|").map(orParts => {
-                let andParts = orParts.split("&");
-                let andMatches = andParts.map(part => parsePart(part.trim()));
-                return joinMatches(andMatches, "&&");
-            });
-            let scanObj2 = joinMatches(andMatches2, "||");
-            let scanMatch2 = scanObj2.scanMatch;
-            let datumMatch2 = scanObj2.datumMatch;
-            let useScan2 = !!filterString2.trim();
-
-            const limitedBuffer = Buffer.from(LOG_LIMIT_FLAG);
-
-
-            await Promise.all(fastArchives().map(async (fastArchive, index) => {
-                const result = await fastArchive.synchronizeData({
-                    range: timeRange,
-                    cacheBust: Querysub.fastRead(() => cacheBustParam.value),
-                    scanFnc: Querysub.fastRead(() => this.props.getScanFnc?.()),
-                    getWantData: async (file) => {
-                        let wantData = await getWantData?.(file);
-                        return (posStart: number, posEnd: number, data: Buffer) => {
-                            let isLimited = false;
-                            for (let i = posStart; i < posEnd && !isLimited; i++) {
-                                if (data[i] === limitedBuffer[0]) {
-                                    for (let j = 1; j < limitedBuffer.length; j++) {
-                                        if (data[i + j] !== limitedBuffer[j]) {
-                                            break;
-                                        }
-                                        if (j === limitedBuffer.length - 1) {
-                                            isLimited = true;
-                                            break;
-                                        }
-                                    }
-                                }
-                            }
-                            if (isLimited) {
-                                this.limitedScanCount++;
-                            }
-
-                            // scanMatch is faster than wantData (generally), so run it first
-                            let matched = scanMatch(posStart, posEnd, data);
-                            if (matched && useScan2) {
-                                matched = scanMatch2(posStart, posEnd, data);
-                            }
-                            if (matched && wantData) {
-                                matched = wantData(posStart, posEnd, data);
-                            }
-                            if (isLimited && matched) {
-                                this.limitedMatchCount++;
-                            }
-                            return matched;
-                        };
-                    },
-                    onData: (datums, file) => {
-                        if (!isLatestSync()) return;
-                        let keptDatums: T[] = [];
-                        for (let i = 0; i < datums.length; i++) {
-                            let datum = datums[i];
-                            if (datumMatch(datum)) {
-                                keptDatums.push(datum);
-                            }
-                        }
-                        onDatums(fastArchive, keptDatums, file);
-                    },
-                    onStats: onStats && ((stats, file) => {
-                        if (!isLatestSync()) return;
-
-                        onStats!(fastArchive, stats, file);
-                        this.allSize += stats.matchedSize + stats.notMatchedSize;
-                        this.matchedSize += stats.matchedSize;
-                        // Update scanned count and sequence number for rendering
-                        scannedValueCount += stats.notMatchedCount + stats.matchedCount;
-                        // Update all data histogram
-                        const bucketIndex = this.getHistogramBucketIndex(file.startTime);
-                        this.histogramAllDataCounts[bucketIndex] += stats.notMatchedCount + stats.matchedCount;
-                        this.histogramSelectedDataCounts[bucketIndex] += stats.matchedCount;
-                        this.histogramAllDataSizes[bucketIndex] += stats.matchedSize + stats.notMatchedSize;
-                        this.histogramSelectedDataSizes[bucketIndex] += stats.matchedSize;
-                        if (scannedValueCount >= HISTOGRAM_RERENDER_INTERVAL) {
-                            scannedValueCount = 0;
-                            let now = Date.now();
-                            if (now - this.lastRenderTime > RENDER_INTERVAL) {
-                                this.lastRenderTime = now;
-                                ifLatest(() => {
-                                    this.state.renderSeqNum++;
-                                });
-                            }
-                        }
-                    }),
-                    onError: (error, file) => {
-                        if (!isLatestSync()) return;
-                        this.fileErrors.push({ file, error });
-                    },
-                    onProgress,
-                    onFinish: () => {
-                        ifLatest(() => {
-                            this.state.renderSeqNum++;
-                        });
-                        setTimeout(() => {
-                            ifLatest(() => {
-                                this.state.finished = true;
-                            });
-                        }, 2500);
-                        onFinish?.();
-                    },
-                });
-                if (result !== "cancelled") {
-                    ifLatest(() => {
-                        this.state.fileMetadata[index] = result.metadata;
-                    });
-                }
-                return result;
-            }));
-
-        } catch (error: any) {
-            ifLatest(() => {
-                this.state.error = error.stack || String(error);
-            });
-        } finally {
-            // Decrement sync count when call finishes (regardless of sequence number)
-            Querysub.commit(() => {
-                this.state.pendingSyncInitializations--;
-            });
-        }
-    }
-    private renderProgressBars() {
-        let progressEntries = Object.entries(this.progressBars);
-
-        if (progressEntries.length === 0) {
-            return null;
-        }
-
-        const getNowTime = lazy(() => Querysub.timeDelayed(1000));
-        if (progressEntries.some(x => x[0].startsWith("Error"))) {
-            progressEntries = progressEntries.filter(x => x[0].startsWith("Error"));
-        }
-        sort(progressEntries, x => x[1].initialTime);
-
-        // Group progress bars by the text after the pipe sign
-        const groupedProgress = new Map<string, {
-            displayName: string;
-            entries: Array<[string, { section: string, value: number, max: number, initialTime: number, lastSetTime: number }]>;
-            totalValue: number;
-            totalMax: number;
-            earliestTime: number;
-            latestTime: number;
-        }>();
-
-        for (const [key, progress] of progressEntries) {
-            const pipeIndex = progress.section.indexOf("|");
-            const displayName = pipeIndex >= 0 ? progress.section.substring(pipeIndex + 1) : progress.section;
-
-            if (!groupedProgress.has(displayName)) {
-                groupedProgress.set(displayName, {
-                    displayName,
-                    entries: [],
-                    totalValue: 0,
-                    totalMax: 0,
-                    earliestTime: progress.initialTime,
-                    latestTime: progress.lastSetTime,
-                });
-            }
-
-            const group = groupedProgress.get(displayName)!;
-            group.entries.push([key, progress]);
-            group.totalValue += progress.value;
-            group.totalMax += progress.max;
-            group.earliestTime = Math.min(group.earliestTime, progress.initialTime);
-            group.latestTime = Math.max(group.latestTime, progress.lastSetTime);
-        }
-
-        return (
-            <div className={css.hbox(10).wrap.alignItems("start").fillWidth}>
-                {!this.state.finished && (
-                    <div
-                        onClick={() => {
-                            this.cancelAllSynchronizes();
-                        }}
-                        className={css.pad2(12).bord2(200, 50, 50).hsl(200, 50, 95).colorhsl(200, 50, 40).button}
-                    >
-                        Cancel
-                    </div>
-                )}
-                {Array.from(groupedProgress.values()).map((group) => {
-                    const fraction = clamp(group.totalMax > 0 ? (group.totalValue / group.totalMax) * 1 : 1, 0, 1);
-                    let progressTime = 0;
-                    if (group.totalValue >= group.totalMax) {
-                        progressTime = group.latestTime;
-                    } else {
-                        progressTime = getNowTime();
-                    }
-                    const elapsedTime = progressTime - group.earliestTime;
-
-                    // Create tooltip showing breakdown by original prefix
-                    const tooltipContent = group.entries.map(([key, progress]) => {
-                        const pipeIndex = progress.section.indexOf("|");
-                        const prefix = pipeIndex >= 0 ? progress.section.substring(0, pipeIndex) : key;
-                        return `${prefix}: ${formatNumber(progress.value)}/${formatNumber(progress.max)}`;
-                    }).join("\n");
-
-                    return (
-                        <div key={group.displayName} className={
-                            // NOTE: There is no point making this proportionally to the time, because we stream the data, so the time overlaps, making everything take most of the time.
-                            css.vbox(4)
-                        }
-                            title={tooltipContent}
-                        >
-                            <div className={css.vbox(2)}>
-                                <div className={css.fontSize(14).colorhsl(0, 0, 20)}>
-                                    {group.displayName}
-                                </div>
-                                <div className={css.fontSize(12).colorhsl(0, 0, 40).width(150)}>
-                                    {formatNumber(group.totalValue)} {group.totalValue < group.totalMax && `/ ${formatNumber(group.totalMax)}`} ({formatTime(elapsedTime)})
-                                </div>
-                            </div>
-                            <div className={css.fillWidth.height(8).bord2(200, 20, 80).hsl(200, 10, 95)}>
-                                <div
-                                    className={css.height(8).hsl(200, 50, 70).width(`${fraction * 100}%`).transition("width 200ms")}
-                                />
-                            </div>
-                        </div>
-                    );
-                })}
-            </div>
-        );
-    }
-
-    private renderHistogram() {
-        this.state.renderSeqNum;
-
-        if (this.histogramAllDataCounts.length === 0) {
-            return null;
-        }
-
-        let countsArray = Array.from(this.histogramAllDataCounts);
-        let selectedArray = Array.from(this.histogramSelectedDataCounts);
-        const maxAllCount = countsArray.reduce((a, b) => Math.max(a, b), 0);
-        const maxSelectedCount = selectedArray.reduce((a, b) => Math.max(a, b), 0);
-        const maxCountForScale = hideAllDataParam.value ? maxSelectedCount : maxAllCount;
-        const selected = css.hsla(120, 40, 50, 1);
-        const all = css.hsla(200, 40, 70, 1);
-
-        return (
-            <div className={css.vbox(10).fillWidth}>
-                <div
-                    className={css.relative.fillWidth.height(100).bord2(0, 0, 40).hbox0.overflowHidden}
-                >
-                    {Array.from(this.histogramAllDataCounts).map((count, index) => {
-                        const selectedCount = this.histogramSelectedDataCounts[index] || 0;
-                        const allSize = this.histogramAllDataSizes[index] || 0;
-                        const selectedSize = this.histogramSelectedDataSizes[index] || 0;
-                        return (
-                            <div
-                                key={index}
-                                className={css.relative.fillHeight.fillWidth.filter("brightness(1.1)", "hover")}
-                                title={`${formatNumber(selectedCount)} filtered (${formatNumber(selectedSize)}B)\n${formatNumber(count)} all (${formatNumber(allSize)}B)\n${formatVeryNiceDateTime(this.histogramStartTime + index * this.histogramBucketTime)}`}
-                            >
-                                {count > 0 && !hideAllDataParam.value && <div
-                                    className={
-                                        all.absolute.bottom(0).left(0).right(1)
-                                            .transition("height 200ms")
-                                            .height(`calc(max(3px, ${(count / maxCountForScale) * 100}%))`)
-                                    }
-                                />}
-
-                                {selectedCount > 0 && <div
-                                    className={
-                                        selected.absolute.bottom(0).left(0).right(1)
-                                            .transition("height 200ms")
-                                            .height(`calc(max(3px, ${(selectedCount / maxCountForScale) * 100}%))`)
-                                    }
-                                />}
-                            </div>
-                        );
-                    })}
-                </div>
-                <div className={css.hbox(20).fontSize(12)}>
-                    {!hideAllDataParam.value && (
-                        <div className={css.hbox(5)}>
-                            <div className={all.size(16, 16)} />
-                            <span>All data (max {formatNumber(maxAllCount)} | {formatNumber(countsArray.reduce((a, b) => a + b, 0))} | {formatNumber(this.allSize)}B)</span>
-                        </div>
-                    )}
-                    <div className={css.hbox(5)}>
-                        <div className={selected.size(16, 16)} />
-                        <span>Filtered data (max {formatNumber(maxSelectedCount)} | {formatNumber(selectedArray.reduce((a, b) => a + b, 0))} | {formatNumber(this.matchedSize)}B)</span>
-                    </div>
-                    <InputLabelURL
-                        label="Only show filtered data"
-                        checkbox
-                        url={hideAllDataParam}
-                        flavor="small"
-                    />
-                </div>
-            </div>
-        );
-    }
-
-    private getOutdatedInfo(): string[] {
-        const currentTimeRange = getTimeRange();
-        const currentFilterString = filterParam.value;
-        const currentArchivePaths = this.props.fastArchives().map(archive => archive.rootPath);
-
-        // Get stored sync parameters
-        const storedParams = this.state.currentSyncParams;
-
-        // If we have no stored params yet, not outdated (first run or loading)
-        if (!storedParams) {
-            return [];
-        }
-
-        let warnings: string[] = [];
-
-        // Check for parameter differences
-        if (storedParams.filterString !== currentFilterString) {
-            warnings.push("filter");
-        }
-        if (storedParams.startTime !== currentTimeRange.startTime) {
-            warnings.push("start time");
-        }
-        if (storedParams.endTime !== currentTimeRange.endTime) {
-            warnings.push("end time");
-        }
-
-        // Check for missing and extra archives
-        const currentArchivePathsSet = new Set(currentArchivePaths);
-        const storedArchivePathsSet = new Set(storedParams.fastArchivePaths);
-
-        for (let storedPath of storedParams.fastArchivePaths) {
-            if (!currentArchivePathsSet.has(storedPath)) {
-                warnings.push(`Missing ${storedPath}`);
-            }
-        }
-        for (let currentPath of currentArchivePaths) {
-            if (!storedArchivePathsSet.has(currentPath)) {
-                warnings.push(`Extra ${currentPath}`);
-            }
-        }
-
-        return warnings;
-    }
-
-    public synchronizeDataThrottled = throttleFunction(500, () => {
-        Querysub.onCommitFinished(() => {
-            logErrors(this.synchronizeData());
-        });
-    });
-    render() {
-        if (throttleRender({ key: "FastArchiveViewer", frameDelay: 30 })) return undefined;
-
-        let totalFileCount = 0;
-        let totalBackblazeByteCount = 0;
-        let totalLocalByteCount = 0;
-        let totalLocalFileCount = 0;
-
-        let readLocalTimes: number[] = [];
-
-
-        for (let fileMetadata of this.state.fileMetadata) {
-            if (!fileMetadata) continue;
-            totalFileCount += fileMetadata.files.length;
-            let backblazeFiles = fileMetadata.files.filter(file => !file.nodeId);
-            let localFiles = fileMetadata.files.filter(file => file.nodeId);
-            if (localFiles.length > 0) {
-                readLocalTimes.push(fileMetadata.createTime || 0);
-            }
-            totalBackblazeByteCount += backblazeFiles.reduce((acc, file) => acc + file.size, 0);
-            totalLocalByteCount += localFiles.reduce((acc, file) => acc + file.size, 0);
-            totalLocalFileCount += localFiles.length;
-        }
-
-        const infoDisplay = (hue: number) => css.pad2(12).bord2(hue, 50, 50).hsl(hue, 50, 95).colorhsl(hue, 50, 40);
-
-
-
-        return (
-            <div className={css.vbox(20).pad2(20).fillBoth}>
-                <div className={css.hbox(20).fillWidth}>
-                    <TimeRangeSelector />
-                    <div className={infoDisplay(280)}>
-                        TODO: Filtering for machineId / threadId, if we find the log loading is too slow
-                    </div>
-                </div>
-
-                <div className={css.vbox(12).fillWidth.paddingBottom(20)}>
-                    <InputLabelURL
-                        label={
-                            <div className={css.hbox(10)}>
-                                <Button onClick={() => {
-                                    void this.synchronizeDataThrottled();
-                                }}>
-                                    Run
-                                </Button>
-                                <span>Filter</span>
-                            </div>
-                        }
-                        url={filterParam}
-                        hot
-                        flavor="large"
-                        fillWidth
-                        onKeyUp={this.synchronizeDataThrottled}
-                        ref2={() => {
-                            if (this.props.runOnLoad) {
-                                void this.synchronizeDataThrottled();
-                            }
-                        }}
-                        noEnterKeyBlur
-                        placeholder="Filter terms, ex x | y & z"
-                    />
-                    <InputLabelURL
-                        label="Case Insensitive"
-                        checkbox
-                        url={caseInsensitiveParam}
-                        onChange={() => {
-                            void this.synchronizeDataThrottled();
-                        }}
-                        flavor="small"
-                    />
-                    <div className={css.vbox(10)}>
-                        {this.state.runCount > 0 && (() => {
-                            const outdatedWarnings = this.getOutdatedInfo();
-                            const errorCount = this.fileErrors.length;
-
-                            // Group errors by file path
-                            const errorsByFile = new Map<string, number>();
-                            for (let errorInfo of this.fileErrors) {
-                                const path = errorInfo.file.path;
-                                errorsByFile.set(path, (errorsByFile.get(path) || 0) + 1);
-                            }
-
-                            const errorTooltip = Array.from(errorsByFile.entries())
-                                .map(([path, count]) => `${path}: ${count} errors`)
-                                .join("\n");
-
-                            return (
-                                <div className={css.hbox(10).wrap}>
-                                    <div
-                                        className={infoDisplay(120)}
-                                        title={this.state.fileMetadata.map(x => x?.files || []).flat().map(x =>
-                                            `${x.path} (${formatNumber(x.size)})`
-                                        ).join("\n")}
-                                    >
-                                        File count: {formatNumber(totalFileCount)}{errorCount > 0 && ` (${errorCount} erred)`}, Backblaze size: {formatNumber(totalBackblazeByteCount)}B (compressed), Disk size: {formatNumber(totalLocalByteCount)}B (uncompressed)
-                                    </div>
-                                    {errorCount > 0 && (
-                                        <div
-                                            className={infoDisplay(45)}
-                                            title={errorTooltip}
-                                        >
-                                            {errorCount} file{errorCount > 1 ? "s" : ""} failed to load
-                                        </div>
-                                    )}
-                                    {outdatedWarnings.length > 0 && (
-                                        <div
-                                            className={infoDisplay(30).button}
-                                            onClick={() => {
-                                                void this.synchronizeDataThrottled();
-                                            }}
-                                            title={outdatedWarnings.join(", ")}
-                                        >
-                                            <div className={css.vbox(4)}>
-                                                <div className={css.boldStyle}>Search parameters outdated - Click to run updated search. Outdated:</div>
-                                                <div className={css.fontSize(12)}>
-                                                    {outdatedWarnings.join(" | ")}
-                                                </div>
-                                            </div>
-                                        </div>
-                                    )}
-                                </div>
-                            );
-                        })()}
-                        {this.state.runCount === 0 && (
-                            <div className={infoDisplay(200).button} onClick={() => {
-                                void this.synchronizeDataThrottled();
-                            }}>
-                                No data downloaded yet. Click here to download data.
-                            </div>
-                        )}
-                        {this.state.finished && (() => {
-
-                            const timeRange = getTimeRange();
-                            if (timeRange.endTime && readLocalTimes.length === 0) return null;
-                            const earliestTime = Math.min(...readLocalTimes);
-
-                            if (earliestTime >= timeRange.endTime) return null;
-
-                            return (
-                                <div
-                                    className={infoDisplay(0).button}
-                                    onClick={() => {
-                                        Querysub.commit(() => {
-                                            cacheBustParam.value = Date.now();
-                                        });
-                                        void this.synchronizeDataThrottled();
-                                    }}
-                                >
-                                    <div className={css.hbox(8).alignItems("center").wrap}>
-                                        <span>Future data has been read</span>
-                                        <span className={css.fontSize(18).boldStyle}>{formatTime(timeRange.endTime - earliestTime)} into the future</span>
-                                        <span>({formatDateTime(earliestTime)})</span>
-                                        <span>Clear here to load possibly missing data data.</span>
-                                    </div>
-                                </div>
-                            );
-                        })()}
-                        {!this.state.finished && <LoaderAurora />}
-                        {this.limitedScanCount > 0 && (
-                            <div className={infoDisplay(60).boldStyle.button}
-                                onClick={() => {
-                                    filterParam.value = LOG_LIMIT_FLAG;
-                                    void this.synchronizeDataThrottled();
-                                }}
-                            >
-                                Click here to see {formatNumber(this.limitedScanCount)} scanned logs were rate limited at a file level. This means lines might be missing. If this happens a lot, use pass {`{ [LOG_LINE_LIMIT_ID]: "INSERT RANDOM GUID HERE" }`} in the error/warn message to limit only the spamming line.
-                            </div>
-                        )}
-                        {this.limitedMatchCount > 0 && filterParam.value && (
-                            <div className={infoDisplay(0).boldStyle.button}
-                                onClick={() => {
-                                    filterParam.value += " & " + LOG_LIMIT_FLAG;
-                                    void this.synchronizeDataThrottled();
-                                }}
-                            >
-                                Click here to see {formatNumber(this.limitedMatchCount)} matched logs were rate limited at a file level. This means lines might be missing. If this happens a lot, use pass {`{ [LOG_LINE_LIMIT_ID]: "INSERT RANDOM GUID HERE" }`} in the error/warn message to limit only the spamming line.
-                            </div>
-                        )}
-                        {this.state.pendingSyncInitializations > 0 && (
-                            <div className={infoDisplay(200)}>
-                                <strong>Pending sync initializations:</strong> {formatNumber(this.state.pendingSyncInitializations)}
-                            </div>
-                        )}
-                    </div>
-
-                    {this.state.error && (
-                        <div className={infoDisplay(0).whiteSpace("pre-wrap")}>
-                            <strong>Error:</strong> {this.state.error}
-                        </div>
-                    )}
-
-                    {this.renderProgressBars()}
-
-                    {this.renderHistogram()}
-
-                    {this.props.children}
-                </div>
-            </div>
-        );
-    }
-}
-
-class LoaderAurora extends qreact.Component {
-    render() {
-        return (
-            <>
-                <style>{`
-					.aurora-container-2 {
-						position: relative;
-						width: 800px;
-						height: 40px;
-						overflow: hidden;
-						background: linear-gradient(180deg, hsl(280, 30%, 12%), hsl(320, 35%, 15%));
-					}
-					.aurora-ribbon-2 {
-						position: absolute;
-						width: 1420px;
-						height: 40px;
-						filter: blur(12px);
-						opacity: 0.9;
-					}
-					
-					/* Different gradient patterns for variety */
-					.aurora-ribbon-2.gradient1 {
-						background: linear-gradient(90deg,
-							hsla(160, 80%, 50%, 0) 0%,
-							hsla(160, 80%, 50%, 0.7) 30%,
-							hsla(280, 80%, 60%, 0.7) 60%,
-							hsla(40, 80%, 55%, 0) 100%);
-					}
-					.aurora-ribbon-2.gradient2 {
-						background: linear-gradient(90deg,
-							hsla(200, 70%, 60%, 0) 0%,
-							hsla(200, 70%, 60%, 0.6) 40%,
-							hsla(320, 70%, 55%, 0.6) 70%,
-							hsla(60, 70%, 50%, 0) 100%);
-					}
-					.aurora-ribbon-2.gradient3 {
-						background: linear-gradient(90deg,
-							hsla(300, 80%, 65%, 0) 0%,
-							hsla(300, 80%, 65%, 0.8) 25%,
-							hsla(180, 80%, 55%, 0.8) 65%,
-							hsla(240, 80%, 60%, 0) 100%);
-					}
-					
-					/* Left to right movement */
-					.aurora-ribbon-2.r1 {
-						top: 0;
-						transform: rotate(-10deg);
-						animation: auroraSlideLeft-2 3s ease-in-out infinite, auroraRotate1-2 6s linear infinite;
-						opacity: 0.7;
-					}
-					.aurora-ribbon-2.r2 {
-						top: 20px;
-						transform: rotate(12deg);
-						animation: auroraSlideRight-2 2.5s ease-in-out infinite, auroraRotate2-2 8s linear infinite reverse;
-						opacity: 0.8;
-					}
-					.aurora-ribbon-2.r3 {
-						top: 40px;
-						transform: rotate(18deg);
-						animation: auroraSlideDiagonal1-2 4s ease-in-out infinite;
-						opacity: 0.6;
-					}
-					.aurora-ribbon-2.r4 {
-						top: -10px;
-						transform: rotate(-15deg);
-						animation: auroraSlideLeft-2 3.5s ease-in-out infinite, auroraRotate3-2 10s linear infinite;
-						opacity: 0.5;
-					}
-					.aurora-ribbon-2.r5 {
-						top: 30px;
-						transform: rotate(8deg);
-						animation: auroraSlideDiagonal2-2 2.8s ease-in-out infinite;
-						opacity: 0.9;
-					}
-					.aurora-ribbon-2.r6 {
-						top: 10px;
-						transform: rotate(-5deg);
-						animation: auroraSlideRight-2 4.2s ease-in-out infinite, auroraRotate1-2 7s linear infinite;
-						opacity: 0.4;
-					}
-					.aurora-ribbon-2.r7 {
-						top: 50px;
-						transform: rotate(22deg);
-						animation: auroraSlideLeft-2 2.2s ease-in-out infinite;
-						opacity: 0.8;
-					}
-					.aurora-ribbon-2.r8 {
-						top: 15px;
-						transform: rotate(-8deg);
-						animation: auroraSlideDiagonal3-2 3.7s ease-in-out infinite, auroraRotate2-2 9s linear infinite;
-						opacity: 0.6;
-					}
-					.aurora-ribbon-2.r9 {
-						top: 35px;
-						transform: rotate(15deg);
-						animation: auroraSlideRight-2 3.1s ease-in-out infinite, auroraRotate1-2 8.5s linear infinite;
-						opacity: 0.7;
-					}
-					.aurora-ribbon-2.r10 {
-						top: -5px;
-						transform: rotate(-12deg);
-						animation: auroraSlideDiagonal1-2 3.8s ease-in-out infinite;
-						opacity: 0.5;
-					}
-					.aurora-ribbon-2.r11 {
-						top: 25px;
-						transform: rotate(6deg);
-						animation: auroraSlideLeft-2 2.9s ease-in-out infinite, auroraRotate3-2 11s linear infinite;
-						opacity: 0.8;
-					}
-					.aurora-ribbon-2.r12 {
-						top: 45px;
-						transform: rotate(-20deg);
-						animation: auroraSlideDiagonal2-2 4.1s ease-in-out infinite, auroraRotate2-2 7.5s linear infinite reverse;
-						opacity: 0.6;
-					}
-					.aurora-ribbon-2.r13 {
-						top: 5px;
-						transform: rotate(10deg);
-						animation: auroraSlideRight-2 2.7s ease-in-out infinite;
-						opacity: 0.9;
-					}
-					.aurora-ribbon-2.r14 {
-						top: 38px;
-						transform: rotate(-7deg);
-						animation: auroraSlideDiagonal3-2 3.3s ease-in-out infinite, auroraRotate1-2 9.5s linear infinite;
-						opacity: 0.4;
-					}
-					.aurora-ribbon-2.r15 {
-						top: 18px;
-						transform: rotate(14deg);
-						animation: auroraSlideLeft-2 4.3s ease-in-out infinite, auroraRotate3-2 8.2s linear infinite reverse;
-						opacity: 0.7;
-					}
-					.aurora-ribbon-2.r16 {
-						top: 12px;
-						transform: rotate(-18deg);
-						animation: auroraSlideDiagonal1-2 2.6s ease-in-out infinite, auroraRotate2-2 10.5s linear infinite;
-						opacity: 0.8;
-					}
-					
-					/* Left to right slide */
-					@keyframes auroraSlideLeft-2 {
-						0% { left: -920px; }
-						100% { left: 900px; }
-					}
-					
-					/* Right to left slide */
-					@keyframes auroraSlideRight-2 {
-						0% { left: 900px; }
-						100% { left: -920px; }
-					}
-					
-					/* Diagonal movements */
-					@keyframes auroraSlideDiagonal1-2 {
-						0% { left: -920px; top: 40px; }
-						50% { left: 400px; top: -5px; }
-						100% { left: 900px; top: 40px; }
-					}
-					
-					@keyframes auroraSlideDiagonal2-2 {
-						0% { left: 900px; top: 30px; }
-						50% { left: 400px; top: 45px; }
-						100% { left: -920px; top: 30px; }
-					}
-					
-					@keyframes auroraSlideDiagonal3-2 {
-						0% { left: -920px; top: 15px; }
-						30% { left: -200px; top: 5px; }
-						70% { left: 600px; top: 25px; }
-						100% { left: 900px; top: 15px; }
-					}
-					
-					/* Rotation animations */
-					@keyframes auroraRotate1-2 {
-						0% { transform: rotate(-10deg); }
-						25% { transform: rotate(-5deg); }
-						50% { transform: rotate(-15deg); }
-						75% { transform: rotate(-8deg); }
-						100% { transform: rotate(-10deg); }
-					}
-					
-					@keyframes auroraRotate2-2 {
-						0% { transform: rotate(12deg); }
-						33% { transform: rotate(18deg); }
-						66% { transform: rotate(8deg); }
-						100% { transform: rotate(12deg); }
-					}
-					
-					@keyframes auroraRotate3-2 {
-						0% { transform: rotate(-15deg); }
-						50% { transform: rotate(-20deg); }
-						100% { transform: rotate(-15deg); }
-					}
-				`}</style>
-                <div className="aurora-container-2">
-                    <div className="aurora-ribbon-2 r1 gradient1"></div>
-                    <div className="aurora-ribbon-2 r2 gradient2"></div>
-                    <div className="aurora-ribbon-2 r3 gradient3"></div>
-                    <div className="aurora-ribbon-2 r4 gradient1"></div>
-                    <div className="aurora-ribbon-2 r5 gradient3"></div>
-                    <div className="aurora-ribbon-2 r6 gradient2"></div>
-                    <div className="aurora-ribbon-2 r7 gradient1"></div>
-                    <div className="aurora-ribbon-2 r8 gradient3"></div>
-                    <div className="aurora-ribbon-2 r9 gradient2"></div>
-                    <div className="aurora-ribbon-2 r10 gradient1"></div>
-                    <div className="aurora-ribbon-2 r11 gradient3"></div>
-                    <div className="aurora-ribbon-2 r12 gradient2"></div>
-                    <div className="aurora-ribbon-2 r13 gradient1"></div>
-                    <div className="aurora-ribbon-2 r14 gradient3"></div>
-                    <div className="aurora-ribbon-2 r15 gradient2"></div>
-                    <div className="aurora-ribbon-2 r16 gradient1"></div>
-                </div>
-            </>
-        );
-    }
-}