querysub 0.326.0 → 0.328.0

package/src/diagnostics/logs/FastArchiveViewer.tsx

@@ -5,7 +5,7 @@ import { DatumStats, FastArchiveAppendable } from "./FastArchiveAppendable";
 import { Querysub } from "../../4-querysub/Querysub";
 import { URLParam } from "../../library-components/URLParam";
 import { TimeRangeSelector, getTimeRange } from "./TimeRangeSelector";
-import { formatNumber, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
+import { formatNumber, formatTime, formatVeryNiceDateTime, formatDateTime } from "socket-function/src/formatting/format";
 import { list, sort, throttleFunction, timeInHour } from "socket-function/src/misc";
 import { css } from "typesafecss";
 import { logErrors } from "../../errors";
@@ -15,22 +15,24 @@ import { ButtonSelector } from "../../library-components/ButtonSelector";
 import { Button } from "../../library-components/Button";
 import { lazy } from "socket-function/src/caching";
 import { LOG_LIMIT_FLAG } from "./diskLogger";
-import { canHaveChildren } from "socket-function/src/types";
+import { MaybePromise, canHaveChildren } from "socket-function/src/types";
 import { niceParse } from "../../niceStringify";
 import { FileMetadata } from "./FastArchiveController";
+import { throttleRender } from "../../functional/throttleRender";
 
 const RENDER_INTERVAL = 1000;
 
 const HISTOGRAM_RERENDER_INTERVAL = 10000;
 
 export const filterParam = new URLParam("filter", "");
-const cacheBustParam = new URLParam("cacheBust", 0);
+export const cacheBustParam = new URLParam("cacheBust", 0);
 const caseInsensitiveParam = new URLParam("caseInsensitive", false);
-
+const hideAllDataParam = new URLParam("hideAllData", false);
 
 export class FastArchiveViewer<T> extends qreact.Component<{
     fastArchives: FastArchiveAppendable<T>[];
-    onStart: () => void;
+    runOnLoad?: boolean;
+    onStart: () => MaybePromise<void>;
     getWantData?: (file: FileMetadata) => Promise<((posStart: number, posEnd: number, data: Buffer) => boolean) | undefined>;
     onDatums: (source: FastArchiveAppendable<T>, datums: T[], metadata: FileMetadata) => void;
     // Called after onData
@@ -38,15 +40,24 @@ export class FastArchiveViewer<T> extends qreact.Component<{
     onFinish?: () => void;
 }> {
     state = t.state({
+        runCount: t.atomic<number>(0),
         // rootPath =>
         fileMetadata: t.atomic<({
             files: FileMetadata[];
             createTime?: number;
         } | undefined)[]>([]),
-        finished: t.atomic(false),
+        finished: t.atomic(true),
         error: t.atomic<string | undefined>(undefined),
         pendingSyncInitializations: t.atomic<number>(0),
 
+        // Current sync parameters - set before synchronizeData is called
+        currentSyncParams: t.atomic<{
+            filterString: string;
+            startTime: number;
+            endTime: number;
+            fastArchivePaths: string[];
+        } | undefined>(undefined),
+
         renderSeqNum: t.atomic<number>(0),
     });
 
@@ -108,6 +119,11 @@ export class FastArchiveViewer<T> extends qreact.Component<{
         this.latestSequenceNumber = this.currentSequenceNumber;
         const mySequenceNumber = this.currentSequenceNumber;
 
+        // Increment run count for each new run
+        Querysub.commit(() => {
+            this.state.runCount++;
+        });
+
         // Helper function to check if this sequence number is still the latest
         const isLatestSync = () => mySequenceNumber === this.latestSequenceNumber;
 
@@ -131,6 +147,15 @@ export class FastArchiveViewer<T> extends qreact.Component<{
 
         let scannedValueCount = 0;
 
+        // Store current sync parameters BEFORE calling synchronizeData
+        ifLatest(() => {
+            this.state.currentSyncParams = {
+                filterString,
+                startTime: timeRange.startTime,
+                endTime: timeRange.endTime,
+                fastArchivePaths: fastArchives.map(archive => archive.rootPath),
+            };
+        });
 
         ifLatest(() => {
             this.state.error = undefined;
@@ -150,7 +175,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
 
             this.histogramStartTime = timeRange.startTime - this.histogramBucketTime * 2;
             this.histogramEndTime = timeRange.endTime + this.histogramBucketTime * 2;
-            const bucketCount = Math.ceil((this.histogramEndTime - this.histogramStartTime) / this.histogramBucketTime);
+            const bucketCount = clamp(Math.ceil((this.histogramEndTime - this.histogramStartTime) / this.histogramBucketTime), 1, 10000);
 
             this.histogramAllDataCounts = new Float64Array(bucketCount);
             this.histogramSelectedDataCounts = new Float64Array(bucketCount);
@@ -181,9 +206,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
             }
         };
         try {
-            ifLatest(() => {
-                onStart();
-            });
+            await onStart();
 
             const caseInsensitive = Querysub.fastRead(() => caseInsensitiveParam.value);
             let caseInsensitiveMapping = new Uint8Array(256);
@@ -408,6 +431,39 @@ export class FastArchiveViewer<T> extends qreact.Component<{
         }
         sort(progressEntries, x => x[1].initialTime);
 
+        // Group progress bars by the text after the pipe sign
+        const groupedProgress = new Map<string, {
+            displayName: string;
+            entries: Array<[string, { section: string, value: number, max: number, initialTime: number, lastSetTime: number }]>;
+            totalValue: number;
+            totalMax: number;
+            earliestTime: number;
+            latestTime: number;
+        }>();
+
+        for (const [key, progress] of progressEntries) {
+            const pipeIndex = progress.section.indexOf("|");
+            const displayName = pipeIndex >= 0 ? progress.section.substring(pipeIndex + 1) : progress.section;
+
+            if (!groupedProgress.has(displayName)) {
+                groupedProgress.set(displayName, {
+                    displayName,
+                    entries: [],
+                    totalValue: 0,
+                    totalMax: 0,
+                    earliestTime: progress.initialTime,
+                    latestTime: progress.lastSetTime,
+                });
+            }
+
+            const group = groupedProgress.get(displayName)!;
+            group.entries.push([key, progress]);
+            group.totalValue += progress.value;
+            group.totalMax += progress.max;
+            group.earliestTime = Math.min(group.earliestTime, progress.initialTime);
+            group.latestTime = Math.max(group.latestTime, progress.lastSetTime);
+        }
+
         return (
             <div className={css.hbox(10).wrap.alignItems("start").fillWidth}>
                 {!this.state.finished && (
@@ -420,33 +476,41 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                         Cancel
                     </div>
                 )}
-                {progressEntries.map(([key, progress]) => {
-                    const fraction = clamp(progress.max > 0 ? (progress.value / progress.max) * 1 : 1, 0, 1);
+                {Array.from(groupedProgress.values()).map((group) => {
+                    const fraction = clamp(group.totalMax > 0 ? (group.totalValue / group.totalMax) * 1 : 1, 0, 1);
                     let progressTime = 0;
-                    if (progress.value >= progress.max) {
-                        progressTime = progress.lastSetTime;
+                    if (group.totalValue >= group.totalMax) {
+                        progressTime = group.latestTime;
                     } else {
                         progressTime = getNowTime();
                     }
-                    const elapsedTime = progressTime - progress.initialTime;
+                    const elapsedTime = progressTime - group.earliestTime;
+
+                    // Create tooltip showing breakdown by original prefix
+                    const tooltipContent = group.entries.map(([key, progress]) => {
+                        const pipeIndex = progress.section.indexOf("|");
+                        const prefix = pipeIndex >= 0 ? progress.section.substring(0, pipeIndex) : key;
+                        return `${prefix}: ${formatNumber(progress.value)}/${formatNumber(progress.max)}`;
+                    }).join("\n");
 
                     return (
-                        <div key={key} className={
+                        <div key={group.displayName} className={
                             // NOTE: There is no point making this proportionally to the time, because we stream the data, so the time overlaps, making everything take most of the time.
                             css.vbox(4)
-                        }>
+                        }
+                            title={tooltipContent}
+                        >
                             <div className={css.vbox(2)}>
                                 <div className={css.fontSize(14).colorhsl(0, 0, 20)}>
-                                    {progress.section}
+                                    {group.displayName}
                                 </div>
                                 <div className={css.fontSize(12).colorhsl(0, 0, 40).width(150)}>
-                                    {formatNumber(progress.value)} {progress.value < progress.max && `/ ${formatNumber(progress.max)}`} ({formatTime(elapsedTime)})
+                                    {formatNumber(group.totalValue)} {group.totalValue < group.totalMax && `/ ${formatNumber(group.totalMax)}`} ({formatTime(elapsedTime)})
                                 </div>
                             </div>
                             <div className={css.fillWidth.height(8).bord2(200, 20, 80).hsl(200, 10, 95)}>
                                 <div
-                                    className={css.height(8).hsl(200, 50, 70).transition("width 200ms")}
-                                    style={{ width: `${fraction * 100}%` }}
+                                    className={css.height(8).hsl(200, 50, 70).width(`${fraction * 100}%`).transition("width 200ms")}
                                 />
                             </div>
                         </div>
@@ -466,6 +530,8 @@ export class FastArchiveViewer<T> extends qreact.Component<{
         let countsArray = Array.from(this.histogramAllDataCounts);
         let selectedArray = Array.from(this.histogramSelectedDataCounts);
         const maxAllCount = countsArray.reduce((a, b) => Math.max(a, b), 0);
+        const maxSelectedCount = selectedArray.reduce((a, b) => Math.max(a, b), 0);
+        const maxCountForScale = hideAllDataParam.value ? maxSelectedCount : maxAllCount;
         const selected = css.hsla(120, 40, 50, 1);
         const all = css.hsla(200, 40, 70, 1);
 
@@ -484,17 +550,19 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                                 className={css.relative.fillHeight.fillWidth.filter("brightness(1.1)", "hover")}
                                 title={`${formatNumber(selectedCount)} filtered (${formatNumber(selectedSize)}B)\n${formatNumber(count)} all (${formatNumber(allSize)}B)\n${formatVeryNiceDateTime(this.histogramStartTime + index * this.histogramBucketTime)}`}
                             >
-                                {count > 0 && <div
+                                {count > 0 && !hideAllDataParam.value && <div
                                     className={
                                         all.absolute.bottom(0).left(0).right(1)
-                                            .height(`calc(max(3px, ${(count / maxAllCount) * 100}%))`)
+                                            .transition("height 200ms")
+                                            .height(`calc(max(3px, ${(count / maxCountForScale) * 100}%))`)
                                     }
                                 />}
 
                                 {selectedCount > 0 && <div
                                     className={
                                         selected.absolute.bottom(0).left(0).right(1)
-                                            .height(`calc(max(3px, ${(selectedCount / maxAllCount) * 100}%))`)
+                                            .transition("height 200ms")
+                                            .height(`calc(max(3px, ${(selectedCount / maxCountForScale) * 100}%))`)
                                     }
                                 />}
                             </div>
@@ -502,19 +570,71 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                     })}
                 </div>
                 <div className={css.hbox(20).fontSize(12)}>
-                    <div className={css.hbox(5)}>
-                        <div className={all.size(16, 16)} />
-                        <span>All data ({formatNumber(countsArray.reduce((a, b) => a + b, 0))} | {formatNumber(this.allSize)}B)</span>
-                    </div>
+                    {!hideAllDataParam.value && (
+                        <div className={css.hbox(5)}>
+                            <div className={all.size(16, 16)} />
+                            <span>All data (max {formatNumber(maxAllCount)} | {formatNumber(countsArray.reduce((a, b) => a + b, 0))} | {formatNumber(this.allSize)}B)</span>
+                        </div>
+                    )}
                     <div className={css.hbox(5)}>
                         <div className={selected.size(16, 16)} />
-                        <span>Filtered data ({formatNumber(selectedArray.reduce((a, b) => a + b, 0))} | {formatNumber(this.matchedSize)}B)</span>
+                        <span>Filtered data (max {formatNumber(maxSelectedCount)} | {formatNumber(selectedArray.reduce((a, b) => a + b, 0))} | {formatNumber(this.matchedSize)}B)</span>
                     </div>
+                    <InputLabelURL
+                        label="Only show filtered data"
+                        checkbox
+                        url={hideAllDataParam}
+                        flavor="small"
+                    />
                 </div>
             </div>
         );
     }
 
+    private getOutdatedInfo(): string[] {
+        const currentTimeRange = getTimeRange();
+        const currentFilterString = filterParam.value;
+        const currentArchivePaths = this.props.fastArchives.map(archive => archive.rootPath);
+
+        // Get stored sync parameters
+        const storedParams = this.state.currentSyncParams;
+
+        // If we have no stored params yet, not outdated (first run or loading)
+        if (!storedParams) {
+            return [];
+        }
+
+        let warnings: string[] = [];
+
+        // Check for parameter differences
+        if (storedParams.filterString !== currentFilterString) {
+            warnings.push("filter");
+        }
+        if (storedParams.startTime !== currentTimeRange.startTime) {
+            warnings.push("start time");
+        }
+        if (storedParams.endTime !== currentTimeRange.endTime) {
+            warnings.push("end time");
+        }
+
+        // Check for missing and extra archives
+        const currentArchivePathsSet = new Set(currentArchivePaths);
+        const storedArchivePathsSet = new Set(storedParams.fastArchivePaths);
+
+        for (let storedPath of storedParams.fastArchivePaths) {
+            if (!currentArchivePathsSet.has(storedPath)) {
+                warnings.push(`Missing ${storedPath}`);
+            }
+        }
+        for (let currentPath of currentArchivePaths) {
+            if (!storedArchivePathsSet.has(currentPath)) {
+                warnings.push(`Extra ${currentPath}`);
+            }
+        }
+
+        return warnings;
+    }
+
     public handleDownload = throttleFunction(500, () => {
         console.log("handleDownload");
         Querysub.onCommitFinished(() => {
@@ -522,6 +642,8 @@ export class FastArchiveViewer<T> extends qreact.Component<{
         });
     });
     render() {
+        if (throttleRender({ key: "FastArchiveViewer", frameDelay: 30 })) return undefined;
+
         let totalFileCount = 0;
         let totalBackblazeByteCount = 0;
         let totalLocalByteCount = 0;
@@ -545,6 +667,8 @@ export class FastArchiveViewer<T> extends qreact.Component<{
 
         const infoDisplay = (hue: number) => css.pad2(12).bord2(hue, 50, 50).hsl(hue, 50, 95).colorhsl(hue, 50, 40);
 
+
+
         return (
             <div className={css.vbox(20).pad2(20).fillBoth}>
                 <div className={css.hbox(20).fillWidth}>
@@ -571,6 +695,11 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                         flavor="large"
                         fillWidth
                         onKeyUp={this.handleDownload}
+                        ref2={() => {
+                            if (this.props.runOnLoad) {
+                                void this.handleDownload();
+                            }
+                        }}
                         noEnterKeyBlur
                         placeholder="Filter terms, ex x | y & z"
                     />
@@ -584,28 +713,71 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                         flavor="small"
                     />
                     <div className={css.vbox(10)}>
-                        {this.state.fileMetadata
-                            && (
+                        {this.state.runCount > 0 && (() => {
+                            const outdatedWarnings = this.getOutdatedInfo();
+                            return (
+                                <div className={css.hbox(10).wrap}>
+                                    <div
+                                        className={infoDisplay(120)}
+                                        title={this.state.fileMetadata.map(x => x?.files || []).flat().map(x =>
+                                            `${x.path} (${formatNumber(x.size)})`
+                                        ).join("\n")}
+                                    >
+                                        File count: {formatNumber(totalFileCount)}, Backblaze size: {formatNumber(totalBackblazeByteCount)}B (compressed), Disk size: {formatNumber(totalLocalByteCount)}B (uncompressed)
+                                    </div>
+                                    {outdatedWarnings.length > 0 && (
+                                        <div
+                                            className={infoDisplay(30).button}
+                                            onClick={() => {
+                                                void this.handleDownload();
+                                            }}
+                                            title={outdatedWarnings.join(", ")}
+                                        >
+                                            <div className={css.vbox(4)}>
+                                                <div className={css.boldStyle}>Search parameters outdated - Click to run updated search. Outdated:</div>
+                                                <div className={css.fontSize(12)}>
+                                                    {outdatedWarnings.join(" | ")}
+                                                </div>
+                                            </div>
+                                        </div>
+                                    )}
+                                </div>
+                            );
+                        })()}
+                        {this.state.runCount === 0 && (
+                            <div className={infoDisplay(200).button} onClick={() => {
+                                void this.handleDownload();
+                            }}>
+                                No data downloaded yet. Click here to download data.
+                            </div>
+                        )}
+                        {this.state.finished && (() => {
+                            if (readLocalTimes.length === 0) return null;
+
+                            const timeRange = getTimeRange();
+                            const earliestTime = Math.min(...readLocalTimes);
+
+                            if (earliestTime >= timeRange.endTime) return null;
+
+                            return (
                                 <div
-                                    className={infoDisplay(120)}
-                                    title={this.state.fileMetadata.map(x => x?.files || []).flat().map(x =>
-                                        `${x.path} (${formatNumber(x.size)})`
-                                    ).join("\n")}
+                                    className={infoDisplay(0).button}
+                                    onClick={() => {
+                                        Querysub.commit(() => {
+                                            cacheBustParam.value = Date.now();
+                                        });
+                                        void this.handleDownload();
+                                    }}
                                 >
-                                    File count: {formatNumber(totalFileCount)}, Backblaze size: {formatNumber(totalBackblazeByteCount)}B (compressed), Disk size: {formatNumber(totalLocalByteCount)}B (uncompressed)
+                                    <div className={css.hbox(8).alignItems("center").wrap}>
+                                        <span>Future data has been read</span>
+                                        <span className={css.fontSize(18).boldStyle}>{formatTime(timeRange.endTime - earliestTime)} into the future</span>
+                                        <span>({formatDateTime(earliestTime)})</span>
+                                        <span>Clear here to load possibly missing data data.</span>
+                                    </div>
                                 </div>
-                            )}
-                        {this.state.finished && <div
-                            className={infoDisplay(60).button}
-                            onClick={() => {
-                                Querysub.commit(() => {
-                                    cacheBustParam.value = Date.now();
-                                });
-                                void this.handleDownload();
-                            }}
-                        >
-                            Snapshot from dates: {readLocalTimes.map(x => formatVeryNiceDateTime(x)).join(" | ")}. Clear here to reload data.
-                        </div>}
+                            );
+                        })()}
                         {!this.state.finished && <LoaderAurora />}
                         {this.limitedScanCount > 0 && (
                             <div className={infoDisplay(60).boldStyle.button}
@@ -614,7 +786,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                                     void this.handleDownload();
                                 }}
                             >
-                                Click here to see {formatNumber(this.limitedScanCount)} scanned logs were rate limited at a file level. We do not track how many were ignored, there might be infinite missing logs.
+                                Click here to see {formatNumber(this.limitedScanCount)} scanned logs were rate limited at a file level. This means lines might be missing. If this happens a lot, use pass {`{ [LOG_LINE_LIMIT_ID]: "INSERT RANDOM GUID HERE" }`} in the error/warn message to limit only the spamming line.
                             </div>
                         )}
                         {this.limitedMatchCount > 0 && filterParam.value && (
@@ -624,7 +796,7 @@ export class FastArchiveViewer<T> extends qreact.Component<{
                                     void this.handleDownload();
                                 }}
                             >
-                                Click here to see {formatNumber(this.limitedMatchCount)} matched logs were rate limited at a file level. We do not track how many were ignored, there might be infinite missing logs.
+                                Click here to see {formatNumber(this.limitedMatchCount)} matched logs were rate limited at a file level. This means lines might be missing. If this happens a lot, use pass {`{ [LOG_LINE_LIMIT_ID]: "INSERT RANDOM GUID HERE" }`} in the error/warn message to limit only the spamming line.
                             </div>
                         )}
                         {this.state.pendingSyncInitializations > 0 && (
@@ -651,7 +823,6 @@ export class FastArchiveViewer<T> extends qreact.Component<{
     }
 }
 
-
 class LoaderAurora extends qreact.Component {
     render() {
         return (