querysub 0.373.0 → 0.375.0

package/src/diagnostics/logs/LogViewer2.tsx

@@ -1,552 +0,0 @@
-module.hotreload = true;
-import { qreact } from "../../4-dom/qreact";
-import { css } from "../../4-dom/css";
-import { URLParam } from "../../library-components/URLParam";
-import { InputLabelURL } from "../../library-components/InputLabel";
-import { Button } from "../../library-components/Button";
-import { formatDateTime, formatDateTimeDetailed, formatNiceDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
-import { TimeRangeSelector, endTimeParam, getTimeRange, startTimeParam } from "./TimeRangeSelector";
-import { FastArchiveAppendable } from "./FastArchiveAppendable";
-import { t } from "../../2-proxy/schema2";
-import { measureFnc } from "socket-function/src/profiling/measure";
-import { logErrors } from "../../errors";
-import { batchFunction, delay, runInSerial } from "socket-function/src/batching";
-import { Querysub } from "../../4-querysub/QuerysubController";
-import { sort, timeInDay, timeInHour } from "socket-function/src/misc";
-import { FastArchiveViewer, cacheBustParam, filterParam, filterParam2 } from "./FastArchiveViewer";
-import { LogDatum, getLoggers, LOG_LIMIT_FLAG, LOG_LINE_LIMIT_FLAG } from "./diskLogger";
-import { ColumnType, Table, TableType } from "../../5-diagnostics/Table";
-import { formatDateJSX } from "../../misc/formatJSX";
-import { InputPicker } from "../../library-components/InputPicker";
-import { JSXFormatter } from "../../5-diagnostics/GenericFormat";
-import { atomic } from "../../2-proxy/PathValueProxyWatcher";
-import { ObjectDisplay } from "./ObjectDisplay";
-import { endTime } from "../misc-pages/archiveViewerShared";
-import { ErrorSuppressionUI } from "./errorNotifications/ErrorSuppressionUI";
-import { FileMetadata } from "./FastArchiveController";
-import { RecentErrorsController, SuppressionListController, getSuppressEntryChecker, getSuppressionFull } from "./errorNotifications/ErrorNotificationController";
-import { SocketFunction } from "socket-function/SocketFunction";
-import { throttleRender } from "../../functional/throttleRender";
-import { isNode } from "typesafecss";
-import { createLogViewerExtractField } from "./logViewerExtractField";
-import { isPublic } from "../../config";
-
-const RENDER_INTERVAL = 1000;
-
-export const testLogs = new FastArchiveAppendable("test/");
-
-const enableLogsURL = new URLParam("enableLogs", false);
-const enableInfosURL = new URLParam("enableInfos", true);
-const enableWarningsURL = new URLParam("enableWarnings", true);
-const enableErrorsURL = new URLParam("enableErrors", true);
-const selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
-const useRelativeTimeURL = new URLParam("useRelativeTime", true);
-
-export const errorNotifyToggleURL = new URLParam("errorNotifyToggle", false);
-const forceGetPublicURL = new URLParam("forceGetPublic", false);
-
-const defaultSelectedFields = {
-    param0: true,
-    time: true,
-    __NAME__: true,
-
-    //__machineId: true,
-    __threadId: true,
-    __entry: true,
-    //__DIR__: true,
-};
-
-export class LogViewer2 extends qreact.Component {
-    state = t.state({
-        datumsSeqNum: t.atomic<number>(0),
-        showCountByFile: t.boolean,
-        showSizeByFile: t.boolean,
-    });
-
-    private example: string | undefined = undefined;
-    private operationSequenceNum = 0;
-    private datumCount = 0;
-    private matchedSize = 0;
-    private notMatchedSize = 0;
-    private errors = 0;
-    private notMatchedCount = 0;
-    private datums: LogDatum[] = [];
-    private countsPerName = new Map<string, number>();
-    private sizePerName = new Map<string, number>();
-
-    private lastRenderTime = 0;
-    private suppressionCounts = new Map<string, number>();
-    private expiredSuppressionCounts = new Map<string, number>();
-    private fastArchiveViewer: FastArchiveViewer<LogDatum> | undefined = undefined;
-
-    rerun() {
-        Querysub.onCommitFinished(async () => {
-            void this.fastArchiveViewer?.synchronizeDataThrottled();
-        });
-    }
-    getLoggers = () => {
-        let logs: FastArchiveAppendable<LogDatum>[] = [];
-        let loggers = getLoggers();
-        if (!loggers) {
-            throw new Error("Loggers not available?");
-        }
-        let { logLogs, warnLogs, infoLogs, errorLogs } = loggers;
-        if (errorNotifyToggleURL.value) {
-            logs.push(errorLogs);
-            logs.push(warnLogs);
-        } else {
-            if (enableLogsURL.value) {
-                logs.push(logLogs);
-            }
-            if (enableInfosURL.value) {
-                logs.push(infoLogs);
-            }
-            if (enableWarningsURL.value) {
-                logs.push(warnLogs);
-            }
-            if (enableErrorsURL.value) {
-                logs.push(errorLogs);
-            }
-        }
-        return logs;
-    };
-
-    render() {
-        //if (throttleRender({ key: "LogViewer2", frameDelay: 30 })) return undefined;
-
-        this.state.datumsSeqNum;
-
-        let suppressionController = SuppressionListController(SocketFunction.browserNodeId());
-        let suppressionList = suppressionController.getSuppressionList();
-        let hasErrorNotifyToggle = errorNotifyToggleURL.value;
-
-        const timeRange = getTimeRange();
-        return (
-            <div className={css.vbox(20).pad2(20).fillBoth}>
-                <div className={css.hbox(10)}>
-                    <div className={css.fontSize(14)}>
-                        Log Viewer
-                    </div>
-                    <InputLabelURL
-                        label="Error Notification Mode" checkbox url={errorNotifyToggleURL}
-                        onChangeValue={(newValue) => {
-                            if (newValue) {
-                                let now = Date.now();
-                                startTimeParam.value = now - timeInDay * 7;
-                                endTimeParam.value = now + timeInHour * 2;
-                            }
-                            this.rerun();
-                        }}
-                    />
-                    {errorNotifyToggleURL.value && <div className={css.hbox(10)}>
-                        <Button onClick={() => {
-                            void RecentErrorsController(SocketFunction.browserNodeId()).raiseTestError.promise("Test error notification");
-                        }}>
-                            Test Error Notification (won't rerun search, but it should show up in the title)
-                        </Button>
-                    </div>}
-                    {!isPublic() && <InputLabelURL
-                        label="Force Public Logs" checkbox url={forceGetPublicURL}
-                        onChangeValue={(newValue) => {
-                            this.rerun();
-                        }}
-                    />}
-                </div>
-
-                {!errorNotifyToggleURL.value && <div className={css.hbox(10)}>
-                    <Button hue={enableLogsURL.value ? 120 : undefined} onClick={() => { enableLogsURL.value = !enableLogsURL.value; this.rerun(); }}>
-                        Logs
-                    </Button>
-                    <Button hue={enableInfosURL.value ? 120 : undefined} onClick={() => { enableInfosURL.value = !enableInfosURL.value; this.rerun(); }}>
-                        Infos
-                    </Button>
-                    <Button hue={enableWarningsURL.value ? 120 : undefined} onClick={() => { enableWarningsURL.value = !enableWarningsURL.value; this.rerun(); }}>
-                        Warnings
-                    </Button>
-                    <Button hue={enableErrorsURL.value ? 120 : undefined} onClick={() => { enableErrorsURL.value = !enableErrorsURL.value; this.rerun(); }}>
-                        Errors
-                    </Button>
-                </div>}
-
-                <FastArchiveViewer
-                    ref2={x => this.fastArchiveViewer = x}
-                    fastArchives={this.getLoggers}
-                    runOnLoad={errorNotifyToggleURL.value}
-                    forceGetPublic={forceGetPublicURL.value}
-                    onStart={async () => {
-                        this.datumCount = 0;
-                        this.notMatchedCount = 0;
-                        this.errors = 0;
-                        this.notMatchedSize = 0;
-                        this.matchedSize = 0;
-                        this.example = undefined;
-                        this.datums = [];
-                        this.suppressionCounts = new Map();
-                        this.expiredSuppressionCounts = new Map();
-                        this.countsPerName = new Map();
-                        this.operationSequenceNum++;
-                        void (async () => {
-                            const currentSequenceNum = this.operationSequenceNum;
-                            while (true) {
-                                await new Promise(resolve => setTimeout(resolve, 1000));
-                                if (this.operationSequenceNum !== currentSequenceNum) {
-                                    break;
-                                }
-                                Querysub.commit(() => {
-                                    this.state.datumsSeqNum++;
-                                });
-                            }
-                        })();
-                        // ALWAYS update it, as the synchronous one might be out of date, and if we use an outdated one extra errors show up.
-                        suppressionList = await suppressionController.getSuppressionList.promise();
-
-                    }}
-                    getScanFnc={() => {
-                        if (!this.state.showCountByFile && !this.state.showSizeByFile) {
-                            return () => false;
-                        }
-                        return createLogViewerExtractField("__NAME__", (value, size) => {
-                            let prevCount = this.countsPerName.get(value) ?? 0;
-                            prevCount++;
-                            this.countsPerName.set(value, prevCount);
-                            let prevSize = this.sizePerName.get(value) ?? 0;
-                            this.sizePerName.set(value, prevSize + size);
-                        });
-                    }}
-                    getWantData={async (file) => {
-                        if (!hasErrorNotifyToggle) return undefined;
-                        // By defaulting to the synchronous one, the list will be updated if there's any changes. However, we will also just get it asynchronously if the list hasn't been updated by the time we call get1data. And because we assign it back to the variable, it'll be cached. 
-                        suppressionList = suppressionList || await suppressionController.getSuppressionList.promise();
-
-                        let suppressionFull = getSuppressionFull({
-                            entries: suppressionList,
-                            blockTimeRange: {
-                                startTime: file.startTime,
-                                endTime: file.endTime,
-                            },
-                            suppressionCounts: this.suppressionCounts,
-                            expiredSuppressionCounts: this.expiredSuppressionCounts,
-                        });
-                        return suppressionFull;
-                    }}
-                    onDatums={(source, datums, file) => {
-                        this.datumCount += datums.length;
-                        if (!this.example && datums.length > 0) {
-                            this.example = JSON.stringify(datums[0]);
-                        }
-                        for (let datum of datums) {
-                            let time = datum.time;
-                            if (time && (time < timeRange.startTime || time > timeRange.endTime)) {
-                                continue;
-                            }
-                            datum.__metadata = file;
-                            this.datums.push(datum);
-                        }
-                    }}
-                    onStats={(source, stats, file) => {
-                        this.matchedSize += stats.matchedSize;
-                        this.notMatchedSize += stats.notMatchedSize;
-                        this.errors += stats.errors;
-                        this.notMatchedCount += stats.notMatchedCount;
-
-                        let now = Date.now();
-                        if (now - this.lastRenderTime > RENDER_INTERVAL) {
-                            this.lastRenderTime = now;
-                            Querysub.commit(() => {
-                                this.state.datumsSeqNum++;
-                            });
-                        }
-                    }}
-                    onFinish={() => {
-                        this.operationSequenceNum++;
-                        Querysub.commit(() => {
-                            sort(this.datums, x => -(x.time || 0));
-                            this.state.datumsSeqNum++;
-                        });
-                    }}
-                >
-                    {errorNotifyToggleURL.value && <ErrorSuppressionUI
-                        dataSeqNum={this.state.datumsSeqNum}
-                        suppressionCounts={this.suppressionCounts}
-                        expiredSuppressionCounts={this.expiredSuppressionCounts}
-                        datums={this.datums}
-                        rerunFilters={() => this.rerun()}
-                    />}
-                    {(() => {
-                        let fieldNames = new Set<string>();
-                        let anyLimited = false;
-                        for (let i = 0; i < Math.min(1000, this.datums.length); i++) {
-                            let datum = this.datums[i];
-                            for (let key of Object.keys(datum)) {
-                                fieldNames.add(key);
-                            }
-                            if (datum[LOG_LIMIT_FLAG]) {
-                                anyLimited = true;
-                            }
-                        }
-                        let lineLimited = false;
-                        for (let i = 0; i < Math.min(1000, this.datums.length); i++) {
-                            let datum = this.datums[i];
-                            if (datum[LOG_LINE_LIMIT_FLAG]) {
-                                lineLimited = true;
-                            }
-                        }
-                        let selectedFields: string[] = [];
-                        for (let field of Object.keys(defaultSelectedFields)) {
-                            if (atomic(selectedFieldsURL.value[field]) === undefined) {
-                                selectedFields.push(field);
-                            }
-                        }
-                        for (let [key, value] of Object.entries(selectedFieldsURL.value)) {
-                            if (value && !selectedFields.includes(key)) {
-                                selectedFields.splice(1, 0, key);
-                            }
-                        }
-                        let columns: TableType<LogDatum>["columns"] = {};
-                        columns["log"] = {
-                            title: "Log",
-                            formatter: (x, context) => {
-                                return <Button onClick={() => console.log(context?.row)}>
-                                    Log
-                                </Button>;
-                            }
-                        };
-                        if (anyLimited) {
-                            columns[LOG_LIMIT_FLAG] = {
-                                title: "Limited",
-                                formatter: x => x && <div className={css.hsl(0, 50, 50).colorhsl(0, 50, 95).boldStyle.pad2(10).ellipsis}
-                                    title="File throttled (other line may be lost lost!)">
-                                    FILE throttled
-                                </div> || undefined
-                            };
-                        }
-                        if (lineLimited) {
-                            columns[LOG_LINE_LIMIT_FLAG] = {
-                                title: "Limited",
-                                formatter: x => x && <div className={css.hsl(60, 50, 50).colorhsl(60, 50, 95).boldStyle.pad2(10).ellipsis}
-                                    title="Line throttled (no lines lost, just count for this line is lower than the actual count)">
-                                    Line throttled
-                                </div> || undefined
-                            };
-                        }
-                        for (let field of selectedFields) {
-                            let column: ColumnType<unknown, LogDatum> = {};
-                            if (field === "time") {
-                                column.formatter = (x: unknown) => useRelativeTimeURL.value ? formatDateJSX(Number(x)) : <span title={formatDateTimeDetailed(Number(x))}>{formatDateTime(Number(x))}</span>;
-                            }
-                            if (!column.formatter) {
-                                column.formatter = (x: unknown) => <ObjectDisplay value={x} />;
-                            }
-                            columns[field] = column;
-                        }
-                        columns["fields"] = {
-                            title: "Fields",
-                            formatter: (x, context) => {
-                                if (!context?.row) return undefined;
-                                let datum = context.row;
-                                let fields = Object.keys(datum);
-                                fields = fields.filter(x => !(x in columns) && !x.startsWith("_"));
-                                return <div className={css.hbox(4, 4).wrap}>
-                                    {fields.map(x => <div
-                                        className={css.pad2(4, 0).hsl(0, 0, 80).button}
-                                        onClick={() => {
-                                            let newValues = { ...selectedFieldsURL.value };
-                                            newValues[x] = true;
-                                            selectedFieldsURL.value = newValues;
-                                        }}
-                                    >
-                                        {x}
-                                    </div>)}
-                                </div>;
-                            }
-                        };
-                        let includedFiles = filterParam2.value.split("|").map(x => x.trim());
-                        return <>
-                            <div className={css.hbox(10)}>
-                                <InputPicker
-                                    label={<div className={css.hbox(10)}>
-                                        <div className={css.flexShrink0}>
-                                            Selected Fields
-                                        </div>
-                                        <Button onClick={() => {
-                                            let newValues = { ...selectedFieldsURL.value };
-                                            for (let key of Object.keys(newValues)) {
-                                                newValues[key] = key in defaultSelectedFields;
-                                            }
-                                            selectedFieldsURL.value = newValues;
-                                        }}>
-                                            Reset
-                                        </Button>
-                                    </div>}
-                                    picked={selectedFields}
-                                    options={Array.from(fieldNames).map(x => ({ value: x, label: x }))}
-                                    addPicked={x => {
-                                        let newValues = { ...selectedFieldsURL.value };
-                                        newValues[x] = true;
-                                        selectedFieldsURL.value = newValues;
-                                    }}
-                                    removePicked={x => {
-                                        let newValues = { ...selectedFieldsURL.value };
-                                        newValues[x] = false;
-                                        selectedFieldsURL.value = newValues;
-                                    }}
-                                />
-                                <InputLabelURL
-                                    label="Use Relative Time"
-                                    checkbox
-                                    url={useRelativeTimeURL}
-                                    onChangeValue={() => {
-                                        Querysub.commit(() => {
-                                            this.state.datumsSeqNum++;
-                                        });
-                                    }}
-                                />
-                            </div>
-                            {this.datums.length === 0 && <div className={css.hsl(40, 50, 50).colorhsl(60, 50, 100).boldStyle.pad2(10).ellipsis}>
-                                No logs matched, either increase the time range or decrease the filter specificity.
-                            </div>}
-                            {includedFiles.length > 0 && includedFiles[0] && <div className={css.hbox(20)}>
-                                <div className={css.hbox(10)}>
-                                    <div className={css.fontSize(18).hsl(0, 50, 50).colorhsl(0, 50, 95).pad2(8, 4)}>
-                                        Only showing {includedFiles.length} files:
-                                    </div>
-                                    <Button onClick={() => {
-                                        filterParam2.value = "";
-                                        this.rerun();
-                                    }}>
-                                        Clear All Filters
-                                    </Button>
-                                </div>
-                                <div className={css.hbox(10, 10).wrap}>
-                                    {includedFiles.filter(x => x).map((fileName) => (
-                                        <div
-                                            key={fileName}
-                                            className={css.pad2(8, 4).hsl(120, 40, 80).bord2(120, 40, 60, 1).button}
-                                            onClick={() => {
-                                                let newFiles = includedFiles.filter(x => x !== fileName);
-                                                filterParam2.value = newFiles.join("|");
-                                            }}
-                                            title={`Click to remove ${fileName} from filter`}
-                                        >
-                                            {fileName} ✕
-                                        </div>
-                                    ))}
-                                </div>
-                            </div>}
-
-                            <div className={css.hbox(10)}>
-                                {this.state.showCountByFile && (() => {
-                                    let counts = Array.from(this.countsPerName.entries());
-                                    sort(counts, x => -x[1]);
-                                    let topCounts = counts.slice(0, 20);
-                                    let totalCount = counts.reduce((sum, [, count]) => sum + count, 0);
-
-                                    return <div className={css.vbox(10)}>
-                                        <div className={css.hbox(10, 10).wrap}>
-                                            {topCounts.map(([name, count]) => {
-                                                let percentage = totalCount > 0 ? (count / totalCount) * 100 : 0;
-                                                let isFiltered = includedFiles.includes(name);
-                                                let size = this.sizePerName.get(name) ?? 0;
-
-                                                return <div
-                                                    key={name}
-                                                    className={css.relative.pad2(8, 4).bord2(210, 30, 60, 1).ellipsis.button}
-                                                    title={isFiltered ? `Click to remove ${name} from filter` : `Click to add ${name} to filter`}
-                                                    onClick={() => {
-                                                        if (isFiltered) {
-                                                            // Remove from filter
-                                                            let newFiles = includedFiles.filter(x => x !== name);
-                                                            filterParam2.value = newFiles.join("|");
-                                                        } else {
-                                                            // Add to filter
-                                                            let newFiles = [...includedFiles.filter(x => x), name];
-                                                            filterParam2.value = newFiles.join("|");
-                                                        }
-                                                    }}
-                                                >
-                                                    <div
-                                                        className={css.absolute.pos(0, 0).height("100%").width(`${percentage}%`).hsl(isFiltered ? 120 : 210, 40, 80)}
-                                                    />
-                                                    <div className={css.relative.maxWidth(200).ellipsis}>
-                                                        {formatNumber(count)} ({formatNumber(size)}B) | {name || "(no name present)"} {isFiltered && "✓" || ""}
-                                                    </div>
-                                                </div>;
-                                            })}
-                                        </div>
-                                    </div>;
-                                })() || <Button onClick={() => {
-                                    this.state.showCountByFile = true;
-                                    this.rerun();
-                                }}>
-                                        Show Count by File
-                                    </Button>}
-
-                                {this.state.showSizeByFile && (() => {
-                                    let sizes = Array.from(this.sizePerName.entries());
-                                    sort(sizes, x => -x[1]);
-                                    let topSizes = sizes.slice(0, 20);
-                                    let totalSize = sizes.reduce((sum, [, size]) => sum + size, 0);
-
-                                    return <div className={css.vbox(10)}>
-                                        <div className={css.hbox(10, 10).wrap}>
-                                            {topSizes.map(([name, size]) => {
-                                                let percentage = totalSize > 0 ? (size / totalSize) * 100 : 0;
-                                                let isFiltered = includedFiles.includes(name);
-                                                let count = this.countsPerName.get(name) ?? 0;
-
-                                                return <div
-                                                    key={name}
-                                                    className={css.relative.pad2(8, 4).bord2(280, 30, 60, 1).ellipsis.button}
-                                                    title={isFiltered ? `Click to remove ${name} from filter` : `Click to add ${name} to filter`}
-                                                    onClick={() => {
-                                                        if (isFiltered) {
-                                                            // Remove from filter
-                                                            let newFiles = includedFiles.filter(x => x !== name);
-                                                            filterParam2.value = newFiles.join("|");
-                                                        } else {
-                                                            // Add to filter
-                                                            let newFiles = [...includedFiles.filter(x => x), name];
-                                                            filterParam2.value = newFiles.join("|");
-                                                        }
-                                                    }}
-                                                >
-                                                    <div
-                                                        className={css.absolute.pos(0, 0).height("100%").width(`${percentage}%`).hsl(isFiltered ? 120 : 280, 40, 80)}
-                                                    />
-                                                    <div className={css.relative.maxWidth(200).ellipsis}>
-                                                        {formatNumber(size)}B ({formatNumber(count)}) | {name || "(no name present)"} {isFiltered && "✓" || ""}
-                                                    </div>
-                                                </div>;
-                                            })}
-                                        </div>
-                                    </div>;
-                                })() || <Button onClick={() => {
-                                    this.state.showSizeByFile = true;
-                                    this.rerun();
-                                }}>
-                                        Show Size by File
-                                    </Button>}
-                            </div>
-                            <Table
-                                rows={this.datums}
-                                columns={columns}
-                                lineLimit={4}
-                                initialLimit={10}
-                                characterLimit={400}
-                                getRowAttributes={row => {
-                                    let hue = -1;
-                                    if (row.__LOG_TYPE === "warn") hue = 40;
-                                    if (row.__LOG_TYPE === "error") hue = 0;
-                                    if (row.__LOG_TYPE === "info") hue = 200;
-                                    return {
-                                        className: hue !== -1 && css.hsl(hue, 40, 50).colorhsl(hue, 0, 100) || undefined,
-                                    };
-                                }}
-                            />
-                        </>;
-                    })()}
-                </FastArchiveViewer>
-            </div>
-        );
-    }
-}