querysub 0.312.0 → 0.314.0

package/src/diagnostics/logs/LogViewer2.tsx

@@ -0,0 +1,349 @@
+module.hotreload = true;
+import { qreact } from "../../4-dom/qreact";
+import { css } from "../../4-dom/css";
+import { URLParam } from "../../library-components/URLParam";
+import { InputLabelURL } from "../../library-components/InputLabel";
+import { Button } from "../../library-components/Button";
+import { formatDateTime, formatNiceDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { TimeRangeSelector, endTimeParam, getTimeRange, startTimeParam } from "./TimeRangeSelector";
+import { FastArchiveAppendable } from "./FastArchiveAppendable";
+import { t } from "../../2-proxy/schema2";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { logErrors } from "../../errors";
+import { batchFunction, runInSerial } from "socket-function/src/batching";
+import { Querysub } from "../../4-querysub/QuerysubController";
+import { sort, timeInDay, timeInHour } from "socket-function/src/misc";
+import { FastArchiveViewer } from "./FastArchiveViewer";
+import { LogDatum, getLoggers, LOG_LIMIT_FLAG } from "./diskLogger";
+import { ColumnType, Table, TableType } from "../../5-diagnostics/Table";
+import { formatDateJSX } from "../../misc/formatJSX";
+import { InputPicker } from "../../library-components/InputPicker";
+import { JSXFormatter } from "../../5-diagnostics/GenericFormat";
+import { atomic } from "../../2-proxy/PathValueProxyWatcher";
+import { ObjectDisplay } from "./ObjectDisplay";
+import { endTime } from "../misc-pages/archiveViewerShared";
+import { ErrorSuppressionUI } from "./errorNotifications/ErrorSuppressionUI";
+import { FileMetadata } from "./FastArchiveController";
+import { SuppressionListController, getSuppressEntryChecker } from "./errorNotifications/ErrorNotificationController";
+import { SocketFunction } from "socket-function/SocketFunction";
+
+const RENDER_INTERVAL = 1000;
+
+export const testLogs = new FastArchiveAppendable("test/");
+
+const enableLogsURL = new URLParam("enableLogs", false);
+const enableInfosURL = new URLParam("enableInfos", true);
+const enableWarningsURL = new URLParam("enableWarnings", true);
+const enableErrorsURL = new URLParam("enableErrors", true);
+const selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
+
+export const errorNotifyToggleURL = new URLParam("errorNotifyToggle", false);
+
+const defaultSelectedFields = {
+    param0: true,
+    time: true,
+    __NAME__: true,
+
+    //__machineId: true,
+    __threadId: true,
+    __entry: true,
+    //__DIR__: true,
+};
+
+export class LogViewer2 extends qreact.Component {
+    state = t.state({
+        datumsSeqNum: t.atomic<number>(0),
+    });
+
+    private example: string | undefined = undefined;
+    private datumCount = 0;
+    private matchedSize = 0;
+    private notMatchedSize = 0;
+    private errors = 0;
+    private notMatchedCount = 0;
+    private datums: LogDatum[] = [];
+
+    private lastRenderTime = 0;
+    private suppressionCounts = new Map<string, number>();
+    private expiredSuppressionCounts = new Map<string, number>();
+    private fastArchiveViewer: FastArchiveViewer<LogDatum> | undefined = undefined;
+
+    rerun() {
+        void this.fastArchiveViewer?.handleDownload();
+    }
+
+    render() {
+
+        this.state.datumsSeqNum;
+
+        let logs: FastArchiveAppendable<LogDatum>[] = [];
+        let loggers = getLoggers();
+        if (!loggers) {
+            return `Loggers not available?`;
+        }
+        let { logLogs, warnLogs, infoLogs, errorLogs } = loggers;
+        if (errorNotifyToggleURL.value) {
+            logs.push(errorLogs);
+            logs.push(warnLogs);
+        } else {
+            if (enableLogsURL.value) {
+                logs.push(logLogs);
+            }
+            if (enableInfosURL.value) {
+                logs.push(infoLogs);
+            }
+            if (enableWarningsURL.value) {
+                logs.push(warnLogs);
+            }
+            if (enableErrorsURL.value) {
+                logs.push(errorLogs);
+            }
+        }
+
+        const timeRange = getTimeRange();
+        return (
+            <div className={css.vbox(20).pad2(20).fillBoth}>
+                <div className={css.hbox(10)}>
+                    <div className={css.fontSize(14)}>
+                        Log Viewer
+                    </div>
+                    <InputLabelURL
+                        label="Error Notification Mode" checkbox url={errorNotifyToggleURL}
+                        onChangeValue={(newValue) => {
+                            if (newValue) {
+                                let now = Date.now();
+                                startTimeParam.value = now - timeInDay * 7;
+                                endTimeParam.value = now + timeInHour * 2;
+                            }
+                            this.rerun();
+                        }}
+                    />
+                </div>
+
+                {!errorNotifyToggleURL.value && <div className={css.hbox(10)}>
+                    <Button hue={enableLogsURL.value ? 120 : undefined} onClick={() => enableLogsURL.value = !enableLogsURL.value}>
+                        Logs
+                    </Button>
+                    <Button hue={enableInfosURL.value ? 120 : undefined} onClick={() => enableInfosURL.value = !enableInfosURL.value}>
+                        Infos
+                    </Button>
+                    <Button hue={enableWarningsURL.value ? 120 : undefined} onClick={() => enableWarningsURL.value = !enableWarningsURL.value}>
+                        Warnings
+                    </Button>
+                    <Button hue={enableErrorsURL.value ? 120 : undefined} onClick={() => enableErrorsURL.value = !enableErrorsURL.value}>
+                        Errors
+                    </Button>
+                </div>}
+
+                <FastArchiveViewer
+                    ref2={x => this.fastArchiveViewer = x}
+                    fastArchives={logs}
+                    onStart={() => {
+                        this.datumCount = 0;
+                        this.notMatchedCount = 0;
+                        this.errors = 0;
+                        this.notMatchedSize = 0;
+                        this.matchedSize = 0;
+                        this.example = undefined;
+                        this.datums = [];
+                        this.suppressionCounts = new Map();
+                        this.expiredSuppressionCounts = new Map();
+                    }}
+                    getWantData={async () => {
+                        if (!Querysub.fastRead(() => errorNotifyToggleURL.value)) return undefined;
+                        let suppressionList = await SuppressionListController(SocketFunction.browserNodeId()).getSuppressionList.promise();
+                        let now = Date.now();
+                        // Newest first, so when we add a new entry we can see what it matches, even if it takes it away from older entries
+                        sort(suppressionList, x => -x.lastUpdateTime);
+                        let checkers = suppressionList.map(x => getSuppressEntryChecker(x));
+                        this.suppressionCounts = new Map(suppressionList.map(x => [x.key, 0]));
+                        this.expiredSuppressionCounts = new Map(suppressionList.map(x => [x.key, 0]));
+                        let updateCounts = batchFunction({ delay: 1000 }, () => {
+                            Querysub.commit(() => {
+                                this.state.datumsSeqNum++;
+                            });
+                        });
+                        return (posStart, posEnd, data, file) => {
+                            for (let checker of checkers) {
+                                if (checker.fnc(data, posStart, posEnd)) {
+                                    if (checker.entry.expiresAt < now) {
+                                        let count = this.expiredSuppressionCounts.get(checker.entry.key) || 0;
+                                        count++;
+                                        this.expiredSuppressionCounts.set(checker.entry.key, count);
+                                        void updateCounts(undefined);
+                                    } else {
+                                        let count = this.suppressionCounts.get(checker.entry.key) || 0;
+                                        count++;
+                                        this.suppressionCounts.set(checker.entry.key, count);
+                                        void updateCounts(undefined);
+                                        return false;
+                                    }
+                                }
+                            }
+                            return true;
+                        };
+                    }}
+                    onDatums={(source, datums, file) => {
+                        this.datumCount += datums.length;
+                        if (!this.example && datums.length > 0) {
+                            this.example = JSON.stringify(datums[0]);
+                        }
+                        for (let datum of datums) {
+                            let time = datum.time;
+                            if (time && (time < timeRange.startTime || time > timeRange.endTime)) {
+                                continue;
+                            }
+                            datum.__metadata = file;
+                            this.datums.push(datum);
+                        }
+                    }}
+                    onStats={(source, stats, file) => {
+                        this.matchedSize += stats.matchedSize;
+                        this.notMatchedSize += stats.notMatchedSize;
+                        this.errors += stats.errors;
+                        this.notMatchedCount += stats.notMatchedCount;
+
+                        let now = Date.now();
+                        if (now - this.lastRenderTime > RENDER_INTERVAL) {
+                            this.lastRenderTime = now;
+                            Querysub.commit(() => {
+                                this.state.datumsSeqNum++;
+                            });
+                        }
+                    }}
+                    onFinish={() => {
+                        Querysub.commit(() => {
+                            sort(this.datums, x => -(x.time || 0));
+                            this.state.datumsSeqNum++;
+                        });
+                    }}
+                >
+                    {errorNotifyToggleURL.value && <ErrorSuppressionUI
+                        dataSeqNum={this.state.datumsSeqNum}
+                        suppressionCounts={this.suppressionCounts}
+                        expiredSuppressionCounts={this.expiredSuppressionCounts}
+                        datums={this.datums}
+                        rerunFilters={() => this.rerun()}
+                    />}
+                    {(() => {
+                        let fieldNames = new Set<string>();
+                        let anyLimited = false;
+                        for (let i = 0; i < Math.min(1000, this.datums.length); i++) {
+                            let datum = this.datums[i];
+                            for (let key of Object.keys(datum)) {
+                                fieldNames.add(key);
+                            }
+                            if (datum[LOG_LIMIT_FLAG]) {
+                                anyLimited = true;
+                            }
+                        }
+                        let selectedFields: string[] = [];
+                        for (let field of Object.keys(defaultSelectedFields)) {
+                            if (atomic(selectedFieldsURL.value[field]) === undefined) {
+                                selectedFields.push(field);
+                            }
+                        }
+                        for (let [key, value] of Object.entries(selectedFieldsURL.value)) {
+                            if (value && !selectedFields.includes(key)) {
+                                selectedFields.splice(1, 0, key);
+                            }
+                        }
+                        let columns: TableType<LogDatum>["columns"] = {};
+                        columns["log"] = {
+                            title: "Log",
+                            formatter: (x, context) => {
+                                return <Button onClick={() => console.log(context?.row)}>
+                                    Log
+                                </Button>;
+                            }
+                        };
+                        if (anyLimited) {
+                            columns[LOG_LIMIT_FLAG] = {
+                                title: "Limited",
+                                formatter: x => x && <div className={css.hsl(0, 50, 50).colorhsl(0, 50, 95).boldStyle.pad2(10).ellipsis}>
+                                    Log Line Throttled
+                                </div> || undefined
+                            };
+                        }
+                        for (let field of selectedFields) {
+                            let column: ColumnType<unknown, LogDatum> = {};
+                            if (field === "time") {
+                                column.formatter = (x: unknown) => formatDateTime(Number(x));
+                            }
+                            if (!column.formatter) {
+                                column.formatter = (x: unknown) => <ObjectDisplay value={x} />;
+                            }
+                            columns[field] = column;
+                        }
+                        columns["fields"] = {
+                            title: "Fields",
+                            formatter: (x, context) => {
+                                if (!context?.row) return undefined;
+                                let datum = context.row;
+                                let fields = Object.keys(datum);
+                                fields = fields.filter(x => !(x in columns) && !x.startsWith("_"));
+                                return <div className={css.hbox(4, 4).wrap}>
+                                    {fields.map(x => <div
+                                        className={css.pad2(4, 0).hsl(0, 0, 80).button}
+                                        onClick={() => {
+                                            let newValues = { ...selectedFieldsURL.value };
+                                            newValues[x] = true;
+                                            selectedFieldsURL.value = newValues;
+                                        }}
+                                    >
+                                        {x}
+                                    </div>)}
+                                </div>;
+                            }
+                        };
+                        return <>
+                            <InputPicker
+                                label={<div className={css.hbox(10)}>
+                                    <div className={css.flexShrink0}>
+                                        Selected Fields
+                                    </div>
+                                    <Button onClick={() => {
+                                        let newValues = { ...selectedFieldsURL.value };
+                                        for (let key of Object.keys(newValues)) {
+                                            newValues[key] = key in defaultSelectedFields;
+                                        }
+                                        selectedFieldsURL.value = newValues;
+                                    }}>
+                                        Reset
+                                    </Button>
+                                </div>}
+                                picked={selectedFields}
+                                options={Array.from(fieldNames).map(x => ({ value: x, label: x }))}
+                                addPicked={x => {
+                                    let newValues = { ...selectedFieldsURL.value };
+                                    newValues[x] = true;
+                                    selectedFieldsURL.value = newValues;
+                                }}
+                                removePicked={x => {
+                                    let newValues = { ...selectedFieldsURL.value };
+                                    newValues[x] = false;
+                                    selectedFieldsURL.value = newValues;
+                                }}
+                            />
+                            <Table
+                                rows={this.datums}
+                                columns={columns}
+                                lineLimit={4}
+                                characterLimit={400}
+                                getRowAttributes={row => {
+                                    let hue = -1;
+                                    if (row.__LOG_TYPE === "warn") hue = 40;
+                                    if (row.__LOG_TYPE === "error") hue = 0;
+                                    if (row.__LOG_TYPE === "info") hue = 200;
+                                    return {
+                                        className: hue !== -1 && css.hsl(hue, 40, 50).colorhsl(hue, 0, 100) || undefined,
+                                    };
+                                }}
+                            />
+                        </>;
+                    })()}
+                </FastArchiveViewer>
+            </div>
+        );
+    }
+}

package/src/diagnostics/logs/TimeRangeSelector.tsx

@@ -0,0 +1,94 @@
+module.hotreload = true;
+
+import { qreact } from "../../4-dom/qreact";
+import { URLParam } from "../../library-components/URLParam";
+import { css } from "../../4-dom/css";
+import { formatTime } from "socket-function/src/formatting/format";
+import { InputLabel } from "../../library-components/InputLabel";
+import { Button } from "../../library-components/Button";
+import { Querysub } from "../../4-querysub/QuerysubController";
+import { timeInHour, timeInMinute } from "socket-function/src/misc";
+
+// URL parameters for time range
+export const startTimeParam = new URLParam("startTime", undefined as number | undefined);
+export const endTimeParam = new URLParam("endTime", undefined as number | undefined);
+
+let now = Date.now();
+/** Get the time range with default values if not set */
+export function getTimeRange(): { startTime: number; endTime: number } {
+    const defaultStart = now - timeInHour * 24;
+    const defaultEnd = now + timeInHour;
+
+    return {
+        startTime: startTimeParam.value ?? defaultStart,
+        endTime: endTimeParam.value ?? defaultEnd,
+    };
+}
+
+export class TimeRangeSelector extends qreact.Component {
+    render() {
+        const timeRange = getTimeRange();
+
+        const resetToLastDay = () => {
+            now = Date.now();
+            startTimeParam.reset();
+            endTimeParam.reset();
+        };
+
+        const formatDateTimeLocal = (timestamp: number) => {
+            // Convert to local datetime-local format: YYYY-MM-DDTHH:MM
+            return new Date(timestamp - new Date().getTimezoneOffset() * 60000)
+                .toISOString()
+                .slice(0, 16);
+        };
+
+        const parseDateTimeLocal = (dateTimeString: string) => {
+            // Parse datetime-local format and convert to timestamp
+            return new Date(dateTimeString).getTime();
+        };
+
+
+        let now = Querysub.nowDelayed(timeInMinute);
+        return (
+            <div className={css.vbox(12).pad2(16).bord2(200, 20, 80).hsl(200, 10, 98)}>
+                <div className={css.hbox(12).wrap}>
+                    <div className={css.hbox(8).wrap.opacity(0.8).width(200)}>
+                        From {formatTime(now - timeRange.startTime)} AGO to {timeRange.endTime > now ? "now" : `${formatTime(now - timeRange.endTime)} AGO`}
+                    </div>
+                    <InputLabel
+                        label="Start Time"
+                        type="datetime-local"
+                        value={formatDateTimeLocal(timeRange.startTime)}
+                        onChange={e => {
+                            startTimeParam.value = parseDateTimeLocal(e.currentTarget.value);
+                        }}
+                        outerClass={!startTimeParam.value && css.opacity(0.5) || ""}
+                    />
+                    <InputLabel
+                        label="End Time"
+                        type="datetime-local"
+                        value={formatDateTimeLocal(timeRange.endTime)}
+                        onChange={e => {
+                            endTimeParam.value = parseDateTimeLocal(e.currentTarget.value);
+                        }}
+                        outerClass={!endTimeParam.value && css.opacity(0.5) || ""}
+                    />
+                    {(endTimeParam.value || startTimeParam.value) && <Button
+                        hue={110} onClick={resetToLastDay}
+                    >
+                        Reset to Last Day
+                    </Button>}
+                    {(!startTimeParam.value || !endTimeParam.value) && <Button
+                        hue={110}
+                        onClick={() => {
+                            startTimeParam.value = timeRange.startTime;
+                            endTimeParam.value = timeRange.endTime;
+                        }}
+                    >
+                        Save Time Range
+                    </Button>}
+                </div>
+            </div>
+        );
+    }
+}