querysub 0.327.0 → 0.329.0

package/src/diagnostics/logs/diskLogger.ts

@@ -6,6 +6,8 @@ import { timeInMinute } from "socket-function/src/misc";
 import { formatTime } from "socket-function/src/formatting/format";
 import { addEpsilons } from "../../bits";
 import { FileMetadata } from "./FastArchiveController";
+import { getPathStr2 } from "../../path";
+import { isPublic } from "../../config";
 // IMPORTANT! We can't have any real imports here, because we are depended on so early in startup!
 
 if (isNode()) {
@@ -35,7 +37,25 @@ export type LogDatum = Record<string, unknown> & {
     /** Dynamically set when matching recent errors only. */
     __matchedOutdatedSuppressionKey?: string;
 };
+export function getLogHash(obj: LogDatum) {
+    return getPathStr2(obj.__threadId || "", obj.time.toString());
+}
+export function getLogFile(obj: LogDatum) {
+    let logType = obj.param0 || "";
+    if (obj.__FILE__) {
+        logType = String(obj.__FILE__);
+    }
+    if (obj[LOG_LINE_LIMIT_ID]) {
+        logType += "::" + String(obj[LOG_LINE_LIMIT_ID]);
+    }
+    return logType;
+
+}
 export const LOG_LIMIT_FLAG = String.fromCharCode(44533) + "LOGS_LIMITED_FLAG-9277640b-d709-4591-ab08-2bb29bbb94f4";
+export const LOG_LINE_LIMIT_FLAG = String.fromCharCode(44534) + "LOGS_LINE_LIMIT_FLAG-dd50ab1f-3021-45e3-82fc-d2702c7a64c8";
+
+/** If this key exists in the logged object, as in a key in one of the objects logged, then we will use the value of it as the limit ID. This is useful as it allows us to either override a limit or limit something independently from other logs in the file. */
+export const LOG_LINE_LIMIT_ID = "LIMIT_LINE_ID";
 
 export const getLoggers = lazy(function () {
     const { FastArchiveAppendable } = require("./FastArchiveAppendable") as typeof import("./FastArchiveAppendable");
@@ -52,6 +72,10 @@ export const getLoggers = lazy(function () {
         errorLogs: new FastArchiveAppendable<LogDatum>("logs-error/"),
     };
 });
+setImmediate(() => {
+    // If we don't import it at all, then it doesn't work client-side. 
+    require("./FastArchiveAppendable") as typeof import("./FastArchiveAppendable");
+});
 const getNotifyErrors = lazy(function () {
     const { notifyWatchersOfError: notifyErrors } = require("./errorNotifications/ErrorNotificationController") as typeof import("./errorNotifications/ErrorNotificationController");
     if (typeof notifyErrors !== "function") {
@@ -88,6 +112,8 @@ let logLimitLookup: {
 
 const LIMIT_PERIOD = timeInMinute * 15;
 const LIMIT_THRESHOLD = 1000;
+const WARN_LIMIT = 100;
+const ERROR_LIMIT = 100;
 
 const logDiskDontShim = logDisk;
 /** NOTE: Calling this directly means we lose __FILE__ tracking. But... that's probably fine... */
@@ -104,6 +130,11 @@ export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknow
         if (logObj.__FILE__) {
             logType = String(logObj.__FILE__);
         }
+        let hasLineLimit = false;
+        if (logObj[LOG_LINE_LIMIT_ID]) {
+            logType += "::" + String(logObj[LOG_LINE_LIMIT_ID]);
+            hasLineLimit = true;
+        }
 
         if (logLimitLookup) {
             if (logObj.time > logLimitLookup.resetTime) {
@@ -120,32 +151,47 @@ export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknow
         let count = logLimitLookup.counts.get(logType) || 0;
         count++;
         logLimitLookup.counts.set(logType, count);
-        if (count > LIMIT_THRESHOLD) {
+        let limit = LIMIT_THRESHOLD;
+        if (type === "warn") {
+            limit = WARN_LIMIT;
+        } else if (type === "error") {
+            limit = ERROR_LIMIT;
+        }
+        if (count > limit) {
             let timeUntilReset = logLimitLookup.resetTime - logObj.time;
-            process.stdout.write(`Log type hit limit, not writing log type to disk for ~${formatTime(timeUntilReset)}: ${logType}\n`);
+            if (hasLineLimit) {
+                process.stdout.write(`Log type hit limit, not writing log type to disk for ~${formatTime(timeUntilReset)}: ${logType}\n`);
+            }
             return;
         }
-        if (count === LIMIT_THRESHOLD) {
-            logObj[LOG_LIMIT_FLAG] = true;
+        if (count >= limit) {
+            if (hasLineLimit) {
+                logObj[LOG_LINE_LIMIT_FLAG] = true;
+            } else {
+                logObj[LOG_LIMIT_FLAG] = true;
+            }
         }
 
-        let loggers = startupDone ? getLoggers() : undefined;
-        if (!loggers) {
-            getLoggers.reset();
-            setImmediate(() => {
-                logDiskDontShim(type, ...args);
-            });
-            return;
-        }
-        const { logLogs, warnLogs, infoLogs, errorLogs } = loggers;
-        if (type === "log") {
-            logLogs.append(logObj);
-        } else if (type === "warn") {
-            warnLogs.append(logObj);
-        } else if (type === "info") {
-            infoLogs.append(logObj);
-        } else {
-            errorLogs.append(logObj);
+        // We don't want developer errors clogging up the error logs. However, they can still notify errors, Because this will only notify nodes that are able to access us (It uses a reverse connection scheme, so instead of talking to nodes that we can access, we only talk to nodes that can access us), Which will mean it will only notify for local services, so the developer still gets error notifications, But our errors won't be spread to all developers. BUT, we will still watch global errors, because we can contact the global server, so developers will still get errors about production issues, even while developing!  
+        if (isPublic()) {
+            let loggers = startupDone ? getLoggers() : undefined;
+            if (!loggers) {
+                getLoggers.reset();
+                setImmediate(() => {
+                    logDiskDontShim(type, ...args);
+                });
+                return;
+            }
+            const { logLogs, warnLogs, infoLogs, errorLogs } = loggers;
+            if (type === "log") {
+                logLogs.append(logObj);
+            } else if (type === "warn") {
+                warnLogs.append(logObj);
+            } else if (type === "info") {
+                infoLogs.append(logObj);
+            } else {
+                errorLogs.append(logObj);
+            }
         }
 
         if (type === "warn" || type === "error") {
@@ -162,12 +208,12 @@ let lastLogTime = 0;
 
 function packageLogObj(type: string, args: unknown[]): LogDatum {
     let now = Date.now();
-    if (now < lastLogTime) {
+    if (now <= lastLogTime) {
         now = addEpsilons(lastLogTime, 1);
     }
     lastLogTime = now;
     let logObj: LogDatum = {
-        time: now,
+        time: 0,
         __LOG_TYPE: type,
     };
     for (let part of globalContextParts) {
@@ -184,5 +230,6 @@ function packageLogObj(type: string, args: unknown[]): LogDatum {
             stringCount++;
         }
     }
+    logObj.time = now;
     return logObj;
 }

package/src/diagnostics/logs/errorNotifications/ErrorDigestPage.tsx

@@ -0,0 +1,409 @@
+module.noserverhotreload = false;
+module.hotreload = true;
+
+
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../../../4-dom/qreact";
+import { getSyncedController } from "../../../library-components/SyncedController";
+import { ErrorDigestController, errorDigestHistory, ErrorDigestInfo } from "./errorDigests";
+import { isManagementUser } from "../../../-0-hooks/hooks";
+import { assertIsManagementUser } from "../../managementPages";
+import { URLParam } from "../../../library-components/URLParam";
+import { tabURL } from "../../../library-components/urlResetGroups";
+import { Anchor, ATag } from "../../../library-components/ATag";
+import { css } from "../../../4-dom/css";
+import { list, sort } from "socket-function/src/misc";
+import { formatDateTime, formatVeryNiceDateTime, formatNumber } from "socket-function/src/formatting/format";
+import { Table } from "../../../5-diagnostics/Table";
+import { Histogram } from "../../../library-components/Histogram";
+import { TabbedUI } from "../../../library-components/TabbedUI";
+import { LogDatum } from "../diskLogger";
+import { t } from "../../../2-proxy/schema2";
+import { getErrorLogsLink } from "./ErrorWarning";
+
+export const digestKeyURL = new URLParam("digestKey", "");
+export const fileDetailTabURL = new URLParam("fileDetailTab", "errors");
+
+export class ErrorDigestPage extends qreact.Component {
+    state = t.state({
+        selectedFile: t.string("")
+    });
+    render() {
+        // Get the current tab mode
+        let currentTab = tabURL.value;
+        let selectedDigestKey = digestKeyURL.value;
+
+        if (currentTab === "detail" && selectedDigestKey) {
+            return this.renderDigestDetail(selectedDigestKey);
+        }
+
+        return this.renderDigestList();
+    }
+
+    renderDigestList() {
+        let controller = ErrorDigestController(SocketFunction.browserNodeId());
+
+        let keys = controller.getDigestKeys();
+
+        if (!keys || keys.length === 0) {
+            return (
+                <div className={css.vbox(16).pad2(24)}>
+                    <div className={css.fontSize(24).fontWeight(600)}>
+                        Error Digest History
+                    </div>
+                    <div>No digest data available</div>
+                </div>
+            );
+        }
+
+        // Sort lexicographically (highest = newest) and take latest 100
+        let sortedKeys = sort(keys as string[], key => key).reverse().slice(0, 100);
+
+        return (
+            <div className={css.vbox(16).paddingTop(24).paddingLeft(24).fillHeight.fillWidth}>
+                <div className={css.fontSize(24).fontWeight(600)}>
+                    Error Digest History
+                </div>
+                <div className={css.fontSize(14).color("gray")}>
+                    Showing latest {Math.min(sortedKeys.length, 100)} digests (sorted by creation time, newest first)
+                </div>
+
+                <div className={css.vbox(8).overflowAuto.fillWidth}>
+                    {sortedKeys.map(key => (
+                        <ATag
+                            key={key}
+                            values={[
+                                tabURL.getOverride("detail"),
+                                digestKeyURL.getOverride(key)
+                            ]}
+                            className={css.pad2(12).hsla(0, 0, 0, 0.1).pointer.hslahover(0, 0, 0, 0.15)}
+                        >
+                            {formatVeryNiceDateTime(+key.split(".")[0])}
+                        </ATag>
+                    ))}
+                </div>
+            </div>
+        );
+    }
+
+    renderDigestDetail(digestKey: string) {
+        let controller = ErrorDigestController(SocketFunction.browserNodeId());
+
+        let digestData = controller.getDigest(digestKey);
+
+        if (!digestData) {
+            return (
+                <div className={css.vbox(16).pad2(24)}>
+                    <div className={css.hbox(16)}>
+                        <ATag
+                            values={[
+                                tabURL.getOverride(""),
+                                digestKeyURL.getOverride("")
+                            ]}
+                        >
+                            ← Back to Digest List
+                        </ATag>
+                    </div>
+                    <div>Loading digest data...</div>
+                </div>
+            );
+        }
+
+        // If a file is selected, show file details
+        if (this.state.selectedFile) {
+            return this.renderFileDetail(digestData, digestKey);
+        }
+
+        return (
+            <div className={css.vbox(24).pad2(24).fillHeight.overflowAuto}>
+                {/* Header */}
+                <div className={css.hbox(16).alignItems("center")}>
+                    <ATag
+                        values={[
+                            tabURL.getOverride(""),
+                            digestKeyURL.getOverride("")
+                        ]}
+                    >
+                        ← Back to Digest List
+                    </ATag>
+                    <div className={css.fontSize(24).fontWeight(600)}>
+                        Digest: {formatVeryNiceDateTime(+digestKey.split(".")[0])}
+                    </div>
+                </div>
+
+                <Anchor
+                    className={css.fontSize(24)}
+                    values={getErrorLogsLink({
+                        startTime: digestData.startTime,
+                        endTime: digestData.endTime,
+                    })}
+                >
+                    View live logs
+                </Anchor>
+
+                {/* Metadata Section */}
+                {this.renderMetadata(digestData)}
+
+                {/* Charts Section */}
+                {this.renderCharts(digestData)}
+
+                {/* Files Table */}
+                {this.renderFilesTable(digestData)}
+            </div>
+        );
+    }
+
+    renderMetadata(digestData: ErrorDigestInfo) {
+        return (
+            <div className={css.vbox(16)}>
+                <div className={css.fontSize(20).fontWeight(600)}>Metadata</div>
+                <div className={css.hbox(32).wrap}>
+                    <div className={css.vbox(4)}>
+                        <div className={css.fontSize(14).colorhsl(0, 0, 60)}>Compressed Bytes</div>
+                        <div className={css.fontSize(16).fontWeight(500)}>
+                            {formatNumber(digestData.totalCompressedBytes)}
+                        </div>
+                    </div>
+                    <div className={css.vbox(4)}>
+                        <div className={css.fontSize(14).colorhsl(0, 0, 60)}>Uncompressed Bytes</div>
+                        <div className={css.fontSize(16).fontWeight(500)}>
+                            {formatNumber(digestData.totalUncompressedBytes)}
+                        </div>
+                    </div>
+                    <div className={css.vbox(4)}>
+                        <div className={css.fontSize(14).colorhsl(0, 0, 60)}>Total Files</div>
+                        <div className={css.fontSize(16).fontWeight(500)}>
+                            {formatNumber(digestData.totalFiles)}
+                        </div>
+                    </div>
+                    <div className={css.vbox(4)}>
+                        <div className={css.fontSize(14).colorhsl(0, 0, 60)}>Scan Duration</div>
+                        <div className={css.fontSize(16).fontWeight(500)}>
+                            {formatNumber(digestData.scanDuration)}ms
+                        </div>
+                    </div>
+                    <div className={css.vbox(4)}>
+                        <div className={css.fontSize(14).colorhsl(0, 0, 60)}>Start Time</div>
+                        <div className={css.fontSize(16).fontWeight(500)}>
+                            {formatVeryNiceDateTime(digestData.startTime)}
+                        </div>
+                    </div>
+                    <div className={css.vbox(4)}>
+                        <div className={css.fontSize(14).colorhsl(0, 0, 60)}>End Time</div>
+                        <div className={css.fontSize(16).fontWeight(500)}>
+                            {formatVeryNiceDateTime(digestData.endTime)}
+                        </div>
+                    </div>
+                </div>
+            </div>
+        );
+    }
+
+    renderCharts(digestData: ErrorDigestInfo) {
+        // Convert histogram data to chart data
+        let unsuppressedErrorsData: { x: number; y: number }[] = [];
+        let unsuppressedWarningsData: { x: number; y: number }[] = [];
+        let corruptData: { x: number; y: number }[] = [];
+        let suppressedData: { x: number; y: number }[] = [];
+
+        for (let [time, data] of digestData.histogram.entries()) {
+            unsuppressedErrorsData.push({ x: time, y: data.unsuppressedErrors });
+            unsuppressedWarningsData.push({ x: time, y: data.unsuppressedWarnings });
+            corruptData.push({ x: time, y: data.corruptErrors + data.corruptWarnings });
+            suppressedData.push({ x: time, y: data.suppressedErrors + data.suppressedWarnings });
+        }
+
+        return (
+            <div className={css.vbox(16)}>
+                <div className={css.fontSize(20).fontWeight(600)}>Error Distribution Over Time</div>
+                <div className={css.vbox(16)}>
+                    <div className={css.hbox(16).wrap}>
+                        <div className={css.fillWidth.minWidth(400).height(300)}>
+                            <Histogram
+                                title="Unsuppressed Errors"
+                                values={unsuppressedErrorsData}
+                                xColumn={{
+                                    title: "Time",
+                                    format: (value) => formatDateTime(value)
+                                }}
+                                yColumn={{ title: "Count" }}
+                            />
+                        </div>
+                        <div className={css.fillWidth.minWidth(400).height(300)}>
+                            <Histogram
+                                title="Unsuppressed Warnings"
+                                values={unsuppressedWarningsData}
+                                xColumn={{
+                                    title: "Time",
+                                    format: (value) => formatDateTime(value)
+                                }}
+                                yColumn={{ title: "Count" }}
+                            />
+                        </div>
+                    </div>
+                    <div className={css.hbox(16).wrap}>
+                        <div className={css.fillWidth.minWidth(400).height(300)}>
+                            <Histogram
+                                title="Corrupt Errors + Warnings"
+                                values={corruptData}
+                                xColumn={{
+                                    title: "Time",
+                                    format: (value) => formatDateTime(value)
+                                }}
+                                yColumn={{ title: "Count" }}
+                            />
+                        </div>
+                        <div className={css.fillWidth.minWidth(400).height(300)}>
+                            <Histogram
+                                title="Suppressed Errors + Warnings"
+                                values={suppressedData}
+                                xColumn={{
+                                    title: "Time",
+                                    format: (value) => formatDateTime(value)
+                                }}
+                                yColumn={{ title: "Count" }}
+                            />
+                        </div>
+                    </div>
+                </div>
+            </div>
+        );
+    }
+
+    renderFilesTable(digestData: ErrorDigestInfo) {
+        // Convert byFile map to table rows
+        let fileRows = Array.from(digestData.byFile.entries()).map(([fileName, fileData]) => {
+            let latestError = fileData.latestErrors[fileData.latestErrors.length - 1];
+            let latestWarning = fileData.latestWarnings[fileData.latestWarnings.length - 1];
+            let latestLog = latestError ?? latestWarning;
+
+            return {
+                fileName,
+                errorCount: fileData.errors,
+                warningCount: fileData.warnings,
+                latestMessage: latestLog?.param0 ?? "No messages",
+                latestTime: latestLog?.time ?? 0
+            };
+        });
+
+        // Sort by error count descending
+        fileRows = sort(fileRows, row => row.errorCount).reverse();
+
+        return (
+            <div className={css.vbox(16)}>
+                <div className={css.fontSize(20).fontWeight(600)}>Files ({fileRows.length} files)</div>
+                <Table
+                    columns={{
+                        fileName: { title: "File Name" },
+                        errorCount: { title: "Errors" },
+                        warningCount: { title: "Warnings" },
+                        latestMessage: {
+                            title: "Latest Message",
+                            formatter: (value: any) => String(value).slice(0, 100) + (String(value).length > 100 ? "..." : "")
+                        }
+                    }}
+                    rows={fileRows}
+                    getRowAttributes={(row: any) => ({
+                        className: css.cursor("pointer").hslhover(210, 20, 95),
+                        onClick: () => {
+                            this.state.selectedFile = row.fileName;
+                        }
+                    })}
+                />
+            </div>
+        );
+    }
+
+    renderFileDetail(digestData: ErrorDigestInfo, digestKey: string) {
+        let selectedFileName = this.state.selectedFile;
+        let fileData = digestData.byFile.get(selectedFileName);
+
+        if (!fileData) {
+            return (
+                <div className={css.vbox(16).pad2(24)}>
+                    <div>File not found</div>
+                </div>
+            );
+        }
+
+        return (
+            <div className={css.vbox(24).pad2(24).fillHeight}>
+                {/* Header with back button */}
+                <div className={css.hbox(16).alignItems("center")}>
+                    <ATag
+                        clickOverride={() => {
+                            this.state.selectedFile = "";
+                        }}
+                        className={css.cursor("pointer")}
+                    >
+                        ← Back to Files Table
+                    </ATag>
+                    <div className={css.fontSize(20).fontWeight(600)}>
+                        {selectedFileName}
+                    </div>
+                </div>
+
+                {/* File stats */}
+                <div className={css.hbox(32)}>
+                    <div className={css.vbox(4)}>
+                        <div className={css.fontSize(14).colorhsl(0, 0, 60)}>Errors</div>
+                        <div className={css.fontSize(16).fontWeight(500)}>{fileData.errors}</div>
+                    </div>
+                    <div className={css.vbox(4)}>
+                        <div className={css.fontSize(14).colorhsl(0, 0, 60)}>Warnings</div>
+                        <div className={css.fontSize(16).fontWeight(500)}>{fileData.warnings}</div>
+                    </div>
+                </div>
+
+                {/* Tabbed interface for errors/warnings */}
+                <div className={css.fillHeight.overflowHidden}>
+                    <TabbedUI
+                        tab={fileDetailTabURL}
+                        tabs={[
+                            {
+                                value: "errors",
+                                title: `Errors (${fileData.errors})`,
+                                contents: this.renderLogEntries(fileData.latestErrors, "errors")
+                            },
+                            {
+                                value: "warnings",
+                                title: `Warnings (${fileData.warnings})`,
+                                contents: this.renderLogEntries(fileData.latestWarnings, "warnings")
+                            }
+                        ]}
+                    />
+                </div>
+            </div>
+        );
+    }
+
+    renderLogEntries(logs: LogDatum[], type: "errors" | "warnings") {
+        if (logs.length === 0) {
+            return <div className={css.colorhsl(0, 0, 60)}>No {type} found</div>;
+        }
+
+        let tableRows = logs.map((log, index) => ({
+            index: logs.length - index,
+            time: formatVeryNiceDateTime(log.time),
+            message: log.param0 ?? "No message",
+            threadId: log.__threadId ?? "Unknown",
+            name: log.__NAME__ ?? "Unknown"
+        }));
+
+        return (
+            <div className={css.fillHeight.overflowAuto}>
+                <Table
+                    columns={{
+                        index: { title: "#" },
+                        time: { title: "Time" },
+                        name: { title: "Source" },
+                        threadId: { title: "Thread" },
+                        message: { title: "Message" }
+                    }}
+                    rows={tableRows}
+                    initialLimit={50}
+                />
+            </div>
+        );
+    }
+}