querysub 0.326.0 → 0.328.0

package/src/diagnostics/logs/LogViewer2.tsx

@@ -4,7 +4,7 @@ import { css } from "../../4-dom/css";
 import { URLParam } from "../../library-components/URLParam";
 import { InputLabelURL } from "../../library-components/InputLabel";
 import { Button } from "../../library-components/Button";
-import { formatDateTime, formatNiceDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { formatDateTime, formatDateTimeDetailed, formatNiceDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { TimeRangeSelector, endTimeParam, getTimeRange, startTimeParam } from "./TimeRangeSelector";
 import { FastArchiveAppendable } from "./FastArchiveAppendable";
 import { t } from "../../2-proxy/schema2";
@@ -13,8 +13,8 @@ import { logErrors } from "../../errors";
 import { batchFunction, runInSerial } from "socket-function/src/batching";
 import { Querysub } from "../../4-querysub/QuerysubController";
 import { sort, timeInDay, timeInHour } from "socket-function/src/misc";
-import { FastArchiveViewer, filterParam } from "./FastArchiveViewer";
-import { LogDatum, getLoggers, LOG_LIMIT_FLAG } from "./diskLogger";
+import { FastArchiveViewer, cacheBustParam, filterParam } from "./FastArchiveViewer";
+import { LogDatum, getLoggers, LOG_LIMIT_FLAG, LOG_LINE_LIMIT_FLAG } from "./diskLogger";
 import { ColumnType, Table, TableType } from "../../5-diagnostics/Table";
 import { formatDateJSX } from "../../misc/formatJSX";
 import { InputPicker } from "../../library-components/InputPicker";
@@ -24,8 +24,10 @@ import { ObjectDisplay } from "./ObjectDisplay";
 import { endTime } from "../misc-pages/archiveViewerShared";
 import { ErrorSuppressionUI } from "./errorNotifications/ErrorSuppressionUI";
 import { FileMetadata } from "./FastArchiveController";
-import { SuppressionListController, getSuppressEntryChecker, getSuppressionFull } from "./errorNotifications/ErrorNotificationController";
+import { RecentErrorsController, SuppressionListController, getSuppressEntryChecker, getSuppressionFull } from "./errorNotifications/ErrorNotificationController";
 import { SocketFunction } from "socket-function/SocketFunction";
+import { throttleRender } from "../../functional/throttleRender";
+import { isNode } from "typesafecss";
 
 const RENDER_INTERVAL = 1000;
 
@@ -36,6 +38,7 @@ const enableInfosURL = new URLParam("enableInfos", true);
 const enableWarningsURL = new URLParam("enableWarnings", true);
 const enableErrorsURL = new URLParam("enableErrors", true);
 const selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, boolean>);
+const useRelativeTimeURL = new URLParam("useRelativeTime", true);
 
 export const errorNotifyToggleURL = new URLParam("errorNotifyToggle", false);
 
@@ -70,10 +73,12 @@ export class LogViewer2 extends qreact.Component {
     private fastArchiveViewer: FastArchiveViewer<LogDatum> | undefined = undefined;
 
     rerun() {
+        cacheBustParam.value = Date.now();
         void this.fastArchiveViewer?.handleDownload();
     }
 
     render() {
+        if (throttleRender({ key: "LogViewer2", frameDelay: 30 })) return undefined;
 
         this.state.datumsSeqNum;
 
@@ -119,11 +124,17 @@ export class LogViewer2 extends qreact.Component {
                                 let now = Date.now();
                                 startTimeParam.value = now - timeInDay * 7;
                                 endTimeParam.value = now + timeInHour * 2;
-                                filterParam.value = "";
                             }
                             this.rerun();
                         }}
                     />
+                    {errorNotifyToggleURL.value && <div className={css.hbox(10)}>
+                        <Button onClick={() => {
+                            void RecentErrorsController(SocketFunction.browserNodeId()).raiseTestError.promise("Test error notification");
+                        }}>
+                            Test Error Notification (won't rerun search, but it should show up in the title)
+                        </Button>
+                    </div>}
                 </div>
 
                 {!errorNotifyToggleURL.value && <div className={css.hbox(10)}>
@@ -144,7 +155,8 @@ export class LogViewer2 extends qreact.Component {
                 <FastArchiveViewer
                     ref2={x => this.fastArchiveViewer = x}
                     fastArchives={logs}
-                    onStart={() => {
+                    runOnLoad={errorNotifyToggleURL.value}
+                    onStart={async () => {
                         this.datumCount = 0;
                         this.notMatchedCount = 0;
                         this.errors = 0;
@@ -167,6 +179,9 @@ export class LogViewer2 extends qreact.Component {
                                 });
                             }
                         })();
+                        // ALWAYS update it, as the synchronous one might be out of date, and if we use an outdated one extra errors show up.
+                        suppressionList = await suppressionController.getSuppressionList.promise();
+
                     }}
                     getWantData={async (file) => {
                         if (!hasErrorNotifyToggle) return undefined;
@@ -238,6 +253,13 @@ export class LogViewer2 extends qreact.Component {
                                 anyLimited = true;
                             }
                         }
+                        let lineLimited = false;
+                        for (let i = 0; i < Math.min(1000, this.datums.length); i++) {
+                            let datum = this.datums[i];
+                            if (datum[LOG_LINE_LIMIT_FLAG]) {
+                                lineLimited = true;
+                            }
+                        }
                         let selectedFields: string[] = [];
                         for (let field of Object.keys(defaultSelectedFields)) {
                             if (atomic(selectedFieldsURL.value[field]) === undefined) {
@@ -261,15 +283,25 @@ export class LogViewer2 extends qreact.Component {
                         if (anyLimited) {
                             columns[LOG_LIMIT_FLAG] = {
                                 title: "Limited",
-                                formatter: x => x && <div className={css.hsl(0, 50, 50).colorhsl(0, 50, 95).boldStyle.pad2(10).ellipsis}>
-                                    Log Line Throttled
+                                formatter: x => x && <div className={css.hsl(0, 50, 50).colorhsl(0, 50, 95).boldStyle.pad2(10).ellipsis}
+                                    title="File throttled (other line may be lost lost!)">
+                                    FILE throttled
+                                </div> || undefined
+                            };
+                        }
+                        if (lineLimited) {
+                            columns[LOG_LINE_LIMIT_FLAG] = {
+                                title: "Limited",
+                                formatter: x => x && <div className={css.hsl(60, 50, 50).colorhsl(60, 50, 95).boldStyle.pad2(10).ellipsis}
+                                    title="Line throttled (no lines lost, just count for this line is lower than the actual count)">
+                                    Line throttled
                                 </div> || undefined
                             };
                         }
                         for (let field of selectedFields) {
                             let column: ColumnType<unknown, LogDatum> = {};
                             if (field === "time") {
-                                column.formatter = (x: unknown) => formatDateTime(Number(x));
+                                column.formatter = (x: unknown) => useRelativeTimeURL.value ? formatDateJSX(Number(x)) : <span title={formatDateTimeDetailed(Number(x))}>{formatDateTime(Number(x))}</span>;
                             }
                             if (!column.formatter) {
                                 column.formatter = (x: unknown) => <ObjectDisplay value={x} />;
@@ -298,38 +330,54 @@ export class LogViewer2 extends qreact.Component {
                             }
                         };
                         return <>
-                            <InputPicker
-                                label={<div className={css.hbox(10)}>
-                                    <div className={css.flexShrink0}>
-                                        Selected Fields
-                                    </div>
-                                    <Button onClick={() => {
+                            <div className={css.hbox(10)}>
+                                <InputPicker
+                                    label={<div className={css.hbox(10)}>
+                                        <div className={css.flexShrink0}>
+                                            Selected Fields
+                                        </div>
+                                        <Button onClick={() => {
+                                            let newValues = { ...selectedFieldsURL.value };
+                                            for (let key of Object.keys(newValues)) {
+                                                newValues[key] = key in defaultSelectedFields;
+                                            }
+                                            selectedFieldsURL.value = newValues;
+                                        }}>
+                                            Reset
+                                        </Button>
+                                    </div>}
+                                    picked={selectedFields}
+                                    options={Array.from(fieldNames).map(x => ({ value: x, label: x }))}
+                                    addPicked={x => {
                                         let newValues = { ...selectedFieldsURL.value };
-                                        for (let key of Object.keys(newValues)) {
-                                            newValues[key] = key in defaultSelectedFields;
-                                        }
+                                        newValues[x] = true;
                                         selectedFieldsURL.value = newValues;
-                                    }}>
-                                        Reset
-                                    </Button>
-                                </div>}
-                                picked={selectedFields}
-                                options={Array.from(fieldNames).map(x => ({ value: x, label: x }))}
-                                addPicked={x => {
-                                    let newValues = { ...selectedFieldsURL.value };
-                                    newValues[x] = true;
-                                    selectedFieldsURL.value = newValues;
-                                }}
-                                removePicked={x => {
-                                    let newValues = { ...selectedFieldsURL.value };
-                                    newValues[x] = false;
-                                    selectedFieldsURL.value = newValues;
-                                }}
-                            />
+                                    }}
+                                    removePicked={x => {
+                                        let newValues = { ...selectedFieldsURL.value };
+                                        newValues[x] = false;
+                                        selectedFieldsURL.value = newValues;
+                                    }}
+                                />
+                                <InputLabelURL
+                                    label="Use Relative Time"
+                                    checkbox
+                                    url={useRelativeTimeURL}
+                                    onChangeValue={() => {
+                                        Querysub.commit(() => {
+                                            this.state.datumsSeqNum++;
+                                        });
+                                    }}
+                                />
+                            </div>
+                            {this.datums.length === 0 && <div className={css.hsl(40, 50, 50).colorhsl(60, 50, 100).boldStyle.pad2(10).ellipsis}>
+                                No logs matched, either increase the time range or decrease the filter specificity.
+                            </div>}
                             <Table
                                 rows={this.datums}
                                 columns={columns}
                                 lineLimit={4}
+                                initialLimit={10}
                                 characterLimit={400}
                                 getRowAttributes={row => {
                                     let hue = -1;

package/src/diagnostics/logs/TimeRangeSelector.tsx

@@ -82,6 +82,14 @@ export class TimeRangeSelector extends qreact.Component {
                     >
                         Set to future data
                     </Button>
+                    <Button
+                        hue={110} onClick={() => {
+                            startTimeParam.value = now - timeInHour;
+                            endTimeParam.value = now + timeInHour * 2;
+                        }}
+                    >
+                        Set to last hour
+                    </Button>
                     <Button
                         hue={110} onClick={() => {
                             startTimeParam.value = now - timeInDay;

package/src/diagnostics/logs/diskLogGlobalContext.ts

@@ -1,7 +1,7 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { lazy } from "socket-function/src/caching";
 import { getExternalIP } from "socket-function/src/networking";
-import { decodeNodeId } from "../../-a-auth/certs";
+import { decodeNodeId, getOwnMachineId, getOwnThreadId } from "../../-a-auth/certs";
 import { getOwnNodeId } from "../../-f-node-discovery/NodeDiscovery";
 import { logErrors } from "../../errors";
 import { addGlobalContext } from "./diskLogger";
@@ -12,9 +12,9 @@ export function addBuiltInContext() {
         let nodeId = getOwnNodeId();
         let nodeParts = decodeNodeId(nodeId);
         return {
-            __machineId: nodeParts?.machineId,
+            __machineId: getOwnMachineId(),
             __mountId: SocketFunction.mountedNodeId,
-            __threadId: nodeParts?.threadId,
+            __threadId: getOwnThreadId(),
             __port: nodeParts?.port,
             __nodeId: nodeId,
             __entry: process.argv[1],

package/src/diagnostics/logs/diskLogger.ts

@@ -6,6 +6,8 @@ import { timeInMinute } from "socket-function/src/misc";
 import { formatTime } from "socket-function/src/formatting/format";
 import { addEpsilons } from "../../bits";
 import { FileMetadata } from "./FastArchiveController";
+import { getPathStr2 } from "../../path";
+import { isPublic } from "../../config";
 // IMPORTANT! We can't have any real imports here, because we are depended on so early in startup!
 
 if (isNode()) {
@@ -35,7 +37,25 @@ export type LogDatum = Record<string, unknown> & {
     /** Dynamically set when matching recent errors only. */
     __matchedOutdatedSuppressionKey?: string;
 };
+export function getLogHash(obj: LogDatum) {
+    return getPathStr2(obj.__threadId || "", obj.time.toString());
+}
+export function getLogFile(obj: LogDatum) {
+    let logType = obj.param0 || "";
+    if (obj.__FILE__) {
+        logType = String(obj.__FILE__);
+    }
+    if (obj[LOG_LINE_LIMIT_ID]) {
+        logType += "::" + String(obj[LOG_LINE_LIMIT_ID]);
+    }
+    return logType;
+
+}
 export const LOG_LIMIT_FLAG = String.fromCharCode(44533) + "LOGS_LIMITED_FLAG-9277640b-d709-4591-ab08-2bb29bbb94f4";
+export const LOG_LINE_LIMIT_FLAG = String.fromCharCode(44534) + "LOGS_LINE_LIMIT_FLAG-dd50ab1f-3021-45e3-82fc-d2702c7a64c8";
+
+/** If this key exists in the logged object, as in a key in one of the objects logged, then we will use the value of it as the limit ID. This is useful as it allows us to either override a limit or limit something independently from other logs in the file. */
+export const LOG_LINE_LIMIT_ID = "LIMIT_LINE_ID";
 
 export const getLoggers = lazy(function () {
     const { FastArchiveAppendable } = require("./FastArchiveAppendable") as typeof import("./FastArchiveAppendable");
@@ -52,6 +72,10 @@ export const getLoggers = lazy(function () {
         errorLogs: new FastArchiveAppendable<LogDatum>("logs-error/"),
     };
 });
+setImmediate(() => {
+    // If we don't import it at all, then it doesn't work client-side. 
+    require("./FastArchiveAppendable") as typeof import("./FastArchiveAppendable");
+});
 const getNotifyErrors = lazy(function () {
     const { notifyWatchersOfError: notifyErrors } = require("./errorNotifications/ErrorNotificationController") as typeof import("./errorNotifications/ErrorNotificationController");
     if (typeof notifyErrors !== "function") {
@@ -88,6 +112,8 @@ let logLimitLookup: {
 
 const LIMIT_PERIOD = timeInMinute * 15;
 const LIMIT_THRESHOLD = 1000;
+const WARN_LIMIT = 100;
+const ERROR_LIMIT = 100;
 
 const logDiskDontShim = logDisk;
 /** NOTE: Calling this directly means we lose __FILE__ tracking. But... that's probably fine... */
@@ -104,6 +130,11 @@ export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknow
         if (logObj.__FILE__) {
             logType = String(logObj.__FILE__);
         }
+        let hasLineLimit = false;
+        if (logObj[LOG_LINE_LIMIT_ID]) {
+            logType += "::" + String(logObj[LOG_LINE_LIMIT_ID]);
+            hasLineLimit = true;
+        }
 
         if (logLimitLookup) {
             if (logObj.time > logLimitLookup.resetTime) {
@@ -120,32 +151,47 @@ export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknow
         let count = logLimitLookup.counts.get(logType) || 0;
         count++;
         logLimitLookup.counts.set(logType, count);
-        if (count > LIMIT_THRESHOLD) {
+        let limit = LIMIT_THRESHOLD;
+        if (type === "warn") {
+            limit = WARN_LIMIT;
+        } else if (type === "error") {
+            limit = ERROR_LIMIT;
+        }
+        if (count > limit) {
             let timeUntilReset = logLimitLookup.resetTime - logObj.time;
-            process.stdout.write(`Log type hit limit, not writing log type to disk for ~${formatTime(timeUntilReset)}: ${logType}\n`);
+            if (hasLineLimit) {
+                process.stdout.write(`Log type hit limit, not writing log type to disk for ~${formatTime(timeUntilReset)}: ${logType}\n`);
+            }
             return;
         }
-        if (count === LIMIT_THRESHOLD) {
-            logObj[LOG_LIMIT_FLAG] = true;
+        if (count >= limit) {
+            if (hasLineLimit) {
+                logObj[LOG_LINE_LIMIT_FLAG] = true;
+            } else {
+                logObj[LOG_LIMIT_FLAG] = true;
+            }
         }
 
-        let loggers = startupDone ? getLoggers() : undefined;
-        if (!loggers) {
-            getLoggers.reset();
-            setImmediate(() => {
-                logDiskDontShim(type, ...args);
-            });
-            return;
-        }
-        const { logLogs, warnLogs, infoLogs, errorLogs } = loggers;
-        if (type === "log") {
-            logLogs.append(logObj);
-        } else if (type === "warn") {
-            warnLogs.append(logObj);
-        } else if (type === "info") {
-            infoLogs.append(logObj);
-        } else {
-            errorLogs.append(logObj);
+        // We don't want developer errors clogging up the error logs. However, they can still notify errors, Because this will only notify nodes that are able to access us (It uses a reverse connection scheme, so instead of talking to nodes that we can access, we only talk to nodes that can access us), Which will mean it will only notify for local services, so the developer still gets error notifications, But our errors won't be spread to all developers. BUT, we will still watch global errors, because we can contact the global server, so developers will still get errors about production issues, even while developing!  
+        if (isPublic()) {
+            let loggers = startupDone ? getLoggers() : undefined;
+            if (!loggers) {
+                getLoggers.reset();
+                setImmediate(() => {
+                    logDiskDontShim(type, ...args);
+                });
+                return;
+            }
+            const { logLogs, warnLogs, infoLogs, errorLogs } = loggers;
+            if (type === "log") {
+                logLogs.append(logObj);
+            } else if (type === "warn") {
+                warnLogs.append(logObj);
+            } else if (type === "info") {
+                infoLogs.append(logObj);
+            } else {
+                errorLogs.append(logObj);
+            }
         }
 
         if (type === "warn" || type === "error") {
@@ -162,12 +208,12 @@ let lastLogTime = 0;
 
 function packageLogObj(type: string, args: unknown[]): LogDatum {
     let now = Date.now();
-    if (now < lastLogTime) {
+    if (now <= lastLogTime) {
         now = addEpsilons(lastLogTime, 1);
     }
     lastLogTime = now;
     let logObj: LogDatum = {
-        time: now,
+        time: 0,
         __LOG_TYPE: type,
     };
     for (let part of globalContextParts) {
@@ -184,5 +230,6 @@ function packageLogObj(type: string, args: unknown[]): LogDatum {
             stringCount++;
         }
     }
+    logObj.time = now;
     return logObj;
 }