querysub 0.355.0 → 0.357.0

package/src/diagnostics/logs/LogViewer2.tsx

@@ -10,7 +10,7 @@ import { FastArchiveAppendable } from "./FastArchiveAppendable";
 import { t } from "../../2-proxy/schema2";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { logErrors } from "../../errors";
-import { batchFunction, runInSerial } from "socket-function/src/batching";
+import { batchFunction, delay, runInSerial } from "socket-function/src/batching";
 import { Querysub } from "../../4-querysub/QuerysubController";
 import { sort, timeInDay, timeInHour } from "socket-function/src/misc";
 import { FastArchiveViewer, cacheBustParam, filterParam, filterParam2 } from "./FastArchiveViewer";
@@ -29,6 +29,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { throttleRender } from "../../functional/throttleRender";
 import { isNode } from "typesafecss";
 import { createLogViewerExtractField } from "./logViewerExtractField";
+import { isPublic } from "../../config";
 
 const RENDER_INTERVAL = 1000;
 
@@ -42,6 +43,7 @@ const selectedFieldsURL = new URLParam("selectedFields", {} as Record<string, bo
 const useRelativeTimeURL = new URLParam("useRelativeTime", true);
 
 export const errorNotifyToggleURL = new URLParam("errorNotifyToggle", false);
+const forceGetPublicURL = new URLParam("forceGetPublic", false);
 
 const defaultSelectedFields = {
     param0: true,
@@ -58,6 +60,7 @@ export class LogViewer2 extends qreact.Component {
     state = t.state({
         datumsSeqNum: t.atomic<number>(0),
         showCountByFile: t.boolean,
+        showSizeByFile: t.boolean,
     });
 
     private example: string | undefined = undefined;
@@ -69,6 +72,7 @@ export class LogViewer2 extends qreact.Component {
     private notMatchedCount = 0;
     private datums: LogDatum[] = [];
     private countsPerName = new Map<string, number>();
+    private sizePerName = new Map<string, number>();
 
     private lastRenderTime = 0;
     private suppressionCounts = new Map<string, number>();
@@ -76,19 +80,15 @@ export class LogViewer2 extends qreact.Component {
     private fastArchiveViewer: FastArchiveViewer<LogDatum> | undefined = undefined;
 
     rerun() {
-        cacheBustParam.value = Date.now();
-        void this.fastArchiveViewer?.synchronizeDataThrottled();
+        Querysub.onCommitFinished(async () => {
+            void this.fastArchiveViewer?.synchronizeDataThrottled();
+        });
     }
-
-    render() {
-        if (throttleRender({ key: "LogViewer2", frameDelay: 30 })) return undefined;
-
-        this.state.datumsSeqNum;
-
+    getLoggers = () => {
         let logs: FastArchiveAppendable<LogDatum>[] = [];
         let loggers = getLoggers();
         if (!loggers) {
-            return `Loggers not available?`;
+            throw new Error("Loggers not available?");
         }
         let { logLogs, warnLogs, infoLogs, errorLogs } = loggers;
         if (errorNotifyToggleURL.value) {
@@ -108,6 +108,13 @@ export class LogViewer2 extends qreact.Component {
                 logs.push(errorLogs);
             }
         }
+        return logs;
+    };
+
+    render() {
+        //if (throttleRender({ key: "LogViewer2", frameDelay: 30 })) return undefined;
+
+        this.state.datumsSeqNum;
 
         let suppressionController = SuppressionListController(SocketFunction.browserNodeId());
         let suppressionList = suppressionController.getSuppressionList();
@@ -138,6 +145,12 @@ export class LogViewer2 extends qreact.Component {
                             Test Error Notification (won't rerun search, but it should show up in the title)
                         </Button>
                     </div>}
+                    {!isPublic() && <InputLabelURL
+                        label="Force Public Logs" checkbox url={forceGetPublicURL}
+                        onChangeValue={(newValue) => {
+                            this.rerun();
+                        }}
+                    />}
                 </div>
 
                 {!errorNotifyToggleURL.value && <div className={css.hbox(10)}>
@@ -157,8 +170,9 @@ export class LogViewer2 extends qreact.Component {
 
                 <FastArchiveViewer
                     ref2={x => this.fastArchiveViewer = x}
-                    fastArchives={logs}
+                    fastArchives={this.getLoggers}
                     runOnLoad={errorNotifyToggleURL.value}
+                    forceGetPublic={forceGetPublicURL.value}
                     onStart={async () => {
                         this.datumCount = 0;
                         this.notMatchedCount = 0;
@@ -188,13 +202,15 @@ export class LogViewer2 extends qreact.Component {
 
                     }}
                     getScanFnc={() => {
-                        if (!this.state.showCountByFile) {
+                        if (!this.state.showCountByFile && !this.state.showSizeByFile) {
                             return () => false;
                         }
-                        return createLogViewerExtractField("name", (value) => {
+                        return createLogViewerExtractField("__NAME__", (value, size) => {
                             let prevCount = this.countsPerName.get(value) ?? 0;
                             prevCount++;
                             this.countsPerName.set(value, prevCount);
+                            let prevSize = this.sizePerName.get(value) ?? 0;
+                            this.sizePerName.set(value, prevSize + size);
                         });
                     }}
                     getWantData={async (file) => {
@@ -418,50 +434,99 @@ export class LogViewer2 extends qreact.Component {
                                 </div>
                             </div>}
 
-                            {this.state.showCountByFile && (() => {
-                                let counts = Array.from(this.countsPerName.entries());
-                                sort(counts, x => -x[1]);
-                                let topCounts = counts.slice(0, 20);
-                                let totalCount = counts.reduce((sum, [, count]) => sum + count, 0);
+                            <div className={css.hbox(10)}>
+                                {this.state.showCountByFile && (() => {
+                                    let counts = Array.from(this.countsPerName.entries());
+                                    sort(counts, x => -x[1]);
+                                    let topCounts = counts.slice(0, 20);
+                                    let totalCount = counts.reduce((sum, [, count]) => sum + count, 0);
 
-                                return <div className={css.vbox(10)}>
-                                    <div className={css.hbox(10, 10).wrap}>
-                                        {topCounts.map(([name, count]) => {
-                                            let percentage = totalCount > 0 ? (count / totalCount) * 100 : 0;
-                                            let isFiltered = includedFiles.includes(name);
+                                    return <div className={css.vbox(10)}>
+                                        <div className={css.hbox(10, 10).wrap}>
+                                            {topCounts.map(([name, count]) => {
+                                                let percentage = totalCount > 0 ? (count / totalCount) * 100 : 0;
+                                                let isFiltered = includedFiles.includes(name);
+                                                let size = this.sizePerName.get(name) ?? 0;
 
-                                            return <div
-                                                key={name}
-                                                className={css.relative.pad2(8, 4).bord2(210, 30, 60, 1).ellipsis.button}
-                                                title={isFiltered ? `Click to remove ${name} from filter` : `Click to add ${name} to filter`}
-                                                onClick={() => {
-                                                    if (isFiltered) {
-                                                        // Remove from filter
-                                                        let newFiles = includedFiles.filter(x => x !== name);
-                                                        filterParam2.value = newFiles.join("|");
-                                                    } else {
-                                                        // Add to filter
-                                                        let newFiles = [...includedFiles.filter(x => x), name];
-                                                        filterParam2.value = newFiles.join("|");
-                                                    }
-                                                }}
-                                            >
-                                                <div
-                                                    className={css.absolute.pos(0, 0).height("100%").width(`${percentage}%`).hsl(isFiltered ? 120 : 210, 40, 80)}
-                                                />
-                                                <div className={css.relative.maxWidth(200).ellipsis}>
-                                                    {formatNumber(count)} | {name || "(no name present)"} {isFiltered && "✓" || ""}
-                                                </div>
-                                            </div>;
-                                        })}
-                                    </div>
-                                </div>;
-                            })() || <Button onClick={() => {
-                                this.state.showCountByFile = true;
-                                this.rerun();
-                            }}>
-                                    Show Count by File
-                                </Button>}
+                                                return <div
+                                                    key={name}
+                                                    className={css.relative.pad2(8, 4).bord2(210, 30, 60, 1).ellipsis.button}
+                                                    title={isFiltered ? `Click to remove ${name} from filter` : `Click to add ${name} to filter`}
+                                                    onClick={() => {
+                                                        if (isFiltered) {
+                                                            // Remove from filter
+                                                            let newFiles = includedFiles.filter(x => x !== name);
+                                                            filterParam2.value = newFiles.join("|");
+                                                        } else {
+                                                            // Add to filter
+                                                            let newFiles = [...includedFiles.filter(x => x), name];
+                                                            filterParam2.value = newFiles.join("|");
+                                                        }
+                                                    }}
+                                                >
+                                                    <div
+                                                        className={css.absolute.pos(0, 0).height("100%").width(`${percentage}%`).hsl(isFiltered ? 120 : 210, 40, 80)}
+                                                    />
+                                                    <div className={css.relative.maxWidth(200).ellipsis}>
+                                                        {formatNumber(count)} ({formatNumber(size)}B) | {name || "(no name present)"} {isFiltered && "✓" || ""}
+                                                    </div>
+                                                </div>;
+                                            })}
+                                        </div>
+                                    </div>;
+                                })() || <Button onClick={() => {
+                                    this.state.showCountByFile = true;
+                                    this.rerun();
+                                }}>
+                                        Show Count by File
+                                    </Button>}
+
+                                {this.state.showSizeByFile && (() => {
+                                    let sizes = Array.from(this.sizePerName.entries());
+                                    sort(sizes, x => -x[1]);
+                                    let topSizes = sizes.slice(0, 20);
+                                    let totalSize = sizes.reduce((sum, [, size]) => sum + size, 0);
+
+                                    return <div className={css.vbox(10)}>
+                                        <div className={css.hbox(10, 10).wrap}>
+                                            {topSizes.map(([name, size]) => {
+                                                let percentage = totalSize > 0 ? (size / totalSize) * 100 : 0;
+                                                let isFiltered = includedFiles.includes(name);
+                                                let count = this.countsPerName.get(name) ?? 0;
+
+                                                return <div
+                                                    key={name}
+                                                    className={css.relative.pad2(8, 4).bord2(280, 30, 60, 1).ellipsis.button}
+                                                    title={isFiltered ? `Click to remove ${name} from filter` : `Click to add ${name} to filter`}
+                                                    onClick={() => {
+                                                        if (isFiltered) {
+                                                            // Remove from filter
+                                                            let newFiles = includedFiles.filter(x => x !== name);
+                                                            filterParam2.value = newFiles.join("|");
+                                                        } else {
+                                                            // Add to filter
+                                                            let newFiles = [...includedFiles.filter(x => x), name];
+                                                            filterParam2.value = newFiles.join("|");
+                                                        }
+                                                    }}
+                                                >
+                                                    <div
+                                                        className={css.absolute.pos(0, 0).height("100%").width(`${percentage}%`).hsl(isFiltered ? 120 : 280, 40, 80)}
+                                                    />
+                                                    <div className={css.relative.maxWidth(200).ellipsis}>
+                                                        {formatNumber(size)}B ({formatNumber(count)}) | {name || "(no name present)"} {isFiltered && "✓" || ""}
+                                                    </div>
+                                                </div>;
+                                            })}
+                                        </div>
+                                    </div>;
+                                })() || <Button onClick={() => {
+                                    this.state.showSizeByFile = true;
+                                    this.rerun();
+                                }}>
+                                        Show Size by File
+                                    </Button>}
+                            </div>
                             <Table
                                 rows={this.datums}
                                 columns={columns}

package/src/diagnostics/logs/TimeRangeSelector.tsx

@@ -20,8 +20,8 @@ export function getTimeRange(): { startTime: number; endTime: number } {
     const defaultEnd = now + timeInHour;
 
     return {
-        startTime: startTimeParam.value ?? defaultStart,
-        endTime: endTimeParam.value ?? defaultEnd,
+        startTime: startTimeParam.value || defaultStart,
+        endTime: endTimeParam.value || defaultEnd,
     };
 }
 
@@ -73,6 +73,9 @@ export class TimeRangeSelector extends qreact.Component {
                         }}
                         outerClass={!endTimeParam.value && css.opacity(0.5) || ""}
                     />
+                </div>
+                <div className={css.hbox(12).wrap}>
+
                     <Button
                         hue={110} onClick={() => {
                             let now = Date.now();

package/src/diagnostics/logs/diskLogger.ts

@@ -98,6 +98,33 @@ const getNotifyErrors = lazy(function () {
     return notifyErrors;
 });
 
+export const getLoggers2 = lazy(function () {
+    const { IndexedLogs } = require("./IndexedLogs/IndexedLogs") as typeof import("./IndexedLogs/IndexedLogs");
+
+    if (!IndexedLogs) {
+        setImmediate(() => {
+            getLoggers2.reset();
+        });
+        return undefined;
+    }
+
+    return {
+        logLogs: new IndexedLogs<LogDatum>({ name: "logs/log", getTime: x => x.time }),
+        warnLogs: new IndexedLogs<LogDatum>({ name: "logs/warn", getTime: x => x.time }),
+        infoLogs: new IndexedLogs<LogDatum>({ name: "logs/info", getTime: x => x.time }),
+        errorLogs: new IndexedLogs<LogDatum>({ name: "logs/error", getTime: x => x.time }),
+    };
+});
+export const getLoggers2Async = lazy(async () => {
+    const { IndexedLogs } = await import("./IndexedLogs/IndexedLogs");
+    return {
+        logLogs: new IndexedLogs<LogDatum>({ name: "logs/log", getTime: x => x.time }),
+        warnLogs: new IndexedLogs<LogDatum>({ name: "logs/warn", getTime: x => x.time }),
+        infoLogs: new IndexedLogs<LogDatum>({ name: "logs/info", getTime: x => x.time }),
+        errorLogs: new IndexedLogs<LogDatum>({ name: "logs/error", getTime: x => x.time }),
+    };
+});
+
 // NOTE: If any message (first param) starts with this, we don't log it to the disk. VERY useful for multi-line logging where it wouldn't make sense in the logs
 // NOTE: This is visible, otherwise it's easy to accidentally copy it, and not know why the text is behaving strangely (not === other seemingly equal text, etc).
 // NOTE: Also hardcoded in measure.ts (in socket-function)
@@ -116,15 +143,7 @@ void Promise.resolve().then(() => {
     startupDone = true;
 });
 
-let logLimitLookup: {
-    resetTime: number;
-    counts: Map<string, number>;
-} | undefined;
 
-const LIMIT_PERIOD = timeInMinute * 15;
-const LIMIT_THRESHOLD = 1000;
-const WARN_LIMIT = 100;
-const ERROR_LIMIT = 100;
 
 const logDiskDontShim = logDisk;
 /** NOTE: Calling this directly means we lose __FILE__ tracking. But... that's probably fine... */
@@ -147,47 +166,12 @@ export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknow
             hasLineLimit = true;
         }
 
-        if (logLimitLookup) {
-            if (logObj.time > logLimitLookup.resetTime) {
-                logLimitLookup = undefined;
-            }
-        }
-        if (!logLimitLookup) {
-            logLimitLookup = {
-                resetTime: logObj.time + LIMIT_PERIOD,
-                counts: new Map(),
-            };
-        }
-
-        let count = logLimitLookup.counts.get(logType) || 0;
-        count++;
-        logLimitLookup.counts.set(logType, count);
-        let limit = LIMIT_THRESHOLD;
-        if (type === "warn") {
-            limit = WARN_LIMIT;
-        } else if (type === "error") {
-            limit = ERROR_LIMIT;
-        }
-        if (count > limit) {
-            let timeUntilReset = logLimitLookup.resetTime - logObj.time;
-            if (hasLineLimit) {
-                process.stdout.write(`Log type hit limit, not writing log type to disk for ~${formatTime(timeUntilReset)}: ${logType}\n`);
-            }
-            return;
-        }
-        if (count >= limit) {
-            if (hasLineLimit) {
-                logObj[LOG_LINE_LIMIT_FLAG] = true;
-            } else {
-                logObj[LOG_LIMIT_FLAG] = true;
-            }
-        }
-
-        // We don't want developer errors clogging up the error logs. However, they can still notify errors, Because this will only notify nodes that are able to access us (It uses a reverse connection scheme, so instead of talking to nodes that we can access, we only talk to nodes that can access us), Which will mean it will only notify for local services, so the developer still gets error notifications, But our errors won't be spread to all developers. BUT, we will still watch global errors, because we can contact the global server, so developers will still get errors about production issues, even while developing!  
-        if (isPublic()) {
-            let loggers = startupDone ? getLoggers() : undefined;
+        // NOTE: Local logs now log to their local disk instead of backblaze, so we can log even if in development (they just won't show up on the remote server).
+        //if (isPublic())
+        {
+            let loggers = startupDone ? getLoggers2() : undefined;
             if (!loggers) {
-                getLoggers.reset();
+                getLoggers2.reset();
                 setImmediate(() => {
                     logDiskDontShim(type, ...args);
                 });

package/src/diagnostics/logs/errorNotifications/ErrorNotificationController.ts

@@ -505,7 +505,7 @@ class URLCache {
         }
     });
 }
-const urlCache = new URLCache();
+export const urlCache = new URLCache();
 
 const limitRecentErrors = measureWrap(function limitRecentErrors(objs: LogDatum[]) {
     sort(objs, x => x.time);
@@ -575,7 +575,7 @@ export class RecentErrors {
             void this.broadcastUpdate(undefined);
         }
     });
-    private broadcastUpdate = batchFunction({ delay: NOTIFICATION_BROADCAST_BATCH }, () => {
+    private broadcastUpdate = batchFunction({ delay: NOTIFICATION_BROADCAST_BATCH, noMeasure: true }, () => {
         recentErrorsChannel.broadcast(true);
     });
 
@@ -637,6 +637,7 @@ export class RecentErrors {
                 },
                 rootPath: appendable.rootPath,
                 noLocalFiles: config.noLocalFiles,
+                forceGetPublic: true,
             });
             // Filter again, as new suppressions change the errors
             await this.updateRecentErrors();

package/src/diagnostics/logs/errorNotifications/errorDigests.tsx

@@ -195,6 +195,7 @@ export async function runDigest() {
                 endTime,
             },
             rootPath: appendable.rootPath,
+            forceGetPublic: true,
         });
         let filesLeft = result.files.slice();
         await Promise.all(list(32).map(() => runThread()));

package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePages.tsx

@@ -0,0 +1,150 @@
+module.hotreload = true;
+import { qreact } from "../../../4-dom/qreact";
+import { css } from "../../../4-dom/css";
+import { t } from "../../../2-proxy/schema2";
+import { Querysub } from "../../../4-querysub/QuerysubController";
+import { sort } from "socket-function/src/misc";
+import { FastArchiveViewer } from "../FastArchiveViewer";
+import { LogDatum, getLoggers } from "../diskLogger";
+import { Table } from "../../../5-diagnostics/Table";
+import { formatDateTime } from "socket-function/src/formatting/format";
+import { FastArchiveAppendable } from "../FastArchiveAppendable";
+import { getTimeRange } from "../TimeRangeSelector";
+import { URLParam } from "../../../library-components/URLParam";
+import { InputLabelURL } from "../../../library-components/InputLabel";
+import { isPublic } from "../../../config";
+
+const forceGetPublicURL = new URLParam("forceGetPublic", false);
+
+export class LifeCyclePages extends qreact.Component {
+    state = t.state({
+        dataSeqNum: t.atomic<number>(0),
+    });
+
+    private datums: LogDatum[] = [];
+    private latestByName = new Map<string, LogDatum>();
+    private fastArchiveViewer: FastArchiveViewer<LogDatum> | undefined = undefined;
+
+    rerun() {
+        Querysub.onCommitFinished(async () => {
+            void this.fastArchiveViewer?.synchronizeDataThrottled();
+        });
+    }
+
+    getLoggers = () => {
+        let logs: FastArchiveAppendable<LogDatum>[] = [];
+        let loggers = getLoggers();
+        if (!loggers) {
+            throw new Error("Loggers not available?");
+        }
+        let { logLogs, warnLogs, infoLogs, errorLogs } = loggers;
+        logs.push(logLogs);
+        logs.push(infoLogs);
+        logs.push(warnLogs);
+        logs.push(errorLogs);
+        return logs;
+    };
+
+    renderContent() {
+        let latestEntries = Array.from(this.latestByName.entries());
+        sort(latestEntries, x => -(x[1].time || 0));
+
+        let rows = latestEntries.map(([name, datum]) => ({
+            name,
+            time: datum.time,
+            datum,
+        }));
+
+        return <>
+            <div className={css.hbox(10)}>
+                <div>
+                    Total unique __NAME__ values: {this.latestByName.size}
+                </div>
+                <div>
+                    Total logs processed: {this.datums.length}
+                </div>
+            </div>
+
+            {rows.length === 0 && <div className={css.hsl(40, 50, 50).colorhsl(60, 50, 100).boldStyle.pad2(10).ellipsis}>
+                No logs matched, either increase the time range or run the search.
+            </div>}
+
+            <Table
+                rows={rows}
+                columns={{
+                    name: {
+                        title: "Name",
+                        formatter: x => <div>{String(x)}</div>
+                    },
+                    time: {
+                        title: "Last Time",
+                        formatter: x => <div>{formatDateTime(Number(x))}</div>
+                    },
+                }}
+                lineLimit={4}
+                initialLimit={20}
+                characterLimit={400}
+            />
+        </>;
+    }
+
+    render() {
+        this.state.dataSeqNum;
+
+        const timeRange = getTimeRange();
+        return (
+            <div className={css.vbox(20).pad2(20).fillBoth}>
+                <div className={css.hbox(10)}>
+                    <div className={css.fontSize(14)}>
+                        Life Cycle Analysis
+                    </div>
+                    {!isPublic() && <InputLabelURL
+                        label="Force Public Logs" checkbox url={forceGetPublicURL}
+                        onChangeValue={(newValue) => {
+                            this.rerun();
+                        }}
+                    />}
+                </div>
+
+                <FastArchiveViewer
+                    ref2={x => this.fastArchiveViewer = x}
+                    fastArchives={this.getLoggers}
+                    runOnLoad={false}
+                    forceGetPublic={forceGetPublicURL.value}
+                    onStart={async () => {
+                        this.datums = [];
+                        this.latestByName = new Map();
+                    }}
+                    onDatums={(source, datums, file) => {
+                        for (let datum of datums) {
+                            let time = datum.time;
+                            if (time && (time < timeRange.startTime || time > timeRange.endTime)) {
+                                continue;
+                            }
+                            this.datums.push(datum);
+                        }
+                    }}
+                    onFinish={() => {
+                        // Sort all datums by time
+                        sort(this.datums, x => x.time || 0);
+
+                        // Get the latest value (last time) for every __NAME__
+                        this.latestByName.clear();
+                        for (let datum of this.datums) {
+                            const name = datum.__NAME__;
+                            if (name) {
+                                this.latestByName.set(name, datum);
+                            }
+                        }
+
+                        Querysub.commit(() => {
+                            this.state.dataSeqNum++;
+                        });
+                    }}
+                >
+                    {this.renderContent()}
+                </FastArchiveViewer>
+            </div>
+        );
+    }
+}