querysub 0.2.0

package/src/diagnostics/benchmark.ts

@@ -0,0 +1,139 @@
+import { addToStats, addToStatsValue, createStatsValue, getStatsTop } from "socket-function/src/profiling/stats";
+import type { PathValue } from "../0-path-value-core/pathValueCore";
+import { registerPeriodic } from "./periodic";
+import { formatTime } from "socket-function/src/formatting/format";
+import { formatStats } from "socket-function/src/profiling/statsFormat";
+import { getPathStr1 } from "../path";
+import { registerResource } from "./trackResources";
+import type { CallSpec } from "../3-path-functions/PathFunctionRunner";
+import { PromiseObj } from "../promise";
+import { lazy } from "socket-function/src/caching";
+import { isNode } from "socket-function/src/misc";
+
+const getCallResultPathFnc = lazy(() => import("../4-querysub/querysubPrediction").then((m) => m.getCallResultPath));
+
+// NOTE: There's no point to track non-predicted calls, because if we don't predict them we won't be watching
+//  them, so we won't know when they finish!
+
+class OverlapRateTracker {
+    public openCount = 0;
+    private openTime = 0;
+    public originalOpenTime = Date.now();
+    private openCloseCount = 0;
+
+    public finishedTime = 0;
+    private finishedCount = 0;
+
+    public openValue() {
+        if (this.openCount === 0) {
+            this.openTime = Date.now();
+        }
+        this.openCount++;
+        this.openCloseCount++;
+    }
+    public closeValue() {
+        this.openCount--;
+        let now = Date.now();
+        if (this.openCount === 0 || now - this.openTime > 5000) {
+            this.finishedTime += now - this.openTime;
+            this.openTime = now;
+            this.finishedCount += this.openCloseCount;
+            this.openCloseCount = 0;
+        }
+    }
+    public viewRate() {
+        if (this.finishedTime === 0) return 0;
+        return this.finishedCount / this.finishedTime * 1000;
+    }
+    public saturatedFraction() {
+        return this.finishedTime / (Date.now() - this.originalOpenTime);
+    }
+    public harvestRate() {
+        let rate = this.viewRate();
+        this.finishedTime = 0;
+        this.finishedCount = 0;
+        this.originalOpenTime = Date.now();
+        return rate;
+    }
+}
+
+export class Benchmark {
+
+    public static predictCount = 0;
+    public static receivedCount = 0;
+
+    private static finishedCalls = createStatsValue();
+    public static overlapRate = new OverlapRateTracker();
+
+    private static predictedCalls = registerResource("Benchmark|predictedCalls", new Map<string, number>());
+
+    private static waitingForCalls = registerResource("Benchmark|waitingForCalls", new Map<string, (() => void)[]>());
+
+    public static onStartPredictCall(path: string) {
+        this.predictedCalls.set(path, Date.now());
+        Benchmark.predictCount++;
+        this.overlapRate.openValue();
+    }
+    public static onReceivedValues(values: PathValue[]) {
+        for (let value of values) {
+            if (value.canGCValue) continue;
+            if (!value.valid) continue;
+            let list = this.waitingForCalls.get(value.path);
+            if (!list) continue;
+            this.waitingForCalls.delete(value.path);
+            for (let resolve of list) {
+                resolve();
+            }
+        }
+
+        let now = Date.now();
+        for (let value of values) {
+            if (!value.path.includes(getPathStr1("Results"))) {
+                continue;
+            }
+            if (value.canGCValue) continue;
+            if (!value.valid) continue;
+            let predictedTime = this.predictedCalls.get(value.path);
+            if (!predictedTime) {
+                continue;
+            }
+            this.overlapRate.closeValue();
+            this.predictedCalls.delete(value.path);
+            this.receivedCount++;
+            addToStatsValue(this.finishedCalls, now - predictedTime);;
+        }
+    }
+
+    public static async waitForCallToFinish(call: CallSpec) {
+        let getCallResultPath = await getCallResultPathFnc();
+        let path = getCallResultPath(call);
+        let list = this.waitingForCalls.get(path);
+        if (!list) {
+            list = [];
+            this.waitingForCalls.set(path, list);
+        }
+        let promiseObj = new PromiseObj();
+        list.push(promiseObj.resolve);
+        await promiseObj.promise;
+    }
+
+    public static logCallsNow() {
+        let stats = this.finishedCalls;
+        this.finishedCalls = createStatsValue();
+        console.log(`Calls ${formatStats(stats)}, rate ${this.overlapRate.harvestRate()}`);
+    }
+
+    public static getLiveCallStats() {
+        return this.finishedCalls;
+    }
+    public static getLiveRate() {
+        return this.overlapRate.viewRate();
+    }
+    public static saturatedFraction() {
+        return this.overlapRate.saturatedFraction();
+    }
+}
+
+if (!isNode()) {
+    registerPeriodic(() => Benchmark.logCallsNow());
+}

package/src/diagnostics/errorLogs/ErrorLogController.ts

@@ -0,0 +1,515 @@
+import { lazy } from "socket-function/src/caching";
+import { LogClass, LOG_FLUSH_INTERVAL, LogBlock, LogBlockInfo, LogRaw, LOG_ARCHIVES, LOG_CLASSES_PATH, logClassesFromString, getLogClassCategorizer, logBlockFromBuffer, logBlockToBuffer, logClassesToString, LOG_BLOCK_EXTENSION, getLogBlockInfo, getLogBlockFileName, addToLogBlock, LogType, isUngroupedClass } from "./ErrorLogCore";
+import { list, throttleFunction, timeInDay, timeInMinute } from "socket-function/src/misc";
+import { getMachineId, getOwnMachineId, getThreadKeyCert } from "../../-a-auth/certs";
+import { getAllNodeIds, getBrowserUrlNode, getOwnNodeId, getOwnThreadId, watchDeltaNodeIds, watchNodeIds } from "../../-f-node-discovery/NodeDiscovery";
+import { getArchives } from "../../-a-archives/archives";
+import { isNode } from "typesafecss";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { requiresNetworkTrustHook } from "../../-d-trust/NetworkTrust2";
+import { ignoreErrors, logErrors, timeoutToError, timeoutToUndefinedSilent } from "../../errors";
+import debugbreak from "debugbreak";
+import { magenta } from "socket-function/src/formatting/logColors";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { registerShutdownHandler } from "../periodic";
+import { isNoNetwork } from "../../config";
+import { ExternalRenderClass, qreact } from "../../4-dom/qreact";
+import { Querysub } from "../../4-querysub/QuerysubController";
+import { atomic, doAtomicWrites } from "../../2-proxy/PathValueProxyWatcher";
+import { isClient } from "../../config2";
+import { ControllerPick, SocketRegistered } from "socket-function/SocketFunctionTypes";
+
+
+const NOTIFY_HISTORY = timeInDay * 7;
+// Only error / fatal are tracked, for now... we might also add warns?
+const logIssueNotifyTypes: LogType[] = ["error", "fatal"];
+
+// We don't log on internal errors, otherwise we might infinitely loop
+function internalError(err: unknown | Error | Promise<unknown>) {
+    if (err && err instanceof Error) {
+        if (!isNode()) {
+            debugger;
+            return;
+        }
+        process.stderr.write((err.stack || err.message) + "\n");
+    } else if (err && err instanceof Promise) {
+        err.catch(internalError);
+    } else {
+        if (!isNode()) {
+            debugger;
+            return;
+        }
+        process.stderr.write("Unknown error: " + String(err) + "\n");
+    }
+}
+
+export function logSilent(message: string, type: LogType) {
+    addLocalLog({ message, time: Date.now() }, type);
+}
+export function logSilentError(message: string) {
+    logSilent(message, "error");
+}
+export function logSilentWarning(message: string) {
+    logSilent(message, "warn");
+}
+
+let nextBlockSeqNum = 1;
+let pendingBlock: LogBlock | undefined;
+export function addLocalLog(log: LogRaw, typeHint: LogType) {
+    // NOTE: Eventually... we should probably ignore logs for non-public machines?
+    //if (isNoNetwork()) return;
+    try {
+        if (!pendingBlock) {
+            ensureBlocksFlushing();
+            let now = Date.now();
+            pendingBlock = {
+                classes: {},
+                startTime: now,
+                endTime: now,
+
+                threadId: "",
+                machineId: "",
+
+                seqNum: nextBlockSeqNum++,
+                count: 0,
+                fatalCount: 0,
+                errorCount: 0,
+                warnCount: 0,
+                infoCount: 0,
+            };
+        }
+
+        if (!logClassifier) {
+            ignoreErrors(initLogClassifier());
+            pendingLogs.push({ log, type: typeHint });
+            return;
+        }
+        let classifier = logClassifier(log, typeHint);
+        pendingBlock.threadId = pendingBlock.threadId || getOwnThreadId();
+        pendingBlock.machineId = pendingBlock.machineId || getOwnMachineId();
+        addToLogBlock(pendingBlock, classifier, log);
+        ErrorLogControllerBase.onLocalLog(log, classifier);
+    } catch (e) {
+        internalError(e);
+    }
+}
+registerShutdownHandler(flushNow);
+async function flushNow() {
+    let newBlock = pendingBlock;
+    pendingBlock = undefined;
+    if (!newBlock) return;
+    newBlock.endTime = Date.now();
+    //console.log(magenta(`Flushing log block ${formatNumber(newBlock.count)} count, seq num ${newBlock.seqNum}, at ${Date.now()}`));
+    if (newBlock.count > 0) {
+        let fileInfo = await logBlockToBuffer(newBlock);
+        await LOG_ARCHIVES().set(fileInfo.fileName, fileInfo.buffer);
+    }
+}
+
+let ensureBlocksFlushing = lazy(() => {
+    setInterval(async () => {
+        await internalError(flushNow());
+    }, LOG_FLUSH_INTERVAL);
+
+    // Read log classifiers every hour, in case we miss an update
+    setInterval(() => {
+        initLogClassifier.reset();
+        logErrors(initLogClassifier());
+    }, timeInMinute * 60);
+});
+
+let pendingLogs: { log: LogRaw; type: LogType }[] = [];
+let logClassifier: ((log: LogRaw, typeHint: LogType) => LogClass) | undefined;
+let initLogClassifier = lazy(async () => {
+    await timeoutToError(60 * 1000, SocketFunction.mountPromise, () => new Error("Logs lost because service never mounted"));
+    let logClasses = await new ErrorLogControllerBase().getClasses();
+    logClassifier = getLogClassCategorizer(logClasses);
+    if (pendingLogs.length > 0) {
+        let logs = pendingLogs;
+        pendingLogs = [];
+        for (let log of logs) {
+            addLocalLog(log.log, log.type);
+        }
+    }
+    return {
+        classify: logClassifier,
+        logClasses: new Map(logClasses.map(a => [a.id, a])),
+    };
+});
+async function onLogClassesUpdated() {
+    initLogClassifier.reset();
+    await initLogClassifier();
+    await ErrorLogControllerBase.recalculateNotifyState();
+}
+
+export class ErrorLogControllerBase {
+    public async scanBlocks(): Promise<LogBlockInfo[]> {
+        let files = await LOG_ARCHIVES().find("", { shallow: true, type: "files" });
+        files = files.filter(a => a.endsWith(LOG_BLOCK_EXTENSION));
+        let infos = files.map(getLogBlockInfo);
+        if (pendingBlock) {
+            infos.push({
+                startTime: pendingBlock.startTime,
+                endTime: pendingBlock.endTime,
+                seqNum: pendingBlock.seqNum,
+                count: pendingBlock.count,
+                fatalCount: pendingBlock.fatalCount,
+                errorCount: pendingBlock.errorCount,
+                warnCount: pendingBlock.warnCount,
+                infoCount: pendingBlock.infoCount,
+                threadId: pendingBlock.threadId,
+                machineId: pendingBlock.machineId,
+                fileName: "",
+            });
+        }
+        return infos;
+    }
+
+    public async readBlocks(config: {
+        files: string[];
+    }): Promise<Buffer[]> {
+        let fileNames = config.files.filter(a =>
+            a.endsWith(LOG_BLOCK_EXTENSION) && !(a.includes("/") || a.includes("\\") || a.includes(".."))
+            || !a
+        );
+        let buffers: Buffer[] = [];
+        async function runThread() {
+            while (fileNames.length > 0) {
+                let fileName = fileNames.pop()!;
+                if (!fileName) {
+                    if (pendingBlock) {
+                        buffers.push((await logBlockToBuffer(pendingBlock)).buffer);
+                    }
+                    continue;
+                }
+                let buffer = await LOG_ARCHIVES().get(fileName);
+                if (!buffer) continue;
+                buffers.push(buffer);
+            }
+        }
+        await Promise.all(list(16).map(runThread));
+        return buffers;
+    }
+    public async readAllLiveBlocks(): Promise<Buffer[]> {
+        let nodeIds = await getAllNodeIds();
+        let blockBuffers: Buffer[] = [];
+        await Promise.allSettled(nodeIds.map(async nodeId => {
+            let controller = ErrorLogController.nodes[nodeId];
+            let buffers = await timeoutToUndefinedSilent(5000, controller.readBlocks({ files: [""] }));
+            if (!buffers) return;
+            console.log(magenta(`Read live data from ${nodeId}`));
+            blockBuffers.push(...buffers);
+        }));
+        return blockBuffers;
+    }
+
+    public async getClasses(): Promise<LogClass[]> {
+        let logClassesBuffer = await LOG_ARCHIVES().get(LOG_CLASSES_PATH);
+        let logClasses = logClassesFromString(logClassesBuffer?.toString() || "");
+        return logClasses;
+    }
+
+    public async updateClass(logClass: LogClass | { id: string; action: "delete" }) {
+        let logClasses = await this.getClasses();
+        let existingIndex = logClasses.findIndex(a => a.id === logClass.id);
+        if ("action" in logClass) {
+            if (logClass.action === "delete") {
+                if (existingIndex !== -1) {
+                    logClasses.splice(existingIndex, 1);
+                }
+            }
+        } else {
+            if (existingIndex !== -1) {
+                logClasses[existingIndex] = logClass;
+            } else {
+                logClasses.push(logClass);
+            }
+        }
+        let buffer = Buffer.from(logClassesToString(logClasses));
+        await LOG_ARCHIVES().set(LOG_CLASSES_PATH, buffer);
+        logErrors((async () => {
+            await onLogClassesUpdated();
+            // Notify ALL other nodes
+            let nodeIdsToTest = await getAllNodeIds();
+            for (let nodeId of nodeIdsToTest) {
+                ignoreErrors(ErrorLogController.nodes[nodeId].onClassesUpdated());
+            }
+        })());
+    }
+
+    public async onClassesUpdated() {
+        ignoreErrors(onLogClassesUpdated());
+    }
+
+    private static joinNotifies(notify: (LogIssueNotify | undefined)[]): LogIssueNotify {
+        // If we don't need to merge, don't so callers can check for ===
+        let finalNotify: LogIssueNotify | undefined = undefined;
+        for (let next of notify) {
+            if (!next) continue;
+            if (!finalNotify) {
+                finalNotify = next;
+                continue;
+            }
+            if (next.oldestUnsuppressed && (!finalNotify.oldestUnsuppressed || next.oldestUnsuppressed.time < finalNotify.oldestUnsuppressed.time)) {
+                if (!finalNotify.oldestUngrouped) {
+                    finalNotify = next;
+                } else {
+                    finalNotify = { ...finalNotify, oldestUnsuppressed: next.oldestUnsuppressed };
+                }
+            }
+            if (next.oldestUngrouped && (!finalNotify.oldestUngrouped || next.oldestUngrouped.time < finalNotify.oldestUngrouped.time)) {
+                if (!finalNotify.oldestUnsuppressed) {
+                    finalNotify = next;
+                } else {
+                    finalNotify = { ...finalNotify, oldestUngrouped: next.oldestUngrouped };
+                }
+            }
+        }
+        return finalNotify || undefined;
+    }
+
+    private static logIssueNotifyEquals(lhs: LogIssueNotify | undefined, rhs: LogIssueNotify | undefined) {
+        return (
+            lhs?.oldestUngrouped?.message === rhs?.oldestUngrouped?.message
+            && lhs?.oldestUnsuppressed?.message === rhs?.oldestUnsuppressed?.message
+            && lhs?.oldestUngrouped?.time === rhs?.oldestUngrouped?.time
+            && lhs?.oldestUnsuppressed?.time === rhs?.oldestUnsuppressed?.time
+        );
+    }
+
+    private static inMemoryNotify: LogIssueNotify | undefined;
+    private static updateInMemoryNotify(notify: LogIssueNotify | undefined) {
+        if (ErrorLogControllerBase.logIssueNotifyEquals(this.inMemoryNotify, notify)) {
+            return;
+        }
+        this.inMemoryNotify = notify;
+        for (let watcher of this.inMemoryNotifyWatchers) {
+            ErrorLogController.nodes[watcher]
+                .liveLogsIssueNotifyCallback(notify)
+                .catch(() => {
+                    this.inMemoryNotifyWatchers.delete(watcher);
+                });
+        }
+        logErrors(ErrorLogControllerBase.recalcAllNotify());
+    }
+    private static inMemoryNotifyWatchers = new Set<string>();
+    public static onLocalLog(log: LogRaw, classObj: LogClass) {
+        if (!logIssueNotifyTypes.includes(classObj.type)) return;
+        let notify: LogIssueNotify | undefined;
+        if (isUngroupedClass(classObj)) {
+            notify = {
+                oldestUnsuppressed: undefined,
+                oldestUngrouped: log,
+            };
+        } else if (log.time > classObj.suppressUntil) {
+            notify = {
+                oldestUnsuppressed: log,
+                oldestUngrouped: undefined,
+            };
+        }
+        if (notify) {
+            this.updateInMemoryNotify(this.joinNotifies([this.inMemoryNotify, notify]));
+        }
+    }
+    public async internalWatchLiveLogs() {
+        let watcher = SocketFunction.getCaller().nodeId;
+        ErrorLogControllerBase.inMemoryNotifyWatchers.add(watcher);
+        // Await, so the client gets it's own errors
+        await ErrorLogController.nodes[watcher].liveLogsIssueNotifyCallback(ErrorLogControllerBase.inMemoryNotify);
+    }
+
+    private static liveLogsIssueNotify = new Map<string, LogIssueNotify | undefined>();
+    public async liveLogsIssueNotifyCallback(notify: LogIssueNotify | undefined) {
+        let caller = SocketFunction.getCaller();
+        let prev = ErrorLogControllerBase.liveLogsIssueNotify.get(caller.nodeId);
+        if (ErrorLogControllerBase.logIssueNotifyEquals(prev, notify)) {
+            return;
+        }
+        console.log(magenta(`Received log notify from ${caller.nodeId} for ${notify?.oldestUngrouped?.message}`));
+        ErrorLogControllerBase.liveLogsIssueNotify.set(caller.nodeId, notify);
+        logErrors(ErrorLogControllerBase.recalcAllNotify());
+    }
+
+
+    private static allNotify: LogIssueNotify | undefined;
+    private static allWatchers = new Set<string>();
+    private static diskNotify: LogIssueNotify | undefined;
+    private static async recalcAllNotify() {
+        if (!ErrorLogControllerBase.watchingLogs) return;
+        let notifies = [
+            ...Array.from(this.liveLogsIssueNotify.values()),
+            this.inMemoryNotify,
+            this.diskNotify
+        ];
+        let { classify } = await initLogClassifier();
+        // Reclassify based on the most recent classifier
+        //  - This only removes classifications, it doesn't add them back. Each node should be watching
+        //      classifiers and rerunning on their data to add classifications back.
+        notifies = notifies.map(x => {
+            if (x?.oldestUngrouped && !isUngroupedClass(classify(x.oldestUngrouped, "error"))) {
+                x = { ...x, oldestUngrouped: undefined };
+            }
+            if (x?.oldestUnsuppressed && x.oldestUnsuppressed.time < classify(x.oldestUnsuppressed, "error").suppressUntil) {
+                x = { ...x, oldestUnsuppressed: undefined };
+            }
+            return x;
+        });
+        let newNotify = this.joinNotifies(notifies);
+        if (this.logIssueNotifyEquals(newNotify, this.allNotify)) {
+            return;
+        }
+        this.allNotify = newNotify;
+        for (let watcher of this.allWatchers) {
+            ErrorLogController.nodes[watcher].logsIssueNotifyCallback(newNotify)
+                .catch(() => {
+                    this.allWatchers.delete(watcher);
+                });
+        }
+    }
+
+    private static watchForNewNodes = lazy(() => {
+        watchDeltaNodeIds(async ({ newNodeIds }) => {
+            for (let nodeId of newNodeIds) {
+                if (nodeId === getOwnNodeId()) continue;
+                if (!ErrorLogControllerBase.liveLogsIssueNotify.has(nodeId)) {
+                    ErrorLogControllerBase.liveLogsIssueNotify.set(nodeId, undefined);
+                }
+                ignoreErrors(ErrorLogController.nodes[nodeId].internalWatchLiveLogs());
+                SocketFunction.onNextDisconnect(nodeId, () => {
+                    // HACK: Remove it, to prevent notifications from errors which we never written to disk.
+                    //  - The only other alternative is to get this error in the UI... somehow, and onto disk,
+                    //      but not if the other node DOES write it to disk, which... gets complicated.
+                    ErrorLogControllerBase.liveLogsIssueNotify.delete(nodeId);
+                });
+            }
+        });
+    });
+
+    private static watchingLogs = false;
+    public static async recalculateNotifyState() {
+        ErrorLogControllerBase.watchForNewNodes();
+
+        // Watch all nodes again, in case they disappear
+        for (let node of await getAllNodeIds()) {
+            if (node === getOwnNodeId()) continue;
+            ignoreErrors(ErrorLogController.nodes[node].internalWatchLiveLogs());
+        }
+
+        let threshold = Date.now() - NOTIFY_HISTORY;
+
+        let { classify, logClasses } = await initLogClassifier();
+        function summarizeBlock(block: LogBlock | undefined) {
+            if (!block) return undefined;
+            let notify: LogIssueNotify | undefined = undefined;
+            for (let classSummary of Object.values(block.classes)) {
+                for (let logRaw of classSummary.examples) {
+                    if (logRaw.time < threshold) continue;
+                    let classObj = classify(logRaw, classSummary.logClassType || logClasses.get(classSummary.logClassId)?.type || "error");
+                    if (!logIssueNotifyTypes.includes(classObj.type)) continue;
+                    if (isUngroupedClass(classObj)) {
+                        if (!notify || !notify.oldestUngrouped || classSummary.firstTime < notify.oldestUngrouped.time) {
+                            if (!notify) notify = { oldestUnsuppressed: undefined, oldestUngrouped: undefined };
+                            if (!notify.oldestUngrouped || notify.oldestUngrouped.time > classSummary.firstTime) {
+                                notify.oldestUngrouped = classSummary.examples[0];
+                            }
+                        }
+                    } else if (logRaw.time > classObj.suppressUntil) {
+                        if (!notify || !notify.oldestUnsuppressed || classSummary.firstTime < notify.oldestUnsuppressed.time) {
+                            if (!notify) notify = { oldestUnsuppressed: undefined, oldestUngrouped: undefined };
+                            if (!notify.oldestUnsuppressed || notify.oldestUnsuppressed.time > classSummary.firstTime) {
+                                notify.oldestUnsuppressed = classSummary.examples[0];
+                            }
+                        }
+                    }
+                }
+            }
+            return notify;
+        }
+
+        // Recalculate in memory state
+        this.updateInMemoryNotify(summarizeBlock(pendingBlock));
+
+        if (ErrorLogControllerBase.watchingLogs) {
+            let blockInfos = await new ErrorLogControllerBase().scanBlocks();
+            blockInfos = blockInfos.filter(a => a.startTime > threshold);
+            let blocks = await new ErrorLogControllerBase().readBlocks({ files: blockInfos.map(a => a.fileName) });
+            let blockObjs = await Promise.all(blocks.map(x => logBlockFromBuffer(x)));
+            let notify = this.joinNotifies(blockObjs.map(summarizeBlock));
+            this.diskNotify = notify;
+        }
+
+        await this.recalcAllNotify();
+    }
+
+    // NOTE: We never unwatch. Watching for LogIssueNotify should be fairly cheap, so this is fine
+    public static startLogWatch = lazy(async () => {
+        ErrorLogControllerBase.watchingLogs = true;
+        await this.recalculateNotifyState();
+    });
+
+    public async watchLogs(callbackClass?: ControllerPick<typeof ErrorLogController, "logsIssueNotifyCallback">) {
+        let self = ErrorLogControllerBase;
+        self.watchingLogs = true;
+
+        let caller = SocketFunction.getCaller();
+        self.allWatchers.add(caller.nodeId);
+
+        await self.startLogWatch();
+
+        let client = callbackClass ? SocketFunction.rehydrateSocketCaller(callbackClass) : ErrorLogController;
+        await client.nodes[caller.nodeId].logsIssueNotifyCallback(self.allNotify);
+    }
+
+    public static logIssueNotifyWatchers = new Set<(notify: LogIssueNotify | undefined) => void>();
+    public async logsIssueNotifyCallback(notify: LogIssueNotify | undefined) {
+        onLogIssueNotify(notify);
+    }
+    public async unwatchLogs() {
+        let self = ErrorLogControllerBase;
+        let caller = SocketFunction.getCaller();
+        self.allWatchers.delete(caller.nodeId);
+    }
+}
+
+function onLogIssueNotify(notify: LogIssueNotify | undefined) {
+    for (let watcher of ErrorLogControllerBase.logIssueNotifyWatchers) {
+        try {
+            watcher(notify);
+        } catch (e) {
+            logErrors(e);
+        }
+    }
+}
+export function watchLogIssueNotify(callback: (notify: LogIssueNotify | undefined) => void) {
+    ErrorLogControllerBase.logIssueNotifyWatchers.add(callback);
+}
+
+export type LogIssueNotify = undefined | {
+    // NOTE: Storing the oldest requires a lot fewer notifications, and is probably what we want
+    //  anyways (it tells us when the problem started).
+    oldestUnsuppressed: LogRaw | undefined;
+    oldestUngrouped: LogRaw | undefined;
+    // NOTE: Count is too hard to count accurately, due to live values being written to disk, etc.
+}
+
+const ErrorLogController = SocketFunction.register(
+    "ErrorLogController-219a7c2e-2da6-4753-9e0e-1f42dc5f6970",
+    new ErrorLogControllerBase(),
+    () => ({
+        scanBlocks: {},
+        readBlocks: {},
+        readAllLiveBlocks: {},
+        getClasses: {},
+        updateClass: {},
+        onClassesUpdated: {},
+
+        watchLogs: {},
+        unwatchLogs: {},
+        liveLogsIssueNotifyCallback: {},
+        logsIssueNotifyCallback: {},
+        internalWatchLiveLogs: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    }),
+);