querysub 0.312.0 → 0.314.0

package/src/diagnostics/errorLogs/ErrorLogController.ts

@@ -1,535 +0,0 @@
-import { lazy } from "socket-function/src/caching";
-import { LogClass, LOG_FLUSH_INTERVAL, LogBlock, LogBlockInfo, LogRaw, LOG_ARCHIVES, LOG_CLASSES_PATH, logClassesFromString, getLogClassCategorizer, logBlockFromBuffer, logBlockToBuffer, logClassesToString, LOG_BLOCK_EXTENSION, getLogBlockInfo, getLogBlockFileName, addToLogBlock, LogType, isUngroupedClass } from "./ErrorLogCore";
-import { list, throttleFunction, timeInDay, timeInMinute } from "socket-function/src/misc";
-import { getMachineId, getOwnMachineId, getThreadKeyCert } from "../../-a-auth/certs";
-import { getAllNodeIds, getBrowserUrlNode, getOwnNodeId, getOwnThreadId, watchDeltaNodeIds, watchNodeIds } from "../../-f-node-discovery/NodeDiscovery";
-import { getArchives } from "../../-a-archives/archives";
-import { isNode } from "typesafecss";
-import { SocketFunction } from "socket-function/SocketFunction";
-import { requiresNetworkTrustHook } from "../../-d-trust/NetworkTrust2";
-import { ignoreErrors, logErrors, timeoutToError, timeoutToUndefinedSilent } from "../../errors";
-import debugbreak from "debugbreak";
-import { blue, magenta } from "socket-function/src/formatting/logColors";
-import { formatNumber } from "socket-function/src/formatting/format";
-import { registerShutdownHandler } from "../periodic";
-import { isNoNetwork } from "../../config";
-import { ExternalRenderClass, qreact } from "../../4-dom/qreact";
-import { Querysub } from "../../4-querysub/QuerysubController";
-import { atomic, doAtomicWrites } from "../../2-proxy/PathValueProxyWatcher";
-import { isClient } from "../../config2";
-import { ControllerPick, SocketRegistered } from "socket-function/SocketFunctionTypes";
-
-const NOTIFY_HISTORY = timeInDay * 7;
-const LOG_STORE_HISTORY = timeInDay * 14;
-// Only error / fatal are tracked, for now... we might also add warns?
-const logIssueNotifyTypes: LogType[] = ["error", "fatal"];
-
-// We don't log on internal errors, otherwise we might infinitely loop
-function internalError(err: unknown | Error | Promise<unknown>) {
-    if (err && err instanceof Error) {
-        if (!isNode()) {
-            debugger;
-            return;
-        }
-        process.stderr.write((err.stack || err.message) + "\n");
-    } else if (err && err instanceof Promise) {
-        err.catch(internalError);
-    } else {
-        if (!isNode()) {
-            debugger;
-            return;
-        }
-        process.stderr.write("Unknown error: " + String(err) + "\n");
-    }
-}
-
-export function logSilent(message: string, type: LogType) {
-    addLocalLog({ message, time: Date.now() }, type);
-}
-export function logSilentError(message: string) {
-    logSilent(message, "error");
-}
-export function logSilentWarning(message: string) {
-    logSilent(message, "warn");
-}
-
-let nextBlockSeqNum = 1;
-let pendingBlock: LogBlock | undefined;
-export function addLocalLog(log: LogRaw, typeHint: LogType) {
-    // NOTE: Eventually... we should ignore logs for non-public machines?
-    //if (isNoNetwork()) return;
-    try {
-        if (!pendingBlock) {
-            ensureBlocksFlushing();
-            let now = Date.now();
-            pendingBlock = {
-                classes: {},
-                startTime: now,
-                endTime: now,
-
-                threadId: "",
-                machineId: "",
-
-                seqNum: nextBlockSeqNum++,
-                count: 0,
-                fatalCount: 0,
-                errorCount: 0,
-                warnCount: 0,
-                infoCount: 0,
-            };
-        }
-
-        if (!logClassifier) {
-            ignoreErrors(initLogClassifier());
-            pendingLogs.push({ log, type: typeHint });
-            return;
-        }
-        let classifier = logClassifier(log, typeHint);
-        // Add threadId and machineId, if we have them
-        try {
-            pendingBlock.threadId = pendingBlock.threadId || getOwnThreadId();
-            pendingBlock.machineId = pendingBlock.machineId || getOwnMachineId();
-        } catch { }
-        addToLogBlock(pendingBlock, classifier, log);
-        ErrorLogControllerBase.onLocalLog(log, classifier);
-    } catch (e) {
-        internalError(e);
-    }
-}
-setTimeout(() => {
-    void initLogClassifier();
-}, 0);
-registerShutdownHandler(flushNow);
-async function flushNow() {
-    let newBlock = pendingBlock;
-    pendingBlock = undefined;
-    if (!newBlock) return;
-    newBlock.endTime = Date.now();
-    //console.log(magenta(`Flushing log block ${formatNumber(newBlock.count)} count, seq num ${newBlock.seqNum}, at ${Date.now()}`));
-    if (newBlock.count > 0) {
-        let fileInfo = await logBlockToBuffer(newBlock);
-        await LOG_ARCHIVES().set(fileInfo.fileName, fileInfo.buffer);
-        if (Math.random() < 0.05) {
-            console.log(blue(`Looking for old log blocks`));
-            let deleteThreshold = Date.now() - LOG_STORE_HISTORY;
-            let files = await LOG_ARCHIVES().find("", { shallow: true, type: "files" });
-            for (let file of files) {
-                if (!file.endsWith(LOG_BLOCK_EXTENSION)) continue;
-                let info = getLogBlockInfo(file);
-                if (info.startTime < deleteThreshold) {
-                    console.log(blue(`Deleting old log block ${file}`));
-                    await LOG_ARCHIVES().del(file);
-                }
-            }
-        }
-    }
-}
-
-let ensureBlocksFlushing = lazy(() => {
-    setInterval(async () => {
-        await internalError(flushNow());
-    }, LOG_FLUSH_INTERVAL);
-
-    // Read log classifiers every hour, in case we miss an update
-    setInterval(() => {
-        initLogClassifier.reset();
-        logErrors(initLogClassifier());
-    }, timeInMinute * 60);
-});
-
-let pendingLogs: { log: LogRaw; type: LogType }[] = [];
-let logClassifier: ((log: LogRaw, typeHint: LogType) => LogClass) | undefined;
-let initLogClassifier = lazy(async () => {
-    if (!isNode()) return null as never;
-    let logClasses = await new ErrorLogControllerBase().getClasses();
-    logClassifier = getLogClassCategorizer(logClasses);
-    if (pendingLogs.length > 0) {
-        let logs = pendingLogs;
-        pendingLogs = [];
-        for (let log of logs) {
-            addLocalLog(log.log, log.type);
-        }
-    }
-    return {
-        classify: logClassifier,
-        logClasses: new Map(logClasses.map(a => [a.id, a])),
-    };
-});
-async function onLogClassesUpdated() {
-    initLogClassifier.reset();
-    await initLogClassifier();
-    await ErrorLogControllerBase.recalculateNotifyState();
-}
-
-export class ErrorLogControllerBase {
-    public async scanBlocks(): Promise<LogBlockInfo[]> {
-        let files = await LOG_ARCHIVES().find("", { shallow: true, type: "files" });
-        files = files.filter(a => a.endsWith(LOG_BLOCK_EXTENSION));
-        let infos = files.map(getLogBlockInfo);
-        if (pendingBlock) {
-            infos.push({
-                startTime: pendingBlock.startTime,
-                endTime: pendingBlock.endTime,
-                seqNum: pendingBlock.seqNum,
-                count: pendingBlock.count,
-                fatalCount: pendingBlock.fatalCount,
-                errorCount: pendingBlock.errorCount,
-                warnCount: pendingBlock.warnCount,
-                infoCount: pendingBlock.infoCount,
-                threadId: pendingBlock.threadId,
-                machineId: pendingBlock.machineId,
-                fileName: "",
-            });
-        }
-        return infos;
-    }
-
-    public async readBlocks(config: {
-        files: string[];
-    }): Promise<Buffer[]> {
-        let fileNames = config.files.filter(a =>
-            a.endsWith(LOG_BLOCK_EXTENSION) && !(a.includes("/") || a.includes("\\") || a.includes(".."))
-            || !a
-        );
-        let buffers: Buffer[] = [];
-        async function runThread() {
-            while (fileNames.length > 0) {
-                let fileName = fileNames.pop()!;
-                if (!fileName) {
-                    if (pendingBlock) {
-                        buffers.push((await logBlockToBuffer(pendingBlock)).buffer);
-                    }
-                    continue;
-                }
-                let buffer = await LOG_ARCHIVES().get(fileName);
-                if (!buffer) continue;
-                buffers.push(buffer);
-            }
-        }
-        await Promise.all(list(16).map(runThread));
-        return buffers;
-    }
-    public async readAllLiveBlocks(): Promise<Buffer[]> {
-        let nodeIds = await getAllNodeIds();
-        let blockBuffers: Buffer[] = [];
-        await Promise.allSettled(nodeIds.map(async nodeId => {
-            let controller = ErrorLogController.nodes[nodeId];
-            let buffers = await timeoutToUndefinedSilent(5000, controller.readBlocks({ files: [""] }));
-            if (!buffers) return;
-            console.log(magenta(`Read live data from ${nodeId}`));
-            blockBuffers.push(...buffers);
-        }));
-        return blockBuffers;
-    }
-
-    public async getClasses(): Promise<LogClass[]> {
-        let logClassesBuffer = await LOG_ARCHIVES().get(LOG_CLASSES_PATH);
-        let logClasses = logClassesFromString(logClassesBuffer?.toString() || "");
-        return logClasses;
-    }
-
-    public async updateClass(logClass: LogClass | { id: string; action: "delete" }) {
-        let logClasses = await this.getClasses();
-        let existingIndex = logClasses.findIndex(a => a.id === logClass.id);
-        if ("action" in logClass) {
-            if (logClass.action === "delete") {
-                if (existingIndex !== -1) {
-                    logClasses.splice(existingIndex, 1);
-                }
-            }
-        } else {
-            if (existingIndex !== -1) {
-                logClasses[existingIndex] = logClass;
-            } else {
-                logClasses.push(logClass);
-            }
-        }
-        let buffer = Buffer.from(logClassesToString(logClasses));
-        await LOG_ARCHIVES().set(LOG_CLASSES_PATH, buffer);
-        logErrors((async () => {
-            await onLogClassesUpdated();
-            // Notify ALL other nodes
-            let nodeIdsToTest = await getAllNodeIds();
-            for (let nodeId of nodeIdsToTest) {
-                ignoreErrors(ErrorLogController.nodes[nodeId].onClassesUpdated());
-            }
-        })());
-    }
-
-    public async onClassesUpdated() {
-        ignoreErrors(onLogClassesUpdated());
-    }
-
-    private static joinNotifies(notify: (LogIssueNotify | undefined)[]): LogIssueNotify {
-        // If we don't need to merge, don't so callers can check for ===
-        let finalNotify: LogIssueNotify | undefined = undefined;
-        for (let next of notify) {
-            if (!next) continue;
-            if (!finalNotify) {
-                finalNotify = next;
-                continue;
-            }
-            if (next.oldestUnsuppressed && (!finalNotify.oldestUnsuppressed || next.oldestUnsuppressed.time < finalNotify.oldestUnsuppressed.time)) {
-                if (!finalNotify.oldestUngrouped) {
-                    finalNotify = next;
-                } else {
-                    finalNotify = { ...finalNotify, oldestUnsuppressed: next.oldestUnsuppressed };
-                }
-            }
-            if (next.oldestUngrouped && (!finalNotify.oldestUngrouped || next.oldestUngrouped.time < finalNotify.oldestUngrouped.time)) {
-                if (!finalNotify.oldestUnsuppressed) {
-                    finalNotify = next;
-                } else {
-                    finalNotify = { ...finalNotify, oldestUngrouped: next.oldestUngrouped };
-                }
-            }
-        }
-        return finalNotify || undefined;
-    }
-
-    private static logIssueNotifyEquals(lhs: LogIssueNotify | undefined, rhs: LogIssueNotify | undefined) {
-        return (
-            lhs?.oldestUngrouped?.message === rhs?.oldestUngrouped?.message
-            && lhs?.oldestUnsuppressed?.message === rhs?.oldestUnsuppressed?.message
-            && lhs?.oldestUngrouped?.time === rhs?.oldestUngrouped?.time
-            && lhs?.oldestUnsuppressed?.time === rhs?.oldestUnsuppressed?.time
-        );
-    }
-
-    private static inMemoryNotify: LogIssueNotify | undefined;
-    private static updateInMemoryNotify(notify: LogIssueNotify | undefined) {
-        if (ErrorLogControllerBase.logIssueNotifyEquals(this.inMemoryNotify, notify)) {
-            return;
-        }
-        this.inMemoryNotify = notify;
-        for (let watcher of this.inMemoryNotifyWatchers) {
-            ErrorLogController.nodes[watcher]
-                .liveLogsIssueNotifyCallback(notify)
-                .catch(() => {
-                    this.inMemoryNotifyWatchers.delete(watcher);
-                });
-        }
-        logErrors(ErrorLogControllerBase.recalcAllNotify());
-    }
-    private static inMemoryNotifyWatchers = new Set<string>();
-    public static onLocalLog(log: LogRaw, classObj: LogClass) {
-        if (!logIssueNotifyTypes.includes(classObj.type)) return;
-        let notify: LogIssueNotify | undefined;
-        if (isUngroupedClass(classObj)) {
-            notify = {
-                oldestUnsuppressed: undefined,
-                oldestUngrouped: log,
-            };
-        } else if (log.time > classObj.suppressUntil) {
-            notify = {
-                oldestUnsuppressed: log,
-                oldestUngrouped: undefined,
-            };
-        }
-        if (notify) {
-            this.updateInMemoryNotify(this.joinNotifies([this.inMemoryNotify, notify]));
-        }
-    }
-    public async internalWatchLiveLogs() {
-        let watcher = SocketFunction.getCaller().nodeId;
-        ErrorLogControllerBase.inMemoryNotifyWatchers.add(watcher);
-        // Await, so the client gets it's own errors
-        await ErrorLogController.nodes[watcher].liveLogsIssueNotifyCallback(ErrorLogControllerBase.inMemoryNotify);
-    }
-
-    private static liveLogsIssueNotify = new Map<string, LogIssueNotify | undefined>();
-    public async liveLogsIssueNotifyCallback(notify: LogIssueNotify | undefined) {
-        let caller = SocketFunction.getCaller();
-        let prev = ErrorLogControllerBase.liveLogsIssueNotify.get(caller.nodeId);
-        if (ErrorLogControllerBase.logIssueNotifyEquals(prev, notify)) {
-            return;
-        }
-        // Don't log when we receive it, otherwise errors show up on all servers which is very confusing and annoying!
-        //console.log(magenta(`Received log notify from ${caller.nodeId} for ${notify?.oldestUngrouped?.message}`));
-        ErrorLogControllerBase.liveLogsIssueNotify.set(caller.nodeId, notify);
-        logErrors(ErrorLogControllerBase.recalcAllNotify());
-    }
-
-
-    private static allNotify: LogIssueNotify | undefined;
-    private static allWatchers = new Set<string>();
-    private static diskNotify: LogIssueNotify | undefined;
-    private static async recalcAllNotify() {
-        if (!ErrorLogControllerBase.watchingLogs) return;
-        let notifies = [
-            ...Array.from(this.liveLogsIssueNotify.values()),
-            this.inMemoryNotify,
-            this.diskNotify
-        ];
-        let { classify } = await initLogClassifier();
-        // Reclassify based on the most recent classifier
-        //  - This only removes classifications, it doesn't add them back. Each node should be watching
-        //      classifiers and rerunning on their data to add classifications back.
-        notifies = notifies.map(x => {
-            if (x?.oldestUngrouped && !isUngroupedClass(classify(x.oldestUngrouped, "error"))) {
-                x = { ...x, oldestUngrouped: undefined };
-            }
-            if (x?.oldestUnsuppressed && x.oldestUnsuppressed.time < classify(x.oldestUnsuppressed, "error").suppressUntil) {
-                x = { ...x, oldestUnsuppressed: undefined };
-            }
-            return x;
-        });
-        let newNotify = this.joinNotifies(notifies);
-        if (this.logIssueNotifyEquals(newNotify, this.allNotify)) {
-            return;
-        }
-        this.allNotify = newNotify;
-        for (let watcher of this.allWatchers) {
-            ErrorLogController.nodes[watcher].logsIssueNotifyCallback(newNotify)
-                .catch(() => {
-                    this.allWatchers.delete(watcher);
-                });
-        }
-    }
-
-    private static watchForNewNodes = lazy(() => {
-        watchDeltaNodeIds(async ({ newNodeIds }) => {
-            for (let nodeId of newNodeIds) {
-                if (nodeId === getOwnNodeId()) continue;
-                if (!ErrorLogControllerBase.liveLogsIssueNotify.has(nodeId)) {
-                    ErrorLogControllerBase.liveLogsIssueNotify.set(nodeId, undefined);
-                }
-                ignoreErrors(ErrorLogController.nodes[nodeId].internalWatchLiveLogs());
-                SocketFunction.onNextDisconnect(nodeId, () => {
-                    // HACK: Remove it, to prevent notifications from errors which we never written to disk.
-                    //  - The only other alternative is to get this error in the UI... somehow, and onto disk,
-                    //      but not if the other node DOES write it to disk, which... gets complicated.
-                    ErrorLogControllerBase.liveLogsIssueNotify.delete(nodeId);
-                });
-            }
-        });
-    });
-
-    private static watchingLogs = false;
-    public static async recalculateNotifyState() {
-        ErrorLogControllerBase.watchForNewNodes();
-
-        // Watch all nodes again, in case they disappear
-        for (let node of await getAllNodeIds()) {
-            if (node === getOwnNodeId()) continue;
-            ignoreErrors(ErrorLogController.nodes[node].internalWatchLiveLogs());
-        }
-
-        let threshold = Date.now() - NOTIFY_HISTORY;
-
-        let { classify, logClasses } = await initLogClassifier();
-        function summarizeBlock(block: LogBlock | undefined) {
-            if (!block) return undefined;
-            let notify: LogIssueNotify | undefined = undefined;
-            for (let classSummary of Object.values(block.classes)) {
-                for (let logRaw of classSummary.examples) {
-                    if (logRaw.time < threshold) continue;
-                    let classObj = classify(logRaw, classSummary.logClassType || logClasses.get(classSummary.logClassId)?.type || "error");
-                    if (!logIssueNotifyTypes.includes(classObj.type)) continue;
-                    if (isUngroupedClass(classObj)) {
-                        if (!notify || !notify.oldestUngrouped || classSummary.firstTime < notify.oldestUngrouped.time) {
-                            if (!notify) notify = { oldestUnsuppressed: undefined, oldestUngrouped: undefined };
-                            if (!notify.oldestUngrouped || notify.oldestUngrouped.time > classSummary.firstTime) {
-                                notify.oldestUngrouped = classSummary.examples[0];
-                            }
-                        }
-                    } else if (logRaw.time > classObj.suppressUntil) {
-                        if (!notify || !notify.oldestUnsuppressed || classSummary.firstTime < notify.oldestUnsuppressed.time) {
-                            if (!notify) notify = { oldestUnsuppressed: undefined, oldestUngrouped: undefined };
-                            if (!notify.oldestUnsuppressed || notify.oldestUnsuppressed.time > classSummary.firstTime) {
-                                notify.oldestUnsuppressed = classSummary.examples[0];
-                            }
-                        }
-                    }
-                }
-            }
-            return notify;
-        }
-
-        // Recalculate in memory state
-        this.updateInMemoryNotify(summarizeBlock(pendingBlock));
-
-        if (ErrorLogControllerBase.watchingLogs) {
-            let blockInfos = await new ErrorLogControllerBase().scanBlocks();
-            blockInfos = blockInfos.filter(a => a.startTime > threshold);
-            let blocks = await new ErrorLogControllerBase().readBlocks({ files: blockInfos.map(a => a.fileName) });
-            let blockObjs = await Promise.all(blocks.map(x => logBlockFromBuffer(x)));
-            let notify = this.joinNotifies(blockObjs.map(summarizeBlock));
-            this.diskNotify = notify;
-        }
-
-        await this.recalcAllNotify();
-    }
-
-    // NOTE: We never unwatch. Watching for LogIssueNotify should be fairly cheap, so this is fine
-    public static startLogWatch = lazy(async () => {
-        ErrorLogControllerBase.watchingLogs = true;
-        await this.recalculateNotifyState();
-    });
-
-    public async watchLogs(callbackClass?: ControllerPick<typeof ErrorLogController, "logsIssueNotifyCallback">) {
-        let self = ErrorLogControllerBase;
-        self.watchingLogs = true;
-
-        let caller = SocketFunction.getCaller();
-        self.allWatchers.add(caller.nodeId);
-
-        await self.startLogWatch();
-
-        let client = callbackClass ? SocketFunction.rehydrateSocketCaller(callbackClass) : ErrorLogController;
-        await client.nodes[caller.nodeId].logsIssueNotifyCallback(self.allNotify);
-    }
-
-    public static logIssueNotifyWatchers = new Set<(notify: LogIssueNotify | undefined) => void>();
-    public async logsIssueNotifyCallback(notify: LogIssueNotify | undefined) {
-        onLogIssueNotify(notify);
-    }
-    public async unwatchLogs() {
-        let self = ErrorLogControllerBase;
-        let caller = SocketFunction.getCaller();
-        self.allWatchers.delete(caller.nodeId);
-    }
-}
-
-function onLogIssueNotify(notify: LogIssueNotify | undefined) {
-    for (let watcher of ErrorLogControllerBase.logIssueNotifyWatchers) {
-        try {
-            watcher(notify);
-        } catch (e) {
-            logErrors(e);
-        }
-    }
-}
-export function watchLogIssueNotify(callback: (notify: LogIssueNotify | undefined) => void) {
-    ErrorLogControllerBase.logIssueNotifyWatchers.add(callback);
-}
-
-export type LogIssueNotify = undefined | {
-    // NOTE: Storing the oldest requires a lot fewer notifications, and is probably what we want
-    //  anyways (it tells us when the problem started).
-    oldestUnsuppressed: LogRaw | undefined;
-    oldestUngrouped: LogRaw | undefined;
-    // NOTE: Count is too hard to count accurately, due to live values being written to disk, etc.
-}
-
-const ErrorLogController = SocketFunction.register(
-    "ErrorLogController-219a7c2e-2da6-4753-9e0e-1f42dc5f6970",
-    new ErrorLogControllerBase(),
-    () => ({
-        scanBlocks: {},
-        readBlocks: {},
-        readAllLiveBlocks: {},
-        getClasses: {},
-        updateClass: {},
-        onClassesUpdated: {},
-
-        watchLogs: {},
-        unwatchLogs: {},
-        liveLogsIssueNotifyCallback: {},
-        logsIssueNotifyCallback: {},
-        internalWatchLiveLogs: {},
-    }),
-    () => ({
-        hooks: [requiresNetworkTrustHook],
-    }),
-);