querysub 0.312.0 → 0.313.0

package/src/diagnostics/logs/errorNotifications/ErrorNotificationController.ts

@@ -0,0 +1,577 @@
+import { isNode } from "typesafecss";
+import { getArchives } from "../../../-a-archives/archives";
+import { SizeLimiter } from "../../SizeLimiter";
+import { FastArchiveAppendable, createLogScanner, objectDelimitterBuffer } from "../FastArchiveAppendable";
+import { LogDatum, getLoggers } from "../diskLogger";
+import os from "os";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { cache, cacheLimited, lazy } from "socket-function/src/caching";
+import { getAllNodeIds } from "../../../-f-node-discovery/NodeDiscovery";
+import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
+import { sort, throttleFunction, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { batchFunction, delay, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { SocketChannel } from "../../../functional/SocketChannel";
+import { measureFnc, measureWrap } from "socket-function/src/profiling/measure";
+import { FastArchiveAppendableControllerBase, getFileMetadataHash } from "../FastArchiveController";
+import fs from "fs";
+import zlib from "zlib";
+import { getSyncedController } from "../../../library-components/SyncedController";
+import { qreact } from "../../../4-dom/qreact";
+import { requiresNetworkTrustHook } from "../../../-d-trust/NetworkTrust2";
+import { assertIsManagementUser } from "../../managementPages";
+import { streamToIteratable } from "../../../misc";
+
+export const MAX_RECENT_ERRORS = 20;
+const MAX_RECENT_ERRORS_PER_FILE = 3;
+
+const BACKBLAZE_POLL_INTERVAL = timeInMinute * 30;
+// The higher we turn this up the less bandwidth and processing time is spent on errors. But also, the longer delay between an error happening and getting an error notification
+const LOCAL_BROADCAST_BATCH = 5000;
+const NOTIFICATION_BROADCAST_BATCH = 5000;
+const SELF_THROTTLE_INTERVAL = 100;
+const SELF_THROTTLE_DELAY = 50;
+
+const VIEW_WINDOW = timeInDay * 7;
+const SUPPRESSION_POLL_INTERVAL = timeInMinute * 15;
+const READ_CHUNK_SIZE = 1024 * 1024 * 10;
+
+const LOCAL_CACHE_LIMIT_BATCH = timeInMinute * 30;
+
+export const NOT_AN_ERROR_EXPIRE_TIME = 2524608000000;
+
+export type SuppressionEntry = {
+    key: string;
+    // Includes, exact case
+    //  - Supports "*" as a wildcard, which matches slower (converts it to a regex)
+    match: string;
+    comment: string;
+    lastUpdateTime: number;
+    expiresAt: number;
+};
+type SuppressionListBase = {
+    entries: {
+        [key: string]: SuppressionEntry;
+    };
+};
+
+type SuppressedChecker = {
+    entry: SuppressionEntry;
+    fnc: (buffer: Buffer, posStart: number, posEnd: number) => boolean;
+}
+
+function getAppendables() {
+    let loggers = getLoggers();
+    if (!loggers) throw new Error("Loggers not available?");
+    // error, warn
+    return [loggers.errorLogs, loggers.warnLogs];
+}
+
+// NOTE: This cache resets eventually, due to it being no longer used every time we update the suppression list. But that's probably fine...
+export const getSuppressEntryChecker = cacheLimited(
+    1000 * 10,
+    function getSuppressEntryChecker(entry: SuppressionEntry): SuppressedChecker {
+        if (entry.match.includes("*")) {
+            try {
+                let regex = new RegExp(entry.match.replaceAll("*", ".*"));
+                return {
+                    entry,
+                    fnc: (buffer: Buffer, posStart: number, posEnd: number) => {
+                        return regex.test(buffer.slice(posStart, posEnd).toString());
+                    }
+                };
+            } catch (error) {
+                console.error(`Failed to create regex for ${JSON.stringify(entry.match)}, ignoring wildcard in it`, { error });
+            }
+        }
+        let matchBuffer = Buffer.from(entry.match);
+        let char0 = matchBuffer[0];
+        return {
+            entry,
+            fnc: (buffer: Buffer, posStart: number, posEnd: number) => {
+                for (let i = posStart; i < posEnd; i++) {
+                    if (matchBuffer.length === 1) return true;
+                    if (buffer[i] === char0) {
+                        for (let j = 1; j < matchBuffer.length; j++) {
+                            let ch2 = buffer[i + j];
+                            if (ch2 !== matchBuffer[j]) {
+                                break;
+                            }
+                            if (j === matchBuffer.length - 1) {
+                                return true;
+                            }
+                        }
+                    }
+                }
+                return false;
+            },
+        };
+    }
+);
+
+
+const suppressionListKey = "suppression-list.json";
+const suppressionListArchive = archiveJSONT<SuppressionListBase>(() =>
+    getArchives("suppression-list"),
+);
+
+class SuppressionList {
+    private getEntries = lazy(async (): Promise<SuppressionListBase> => {
+        await runInfinitePollCallAtStart(SUPPRESSION_POLL_INTERVAL, async () => {
+            let entries = await suppressionListArchive.get(suppressionListKey);
+            if (!entries) {
+                entries = { entries: {} };
+            }
+            await suppressionListArchive.set(suppressionListKey, entries);
+            this.getEntries.set(Promise.resolve(entries));
+        });
+        // Infinite poll will have set this, so we don't infinitely loop
+        return await this.getEntries();
+    });
+
+    public async filterObjsToNonSuppressed(objs: LogDatum[]): Promise<LogDatum[]> {
+        // NOTE: Streamed data should be rare enough
+        let parts: Buffer[] = [];
+        for (let obj of objs) {
+            parts.push(Buffer.from(JSON.stringify(obj)));
+            parts.push(objectDelimitterBuffer);
+        }
+        let buffer = Buffer.concat(parts);
+        let scanner = await this.scanForRecentErrors();
+        await scanner.onData(buffer);
+        return await scanner.finish();
+    }
+    public async scanForRecentErrors(): Promise<{
+        onData: (data: Buffer) => void;
+        finish: () => Promise<LogDatum[]>;
+    }> {
+        let entries = await this.getEntries();
+        let now = Date.now();
+        let checkers = Object.values(entries.entries)
+            .map(entry => getSuppressEntryChecker(entry))
+            ;
+        let datums: LogDatum[] = [];
+        let callback = createLogScanner({
+            onParsedData: (posStart, posEnd, buffer) => {
+                if (buffer === "done") {
+                    return;
+                }
+                let outdatedSuppressionKey: string | undefined = undefined;
+                for (let checker of checkers) {
+                    if (checker.fnc(buffer, posStart, posEnd)) {
+                        if (checker.entry.expiresAt < now) {
+                            outdatedSuppressionKey = checker.entry.key;
+                            continue;
+                        }
+                        return;
+                    }
+                }
+                let obj: LogDatum;
+                try {
+                    obj = JSON.parse(buffer.slice(posStart, posEnd).toString()) as LogDatum;
+                } catch (e: any) {
+                    process.stderr.write(`Failed to parse log datum in around ${buffer.slice(posStart, posEnd).slice(0, 100).toString("hex")}, error is:\n${e.stack}`);
+                    return;
+                }
+                if (outdatedSuppressionKey) {
+                    obj.__matchedOutdatedSuppressionKey = outdatedSuppressionKey;
+                }
+                datums.push(obj);
+            },
+        });
+        let lastWaitTime = Date.now();
+        return {
+            onData: runInSerial(async (buffer) => {
+                // TODO: Maybe we should add this pattern to batching.ts? Basically, if we get called fast, we allow the calls through. BUT, if we called slowly OR we are doing a lot of processing (and so we are working for all of SELF_THROTTLE_INTERVAL), then we wait. This prevents this from taking over the machine. The back off is steep though, and if the machine is lagging we might reduce to a trickle, just getting 1 call in per SELF_THROTTLE_DELAY + synchronous lag from work in other parts of the program.
+                let now = Date.now();
+                if (now - lastWaitTime > SELF_THROTTLE_INTERVAL) {
+                    await delay(SELF_THROTTLE_DELAY);
+                    lastWaitTime = now;
+                }
+                await callback(buffer);
+            }),
+            finish: async () => {
+                await callback("done");
+                // NOTE: We COULD limit as we run, however... how many errors are we really going to encounter that AREN'T suppressed? Suppression is supposed to prevent overload anyways. I guess worst case scenario, yes, we could get overloaded, but... if we are logging more NEW errors than we can store in memory, we have bigger problems...
+                return limitRecentErrors(datums);
+            },
+        };
+    }
+    public async setSuppressionEntry(entry: SuppressionEntry) {
+        let entries = await this.getEntries();
+        entry.lastUpdateTime = Date.now();
+        entries.entries[entry.key] = entry;
+        await suppressionListArchive.set(suppressionListKey, entries);
+        await recentErrors.onSuppressionChanged();
+    }
+    public async removeSuppressionEntry(key: string) {
+        let entries = await this.getEntries();
+        delete entries.entries[key];
+        await suppressionListArchive.set(suppressionListKey, entries);
+        await recentErrors.onSuppressionChanged();
+    }
+
+    public async getSuppressionList(): Promise<SuppressionEntry[]> {
+        return Object.values((await this.getEntries()).entries);
+    }
+}
+const suppressionList = new SuppressionList();
+export const SuppressionListController = getSyncedController(SocketFunction.register(
+    "SuppressionListController-08f985d8-8d06-4041-ac4b-44566c54615d",
+    suppressionList,
+    () => ({
+        setSuppressionEntry: {},
+        removeSuppressionEntry: {},
+        getSuppressionList: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    }),
+    {
+        noFunctionMeasure: true,
+    }
+), {
+    reads: {
+        getSuppressionList: ["suppression-list"],
+    },
+    writes: {
+        setSuppressionEntry: ["suppression-list", "recent-errors"],
+        removeSuppressionEntry: ["suppression-list", "recent-errors"],
+    },
+});
+
+class URLCache {
+    private root = isNode() && os.homedir() + "/backblaze-cache/" || "";
+    private sizeLimiter = new SizeLimiter({
+        diskRoot: this.root,
+        maxBytes: 1024 * 1024 * 1024 * 10,
+        // Basically... enough cache for a week of hourly data, for 1000 servers. At which point, this whole system will suck (too many servers to check), and we might have to change it
+        //  - Ex, by only using backblaze, and aggregating logs there, to reduce file counts
+        //  - Or... just by using elasticsearch...
+        //  - Or... just have dedicated search servers, which store the data in memory. We will have few people querying, so having 100 servers all execute a 60s search query is fine.
+        maxFiles: 24 * 7 * 1000,
+        maxDiskFraction: 0.1,
+        maxTotalDiskFraction: 0.96,
+        minBytes: 0,
+    });
+    // Returns the PATH to it on disk, so you can then parse it via a stream, so you never have to store it all at once in memory.
+    public async getURLLocalPath(url: string, hash: string): Promise<string | undefined> {
+        if (!isNode()) return undefined;
+
+        // Create cache directory if it doesn't exist
+        if (!fs.existsSync(this.root)) {
+            await fs.promises.mkdir(this.root, { recursive: true });
+        }
+
+        const filePath = this.root + hash;
+
+        try {
+            // Check if file already exists
+            const stats = await fs.promises.stat(filePath);
+            if (stats.isFile()) {
+                void this.applyLimitting([]);
+                return filePath;
+            }
+        } catch (e: any) {
+            // File doesn't exist, need to download it
+            if (e.code !== "ENOENT") {
+                throw e;
+            }
+        }
+
+        try {
+            // Download and stream to disk
+            const response = await fetch(url);
+            if (!response.ok) {
+                throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+            }
+
+            if (!response.body) {
+                throw new Error(`Response body is undefined for ${url}`);
+            }
+
+            // Create write stream
+            let tempPath = filePath + ".temp";
+            const writeStream = fs.createWriteStream(tempPath);
+            const reader = response.body.getReader();
+
+            try {
+                for await (const chunk of streamToIteratable(reader)) {
+                    if (!chunk) continue;
+                    let result = writeStream.write(Buffer.from(chunk));
+                    if (!result) {
+                        await new Promise<void>(resolve => writeStream.once("drain", resolve));
+                    }
+                }
+
+                await new Promise<void>((resolve, reject) => {
+                    writeStream.end((err: any) => {
+                        if (err) reject(err);
+                        else resolve();
+                    });
+                });
+
+                // Trigger cache cleanup in the background
+                void this.applyLimitting([]);
+
+                await fs.promises.rename(tempPath, filePath);
+                return filePath;
+            } catch (error) {
+                // Clean up partial file on error
+                try {
+                    await fs.promises.unlink(tempPath);
+                } catch {
+                    // Ignore cleanup errors
+                }
+                throw error;
+            } finally {
+                reader.releaseLock();
+            }
+        } catch (error) {
+            console.error(`Failed to download and cache file from ${url}:`, error);
+            return undefined;
+        }
+    }
+    private applyLimitting = batchFunction({ delay: LOCAL_CACHE_LIMIT_BATCH }, async () => {
+        const files = await fs.promises.readdir(this.root);
+        const fileInfos: { time: number; bytes: number; path: string; }[] = [];
+
+        for (const file of files) {
+            const filePath = this.root + file;
+            const stats = await fs.promises.stat(filePath);
+            if (stats.isFile()) {
+                fileInfos.push({
+                    time: stats.atimeMs,
+                    bytes: stats.size,
+                    path: filePath,
+                });
+            }
+        }
+
+        const result = await this.sizeLimiter.limit(fileInfos);
+
+        for (const fileToRemove of result.remove) {
+            await fs.promises.unlink(fileToRemove.path);
+        }
+    });
+}
+const urlCache = new URLCache();
+
+const limitRecentErrors = measureWrap(function limitRecentErrors(objs: LogDatum[]) {
+    sort(objs, x => x.time);
+    let recent: LogDatum[] = [];
+    let countByFile = new Map<string, number>();
+    // NOTE: We iterate backwards, because... usually new logs come in at the end, and are pushed, so we want to sort by time (that way we often don't have to resort by much). And if we sort by time, the newest at at the end!
+    for (let i = objs.length - 1; i >= 0; i--) {
+        let obj = objs[i];
+        let file = String(obj.__FILE__) || "";
+        let count = countByFile.get(file) || 0;
+        count++;
+        if (count > MAX_RECENT_ERRORS_PER_FILE) continue;
+        countByFile.set(file, count);
+        recent.push(obj);
+        if (recent.length >= MAX_RECENT_ERRORS) break;
+    }
+    return recent;
+});
+
+class RecentErrors {
+
+    private initialize = lazy(async () => {
+        errorWatcherBase.watch(x => this.addErrors(x));
+        await this.scanNow({});
+        runInfinitePoll(BACKBLAZE_POLL_INTERVAL, async () => {
+            await this.scanNow({ noLocalFiles: true });
+        });
+    });
+
+    private _recentErrors: LogDatum[] = [];
+    private updateRecentErrors = (objs: LogDatum[]) => {
+        let newRecentErrors = limitRecentErrors(objs);
+        // If any changed
+        let prev = new Set(this._recentErrors);
+        let newErrors = new Set(newRecentErrors);
+        function hasAnyChanged() {
+            for (let obj of newRecentErrors) {
+                if (!prev.has(obj)) {
+                    return true;
+                }
+            }
+            for (let obj of prev) {
+                if (!newErrors.has(obj)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+        if (hasAnyChanged()) {
+            this._recentErrors = newRecentErrors;
+            void this.broadcastUpdate(undefined);
+        }
+    };
+    private broadcastUpdate = batchFunction({ delay: NOTIFICATION_BROADCAST_BATCH }, () => {
+        recentErrorsChannel.broadcast(true);
+    });
+
+    private addErrors(objs: LogDatum[]) {
+        for (let obj of objs) {
+            this._recentErrors.push(obj);
+        }
+        this.updateRecentErrors(this._recentErrors);
+    }
+
+    public async onSuppressionChanged() {
+        this.updateRecentErrors(await suppressionList.filterObjsToNonSuppressed(this._recentErrors));
+    }
+
+    private scannedHashes = new Set<string>();
+    private scanNow = runInSerial(async (config: {
+        noLocalFiles?: boolean;
+    }) => {
+        for (let appendable of getAppendables()) {
+            let startTime = Date.now() - VIEW_WINDOW;
+            let endTime = Date.now() + timeInHour * 2;
+            let result = await new FastArchiveAppendableControllerBase().startSynchronizeInternal({
+                range: {
+                    startTime,
+                    endTime,
+                },
+                rootPath: appendable.rootPath,
+                noLocalFiles: config.noLocalFiles,
+            });
+            // Filter again, as new suppressions change the errors
+            this.updateRecentErrors(await suppressionList.filterObjsToNonSuppressed(this._recentErrors));
+            let recentLimit = 0;
+            const applyRecentLimit = () => {
+                if (this._recentErrors.length < MAX_RECENT_ERRORS) return;
+                recentLimit = this._recentErrors[0].time - timeInHour * 2;
+            };
+            applyRecentLimit();
+            for (let file of result.files) {
+                let path: string | undefined = undefined;
+                let size: number | undefined = undefined;
+                try {
+                    // Skip if it is older than even our throttled values!
+                    if (file.startTime < recentLimit) continue;
+                    // Don't skip local files, we only do them once at the start, or on demand when we want to recalc all anyways!
+                    if (!file.nodeId) {
+                        let hash = getFileMetadataHash(file);
+                        if (this.scannedHashes.has(hash)) continue;
+                        this.scannedHashes.add(hash);
+                    }
+
+                    let hash = getFileMetadataHash(file);
+                    path = await urlCache.getURLLocalPath(file.url, hash);
+                    if (!path) continue;
+                    let scanner = await suppressionList.scanForRecentErrors();
+
+                    // Stream decompress the file while reading it
+                    size = await fs.promises.stat(path).then(x => x.size);
+                    if (!size) {
+                        console.error(`Deleting empty cached file ${path} for ${file.url} `);
+                        // NOTE: This means we will repeatedly download empty files, but... that should be fairly fast...
+                        await fs.promises.unlink(path);
+                        continue;
+                    }
+                    let sizeT = size;
+                    let fd = await fs.promises.open(path, "r");
+                    try {
+                        await new Promise<void>(async (resolve, reject) => {
+                            const gunzip = zlib.createGunzip();
+
+                            gunzip.on("data", (chunk: Buffer) => {
+                                void scanner.onData(chunk);
+                            });
+
+                            gunzip.on("end", async () => {
+                                try {
+                                    resolve();
+                                } catch (error) {
+                                    reject(error);
+                                }
+                            });
+
+                            gunzip.on("error", reject);
+
+                            try {
+                                for (let i = 0; i < sizeT; i += READ_CHUNK_SIZE) {
+                                    let chunkSize = Math.min(READ_CHUNK_SIZE, sizeT - i);
+                                    let buffer = Buffer.alloc(chunkSize);
+                                    await fd.read(buffer, 0, chunkSize, i);
+                                    let result = gunzip.write(buffer);
+                                    if (!result) {
+                                        await new Promise(resolve => gunzip.once("drain", resolve));
+                                    }
+                                }
+                                gunzip.end();
+                            } catch (error) {
+                                reject(error);
+                            }
+                        });
+                    } finally {
+                        await fd.close();
+                    }
+                    let newErrors = await scanner.finish();
+                    this.addErrors(newErrors);
+                    applyRecentLimit();
+                } catch (e: any) {
+                    console.error(`Failed to scan file ${file.url}, size is ${size}, error is:\n${e.stack}`);
+                }
+            }
+        }
+    });
+
+
+    public async getRecentErrors(): Promise<LogDatum[]> {
+        await this.initialize();
+        return this._recentErrors;
+    }
+
+    // Rescans all local, and new backblaze
+    public async rescanAllErrorsNow() {
+        await this.scanNow({});
+        return this._recentErrors;
+    }
+}
+const recentErrors = new RecentErrors();
+export const RecentErrorsController = getSyncedController(SocketFunction.register(
+    "RecentErrorsController-8450c626-2a06-4eee-81cb-67d4c2fa8155",
+    recentErrors,
+    () => ({
+        getRecentErrors: {},
+        rescanAllErrorsNow: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    }),
+), {
+    reads: {
+        getRecentErrors: ["recent-errors"],
+    },
+    writes: {
+        rescanAllErrorsNow: ["recent-errors"],
+    },
+});
+
+const recentErrorsChannel = new SocketChannel<true>("recent-errors-eeceb0c8-4086-4ab3-b3ff-fa9fd5282e14");
+
+export const watchRecentErrors = lazy(function watchRecentErrors() {
+    recentErrorsChannel.watch(async () => {
+        // Only 1 function, so just refresh all...
+        RecentErrorsController.refreshAll();
+    });
+});
+
+
+
+
+export const notifyWatchersOfError = batchFunction({
+    delay: LOCAL_BROADCAST_BATCH,
+},
+    async (objs: LogDatum[]) => {
+        objs = await suppressionList.filterObjsToNonSuppressed(objs);
+        errorWatcherBase.broadcast(objs);
+    }
+);
+
+const errorWatcherBase = new SocketChannel<LogDatum[]>("error-watcher-38de08cd-3247-4f75-9ac0-7919b240607d");