querysub 0.374.0 → 0.376.0

package/src/diagnostics/logs/errorNotifications/ErrorNotificationController.ts

@@ -1,756 +0,0 @@
-import { isNode } from "typesafecss";
-import { getArchives } from "../../../-a-archives/archives";
-import { SizeLimiter } from "../../SizeLimiter";
-import { FastArchiveAppendable, createLogScanner, objectDelimitterBuffer } from "../FastArchiveAppendable";
-import { LogDatum, getLogHash, getLoggers } from "../diskLogger";
-import os from "os";
-import { SocketFunction } from "socket-function/SocketFunction";
-import { cache, cacheLimited, lazy } from "socket-function/src/caching";
-import { getAllNodeIds } from "../../../-f-node-discovery/NodeDiscovery";
-import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
-import { sort, throttleFunction, timeInDay, timeInHour, timeInMinute } from "socket-function/src/misc";
-import { formatNumber } from "socket-function/src/formatting/format";
-import { batchFunction, delay, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { SocketChannel } from "../../../functional/SocketChannel";
-import { measureFnc, measureWrap } from "socket-function/src/profiling/measure";
-import { FastArchiveAppendableControllerBase, getFileMetadataHash } from "../FastArchiveController";
-import fs from "fs";
-import zlib from "zlib";
-import { getSyncedController } from "../../../library-components/SyncedController";
-import { qreact } from "../../../4-dom/qreact";
-import { requiresNetworkTrustHook } from "../../../-d-trust/NetworkTrust2";
-import { assertIsManagementUser } from "../../managementPages";
-import { streamToIteratable } from "../../../misc";
-import { fsExistsAsync } from "../../../fs";
-import { getPathStr2 } from "../../../path";
-
-export const MAX_RECENT_ERRORS = 20;
-const MAX_RECENT_ERRORS_PER_FILE = 3;
-
-const BACKBLAZE_POLL_INTERVAL = timeInMinute * 30;
-// The higher we turn this up the less bandwidth and processing time is spent on errors. But also, the longer delay between an error happening and getting an error notification
-const LOCAL_BROADCAST_BATCH = 5000;
-const NOTIFICATION_BROADCAST_BATCH = 5000;
-const SELF_THROTTLE_INTERVAL = 100;
-const SELF_THROTTLE_DELAY = 50;
-
-const VIEW_WINDOW = timeInDay * 7;
-const SUPPRESSION_POLL_INTERVAL = timeInMinute * 15;
-const READ_CHUNK_SIZE = 1024 * 1024 * 10;
-
-const LOCAL_CACHE_LIMIT_BATCH = timeInMinute * 30;
-
-export const NOT_AN_ERROR_EXPIRE_TIME = 2524608000000;
-
-export type SuppressionEntry = {
-    key: string;
-    // Includes, exact case
-    //  - Supports "*" as a wildcard, which matches slower (converts it to a regex)
-    match: string;
-    comment: string;
-    lastUpdateTime: number;
-    expiresAt: number;
-};
-type SuppressionListBase = {
-    entries: {
-        [key: string]: SuppressionEntry;
-    };
-};
-
-type SuppressedChecker = {
-    entry: SuppressionEntry;
-    fnc: (buffer: Buffer, posStart: number, posEnd: number) => boolean;
-}
-
-export function getErrorAppendables() {
-    let loggers = getLoggers();
-    if (!loggers) throw new Error("Loggers not available?");
-    // error, warn
-    return [loggers.errorLogs, loggers.warnLogs];
-}
-
-// NOTE: This cache resets eventually, due to it being no longer used every time we update the suppression list. But that's probably fine...
-export const getSuppressEntryChecker = cacheLimited(
-    1000 * 10,
-    function getSuppressEntryChecker(entry: SuppressionEntry): SuppressedChecker {
-        if (entry.match.includes("*")) {
-            try {
-                let regex = new RegExp(entry.match.replaceAll("*", ".*"));
-                return {
-                    entry,
-                    fnc: (buffer: Buffer, posStart: number, posEnd: number) => {
-                        return regex.test(buffer.slice(posStart, posEnd).toString());
-                    }
-                };
-            } catch (error) {
-                console.error(`Failed to create regex for ${JSON.stringify(entry.match)}, ignoring wildcard in it`, { error });
-            }
-        }
-        let matchBuffer = Buffer.from(entry.match);
-        let char0 = matchBuffer[0];
-        return {
-            entry,
-            fnc: (buffer: Buffer, posStart: number, posEnd: number) => {
-                for (let i = posStart; i < posEnd; i++) {
-                    if (matchBuffer.length === 1) return true;
-                    if (buffer[i] === char0) {
-                        for (let j = 1; j < matchBuffer.length; j++) {
-                            let ch2 = buffer[i + j];
-                            if (ch2 !== matchBuffer[j]) {
-                                break;
-                            }
-                            if (j === matchBuffer.length - 1) {
-                                return true;
-                            }
-                        }
-                    }
-                }
-                return false;
-            },
-        };
-    }
-);
-
-export const getSuppressionFull = measureWrap(function getSuppressionFull(config: {
-    entries: SuppressionEntry[];
-    blockTimeRange: {
-        startTime: number;
-        endTime: number;
-    };
-    suppressionCounts?: Map<string, number>;
-    expiredSuppressionCounts?: Map<string, number>;
-    // => wants data
-}): ((posStart: number, posEnd: number, data: Buffer, obj?: { outdatedSuppressionKey?: string }) => boolean) {
-    let { entries, blockTimeRange } = config;
-    const { suppressionCounts, expiredSuppressionCounts } = config;
-    // Add some buffer, just in case entries get added a bit later, or early.
-    let startTime = blockTimeRange.startTime - timeInHour;
-    let endTime = blockTimeRange.endTime + timeInHour;
-
-    sort(entries, x => -x.lastUpdateTime);
-
-    let checkers = entries.map(x => getSuppressEntryChecker(x));
-
-    let definitelyNotExpired = checkers.filter(x => x.entry.expiresAt > endTime);
-    let definitelyExpired = checkers.filter(x => x.entry.expiresAt < startTime);
-    let maybeExpired = checkers.filter(x => x.entry.expiresAt >= startTime && x.entry.expiresAt <= endTime);
-
-    return (posStart, posEnd, data, obj) => {
-        let suppressed = false;
-        for (let checker of definitelyNotExpired) {
-            if (checker.fnc(data, posStart, posEnd)) {
-                if (!suppressionCounts && !expiredSuppressionCounts && !obj) {
-                    return false;
-                }
-                suppressed = true;
-                if (!suppressionCounts) break;
-
-                let count = suppressionCounts.get(checker.entry.key) || 0;
-                count++;
-                suppressionCounts.set(checker.entry.key, count);
-            }
-        }
-
-        // Handle definitelyExpired - these are outdated suppressions
-        let mostRecentOutdatedSuppressionKey: string | undefined = undefined;
-        let mostRecentOutdatedSuppressionTime = 0;
-
-        // Handle maybeExpired - need to parse timestamp to check if suppression was active
-        if (maybeExpired.length > 0 && (suppressionCounts || expiredSuppressionCounts || obj)) {
-            let logTime = 0;
-            try {
-                let logEntry = JSON.parse(data.slice(posStart, posEnd).toString()) as LogDatum;
-                if (typeof logEntry.time === "number") {
-                    logTime = logEntry.time;
-                }
-            } catch { }
-
-            for (let checker of maybeExpired) {
-                if (checker.fnc(data, posStart, posEnd)) {
-                    if (checker.entry.expiresAt >= logTime) {
-                        suppressed = true;
-                        if (suppressionCounts) {
-                            let count = suppressionCounts.get(checker.entry.key) || 0;
-                            count++;
-                            suppressionCounts.set(checker.entry.key, count);
-                        }
-                    } else {
-
-                        if (checker.entry.expiresAt > mostRecentOutdatedSuppressionTime) {
-                            mostRecentOutdatedSuppressionKey = checker.entry.key;
-                            mostRecentOutdatedSuppressionTime = checker.entry.expiresAt;
-                        }
-                        // Even if we don't want the expired suppression counts, we might want the normal suppression counts, so we have to keep going. 
-                        if (expiredSuppressionCounts) {
-                            let count = expiredSuppressionCounts.get(checker.entry.key) || 0;
-                            count++;
-                            expiredSuppressionCounts.set(checker.entry.key, count);
-                        }
-                    }
-                }
-            }
-        }
-
-        if (expiredSuppressionCounts || obj) {
-            for (let checker of definitelyExpired) {
-                if (checker.fnc(data, posStart, posEnd)) {
-                    // First match is the most recent (entries are sorted by lastUpdateTime desc)
-                    if (checker.entry.expiresAt > mostRecentOutdatedSuppressionTime) {
-                        mostRecentOutdatedSuppressionKey = checker.entry.key;
-                    }
-                    if (!expiredSuppressionCounts) break;
-                    let count = expiredSuppressionCounts.get(checker.entry.key) || 0;
-                    count++;
-                    expiredSuppressionCounts.set(checker.entry.key, count);
-                }
-            }
-        }
-
-        // Set the most recent outdated suppression key if we found any and weren't suppressed
-        if (obj && mostRecentOutdatedSuppressionKey) {
-            obj.outdatedSuppressionKey = mostRecentOutdatedSuppressionKey;
-        }
-
-        return !suppressed;
-    };
-});
-
-
-const suppressionListKey = "suppression-list.json";
-const suppressionListArchive = archiveJSONT<SuppressionListBase>(() =>
-    getArchives("suppression-list"),
-);
-const suppressionUpdatedChannel = new SocketChannel<boolean>("suppression-updated");
-
-export async function getSuppressionListRaw(): Promise<SuppressionListBase> {
-    let entries = await suppressionListArchive.get(suppressionListKey);
-    if (!entries) {
-        entries = { entries: {} };
-    }
-    return entries;
-}
-
-class SuppressionList {
-    private init = lazy(async () => {
-        suppressionUpdatedChannel.watch(async () => {
-            await this.updateEntriesNow();
-            await recentErrors.onSuppressionChanged();
-        });
-        await runInfinitePollCallAtStart(SUPPRESSION_POLL_INTERVAL, async () => {
-            await this.updateEntriesNow();
-        });
-    });
-    private cacheEntries: SuppressionListBase | undefined = undefined;
-    public updateEntriesNow = async () => {
-        let entries = await getSuppressionListRaw();
-        if (!entries) {
-            entries = { entries: {} };
-        }
-        this.cacheEntries = entries;
-    };
-    private async getEntries(): Promise<SuppressionListBase> {
-        await this.init();
-        if (!this.cacheEntries) {
-            throw new Error("Cache entries not set? Should be impossible.");
-        }
-        // Infinite poll will have set this, so we don't infinitely loop
-        return this.cacheEntries;
-    }
-
-    public async filterObjsToNonSuppressed(objs: LogDatum[]): Promise<LogDatum[]> {
-        // NOTE: Streamed data should be rare enough, that handling this inefficiently is okay.
-        if (objs.length === 0) return [];
-        let startTime = objs[0].time;
-        let endTime = objs[objs.length - 1].time;
-        let parts: Buffer[] = [];
-        for (let obj of objs) {
-            parts.push(Buffer.from(JSON.stringify(obj)));
-            parts.push(objectDelimitterBuffer);
-            if (obj.time < startTime) {
-                startTime = obj.time;
-            }
-            if (obj.time > endTime) {
-                endTime = obj.time;
-            }
-        }
-        let buffer = Buffer.concat(parts);
-        let scanner = await this.scanForRecentErrors({
-            debugName: "filterObjsToNonSuppressed",
-            startTime,
-            endTime,
-        });
-        await scanner.onData(buffer);
-        return await scanner.finish();
-    }
-    public async scanForRecentErrors(config: {
-        debugName: string;
-        startTime: number;
-        endTime: number;
-    }): Promise<{
-        onData: (data: Buffer) => void;
-        finish: () => Promise<LogDatum[]>;
-    }> {
-        let entries = await this.getEntries();
-        let suppressionFull = getSuppressionFull({
-            entries: Object.values(entries.entries),
-            blockTimeRange: {
-                startTime: config.startTime,
-                endTime: config.endTime,
-            },
-        });
-        let datums: LogDatum[] = [];
-        // Create an object which we'll reuse that will be the output object for the suppression key. 
-        // for the suppression key. 
-        let obj: { outdatedSuppressionKey?: string } = {};
-        let callback = createLogScanner({
-            debugName: config.debugName,
-            onParsedData: (posStart, posEnd, buffer) => {
-                if (buffer === "done") {
-                    return;
-                }
-                let result = suppressionFull(posStart, posEnd, buffer, obj);
-
-                if (!result) return;
-
-                let datum: LogDatum;
-                try {
-                    datum = JSON.parse(buffer.slice(posStart, posEnd).toString()) as LogDatum;
-                } catch (e: any) {
-                    process.stderr.write(`Failed to parse log datum in around ${buffer.slice(posStart, posEnd).slice(0, 100).toString("hex")}, in source ${config.debugName}, error is:\n${e.stack}`);
-                    return;
-                }
-                if (obj.outdatedSuppressionKey) {
-                    datum.__matchedOutdatedSuppressionKey = obj.outdatedSuppressionKey;
-                }
-                datums.push(datum);
-            },
-        });
-        let lastWaitTime = Date.now();
-        const stream = runInSerial(async (buffer: Buffer | "done") => {
-            // TODO: Maybe we should add this pattern to batching.ts? Basically, if we get called fast, we allow the calls through. BUT, if we called slowly OR we are doing a lot of processing (and so we are working for all of SELF_THROTTLE_INTERVAL), then we wait. This prevents this from taking over the machine. The back off is steep though, and if the machine is lagging we might reduce to a trickle, just getting 1 call in per SELF_THROTTLE_DELAY + synchronous lag from work in other parts of the program.
-            let now = Date.now();
-            if (now - lastWaitTime > SELF_THROTTLE_INTERVAL) {
-                await delay(SELF_THROTTLE_DELAY);
-                lastWaitTime = now;
-            }
-            await callback(buffer);
-        });
-        return {
-            onData: stream,
-            finish: async () => {
-                await stream("done");
-                // NOTE: We COULD limit as we run, however... how many errors are we really going to encounter that AREN'T suppressed? Suppression is supposed to prevent overload anyways. I guess worst case scenario, yes, we could get overloaded, but... if we are logging more NEW errors than we can store in memory, we have bigger problems...
-                return limitRecentErrors(datums);
-            },
-        };
-    }
-    public async setSuppressionEntry(entry: SuppressionEntry) {
-        let entries = await this.getEntries();
-        entry.lastUpdateTime = Date.now();
-        entries.entries[entry.key] = entry;
-        await suppressionListArchive.set(suppressionListKey, entries);
-        suppressionUpdatedChannel.broadcast(true);
-        await recentErrors.onSuppressionChanged();
-    }
-    public async removeSuppressionEntry(key: string) {
-        let entries = await this.getEntries();
-        delete entries.entries[key];
-        await suppressionListArchive.set(suppressionListKey, entries);
-        suppressionUpdatedChannel.broadcast(true);
-        await recentErrors.onSuppressionChanged();
-    }
-
-    public async getSuppressionList(): Promise<SuppressionEntry[]> {
-        let entries = Object.values((await this.getEntries()).entries);
-        return entries;
-    }
-}
-export const suppressionList = new SuppressionList();
-export const SuppressionListController = getSyncedController(SocketFunction.register(
-    "SuppressionListController-08f985d8-8d06-4041-ac4b-44566c54615d",
-    suppressionList,
-    () => ({
-        setSuppressionEntry: {},
-        removeSuppressionEntry: {},
-        getSuppressionList: {},
-    }),
-    () => ({
-        hooks: [assertIsManagementUser],
-    }),
-    {
-        noFunctionMeasure: true,
-    }
-), {
-    reads: {
-        getSuppressionList: ["suppression-list"],
-    },
-    writes: {
-        setSuppressionEntry: ["suppression-list", "recent-errors"],
-        removeSuppressionEntry: ["suppression-list", "recent-errors"],
-    },
-});
-
-class URLCache {
-    private root = isNode() && os.homedir() + "/backblaze-cache/" || "";
-    private sizeLimiter = new SizeLimiter({
-        diskRoot: this.root,
-        maxBytes: 1024 * 1024 * 1024 * 10,
-        // Basically... enough cache for a week of hourly data, for 1000 servers. At which point, this whole system will suck (too many servers to check), and we might have to change it
-        //  - Ex, by only using backblaze, and aggregating logs there, to reduce file counts
-        //  - Or... just by using elasticsearch...
-        //  - Or... just have dedicated search servers, which store the data in memory. We will have few people querying, so having 100 servers all execute a 60s search query is fine.
-        maxFiles: 24 * 7 * 1000,
-        maxDiskFraction: 0.1,
-        maxTotalDiskFraction: 0.96,
-        minBytes: 0,
-    });
-    // Returns the PATH to it on disk, so you can then parse it via a stream, so you never have to store it all at once in memory.
-    public async getURLLocalPath(url: string, hash: string): Promise<string | undefined> {
-        if (!isNode()) return undefined;
-
-        // Create cache directory if it doesn't exist
-        if (!await fsExistsAsync(this.root)) {
-            await fs.promises.mkdir(this.root, { recursive: true });
-        }
-
-        const filePath = this.root + hash;
-
-        try {
-            // Check if file already exists
-            const stats = await fs.promises.stat(filePath);
-            if (stats.isFile()) {
-                void this.applyLimitting([]);
-                return filePath;
-            }
-        } catch (e: any) {
-            // File doesn't exist, need to download it
-            if (e.code !== "ENOENT") {
-                throw e;
-            }
-        }
-
-        try {
-            // Download and stream to disk
-            const response = await fetch(url);
-            if (!response.ok) {
-                throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
-            }
-
-            if (!response.body) {
-                throw new Error(`Response body is undefined for ${url}`);
-            }
-
-            // Create write stream
-            let tempPath = filePath + ".temp";
-            const writeStream = fs.createWriteStream(tempPath);
-            const reader = response.body.getReader();
-
-            try {
-                for await (const chunk of streamToIteratable(reader)) {
-                    if (!chunk) continue;
-                    let result = writeStream.write(Buffer.from(chunk));
-                    if (!result) {
-                        await new Promise<void>(resolve => writeStream.once("drain", resolve));
-                    }
-                }
-
-                await new Promise<void>((resolve, reject) => {
-                    writeStream.end((err: any) => {
-                        if (err) reject(err);
-                        else resolve();
-                    });
-                });
-
-                // Trigger cache cleanup in the background
-                void this.applyLimitting([]);
-
-                await fs.promises.rename(tempPath, filePath);
-                return filePath;
-            } catch (error) {
-                // Clean up partial file on error
-                try {
-                    await fs.promises.unlink(tempPath);
-                } catch {
-                    // Ignore cleanup errors
-                }
-                throw error;
-            } finally {
-                reader.releaseLock();
-            }
-        } catch (error) {
-            console.error(`Failed to download and cache file from ${url}:`, error);
-            return undefined;
-        }
-    }
-    private applyLimitting = batchFunction({ delay: LOCAL_CACHE_LIMIT_BATCH }, async () => {
-        const files = await fs.promises.readdir(this.root);
-        const fileInfos: { time: number; bytes: number; path: string; }[] = [];
-
-        for (const file of files) {
-            const filePath = this.root + file;
-            const stats = await fs.promises.stat(filePath);
-            if (stats.isFile()) {
-                fileInfos.push({
-                    time: stats.atimeMs,
-                    bytes: stats.size,
-                    path: filePath,
-                });
-            }
-        }
-
-        const result = await this.sizeLimiter.limit(fileInfos);
-
-        for (const fileToRemove of result.remove) {
-            await fs.promises.unlink(fileToRemove.path);
-        }
-    });
-}
-export const urlCache = new URLCache();
-
-const limitRecentErrors = measureWrap(function limitRecentErrors(objs: LogDatum[]) {
-    sort(objs, x => x.time);
-    let recent: LogDatum[] = [];
-    let foundHashes = new Set<string>();
-    let countByFile = new Map<string, number>();
-    // NOTE: We iterate backwards, because... usually new logs come in at the end, and are pushed, so we want to sort by time (that way we often don't have to resort by much). And if we sort by time, the newest at at the end!
-    for (let i = objs.length - 1; i >= 0; i--) {
-        let obj = objs[i];
-        let file = String(obj.__FILE__) || "";
-        let count = countByFile.get(file) || 0;
-        if (count > MAX_RECENT_ERRORS_PER_FILE) continue;
-        count++;
-        let hash = getLogHash(obj);
-        if (foundHashes.has(hash)) continue;
-        foundHashes.add(hash);
-        if (count > MAX_RECENT_ERRORS_PER_FILE) continue;
-        countByFile.set(file, count);
-        recent.push(obj);
-        if (recent.length >= MAX_RECENT_ERRORS) break;
-    }
-    return recent;
-});
-
-export class RecentErrors {
-
-    constructor(private addErrorsCallback?: (objs: LogDatum[]) => void | Promise<void>) {
-        this.addErrorsCallback = addErrorsCallback;
-    }
-
-    // TODO: Uninitialize (stopping the infinite polling), if all of our recent errors watchers go away.
-    private initialize = lazy(async () => {
-        errorWatcherBase.watch(x => {
-            void this.addErrors(x);
-        });
-        await this.scanNow({});
-        runInfinitePoll(BACKBLAZE_POLL_INTERVAL, async () => {
-            await this.scanNow({ noLocalFiles: true });
-        });
-    });
-
-    private _recentErrors: LogDatum[] = [];
-    private _lastSentErrors: LogDatum[] = [];
-    private updateRecentErrors = runInSerial(async () => {
-        let newList = this._recentErrors;
-        newList = await suppressionList.filterObjsToNonSuppressed(newList);
-        newList = limitRecentErrors(newList);
-        this._recentErrors = newList;
-        // If any changed
-        let prev = new Set(this._lastSentErrors);
-        let newErrors = new Set(newList);
-        function hasAnyChanged() {
-            for (let obj of newList) {
-                if (!prev.has(obj)) {
-                    return true;
-                }
-            }
-            for (let obj of prev) {
-                if (!newErrors.has(obj)) {
-                    return true;
-                }
-            }
-            return false;
-        }
-        this._lastSentErrors = newList;
-        if (hasAnyChanged()) {
-            void this.broadcastUpdate(undefined);
-        }
-    });
-    private broadcastUpdate = batchFunction({ delay: NOTIFICATION_BROADCAST_BATCH, noMeasure: true }, () => {
-        recentErrorsChannel.broadcast(true);
-    });
-
-    private addErrors = runInSerial(async (objs: LogDatum[]) => {
-        if (objs.length === 0) return;
-
-        if (this.addErrorsCallback) {
-            await this.addErrorsCallback(objs);
-            return;
-        }
-        for (let obj of objs) {
-            this._recentErrors.push(obj);
-        }
-        await this.updateRecentErrors();
-    });
-
-    private lastSuppressionList = new Map<string, SuppressionEntry>();
-    public onSuppressionChanged = runInSerial(async () => {
-        let newSuppressionList = new Map((await suppressionList.getSuppressionList()).map(x => [x.key, x]));
-        let prev = this.lastSuppressionList;
-        function anyReduced() {
-            for (let newEntry of newSuppressionList.values()) {
-                let oldEntry = prev.get(newEntry.key);
-                if (oldEntry && newEntry.expiresAt < oldEntry.expiresAt) {
-                    return true;
-                }
-            }
-            for (let oldEntry of prev.values()) {
-                if (!newSuppressionList.has(oldEntry.key)) {
-                    return true;
-                }
-            }
-            return false;
-        }
-        if (anyReduced()) {
-            console.info("Suppression has been reduced (entries removed or expiry times decreased), performing full rescan to find any revealed values.");
-            this.scannedHashes.clear();
-            void this.scanNow({});
-        }
-        this.lastSuppressionList = newSuppressionList;
-        await this.updateRecentErrors();
-    });
-
-    private scannedHashes = new Set<string>();
-    private scanNow = runInSerial(async (config: {
-        noLocalFiles?: boolean;
-    }) => {
-        // If we're scanning everything, we should update the suppression list, because it might have been changed remotely, and we might be scanning everything because the user clicked refresh. 
-        if (!this.lastSuppressionList || !config.noLocalFiles) {
-            this.lastSuppressionList = new Map((await suppressionList.getSuppressionList()).map(x => [x.key, x]));
-        }
-        for (let appendable of getErrorAppendables()) {
-            let startTime = Date.now() - VIEW_WINDOW;
-            let endTime = Date.now() + timeInHour * 2;
-            let result = await new FastArchiveAppendableControllerBase().startSynchronizeInternal({
-                range: {
-                    startTime,
-                    endTime,
-                },
-                rootPath: appendable.rootPath,
-                noLocalFiles: config.noLocalFiles,
-                forceGetPublic: true,
-            });
-            // Filter again, as new suppressions change the errors
-            await this.updateRecentErrors();
-            let recentLimit = 0;
-            const applyRecentLimit = () => {
-                if (this._recentErrors.length < MAX_RECENT_ERRORS) return;
-                recentLimit = this._recentErrors[0].time - timeInHour * 2;
-            };
-            applyRecentLimit();
-            for (let file of result.files) {
-                let path: string | undefined = undefined;
-                let size: number | undefined = undefined;
-                try {
-                    // Skip if it is older than even our throttled values!
-                    if (file.startTime < recentLimit) continue;
-                    // Don't skip local files, we only do them once at the start, or on demand when we want to recalc all anyways!
-                    if (!file.nodeId) {
-                        let hash = getFileMetadataHash(file);
-                        if (this.scannedHashes.has(hash)) continue;
-                        this.scannedHashes.add(hash);
-                    }
-
-                    let hash = getFileMetadataHash(file);
-                    path = await urlCache.getURLLocalPath(file.url, hash);
-                    if (!path) continue;
-                    let scanner = await suppressionList.scanForRecentErrors({
-                        debugName: file.url,
-                        startTime: file.startTime,
-                        endTime: file.endTime,
-                    });
-
-                    // Stream decompress the file while reading it
-                    size = await fs.promises.stat(path).then(x => x.size);
-                    if (!size) {
-                        console.error(`Deleting empty cached file ${path} for ${file.url} `);
-                        // NOTE: This means we will repeatedly download empty files, but... that should be fairly fast...
-                        await fs.promises.unlink(path);
-                        continue;
-                    }
-                    const fileStream = fs.createReadStream(path);
-                    const gunzip = zlib.createGunzip();
-                    const decompressedStream = fileStream.pipe(gunzip);
-                    for await (const chunk of decompressedStream) {
-                        scanner.onData(chunk);
-                    }
-                    let newErrors = await scanner.finish();
-                    await this.addErrors(newErrors);
-                    applyRecentLimit();
-                } catch (e: any) {
-                    console.error(`Failed to scan file ${file.url}, size is ${size}, error is:\n${e.stack}`);
-                }
-            }
-        }
-    });
-
-
-    public async getRecentErrors(): Promise<LogDatum[]> {
-        await this.initialize();
-        return this._recentErrors;
-    }
-
-    // Rescans all local, and new backblaze
-    public async rescanAllErrorsNow() {
-        await this.scanNow({});
-        return this._recentErrors;
-    }
-
-    public async raiseTestError(...params: unknown[]) {
-        console.error(...params);
-    }
-}
-const recentErrors = new RecentErrors();
-export const RecentErrorsController = getSyncedController(SocketFunction.register(
-    "RecentErrorsController-8450c626-2a06-4eee-81cb-67d4c2fa8155",
-    recentErrors,
-    () => ({
-        getRecentErrors: {},
-        rescanAllErrorsNow: {},
-        raiseTestError: {},
-    }),
-    () => ({
-        hooks: [assertIsManagementUser],
-    }),
-), {
-    reads: {
-        getRecentErrors: ["recent-errors"],
-    },
-    writes: {
-        rescanAllErrorsNow: ["recent-errors"],
-    },
-});
-
-export const recentErrorsChannel = new SocketChannel<true>("recent-errors-eeceb0c8-4086-4ab3-b3ff-fa9fd5282e14");
-
-export const watchRecentErrors = lazy(function watchRecentErrors() {
-    recentErrorsChannel.watch(async () => {
-        // Only 1 function, so just refresh all...
-        RecentErrorsController.refreshAll();
-    });
-});
-
-
-
-
-export const notifyWatchersOfError = batchFunction({
-    delay: LOCAL_BROADCAST_BATCH,
-},
-    async (objs: LogDatum[]) => {
-        objs = await suppressionList.filterObjsToNonSuppressed(objs);
-        if (objs.length > 0) {
-            errorWatcherBase.broadcast(objs);
-        }
-    }
-);
-
-export const errorWatcherBase = new SocketChannel<LogDatum[]>("error-watcher-38de08cd-3247-4f75-9ac0-7919b240607d");