querysub 0.327.0 → 0.329.0

package/src/diagnostics/logs/errorNotifications/ErrorNotificationController.ts

@@ -2,7 +2,7 @@ import { isNode } from "typesafecss";
 import { getArchives } from "../../../-a-archives/archives";
 import { SizeLimiter } from "../../SizeLimiter";
 import { FastArchiveAppendable, createLogScanner, objectDelimitterBuffer } from "../FastArchiveAppendable";
-import { LogDatum, getLoggers } from "../diskLogger";
+import { LogDatum, getLogHash, getLoggers } from "../diskLogger";
 import os from "os";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { cache, cacheLimited, lazy } from "socket-function/src/caching";
@@ -21,6 +21,8 @@ import { qreact } from "../../../4-dom/qreact";
 import { requiresNetworkTrustHook } from "../../../-d-trust/NetworkTrust2";
 import { assertIsManagementUser } from "../../managementPages";
 import { streamToIteratable } from "../../../misc";
+import { fsExistsAsync } from "../../../fs";
+import { getPathStr2 } from "../../../path";
 
 export const MAX_RECENT_ERRORS = 20;
 const MAX_RECENT_ERRORS_PER_FILE = 3;
@@ -60,7 +62,7 @@ type SuppressedChecker = {
     fnc: (buffer: Buffer, posStart: number, posEnd: number) => boolean;
 }
 
-function getAppendables() {
+export function getErrorAppendables() {
     let loggers = getLoggers();
     if (!loggers) throw new Error("Loggers not available?");
     // error, warn
@@ -151,18 +153,17 @@ export const getSuppressionFull = measureWrap(function getSuppressionFull(config
 
         // Handle definitelyExpired - these are outdated suppressions
         let mostRecentOutdatedSuppressionKey: string | undefined = undefined;
+        let mostRecentOutdatedSuppressionTime = 0;
 
         // Handle maybeExpired - need to parse timestamp to check if suppression was active
         if (maybeExpired.length > 0 && (suppressionCounts || expiredSuppressionCounts || obj)) {
-            const getLogTime = () => {
-                try {
-                    let logEntry = JSON.parse(data.slice(posStart, posEnd).toString()) as LogDatum;
-                    return typeof logEntry.time === "number" ? logEntry.time : 0;
-                } catch {
-                    return 0;
+            let logTime = 0;
+            try {
+                let logEntry = JSON.parse(data.slice(posStart, posEnd).toString()) as LogDatum;
+                if (typeof logEntry.time === "number") {
+                    logTime = logEntry.time;
                 }
-            };
-            let logTime = getLogTime();
+            } catch { }
 
             for (let checker of maybeExpired) {
                 if (checker.fnc(data, posStart, posEnd)) {
@@ -174,8 +175,10 @@ export const getSuppressionFull = measureWrap(function getSuppressionFull(config
                             suppressionCounts.set(checker.entry.key, count);
                         }
                     } else {
-                        if (!mostRecentOutdatedSuppressionKey) {
+
+                        if (checker.entry.expiresAt > mostRecentOutdatedSuppressionTime) {
                             mostRecentOutdatedSuppressionKey = checker.entry.key;
+                            mostRecentOutdatedSuppressionTime = checker.entry.expiresAt;
                         }
                         // Even if we don't want the expired suppression counts, we might want the normal suppression counts, so we have to keep going. 
                         if (expiredSuppressionCounts) {
@@ -192,7 +195,7 @@ export const getSuppressionFull = measureWrap(function getSuppressionFull(config
             for (let checker of definitelyExpired) {
                 if (checker.fnc(data, posStart, posEnd)) {
                     // First match is the most recent (entries are sorted by lastUpdateTime desc)
-                    if (!mostRecentOutdatedSuppressionKey) {
+                    if (checker.entry.expiresAt > mostRecentOutdatedSuppressionTime) {
                         mostRecentOutdatedSuppressionKey = checker.entry.key;
                     }
                     if (!expiredSuppressionCounts) break;
@@ -204,7 +207,7 @@ export const getSuppressionFull = measureWrap(function getSuppressionFull(config
         }
 
         // Set the most recent outdated suppression key if we found any and weren't suppressed
-        if (obj && mostRecentOutdatedSuppressionKey && !suppressed) {
+        if (obj && mostRecentOutdatedSuppressionKey) {
             obj.outdatedSuppressionKey = mostRecentOutdatedSuppressionKey;
         }
 
@@ -219,18 +222,27 @@ const suppressionListArchive = archiveJSONT<SuppressionListBase>(() =>
 );
 const suppressionUpdatedChannel = new SocketChannel<boolean>("suppression-updated");
 
+export async function getSuppressionListRaw(): Promise<SuppressionListBase> {
+    let entries = await suppressionListArchive.get(suppressionListKey);
+    if (!entries) {
+        entries = { entries: {} };
+    }
+    return entries;
+}
+
 class SuppressionList {
     private init = lazy(async () => {
-        suppressionUpdatedChannel.watch(() => {
-            void this.updateEntriesNow();
+        suppressionUpdatedChannel.watch(async () => {
+            await this.updateEntriesNow();
+            await recentErrors.onSuppressionChanged();
         });
         await runInfinitePollCallAtStart(SUPPRESSION_POLL_INTERVAL, async () => {
             await this.updateEntriesNow();
         });
     });
     private cacheEntries: SuppressionListBase | undefined = undefined;
-    private updateEntriesNow = async () => {
-        let entries = await suppressionListArchive.get(suppressionListKey);
+    public updateEntriesNow = async () => {
+        let entries = await getSuppressionListRaw();
         if (!entries) {
             entries = { entries: {} };
         }
@@ -336,14 +348,14 @@ class SuppressionList {
         let entries = await this.getEntries();
         entry.lastUpdateTime = Date.now();
         entries.entries[entry.key] = entry;
-        void suppressionListArchive.set(suppressionListKey, entries);
+        await suppressionListArchive.set(suppressionListKey, entries);
         suppressionUpdatedChannel.broadcast(true);
         await recentErrors.onSuppressionChanged();
     }
     public async removeSuppressionEntry(key: string) {
         let entries = await this.getEntries();
         delete entries.entries[key];
-        void suppressionListArchive.set(suppressionListKey, entries);
+        await suppressionListArchive.set(suppressionListKey, entries);
         suppressionUpdatedChannel.broadcast(true);
         await recentErrors.onSuppressionChanged();
     }
@@ -353,7 +365,7 @@ class SuppressionList {
         return entries;
     }
 }
-const suppressionList = new SuppressionList();
+export const suppressionList = new SuppressionList();
 export const SuppressionListController = getSyncedController(SocketFunction.register(
     "SuppressionListController-08f985d8-8d06-4041-ac4b-44566c54615d",
     suppressionList,
@@ -397,7 +409,7 @@ class URLCache {
         if (!isNode()) return undefined;
 
         // Create cache directory if it doesn't exist
-        if (!fs.existsSync(this.root)) {
+        if (!await fsExistsAsync(this.root)) {
             await fs.promises.mkdir(this.root, { recursive: true });
         }
 
@@ -498,13 +510,18 @@ const urlCache = new URLCache();
 const limitRecentErrors = measureWrap(function limitRecentErrors(objs: LogDatum[]) {
     sort(objs, x => x.time);
     let recent: LogDatum[] = [];
+    let foundHashes = new Set<string>();
     let countByFile = new Map<string, number>();
     // NOTE: We iterate backwards, because... usually new logs come in at the end, and are pushed, so we want to sort by time (that way we often don't have to resort by much). And if we sort by time, the newest at at the end!
     for (let i = objs.length - 1; i >= 0; i--) {
         let obj = objs[i];
         let file = String(obj.__FILE__) || "";
         let count = countByFile.get(file) || 0;
+        if (count > MAX_RECENT_ERRORS_PER_FILE) continue;
         count++;
+        let hash = getLogHash(obj);
+        if (foundHashes.has(hash)) continue;
+        foundHashes.add(hash);
         if (count > MAX_RECENT_ERRORS_PER_FILE) continue;
         countByFile.set(file, count);
         recent.push(obj);
@@ -513,8 +530,13 @@ const limitRecentErrors = measureWrap(function limitRecentErrors(objs: LogDatum[
     return recent;
 });
 
-class RecentErrors {
+export class RecentErrors {
+
+    constructor(private addErrorsCallback?: (objs: LogDatum[]) => void | Promise<void>) {
+        this.addErrorsCallback = addErrorsCallback;
+    }
 
+    // TODO: Uninitialize (stopping the infinite polling), if all of our recent errors watchers go away.
     private initialize = lazy(async () => {
         errorWatcherBase.watch(x => {
             void this.addErrors(x);
@@ -526,7 +548,7 @@ class RecentErrors {
     });
 
     private _recentErrors: LogDatum[] = [];
-    private updateRecentErrors = async (objs: LogDatum[]) => {
+    private updateRecentErrors = runInSerial(async (objs: LogDatum[]) => {
         objs = await suppressionList.filterObjsToNonSuppressed(objs);
         let newRecentErrors = limitRecentErrors(objs);
         // If any changed
@@ -549,28 +571,60 @@ class RecentErrors {
             this._recentErrors = newRecentErrors;
             void this.broadcastUpdate(undefined);
         }
-    };
+    });
     private broadcastUpdate = batchFunction({ delay: NOTIFICATION_BROADCAST_BATCH }, () => {
         recentErrorsChannel.broadcast(true);
     });
 
-    private async addErrors(objs: LogDatum[]) {
+    private addErrors = runInSerial(async (objs: LogDatum[]) => {
         if (objs.length === 0) return;
+
+        if (this.addErrorsCallback) {
+            await this.addErrorsCallback(objs);
+            return;
+        }
         for (let obj of objs) {
             this._recentErrors.push(obj);
         }
         await this.updateRecentErrors(this._recentErrors);
-    }
+    });
 
-    public async onSuppressionChanged() {
+    private lastSuppressionList = new Map<string, SuppressionEntry>();
+    public onSuppressionChanged = runInSerial(async () => {
+        let newSuppressionList = new Map((await suppressionList.getSuppressionList()).map(x => [x.key, x]));
+        let prev = this.lastSuppressionList;
+        function anyReduced() {
+            for (let newEntry of newSuppressionList.values()) {
+                let oldEntry = prev.get(newEntry.key);
+                if (oldEntry && newEntry.expiresAt < oldEntry.expiresAt) {
+                    return true;
+                }
+            }
+            for (let oldEntry of prev.values()) {
+                if (!newSuppressionList.has(oldEntry.key)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+        if (anyReduced()) {
+            console.info("Suppression has been reduced (entries removed or expiry times decreased), performing full rescan to find any revealed values.");
+            this.scannedHashes.clear();
+            void this.scanNow({});
+        }
+        this.lastSuppressionList = newSuppressionList;
         await this.updateRecentErrors(this._recentErrors);
-    }
+    });
 
     private scannedHashes = new Set<string>();
     private scanNow = runInSerial(async (config: {
         noLocalFiles?: boolean;
     }) => {
-        for (let appendable of getAppendables()) {
+        // If we're scanning everything, we should update the suppression list, because it might have been changed remotely, and we might be scanning everything because the user clicked refresh. 
+        if (!this.lastSuppressionList || !config.noLocalFiles) {
+            this.lastSuppressionList = new Map((await suppressionList.getSuppressionList()).map(x => [x.key, x]));
+        }
+        for (let appendable of getErrorAppendables()) {
             let startTime = Date.now() - VIEW_WINDOW;
             let endTime = Date.now() + timeInHour * 2;
             let result = await new FastArchiveAppendableControllerBase().startSynchronizeInternal({
@@ -619,43 +673,11 @@ class RecentErrors {
                         await fs.promises.unlink(path);
                         continue;
                     }
-                    let sizeT = size;
-                    let fd = await fs.promises.open(path, "r");
-                    try {
-                        await new Promise<void>(async (resolve, reject) => {
-                            const gunzip = zlib.createGunzip();
-
-                            gunzip.on("data", (chunk: Buffer) => {
-                                void scanner.onData(chunk);
-                            });
-
-                            gunzip.on("end", async () => {
-                                try {
-                                    resolve();
-                                } catch (error) {
-                                    reject(error);
-                                }
-                            });
-
-                            gunzip.on("error", reject);
-
-                            try {
-                                for (let i = 0; i < sizeT; i += READ_CHUNK_SIZE) {
-                                    let chunkSize = Math.min(READ_CHUNK_SIZE, sizeT - i);
-                                    let buffer = Buffer.alloc(chunkSize);
-                                    await fd.read(buffer, 0, chunkSize, i);
-                                    let result = gunzip.write(buffer);
-                                    if (!result) {
-                                        await new Promise(resolve => gunzip.once("drain", resolve));
-                                    }
-                                }
-                                gunzip.end();
-                            } catch (error) {
-                                reject(error);
-                            }
-                        });
-                    } finally {
-                        await fd.close();
+                    const fileStream = fs.createReadStream(path);
+                    const gunzip = zlib.createGunzip();
+                    const decompressedStream = fileStream.pipe(gunzip);
+                    for await (const chunk of decompressedStream) {
+                        scanner.onData(chunk);
                     }
                     let newErrors = await scanner.finish();
                     await this.addErrors(newErrors);
@@ -678,6 +700,10 @@ class RecentErrors {
         await this.scanNow({});
         return this._recentErrors;
     }
+
+    public async raiseTestError(...params: unknown[]) {
+        console.error(...params);
+    }
 }
 const recentErrors = new RecentErrors();
 export const RecentErrorsController = getSyncedController(SocketFunction.register(
@@ -686,6 +712,7 @@ export const RecentErrorsController = getSyncedController(SocketFunction.registe
     () => ({
         getRecentErrors: {},
         rescanAllErrorsNow: {},
+        raiseTestError: {},
     }),
     () => ({
         hooks: [assertIsManagementUser],
@@ -722,4 +749,4 @@ export const notifyWatchersOfError = batchFunction({
     }
 );
 
-const errorWatcherBase = new SocketChannel<LogDatum[]>("error-watcher-38de08cd-3247-4f75-9ac0-7919b240607d");
+export const errorWatcherBase = new SocketChannel<LogDatum[]>("error-watcher-38de08cd-3247-4f75-9ac0-7919b240607d");

package/src/diagnostics/logs/errorNotifications/ErrorSuppressionUI.tsx

@@ -10,6 +10,8 @@ import { nextId, sort, timeInDay } from "socket-function/src/misc";
 import { formatNumber, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
 import { formatDateJSX } from "../../../misc/formatJSX";
 import { LogDatum } from "../diskLogger";
+import { measureFnc } from "socket-function/src/profiling/measure";
+import { throttleRender } from "../../../functional/throttleRender";
 
 export class ErrorSuppressionUI extends qreact.Component<{
     dataSeqNum: number;
@@ -20,8 +22,10 @@ export class ErrorSuppressionUI extends qreact.Component<{
 }> {
     state = t.state({
         matchedInput: t.string(""),
+        renderLimit: t.number(10)
     });
 
+    @measureFnc
     private calculatePreviewMatchCount(pattern: string): number {
         if (!pattern.trim()) return 0;
 
@@ -50,6 +54,8 @@ export class ErrorSuppressionUI extends qreact.Component<{
     }
 
     public render() {
+        if (throttleRender({ key: "ErrorSuppressionUI", frameDelay: 30 })) return undefined;
+
         this.props.dataSeqNum;
         const controller = SuppressionListController(SocketFunction.browserNodeId());
         const entries = (controller.getSuppressionList() || []);
@@ -117,6 +123,27 @@ export class ErrorSuppressionUI extends qreact.Component<{
                 >
                     Fixed
                 </Button>
+                <Button
+                    onClick={() => {
+                        let value = this.state.matchedInput;
+                        this.state.matchedInput = "";
+                        void Querysub.onCommitFinished(async () => {
+                            await controller.setSuppressionEntry.promise({
+                                key: nextId(),
+                                match: value,
+                                comment: "",
+                                lastUpdateTime: Date.now(),
+                                expiresAt: Date.now(),
+                            });
+                            Querysub.commit(() => {
+                                this.props.rerunFilters();
+                            });
+                        });
+                    }}
+                    title="Fixed immediately, any future errors even that happen right now will trigger again. "
+                >
+                    Fixed Now
+                </Button>
                 <Button onClick={() => {
                     let value = this.state.matchedInput;
                     this.state.matchedInput = "";
@@ -137,8 +164,12 @@ export class ErrorSuppressionUI extends qreact.Component<{
                 </Button>
             </div>
 
-            <div className={css.vbox(8).fillWidth.overflowAuto.maxHeight("30vh")}>
-                {entries.map((entry) => {
+            <div className={css.pad2(12).bord2(200, 40, 85).hsl(200, 40, 95).fillWidth}>
+                <strong>Note:</strong> Suppression time updates don't automatically rerun the search. Click Run to rerun the search.
+            </div>
+
+            <div className={css.vbox(8).fillWidth.overflowAuto.maxHeight("20vh")}>
+                {entries.slice(0, this.state.renderLimit).map((entry) => {
                     const updateEntry = (changes: Partial<SuppressionEntry>) => {
                         let newEntry = { ...entry, ...changes };
                         void Querysub.onCommitFinished(async () => {
@@ -152,7 +183,7 @@ export class ErrorSuppressionUI extends qreact.Component<{
                         className={
                             css.hbox(8).pad2(12).bord2(0, 0, 10).fillWidth
                             //+ (entry.expiresAt < Date.now() && expiredCount > 0 && css.opacity(0.5))
-                            + ((count === 0 && expiredCount === 0) && css.opacity(0.6))
+                            + ((expiredCount === 0) && css.opacity(0.6))
                             + (
                                 count > 0 && entry.expiresAt !== NOT_AN_ERROR_EXPIRE_TIME && css.hsla(0, 50, 50, 0.5)
                                 || css.hsla(0, 0, 0, 0.1)
@@ -226,6 +257,9 @@ export class ErrorSuppressionUI extends qreact.Component<{
                         </Button>
                     </div>;
                 })}
+                {entries.length > this.state.renderLimit && <Button onClick={() => this.state.renderLimit *= 2}>
+                    Load More
+                </Button>}
             </div>
         </div>;
     }

package/src/diagnostics/logs/errorNotifications/ErrorWarning.tsx

@@ -1,12 +1,12 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { qreact } from "../../../4-dom/qreact";
 import { css } from "../../../4-dom/css";
-import { isCurrentUserSuperUser } from "../../../user-implementation/userData";
+import { isCurrentUserSuperUser, user_data } from "../../../user-implementation/userData";
 import { RecentErrorsController, SuppressionListController, watchRecentErrors, MAX_RECENT_ERRORS, NOT_AN_ERROR_EXPIRE_TIME, SuppressionEntry } from "./ErrorNotificationController";
 import { t } from "../../../2-proxy/schema2";
 import { InputLabel } from "../../../library-components/InputLabel";
 import { Button } from "../../../library-components/Button";
-import { ATag } from "../../../library-components/ATag";
+import { ATag, Anchor, URLOverride } from "../../../library-components/ATag";
 import { managementPageURL, showingManagementURL } from "../../managementPages";
 import { errorNotifyToggleURL } from "../LogViewer2";
 import { Querysub } from "../../../4-querysub/QuerysubController";
@@ -16,6 +16,25 @@ import { Icon } from "../../../library-components/icons";
 import { filterParam } from "../FastArchiveViewer";
 import { endTimeParam, startTimeParam } from "../TimeRangeSelector";
 import { formatDateJSX } from "../../../misc/formatJSX";
+import { atomic } from "../../../2-proxy/PathValueProxyWatcher";
+
+export function getErrorLogsLink(config?: {
+    startTime?: number;
+    endTime?: number;
+}): URLOverride[] {
+    let startTime = config?.startTime ?? Date.now() - timeInDay * 1;
+    let endTime = config?.endTime ?? Date.now() + timeInHour * 2;
+    return [
+        showingManagementURL.getOverride(true),
+        managementPageURL.getOverride("LogViewer2"),
+        errorNotifyToggleURL.getOverride(true),
+        filterParam.getOverride(""),
+
+        // NOTE: While loading a weeks worth of logs clientside is a bit slow. Scanning serverside is not nearly as bad, as it can be done over hours, but... we want the page to be snappy, loading in seconds, so... just use a day, and we might reduce it even further if needed...
+        startTimeParam.getOverride(startTime),
+        endTimeParam.getOverride(endTime),
+    ];
+}
 
 export class ErrorWarning extends qreact.Component {
     state = t.state({
@@ -67,22 +86,36 @@ export class ErrorWarning extends qreact.Component {
                 </style>
             </Button>;
 
-        const logLink = [
-            showingManagementURL.getOverride(true),
-            managementPageURL.getOverride("LogViewer2"),
-            errorNotifyToggleURL.getOverride(true),
-            filterParam.getOverride(""),
-            startTimeParam.getOverride(Date.now() - timeInDay * 7),
-            endTimeParam.getOverride(Date.now() + timeInHour * 2),
-        ];
+        let discordURLWarning: qreact.ComponentChildren = undefined;
+        if (!atomic(user_data().secure.notifyDiscordWebhookURL)) {
+            discordURLWarning = (
+                <Anchor
+                    target="_blank"
+                    title="Can't send application notifications to developers due to missing Discord hook URL. Click here and set it."
+                    values={[
+                        showingManagementURL.getOverride(true),
+                        managementPageURL.getOverride("SecurityPage"),
+                    ]}
+                >
+                    <Button hue={0}>
+                        ⚠️ Missing Discord Hook URL <span className={css.filter("invert(1)")}>📞</span>
+                    </Button>
+                </Anchor>
+            );
+        }
+
+        const logLink = getErrorLogsLink();
 
         if (!recentErrors || recentErrors.length === 0) {
-            return <span className={css.hbox(8)}>
-                <ATag target="_blank" values={logLink}>
-                    No Errors
-                </ATag>
-                {refreshButton}
-            </span>;
+            return (
+                <span className={css.hbox(8)}>
+                    <ATag target="_blank" values={logLink}>
+                        No Errors
+                    </ATag>
+                    {refreshButton}
+                    {discordURLWarning}
+                </span>
+            );
         }
 
         // Count unique files
@@ -128,6 +161,7 @@ export class ErrorWarning extends qreact.Component {
                         View Logs
                     </ATag>
                     {refreshButton}
+                    {discordURLWarning}
                 </div>
 
                 {topExpired &&