querysub 0.377.0 → 0.378.0

package/bin/error-watch-public.js

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+process.argv.push("--nobreak");
+process.argv.push("--public");
+
+require("typenode");
+require("../src/diagnostics/logs/errorNotifications2/errorWatchEntry.ts");

package/bin/error-watch.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+
+process.argv.push("--local");
+
+require("typenode");
+require("../src/diagnostics/logs/errorNotifications2/errorWatchEntry.ts");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.377.0",
+  "version": "0.378.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -12,9 +12,10 @@
     "servershardedtest": "yarn server --authority ./pathremain.json & yarn server --authority ./patha.json & yarn server --authority ./pathb.json & yarn server --authority ./pathc.json & yarn server --authority ./pathd.json",
     "type": "yarn tsc --noEmit",
     "depend": "yarn --silent depcruise src --include-only \"^src\" --config --output-type dot | dot -T svg > dependency-graph.svg",
-    "t": "yarn typenode ./src/diagnostics/logs/lifeCycleAnalysis/testHoist.ts",
+    "t": "yarn typenode ./test.ts",
+    "t2": "yarn typenode ./test2.ts",
     "test-wat": "yarn typenode ./src/wat/watCompiler.ts",
-    "error-watch": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx",
+    "error-watch": "yarn typenode ./src/diagnostics/logs/errorNotifications2/errorWatchEntry.ts",
     "error-email": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx",
     "build-native": "cd src/diagnostics/logs/IndexedLogs && node-gyp rebuild"
   },
@@ -33,7 +34,9 @@
     "join": "./bin/join.js",
     "join-public": "./bin/join-public.js",
     "movelogs": "./bin/movelogs.js",
-    "addsuperuser": "./bin/addsuperuser.js"
+    "addsuperuser": "./bin/addsuperuser.js",
+    "error-watch": "./bin/error-watch.js",
+    "error-watch-public": "./bin/error-watch-public.js"
   },
   "dependencies": {
     "@types/fs-ext": "^2.0.3",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -113,6 +113,13 @@ export function isNodeIdOnOwnMachineId(nodeId: string): boolean {
     return certs.getMachineId(nodeId) === getOwnMachineId() || nodeId.startsWith("127-0-0-1.");
 }
 
+/** Often, when getting all nodes, you're going to receive duplicates of all local nodes with the duplicate containing this prefix. Using the local ID is faster as it will actually use the local IP.
+    TODO: We need a robust way to give a deduplicated set that prefers the local ids. At the moment we just filter out the id locals if we can't have duplicates.
+ */
+export function isNodeIdLocal(nodeId: string): boolean {
+    return nodeId.startsWith("127-0-0-1.");
+}
+
 let nodeOverrides: string[] | undefined;
 let beforeGetNodeAllId = async () => { };
 export async function getAllNodeIds() {

package/src/-g-core-values/NodeCapabilities.ts

@@ -16,15 +16,14 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { isNoNetwork } from "../config";
 import { getDebuggerUrl } from "../diagnostics/listenOnDebugger";
 import { hackDevtoolsWebsocketForward } from "./oneTimeForward";
-import { getOwnMachineId, decodeNodeId, decodeNodeIdAssert } from "../-a-auth/certs";
+import { getOwnMachineId, decodeNodeId, decodeNodeIdAssert, getMachineId } from "../-a-auth/certs";
 import { sort } from "socket-function/src/misc";
 import { getPathStr2 } from "../path";
+import { PromiseObj } from "../promise";
 setImmediate(() => {
     import("../diagnostics/MachineThreadInfo");
 });
 
-let loadTime = Date.now();
-
 let controllerNodeIdCache = new Map<string, string | Promise<string | undefined>>();
 
 // NOTE: If this becomes slow (because we are just trying all servers), we could start to store capabilities
@@ -39,32 +38,45 @@ export async function getControllerNodeId(
     if (cached && typeof cached !== "string") {
         cached = await cached;
     }
+    // TODO: We are relying on is node connected to do a lot of work here. If the node's connected, but somehow doesn't work, we can get in an unrecoverable state, even if there are plenty of other working nodes.
     if (cached && !SocketFunction.isNodeConnected(cached)) {
         controllerNodeIdCache.delete(controller._classGuid);
         cached = undefined;
     }
     if (cached) return cached;
-    let promise = getInternal();
+    let promise = getInternal(retryCount);
     controllerNodeIdCache.set(controller._classGuid, promise);
     return promise;
 
-    async function getInternal() {
-
+    async function getInternal(retryCount: number) {
         let nodeIdsToTest = await getAllNodeIds();
         // Shuffle, so we aren't always using the same node!
         nodeIdsToTest = shuffle(nodeIdsToTest, Date.now());
-        for (let nodeId of nodeIdsToTest) {
+        let resolvedNode = new PromiseObj<string>();
+
+        if (!quiet) {
+            console.log(`Checking ${nodeIdsToTest.length} nodes for capability ${controller._classGuid}`);
+        }
+        let allFinished = Promise.all(nodeIdsToTest.map(async nodeId => {
             if (await doesNodeExposeController(nodeId, controller)) {
                 if (!quiet) {
                     let duration = Date.now() - initialTime;
                     console.log(green(`Resolved capability ${controller._classGuid} with node ${nodeId}, in ${formatTime(duration)}`));
                 }
-                return nodeId;
+                resolvedNode.resolve(nodeId);
             }
+        }));
+        let result = await Promise.race([allFinished, resolvedNode.promise]);
+        if (typeof result === "string") {
+            return result;
         }
+
         if (retryCount > 0) {
+            if (!quiet) {
+                console.log(`Did not find capability in ${nodeIdsToTest.length} nodes, retrying... ${controller._classGuid}`);
+            }
             await delay(2000);
-            return getControllerNodeId(controller, quiet, retryCount - 1, initialTime);
+            return getInternal(retryCount - 1);
         }
         if (!quiet) {
             console.warn(yellow(`Could not find a node that exposes controller ${controller._classGuid}, tried: ${JSON.stringify(nodeIdsToTest)}`));
@@ -81,7 +93,11 @@ export async function getControllerNodeIdList(
     await Promise.all(nodeIdsToTest.map(async nodeId => {
         let result = await doesNodeExposeController(nodeId, controller);
         if (result) {
-            passedNodeIds.set(nodeId, result);
+            let entryPoint = await NodeCapabilitiesController.nodes[nodeId].getEntryPoint();
+            passedNodeIds.set(nodeId, {
+                machineId: getMachineId(nodeId),
+                entryPoint,
+            });
         }
     }));
 
@@ -99,13 +115,11 @@ export async function getControllerNodeIdList(
 }
 
 
-export async function doesNodeExposeController(reconnectNodeId: string, controller: SocketRegistered<{}>): Promise<{ machineId: string, entryPoint: string } | false> {
+export async function doesNodeExposeController(reconnectNodeId: string, controller: SocketRegistered<{}>): Promise<boolean> {
     let exposedControllers = await timeoutToUndefinedSilent(10_000, NodeCapabilitiesController.nodes[reconnectNodeId].getExposedControllers());
-    let machineId = await getOwnMachineId();
-    let entryPoint = await new NodeCapabilitiesControllerBase().getEntryPoint();
 
     if (exposedControllers?.includes(controller._classGuid)) {
-        return { machineId, entryPoint };
+        return true;
     }
     return false;
 }

package/src/3-path-functions/PathFunctionRunnerMain.ts

@@ -40,10 +40,6 @@ async function main() {
 
     await Querysub.hostService("PathFunctionRunnerMain");
 
-    if (!isPublic()) {
-        void IndexedLogs.runLogMoveLoop();
-    }
-
     // Use a fairly high stick time (the default is 10s), because having wait to sync data is very slow,
     //  and the function runner SHOULD have more memory than the clients, and much faster network speeds
     //  (it should be on the same local network as the path value authorities).

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -26,6 +26,9 @@ import { LimitGroup } from "../../../functional/limitProcessing";
 import { getAllNodeIds } from "../../../-f-node-discovery/NodeDiscovery";
 import { NodeCapabilitiesController } from "../../../-g-core-values/NodeCapabilities";
 import { getLoggers2Async } from "../diskLogger";
+import { watchAllValues } from "../errorNotifications2/logWatcher";
+// 
+watchAllValues;
 
 export type TimeFilePathWithSize = TimeFilePath & {
     size: number;
@@ -33,7 +36,8 @@ export type TimeFilePathWithSize = TimeFilePath & {
     sourceName: string;
 };
 
-let loggerByName = new Map<string, IndexedLogs<unknown>>();
+export let loggerByName = new Map<string, IndexedLogs<unknown>>();
+
 
 
 export class IndexedLogs<T> {
@@ -218,11 +222,30 @@ export class IndexedLogs<T> {
         return groups;
     }
 
+    private errorWatchers = new Set<(datum: T) => void>();
+    public watchErrors(callback: (datum: T) => void): () => void {
+        console.log(blue(`Watching errors: ${this.config.name}`));
+        this.errorWatchers.add(callback);
+        return () => {
+            this.errorWatchers.delete(callback);
+        };
+    }
+
     public append(datum: T) {
         this.getCurrentLogStream().append(datum);
         if (IndexedLogs.shouldRunLoop) {
             this.runLogMoverLoop();
         }
+
+        for (let callback of this.errorWatchers.values()) {
+            console.log(blue(`Calling error callback: ${this.config.name}`));
+            try {
+                callback(datum);
+            } catch (e) {
+                this.errorWatchers.delete(callback);
+                console.warn("Error in watchErrors callback, removing callback (likely just a client disconnect)", e);
+            }
+        }
     }
 
     private machineNodeCache = new Map<string, string>();

package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx

@@ -4,7 +4,7 @@ import { css } from "../../../4-dom/css";
 import { t } from "../../../2-proxy/schema2";
 import { Button } from "../../../library-components/Button";
 import { InputLabel, InputLabelURL } from "../../../library-components/InputLabel";
-import { getLoggers2, getLoggers2Async, LogDatum } from "../diskLogger";
+import { getLoggers2Async, LogDatum } from "../diskLogger";
 import { list, timeInDay, keyByArray, sort, throttleFunction } from "socket-function/src/misc";
 import { formatDateTime, formatDateTimeDetailed, formatNumber, formatTime, formatPercent } from "socket-function/src/formatting/format";
 import { IndexedLogs, TimeFilePathWithSize } from "./IndexedLogs";

package/src/diagnostics/logs/diskLogger.ts

@@ -7,6 +7,7 @@ import { formatTime } from "socket-function/src/formatting/format";
 import { addEpsilons } from "../../bits";
 import { getPathStr2 } from "../../path";
 import { isPublic } from "../../config";
+import type { IndexedLogs } from "./IndexedLogs/IndexedLogs";
 // IMPORTANT! We can't have any real imports here, because we are depended on so early in startup!
 
 if (isNode()) {
@@ -65,32 +66,35 @@ export const LOG_LINE_LIMIT_FLAG = String.fromCharCode(44534) + "LOGS_LINE_LIMIT
 /** If this key exists in the logged object, as in a key in one of the objects logged, then we will use the value of it as the limit ID. This is useful as it allows us to either override a limit or limit something independently from other logs in the file. */
 export const LOG_LINE_LIMIT_ID = "LIMIT_LINE_ID";
 
-export const getLoggers2 = lazy(function () {
-    const { IndexedLogs } = require("./IndexedLogs/IndexedLogs") as typeof import("./IndexedLogs/IndexedLogs");
-
-    if (!IndexedLogs) {
-        setImmediate(() => {
-            getLoggers2.reset();
-        });
-        return undefined;
-    }
-
-    return {
-        logLogs: new IndexedLogs<LogDatum>({ name: "logs/log", getTime: x => x.time }),
-        warnLogs: new IndexedLogs<LogDatum>({ name: "logs/warn", getTime: x => x.time }),
-        infoLogs: new IndexedLogs<LogDatum>({ name: "logs/info", getTime: x => x.time }),
-        errorLogs: new IndexedLogs<LogDatum>({ name: "logs/error", getTime: x => x.time }),
-    };
-});
+let loggersObj: {
+    logLogs: IndexedLogs<LogDatum>;
+    warnLogs: IndexedLogs<LogDatum>;
+    infoLogs: IndexedLogs<LogDatum>;
+    errorLogs: IndexedLogs<LogDatum>;
+} | undefined;
 export const getLoggers2Async = lazy(async () => {
     const { IndexedLogs } = await import("./IndexedLogs/IndexedLogs");
-    return {
+    loggersObj = {
         logLogs: new IndexedLogs<LogDatum>({ name: "logs/log", getTime: x => x.time }),
         warnLogs: new IndexedLogs<LogDatum>({ name: "logs/warn", getTime: x => x.time }),
         infoLogs: new IndexedLogs<LogDatum>({ name: "logs/info", getTime: x => x.time }),
         errorLogs: new IndexedLogs<LogDatum>({ name: "logs/error", getTime: x => x.time }),
     };
+    return loggersObj;
 });
+export function getLogLogs(): Promise<IndexedLogs<LogDatum>> {
+    return getLoggers2Async().then(x => x.logLogs);
+}
+export function getWarnLogs(): Promise<IndexedLogs<LogDatum>> {
+    return getLoggers2Async().then(x => x.warnLogs);
+}
+export function getInfoLogs(): Promise<IndexedLogs<LogDatum>> {
+    return getLoggers2Async().then(x => x.infoLogs);
+}
+export function getErrorLogs(): Promise<IndexedLogs<LogDatum>> {
+    return getLoggers2Async().then(x => x.errorLogs);
+}
+
 
 // NOTE: If any message (first param) starts with this, we don't log it to the disk. VERY useful for multi-line logging where it wouldn't make sense in the logs
 // NOTE: This is visible, otherwise it's easy to accidentally copy it, and not know why the text is behaving strangely (not === other seemingly equal text, etc).
@@ -105,10 +109,6 @@ export function addGlobalContext(fnc: () => { [key: string]: unknown }) {
     globalContextParts.push(fnc);
 }
 
-let startupDone = false;
-void Promise.resolve().then(() => {
-    startupDone = true;
-});
 
 
 
@@ -136,15 +136,14 @@ export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknow
         // NOTE: Local logs now log to their local disk instead of backblaze, so we can log even if in development (they just won't show up on the remote server).
         //if (isPublic())
         {
-            let loggers = startupDone ? getLoggers2() : undefined;
-            if (!loggers) {
-                getLoggers2.reset();
-                setImmediate(() => {
+            let promise = getLoggers2Async();
+            if (!loggersObj) {
+                void promise.finally(() => {
                     logDiskDontShim(type, ...args);
                 });
                 return;
             }
-            const { logLogs, warnLogs, infoLogs, errorLogs } = loggers;
+            const { logLogs, warnLogs, infoLogs, errorLogs } = loggersObj;
             if (type === "log") {
                 logLogs.append(logObj);
             } else if (type === "warn") {

package/src/diagnostics/logs/diskShimConsoleLogs.ts

@@ -20,6 +20,10 @@ export function shimConsoleLogs() {
         let console = globalThis.console;
         let originalFnc = console[fncName];
         console[fncName] = (...args: any[]) => {
+            if (!(globalThis as any).shimmedConsoleLogs) {
+                if (fncName === "info") return;
+                return originalFnc(...args);
+            }
             try {
                 if (
                     args.length > 0

package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx

@@ -0,0 +1,276 @@
+import { qreact } from "../../../4-dom/qreact";
+import { t } from "../../../2-proxy/schema2";
+import { css } from "typesafecss";
+import { ErrorNotificationsController, watchUnmatchedErrors } from "./errorNotifications";
+import { LogDatum } from "../diskLogger";
+import { sort, nextId } from "socket-function/src/misc";
+import { formatVeryNiceDateTime } from "socket-function/src/formatting/format";
+import { Querysub } from "../../../4-querysub/QuerysubController";
+import { InputLabel } from "../../../library-components/InputLabel";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { isPublic } from "../../../config";
+
+export class ErrorNotificationPage extends qreact.Component {
+    state = t.state({
+        unmatchedErrors: t.atomic<LogDatum[]>([]),
+        suppressionMatches: t.atomic<Map<string, {
+            history: Map<number, { count: number }>;
+            exampleIndex: number;
+            examples: LogDatum[];
+        }>>(new Map()),
+        suppressionEntries: t.atomic<Array<{
+            id: string;
+            description: string;
+            pattern: string;
+            createdTime: number;
+            lastUpdatedTime: number;
+        }>>([]),
+        addingNewSuppression: t.atomic<boolean>(false),
+        newSuppressionDescription: t.atomic<string>(""),
+        newSuppressionPattern: t.atomic<string>(""),
+        initError: t.atomic<string | undefined>(undefined),
+        isLoading: t.atomic<boolean>(true),
+        testErrorMessage: t.atomic<string>("example error"),
+    });
+
+
+    unsubscribe?: () => void;
+
+    componentDidMount() {
+        Querysub.onCommitFinished(async () => {
+            try {
+                this.unsubscribe = await watchUnmatchedErrors({
+                    callback: (datums: LogDatum[]) => {
+                        Querysub.commit(() => {
+                            this.state.unmatchedErrors = [...this.state.unmatchedErrors, ...datums];
+                        });
+                    },
+                });
+
+                let controller = ErrorNotificationsController(SocketFunction.browserNodeId());
+                let data = await controller.getData.promise();
+                let suppressionEntries = await controller.getSuppressionEntries.promise();
+
+                Querysub.commit(() => {
+                    this.state.unmatchedErrors = data.unmatchedErrors;
+                    this.state.suppressionMatches = data.suppressionMatches;
+                    this.state.suppressionEntries = suppressionEntries;
+                    this.state.initError = undefined;
+                    this.state.isLoading = false;
+                });
+            } catch (error) {
+                Querysub.commit(() => {
+                    this.state.initError = error instanceof Error ? error.message : String(error);
+                    this.state.isLoading = false;
+                });
+            }
+        });
+    }
+
+    componentWillUnmount() {
+        this.unsubscribe?.();
+    }
+
+    render() {
+        if (this.state.isLoading) {
+            return <div className={css.vbox(16).pad2(16)}>
+                <h2>Error Notifications</h2>
+                <div>Loading error notifications...</div>
+            </div>;
+        }
+
+        if (this.state.initError) {
+            return <div className={css.vbox(16).pad2(16)}>
+                <h2>Error Notifications</h2>
+                <div className={css.pad2(12).bord2(0, 80, 50).hsl(0, 80, 95).colorhsl(0, 80, 30)}>
+                    <strong>Error loading error notifications:</strong>
+                    <pre className={css.pad2(8).hsl(0, 0, 98).bord2(0, 0, 85)}>
+                        {this.state.initError}
+                    </pre>
+                </div>
+            </div>;
+        }
+
+        let sortedErrors = [...this.state.unmatchedErrors];
+        sort(sortedErrors, x => -x.time);
+        let recentErrors = sortedErrors.slice(0, 10);
+
+        let sortedSuppressions = [...this.state.suppressionEntries];
+        sort(sortedSuppressions, x => -x.lastUpdatedTime);
+
+        return <div className={css.vbox(16).pad2(16)}>
+            <h2>Error Notifications</h2>
+
+            {!isPublic() && <h1>
+                IMPORTANT! You MUST run `yarn error-watch` to get errors notifications in the dev environment.
+            </h1>}
+
+            <div className={css.hbox(12).pad2(12).bord2(30, 50, 60).hsl(30, 50, 95)}>
+                <InputLabel
+                    label="Test Error Message"
+                    value={this.state.testErrorMessage}
+                    onChangeValue={(value) => {
+                        this.state.testErrorMessage = value;
+                    }}
+                    className={css.width(400)}
+                />
+                <button
+                    className={css.pad2(12, 8).button.bord2(30, 80, 50).hsl(30, 80, 85)}
+                    onClick={() => {
+                        let message = this.state.testErrorMessage;
+                        Querysub.onCommitFinished(async () => {
+                            let controller = ErrorNotificationsController(SocketFunction.browserNodeId());
+                            await controller.logTestError.promise(message);
+                        });
+                    }}
+                >
+                    Log Test Error
+                </button>
+            </div>
+
+            <div className={css.vbox(12)}>
+                <h3>Unmatched Errors ({this.state.unmatchedErrors.length})</h3>
+                {recentErrors.length === 0 && <div>No unmatched errors</div>}
+                <div className={css.vbox(8)}>
+                    {recentErrors.map((error, idx) => (
+                        <div key={idx} className={css.pad2(12).bord2(0, 30, 70).hsl(0, 50, 95).vbox(4)}>
+                            <div className={css.hbox(8)}>
+                                <strong>{formatVeryNiceDateTime(error.time)}</strong>
+                                {error.machineId && <span>Machine: {error.machineId}</span>}
+                            </div>
+                            <pre className={css.overflowAuto.pad2(8).hsl(0, 0, 98).bord2(0, 0, 85)}>
+                                {JSON.stringify(error)}
+                            </pre>
+                        </div>
+                    ))}
+                </div>
+            </div>
+
+            <div className={css.vbox(12)}>
+                <div className={css.hbox(12)}>
+                    <h3>Suppressions ({sortedSuppressions.length})</h3>
+                    <button
+                        className={css.pad2(12, 8).button.bord2(120, 80, 50).hsl(120, 80, 90)}
+                        onClick={() => {
+                            this.state.addingNewSuppression = !this.state.addingNewSuppression;
+                        }}
+                    >
+                        {this.state.addingNewSuppression ? "Cancel" : "Add Suppression"}
+                    </button>
+                </div>
+
+                {this.state.addingNewSuppression && (
+                    <div className={css.vbox(8).pad2(12).bord2(120, 50, 60).hsl(120, 50, 95)}>
+                        <InputLabel
+                            label="Description"
+                            value={this.state.newSuppressionDescription}
+                            onChangeValue={(value) => {
+                                this.state.newSuppressionDescription = value;
+                            }}
+                            className={css.width(500)}
+                        />
+                        <InputLabel
+                            label="Pattern (regex)"
+                            value={this.state.newSuppressionPattern}
+                            onChangeValue={(value) => {
+                                this.state.newSuppressionPattern = value;
+                            }}
+                            className={css.width(500)}
+                        />
+                        <button
+                            className={css.pad2(12, 8).button.bord2(120, 80, 50).hsl(120, 80, 85)}
+                            onClick={() => {
+                                let description = this.state.newSuppressionDescription;
+                                let pattern = this.state.newSuppressionPattern;
+                                if (!description || !pattern) {
+                                    alert("Description and pattern are required");
+                                    return;
+                                }
+
+                                Querysub.onCommitFinished(async () => {
+                                    let now = Date.now();
+                                    let newEntry = {
+                                        id: nextId(),
+                                        description,
+                                        pattern,
+                                        createdTime: now,
+                                        lastUpdatedTime: now,
+                                    };
+                                    let controller = ErrorNotificationsController(SocketFunction.browserNodeId());
+                                    await controller.setSuppressionEntry.promise(newEntry);
+
+                                    let suppressionEntries = await controller.getSuppressionEntries.promise();
+                                    Querysub.commit(() => {
+                                        this.state.suppressionEntries = suppressionEntries;
+                                        this.state.addingNewSuppression = false;
+                                        this.state.newSuppressionDescription = "";
+                                        this.state.newSuppressionPattern = "";
+                                    });
+                                });
+                            }}
+                        >
+                            Save Suppression
+                        </button>
+                    </div>
+                )}
+
+                <div className={css.vbox(12)}>
+                    {sortedSuppressions.map((suppression) => {
+                        let matchData = this.state.suppressionMatches.get(suppression.id);
+                        let totalCount = 0;
+                        if (matchData) {
+                            for (let [chunk, data] of matchData.history) {
+                                totalCount += data.count;
+                            }
+                        }
+
+                        return <div key={suppression.id} className={css.pad2(12).bord2(200, 30, 70).hsl(200, 30, 95).vbox(8)}>
+                            <div className={css.hbox(12)}>
+                                <div className={css.flexGrow(1).vbox(4)}>
+                                    <strong>{suppression.description}</strong>
+                                    <div className={css.hbox(12)}>
+                                        <span>Pattern: <code className={css.pad2(4, 2).hsl(0, 0, 90)}>{suppression.pattern}</code></span>
+                                        <span>Total Matches: {totalCount}</span>
+                                    </div>
+                                    <div className={css.hbox(12)}>
+                                        <span>Created: {formatVeryNiceDateTime(suppression.createdTime)}</span>
+                                        <span>Updated: {formatVeryNiceDateTime(suppression.lastUpdatedTime)}</span>
+                                    </div>
+                                </div>
+                                <button
+                                    className={css.pad2(8, 6).button.bord2(0, 80, 50).hsl(0, 80, 90)}
+                                    onClick={() => {
+                                        if (!confirm(`Delete suppression "${suppression.description}"?`)) return;
+
+                                        Querysub.onCommitFinished(async () => {
+                                            let controller = ErrorNotificationsController(SocketFunction.browserNodeId());
+                                            await controller.deleteSuppressionEntry.promise(suppression.id);
+
+                                            let suppressionEntries = await controller.getSuppressionEntries.promise();
+                                            Querysub.commit(() => {
+                                                this.state.suppressionEntries = suppressionEntries;
+                                            });
+                                        });
+                                    }}
+                                >
+                                    Delete
+                                </button>
+                            </div>
+
+                            {matchData && matchData.examples.length > 0 && (
+                                <div className={css.vbox(8)}>
+                                    <strong>Recent Examples ({matchData.examples.length}):</strong>
+                                    {matchData.examples.slice(0, 3).map((example, idx) => (
+                                        <pre key={idx} className={css.overflowAuto.pad2(8).hsl(0, 0, 98).bord2(0, 0, 85).fontSize(12)}>
+                                            {JSON.stringify(example, null, 2)}
+                                        </pre>
+                                    ))}
+                                </div>
+                            )}
+                        </div>;
+                    })}
+                </div>
+            </div>
+        </div>;
+    }
+}

package/src/diagnostics/logs/errorNotifications2/errorNotifications.ts

@@ -0,0 +1,269 @@
+import { cacheLimited, lazy } from "socket-function/src/caching";
+import { nestArchives } from "../../../-a-archives/archives";
+import { getArchivesBackblaze } from "../../../-a-archives/archivesBackBlaze";
+import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
+import { Querysub } from "../../../4-querysub/QuerysubController";
+import { getDomain } from "../../../config";
+import { MachineInfo } from "../../../deployManager/machineSchema";
+import { createMatchesPattern } from "../IndexedLogs/bufferSearchFindMatcher";
+import { LogDatum, getErrorLogs } from "../diskLogger";
+import { watchAllValues } from "./logWatcher";
+import { batchFunction, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { nextId, timeInMinute } from "socket-function/src/misc";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { assertIsManagementUser } from "../../managementPages";
+import { getSyncedController } from "../../../library-components/SyncedController";
+import { getControllerNodeId } from "../../../-g-core-values/NodeCapabilities";
+
+
+type SuppressionEntry = {
+    id: string;
+    description: string;
+    pattern: string;
+
+    createdTime: number;
+    lastUpdatedTime: number;
+};
+const suppression = archiveJSONT<SuppressionEntry>(() => nestArchives("logs/error-suppression/", getArchivesBackblaze(getDomain())));
+let suppressionCache: SuppressionEntry[] = [];
+let ensureWatching = lazy(async () => {
+    await runInfinitePollCallAtStart(timeInMinute * 5, updateNow);
+});
+async function updateNow() {
+    suppressionCache = await suppression.values();
+}
+async function getSuppressionEntries(): Promise<SuppressionEntry[]> {
+    await ensureWatching();
+    return suppressionCache;
+}
+
+let getMatcher = cacheLimited(10000, (text: string) => createMatchesPattern(Buffer.from(text), false));
+
+const MAX_UNMATCHED = 10000;
+const MAX_EXAMPLES = 100;
+let unmatchedIndex = 0;
+let unmatchedErrors: LogDatum[] = [];
+let suppressionMatches = new Map<string, {
+    // getHistoryChunk =>
+    history: Map<number, {
+        count: number;
+    }>;
+    exampleIndex: number;
+    examples: LogDatum[];
+}>();
+const chunkUnit = timeInMinute * 5;
+export function getHistoryChunk(time: number) {
+    return Math.floor(time / chunkUnit) * chunkUnit;
+}
+export function getChunkEndTime(chunk: number) {
+    chunk = getHistoryChunk(chunk);
+    return chunk + chunkUnit;
+}
+
+export async function exposeErrorWatchService() {
+    SocketFunction.expose(ErrorNotificationServiceBase);
+
+    let errorLogs = await getErrorLogs();
+    for await (let error of watchAllValues(errorLogs)) {
+        let errorBuffer = Buffer.from(JSON.stringify(error));
+        let suppressionEntries = await getSuppressionEntries();
+        let anyMatches = false;
+        for (let suppressionEntry of suppressionEntries) {
+            let matcher = getMatcher(suppressionEntry.pattern);
+            if (!matcher(errorBuffer)) continue;
+            anyMatches = true;
+            let history = suppressionMatches.get(suppressionEntry.id);
+            if (!history) {
+                history = {
+                    history: new Map(),
+                    exampleIndex: 0,
+                    examples: [],
+                };
+                suppressionMatches.set(suppressionEntry.id, history);
+            }
+
+            if (history.examples.length > MAX_EXAMPLES) {
+                history.examples[history.exampleIndex] = error;
+                history.exampleIndex = (history.exampleIndex + 1) % MAX_EXAMPLES;
+            } else {
+                history.examples.push(error);
+            }
+
+            let historyChunk = getHistoryChunk(error.time);
+            let historyEntry = history.history.get(historyChunk);
+            if (!historyEntry) {
+                historyEntry = { count: 0 };
+                history.history.set(historyChunk, historyEntry);
+            }
+            historyEntry.count++;
+        }
+        if (!anyMatches) {
+            if (unmatchedErrors.length > unmatchedIndex) {
+                unmatchedErrors[unmatchedIndex] = error;
+                unmatchedIndex = (unmatchedIndex + 1) % MAX_UNMATCHED;
+            } else {
+                unmatchedErrors.push(error);
+            }
+            void ErrorNotificationService.triggerUnmatchedError(error);
+        }
+    }
+}
+
+class ErrorNotificationService {
+    public async getData() {
+        return {
+            unmatchedErrors,
+            suppressionMatches,
+        };
+    }
+
+    private static watchersSERVICE = new Set<string>();
+    public async watchUnmatchedErrorsSERVICE(): Promise<void> {
+        let caller = SocketFunction.getCaller();
+        ErrorNotificationService.watchersSERVICE.add(caller.nodeId);
+    }
+
+    public static triggerUnmatchedError = batchFunction({ delay: 100 }, (datums: LogDatum[]) => {
+        for (let nodeId of ErrorNotificationService.watchersSERVICE) {
+            void (async () => {
+                try {
+                    await ErrorNotificationDataBase.nodes[nodeId].receiveErrorHTTP(datums);
+                } catch {
+                    ErrorNotificationService.watchersSERVICE.delete(nodeId);
+                }
+            })();
+        }
+    });
+}
+
+const ErrorNotificationServiceBase = SocketFunction.register(
+    "ErrorNotificationService-019c9cae-8333-7708-a4e7-500f5fc23175",
+    new ErrorNotificationService(),
+    () => ({
+        getData: {},
+        watchUnmatchedErrorsSERVICE: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    }),
+    {
+        noAutoExpose: true,
+    }
+);
+
+class ErrorNotificationData {
+    public async getData() {
+        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
+        if (!controllerNodeId) {
+            throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
+        }
+        return await ErrorNotificationServiceBase.nodes[controllerNodeId].getData();
+    }
+
+    static browserCallbacks = new Set<(datum: LogDatum[]) => void>();
+    public static async watchUnmatchedErrors(config: {
+        callback: (datum: LogDatum[]) => void;
+    }) {
+        ErrorNotificationData.browserCallbacks.add(config.callback);
+        await ErrorNotificationData.ensureWatchingErrorsBrowser();
+        return () => {
+            ErrorNotificationData.browserCallbacks.delete(config.callback);
+        };
+    }
+
+    private static ensureWatchingErrorsBrowser = lazy(async () => {
+        await ErrorNotificationDataBase.nodes[SocketFunction.getBrowserNodeId()].watchUnmatchedErrorsHTTP();
+    });
+
+    public async receiveErrorBrowser(datums: LogDatum[]) {
+        for (let callback of ErrorNotificationData.browserCallbacks) {
+            callback(datums);
+        }
+    }
+
+    private watchersHTTP = new Set<string>();
+    public async watchUnmatchedErrorsHTTP() {
+        let caller = SocketFunction.getCaller();
+        this.watchersHTTP.add(caller.nodeId);
+        await ErrorNotificationData.ensureWatchingErrorsHTTP();
+    }
+
+    private static ensureWatchingErrorsHTTP = lazy(async () => {
+        let controllerNodeId = await getControllerNodeId(ErrorNotificationServiceBase);
+        if (!controllerNodeId) {
+            ErrorNotificationData.ensureWatchingErrorsHTTP.reset();
+            throw new Error(`Could not find node exposing controller ErrorNotificationServiceBase`);
+        }
+        SocketFunction.onNextDisconnect(controllerNodeId, () => {
+            ErrorNotificationData.ensureWatchingErrorsHTTP.reset();
+        });
+        await ErrorNotificationServiceBase.nodes[controllerNodeId].watchUnmatchedErrorsSERVICE();
+    });
+
+    public async receiveErrorHTTP(datums: LogDatum[]) {
+        for (let nodeId of this.watchersHTTP) {
+            void (async () => {
+                try {
+                    await ErrorNotificationDataBase.nodes[nodeId].receiveErrorBrowser(datums);
+                } catch {
+                    this.watchersHTTP.delete(nodeId);
+                }
+            })();
+        }
+    }
+
+    public async getSuppressionEntries() {
+        return await getSuppressionEntries();
+    }
+
+    public async setSuppressionEntry(entry: SuppressionEntry) {
+        await suppression.set(entry.id, entry);
+        let prevEntry = suppressionCache.findIndex(e => e.id === entry.id);
+        if (prevEntry !== -1) {
+            suppressionCache[prevEntry] = entry;
+        } else {
+            suppressionCache.push(entry);
+        }
+        void updateNow();
+    }
+
+    public async deleteSuppressionEntry(id: string) {
+        await suppression.delete(id);
+        let prevEntry = suppressionCache.findIndex(e => e.id === id);
+        if (prevEntry !== -1) {
+            suppressionCache.splice(prevEntry, 1);
+        }
+        void updateNow();
+    }
+
+    public async logTestError(errorMessage: string) {
+        console.error(errorMessage);
+    }
+}
+
+const ErrorNotificationDataBase = SocketFunction.register(
+    "ErrorNotificationData-019c9cae-8333-7708-a4e7-500f5fc23174",
+    new ErrorNotificationData(),
+    () => ({
+        getData: {},
+        receiveErrorBrowser: {},
+        receiveErrorHTTP: {},
+        watchUnmatchedErrorsHTTP: {},
+        getSuppressionEntries: {},
+        setSuppressionEntry: {},
+        deleteSuppressionEntry: {},
+        logTestError: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    })
+);
+export const ErrorNotificationsController = getSyncedController(ErrorNotificationDataBase);
+
+export function watchUnmatchedErrors(config: {
+    callback: (datum: LogDatum[]) => void;
+}) {
+    return ErrorNotificationData.watchUnmatchedErrors({
+        callback: config.callback,
+    });
+}

package/src/diagnostics/logs/errorNotifications2/errorWatchEntry.ts

@@ -0,0 +1,13 @@
+import "../../../forceProduction";
+import "../../../inject";
+
+import { logErrors } from "../../../errors";
+import { Querysub } from "../../../4-querysub/QuerysubController";
+import { exposeErrorWatchService } from "./errorNotifications";
+
+logErrors(main());
+
+async function main() {
+    await Querysub.hostService("ErrorWatch");
+    await exposeErrorWatchService();
+}

package/src/diagnostics/logs/errorNotifications2/logWatcher.ts

@@ -0,0 +1,104 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { isNodeIdLocal, isOwnNodeId, watchDeltaNodeIds } from "../../../-f-node-discovery/NodeDiscovery";
+import { IndexedLogs, loggerByName } from "../IndexedLogs/IndexedLogs";
+import { nextId } from "socket-function/src/misc";
+import { assertIsManagementUser, isManagementUser } from "../../managementPages";
+import { ignoreErrors } from "../../../errors";
+import { blue } from "socket-function/src/formatting/logColors";
+import { SocketChannel } from "../../../functional/SocketChannel";
+import { batchFunction } from "socket-function/src/batching";
+import { isPublic } from "../../../config";
+
+export async function* watchAllValues<T>(logs: IndexedLogs<T>): AsyncGenerator<T> {
+    let callbackId = nextId();
+    let buffer: T[] = [];
+    let resolveNext: ((value: T) => void) | undefined;
+
+    clientCallbacks.set(callbackId, (datums: unknown[]) => {
+        for (let i = 1; i < datums.length; i++) {
+            buffer.push(datums[i] as T);
+        }
+        let datum = datums[0] as T;
+        if (resolveNext) {
+            resolveNext(datum as T);
+            resolveNext = undefined;
+        } else {
+            buffer.push(datum as T);
+        }
+    });
+
+    watchDeltaNodeIds((delta) => {
+        for (let nodeId of delta.newNodeIds) {
+            if (isOwnNodeId(nodeId)) continue;
+            let isNodeLocal = isNodeIdLocal(nodeId);
+            if (isNodeLocal !== isPublic()) continue;
+            ignoreErrors(ErrorNotificationControllerRegistered.nodes[nodeId].watchErrors({
+                loggerName: logs.config.name,
+                callbackId,
+            }));
+        }
+    });
+
+    try {
+        while (true) {
+            if (buffer.length > 0) {
+                yield buffer.shift()!;
+            } else {
+                yield await new Promise<T>((resolve) => {
+                    resolveNext = resolve;
+                });
+            }
+        }
+    } finally {
+        clientCallbacks.delete(callbackId);
+    }
+}
+
+
+
+let clientCallbacks = new Map<string, (datums: unknown[]) => void>();
+
+class ErrorNotificationController {
+    public async onError(config: {
+        callbackId: string;
+        datums: unknown[];
+    }) {
+        let callback = clientCallbacks.get(config.callbackId);
+        if (!callback) {
+            throw new Error(`Callback ${config.callbackId} not found`);
+        }
+        callback(config.datums);
+    }
+
+    public async watchErrors(config: {
+        loggerName: string;
+        callbackId: string;
+    }): Promise<void> {
+        let caller = SocketFunction.getCaller();
+        let logger = loggerByName.get(config.loggerName);
+        if (!logger) {
+            throw new Error(`Logger ${config.loggerName} not found`);
+        }
+
+        let unsubscribe = logger.watchErrors(batchFunction({ delay: 100 }, (datums: unknown[]) => {
+            ErrorNotificationControllerRegistered.nodes[caller.nodeId].onError({
+                callbackId: config.callbackId,
+                datums,
+            }).catch(() => {
+                unsubscribe();
+            });
+        }));
+    }
+}
+
+const ErrorNotificationControllerRegistered = SocketFunction.register(
+    "ErrorNotificationController-019c9cae-8333-7708-a4e7-500f5fc23173",
+    new ErrorNotificationController(),
+    () => ({
+        onError: {},
+        watchErrors: {}
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    })
+);

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -20,16 +20,30 @@ IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs shou
 
 
 
+//todonext
 
-2) Create lot of remote server logs
-    - Via our refresh loop
 
-3) Verify search is still fast, and works correctly!
-    - Try to get a rare value in that is in the logs, but is hard to find
-    Good rare search. Checks for sending emails, which we do very rarely. If we pack in 10GB of logs after this, it will really stress our indexing system.
-        - At the moment our pending index has a good hit rate (0 false positives out of 1720). We'll see if this continues for our remote index.
-    https://127-0-0-1.querysubtest.com:7007/?hot&readPublicLogs&showingmanagement&machineview=services&managementpage=LogViewer3&searchText=Sending%20email&selectedFields=%7B%22path%22%3Atrue%7D
-    
+1) Add button that will log an error serverside, to ensure this system works
+    - Allow typing any error we want
+2) Write smaller component just for watching
+    - Lazily the component, and only if super user (just uncomment the code in Page.tsx)
+
+3) Fix up the error watching UI
+
+
+4) Publish
+
+5) Deploy watcher service to remote (yarn error-watch-public)
+
+5) Test on actual server, with actual logs
+
+
+// single service, no history search, discord messaging, suppression and suppression page
+//  - Storing suppression history just in memory? (although suppression values, obviously, on disk)
+//  - receive all errors, and service will do suppression
+
+// Start with page itself showing errors and suppressing them, for debugging/testing
+// - Ai to automatically create suppression searches, and combine existing ones, etc
 
 
 Rewrite error notification code
@@ -48,6 +62,9 @@ Rewrite error notification code
 
 
 0) Add LZ4 compression to socket-function by default
+    - Setup local tester for it to start
+        - with non-synced endpoint, just surely socket-function
+        - purely nodejs, just talking to itself
     - Allow setting "compress" to "none" or "lz4" or "zip" or "zip0" or "zip3", etc, for levels.
         - default is "lz4"
     - REQUIRES feature checking the remote, to make sure it is new enough to accept this.
@@ -76,6 +93,16 @@ Rewrite error notification code
 
 
 
+//todonext
+// Reorganize our lifecycles ideas
+//  - Scans over past logs, and only finished, never pending
+//         - Probably... realtime creating cache, BUT, if we add new lifecycles, we then only add them on demand. So when we are in the debugging mindset, and adding and changing them, it's fast (only as needed), but when we come back later when we aren't in the middle of debugging, it'll be super fast so we can fix the bug quickly and move on.
+//      - OR... we can do pending dynamically, but we don't cache it? Hmm... then what's the point? Hmm...
+//      - MAYBE, no caching, but we restrict the time by a lot? HMM...
+//      - I mean, we have to implement it without caching anyways to start, so we can just do that, and then cache it later if we see the need...
+//          - WELL, we need SOME kind of caching... Maybe... we DO use a strict time range, and then... if we've scanned a pending file in the past, we can cache that, because we know we've got all of it's values. Hmm...
+//  - Use lifecycles to debug rejections. To a point, until it is likely we are out of sync, then we should write the sync verification code.
+
 
 // todonext;
 // Hmm... so... should we index it, so we can search it? HMM... I think we might want to?

package/src/diagnostics/managementPages.tsx

@@ -89,6 +89,12 @@ export async function registerManagementPages2(config: {
         componentName: "LogViewer3",
         getModule: () => import("./logs/IndexedLogs/LogViewer3"),
     });
+    inputPages.push({
+        title: "Error Notifications",
+        componentName: "ErrorNotificationPage",
+        controllerName: "",
+        getModule: () => import("./logs/errorNotifications2/ErrorNotificationPage"),
+    });
     inputPages.push({
         title: "Security",
         componentName: "SecurityPage",

package/src/server.ts

@@ -13,16 +13,8 @@ import { formatTime } from "socket-function/src/formatting/format";
 //  we DO want to register some static diagnostics from Querysub, so... import it explicitly.
 import "./4-querysub/Querysub";
 import { getOurAuthorities } from "./config2";
-import { getDomain, isPublic } from "./config";
 
-import yargs from "yargs";
 import { Querysub } from "./4-querysub/Querysub";
-import { createSourceCheck } from "./user-implementation/canSeeSource";
-let yargObj = isNodeTrue() && yargs(process.argv)
-    .option("authority", { type: "string", desc: `Defines the base paths we are an authority on (the domain is prepended to them). Either a file path to a JSON(AuthorityPath[]), or a base64 representation of the JSON(AuthorityPath[]).` })
-    .option("verbose", { type: "boolean", desc: "Log all writes and reads" })
-    .argv || {}
-    ;
 
 logErrors(main());
 

package/test.ts

@@ -1,8 +1,18 @@
+import { chdir } from "process";
+chdir("D:/repos/qs-cyoa/");
+
+import "./inject";
+
 import { Querysub } from "./src/4-querysub/QuerysubController";
-// Import querysub, so inject is run
-Querysub;
-import { testMain } from "./testEntry2";
+import { getErrorLogs, getLoggers2Async } from "./src/diagnostics/logs/diskLogger";
+import { watchAllValues } from "./src/diagnostics/logs/errorNotifications2/logWatcher";
+
+async function main() {
+    await Querysub.hostService("testwatcher");
+    let errorLogs = await getErrorLogs();
+    for await (let error of watchAllValues(errorLogs)) {
+        process.stdout.write(JSON.stringify(error) + "\n");
+    }
+}
 
-testMain().catch(console.error).finally(() => {
-    process.exit(0);
-});
+main().catch(console.error).finally(() => process.exit(0));

package/test2.ts

@@ -0,0 +1,20 @@
+import { chdir } from "process";
+chdir("D:/repos/qs-cyoa/");
+
+import "./inject";
+
+import { timeInSecond } from "socket-function/src/misc";
+import { getErrorLogs, getLoggers2Async } from "./src/diagnostics/logs/diskLogger";
+import { watchAllValues } from "./src/diagnostics/logs/errorNotifications2/logWatcher";
+import { delay } from "socket-function/src/batching";
+import { Querysub } from "./src/4-querysub/QuerysubController";
+
+async function main() {
+    await Querysub.hostService("test");
+    while (true) {
+        console.error("Test error");
+        await delay(timeInSecond * 5);
+    }
+}
+
+main().catch(console.error).finally(() => process.exit(0));

package/src/diagnostics/logs/errorNotifications2/errorNotifications2.ts

@@ -1,9 +0,0 @@
-
-//todonext
-
-// single service, no history search, discord messaging, suppression and suppression page
-//  - Storing suppression history just in memory? (although suppression values, obviously, on disk)
-//  - receive all errors, and service will do suppression
-
-// Start with page itself showing errors and suppressing them, for debugging/testing
-// - Ai to automatically create suppression searches, and combine existing ones, etc