querysub 0.377.0 → 0.379.0

package/src/diagnostics/logs/errorNotifications2/errorWatcher.ts

@@ -0,0 +1,168 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { lazy } from "socket-function/src/caching";
+import { nextId } from "socket-function/src/misc";
+import { t } from "../../../2-proxy/schema2";
+import { Querysub } from "../../../4-querysub/QuerysubController";
+import { LogDatum } from "../diskLogger";
+import { SuppressionMatch, watchUnmatchedErrors, ErrorNotificationsController } from "./errorNotifications";
+
+let errorNotificationsLocalSchema = Querysub.createLocalSchema("errorNotifications", {
+    unmatchedErrors: t.atomic<LogDatum[]>([]),
+    suppressionMatches: t.atomic<Map<string, SuppressionMatch>>(new Map()),
+    suppressionEntries: t.atomic<Array<{
+        id: string;
+        notes?: string;
+        pattern: string;
+        timeout: number;
+        createdTime: number;
+        lastUpdatedTime: number;
+    }>>([]),
+    isLoading: t.atomic<boolean>(true),
+    initError: t.atomic<string | undefined>(undefined),
+});
+
+export let getErrorNotificationsManager = lazy(() => {
+    let state = errorNotificationsLocalSchema();
+
+    void (async () => {
+        try {
+            await watchUnmatchedErrors({
+                errorCallback: (datums: LogDatum[]) => {
+                    Querysub.commit(() => {
+                        state.unmatchedErrors = [...state.unmatchedErrors, ...datums];
+                    });
+                },
+                suppressionCallback: (matches) => {
+                    Querysub.commit(() => {
+                        let newMatches = new Map(state.suppressionMatches);
+                        let hasNewSuppression = false;
+                        for (let match of matches) {
+                            newMatches.set(match.id, match);
+                            if (!state.suppressionEntries.find(e => e.id === match.id)) {
+                                hasNewSuppression = true;
+                            }
+                        }
+                        state.suppressionMatches = newMatches;
+                        if (hasNewSuppression) {
+                            void reloadData();
+                        }
+                    });
+                },
+            });
+
+            let controller = ErrorNotificationsController(SocketFunction.browserNodeId());
+            let data = await controller.getData.promise();
+            let suppressionEntries = await controller.getSuppressionEntries.promise();
+
+            Querysub.commit(() => {
+                state.unmatchedErrors = data.unmatchedErrors;
+                state.suppressionMatches = data.suppressionMatches;
+                state.suppressionEntries = suppressionEntries;
+                state.initError = undefined;
+                state.isLoading = false;
+            });
+        } catch (error) {
+            Querysub.commit(() => {
+                state.initError = error instanceof Error && error.message || String(error);
+                state.isLoading = false;
+            });
+        }
+    })();
+
+    async function reloadData() {
+        let controller = ErrorNotificationsController(SocketFunction.browserNodeId());
+        let data = await controller.getData.promise();
+        let suppressionEntries = await controller.getSuppressionEntries.promise();
+        Querysub.commit(() => {
+            state.unmatchedErrors = data.unmatchedErrors;
+            state.suppressionMatches = data.suppressionMatches;
+            state.suppressionEntries = suppressionEntries;
+        });
+    }
+
+    return {
+        state,
+        async addSuppression(pattern: string, timeout: number) {
+            if (!pattern) {
+                throw new Error("Pattern is required");
+            }
+
+            let now = Date.now();
+            let newEntry = {
+                id: nextId(),
+                pattern,
+                timeout,
+                createdTime: now,
+                lastUpdatedTime: now,
+            };
+            let controller = ErrorNotificationsController(SocketFunction.browserNodeId());
+            await controller.setSuppressionEntry.promise(newEntry);
+            await reloadData();
+        },
+        async updateSuppression(id: string, pattern: string) {
+            if (!pattern) {
+                throw new Error("Pattern is required");
+            }
+
+            const existingEntry = Querysub.fastRead(() => state.suppressionEntries.find((s: { id: string; }) => s.id === id));
+            if (!existingEntry) {
+                throw new Error(`Suppression with id ${id} not found`);
+            }
+
+            let updatedEntry = {
+                id: existingEntry.id,
+                notes: existingEntry.notes,
+                pattern,
+                timeout: existingEntry.timeout,
+                createdTime: existingEntry.createdTime,
+                lastUpdatedTime: Date.now(),
+            };
+            let controller = ErrorNotificationsController(SocketFunction.browserNodeId());
+            await controller.setSuppressionEntry.promise(updatedEntry);
+            await reloadData();
+        },
+        async updateNotes(id: string, notes: string | undefined) {
+            const existingEntry = Querysub.fastRead(() => state.suppressionEntries.find((s: { id: string; }) => s.id === id));
+            if (!existingEntry) {
+                throw new Error(`Suppression with id ${id} not found`);
+            }
+
+            Querysub.commit(() => {
+                const idx = state.suppressionEntries.findIndex((s: { id: string; }) => s.id === id);
+                if (idx !== -1) {
+                    let updated = [...state.suppressionEntries];
+                    updated[idx] = { ...updated[idx], notes, lastUpdatedTime: Date.now() };
+                    state.suppressionEntries = updated;
+                }
+            });
+
+            let controller = ErrorNotificationsController(SocketFunction.browserNodeId());
+            await controller.updateSuppressionNotes.promise(id, notes);
+        },
+        async updateTimeout(id: string, timeout: number) {
+            const existingEntry = Querysub.fastRead(() => state.suppressionEntries.find((s: { id: string; }) => s.id === id));
+            if (!existingEntry) {
+                throw new Error(`Suppression with id ${id} not found`);
+            }
+
+            let updatedEntry = {
+                ...existingEntry,
+                timeout,
+                lastUpdatedTime: Date.now(),
+            };
+
+            let controller = ErrorNotificationsController(SocketFunction.browserNodeId());
+            await controller.setSuppressionEntry.promise(updatedEntry);
+            await reloadData();
+        },
+        async deleteSuppression(id: string) {
+            let controller = ErrorNotificationsController(SocketFunction.browserNodeId());
+            await controller.deleteSuppressionEntry.promise(id);
+            await reloadData();
+        },
+        async logTestError(message: string) {
+            let controller = ErrorNotificationsController(SocketFunction.browserNodeId());
+            await controller.logTestError.promise(message);
+        },
+    };
+});

package/src/diagnostics/logs/errorNotifications2/logWatcher.ts

@@ -0,0 +1,104 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { isNodeIdLocal, isOwnNodeId, watchDeltaNodeIds } from "../../../-f-node-discovery/NodeDiscovery";
+import { IndexedLogs, loggerByName } from "../IndexedLogs/IndexedLogs";
+import { nextId } from "socket-function/src/misc";
+import { assertIsManagementUser, isManagementUser } from "../../managementPages";
+import { ignoreErrors } from "../../../errors";
+import { blue } from "socket-function/src/formatting/logColors";
+import { SocketChannel } from "../../../functional/SocketChannel";
+import { batchFunction } from "socket-function/src/batching";
+import { isPublic } from "../../../config";
+
+export async function* watchAllValues<T>(logs: IndexedLogs<T>): AsyncGenerator<T> {
+    let callbackId = nextId();
+    let buffer: T[] = [];
+    let resolveNext: ((value: T) => void) | undefined;
+
+    clientCallbacks.set(callbackId, (datums: unknown[]) => {
+        for (let i = 1; i < datums.length; i++) {
+            buffer.push(datums[i] as T);
+        }
+        let datum = datums[0] as T;
+        if (resolveNext) {
+            resolveNext(datum as T);
+            resolveNext = undefined;
+        } else {
+            buffer.push(datum as T);
+        }
+    });
+
+    watchDeltaNodeIds((delta) => {
+        for (let nodeId of delta.newNodeIds) {
+            if (isOwnNodeId(nodeId)) continue;
+            let isNodeLocal = isNodeIdLocal(nodeId);
+            if (isNodeLocal !== isPublic()) continue;
+            ignoreErrors(ErrorNotificationControllerRegistered.nodes[nodeId].watchErrors({
+                loggerName: logs.config.name,
+                callbackId,
+            }));
+        }
+    });
+
+    try {
+        while (true) {
+            if (buffer.length > 0) {
+                yield buffer.shift()!;
+            } else {
+                yield await new Promise<T>((resolve) => {
+                    resolveNext = resolve;
+                });
+            }
+        }
+    } finally {
+        clientCallbacks.delete(callbackId);
+    }
+}
+
+
+
+let clientCallbacks = new Map<string, (datums: unknown[]) => void>();
+
+class ErrorNotificationController {
+    public async onError(config: {
+        callbackId: string;
+        datums: unknown[];
+    }) {
+        let callback = clientCallbacks.get(config.callbackId);
+        if (!callback) {
+            throw new Error(`Callback ${config.callbackId} not found`);
+        }
+        callback(config.datums);
+    }
+
+    public async watchErrors(config: {
+        loggerName: string;
+        callbackId: string;
+    }): Promise<void> {
+        let caller = SocketFunction.getCaller();
+        let logger = loggerByName.get(config.loggerName);
+        if (!logger) {
+            throw new Error(`Logger ${config.loggerName} not found`);
+        }
+
+        let unsubscribe = logger.watchErrors(batchFunction({ delay: 100 }, (datums: unknown[]) => {
+            ErrorNotificationControllerRegistered.nodes[caller.nodeId].onError({
+                callbackId: config.callbackId,
+                datums,
+            }).catch(() => {
+                unsubscribe();
+            });
+        }));
+    }
+}
+
+const ErrorNotificationControllerRegistered = SocketFunction.register(
+    "ErrorNotificationController-019c9cae-8333-7708-a4e7-500f5fc23173",
+    new ErrorNotificationController(),
+    () => ({
+        onError: {},
+        watchErrors: {}
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    })
+);

package/src/diagnostics/logs/errorNotifications2/openRouterHelper.ts

@@ -0,0 +1,77 @@
+export async function callOpenRouter(apiKey: string, prompt: string): Promise<string> {
+    const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
+        method: "POST",
+        headers: {
+            "Authorization": `Bearer ${apiKey}`,
+            "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+            model: "google/gemini-3-flash-preview",
+            messages: [
+                {
+                    role: "user",
+                    content: prompt,
+                },
+            ],
+        }),
+    });
+
+    if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(`OpenRouter API error: ${response.status} ${response.statusText} - ${errorText}`);
+    }
+
+    const data = await response.json();
+
+    if (!data.choices || !data.choices[0] || !data.choices[0].message || !data.choices[0].message.content) {
+        throw new Error(`Unexpected OpenRouter API response format: ${JSON.stringify(data)}`);
+    }
+
+    return data.choices[0].message.content.trim();
+}
+
+export async function callOpenRouterJSON<T>(apiKey: string, prompt: string): Promise<T> {
+    const maxRetries = 3;
+    let lastError: Error | undefined;
+
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+        try {
+            const response = await fetch("https://openrouter.ai/api/v1/chat/completions", {
+                method: "POST",
+                headers: {
+                    "Authorization": `Bearer ${apiKey}`,
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                    model: "google/gemini-3-flash-preview",
+                    messages: [
+                        {
+                            role: "user",
+                            content: prompt,
+                        },
+                    ],
+                    response_format: { type: "json_object" },
+                }),
+            });
+
+            if (!response.ok) {
+                const errorText = await response.text();
+                throw new Error(`OpenRouter API error: ${response.status} ${response.statusText} - ${errorText}`);
+            }
+
+            const data = await response.json();
+
+            if (!data.choices || !data.choices[0] || !data.choices[0].message || !data.choices[0].message.content) {
+                throw new Error(`Unexpected OpenRouter API response format: ${JSON.stringify(data)}`);
+            }
+
+            const content = data.choices[0].message.content.trim();
+            const parsed = JSON.parse(content) as T;
+            return parsed;
+        } catch (error) {
+            lastError = error instanceof Error ? error : new Error(String(error));
+        }
+    }
+
+    throw new Error(`Failed after ${maxRetries} attempts: ${lastError?.message}`);
+}

package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx

@@ -20,34 +20,22 @@ IMPORTANT! Now I am properly calling shutdown, so none of the streamed logs shou
 
 
 
+//todonext
 
-2) Create lot of remote server logs
-    - Via our refresh loop
+5) Deploy watcher service to remote (yarn error-watch-public)
+6) Deploy CYOA
 
-3) Verify search is still fast, and works correctly!
-    - Try to get a rare value in that is in the logs, but is hard to find
-    Good rare search. Checks for sending emails, which we do very rarely. If we pack in 10GB of logs after this, it will really stress our indexing system.
-        - At the moment our pending index has a good hit rate (0 false positives out of 1720). We'll see if this continues for our remote index.
-    https://127-0-0-1.querysubtest.com:7007/?hot&readPublicLogs&showingmanagement&machineview=services&managementpage=LogViewer3&searchText=Sending%20email&selectedFields=%7B%22path%22%3Atrue%7D
-    
+5) Test on actual server, with actual logs
 
 
-Rewrite error notification code
-    - Single service, which talks to all machines
-        - We'll have a dev mode, mostly for testing, although we won't usually run it.
-        - BROADCASTS messages to all http servers so they show them
-            - Broadcast messages only has one type now "error notification", but add TODO for more types, which may display in different ways (via toast, etc)
-    - Maybe only streaming? We'll miss some, but... not that much...
-    - ALSO owns the discord messaging code
-    - Suppression... will work in the service itself. Because there shouldn't be THAT many errors happening!
-    - Suppression will use AI to group?
-    - BUT, we will still have page that shows suppression list, with counts and times
-        - Page links to search, which tries to find those errors (we should use the same format for suppression as search, so it should be fairly easy to do that)
 
 
 
 
 0) Add LZ4 compression to socket-function by default
+    - Setup local tester for it to start
+        - with non-synced endpoint, just surely socket-function
+        - purely nodejs, just talking to itself
     - Allow setting "compress" to "none" or "lz4" or "zip" or "zip0" or "zip3", etc, for levels.
         - default is "lz4"
     - REQUIRES feature checking the remote, to make sure it is new enough to accept this.
@@ -76,6 +64,16 @@ Rewrite error notification code
 
 
 
+//todonext
+// Reorganize our lifecycles ideas
+//  - Scans over past logs, and only finished, never pending
+//         - Probably... realtime creating cache, BUT, if we add new lifecycles, we then only add them on demand. So when we are in the debugging mindset, and adding and changing them, it's fast (only as needed), but when we come back later when we aren't in the middle of debugging, it'll be super fast so we can fix the bug quickly and move on.
+//      - OR... we can do pending dynamically, but we don't cache it? Hmm... then what's the point? Hmm...
+//      - MAYBE, no caching, but we restrict the time by a lot? HMM...
+//      - I mean, we have to implement it without caching anyways to start, so we can just do that, and then cache it later if we see the need...
+//          - WELL, we need SOME kind of caching... Maybe... we DO use a strict time range, and then... if we've scanned a pending file in the past, we can cache that, because we know we've got all of it's values. Hmm...
+//  - Use lifecycles to debug rejections. To a point, until it is likely we are out of sync, then we should write the sync verification code.
+
 
 // todonext;
 // Hmm... so... should we index it, so we can search it? HMM... I think we might want to?

package/src/diagnostics/managementPages.tsx

@@ -89,6 +89,12 @@ export async function registerManagementPages2(config: {
         componentName: "LogViewer3",
         getModule: () => import("./logs/IndexedLogs/LogViewer3"),
     });
+    inputPages.push({
+        title: "Error Notifications",
+        componentName: "ErrorNotificationPage",
+        controllerName: "",
+        getModule: () => import("./logs/errorNotifications2/ErrorNotificationPage"),
+    });
     inputPages.push({
         title: "Security",
         componentName: "SecurityPage",
@@ -269,6 +275,8 @@ export function renderIsManagementUser() {
     return isManagementUserValue().value;
 }
 
+const ErrorWarning = createLazyComponent(() => import("./logs/errorNotifications2/ErrorWarning"))("ErrorWarning");
+
 class ManagementRoot extends qreact.Component {
     state = {
         ready: false
@@ -313,6 +321,7 @@ class ManagementRoot extends qreact.Component {
                         .color("white")
                         .color("hsl(0, 0%, 7%)")
                         .pointerEvents("none")
+                    + " ManagementRoot-page"
                 }>
                     <style>
                         {`
@@ -321,7 +330,8 @@ class ManagementRoot extends qreact.Component {
                             }
                         `}
                     </style>
-                    <div class={css.fillWidth.hbox(30, 10).wrap.hsl(245, 25, 60).pad2(10).pointerEvents("all")}>
+                    <div class={css.fillWidth.hbox(30, 10).wrap.hsl(245, 25, 80).pad2(10).pointerEvents("all")}>
+                        <ErrorWarning />
                         {pages.map(page =>
                             <ATag values={[{ param: managementPageURL, value: page.componentName }]}>{page.title}</ATag>
                         )}
@@ -331,7 +341,7 @@ class ManagementRoot extends qreact.Component {
                             class={
                                 css.fillBoth.vbox0.overflowAuto.hsl(145, 50, 75)
                                     .pointerEvents("all")
-                                + " ManagementRoot-page"
+
                             }
                             onKeyDown={e => {
                                 if (e.currentTarget === e.target) {

package/src/server.ts

@@ -13,16 +13,8 @@ import { formatTime } from "socket-function/src/formatting/format";
 //  we DO want to register some static diagnostics from Querysub, so... import it explicitly.
 import "./4-querysub/Querysub";
 import { getOurAuthorities } from "./config2";
-import { getDomain, isPublic } from "./config";
 
-import yargs from "yargs";
 import { Querysub } from "./4-querysub/Querysub";
-import { createSourceCheck } from "./user-implementation/canSeeSource";
-let yargObj = isNodeTrue() && yargs(process.argv)
-    .option("authority", { type: "string", desc: `Defines the base paths we are an authority on (the domain is prepended to them). Either a file path to a JSON(AuthorityPath[]), or a base64 representation of the JSON(AuthorityPath[]).` })
-    .option("verbose", { type: "boolean", desc: "Log all writes and reads" })
-    .argv || {}
-    ;
 
 logErrors(main());
 

package/src/user-implementation/SecurityPage.tsx

@@ -59,6 +59,12 @@ class ConfigTab extends qreact.Component {
                     value={user_data().secure.notifyDiscordWebhookURL}
                     onChangeValue={value => user_functions.setNotifyDiscordWebhookURL({ webhookURL: value })}
                 />
+                <InputLabel
+                    label={"OpenRouter API Key"}
+                    edit
+                    value={user_data().secure.openRouterAPIKey}
+                    onChangeValue={value => user_functions.setOpenRouterAPIKey({ apiKey: value })}
+                />
                 <Button onClick={() => user_functions.testSendDiscordMessage()}>
                     Test Discord Message
                 </Button>

package/src/user-implementation/userData.ts

@@ -13,7 +13,7 @@ import { MAX_ACCEPTED_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
 import { createURLSync } from "../library-components/URLParam";
 import { devDebugbreak, getDomain, getEmailDomain, isDevDebugbreak, isRecovery } from "../config";
 import { delay } from "socket-function/src/batching";
-import { enableErrorNotifications } from "../library-components/errorNotifications";
+import { enableErrorNotifications } from "../library-components/uncaughtToast";
 import { clamp } from "../misc";
 import { sha256 } from "js-sha256";
 import { logDisk } from "../diagnostics/logs/diskLogger";
@@ -166,6 +166,7 @@ const { data, functions } = Querysub.syncSchema<{
         signupsOpen?: boolean;
         postmarkAPIKey?: string;
         notifyDiscordWebhookURL?: string;
+        openRouterAPIKey?: string;
 
         // NOTE: For now these will only be blocked at a user level. If we run into issues, we can
         //  make Querysub have configurable IP blocking that accepts a synced read function, which
@@ -223,6 +224,7 @@ const { data, functions } = Querysub.syncSchema<{
         specialSetInviteCount,
         setPostmarkAPIKey,
         setNotifyDiscordWebhookURL,
+        setOpenRouterAPIKey,
         specialSetUserType,
         registerPageLoadTime,
         banUser, unbanUser,
@@ -978,6 +980,9 @@ function setPostmarkAPIKey(config: { apiKey: string; }) {
 function setNotifyDiscordWebhookURL(config: { webhookURL: string; }) {
     data().secure.notifyDiscordWebhookURL = config.webhookURL;
 }
+function setOpenRouterAPIKey(config: { apiKey: string; }) {
+    data().secure.openRouterAPIKey = config.apiKey;
+}
 
 
 function banUser(config: { userId: string; }) {

package/test.ts

@@ -1,8 +1,18 @@
+import { chdir } from "process";
+chdir("D:/repos/qs-cyoa/");
+
+import "./inject";
+
 import { Querysub } from "./src/4-querysub/QuerysubController";
-// Import querysub, so inject is run
-Querysub;
-import { testMain } from "./testEntry2";
+import { getErrorLogs, getLoggers2Async } from "./src/diagnostics/logs/diskLogger";
+import { watchAllValues } from "./src/diagnostics/logs/errorNotifications2/logWatcher";
+
+async function main() {
+    await Querysub.hostService("testwatcher");
+    let errorLogs = await getErrorLogs();
+    for await (let error of watchAllValues(errorLogs)) {
+        process.stdout.write(JSON.stringify(error) + "\n");
+    }
+}
 
-testMain().catch(console.error).finally(() => {
-    process.exit(0);
-});
+main().catch(console.error).finally(() => process.exit(0));

package/test2.ts

@@ -0,0 +1,20 @@
+import { chdir } from "process";
+chdir("D:/repos/qs-cyoa/");
+
+import "./inject";
+
+import { timeInSecond } from "socket-function/src/misc";
+import { getErrorLogs, getLoggers2Async } from "./src/diagnostics/logs/diskLogger";
+import { watchAllValues } from "./src/diagnostics/logs/errorNotifications2/logWatcher";
+import { delay } from "socket-function/src/batching";
+import { Querysub } from "./src/4-querysub/QuerysubController";
+
+async function main() {
+    await Querysub.hostService("test");
+    while (true) {
+        console.error("Test error");
+        await delay(timeInSecond * 5);
+    }
+}
+
+main().catch(console.error).finally(() => process.exit(0));

package/src/diagnostics/logs/errorNotifications2/errorNotifications2.ts

@@ -1,9 +0,0 @@
-
-//todonext
-
-// single service, no history search, discord messaging, suppression and suppression page
-//  - Storing suppression history just in memory? (although suppression values, obviously, on disk)
-//  - receive all errors, and service will do suppression
-
-// Start with page itself showing errors and suppressing them, for debugging/testing
-// - Ai to automatically create suppression searches, and combine existing ones, etc