querysub 0.327.0 → 0.329.0

package/src/functional/throttleRender.ts

@@ -0,0 +1,134 @@
+
+
+// NOTE: Many cases we don't know if we want to throttle, or how long we want to throttle for, until we start the watcher, so this has to be done inside our render, instead of in ProxyWatcher.
+
+import { cache } from "socket-function/src/caching";
+import { proxyWatcher, SyncWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { qreact } from "../4-dom/qreact";
+import { onNextPaint } from "./onNextPaint";
+
+// Throttles calls that have the same throttleKey
+/** Used near the start of your render, like so:
+        if (throttleRender({ key: "NodeControls", frameDelay: 0, frameDelaySmear: 60 })) return undefined;
+    - Doesn't break watches, so debug tools will still show you are watching them
+    - Doesn't break the rendered output (uses the last fully rendered output)
+    - Pauses rendering until the throttle finishes, then forcefully renders once.
+    - Always delays by at least 1 frame.
+*/
+export function throttleRender(config: {
+    // Throttles are smeared perkey
+    key: string;
+    // We pick a large value in this range as throttleKey overlaps, allowing you to smear a large number of changes over a larger period of time, BUT, while also waiting less time if there are few changes!
+    //  - Throttles won't necessarily finish in the same order you call them (we have to wrap around in the range after we hit the end, so later calls might re-trigger earlier
+    frameDelay: number;
+    // Range is [frameDelay, frameDelay + smear]
+    frameDelaySmear?: number;
+    // Delays for additional frames. Ex, set frameDelay to 0, smear to 60, and count to 30. The first delay is 0, then 30, then 60
+    frameDelayCount?: number;
+}): boolean {
+    let watcher = proxyWatcher.getTriggeredWatcher();
+    // Never throttle the first render, as that would be noticeable and always unintended
+    if (watcher.syncRunCount === 0) return false;
+    let { schedule, inSchedule, isTriggered, runScheduler } = throttleManager(config.key);
+    if (isTriggered(watcher)) {
+        //console.log(`Triggering throttle render for ${watcher.debugName} on frame ${getFrameNumber()}`);
+        return false;
+    }
+    runScheduler();
+    if (inSchedule.has(watcher)) {
+        //console.log(`Skipping throttle render for ${watcher.debugName} because it is already in the schedule`);
+        proxyWatcher.reuseLastWatches();
+        qreact.cancelRender();
+        return true;
+    }
+    inSchedule.add(watcher);
+    //console.log(`Adding throttle render for ${watcher.debugName}`);
+
+
+    let count = config.frameDelayCount || 1;
+    let smear = config.frameDelaySmear ?? 0;
+    let curIndex = config.frameDelay;
+    let endIndex = config.frameDelay + smear;
+    let targetFillCount = 0;
+    // Annoying algorithm to find the lowest slow available...
+    while (true) {
+        let cur = schedule[curIndex];
+        if ((cur?.length || 0) === targetFillCount) {
+            schedule[curIndex] = schedule[curIndex] || [];
+            schedule[curIndex]!.push(watcher);
+            for (let i = 1; i < count; i++) {
+                schedule[curIndex + i] = schedule[curIndex + i] || [];
+                schedule[curIndex + i]!.push("delay");
+            }
+            break;
+        }
+        curIndex++;
+        if (curIndex > endIndex) {
+            curIndex = config.frameDelay;
+            targetFillCount++;
+        }
+    }
+
+    proxyWatcher.reuseLastWatches();
+    qreact.cancelRender();
+    return true;
+}
+
+let throttleManager = cache((key: string) => {
+    let schedule: ((SyncWatcher | "delay")[] | undefined)[] = [];
+    let inSchedule = new Set<SyncWatcher>();
+    let triggered = new Set<SyncWatcher>();
+    let runningScheduler = false;
+    function isTriggered(watcher: SyncWatcher) {
+        return triggered.has(watcher);
+    }
+
+    return {
+        schedule,
+        inSchedule,
+        isTriggered,
+        runScheduler: () => {
+            if (runningScheduler) return;
+            runningScheduler = true;
+            void onNextPaint().finally(runNextTick);
+            function runNextTick() {
+                let next = schedule.shift();
+                void onNextPaint().finally(() => {
+                    triggered.clear();
+                });
+                if (next) {
+                    for (let watcher of next) {
+                        if (watcher === "delay") continue;
+                        inSchedule.delete(watcher);
+                        triggered.add(watcher);
+                        //console.log(`Triggering throttle render for ${watcher.debugName}`);
+                        watcher.explicitlyTrigger({
+                            paths: new Set(),
+                            pathSources: new Set(),
+                            newParentsSynced: new Set(),
+                            extraReasons: ["throttleRender trigger"],
+                        });
+                    }
+                }
+                if (schedule.length > 0) {
+                    void onNextPaint().finally(runNextTick);
+                } else {
+                    runningScheduler = false;
+                }
+            }
+        },
+    };
+});
+
+
+
+let rendersInFrame = 0;
+export function countRendersInFrame() {
+    if (rendersInFrame === 0) {
+        void onNextPaint().finally(() => {
+            //console.log(`Render in frame ${rendersInFrame}`);
+            rendersInFrame = 0;
+        });
+    }
+    rendersInFrame++;
+}

package/src/library-components/ATag.tsx

@@ -3,6 +3,7 @@ import { css, isNode } from "typesafecss";
 import { URLParam, parseSearchString, encodeSearchString } from "./URLParam";
 import { qreact } from "../4-dom/qreact";
 import { niceStringify } from "../niceStringify";
+import { getDomain } from "../config";
 
 export type URLOverride<T = unknown> = {
     param: URLParam<T>;
@@ -118,8 +119,7 @@ export const Anchor = ATag;
 export const Link = ATag;
 
 export function createLink(values: URLOverride[]) {
-    if (isNode()) throw new Error(`Create link is not implemented yet on node.`);
-    let urlObj = new URL(document.location.href);
+    let urlObj = new URL(isNode() ? "https://" + getDomain() : document.location.href);
     let params = parseSearchString(urlObj.search);
     for (let value of values) {
         params[value.param.urlKey] = value.value;

package/src/library-components/SyncedController.ts

@@ -109,7 +109,7 @@ export function getSyncedController<T extends SocketRegistered>(
     resetAll(): void;
     refreshAll(): void;
     isAnyLoading(): boolean;
-    base: T;
+    __baseController: T;
 } {
     if (isNode()) {
         let result = cache((nodeId: string) => {
@@ -161,7 +161,7 @@ export function getSyncedController<T extends SocketRegistered>(
         result.isAnyLoading = () => {
             notAllowedOnServer();
         };
-        result.base = controller;
+        result.__baseController = controller;
         return result;
     }
     let id = nextId();
@@ -416,6 +416,6 @@ export function getSyncedController<T extends SocketRegistered>(
             }
         });
     };
-    result.base = controller;
+    result.__baseController = controller;
     return result;
 }

package/src/misc.ts

@@ -126,6 +126,19 @@ export function partialCopyObject(data: unknown, maxFields: number = 500): unkno
 
             // Handle plain objects
             const result: Record<string, unknown> = {};
+
+            // Special case for Error objects - include stack and message
+            if (value instanceof Error) {
+                fieldCount++;
+                if (fieldCount < maxFields) {
+                    result.message = value.message;
+                }
+                fieldCount++;
+                if (fieldCount < maxFields) {
+                    result.stack = value.stack;
+                }
+            }
+
             for (const key of Object.keys(value as object)) {
                 fieldCount++;
                 if (fieldCount >= maxFields) break;
@@ -172,4 +185,9 @@ export function streamToIteratable<T>(reader: {
             }
         }
     };
+}
+
+const AsyncFunction = (async () => { }).constructor;
+export function isAsyncFunction(func: unknown): boolean {
+    return func instanceof AsyncFunction;
 }

package/src/misc2.ts

@@ -1,5 +1,111 @@
+import { delay } from "socket-function/src/batching";
+import { formatTime } from "socket-function/src/formatting/format";
+import { timeInHour } from "socket-function/src/misc";
 import { atomic } from "./2-proxy/PathValueProxyWatcher";
 
 export function isStrSimilar(a: string | undefined, b: string | undefined) {
     return atomic(a)?.toLowerCase().trim().replaceAll(" ", "") === atomic(b)?.toLowerCase().trim().replaceAll(" ", "");
+}
+
+export interface StreamLike {
+    on(event: "data", listener: (chunk: Buffer) => void): void;
+    on(event: "end", listener: () => void): void;
+    on(event: "error", listener: (error: Error) => void): void;
+}
+export async function* streamToAsyncIterable(stream: StreamLike): AsyncIterable<Buffer> {
+    let pendingChunks: Buffer[] = [];
+    let streamEnded = false;
+    let streamError: Error | undefined = undefined;
+    let resolveNext: (() => void) | undefined = undefined;
+
+    stream.on("data", (chunk: Buffer) => {
+        pendingChunks.push(chunk);
+        if (resolveNext) {
+            resolveNext();
+            resolveNext = undefined;
+        }
+    });
+
+    stream.on("end", () => {
+        streamEnded = true;
+        if (resolveNext) {
+            resolveNext();
+            resolveNext = undefined;
+        }
+    });
+
+    stream.on("error", (error) => {
+        streamError = error;
+        if (resolveNext) {
+            resolveNext();
+            resolveNext = undefined;
+        }
+    });
+
+    while (!streamEnded && !streamError) {
+        if (pendingChunks.length > 0) {
+            yield pendingChunks.shift()!;
+        } else {
+            await new Promise<void>((resolve) => {
+                resolveNext = resolve;
+            });
+        }
+    }
+
+    if (streamError) {
+        throw streamError;
+    }
+
+    while (pendingChunks.length > 0) {
+        yield pendingChunks.shift()!;
+    }
+}
+
+
+export async function runScheduler(hours: number[], func: () => Promise<void>) {
+    while (true) {
+        try {
+            const now = new Date();
+
+            // Find the next check time from our list
+            const sortedHours = [...hours].sort((a, b) => a - b);
+            let targetTime = new Date(now);
+
+            // Find the next check hour after the current time
+            let nextHour = sortedHours.find(hour => {
+                const checkTime = new Date(now);
+                checkTime.setHours(hour, 0, 0, 0);
+                return checkTime > now;
+            });
+
+            if (nextHour === undefined) {
+                // No more checks today, take the first one tomorrow
+                nextHour = sortedHours[0];
+                targetTime.setDate(targetTime.getDate() + 1);
+            }
+
+            targetTime.setHours(nextHour, 0, 0, 0);
+
+            console.log(`Next target time: ${targetTime.toLocaleString()}`);
+
+            // Wait and check every 15 minutes until it's time
+            while (true) {
+                const currentTime = new Date();
+                const timeUntilCheck = targetTime.getTime() - currentTime.getTime();
+
+                if (timeUntilCheck <= 0) {
+                    console.log("Time to run the check!");
+                    await func();
+                    break; // Break out of inner loop to calculate next check time
+                }
+
+                console.log(`Time until next check: ${formatTime(timeUntilCheck)}`);
+                await delay(timeInHour / 4); // Wait 15 minutes (1/4 of an hour)
+            }
+        } catch (error) {
+            console.error(error);
+            // Even if there's an error, wait 15 minutes before trying again
+            await delay(timeInHour / 4);
+        }
+    }
 }

package/src/user-implementation/SecurityPage.tsx

@@ -13,7 +13,7 @@ import { redButton } from "../library-components/colors";
 
 export class SecurityPage extends qreact.Component {
     render() {
-        return <div class={css.pad(10)}>
+        return <div class={css.pad(10).fillWidth}>
             <TabbedUI
                 tabs={[
                     { value: "config", title: "Config", contents: <ConfigTab /> },
@@ -49,13 +49,19 @@ class ConfigTab extends qreact.Component {
                 <InputLabel
                     label={"Postmark API Key"}
                     edit
-                    editClass={css.ellipsis.maxWidth("100px")}
+                    fillWidth
                     value={user_data().secure.postmarkAPIKey}
                     onChangeValue={value => user_functions.setPostmarkAPIKey({ apiKey: value })}
                 />
-                {isCurrentUserSuperUser() && <Button onClick={() => user_functions.test()}>
-                    test
-                </Button>}
+                <InputLabel
+                    label={"Discord Webhook URL"}
+                    edit
+                    value={user_data().secure.notifyDiscordWebhookURL}
+                    onChangeValue={value => user_functions.setNotifyDiscordWebhookURL({ webhookURL: value })}
+                />
+                <Button onClick={() => user_functions.testSendDiscordMessage()}>
+                    Test Discord Message
+                </Button>
             </>
         );
     }

package/src/user-implementation/userData.ts

@@ -6,17 +6,21 @@ import { registerAliveChecker } from "../2-proxy/garbageCollection";
 import { generateLoginEmail } from "./loginEmail";
 import { logErrors } from "../errors";
 import { PermissionsParameters } from "../3-path-functions/syncSchema";
-import { sendEmail_postmark } from "../email/postmark";
+import { sendEmail_postmark } from "../email_ims_notifications/postmark";
 import { isClient } from "../config2";
 import { getExternalIP } from "../misc/networking";
 import { MAX_ACCEPTED_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
 import { createURLSync } from "../library-components/URLParam";
-import { devDebugbreak, getDomain, isDevDebugbreak, isRecovery } from "../config";
+import { devDebugbreak, getDomain, getEmailDomain, isDevDebugbreak, isRecovery } from "../config";
 import { delay } from "socket-function/src/batching";
 import { enableErrorNotifications } from "../library-components/errorNotifications";
 import { clamp } from "../misc";
 import { sha256 } from "js-sha256";
 import { logDisk } from "../diagnostics/logs/diskLogger";
+import { createLink } from "../library-components/ATag";
+import { showingManagementURL, managementPageURL } from "../diagnostics/managementPages";
+import { sendDiscordMessage } from "../email_ims_notifications/discord";
+import { formatDateTime } from "socket-function/src/formatting/format";
 
 /*
     IMPORTANT!
@@ -161,6 +165,7 @@ const { data, functions } = Querysub.syncSchema<{
     secure: {
         signupsOpen?: boolean;
         postmarkAPIKey?: string;
+        notifyDiscordWebhookURL?: string;
 
         // NOTE: For now these will only be blocked at a user level. If we run into issues, we can
         //  make Querysub have configurable IP blocking that accepts a synced read function, which
@@ -217,18 +222,25 @@ const { data, functions } = Querysub.syncSchema<{
         specialBlockIP, specialUnblockIP,
         specialSetInviteCount,
         setPostmarkAPIKey,
+        setNotifyDiscordWebhookURL,
         specialSetUserType,
         registerPageLoadTime,
         banUser, unbanUser,
-        test: () => {
+        testSendDiscordMessage: () => {
             assertUserType("superuser");
-            let ip = Querysub.getCallerIP();
-            let machineId = Querysub.getCallerMachineId();
-            data().users["local"].allowedIPs[ip] = atomicObjectWrite({ time: Date.now(), machineId });
-            data().users["local"].machineIds[machineId] = atomicObjectWrite({ time: Date.now(), ip });
-            data().machineSecure[machineId].userId = "local";
-            data().users["local"].userId = "local";
-        }
+            const webhookURL = atomic(data().secure.notifyDiscordWebhookURL);
+            if (!webhookURL) {
+                throw new Error("No Discord webhook URL set");
+            }
+            let callerUserId = getCurrentUserAssert();
+            let email = data().users[callerUserId].email;
+            Querysub.onCommitFinished(async () => {
+                await sendDiscordMessage({
+                    webhookURL,
+                    message: `Discord webhook test at ${formatDateTime(Date.now())} by ${email} (${JSON.stringify(callerUserId)})`,
+                });
+            });
+        },
     },
     module,
     moduleId: "userData2",
@@ -269,11 +281,13 @@ const { data, functions } = Querysub.syncSchema<{
         },
     },
     functionMetadata: {
+        // There's no point to predict sending emails, or inviting users, or registering the page load time or sending discord messages, As the main function of these can only be done server-side. 
+        // verifyMachineId is used when logging in with the login token. We could predict it, but there's no real reason to because if we do, all of our reads are gonna fail until it runs, Which will actually be really confusing to the user and look like a bug. 
         sendLoginEmail: { nopredict: true },
         verifyMachineId: { nopredict: true },
         inviteUser: { nopredict: true },
-        setPostmarkAPIKey: { nopredict: true },
         registerPageLoadTime: { nopredict: true },
+        testSendDiscordMessage: { nopredict: true },
     },
 });
 
@@ -593,9 +607,14 @@ function sendLoginEmail(config: {
         const machineId = Querysub.getCallerMachineId();
         const now = Querysub.getCallTime();
 
+        // Check for the API key, even though we don't immediately use it, so that we can get good errors to tell the user (likely the developer) to setup the postmark API key. 
         const apiKey = atomic(data().secure.postmarkAPIKey);
         if (!apiKey) {
-            throw new Error(`No postmark API key, so we can't send login emails. Run "yarn setemailkey <key>" first.`);
+            let link = createLink([
+                showingManagementURL.getOverride(true),
+                managementPageURL.getOverride("SecurityPage"),
+            ]);
+            throw new Error(`No postmark API key, so we can't send login emails. Set the key at ${link}.`);
         }
 
         {
@@ -670,18 +689,36 @@ function sendLoginEmail(config: {
             timeoutTime,
         });
         Querysub.onCommitFinished(async () => {
-            await sendEmail_postmark({
-                apiKey,
-                to: email,
-                // TODO: Allow configuring this (defaulting to getDomain() if unconfigured). For now we hardcode it, because it takes
-                //  a while to verify a new postmark email, and we don't even know what final domain we will be using.
-                from: "login@querysub.com",
+            await sendEmail({
+                to: [email],
+                fromPrefix: "login",
                 subject,
                 contents,
             });
         });
     });
 }
+export async function sendEmail(config: {
+    to: string[];
+    // The domain should be getDomain
+    // TODO: Actually use git domain, for now it's hardcoded, because setting up a new   
+    fromPrefix: string;
+    subject: string;
+    contents: preact.VNode;
+}) {
+    let key = await Querysub.commitAsync(() => atomic(data().secure.postmarkAPIKey));
+    if (!key) {
+        console.warn(`No postmark API key setup, so we can't send email`);
+        return;
+    }
+    await sendEmail_postmark({
+        apiKey: key,
+        to: config.to,
+        from: `${config.fromPrefix}@${getEmailDomain()}`,
+        subject: config.subject,
+        contents: config.contents,
+    });
+}
 
 function verifyMachineId(config: {
     loginToken: string
@@ -938,11 +975,8 @@ export function scriptCreateUser(config: {
 function setPostmarkAPIKey(config: { apiKey: string; }) {
     data().secure.postmarkAPIKey = config.apiKey;
 }
-
-export function scriptSetPostmarkAPIKey(config: {
-    apiKey: string;
-}) {
-    setPostmarkAPIKey(config);
+function setNotifyDiscordWebhookURL(config: { webhookURL: string; }) {
+    data().secure.notifyDiscordWebhookURL = config.webhookURL;
 }
 
 

package/testEntry2.ts

@@ -5,21 +5,34 @@ import { shutdown } from "./src/diagnostics/periodic";
 import { testTCPIsListening } from "socket-function/src/networking";
 import { Querysub } from "./src/4-querysub/QuerysubController";
 import { timeInSecond } from "socket-function/src/misc";
+import { LOG_LINE_LIMIT_ID } from "./src/diagnostics/logs/diskLogger";
+import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
 
 export async function testMain() {
     Querysub;
+    await waitForFirstTimeSync();
+
     //let test = await testTCPIsListening("1.1.1.1", 443);
     //console.log(test);
     // Writing heartbeat 2025/09/14 08:37:46 PM for self (5ac8a2fa78fce4ea.971ed8b01743d123.querysubtest.com:13900)
     await delay(timeInSecond);
     await Querysub.hostService("test");
     await delay(timeInSecond * 5);
+
+    console.error(`A completely new error that is not suppressed. .`);
+    await delay(timeInSecond * 15);
+    await shutdown();
+    return;
+
+
     // console.log(getOwnThreadId());
     // Log an error every 30 seconds forever. 
     while (true) {
         console.error(`Test warning for im testing ${Date.now()}`);
         await delay(timeInSecond * 30);
     }
+
+
     // console.log(getOwnThreadId());
     // await shutdown();
     //await Querysub.hostService("test");
@@ -36,8 +49,4 @@ export async function testMain() {
     // }
     await delay(timeInSecond * 15);
     await shutdown();
-}
-async function main() {
-
-}
-
+}

package/src/user-implementation/setEmailKey.ts

@@ -1,25 +0,0 @@
-import { Querysub } from "../4-querysub/Querysub";
-import { pathValueCommitter } from "../0-path-value-core/PathValueCommitter";
-import { scriptSetPostmarkAPIKey } from "./userData";
-import { isNodeTrue } from "socket-function/src/misc";
-import yargs from "yargs";
-
-let yargObj = isNodeTrue() && yargs(process.argv)
-    .option("key", { type: "string", desc: `The key to set for the email.` })
-    .argv || {}
-    ;
-
-async function main() {
-    const howToCall = ` Call with yarn setemailkey --key <key>`;
-    const key = yargObj.key;
-    if (!key) throw new Error("No key provided." + howToCall);
-
-    await Querysub.hostService("setEmailKey");
-
-    await Querysub.commitSynced(() => {
-        scriptSetPostmarkAPIKey({ apiKey: key });
-    });
-
-    await pathValueCommitter.waitForValuesToCommit();
-}
-main().catch(e => console.log(e)).finally(() => process.exit());