querysub 0.327.0 → 0.328.0

package/src/functional/throttleRender.ts

@@ -0,0 +1,134 @@
+
+
+// NOTE: Many cases we don't know if we want to throttle, or how long we want to throttle for, until we start the watcher, so this has to be done inside our render, instead of in ProxyWatcher.
+
+import { cache } from "socket-function/src/caching";
+import { proxyWatcher, SyncWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { qreact } from "../4-dom/qreact";
+import { onNextPaint } from "./onNextPaint";
+
+// Throttles calls that have the same throttleKey
+/** Used near the start of your render, like so:
+        if (throttleRender({ key: "NodeControls", frameDelay: 0, frameDelaySmear: 60 })) return undefined;
+    - Doesn't break watches, so debug tools will still show you are watching them
+    - Doesn't break the rendered output (uses the last fully rendered output)
+    - Pauses rendering until the throttle finishes, then forcefully renders once.
+    - Always delays by at least 1 frame.
+*/
+export function throttleRender(config: {
+    // Throttles are smeared perkey
+    key: string;
+    // We pick a large value in this range as throttleKey overlaps, allowing you to smear a large number of changes over a larger period of time, BUT, while also waiting less time if there are few changes!
+    //  - Throttles won't necessarily finish in the same order you call them (we have to wrap around in the range after we hit the end, so later calls might re-trigger earlier
+    frameDelay: number;
+    // Range is [frameDelay, frameDelay + smear]
+    frameDelaySmear?: number;
+    // Delays for additional frames. Ex, set frameDelay to 0, smear to 60, and count to 30. The first delay is 0, then 30, then 60
+    frameDelayCount?: number;
+}): boolean {
+    let watcher = proxyWatcher.getTriggeredWatcher();
+    // Never throttle the first render, as that would be noticeable and always unintended
+    if (watcher.syncRunCount === 0) return false;
+    let { schedule, inSchedule, isTriggered, runScheduler } = throttleManager(config.key);
+    if (isTriggered(watcher)) {
+        //console.log(`Triggering throttle render for ${watcher.debugName} on frame ${getFrameNumber()}`);
+        return false;
+    }
+    runScheduler();
+    if (inSchedule.has(watcher)) {
+        //console.log(`Skipping throttle render for ${watcher.debugName} because it is already in the schedule`);
+        proxyWatcher.reuseLastWatches();
+        qreact.cancelRender();
+        return true;
+    }
+    inSchedule.add(watcher);
+    //console.log(`Adding throttle render for ${watcher.debugName}`);
+
+
+    let count = config.frameDelayCount || 1;
+    let smear = config.frameDelaySmear ?? 0;
+    let curIndex = config.frameDelay;
+    let endIndex = config.frameDelay + smear;
+    let targetFillCount = 0;
+    // Annoying algorithm to find the lowest slow available...
+    while (true) {
+        let cur = schedule[curIndex];
+        if ((cur?.length || 0) === targetFillCount) {
+            schedule[curIndex] = schedule[curIndex] || [];
+            schedule[curIndex]!.push(watcher);
+            for (let i = 1; i < count; i++) {
+                schedule[curIndex + i] = schedule[curIndex + i] || [];
+                schedule[curIndex + i]!.push("delay");
+            }
+            break;
+        }
+        curIndex++;
+        if (curIndex > endIndex) {
+            curIndex = config.frameDelay;
+            targetFillCount++;
+        }
+    }
+
+    proxyWatcher.reuseLastWatches();
+    qreact.cancelRender();
+    return true;
+}
+
+let throttleManager = cache((key: string) => {
+    let schedule: ((SyncWatcher | "delay")[] | undefined)[] = [];
+    let inSchedule = new Set<SyncWatcher>();
+    let triggered = new Set<SyncWatcher>();
+    let runningScheduler = false;
+    function isTriggered(watcher: SyncWatcher) {
+        return triggered.has(watcher);
+    }
+
+    return {
+        schedule,
+        inSchedule,
+        isTriggered,
+        runScheduler: () => {
+            if (runningScheduler) return;
+            runningScheduler = true;
+            void onNextPaint().finally(runNextTick);
+            function runNextTick() {
+                let next = schedule.shift();
+                void onNextPaint().finally(() => {
+                    triggered.clear();
+                });
+                if (next) {
+                    for (let watcher of next) {
+                        if (watcher === "delay") continue;
+                        inSchedule.delete(watcher);
+                        triggered.add(watcher);
+                        //console.log(`Triggering throttle render for ${watcher.debugName}`);
+                        watcher.explicitlyTrigger({
+                            paths: new Set(),
+                            pathSources: new Set(),
+                            newParentsSynced: new Set(),
+                            extraReasons: ["throttleRender trigger"],
+                        });
+                    }
+                }
+                if (schedule.length > 0) {
+                    void onNextPaint().finally(runNextTick);
+                } else {
+                    runningScheduler = false;
+                }
+            }
+        },
+    };
+});
+
+
+
+let rendersInFrame = 0;
+export function countRendersInFrame() {
+    if (rendersInFrame === 0) {
+        void onNextPaint().finally(() => {
+            //console.log(`Render in frame ${rendersInFrame}`);
+            rendersInFrame = 0;
+        });
+    }
+    rendersInFrame++;
+}

package/src/library-components/ATag.tsx

@@ -3,6 +3,7 @@ import { css, isNode } from "typesafecss";
 import { URLParam, parseSearchString, encodeSearchString } from "./URLParam";
 import { qreact } from "../4-dom/qreact";
 import { niceStringify } from "../niceStringify";
+import { getDomain } from "../config";
 
 export type URLOverride<T = unknown> = {
     param: URLParam<T>;
@@ -118,8 +119,7 @@ export const Anchor = ATag;
 export const Link = ATag;
 
 export function createLink(values: URLOverride[]) {
-    if (isNode()) throw new Error(`Create link is not implemented yet on node.`);
-    let urlObj = new URL(document.location.href);
+    let urlObj = new URL(isNode() ? "https://" + getDomain() : document.location.href);
     let params = parseSearchString(urlObj.search);
     for (let value of values) {
         params[value.param.urlKey] = value.value;

package/src/misc.ts

@@ -126,6 +126,19 @@ export function partialCopyObject(data: unknown, maxFields: number = 500): unkno
 
             // Handle plain objects
             const result: Record<string, unknown> = {};
+
+            // Special case for Error objects - include stack and message
+            if (value instanceof Error) {
+                fieldCount++;
+                if (fieldCount < maxFields) {
+                    result.message = value.message;
+                }
+                fieldCount++;
+                if (fieldCount < maxFields) {
+                    result.stack = value.stack;
+                }
+            }
+
             for (const key of Object.keys(value as object)) {
                 fieldCount++;
                 if (fieldCount >= maxFields) break;

package/src/misc2.ts

@@ -2,4 +2,58 @@ import { atomic } from "./2-proxy/PathValueProxyWatcher";
 
 export function isStrSimilar(a: string | undefined, b: string | undefined) {
     return atomic(a)?.toLowerCase().trim().replaceAll(" ", "") === atomic(b)?.toLowerCase().trim().replaceAll(" ", "");
+}
+
+export interface StreamLike {
+    on(event: "data", listener: (chunk: Buffer) => void): void;
+    on(event: "end", listener: () => void): void;
+    on(event: "error", listener: (error: Error) => void): void;
+}
+export async function* streamToAsyncIterable(stream: StreamLike): AsyncIterable<Buffer> {
+    let pendingChunks: Buffer[] = [];
+    let streamEnded = false;
+    let streamError: Error | undefined = undefined;
+    let resolveNext: (() => void) | undefined = undefined;
+
+    stream.on("data", (chunk: Buffer) => {
+        pendingChunks.push(chunk);
+        if (resolveNext) {
+            resolveNext();
+            resolveNext = undefined;
+        }
+    });
+
+    stream.on("end", () => {
+        streamEnded = true;
+        if (resolveNext) {
+            resolveNext();
+            resolveNext = undefined;
+        }
+    });
+
+    stream.on("error", (error) => {
+        streamError = error;
+        if (resolveNext) {
+            resolveNext();
+            resolveNext = undefined;
+        }
+    });
+
+    while (!streamEnded && !streamError) {
+        if (pendingChunks.length > 0) {
+            yield pendingChunks.shift()!;
+        } else {
+            await new Promise<void>((resolve) => {
+                resolveNext = resolve;
+            });
+        }
+    }
+
+    if (streamError) {
+        throw streamError;
+    }
+
+    while (pendingChunks.length > 0) {
+        yield pendingChunks.shift()!;
+    }
 }

package/src/user-implementation/SecurityPage.tsx

@@ -13,7 +13,7 @@ import { redButton } from "../library-components/colors";
 
 export class SecurityPage extends qreact.Component {
     render() {
-        return <div class={css.pad(10)}>
+        return <div class={css.pad(10).fillWidth}>
             <TabbedUI
                 tabs={[
                     { value: "config", title: "Config", contents: <ConfigTab /> },
@@ -49,13 +49,19 @@ class ConfigTab extends qreact.Component {
                 <InputLabel
                     label={"Postmark API Key"}
                     edit
-                    editClass={css.ellipsis.maxWidth("100px")}
+                    fillWidth
                     value={user_data().secure.postmarkAPIKey}
                     onChangeValue={value => user_functions.setPostmarkAPIKey({ apiKey: value })}
                 />
-                {isCurrentUserSuperUser() && <Button onClick={() => user_functions.test()}>
-                    test
-                </Button>}
+                <InputLabel
+                    label={"Discord Webhook URL"}
+                    edit
+                    value={user_data().secure.notifyDiscordWebhookURL}
+                    onChangeValue={value => user_functions.setNotifyDiscordWebhookURL({ webhookURL: value })}
+                />
+                <Button onClick={() => user_functions.testSendDiscordMessage()}>
+                    Test Discord Message
+                </Button>
             </>
         );
     }

package/src/user-implementation/userData.ts

@@ -6,7 +6,7 @@ import { registerAliveChecker } from "../2-proxy/garbageCollection";
 import { generateLoginEmail } from "./loginEmail";
 import { logErrors } from "../errors";
 import { PermissionsParameters } from "../3-path-functions/syncSchema";
-import { sendEmail_postmark } from "../email/postmark";
+import { sendEmail_postmark } from "../email_ims_notifications/postmark";
 import { isClient } from "../config2";
 import { getExternalIP } from "../misc/networking";
 import { MAX_ACCEPTED_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
@@ -17,6 +17,10 @@ import { enableErrorNotifications } from "../library-components/errorNotificatio
 import { clamp } from "../misc";
 import { sha256 } from "js-sha256";
 import { logDisk } from "../diagnostics/logs/diskLogger";
+import { createLink } from "../library-components/ATag";
+import { showingManagementURL, managementPageURL } from "../diagnostics/managementPages";
+import { sendDiscordMessage } from "../email_ims_notifications/discord";
+import { formatDateTime } from "socket-function/src/formatting/format";
 
 /*
     IMPORTANT!
@@ -161,6 +165,7 @@ const { data, functions } = Querysub.syncSchema<{
     secure: {
         signupsOpen?: boolean;
         postmarkAPIKey?: string;
+        notifyDiscordWebhookURL?: string;
 
         // NOTE: For now these will only be blocked at a user level. If we run into issues, we can
         //  make Querysub have configurable IP blocking that accepts a synced read function, which
@@ -217,18 +222,25 @@ const { data, functions } = Querysub.syncSchema<{
         specialBlockIP, specialUnblockIP,
         specialSetInviteCount,
         setPostmarkAPIKey,
+        setNotifyDiscordWebhookURL,
         specialSetUserType,
         registerPageLoadTime,
         banUser, unbanUser,
-        test: () => {
+        testSendDiscordMessage: () => {
             assertUserType("superuser");
-            let ip = Querysub.getCallerIP();
-            let machineId = Querysub.getCallerMachineId();
-            data().users["local"].allowedIPs[ip] = atomicObjectWrite({ time: Date.now(), machineId });
-            data().users["local"].machineIds[machineId] = atomicObjectWrite({ time: Date.now(), ip });
-            data().machineSecure[machineId].userId = "local";
-            data().users["local"].userId = "local";
-        }
+            const webhookURL = atomic(data().secure.notifyDiscordWebhookURL);
+            if (!webhookURL) {
+                throw new Error("No Discord webhook URL set");
+            }
+            let callerUserId = getCurrentUserAssert();
+            let email = data().users[callerUserId].email;
+            Querysub.onCommitFinished(async () => {
+                await sendDiscordMessage({
+                    webhookURL,
+                    message: `Discord webhook test at ${formatDateTime(Date.now())} by ${email} (${JSON.stringify(callerUserId)})`,
+                });
+            });
+        },
     },
     module,
     moduleId: "userData2",
@@ -269,11 +281,13 @@ const { data, functions } = Querysub.syncSchema<{
         },
     },
     functionMetadata: {
+        // There's no point to predict sending emails, or inviting users, or registering the page load time or sending discord messages, As the main function of these can only be done server-side. 
+        // verifyMachineId is used when logging in with the login token. We could predict it, but there's no real reason to because if we do, all of our reads are gonna fail until it runs, Which will actually be really confusing to the user and look like a bug. 
         sendLoginEmail: { nopredict: true },
         verifyMachineId: { nopredict: true },
         inviteUser: { nopredict: true },
-        setPostmarkAPIKey: { nopredict: true },
         registerPageLoadTime: { nopredict: true },
+        testSendDiscordMessage: { nopredict: true },
     },
 });
 
@@ -595,7 +609,11 @@ function sendLoginEmail(config: {
 
         const apiKey = atomic(data().secure.postmarkAPIKey);
         if (!apiKey) {
-            throw new Error(`No postmark API key, so we can't send login emails. Run "yarn setemailkey <key>" first.`);
+            let link = createLink([
+                showingManagementURL.getOverride(true),
+                managementPageURL.getOverride("SecurityPage"),
+            ]);
+            throw new Error(`No postmark API key, so we can't send login emails. Set the key at ${link}.`);
         }
 
         {
@@ -938,11 +956,8 @@ export function scriptCreateUser(config: {
 function setPostmarkAPIKey(config: { apiKey: string; }) {
     data().secure.postmarkAPIKey = config.apiKey;
 }
-
-export function scriptSetPostmarkAPIKey(config: {
-    apiKey: string;
-}) {
-    setPostmarkAPIKey(config);
+function setNotifyDiscordWebhookURL(config: { webhookURL: string; }) {
+    data().secure.notifyDiscordWebhookURL = config.webhookURL;
 }
 
 

package/testEntry2.ts

@@ -5,21 +5,34 @@ import { shutdown } from "./src/diagnostics/periodic";
 import { testTCPIsListening } from "socket-function/src/networking";
 import { Querysub } from "./src/4-querysub/QuerysubController";
 import { timeInSecond } from "socket-function/src/misc";
+import { LOG_LINE_LIMIT_ID } from "./src/diagnostics/logs/diskLogger";
+import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
 
 export async function testMain() {
     Querysub;
+    await waitForFirstTimeSync();
+
     //let test = await testTCPIsListening("1.1.1.1", 443);
     //console.log(test);
     // Writing heartbeat 2025/09/14 08:37:46 PM for self (5ac8a2fa78fce4ea.971ed8b01743d123.querysubtest.com:13900)
     await delay(timeInSecond);
     await Querysub.hostService("test");
     await delay(timeInSecond * 5);
+
+    console.error(`This should show up locally, but not remotely.`);
+    await delay(timeInSecond * 15);
+    await shutdown();
+    return;
+
+
     // console.log(getOwnThreadId());
     // Log an error every 30 seconds forever. 
     while (true) {
         console.error(`Test warning for im testing ${Date.now()}`);
         await delay(timeInSecond * 30);
     }
+
+
     // console.log(getOwnThreadId());
     // await shutdown();
     //await Querysub.hostService("test");
@@ -36,8 +49,4 @@ export async function testMain() {
     // }
     await delay(timeInSecond * 15);
     await shutdown();
-}
-async function main() {
-
-}
-
+}

package/src/user-implementation/setEmailKey.ts

@@ -1,25 +0,0 @@
-import { Querysub } from "../4-querysub/Querysub";
-import { pathValueCommitter } from "../0-path-value-core/PathValueCommitter";
-import { scriptSetPostmarkAPIKey } from "./userData";
-import { isNodeTrue } from "socket-function/src/misc";
-import yargs from "yargs";
-
-let yargObj = isNodeTrue() && yargs(process.argv)
-    .option("key", { type: "string", desc: `The key to set for the email.` })
-    .argv || {}
-    ;
-
-async function main() {
-    const howToCall = ` Call with yarn setemailkey --key <key>`;
-    const key = yargObj.key;
-    if (!key) throw new Error("No key provided." + howToCall);
-
-    await Querysub.hostService("setEmailKey");
-
-    await Querysub.commitSynced(() => {
-        scriptSetPostmarkAPIKey({ apiKey: key });
-    });
-
-    await pathValueCommitter.waitForValuesToCommit();
-}
-main().catch(e => console.log(e)).finally(() => process.exit());