querysub 0.326.0 → 0.328.0

package/src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx

@@ -1,75 +1,221 @@
 import { batchFunction, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { getControllerNodeId } from "../../../-g-core-values/NodeCapabilities";
-import { RecentErrorsController, recentErrorsChannel, watchRecentErrors } from "./ErrorNotificationController";
-import { timeInSecond } from "socket-function/src/misc";
-import { formatDateTime } from "socket-function/src/formatting/format";
+import { RecentErrors, RecentErrorsController, errorWatcherBase, recentErrorsChannel, suppressionList, watchRecentErrors } from "./ErrorNotificationController";
+import { sort, timeInDay, timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { formatDate, formatDateTime } from "socket-function/src/formatting/format";
+import { getDomain } from "../../../config";
+import { Querysub, QuerysubController } from "../../../4-querysub/QuerysubController";
+import { formatPercent } from "socket-function/src/formatting/format";
+import { LogDatum, getLogFile } from "../diskLogger";
+import { sendDiscordMessage } from "../../../email_ims_notifications/discord";
+import { user_data } from "../../../user-implementation/userData";
+import { createLink } from "../../../library-components/ATag";
+import { getLogsLinkParts } from "./ErrorWarning";
 
+const MAX_IMS_PER_DAY = 3;
+const MAX_IMS_PER_HOURS = 1;
+const MAX_IMS_PER_FILE_PER_DAY = 1;
+const MAX_PER_FILE_PER_IM = 2;
+const MAX_PER_IM = 10;
 
+// Wait a bit, because it's likely if there's one error, there are more errors. 
+const BATCH_TIME = timeInSecond * 30;
+const DIGEST_INTERVAL = timeInDay * 3;
 
-// 3) Dismissing of certain errors is not working. They keep coming back.
-//      - I think our expiry date comparison code might be wrong. It seems like once they leave the maybe expired range they come back immediately. If we can reproduce this locally, it'll be trivial to debug because the suppression stuff is global, so we can just see if there are any errors, and if there are, we break in on them. 
-// UGH... To debug this, we need to ignore the changes and then we need to work on the other stuff and then we need to come back later and see if those changes have shown up again. We basically need to debug it when it happens. We can't debug it now. It's too late, Now the errors should be showing up because they are expired. 
 
+// 4) Deploy
+// 4) Create local errors and make sure the remote server doesn't get them 
+// 5) Create remote errors and make sure that the all servers get them (Local and remote servers)
 
-// The constant error notifications might be fixed now. We'll see tomorrow after all the rolling updates finish. 
-// 4) fix whatever's causing constant error notifications. Something is broadcasting on the Recent Errors Change channel constantly. 
-//      - I guess usually there's no server that's going to be listening on it. So it's... Not that big of a deal, but it's still annoying. 
 
+// Abstract out RecentErrors scanning, so we can reuse it
+//  (appendables, timeRange, callback)
+// - Oh, we also need a way to tell it to not limit to recent errors.
+// - And also to disable throttling. 
 
-//  4.1) update channel watching so you can specify that you want to watch only on a specific node ID and then update our code so we only watch it on the controller node ID that we're interfacing with. 
 
+// createDigest(appendables, timeRange) => Promise<DigestInfo>
 
-// 5) Verify our suppression updates broadcast across the channel correctly, causing us to be able to suppress a notification and our watching script to then stop seeing the new updates. Realistically, it's the calling script that stops setting them, but same thing. 
-
+// 7) Write the digest script, which does a fresh scan on just a specific range of data using start synchronize internal and suppression list, etc. etc. 
+//      - The most recent ten unsuppressed errors. Counts of unsuppressed errors by file. Or by line id? If they have it. Counts of suppressed and unsuppressed errors by hour group. 
+//      - Oh, also everything we do for errors do for warnings as well, and split the values. We want warnings and errors. 
+//      - Time the entire thing, and put that, and the profile, in the digest too! That will give us a good gauge on if the errors/suppressions are getting slow (due to a lot of errors, or a lot of suppression checks!)
+// store the digest results in Back Blaze. 
+// 8) Write a page that will show the results from Backblaze, allowing you to see a list of all of the digests and to view a specific one. Just showing all the data in a chart, whatever. Showed two charts, one with just the unsuppressed errors, and the other with both suppressed and unsuppressed, because you really only care about suppressed when you're looking at the fraction of suppressed to unsuppressed, and the sheer quantity of unsuppressed. 
+// 9) send an email every time period, and also send an IM that has smaller information
+//      - Both will link to the actual web page that has the digest, deep linking to the specific tabs. 
+//      - Show the chart in the email as well, but just format it like ASCII Because image dependencies are annoying and I don't want to implement them right now as it might take a few days to get working. 
 
 
-// 5) Set up the Instant Messaging Sending API. 
-//  - Discord. With beeper it won't really matter what we're messaging. We could also do WhatsApp. It's really all the same.  
 
-// 6) Set up all the code to properly rate limit IMs, batch them, link back to the log page, etc. 
-//      - Just link to the error page for the last week. We don't need to link to anything specific. 
-//      - properly getting the node ID that we're going to be watching, and if it goes down, getting a new one, and ignoring messages from the old node. 
-//      - And if no node exists, we need to warn and then wait. 
+// NOTE: Yes, this is stored in memory, so if the server reboots or if this script keeps crashing, we might send a lot of instant messages. However, one, Discord will probably late rate limit us, and two, this means something is really wrong, especially if it happens a lot, and we really should fix it right away. 
+// NOTE: If we decide not to send IMs, we don't queue them up, because that's just extremely annoying. The point of the limit isn't to send the maximum we can that stays just under the limit! The point of the limit is if a lot happens at once, ignore most of it. 
+type IMInfo = {
+    time: number;
+    perFile: {
+        [file: string]: LogDatum[];
+    };
+};
+
+let imHistory: IMInfo[] = [];
+function filterIMInfo(info: IMInfo): {
+    info: IMInfo;
+    countFiltered: number;
+} {
+    let countFiltered = 0;
+    // Don't prefer the most warnings, prefer The oldest warnings, which are generally the ones that are first in the object. 
+    // filter based on files per im, and max per im
+    for (let [key, value] of Object.entries(info.perFile)) {
+        if (value.length > MAX_PER_FILE_PER_IM) {
+            countFiltered += value.length - MAX_PER_FILE_PER_IM;
+            value = value.slice(0, MAX_PER_FILE_PER_IM);
+        }
+        info.perFile[key] = value;
+    }
+    let entries = Object.entries(info.perFile);
+    if (entries.length > MAX_PER_IM) {
+        let removed = entries.slice(MAX_PER_IM);
+        for (let [key, value] of removed) {
+            countFiltered += value.length;
+        }
+        entries = entries.slice(0, MAX_PER_IM);
+    }
+    info.perFile = Object.fromEntries(entries);
+    // Also ignore files if they've been mentioned too many times today. 
+    let dayThreshold = Date.now() - timeInDay;
+    let historyInDay = imHistory.filter(x => x.time > dayThreshold);
+    let countByFile = new Map<string, number>();
+    for (let obj of historyInDay) {
+        for (let [key, value] of Object.entries(obj.perFile)) {
+            countByFile.set(key, (countByFile.get(key) || 0) + value.length);
+        }
+    }
+    for (let key of Object.keys(info.perFile)) {
+        let count = countByFile.get(key) || 0;
+        if (count >= MAX_IMS_PER_FILE_PER_DAY) {
+            countFiltered += count;
+            delete info.perFile[key];
+        }
+    }
+    return {
+        info,
+        countFiltered,
+    };
+}
+function canSendNow(info: IMInfo) {
+    let dayThreshold = Date.now() - timeInDay;
+    let historyInDay = imHistory.filter(x => x.time > dayThreshold);
+    let hourThreshold = Date.now() - timeInHour;
+    let historyInHour = historyInDay.filter(x => x.time > hourThreshold);
+
+    if (historyInDay.length >= MAX_IMS_PER_DAY) {
+        return false;
+    }
+    if (historyInHour.length >= MAX_IMS_PER_HOURS) {
+        return false;
+    }
+    return true;
+}
 
+const sendIMs = batchFunction(({ delay: BATCH_TIME }), async (logsAll: LogDatum[][]) => {
+    let logs = logsAll.flat();
+    let infoBase: IMInfo = {
+        time: Date.now(),
+        perFile: {},
+    };
+    for (let log of logs) {
+        let file = getLogFile(log);
+        let array = infoBase.perFile[file];
+        if (!array) {
+            array = [];
+            infoBase.perFile[file] = array;
+        }
+        array.push(log);
+    }
+    let { info, countFiltered } = filterIMInfo(infoBase);
+    if (canSendNow(info)) {
+        imHistory.push(info);
+        let webhookURL = await Querysub.commitAsync(() => {
+            return user_data().secure.notifyDiscordWebhookURL;
+        });
+        if (!webhookURL) {
+            console.error(`No Discord webhook URL set, cannot send warning instant messages`);
+            return;
+        }
+        let url = createLink(getLogsLinkParts());
+        let message = Object.values(info.perFile).flat().map(
+            x => `[${formatDateTime(x.time)}](${url}) | ${x.param0} (${x.__NAME__})`
+        ).join("\n");
+        if (countFiltered > 0) {
+            message += `\n+${countFiltered} more errors`;
+        }
+        void sendDiscordMessage({
+            webhookURL,
+            message,
+        });
+    }
+});
 
-// 7) Write the digest script, which is very different, but will run in the same entry.
-//      - Separate warnings and errors and also bucket by time bucket
-//      - suppressed errors by time bucket (but no type, as we definitely don't want to parse all suppressed errors...)
-//      - Time the entire thing, and put that, and the profile, in the digest too! That will give us a good gauge on if the errors/suppressions are getting slow (due to a lot of errors, or a lot of suppression checks!)
-// 8) Write a page that shows the results of the digest in tabs, writing the digest probably just to backblaze 
-//      - For now, just have two tabs, one for errors and one for warnings. 
-//      - If we're going to do a full scan, we might as well show time series data as well. It's trivial. 
-//      - Also track the number of suppressed errors as well. We won't have details on these such as a breakdown, but we can at least show the count (and the count by time)
-// 9) send an email every time period, and also send an IM that has smaller information
-//      - Both will link to the actual web page that has the digest, deep linking to the specific tabs. 
-//      - Show the chart in the email as well, but just format it like ASCII Because image dependencies are annoying and I don't want to implement them right now as it might take a few days to get working. 
 
 async function runIMNotifies() {
-    let controllerNodeId = await getControllerNodeId(RecentErrorsController.base);
-    if (!controllerNodeId) throw new Error("No controller node id found");
-    //todonext
-    //  Temporary hardcode to use the local server
-    controllerNodeId = "127-0-0-1.querysubtest.com:7007";
-
-    let controller = RecentErrorsController.base.nodes[controllerNodeId];
-    recentErrorsChannel.watch(() => {
-        void updateRecentErrors(undefined);
-    });
-    const updateRecentErrors = batchFunction(
-        //todonext
-        //  Increase this after we finish testing
-        { delay: 1000 },
-        async function updateRecentErrors() {
-            let recentErrors = await controller.getRecentErrors();
-            console.log(`Received ${recentErrors.length} recent errors at ${formatDateTime(Date.now())}`);
-            for (let error of recentErrors) {
-                console.log(`    ${error.param0}`);
+    await Querysub.hostService("error-notifications");
+
+    // NOTE: This should be fine, as realistically how many errors are we going to see in the last two weeks. At 10 million, we're still probably only going to allocate 160 megs of memory, assuming we allocate 16 bytes per number. If we have more than 16 million for a single thread, it'll fail because the max set size... However, they will likely be somewhat distributed between threads. 
+    let errorByThreadIdByDay = new Map<number, Map<string, Set<number>>>();
+    function getDay(time: number) {
+        return Math.floor(time / timeInDay) * timeInDay;
+    }
+    function isDuplicate(obj: LogDatum): boolean {
+        // Checks if it's a duplicate, and if it's not, adds it. 
+        let day = getDay(obj.time);
+        let threadId = obj.__threadId || "";
+        let dayMap = errorByThreadIdByDay.get(day);
+        if (!dayMap) {
+            dayMap = new Map<string, Set<number>>();
+            errorByThreadIdByDay.set(day, dayMap);
+        }
+        let threadIdMap = dayMap.get(threadId);
+        if (!threadIdMap) {
+            threadIdMap = new Set<number>();
+            dayMap.set(threadId, threadIdMap);
+        }
+        if (threadIdMap.has(obj.time)) {
+            return true;
+        }
+        threadIdMap.add(obj.time);
+        return false;
+
+    }
+    function clearOldDays() {
+        // Clear all the days that are more than 14 days older than our current day.
+        let now = Date.now();
+        let currentDay = getDay(now);
+        let cutOffDay = currentDay - 14;
+        for (let day of errorByThreadIdByDay.keys()) {
+            if (day < cutOffDay) {
+                errorByThreadIdByDay.delete(day);
+                console.log(`Cleared old day ${formatDateTime(day)}`);
             }
-            console.log();
-            console.log();
         }
-    );
-    await updateRecentErrors(undefined);
+    }
+    errorWatcherBase.watch(async (objs) => {
+        clearOldDays();
+        objs = await suppressionList.filterObjsToNonSuppressed(objs);
+        objs = objs.filter(x => !isDuplicate(x));
+        if (objs.length === 0) return;
+        // The oldest first as they are most likely the cause. 
+        sort(objs, x => -x.time);
+        void sendIMs(objs);
+        console.log();
+        console.log();
+        console.log(`Received ${objs.length} recent errors at ${formatDateTime(Date.now())}`);
+        for (let obj of objs) {
+            console.log(`    ${obj.param0}`);
+        }
+        console.log();
+        console.log();
+    });
+
 }
 
 async function main() {

package/src/diagnostics/logs/lifeCycleAnalysis/spec.md

@@ -91,8 +91,9 @@ Make sure we check our life cycles for nodes being added and removed to make sur
 
 Check the startup lifecycle to make sure we can detect the nodes pretty fast and in parallel, instead of serial
 
-10) Verify old user/fast-log-cache machine folders are deleted
-
+11) Take all of the errors that we are ignoring and use the life cycles to detect why they're happening. A lot of them really shouldn't be happening. 
+    - Receiving values from different authorities and the ones we're watching is weird. Why does that keep happening?
+    - And we keep running into audit mismatches? Why does that keep happening? Is it only because of our local development server? 
 
 SPECIAL UI links for certain errors in log view
     - Probably dynamically created, based on contents of log

package/src/diagnostics/managementPages.tsx

@@ -90,6 +90,11 @@ export async function registerManagementPages2(config: {
         componentName: "LogViewer2",
         getModule: () => import("./logs/LogViewer2"),
     });
+    inputPages.push({
+        title: "Security",
+        componentName: "SecurityPage",
+        getModule: () => import("../user-implementation/SecurityPage"),
+    });
     inputPages.push({
         title: "Audit Paths",
         componentName: "AuditLogPage",

package/src/email_ims_notifications/discord.tsx

@@ -0,0 +1,203 @@
+import { httpsRequest } from "../https";
+
+export interface DiscordEmbedFooter {
+    text: string;
+    icon_url?: string;
+}
+
+export interface DiscordEmbedImage {
+    url: string;
+}
+
+export interface DiscordEmbedThumbnail {
+    url: string;
+}
+
+export interface DiscordEmbedVideo {
+    url: string;
+}
+
+export interface DiscordEmbedProvider {
+    name?: string;
+    url?: string;
+}
+
+export interface DiscordEmbedAuthor {
+    name: string;
+    url?: string;
+    icon_url?: string;
+}
+
+export interface DiscordEmbedField {
+    name: string;
+    value: string;
+    inline?: boolean;
+}
+
+export interface DiscordEmbed {
+    title?: string;
+    type?: "rich" | "image" | "video" | "gifv" | "article" | "link";
+    description?: string;
+    url?: string;
+    timestamp?: string; // ISO8601 timestamp
+    color?: number; // integer color value
+    footer?: DiscordEmbedFooter;
+    image?: DiscordEmbedImage;
+    thumbnail?: DiscordEmbedThumbnail;
+    video?: DiscordEmbedVideo;
+    provider?: DiscordEmbedProvider;
+    author?: DiscordEmbedAuthor;
+    fields?: DiscordEmbedField[];
+}
+
+export interface DiscordAllowedMentions {
+    parse?: ("roles" | "users" | "everyone")[];
+    roles?: string[];
+    users?: string[];
+    replied_user?: boolean;
+}
+
+export interface DiscordAttachment {
+    id: string;
+    filename: string;
+    description?: string;
+    content_type?: string;
+    size: number;
+    url: string;
+    proxy_url: string;
+}
+
+export interface DiscordEmoji {
+    id?: string;
+    name?: string;
+    animated?: boolean;
+}
+
+export interface DiscordSelectOption {
+    label: string;
+    value: string;
+    description?: string;
+    emoji?: DiscordEmoji;
+    default?: boolean;
+}
+
+export interface DiscordButtonComponent {
+    type: 2; // Button component type
+    style: 1 | 2 | 3 | 4 | 5; // Primary, Secondary, Success, Danger, Link
+    label?: string;
+    emoji?: DiscordEmoji;
+    custom_id?: string; // Required for non-link buttons
+    url?: string; // Required for link buttons
+    disabled?: boolean;
+}
+
+export interface DiscordSelectMenuComponent {
+    type: 3; // String select menu component type
+    custom_id: string;
+    options: DiscordSelectOption[];
+    placeholder?: string;
+    min_values?: number;
+    max_values?: number;
+    disabled?: boolean;
+}
+
+export interface DiscordUserSelectMenuComponent {
+    type: 5; // User select menu component type
+    custom_id: string;
+    placeholder?: string;
+    min_values?: number;
+    max_values?: number;
+    disabled?: boolean;
+}
+
+export interface DiscordRoleSelectMenuComponent {
+    type: 6; // Role select menu component type
+    custom_id: string;
+    placeholder?: string;
+    min_values?: number;
+    max_values?: number;
+    disabled?: boolean;
+}
+
+export interface DiscordMentionableSelectMenuComponent {
+    type: 7; // Mentionable (users + roles) select menu component type
+    custom_id: string;
+    placeholder?: string;
+    min_values?: number;
+    max_values?: number;
+    disabled?: boolean;
+}
+
+export interface DiscordChannelSelectMenuComponent {
+    type: 8; // Channel select menu component type
+    custom_id: string;
+    placeholder?: string;
+    min_values?: number;
+    max_values?: number;
+    disabled?: boolean;
+    channel_types?: number[]; // Array of channel type integers to filter by
+}
+
+export interface DiscordTextInputComponent {
+    type: 4; // Text input component type
+    custom_id: string;
+    style: 1 | 2; // Short (1) or Paragraph (2)
+    label: string;
+    min_length?: number;
+    max_length?: number;
+    required?: boolean;
+    value?: string;
+    placeholder?: string;
+}
+
+export type DiscordComponent =
+    | DiscordButtonComponent
+    | DiscordSelectMenuComponent
+    | DiscordUserSelectMenuComponent
+    | DiscordRoleSelectMenuComponent
+    | DiscordMentionableSelectMenuComponent
+    | DiscordChannelSelectMenuComponent
+    | DiscordTextInputComponent;
+
+export interface DiscordActionRow {
+    type: 1; // Action row component type
+    components: DiscordComponent[];
+}
+
+export interface DiscordWebhookMessage {
+    content?: string;
+    username?: string;
+    avatar_url?: string;
+    tts?: boolean;
+    embeds?: DiscordEmbed[];
+    allowed_mentions?: DiscordAllowedMentions;
+    components?: DiscordActionRow[];
+    files?: any[]; // File attachments - these are handled differently in multipart requests
+    payload_json?: string;
+    attachments?: DiscordAttachment[];
+    flags?: number;
+    thread_name?: string;
+}
+
+export async function sendDiscordMessage(config: {
+    webhookURL: string;
+    message: DiscordWebhookMessage | string;
+}) {
+    const { webhookURL, message } = config;
+
+    const payload: DiscordWebhookMessage = typeof message === "string"
+        ? { content: message }
+        : message;
+
+    await httpsRequest(
+        webhookURL,
+        Buffer.from(JSON.stringify(payload)),
+        "POST",
+        false,
+        {
+            headers: {
+                "Content-Type": "application/json",
+            },
+        }
+    );
+}

package/src/fs.ts

@@ -69,4 +69,13 @@ export async function* readDirRecursive(dir: string): AsyncGenerator<string> {
             } catch { }
         }
     } catch { }
+}
+
+export async function fsExistsAsync(path: string) {
+    try {
+        await fs.promises.stat(path);
+        return true;
+    } catch {
+        return false;
+    }
 }

package/src/functional/SocketChannel.ts

@@ -55,6 +55,7 @@ export class SocketChannel<T> {
     }
 
     private watchAllNodes = lazy(async () => {
+        // NOTE: By watching instead of having nodes broadcast, it naturally prevents non-public servers from sending messages to public servers. This is really nice, and helps keep non-public servers (development servers), actually non-public...
         watchNodeIds((nodeIds) => {
             for (let nodeId of nodeIds) {
                 void errorToUndefinedSilent(this.controller.nodes[nodeId]._internal_watchMessages());
@@ -68,4 +69,12 @@ export class SocketChannel<T> {
             this.localWatchers.delete(callback);
         };
     }
+    // NOTE: We also get notifications for watching on all nodes, which should be fine...
+    public async watchSingleNode(nodeId: string, callback: (message: T) => void) {
+        await this.controller.nodes[nodeId]._internal_watchMessages();
+        this.localWatchers.add(callback);
+        return () => {
+            this.localWatchers.delete(callback);
+        };
+    }
 }

package/src/functional/throttleRender.ts

@@ -0,0 +1,134 @@
+
+
+// NOTE: Many cases we don't know if we want to throttle, or how long we want to throttle for, until we start the watcher, so this has to be done inside our render, instead of in ProxyWatcher.
+
+import { cache } from "socket-function/src/caching";
+import { proxyWatcher, SyncWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { qreact } from "../4-dom/qreact";
+import { onNextPaint } from "./onNextPaint";
+
+// Throttles calls that have the same throttleKey
+/** Used near the start of your render, like so:
+        if (throttleRender({ key: "NodeControls", frameDelay: 0, frameDelaySmear: 60 })) return undefined;
+    - Doesn't break watches, so debug tools will still show you are watching them
+    - Doesn't break the rendered output (uses the last fully rendered output)
+    - Pauses rendering until the throttle finishes, then forcefully renders once.
+    - Always delays by at least 1 frame.
+*/
+export function throttleRender(config: {
+    // Throttles are smeared perkey
+    key: string;
+    // We pick a large value in this range as throttleKey overlaps, allowing you to smear a large number of changes over a larger period of time, BUT, while also waiting less time if there are few changes!
+    //  - Throttles won't necessarily finish in the same order you call them (we have to wrap around in the range after we hit the end, so later calls might re-trigger earlier
+    frameDelay: number;
+    // Range is [frameDelay, frameDelay + smear]
+    frameDelaySmear?: number;
+    // Delays for additional frames. Ex, set frameDelay to 0, smear to 60, and count to 30. The first delay is 0, then 30, then 60
+    frameDelayCount?: number;
+}): boolean {
+    let watcher = proxyWatcher.getTriggeredWatcher();
+    // Never throttle the first render, as that would be noticeable and always unintended
+    if (watcher.syncRunCount === 0) return false;
+    let { schedule, inSchedule, isTriggered, runScheduler } = throttleManager(config.key);
+    if (isTriggered(watcher)) {
+        //console.log(`Triggering throttle render for ${watcher.debugName} on frame ${getFrameNumber()}`);
+        return false;
+    }
+    runScheduler();
+    if (inSchedule.has(watcher)) {
+        //console.log(`Skipping throttle render for ${watcher.debugName} because it is already in the schedule`);
+        proxyWatcher.reuseLastWatches();
+        qreact.cancelRender();
+        return true;
+    }
+    inSchedule.add(watcher);
+    //console.log(`Adding throttle render for ${watcher.debugName}`);
+
+
+    let count = config.frameDelayCount || 1;
+    let smear = config.frameDelaySmear ?? 0;
+    let curIndex = config.frameDelay;
+    let endIndex = config.frameDelay + smear;
+    let targetFillCount = 0;
+    // Annoying algorithm to find the lowest slow available...
+    while (true) {
+        let cur = schedule[curIndex];
+        if ((cur?.length || 0) === targetFillCount) {
+            schedule[curIndex] = schedule[curIndex] || [];
+            schedule[curIndex]!.push(watcher);
+            for (let i = 1; i < count; i++) {
+                schedule[curIndex + i] = schedule[curIndex + i] || [];
+                schedule[curIndex + i]!.push("delay");
+            }
+            break;
+        }
+        curIndex++;
+        if (curIndex > endIndex) {
+            curIndex = config.frameDelay;
+            targetFillCount++;
+        }
+    }
+
+    proxyWatcher.reuseLastWatches();
+    qreact.cancelRender();
+    return true;
+}
+
+let throttleManager = cache((key: string) => {
+    let schedule: ((SyncWatcher | "delay")[] | undefined)[] = [];
+    let inSchedule = new Set<SyncWatcher>();
+    let triggered = new Set<SyncWatcher>();
+    let runningScheduler = false;
+    function isTriggered(watcher: SyncWatcher) {
+        return triggered.has(watcher);
+    }
+
+    return {
+        schedule,
+        inSchedule,
+        isTriggered,
+        runScheduler: () => {
+            if (runningScheduler) return;
+            runningScheduler = true;
+            void onNextPaint().finally(runNextTick);
+            function runNextTick() {
+                let next = schedule.shift();
+                void onNextPaint().finally(() => {
+                    triggered.clear();
+                });
+                if (next) {
+                    for (let watcher of next) {
+                        if (watcher === "delay") continue;
+                        inSchedule.delete(watcher);
+                        triggered.add(watcher);
+                        //console.log(`Triggering throttle render for ${watcher.debugName}`);
+                        watcher.explicitlyTrigger({
+                            paths: new Set(),
+                            pathSources: new Set(),
+                            newParentsSynced: new Set(),
+                            extraReasons: ["throttleRender trigger"],
+                        });
+                    }
+                }
+                if (schedule.length > 0) {
+                    void onNextPaint().finally(runNextTick);
+                } else {
+                    runningScheduler = false;
+                }
+            }
+        },
+    };
+});
+
+
+
+let rendersInFrame = 0;
+export function countRendersInFrame() {
+    if (rendersInFrame === 0) {
+        void onNextPaint().finally(() => {
+            //console.log(`Render in frame ${rendersInFrame}`);
+            rendersInFrame = 0;
+        });
+    }
+    rendersInFrame++;
+}