querysub 0.328.0 → 0.330.0

package/src/diagnostics/logs/errorNotifications/errorDigests.tsx

@@ -1,8 +1,291 @@
-type DigestInfo = {
-    histogram: {
-        [timeGroup: number]: number;
+import { insertIntoSortedList, list, timeInDay, timeInHour } from "socket-function/src/misc";
+import { runScheduler } from "../../../misc2";
+import { getErrorAppendables, getSuppressionFull, getSuppressionListRaw, suppressionList } from "./ErrorNotificationController";
+import { createLogScanner } from "../FastArchiveAppendable";
+import { LogDatum, getLogFile, getLoggers } from "../diskLogger";
+import { FastArchiveAppendableControllerBase, FileMetadata } from "../FastArchiveController";
+import { httpsRequest } from "socket-function/src/https";
+import { Zip } from "../../../zip";
+import { encodeCborx } from "../../../misc/cloneHelpers";
+import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
+import { nestArchives } from "../../../-a-archives/archives";
+import { getArchivesBackblaze } from "../../../-a-archives/archivesBackBlaze";
+import { getDomain } from "../../../config";
+import { getOwnMachineId, getOwnThreadId } from "../../../-a-auth/certs";
+import { sendErrorDigestEmail } from "./errorDigestEmail";
+import { archiveCborT } from "../../../-a-archives/archivesCborT";
+import { Querysub } from "../../../4-querysub/Querysub";
+import { sendEmail } from "../../../user-implementation/userData";
+import { qreact } from "../../../4-dom/qreact";
+import { getSyncedController } from "../../../library-components/SyncedController";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { assertIsManagementUser } from "../../managementPages";
+
+const LATEST_ERRORS_COUNT_PER_FILE = 20;
+
+export type ErrorDigestInfo = {
+    key: string;
+    // timeGroupEnd =>
+    histogram: Map<number, {
+        suppressedErrors: number;
+        unsuppressedErrors: number;
+        suppressedWarnings: number;
+        unsuppressedWarnings: number;
+
+        corruptErrors: number;
+        corruptWarnings: number;
+
+        firstCorruptError?: string;
+        firstCorruptWarning?: string;
+    }>;
+    // file =>
+    byFile: Map<string, {
+        errors: number;
+        warnings: number;
+        // Sorted from oldest to newest
+        latestErrors: LogDatum[];
+        latestWarnings: LogDatum[];
+    }>;
+    totalCompressedBytes: number;
+    totalUncompressedBytes: number;
+    totalFiles: number;
+
+    scanDuration: number;
+    scanStartTime: number;
+    scanEndTime: number;
+    startTime: number;
+    endTime: number;
+};
+class ErrorDigest {
+    public async getDigestKeys() {
+        return errorDigestHistory.keys();
+    }
+    public async getDigest(key: string) {
+        return errorDigestHistory.get(key);
+    }
+}
+
+export const ErrorDigestController = getSyncedController(SocketFunction.register(
+    "ErrorDigestController-e5996b95-dcfc-412e-a104-80ed2c2d5933",
+    new ErrorDigest(),
+    () => ({
+        getDigestKeys: {},
+        getDigest: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    })
+));
+
+export const errorDigestHistory = archiveCborT<ErrorDigestInfo>(() => nestArchives("error-digests/", getArchivesBackblaze(getDomain())));
+
+function getClosest(value: number, choices: number[]) {
+    let dist = Number.POSITIVE_INFINITY;
+    let closest: number = choices[0];
+    for (let choice of choices) {
+        let curDist = Math.abs(value - choice);
+        if (curDist < dist) {
+            dist = curDist;
+            closest = choice;
+        }
+    }
+    return closest;
+}
+
+async function runDigest() {
+    console.log("Running error digest gathering");
+    // Find the previous day
+    let endTime = getClosest(
+        Date.now(),
+        [
+            new Date().setHours(11),
+            new Date(Date.now() - timeInDay).setHours(11),
+        ]
+    );
+    let endDate = new Date(endTime);
+    endDate.setMinutes(0);
+    endDate.setSeconds(0);
+    endDate.setMilliseconds(0);
+    endTime = endDate.getTime();
+    let startTime = new Date(endTime - timeInDay).getTime();
+    let scanStartTime = Date.now();
+
+    let digestInfo: ErrorDigestInfo = {
+        key: `${Date.now()}_${getOwnMachineId()}_${getOwnThreadId()}`,
+        histogram: new Map(),
+        byFile: new Map(),
+        scanDuration: 0,
+        scanStartTime,
+        scanEndTime: Date.now(),
+        startTime,
+        endTime,
+        totalCompressedBytes: 0,
+        totalUncompressedBytes: 0,
+        totalFiles: 0,
     };
-    topByFile: {
-        [file: string]: number;
-    };
-};
+
+    let entries = await getSuppressionListRaw();
+    let suppressionFull = getSuppressionFull({
+        entries: Object.values(entries.entries),
+        blockTimeRange: {
+            startTime,
+            endTime,
+        },
+    });
+
+    function getTimeGroup(time: number) {
+        return Math.floor((time - startTime) / timeInHour) * timeInHour + startTime;
+    }
+
+
+    let appendables = getErrorAppendables();
+    for (let appendable of appendables) {
+        let isError = true;
+        if (appendable.rootPath.includes("warn")) {
+            isError = false;
+        } else if (appendable.rootPath.includes("error")) {
+            isError = true;
+        } else {
+            throw new Error(`Unhandled appendable root path: ${appendable.rootPath}`);
+        }
+        function registerCount(time: number, isSuppressed: boolean, corruptError?: string) {
+            let timeGroup = getTimeGroup(time);
+            let obj = digestInfo.histogram.get(timeGroup);
+            if (!obj) {
+                obj = {
+                    suppressedErrors: 0,
+                    unsuppressedErrors: 0,
+                    suppressedWarnings: 0,
+                    unsuppressedWarnings: 0,
+                    corruptErrors: 0,
+                    corruptWarnings: 0,
+                };
+                digestInfo.histogram.set(timeGroup, obj);
+            }
+            if (isError) {
+                if (isSuppressed) {
+                    obj.suppressedErrors++;
+                } else {
+                    obj.unsuppressedErrors++;
+                }
+                if (corruptError) {
+                    obj.corruptErrors++;
+                    if (!obj.firstCorruptError) {
+                        obj.firstCorruptError = corruptError;
+                    }
+                }
+            } else {
+                if (isSuppressed) {
+                    obj.suppressedWarnings++;
+                } else {
+                    obj.unsuppressedWarnings++;
+                }
+                if (corruptError) {
+                    obj.corruptWarnings++;
+                    if (!obj.firstCorruptWarning) {
+                        obj.firstCorruptWarning = corruptError;
+                    }
+                }
+            }
+        }
+        console.log(`Gathering files for ${appendable.rootPath}`);
+        let result = await new FastArchiveAppendableControllerBase().startSynchronizeInternal({
+            range: {
+                startTime,
+                endTime,
+            },
+            rootPath: appendable.rootPath,
+        });
+        let filesLeft = result.files.slice();
+        await Promise.all(list(32).map(() => runThread()));
+        async function runThread() {
+            while (true) {
+                let file = filesLeft.shift();
+                if (!file) {
+                    return;
+                }
+                await processFile(file);
+            }
+        }
+        async function processFile(file: FileMetadata) {
+            try {
+                console.log(`Processing file ${file.path}`);
+                let compressed = await httpsRequest(file.url);
+                let data = await Zip.gunzip(compressed);
+                digestInfo.totalCompressedBytes += compressed.length;
+                digestInfo.totalUncompressedBytes += data.length;
+                digestInfo.totalFiles++;
+
+                let callback = createLogScanner({
+                    debugName: "digestScanner",
+                    onParsedData: (posStart, posEnd, buffer) => {
+                        if (buffer === "done") {
+                            return;
+                        }
+                        let result = suppressionFull(posStart, posEnd, buffer);
+                        if (!result) {
+                            registerCount(file.endTime, true);
+                            return;
+                        }
+
+                        let datum: LogDatum;
+                        try {
+                            datum = JSON.parse(buffer.slice(posStart, posEnd).toString()) as LogDatum;
+                        } catch (e: any) {
+                            let message = `Failed to parse log datum in around ${buffer.slice(posStart, posEnd).slice(0, 100).toString("hex")}, error is:\n${e.stack}`;
+                            process.stderr.write(message);
+                            registerCount(file.endTime, false, message);
+                            return;
+                        }
+                        registerCount(datum.time, false);
+
+                        let fileGroup = getLogFile(datum);
+                        let obj = digestInfo.byFile.get(fileGroup);
+                        if (!obj) {
+                            obj = {
+                                errors: 0,
+                                warnings: 0,
+                                latestErrors: [],
+                                latestWarnings: [],
+                            };
+                            digestInfo.byFile.set(fileGroup, obj);
+                        }
+                        if (isError) {
+                            obj.errors++;
+                        } else {
+                            obj.warnings++;
+                        }
+                        let list = isError ? obj.latestErrors : obj.latestWarnings;
+                        if (list.length === 0 || datum.time >= list[0].time) {
+                            // NOTE: This should almost never trigger, so the search, and even worse, the splice, should almost never happen
+                            insertIntoSortedList(list, x => x.time, datum);
+                            if (list.length > LATEST_ERRORS_COUNT_PER_FILE) {
+                                list.splice(0, list.length - LATEST_ERRORS_COUNT_PER_FILE);
+                            }
+                        }
+                    },
+                });
+                await callback(data);
+                await callback("done");
+            } catch (e: any) {
+                console.warn(`Failed to process file ${file.path}, error: ${e.stack}`);
+            }
+            let progress = result.files.length - filesLeft.length + 1;
+            console.log(`Processed file ${file.path} (${progress} / ${result.files.length}) in ${appendable.rootPath}`);
+        }
+    }
+
+
+    let scanEndTime = Date.now();
+    digestInfo.scanDuration = scanEndTime - scanStartTime;
+    digestInfo.scanEndTime = scanEndTime;
+    await errorDigestHistory.set(digestInfo.key, digestInfo);
+    await sendErrorDigestEmail(digestInfo);
+}
+
+export async function runDigestLoop() {
+    await Querysub.hostService("error-digests");
+
+    // TODO: We might want to change the scheduler to run only on some days, adding support to weekday filtering as well (ex, just monday, wednesday, friday)
+    await runScheduler([12], runDigest);
+}

package/src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx

@@ -10,7 +10,7 @@ import { LogDatum, getLogFile } from "../diskLogger";
 import { sendDiscordMessage } from "../../../email_ims_notifications/discord";
 import { user_data } from "../../../user-implementation/userData";
 import { createLink } from "../../../library-components/ATag";
-import { getLogsLinkParts } from "./ErrorWarning";
+import { getErrorLogsLink } from "./ErrorWarning";
 
 const MAX_IMS_PER_DAY = 3;
 const MAX_IMS_PER_HOURS = 1;
@@ -20,32 +20,8 @@ const MAX_PER_IM = 10;
 
 // Wait a bit, because it's likely if there's one error, there are more errors. 
 const BATCH_TIME = timeInSecond * 30;
-const DIGEST_INTERVAL = timeInDay * 3;
-
-
-// 4) Deploy
-// 4) Create local errors and make sure the remote server doesn't get them 
-// 5) Create remote errors and make sure that the all servers get them (Local and remote servers)
-
-
-// Abstract out RecentErrors scanning, so we can reuse it
-//  (appendables, timeRange, callback)
-// - Oh, we also need a way to tell it to not limit to recent errors.
-// - And also to disable throttling. 
-
-
-// createDigest(appendables, timeRange) => Promise<DigestInfo>
-
-// 7) Write the digest script, which does a fresh scan on just a specific range of data using start synchronize internal and suppression list, etc. etc. 
-//      - The most recent ten unsuppressed errors. Counts of unsuppressed errors by file. Or by line id? If they have it. Counts of suppressed and unsuppressed errors by hour group. 
-//      - Oh, also everything we do for errors do for warnings as well, and split the values. We want warnings and errors. 
-//      - Time the entire thing, and put that, and the profile, in the digest too! That will give us a good gauge on if the errors/suppressions are getting slow (due to a lot of errors, or a lot of suppression checks!)
-// store the digest results in Back Blaze. 
-// 8) Write a page that will show the results from Backblaze, allowing you to see a list of all of the digests and to view a specific one. Just showing all the data in a chart, whatever. Showed two charts, one with just the unsuppressed errors, and the other with both suppressed and unsuppressed, because you really only care about suppressed when you're looking at the fraction of suppressed to unsuppressed, and the sheer quantity of unsuppressed. 
-// 9) send an email every time period, and also send an IM that has smaller information
-//      - Both will link to the actual web page that has the digest, deep linking to the specific tabs. 
-//      - Show the chart in the email as well, but just format it like ASCII Because image dependencies are annoying and I don't want to implement them right now as it might take a few days to get working. 
 
+// 11) Deploy services to service
 
 
 // NOTE: Yes, this is stored in memory, so if the server reboots or if this script keeps crashing, we might send a lot of instant messages. However, one, Discord will probably late rate limit us, and two, this means something is really wrong, especially if it happens a lot, and we really should fix it right away. 
@@ -142,7 +118,7 @@ const sendIMs = batchFunction(({ delay: BATCH_TIME }), async (logsAll: LogDatum[
             console.error(`No Discord webhook URL set, cannot send warning instant messages`);
             return;
         }
-        let url = createLink(getLogsLinkParts());
+        let url = createLink(getErrorLogsLink());
         let message = Object.values(info.perFile).flat().map(
             x => `[${formatDateTime(x.time)}](${url}) | ${x.param0} (${x.__NAME__})`
         ).join("\n");
@@ -184,7 +160,6 @@ async function runIMNotifies() {
         }
         threadIdMap.add(obj.time);
         return false;
-
     }
     function clearOldDays() {
         // Clear all the days that are more than 14 days older than our current day.

package/src/diagnostics/logs/lifeCycleAnalysis/spec.md

@@ -4,24 +4,7 @@ Very small amount of data
     https://127-0-0-1.querysubtest.com:7007/?hot&enableLogs&page=login&filter=%22431%22&showingmanagement&endTime=1755140880000&startTime=1754950020000&managementpage=LogViewer2
 
 
-5) IM error notifications - allow immediately knowing about production issues, for better testing
-    - Create a dedicated entry point which acts like a client of the HTTP server, using RecentErrorControllers.getRecentErrors
-        - Getting it working in a script will be interesting, but... in theory it should just work?
-    - Just for new errors
-    - Using backblaze to track when we send it, so we can heavily limit IMs and email
-    - IM api key tracked in secrets (like email api key)
-    - Once we get it working, deploy to production
-
-6) IM + email digests (daily / weekly?)
-    - a very short digest for the instant message which then links to a page on the site with a larger digest
-        - which has tabs, and each part in the instant message links to the correct tab
-    - Augments the error notifications entry point, having it also queue stuff up for digests. 
-    - Some notifications will never be immediate and will always be only in digests. 
-    - For now this will just be for:
-        - non-suppressed errors
-        - suppressed errors
-
-
+AFTER digests, go back to adding application code, as the framework is getting boring...
 
 5) Life cycle analyzer
     - Implement regular range lifecycles first (matching an === object field)
@@ -95,6 +78,13 @@ Check the startup lifecycle to make sure we can detect the nodes pretty fast and
     - Receiving values from different authorities and the ones we're watching is weird. Why does that keep happening?
     - And we keep running into audit mismatches? Why does that keep happening? Is it only because of our local development server? 
 
+DEBUG: Deploy hash updates. 
+    - Forced refresh now, and then immediately refreshing is STILL not giving us the latest code. Even though we waited for everything to reload the UI, which took forever.
+    - It's probably an issue with the routing information being out of date, I think it's cached in Cloudflare. We could at least use life cycles to verify the values we have, and then if they're different than the values in the client, then I guess it must be in Cloudflare. We can also verify our timing, as I'm pretty sure we're supposed to be waiting for the cloud flare values to update, and if we're not, then that's a problem.
+
+DEBUG: Suppression creation propagation 
+    - It didn't propagate to all the servers?
+
 SPECIAL UI links for certain errors in log view
     - Probably dynamically created, based on contents of log
     - LINKS to filters for all these special errors on a special page

package/src/diagnostics/managementPages.tsx

@@ -90,6 +90,12 @@ export async function registerManagementPages2(config: {
         componentName: "LogViewer2",
         getModule: () => import("./logs/LogViewer2"),
     });
+    inputPages.push({
+        title: "Error Digests",
+        componentName: "ErrorDigestPage",
+        controllerName: "ErrorDigestController",
+        getModule: () => import("./logs/errorNotifications/ErrorDigestPage"),
+    });
     inputPages.push({
         title: "Security",
         componentName: "SecurityPage",
@@ -181,22 +187,29 @@ export async function registerManagementPages2(config: {
         // Wait, so the import system knows the modules are async imports
         await delay(0);
         for (let page of inputPages) {
-            // NOTE: If we split this into a module for component/controller, we need to make sure we
-            //  import both serverside, so we can whitelist them for import clientside.
-            let mod = await page.getModule();
-            if (!page.controllerName) continue;
-            if (!(page.controllerName in mod)) {
-                console.error(`Controller ${page.controllerName} not found in module`, mod);
-                throw new Error(`Controller ${page.controllerName} not found in module`);
-            }
-            let controller = mod[page.controllerName] as SocketRegistered;
-            if (!controller) {
-                throw new Error(`Controller ${page.controllerName} not found in module`);
-            }
-            if (!controller._classGuid) {
-                throw new Error(`Controller ${page.controllerName} does not have a class guid`);
+            try {
+                // NOTE: If we split this into a module for component/controller, we need to make sure we
+                //  import both serverside, so we can whitelist them for import clientside.
+                let mod = await page.getModule();
+                if (!page.controllerName) continue;
+                if (!(page.controllerName in mod)) {
+                    console.error(`Controller ${page.controllerName} not found in module`, mod);
+                    throw new Error(`Controller ${page.controllerName} not found in module`);
+                }
+                let controller = mod[page.controllerName] as SocketRegistered;
+                if ((controller as any)?.__baseController) {
+                    controller = (controller as any).__baseController;
+                }
+                if (!controller) {
+                    throw new Error(`Controller ${page.controllerName} not found in module`);
+                }
+                if (!controller._classGuid) {
+                    throw new Error(`Controller ${page.controllerName} does not have a class guid`);
+                }
+                SocketFunction.expose(controller);
+            } catch (e: any) {
+                console.error(`Error when registering management page ${page.controllerName} in ${page.componentName}: ${e.stack}`);
             }
-            SocketFunction.expose(controller);
         }
     } else {
         for (let page of inputPages) {

package/src/email_ims_notifications/postmark.tsx

@@ -8,7 +8,7 @@ import { renderToString } from "../library-components/renderToString";
 
 export async function sendEmail_postmark(config: {
     apiKey: string;
-    to: string;
+    to: string[];
     from: string;
     subject: string;
     contents: preact.VNode;
@@ -18,13 +18,13 @@ export async function sendEmail_postmark(config: {
     if (Querysub.isInSyncedCall()) {
         throw new Error("sendEmail_sendgrid should not be called in a synced call, as this might result in multiple sends. Instead, use Querysub.onCommitFinished to call after the synced call");
     }
-    console.log(`${magenta("Sending email")} to ${green(config.to)} with subject ${config.subject}`);
+    console.log(`${magenta("Sending email")} to ${green(config.to.join(", "))} with subject ${config.subject}`);
     let htmlContent = renderToString(config.contents);
     await httpsRequest(
         "https://api.postmarkapp.com/email",
         Buffer.from(JSON.stringify({
             From: config.from,
-            To: config.to,
+            To: config.to.join(","),
             Subject: config.subject,
             HtmlBody: htmlContent,
         })),

package/src/library-components/SyncedController.ts

@@ -109,7 +109,7 @@ export function getSyncedController<T extends SocketRegistered>(
     resetAll(): void;
     refreshAll(): void;
     isAnyLoading(): boolean;
-    base: T;
+    __baseController: T;
 } {
     if (isNode()) {
         let result = cache((nodeId: string) => {
@@ -161,7 +161,7 @@ export function getSyncedController<T extends SocketRegistered>(
         result.isAnyLoading = () => {
             notAllowedOnServer();
         };
-        result.base = controller;
+        result.__baseController = controller;
         return result;
     }
     let id = nextId();
@@ -416,6 +416,6 @@ export function getSyncedController<T extends SocketRegistered>(
             }
         });
     };
-    result.base = controller;
+    result.__baseController = controller;
     return result;
 }

package/src/misc.ts

@@ -185,4 +185,9 @@ export function streamToIteratable<T>(reader: {
             }
         }
     };
+}
+
+const AsyncFunction = (async () => { }).constructor;
+export function isAsyncFunction(func: unknown): boolean {
+    return func instanceof AsyncFunction;
 }

package/src/misc2.ts

@@ -1,3 +1,6 @@
+import { delay } from "socket-function/src/batching";
+import { formatTime } from "socket-function/src/formatting/format";
+import { timeInHour } from "socket-function/src/misc";
 import { atomic } from "./2-proxy/PathValueProxyWatcher";
 
 export function isStrSimilar(a: string | undefined, b: string | undefined) {
@@ -56,4 +59,53 @@ export async function* streamToAsyncIterable(stream: StreamLike): AsyncIterable<
     while (pendingChunks.length > 0) {
         yield pendingChunks.shift()!;
     }
+}
+
+
+export async function runScheduler(hours: number[], func: () => Promise<void>) {
+    while (true) {
+        try {
+            const now = new Date();
+
+            // Find the next check time from our list
+            const sortedHours = [...hours].sort((a, b) => a - b);
+            let targetTime = new Date(now);
+
+            // Find the next check hour after the current time
+            let nextHour = sortedHours.find(hour => {
+                const checkTime = new Date(now);
+                checkTime.setHours(hour, 0, 0, 0);
+                return checkTime > now;
+            });
+
+            if (nextHour === undefined) {
+                // No more checks today, take the first one tomorrow
+                nextHour = sortedHours[0];
+                targetTime.setDate(targetTime.getDate() + 1);
+            }
+
+            targetTime.setHours(nextHour, 0, 0, 0);
+
+            console.log(`Next target time: ${targetTime.toLocaleString()}`);
+
+            // Wait and check every 15 minutes until it's time
+            while (true) {
+                const currentTime = new Date();
+                const timeUntilCheck = targetTime.getTime() - currentTime.getTime();
+
+                if (timeUntilCheck <= 0) {
+                    console.log("Time to run the check!");
+                    await func();
+                    break; // Break out of inner loop to calculate next check time
+                }
+
+                console.log(`Time until next check: ${formatTime(timeUntilCheck)}`);
+                await delay(timeInHour / 4); // Wait 15 minutes (1/4 of an hour)
+            }
+        } catch (error) {
+            console.error(error);
+            // Even if there's an error, wait 15 minutes before trying again
+            await delay(timeInHour / 4);
+        }
+    }
 }

package/src/user-implementation/userData.ts

@@ -11,7 +11,7 @@ import { isClient } from "../config2";
 import { getExternalIP } from "../misc/networking";
 import { MAX_ACCEPTED_CHANGE_AGE } from "../0-path-value-core/pathValueCore";
 import { createURLSync } from "../library-components/URLParam";
-import { devDebugbreak, getDomain, isDevDebugbreak, isRecovery } from "../config";
+import { devDebugbreak, getDomain, getEmailDomain, isDevDebugbreak, isRecovery } from "../config";
 import { delay } from "socket-function/src/batching";
 import { enableErrorNotifications } from "../library-components/errorNotifications";
 import { clamp } from "../misc";
@@ -607,6 +607,7 @@ function sendLoginEmail(config: {
         const machineId = Querysub.getCallerMachineId();
         const now = Querysub.getCallTime();
 
+        // Check for the API key, even though we don't immediately use it, so that we can get good errors to tell the user (likely the developer) to setup the postmark API key. 
         const apiKey = atomic(data().secure.postmarkAPIKey);
         if (!apiKey) {
             let link = createLink([
@@ -688,18 +689,36 @@ function sendLoginEmail(config: {
             timeoutTime,
         });
         Querysub.onCommitFinished(async () => {
-            await sendEmail_postmark({
-                apiKey,
-                to: email,
-                // TODO: Allow configuring this (defaulting to getDomain() if unconfigured). For now we hardcode it, because it takes
-                //  a while to verify a new postmark email, and we don't even know what final domain we will be using.
-                from: "login@querysub.com",
+            await sendEmail({
+                to: [email],
+                fromPrefix: "login",
                 subject,
                 contents,
             });
         });
     });
 }
+export async function sendEmail(config: {
+    to: string[];
+    // The domain should be getDomain
+    // TODO: Actually use git domain, for now it's hardcoded, because setting up a new   
+    fromPrefix: string;
+    subject: string;
+    contents: preact.VNode;
+}) {
+    let key = await Querysub.commitAsync(() => atomic(data().secure.postmarkAPIKey));
+    if (!key) {
+        console.warn(`No postmark API key setup, so we can't send email`);
+        return;
+    }
+    await sendEmail_postmark({
+        apiKey: key,
+        to: config.to,
+        from: `${config.fromPrefix}@${getEmailDomain()}`,
+        subject: config.subject,
+        contents: config.contents,
+    });
+}
 
 function verifyMachineId(config: {
     loginToken: string

package/testEntry2.ts

@@ -19,7 +19,7 @@ export async function testMain() {
     await Querysub.hostService("test");
     await delay(timeInSecond * 5);
 
-    console.error(`This should show up locally, but not remotely.`);
+    console.error(`A completely new error that is not suppressed. .`);
     await delay(timeInSecond * 15);
     await shutdown();
     return;