querysub 0.343.0 → 0.345.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.343.0",
+  "version": "0.345.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -15,7 +15,8 @@
     "test": "yarn typenode ./test.ts",
     "test3": "yarn typenode ./src/test/test.tsx --local",
     "test2": "yarn typenode ./src/4-dom/qreactTest.tsx --local",
-    "error-watch": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx"
+    "error-watch": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx",
+    "error-email": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx"
   },
   "bin": {
     "deploy": "./bin/deploy.js",

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -1636,6 +1636,7 @@ export class PathValueProxyWatcher {
                     // The calls have to happen after our local writes. This is because they are likely to
                     //  influence the local writes, and we don't want our local writes to be always invalidated
                     call.runAtTime = getNextTime();
+                    call.fromProxy = watcher.debugName;
                     logErrors(runCall(call, metadata));
                     watcher.options.onCallCommit?.(call, metadata);
                 }

package/src/3-path-functions/PathFunctionHelpers.ts

@@ -121,7 +121,7 @@ export function interceptCallsBase<T>(
 }
 interceptCalls.declare(interceptCallsBase);
 
-
+/** Writes the function call allowing interceptors to capture it (vs runCall which just runs it). */
 export function writeFunctionCall(config: {
     domainName: string;
     moduleId: string;

package/src/3-path-functions/PathFunctionRunner.ts

@@ -91,6 +91,9 @@ export interface CallSpec {
     callerIP: string;
     runAtTime: Time;
 
+    // Not just used for debugging, also used to add special proxy-related warnings. 
+    fromProxy?: string;
+
     filterable?: Filterable;
 }
 export function debugCallSpec(spec: CallSpec): string {

package/src/3-path-functions/syncSchema.ts

@@ -48,6 +48,9 @@ export type FunctionMetadata<F = unknown> = {
      */
     delayCommit?: boolean;
 
+    /** By default, we try to finish calls in the start order when using Querysub.commitAsync, unless the caller explicitly says not to. However, if this is set, then we won't have this call finish or cause other calls to be finished in the start order. This makes locking less safe, but can be useful for long-running functions that shouldn't block other functions and which the order doesn't matter. */
+    noFinishInStartOrder?: boolean;
+
     /** Too many locks can lag the server, and eventually cause crashes. Consider using Querysub.noLocks(() => ...) around code that is accessing too many values, assuming you don't want to lock them. However, if absolutely required, you can override max locks to allow as many locks to be created as you want until the server crashses... */
     maxLocksOverride?: number;
 };
@@ -363,7 +366,7 @@ export function syncSchema<Schema>(schema?: Schema2): SyncSchemaResult<Schema> {
                     moduleId,
                     functionId: name,
                     args,
-                    metadata,
+                    metadata: metadata
                 });
             }, {
                 debug() {

package/src/4-deploy/edgeClientWatcher.tsx

@@ -18,6 +18,8 @@ import { Button } from "../library-components/Button";
 import { updateRootDiscoveryLocation } from "../-f-node-discovery/NodeDiscovery";
 
 const SWITCH_SERVER_TIMEOUT = timeInSecond * 15;
+const MAX_DISPLAY_INTERVAL = timeInMinute * 5;
+const FINAL_DELAY = timeInSecond * 30;
 
 let lastHashServer = "";
 export function startEdgeNotifier() {
@@ -61,113 +63,105 @@ const notifyClients = throttleFunction(timeInMinute, async function notifyClient
 // Track current notification state
 let currentNotification: { close: () => void } | null = null;
 let curHash = "";
+let lastShownTime = 0;
 function onLiveHashChange(liveHash: string, refreshThresholdTime: number) {
     console.log(blue(`Received client liveHash ${liveHash}, prev hash: ${curHash}`));
     if (liveHash === curHash) return;
     let prevHash = curHash;
     // Don't notify the user right away. Hopefully they refresh naturally, and we never have to notify them at all!
     // Also, refresh BEFORE the server dies, not exactly when it is about to die
-    let notifyIntervals = [0.4, 0.75, 0.95];
+    let delays = [0.4, 0.75, 1].map(x => delay(x * refreshThresholdTime));
+
     console.log(blue(`Client liveHash changed ${liveHash}, prev hash: ${prevHash}`));
-    // If we are replacing an already existing notification, don't show immediately
-    let skipFirst = false;
-    if (currentNotification) {
-        currentNotification.close();
-        currentNotification = null;
-        skipFirst = true;
-    }
-    curHash = liveHash;
 
-    let duration = refreshThresholdTime - Date.now();
+    curHash = liveHash;
 
     // Start notification loop
     void (async () => {
         // Show notifications at intervals
-        for (let i = 0; i < notifyIntervals.length - 1; i++) {
+        for (let delayTime of delays) {
+            await delayTime;
             // Don't show if a newer notification is active
             if (curHash !== liveHash) return;
 
-            let waitDuration = (notifyIntervals[i + 1] - notifyIntervals[i]) * duration;
-            // If the duration is short, and it's not the last one, skip it
-            if (i < notifyIntervals.length - 2 && waitDuration <= 30 * 1000) continue;
-
             // Update the URL override for manual refreshes
             if (!liveHash.startsWith("forced")) {
                 Querysub.localCommit(() => {
                     liveHashOverrideURL.value = { liveHash, time: Date.now() };
                 });
             }
-            if (!skipFirst) {
-                if (currentNotification) {
-                    currentNotification.close();
-                    currentNotification = null;
-                }
-                // Show notification modal
-                let nextNotification = {
-                    close: showModal({
-                        onClose: () => {
-                            if (currentNotification !== nextNotification) return;
-                            currentNotification = null;
-                        },
-                        content: (
-                            <div
-                                title={`Live Hash: ${liveHash}, prev hash: ${prevHash}`}
-                                className={
-                                    css.vbox(10).pad(20)
-                                        .hsla(0, 0, 0, 0.8)
-                                        .position("fixed")
-                                        .bottom(10)
-                                        .right(10)
-                                        .zIndex(1000)
-                                    + " keepModalsOpen"
-                                }
-                            >
-                                <div className={css.vbox(10).maxWidth(250)}>
-                                    <h3 className={css.margin(0)}>Server Update Available</h3>
-                                    <div>The server has been updated. Please refresh the page to ensure you don't experience compatibility issues.</div>
-                                    {i !== notifyIntervals.length - 2 && <div>This notification will be shown again at {formatNiceDateTime(Date.now() + waitDuration)}</div>}
-                                    {i === notifyIntervals.length - 2 && <div>The page will automatically refresh at {formatNiceDateTime(Date.now() + waitDuration)}</div>}
-                                </div>
-                                <div className={css.hbox(10).justifyContent("flex-end")}>
-                                    <button
-                                        className={css.pad(8, 16).hsl(200, 50, 50).color("white").pointer}
-                                        onClick={(e) => {
-                                            if (!liveHash.startsWith("forced")) {
-                                                Querysub.localCommit(() => {
-                                                    liveHashOverrideURL.value = { liveHash, time: Date.now() };
-                                                });
-                                            }
-                                            window.location.reload();
-                                        }}
-                                    >
-                                        Refresh Now
-                                    </button>
-                                    <button
-                                        className={css.pad(8, 16).pointer}
-                                        onClick={(e) => {
-                                            if (currentNotification) {
-                                                currentNotification.close();
-                                                currentNotification = null;
-                                            }
-                                        }}
-                                    >
-                                        Dismiss
-                                    </button>
-                                </div>
-                            </div>
-                        )
-                    }).close
-                };
-                currentNotification = nextNotification;
-            }
-            skipFirst = false;
 
-            console.log(red(`Notify again in ${formatTime(waitDuration)}`));
-            await delay(waitDuration);
+            let timeSinceLastShown = Date.now() - lastShownTime;
+            let isLastDelay = delayTime === delays.at(-1);
+            if (timeSinceLastShown < MAX_DISPLAY_INTERVAL && !isLastDelay) continue;
+
+            lastShownTime = Date.now();
+
+            if (currentNotification) {
+                currentNotification.close();
+                currentNotification = null;
+            }
+            // Show notification modal
+            let nextNotification = {
+                close: showModal({
+                    onClose: () => {
+                        if (currentNotification !== nextNotification) return;
+                        currentNotification = null;
+                    },
+                    content: (
+                        <div
+                            title={`Live Hash: ${liveHash}, prev hash: ${prevHash}`}
+                            className={
+                                css.vbox(10).pad(20)
+                                    .hsla(0, 0, 0, 0.8)
+                                    .position("fixed")
+                                    .bottom(10)
+                                    .right(10)
+                                    .zIndex(1000)
+                                + " keepModalsOpen"
+                            }
+                        >
+                            <div className={css.vbox(10).maxWidth(250)}>
+                                <h3 className={css.margin(0)}>Server Update Available</h3>
+                                <div>The server has been updated. Please refresh the page to ensure you don't experience compatibility issues.</div>
+                                {isLastDelay && <div>The page will automatically refresh shortly</div>}
+                            </div>
+                            <div className={css.hbox(10).justifyContent("flex-end")}>
+                                <button
+                                    className={css.pad(8, 16).hsl(200, 50, 50).color("white").pointer}
+                                    onClick={(e) => {
+                                        if (!liveHash.startsWith("forced")) {
+                                            Querysub.localCommit(() => {
+                                                liveHashOverrideURL.value = { liveHash, time: Date.now() };
+                                            });
+                                        }
+                                        window.location.reload();
+                                    }}
+                                >
+                                    Refresh Now
+                                </button>
+                                <button
+                                    className={css.pad(8, 16).pointer}
+                                    onClick={(e) => {
+                                        if (currentNotification) {
+                                            currentNotification.close();
+                                            currentNotification = null;
+                                        }
+                                    }}
+                                >
+                                    Dismiss
+                                </button>
+                            </div>
+                        </div>
+                    )
+                }).close
+            };
+            currentNotification = nextNotification;
         }
 
         // Only force refresh if this is still the current notification
-        if (curHash === liveHash) {
+        if (curHash === liveHash && currentNotification) {
+            await delay(FINAL_DELAY);
             window.location.reload();
         }
     })();

package/src/4-querysub/querysubPrediction.ts

@@ -418,7 +418,7 @@ export function getCallWrites(config: {
 
     let finishInStartOrder: number | boolean | undefined;
 
-    if (config.useFinishReordering) {
+    if (config.useFinishReordering && !config.metadata?.noFinishInStartOrder && call.fromProxy) {
         let triggerCaller = getCurrentCallCreationProxy();
         if (!triggerCaller) {
             require("debugbreak")(2);

package/src/diagnostics/logs/diskLogger.ts

@@ -40,10 +40,21 @@ export type LogDatum = Record<string, unknown> & {
 export function getLogHash(obj: LogDatum) {
     return getPathStr2(obj.__threadId || "", obj.time.toString());
 }
+const errorMessageFileRegex = /\n    at .+?\(([^:]+):\d+:\d+\)/;
+
 export function getLogFile(obj: LogDatum) {
     let logType = obj.param0 || "";
     if (obj.__FILE__) {
         logType = String(obj.__FILE__);
+        // /root/machine-services/*/git/
+        if (logType.startsWith("/root/machine-services/")) {
+            logType = logType.split("/").slice(4).join("/");
+        }
+    } else {
+        let match = logType.match(errorMessageFileRegex);
+        if (match) {
+            logType = match[1];
+        }
     }
     if (obj[LOG_LINE_LIMIT_ID]) {
         logType += "::" + String(obj[LOG_LINE_LIMIT_ID]);

package/src/diagnostics/logs/errorNotifications/errorDigestEmail.tsx

@@ -23,6 +23,14 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
     }
 
     let errors: {
+        file: string;
+        errorsInFile: number;
+        warningsInFile: number;
+        message: string;
+        messageTime: string;
+    }[] = [];
+    let warnings: {
+        file: string;
         errorsInFile: number;
         warningsInFile: number;
         message: string;
@@ -38,6 +46,8 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
     let corruptWarning = "";
 
     let failingFiles = 0;
+    let errorFiles = 0;
+    let warningFiles = 0;
 
     for (let value of digestInfo.histogram.values()) {
         errorCount += value.unsuppressedErrors;
@@ -52,25 +62,44 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
         if (value.firstCorruptWarning && !corruptWarning) {
             corruptWarning = value.firstCorruptWarning;
         }
-        if (value.unsuppressedErrors > 0) {
-            failingFiles++;
-        }
     }
 
-    for (let value of digestInfo.byFile.values()) {
+    for (let [file, value] of digestInfo.byFile) {
         for (let error of value.latestErrors.slice(-MAX_COUNT_PER_FILE)) {
             errors.push({
+                file,
                 errorsInFile: value.errors,
                 warningsInFile: value.warnings,
                 message: `${error.param0} (${error.__NAME__})`,
                 messageTime: formatDateTime(error.time),
             });
         }
+
+        for (let warning of value.latestWarnings.slice(-MAX_COUNT_PER_FILE)) {
+            warnings.push({
+                file,
+                errorsInFile: value.errors,
+                warningsInFile: value.warnings,
+                message: `${warning.param0} (${warning.__NAME__})`,
+                messageTime: formatDateTime(warning.time),
+            });
+        }
+
+        if (value.errors > 0) {
+            failingFiles++;
+            errorFiles++;
+        }
+        if (value.warnings > 0) {
+            warningFiles++;
+        }
     }
 
     sort(errors, x => -x.errorsInFile);
     errors = errors.slice(0, MAX_COUNT);
 
+    sort(warnings, x => -x.warningsInFile);
+    warnings = warnings.slice(0, MAX_COUNT);
+
     let link = createLink([
         showingManagementURL.getOverride(true),
         managementPageURL.getOverride("ErrorDigestPage"),
@@ -81,7 +110,7 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
     await sendEmail({
         to: notifyEmails,
         fromPrefix: "error-digest",
-        subject: `${errorCount} err | >= ${formatNumber(failingFiles)} lines | ${warningCount} warn${corruptErrors + corruptWarnings > 0 ? ` | ${corruptErrors + corruptWarnings} corrupt` : ""} | ${suppressedErrors + suppressedWarnings} suppressed | ${formatTime(digestInfo.scanDuration)} | ${formatNumber(digestInfo.totalCompressedBytes)} / ${formatNumber(digestInfo.totalUncompressedBytes)} | ${formatNumber(digestInfo.totalFiles)} log files`,
+        subject: `${errorCount} err | ~${formatNumber(failingFiles)} lines | ${warningCount} warn${corruptErrors + corruptWarnings > 0 ? ` | ${corruptErrors + corruptWarnings} corrupt` : ""} | ${suppressedErrors + suppressedWarnings} hid | ${formatTime(digestInfo.scanDuration)} | ${formatNumber(digestInfo.totalUncompressedBytes)} | ${formatNumber(digestInfo.totalFiles)} log files`,
         contents: <div>
             <h2>Error Summary</h2>
             <ul style="list-style-type: none; padding-left: 0;">
@@ -89,12 +118,14 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
                     <strong style="color: #dc3545;">Errors:</strong>
                     <span style="background-color: #dc3545; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold; margin-left: 8px;">{errorCount}</span>
                     <span style="color: #dc3545;"> unsuppressed</span>
+                    <span style="color: #6c757d; margin-left: 8px;">({formatNumber(errorFiles)} files)</span>
                     {suppressedErrors > 0 && <span>, <span style="color: #6c757d;">{formatNumber(suppressedErrors)} suppressed</span></span>}
                 </li>
                 <li>
                     <strong style="color: #fd7e14;">Warnings:</strong>
                     <span style="background-color: #fd7e14; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold; margin-left: 8px;">{warningCount}</span>
                     <span style="color: #fd7e14;"> unsuppressed</span>
+                    <span style="color: #6c757d; margin-left: 8px;">({formatNumber(warningFiles)} files)</span>
                     {suppressedWarnings > 0 && <span>, <span style="color: #6c757d;">{formatNumber(suppressedWarnings)} suppressed</span></span>}
                 </li>
             </ul>
@@ -147,11 +178,12 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
             <a href={link} style="display: block; margin-bottom: 20px; padding: 10px; background-color: #007bff; color: white; text-decoration: none; border-radius: 4px; text-align: center;">View live logs</a>
 
             {errors.length > 0 && <div>
-                <h2 style="color: #495057;">Recent Errors (<span style="color: #dc3545; font-weight: bold;">{errors.length}</span> shown)</h2>
+                <h2 style="color: #495057;">Recent Errors (<span style="color: #dc3545; font-weight: bold;">{errors.length}</span> files shown, <span style="color: #dc3545; font-weight: bold;">{errors.reduce((acc, error) => acc + error.errorsInFile, 0)}</span> errors)</h2>
                 <table style="border-collapse: collapse; width: 100%; margin-top: 10px;">
                     <thead>
                         <tr style="background-color: #495057; color: white;">
                             <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">Time</th>
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">File</th>
                             <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">Message</th>
                             <th style="border: 1px solid #6c757d; padding: 8px; text-align: right; background-color: #dc3545;">Errors in File</th>
                             <th style="border: 1px solid #6c757d; padding: 8px; text-align: right; background-color: #fd7e14;">Warnings in File</th>
@@ -161,6 +193,7 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
                         {errors.map((error, index) => (
                             <tr key={index} style={`background-color: ${index % 2 === 0 ? "#f8f9fa" : "white"}`}>
                                 <td style="border: 1px solid #ddd; padding: 8px;">{error.messageTime}</td>
+                                <td style="border: 1px solid #ddd; padding: 8px; font-family: monospace; font-size: 12px;">{error.file}</td>
                                 <td style="border: 1px solid #ddd; padding: 8px;">{error.message}</td>
                                 <td style="border: 1px solid #ddd; padding: 8px; text-align: right; background-color: #f8d7da; color: #721c24; font-weight: bold;">{error.errorsInFile}</td>
                                 <td style="border: 1px solid #ddd; padding: 8px; text-align: right; background-color: #fff3cd; color: #856404; font-weight: bold;">{error.warningsInFile}</td>
@@ -169,6 +202,32 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
                     </tbody>
                 </table>
             </div>}
+
+            {warnings.length > 0 && <div style="margin-top: 20px;">
+                <h2 style="color: #495057;">Recent Warnings (<span style="color: #fd7e14; font-weight: bold;">{warnings.length}</span> files shown, <span style="color: #fd7e14; font-weight: bold;">{warnings.reduce((acc, warning) => acc + warning.warningsInFile, 0)}</span> warnings)</h2>
+                <table style="border-collapse: collapse; width: 100%; margin-top: 10px;">
+                    <thead>
+                        <tr style="background-color: #495057; color: white;">
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">Time</th>
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">File</th>
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">Message</th>
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: right; background-color: #dc3545;">Errors in File</th>
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: right; background-color: #fd7e14;">Warnings in File</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {warnings.map((warning, index) => (
+                            <tr key={index} style={`background-color: ${index % 2 === 0 ? "#f8f9fa" : "white"}`}>
+                                <td style="border: 1px solid #ddd; padding: 8px;">{warning.messageTime}</td>
+                                <td style="border: 1px solid #ddd; padding: 8px; font-family: monospace; font-size: 12px;">{warning.file}</td>
+                                <td style="border: 1px solid #ddd; padding: 8px;">{warning.message}</td>
+                                <td style="border: 1px solid #ddd; padding: 8px; text-align: right; background-color: #f8d7da; color: #721c24; font-weight: bold;">{warning.errorsInFile}</td>
+                                <td style="border: 1px solid #ddd; padding: 8px; text-align: right; background-color: #fff3cd; color: #856404; font-weight: bold;">{warning.warningsInFile}</td>
+                            </tr>
+                        ))}
+                    </tbody>
+                </table>
+            </div>}
         </div>,
     });
 }

package/src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx

@@ -1,7 +1,14 @@
-import { runDigestLoop } from "./errorDigests";
+import { Querysub } from "../../../4-querysub/QuerysubController";
+import { runDigest, runDigestLoop } from "./errorDigests";
 
 async function main() {
-    await runDigestLoop();
+    if (process.argv.includes("--now")) {
+        await Querysub.hostService("error-digests-now");
+
+        await runDigest();
+    } else {
+        await runDigestLoop();
+    }
 }
 // The digest loop should never exit, and if it does, we probably want to terminate ourselves so that the service manager will restart us, hopefully putting us back in a good state. 
 main().catch(console.error).finally(() => process.exit());

package/src/diagnostics/logs/errorNotifications/errorDigests.tsx

@@ -92,7 +92,7 @@ function getClosest(value: number, choices: number[]) {
     return closest;
 }
 
-async function runDigest() {
+export async function runDigest() {
     console.log("Running error digest gathering");
     // Find the previous day
     let endTime = getClosest(

package/src/misc/cloneHelpers.ts

@@ -1,6 +1,9 @@
 import cborx from "cbor-x";
 import { lazy } from "socket-function/src/caching";
-const cborxInstance = lazy(() => new cborx.Encoder({ structuredClone: true }));
+const cborxInstance = lazy(() => new cborx.Encoder({
+    structuredClone: true,
+
+}));
 export function deepCloneCborx<T>(value: T): T {
     return decodeCborx(encodeCborx(value));
 }