querysub 0.343.0 → 0.344.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.343.0",
+  "version": "0.344.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -15,7 +15,8 @@
     "test": "yarn typenode ./test.ts",
     "test3": "yarn typenode ./src/test/test.tsx --local",
     "test2": "yarn typenode ./src/4-dom/qreactTest.tsx --local",
-    "error-watch": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx"
+    "error-watch": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorWatchEntry.tsx",
+    "error-email": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx"
   },
   "bin": {
     "deploy": "./bin/deploy.js",

package/src/diagnostics/logs/diskLogger.ts

@@ -40,10 +40,21 @@ export type LogDatum = Record<string, unknown> & {
 export function getLogHash(obj: LogDatum) {
     return getPathStr2(obj.__threadId || "", obj.time.toString());
 }
+const errorMessageFileRegex = /\n    at .+?\(([^:]+):\d+:\d+\)/;
+
 export function getLogFile(obj: LogDatum) {
     let logType = obj.param0 || "";
     if (obj.__FILE__) {
         logType = String(obj.__FILE__);
+        // /root/machine-services/*/git/
+        if (logType.startsWith("/root/machine-services/")) {
+            logType = logType.split("/").slice(4).join("/");
+        }
+    } else {
+        let match = logType.match(errorMessageFileRegex);
+        if (match) {
+            logType = match[1];
+        }
     }
     if (obj[LOG_LINE_LIMIT_ID]) {
         logType += "::" + String(obj[LOG_LINE_LIMIT_ID]);

package/src/diagnostics/logs/errorNotifications/errorDigestEmail.tsx

@@ -23,6 +23,14 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
     }
 
     let errors: {
+        file: string;
+        errorsInFile: number;
+        warningsInFile: number;
+        message: string;
+        messageTime: string;
+    }[] = [];
+    let warnings: {
+        file: string;
         errorsInFile: number;
         warningsInFile: number;
         message: string;
@@ -38,6 +46,8 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
     let corruptWarning = "";
 
     let failingFiles = 0;
+    let errorFiles = 0;
+    let warningFiles = 0;
 
     for (let value of digestInfo.histogram.values()) {
         errorCount += value.unsuppressedErrors;
@@ -52,25 +62,44 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
         if (value.firstCorruptWarning && !corruptWarning) {
             corruptWarning = value.firstCorruptWarning;
         }
-        if (value.unsuppressedErrors > 0) {
-            failingFiles++;
-        }
     }
 
-    for (let value of digestInfo.byFile.values()) {
+    for (let [file, value] of digestInfo.byFile) {
         for (let error of value.latestErrors.slice(-MAX_COUNT_PER_FILE)) {
             errors.push({
+                file,
                 errorsInFile: value.errors,
                 warningsInFile: value.warnings,
                 message: `${error.param0} (${error.__NAME__})`,
                 messageTime: formatDateTime(error.time),
             });
         }
+
+        for (let warning of value.latestWarnings.slice(-MAX_COUNT_PER_FILE)) {
+            warnings.push({
+                file,
+                errorsInFile: value.errors,
+                warningsInFile: value.warnings,
+                message: `${warning.param0} (${warning.__NAME__})`,
+                messageTime: formatDateTime(warning.time),
+            });
+        }
+
+        if (value.errors > 0) {
+            failingFiles++;
+            errorFiles++;
+        }
+        if (value.warnings > 0) {
+            warningFiles++;
+        }
     }
 
     sort(errors, x => -x.errorsInFile);
     errors = errors.slice(0, MAX_COUNT);
 
+    sort(warnings, x => -x.warningsInFile);
+    warnings = warnings.slice(0, MAX_COUNT);
+
     let link = createLink([
         showingManagementURL.getOverride(true),
         managementPageURL.getOverride("ErrorDigestPage"),
@@ -81,7 +110,7 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
     await sendEmail({
         to: notifyEmails,
         fromPrefix: "error-digest",
-        subject: `${errorCount} err | >= ${formatNumber(failingFiles)} lines | ${warningCount} warn${corruptErrors + corruptWarnings > 0 ? ` | ${corruptErrors + corruptWarnings} corrupt` : ""} | ${suppressedErrors + suppressedWarnings} suppressed | ${formatTime(digestInfo.scanDuration)} | ${formatNumber(digestInfo.totalCompressedBytes)} / ${formatNumber(digestInfo.totalUncompressedBytes)} | ${formatNumber(digestInfo.totalFiles)} log files`,
+        subject: `${errorCount} err | ~${formatNumber(failingFiles)} lines | ${warningCount} warn${corruptErrors + corruptWarnings > 0 ? ` | ${corruptErrors + corruptWarnings} corrupt` : ""} | ${suppressedErrors + suppressedWarnings} hid | ${formatTime(digestInfo.scanDuration)} | ${formatNumber(digestInfo.totalUncompressedBytes)} | ${formatNumber(digestInfo.totalFiles)} log files`,
         contents: <div>
             <h2>Error Summary</h2>
             <ul style="list-style-type: none; padding-left: 0;">
@@ -89,12 +118,14 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
                     <strong style="color: #dc3545;">Errors:</strong>
                     <span style="background-color: #dc3545; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold; margin-left: 8px;">{errorCount}</span>
                     <span style="color: #dc3545;"> unsuppressed</span>
+                    <span style="color: #6c757d; margin-left: 8px;">({formatNumber(errorFiles)} files)</span>
                     {suppressedErrors > 0 && <span>, <span style="color: #6c757d;">{formatNumber(suppressedErrors)} suppressed</span></span>}
                 </li>
                 <li>
                     <strong style="color: #fd7e14;">Warnings:</strong>
                     <span style="background-color: #fd7e14; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold; margin-left: 8px;">{warningCount}</span>
                     <span style="color: #fd7e14;"> unsuppressed</span>
+                    <span style="color: #6c757d; margin-left: 8px;">({formatNumber(warningFiles)} files)</span>
                     {suppressedWarnings > 0 && <span>, <span style="color: #6c757d;">{formatNumber(suppressedWarnings)} suppressed</span></span>}
                 </li>
             </ul>
@@ -147,11 +178,12 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
             <a href={link} style="display: block; margin-bottom: 20px; padding: 10px; background-color: #007bff; color: white; text-decoration: none; border-radius: 4px; text-align: center;">View live logs</a>
 
             {errors.length > 0 && <div>
-                <h2 style="color: #495057;">Recent Errors (<span style="color: #dc3545; font-weight: bold;">{errors.length}</span> shown)</h2>
+                <h2 style="color: #495057;">Recent Errors (<span style="color: #dc3545; font-weight: bold;">{errors.length}</span> files shown, <span style="color: #dc3545; font-weight: bold;">{errors.reduce((acc, error) => acc + error.errorsInFile, 0)}</span> errors)</h2>
                 <table style="border-collapse: collapse; width: 100%; margin-top: 10px;">
                     <thead>
                         <tr style="background-color: #495057; color: white;">
                             <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">Time</th>
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">File</th>
                             <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">Message</th>
                             <th style="border: 1px solid #6c757d; padding: 8px; text-align: right; background-color: #dc3545;">Errors in File</th>
                             <th style="border: 1px solid #6c757d; padding: 8px; text-align: right; background-color: #fd7e14;">Warnings in File</th>
@@ -161,6 +193,7 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
                         {errors.map((error, index) => (
                             <tr key={index} style={`background-color: ${index % 2 === 0 ? "#f8f9fa" : "white"}`}>
                                 <td style="border: 1px solid #ddd; padding: 8px;">{error.messageTime}</td>
+                                <td style="border: 1px solid #ddd; padding: 8px; font-family: monospace; font-size: 12px;">{error.file}</td>
                                 <td style="border: 1px solid #ddd; padding: 8px;">{error.message}</td>
                                 <td style="border: 1px solid #ddd; padding: 8px; text-align: right; background-color: #f8d7da; color: #721c24; font-weight: bold;">{error.errorsInFile}</td>
                                 <td style="border: 1px solid #ddd; padding: 8px; text-align: right; background-color: #fff3cd; color: #856404; font-weight: bold;">{error.warningsInFile}</td>
@@ -169,6 +202,32 @@ export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
                     </tbody>
                 </table>
             </div>}
+
+            {warnings.length > 0 && <div style="margin-top: 20px;">
+                <h2 style="color: #495057;">Recent Warnings (<span style="color: #fd7e14; font-weight: bold;">{warnings.length}</span> files shown, <span style="color: #fd7e14; font-weight: bold;">{warnings.reduce((acc, warning) => acc + warning.warningsInFile, 0)}</span> warnings)</h2>
+                <table style="border-collapse: collapse; width: 100%; margin-top: 10px;">
+                    <thead>
+                        <tr style="background-color: #495057; color: white;">
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">Time</th>
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">File</th>
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">Message</th>
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: right; background-color: #dc3545;">Errors in File</th>
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: right; background-color: #fd7e14;">Warnings in File</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {warnings.map((warning, index) => (
+                            <tr key={index} style={`background-color: ${index % 2 === 0 ? "#f8f9fa" : "white"}`}>
+                                <td style="border: 1px solid #ddd; padding: 8px;">{warning.messageTime}</td>
+                                <td style="border: 1px solid #ddd; padding: 8px; font-family: monospace; font-size: 12px;">{warning.file}</td>
+                                <td style="border: 1px solid #ddd; padding: 8px;">{warning.message}</td>
+                                <td style="border: 1px solid #ddd; padding: 8px; text-align: right; background-color: #f8d7da; color: #721c24; font-weight: bold;">{warning.errorsInFile}</td>
+                                <td style="border: 1px solid #ddd; padding: 8px; text-align: right; background-color: #fff3cd; color: #856404; font-weight: bold;">{warning.warningsInFile}</td>
+                            </tr>
+                        ))}
+                    </tbody>
+                </table>
+            </div>}
         </div>,
     });
 }

package/src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx

@@ -1,7 +1,14 @@
-import { runDigestLoop } from "./errorDigests";
+import { Querysub } from "../../../4-querysub/QuerysubController";
+import { runDigest, runDigestLoop } from "./errorDigests";
 
 async function main() {
-    await runDigestLoop();
+    if (process.argv.includes("--now")) {
+        await Querysub.hostService("error-digests-now");
+
+        await runDigest();
+    } else {
+        await runDigestLoop();
+    }
 }
 // The digest loop should never exit, and if it does, we probably want to terminate ourselves so that the service manager will restart us, hopefully putting us back in a good state. 
 main().catch(console.error).finally(() => process.exit());

package/src/diagnostics/logs/errorNotifications/errorDigests.tsx

@@ -92,7 +92,7 @@ function getClosest(value: number, choices: number[]) {
     return closest;
 }
 
-async function runDigest() {
+export async function runDigest() {
     console.log("Running error digest gathering");
     // Find the previous day
     let endTime = getClosest(

package/src/misc/cloneHelpers.ts

@@ -1,6 +1,9 @@
 import cborx from "cbor-x";
 import { lazy } from "socket-function/src/caching";
-const cborxInstance = lazy(() => new cborx.Encoder({ structuredClone: true }));
+const cborxInstance = lazy(() => new cborx.Encoder({
+    structuredClone: true,
+
+}));
 export function deepCloneCborx<T>(value: T): T {
     return decodeCborx(encodeCborx(value));
 }