querysub 0.327.0 → 0.329.0

package/src/diagnostics/logs/errorNotifications/errorDigestEmail.tsx

@@ -0,0 +1,174 @@
+import { formatDate, formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { getNotifyEmails } from "../../../config";
+import { sendEmail_postmark } from "../../../email_ims_notifications/postmark";
+import { sendEmail_sendgrid } from "../../../email_ims_notifications/sendgrid";
+import { sendEmail } from "../../../user-implementation/userData";
+import { ErrorDigestInfo } from "./errorDigests";
+import { qreact } from "../../../4-dom/qreact";
+import { LogDatum } from "../diskLogger";
+import { sort } from "socket-function/src/misc";
+import { createLink } from "../../../library-components/ATag";
+import { getErrorLogsLink } from "./ErrorWarning";
+import { managementPageURL, showingManagementURL } from "../../managementPages";
+import { digestKeyURL } from "./ErrorDigestPage";
+import { tabURL } from "../../../library-components/urlResetGroups";
+
+const MAX_COUNT_PER_FILE = 2;
+const MAX_COUNT = 100;
+
+export async function sendErrorDigestEmail(digestInfo: ErrorDigestInfo) {
+    let notifyEmails = getNotifyEmails();
+    if (notifyEmails.length === 0) {
+        throw new Error(`No notify emails set, so we can't send email. Set in querysub.json config file (beside your package.json) with the { "notifyemails": ["email1@example.com", "email2@example.com"] }, or as a command line argument with --notifyemails "email1@example.com"`);
+    }
+
+    let errors: {
+        errorsInFile: number;
+        warningsInFile: number;
+        message: string;
+        messageTime: string;
+    }[] = [];
+    let errorCount = 0;
+    let warningCount = 0;
+    let suppressedErrors = 0;
+    let suppressedWarnings = 0;
+    let corruptErrors = 0;
+    let corruptWarnings = 0;
+
+    let corruptWarning = "";
+
+    let failingFiles = 0;
+
+    for (let value of digestInfo.histogram.values()) {
+        errorCount += value.unsuppressedErrors;
+        warningCount += value.unsuppressedWarnings;
+        suppressedErrors += value.suppressedErrors;
+        suppressedWarnings += value.suppressedWarnings;
+        corruptErrors += value.corruptErrors;
+        corruptWarnings += value.corruptWarnings;
+        if (value.firstCorruptError && !corruptWarning) {
+            corruptWarning = value.firstCorruptError;
+        }
+        if (value.firstCorruptWarning && !corruptWarning) {
+            corruptWarning = value.firstCorruptWarning;
+        }
+        if (value.unsuppressedErrors > 0) {
+            failingFiles++;
+        }
+    }
+
+    for (let value of digestInfo.byFile.values()) {
+        for (let error of value.latestErrors.slice(-MAX_COUNT_PER_FILE)) {
+            errors.push({
+                errorsInFile: value.errors,
+                warningsInFile: value.warnings,
+                message: `${error.param0} (${error.__NAME__})`,
+                messageTime: formatDateTime(error.time),
+            });
+        }
+    }
+
+    sort(errors, x => -x.errorsInFile);
+    errors = errors.slice(0, MAX_COUNT);
+
+    let link = createLink([
+        showingManagementURL.getOverride(true),
+        managementPageURL.getOverride("ErrorDigestPage"),
+        digestKeyURL.getOverride(digestInfo.key),
+        tabURL.getOverride("detail"),
+    ]);
+
+    await sendEmail({
+        to: notifyEmails,
+        fromPrefix: "error-digest",
+        subject: `${errorCount} errors | ${formatNumber(failingFiles)} failing files | ${warningCount} warnings${corruptErrors + corruptWarnings > 0 ? ` | ${corruptErrors + corruptWarnings} corrupt` : ""} | ${formatTime(digestInfo.scanDuration)} | ${formatNumber(digestInfo.totalCompressedBytes)} / ${formatNumber(digestInfo.totalUncompressedBytes)} | ${formatNumber(digestInfo.totalFiles)} files`,
+        contents: <div>
+            <h2>Error Summary</h2>
+            <ul style="list-style-type: none; padding-left: 0;">
+                <li style="margin-bottom: 8px;">
+                    <strong style="color: #dc3545;">Errors:</strong>
+                    <span style="background-color: #dc3545; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold; margin-left: 8px;">{errorCount}</span>
+                    <span style="color: #dc3545;"> unsuppressed</span>
+                    {suppressedErrors > 0 && <span>, <span style="color: #6c757d;">{formatNumber(suppressedErrors)} suppressed</span></span>}
+                </li>
+                <li>
+                    <strong style="color: #fd7e14;">Warnings:</strong>
+                    <span style="background-color: #fd7e14; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold; margin-left: 8px;">{warningCount}</span>
+                    <span style="color: #fd7e14;"> unsuppressed</span>
+                    {suppressedWarnings > 0 && <span>, <span style="color: #6c757d;">{formatNumber(suppressedWarnings)} suppressed</span></span>}
+                </li>
+            </ul>
+
+            {corruptWarning && <div>
+                <h2 style="color: #dc3545;">
+                    Corrupt Lines
+                    <span style="background-color: #dc3545; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold; margin-left: 8px;">{formatNumber(corruptErrors + corruptWarnings)}</span>
+                    , example:
+                </h2>
+                <p style="color: #dc3545; font-weight: bold; background-color: #f8d7da; padding: 10px; border-left: 4px solid #dc3545;">{corruptWarning}</p>
+            </div>}
+
+            <div style="background-color: #f8f9fa; border: 1px solid #dee2e6; border-radius: 8px; padding: 20px; margin-bottom: 20px;">
+                <table style="width: 100%; border-collapse: collapse;">
+                    <tr>
+                        <td style="text-align: center; padding: 10px;">
+                            <div style="color: #6c757d; font-size: 12px; text-transform: uppercase; letter-spacing: 0.5px;">Total Files</div>
+                            <div style="background-color: #17a2b8; color: white; padding: 8px 12px; border-radius: 6px; font-weight: bold; font-size: 18px; margin-top: 4px;">{formatNumber(digestInfo.totalFiles)}</div>
+                        </td>
+                        <td style="text-align: center; padding: 10px;">
+                            <div style="color: #6c757d; font-size: 12px; text-transform: uppercase; letter-spacing: 0.5px;">Compressed Bytes</div>
+                            <div style="background-color: #ffc107; color: #212529; padding: 8px 12px; border-radius: 6px; font-weight: bold; font-size: 18px; margin-top: 4px;">{formatNumber(digestInfo.totalCompressedBytes)}</div>
+                        </td>
+                        <td style="text-align: center; padding: 10px;">
+                            <div style="color: #6c757d; font-size: 12px; text-transform: uppercase; letter-spacing: 0.5px;">Uncompressed Bytes</div>
+                            <div style="background-color: #6f42c1; color: white; padding: 8px 12px; border-radius: 6px; font-weight: bold; font-size: 18px; margin-top: 4px;">{formatNumber(digestInfo.totalUncompressedBytes)}</div>
+                        </td>
+                    </tr>
+                </table>
+                <div style="border-top: 1px solid #dee2e6; padding-top: 15px; margin-top: 15px;">
+                    <table style="width: 100%; border-collapse: collapse;">
+                        <tr>
+                            <td style="text-align: center; padding: 10px; width: 200px;">
+                                <div style="color: #6c757d; font-size: 12px; text-transform: uppercase; letter-spacing: 0.5px;">Scan Duration</div>
+                                <div style="background-color: #007bff; color: white; padding: 8px 12px; border-radius: 6px; font-weight: bold; font-size: 18px; margin-top: 4px; font-family: monospace;">{formatTime(digestInfo.scanDuration)}</div>
+                            </td>
+                            <td style="padding: 10px; vertical-align: top;">
+                                <div style="color: #6c757d; font-size: 12px; text-transform: uppercase; letter-spacing: 0.5px; margin-bottom: 8px;">Time Range</div>
+                                <div style="font-size: 14px;">
+                                    <div style="margin-bottom: 8px;">From:&nbsp;&nbsp;<span style="background-color: #28a745; color: white; padding: 4px 8px; border-radius: 4px; font-weight: bold;">{formatDateTime(digestInfo.startTime)}</span></div>
+                                    <div>To:&nbsp;&nbsp;<span style="background-color: #dc3545; color: white; padding: 4px 8px; border-radius: 4px; font-weight: bold;">{formatDateTime(digestInfo.endTime)}</span></div>
+                                </div>
+                            </td>
+                        </tr>
+                    </table>
+                </div>
+            </div>
+
+            <a href={link} style="display: block; margin-bottom: 20px; padding: 10px; background-color: #007bff; color: white; text-decoration: none; border-radius: 4px; text-align: center;">View live logs</a>
+
+            {errors.length > 0 && <div>
+                <h2 style="color: #495057;">Recent Errors (<span style="color: #dc3545; font-weight: bold;">{errors.length}</span> shown)</h2>
+                <table style="border-collapse: collapse; width: 100%; margin-top: 10px;">
+                    <thead>
+                        <tr style="background-color: #495057; color: white;">
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">Time</th>
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: left;">Message</th>
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: right; background-color: #dc3545;">Errors in File</th>
+                            <th style="border: 1px solid #6c757d; padding: 8px; text-align: right; background-color: #fd7e14;">Warnings in File</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {errors.map((error, index) => (
+                            <tr key={index} style={`background-color: ${index % 2 === 0 ? "#f8f9fa" : "white"}`}>
+                                <td style="border: 1px solid #ddd; padding: 8px;">{error.messageTime}</td>
+                                <td style="border: 1px solid #ddd; padding: 8px;">{error.message}</td>
+                                <td style="border: 1px solid #ddd; padding: 8px; text-align: right; background-color: #f8d7da; color: #721c24; font-weight: bold;">{error.errorsInFile}</td>
+                                <td style="border: 1px solid #ddd; padding: 8px; text-align: right; background-color: #fff3cd; color: #856404; font-weight: bold;">{error.warningsInFile}</td>
+                            </tr>
+                        ))}
+                    </tbody>
+                </table>
+            </div>}
+        </div>,
+    });
+}

package/src/diagnostics/logs/errorNotifications/errorDigests.tsx

@@ -0,0 +1,291 @@
+import { insertIntoSortedList, list, timeInDay, timeInHour } from "socket-function/src/misc";
+import { runScheduler } from "../../../misc2";
+import { getErrorAppendables, getSuppressionFull, getSuppressionListRaw, suppressionList } from "./ErrorNotificationController";
+import { createLogScanner } from "../FastArchiveAppendable";
+import { LogDatum, getLogFile, getLoggers } from "../diskLogger";
+import { FastArchiveAppendableControllerBase, FileMetadata } from "../FastArchiveController";
+import { httpsRequest } from "socket-function/src/https";
+import { Zip } from "../../../zip";
+import { encodeCborx } from "../../../misc/cloneHelpers";
+import { archiveJSONT } from "../../../-a-archives/archivesJSONT";
+import { nestArchives } from "../../../-a-archives/archives";
+import { getArchivesBackblaze } from "../../../-a-archives/archivesBackBlaze";
+import { getDomain } from "../../../config";
+import { getOwnMachineId, getOwnThreadId } from "../../../-a-auth/certs";
+import { sendErrorDigestEmail } from "./errorDigestEmail";
+import { archiveCborT } from "../../../-a-archives/archivesCborT";
+import { Querysub } from "../../../4-querysub/Querysub";
+import { sendEmail } from "../../../user-implementation/userData";
+import { qreact } from "../../../4-dom/qreact";
+import { getSyncedController } from "../../../library-components/SyncedController";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { assertIsManagementUser } from "../../managementPages";
+
+const LATEST_ERRORS_COUNT_PER_FILE = 20;
+
+export type ErrorDigestInfo = {
+    key: string;
+    // timeGroupEnd =>
+    histogram: Map<number, {
+        suppressedErrors: number;
+        unsuppressedErrors: number;
+        suppressedWarnings: number;
+        unsuppressedWarnings: number;
+
+        corruptErrors: number;
+        corruptWarnings: number;
+
+        firstCorruptError?: string;
+        firstCorruptWarning?: string;
+    }>;
+    // file =>
+    byFile: Map<string, {
+        errors: number;
+        warnings: number;
+        // Sorted from oldest to newest
+        latestErrors: LogDatum[];
+        latestWarnings: LogDatum[];
+    }>;
+    totalCompressedBytes: number;
+    totalUncompressedBytes: number;
+    totalFiles: number;
+
+    scanDuration: number;
+    scanStartTime: number;
+    scanEndTime: number;
+    startTime: number;
+    endTime: number;
+};
+class ErrorDigest {
+    public async getDigestKeys() {
+        return errorDigestHistory.keys();
+    }
+    public async getDigest(key: string) {
+        return errorDigestHistory.get(key);
+    }
+}
+
+export const ErrorDigestController = getSyncedController(SocketFunction.register(
+    "ErrorDigestController-e5996b95-dcfc-412e-a104-80ed2c2d5933",
+    new ErrorDigest(),
+    () => ({
+        getDigestKeys: {},
+        getDigest: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    })
+));
+
+export const errorDigestHistory = archiveCborT<ErrorDigestInfo>(() => nestArchives("error-digests/", getArchivesBackblaze(getDomain())));
+
+function getClosest(value: number, choices: number[]) {
+    let dist = Number.POSITIVE_INFINITY;
+    let closest: number = choices[0];
+    for (let choice of choices) {
+        let curDist = Math.abs(value - choice);
+        if (curDist < dist) {
+            dist = curDist;
+            closest = choice;
+        }
+    }
+    return closest;
+}
+
+async function runDigest() {
+    console.log("Running error digest gathering");
+    // Find the previous day
+    let endTime = getClosest(
+        Date.now(),
+        [
+            new Date().setHours(11),
+            new Date(Date.now() - timeInDay).setHours(11),
+        ]
+    );
+    let endDate = new Date(endTime);
+    endDate.setMinutes(0);
+    endDate.setSeconds(0);
+    endDate.setMilliseconds(0);
+    endTime = endDate.getTime();
+    let startTime = new Date(endTime - timeInDay).getTime();
+    let scanStartTime = Date.now();
+
+    let digestInfo: ErrorDigestInfo = {
+        key: `${Date.now()}_${getOwnMachineId()}_${getOwnThreadId()}`,
+        histogram: new Map(),
+        byFile: new Map(),
+        scanDuration: 0,
+        scanStartTime,
+        scanEndTime: Date.now(),
+        startTime,
+        endTime,
+        totalCompressedBytes: 0,
+        totalUncompressedBytes: 0,
+        totalFiles: 0,
+    };
+
+    let entries = await getSuppressionListRaw();
+    let suppressionFull = getSuppressionFull({
+        entries: Object.values(entries.entries),
+        blockTimeRange: {
+            startTime,
+            endTime,
+        },
+    });
+
+    function getTimeGroup(time: number) {
+        return Math.floor((time - startTime) / timeInHour) * timeInHour + startTime;
+    }
+
+
+    let appendables = getErrorAppendables();
+    for (let appendable of appendables) {
+        let isError = true;
+        if (appendable.rootPath.includes("warn")) {
+            isError = false;
+        } else if (appendable.rootPath.includes("error")) {
+            isError = true;
+        } else {
+            throw new Error(`Unhandled appendable root path: ${appendable.rootPath}`);
+        }
+        function registerCount(time: number, isSuppressed: boolean, corruptError?: string) {
+            let timeGroup = getTimeGroup(time);
+            let obj = digestInfo.histogram.get(timeGroup);
+            if (!obj) {
+                obj = {
+                    suppressedErrors: 0,
+                    unsuppressedErrors: 0,
+                    suppressedWarnings: 0,
+                    unsuppressedWarnings: 0,
+                    corruptErrors: 0,
+                    corruptWarnings: 0,
+                };
+                digestInfo.histogram.set(timeGroup, obj);
+            }
+            if (isError) {
+                if (isSuppressed) {
+                    obj.suppressedErrors++;
+                } else {
+                    obj.unsuppressedErrors++;
+                }
+                if (corruptError) {
+                    obj.corruptErrors++;
+                    if (!obj.firstCorruptError) {
+                        obj.firstCorruptError = corruptError;
+                    }
+                }
+            } else {
+                if (isSuppressed) {
+                    obj.suppressedWarnings++;
+                } else {
+                    obj.unsuppressedWarnings++;
+                }
+                if (corruptError) {
+                    obj.corruptWarnings++;
+                    if (!obj.firstCorruptWarning) {
+                        obj.firstCorruptWarning = corruptError;
+                    }
+                }
+            }
+        }
+        console.log(`Gathering files for ${appendable.rootPath}`);
+        let result = await new FastArchiveAppendableControllerBase().startSynchronizeInternal({
+            range: {
+                startTime,
+                endTime,
+            },
+            rootPath: appendable.rootPath,
+        });
+        let filesLeft = result.files.slice();
+        await Promise.all(list(32).map(() => runThread()));
+        async function runThread() {
+            while (true) {
+                let file = filesLeft.shift();
+                if (!file) {
+                    return;
+                }
+                await processFile(file);
+            }
+        }
+        async function processFile(file: FileMetadata) {
+            try {
+                console.log(`Processing file ${file.path}`);
+                let compressed = await httpsRequest(file.url);
+                let data = await Zip.gunzip(compressed);
+                digestInfo.totalCompressedBytes += compressed.length;
+                digestInfo.totalUncompressedBytes += data.length;
+                digestInfo.totalFiles++;
+
+                let callback = createLogScanner({
+                    debugName: "digestScanner",
+                    onParsedData: (posStart, posEnd, buffer) => {
+                        if (buffer === "done") {
+                            return;
+                        }
+                        let result = suppressionFull(posStart, posEnd, buffer);
+                        if (!result) {
+                            registerCount(file.endTime, true);
+                            return;
+                        }
+
+                        let datum: LogDatum;
+                        try {
+                            datum = JSON.parse(buffer.slice(posStart, posEnd).toString()) as LogDatum;
+                        } catch (e: any) {
+                            let message = `Failed to parse log datum in around ${buffer.slice(posStart, posEnd).slice(0, 100).toString("hex")}, error is:\n${e.stack}`;
+                            process.stderr.write(message);
+                            registerCount(file.endTime, false, message);
+                            return;
+                        }
+                        registerCount(datum.time, false);
+
+                        let fileGroup = getLogFile(datum);
+                        let obj = digestInfo.byFile.get(fileGroup);
+                        if (!obj) {
+                            obj = {
+                                errors: 0,
+                                warnings: 0,
+                                latestErrors: [],
+                                latestWarnings: [],
+                            };
+                            digestInfo.byFile.set(fileGroup, obj);
+                        }
+                        if (isError) {
+                            obj.errors++;
+                        } else {
+                            obj.warnings++;
+                        }
+                        let list = isError ? obj.latestErrors : obj.latestWarnings;
+                        if (list.length === 0 || datum.time >= list[0].time) {
+                            // NOTE: This should almost never trigger, so the search, and even worse, the splice, should almost never happen
+                            insertIntoSortedList(list, x => x.time, datum);
+                            if (list.length > LATEST_ERRORS_COUNT_PER_FILE) {
+                                list.splice(0, list.length - LATEST_ERRORS_COUNT_PER_FILE);
+                            }
+                        }
+                    },
+                });
+                await callback(data);
+                await callback("done");
+            } catch (e: any) {
+                console.warn(`Failed to process file ${file.path}, error: ${e.stack}`);
+            }
+            let progress = result.files.length - filesLeft.length + 1;
+            console.log(`Processed file ${file.path} (${progress} / ${result.files.length}) in ${appendable.rootPath}`);
+        }
+    }
+
+
+    let scanEndTime = Date.now();
+    digestInfo.scanDuration = scanEndTime - scanStartTime;
+    digestInfo.scanEndTime = scanEndTime;
+    await errorDigestHistory.set(digestInfo.key, digestInfo);
+    await sendErrorDigestEmail(digestInfo);
+}
+
+export async function runDigestLoop() {
+    await Querysub.hostService("error-digests");
+
+    // TODO: We might want to change the scheduler to run only on some days, adding support to weekday filtering as well (ex, just monday, wednesday, friday)
+    await runScheduler([12], runDigest);
+}

package/src/diagnostics/logs/errorNotifications/errorLoopEntry.tsx

@@ -0,0 +1,7 @@
+import { runDigestLoop } from "./errorDigests";
+
+async function main() {
+    await runDigestLoop();
+}
+// The digest loop should never exit, and if it does, we probably want to terminate ourselves so that the service manager will restart us, hopefully putting us back in a good state. 
+main().catch(console.error).finally(() => process.exit());