querysub 0.312.0 → 0.313.0

package/src/diagnostics/logs/diskLogger.ts

@@ -1,88 +1,72 @@
-
-import { batchFunction, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { nextId, timeInDay, timeInHour } from "socket-function/src/misc";
-import { getStorageDir, getStorageFolder } from "../../fs";
-import fs from "fs";
-import { canHaveChildren } from "socket-function/src/types";
-import { SizeLimiter } from "../SizeLimiter";
-import { SocketFunction } from "socket-function/SocketFunction";
 import { isNode } from "typesafecss";
-import { logGitHashes } from "./logGitHashes";
-import { Zip } from "../../zip";
-import { formatNumber } from "socket-function/src/formatting/format";
+import { partialCopyObject } from "../../misc";
+import { canHaveChildren } from "socket-function/src/types";
+import { lazy } from "socket-function/src/caching";
+import { timeInMinute } from "socket-function/src/misc";
+import { formatTime } from "socket-function/src/formatting/format";
+import { addEpsilons } from "../../bits";
+import { FileMetadata } from "./FastArchiveController";
+// IMPORTANT! We can't have any real imports here, because we are depended on so early in startup!
 
 if (isNode()) {
     // Delayed setup, as we depend on diskLogger early, and we don't want to force high level
     //  modules to be required before their level
     setImmediate((async () => {
-        await import("./DiskLoggerPage");
         const { addBuiltInContext } = await import("./diskLogGlobalContext");
         addBuiltInContext();
+        const { logGitHashes } = await import("./logGitHashes");
         await logGitHashes();
     }));
 }
 
-// TODO: If we run into size problems with our logs, we might want to use compression?
-//  - We want chunks to be < 4096, but... we might still be able to compress a bit?
-
-let folder = getStorageFolder("disklogs");
-
-let SIZE_LIMIT = new SizeLimiter({
-    diskRoot: getStorageDir(),
-
-    maxBytes: 1024 * 1024 * 1024 * 1,
-    minBytes: 1024 * 1024 * 10,
-    maxDiskFraction: 0.02,
-    maxTotalDiskFraction: 0.95,
-    maxFiles: 1000,
-});
-
-const LOG_FILE_DURATION = timeInDay;
-
-export type LogObj = {
-    [key: string]: unknown;
+// NOTE: When logging we spread objects. If we encounter strings, we set the field `param${index}`
+export type LogDatum = Record<string, unknown> & {
     time: number;
+    __LOG_TYPE: string;
+    __machineId?: string;
+    __threadId?: string;
+    __entry?: string;
+    param0?: string;
+    __DIR__?: string;
+    __NAME__?: string;
+    __LINE__?: string;
+    /** Dynamically set at runtime in the frontend. */
+    __metadata?: FileMetadata;
+    /** Dynamically set when matching recent errors only. */
+    __matchedOutdatedSuppressionKey?: string;
 };
-
-// NOTE: This is visible, otherwise it's easy to accidentally copy it, and not know why
-//  the text is behaving strangely (not === other seemingly equal text, etc).
-// NOTE: Also hardcoded in measure.ts (in socket-function)
-export const noDiskLogPrefix = "█ ";
-export const diskLog = logDisk;
-export function logDisk(...args: unknown[]) {
-    if (!isNode()) return;
-    try {
-        if (args.length === 0) return;
-        // Move the first string argument to the front
-        if (args.length > 0 && typeof args[0] !== "string" && args.some(x => typeof x === "string")) {
-            let strIndex = args.findIndex(x => typeof x === "string");
-            let str = args[strIndex];
-            args.splice(strIndex, 1);
-            args.unshift(str);
-        }
-        if (args.length > 0 && typeof args[0] === "string" && args[0].trim().length === 0) return;
-        if (args.length > 0 && typeof args[0] === "string" && args[0].startsWith(noDiskLogPrefix)) return;
-        let logObj = packageLogObj(args);
-        logQueue(logObj);
-    } catch (e: any) {
-        process.stdout.write(`Error writing to disk logs: ${e.stack || e}\n\t${String(args[0])}\n`);
+export const LOG_LIMIT_FLAG = String.fromCharCode(44533) + "LOGS_LIMITED_FLAG-9277640b-d709-4591-ab08-2bb29bbb94f4";
+
+export const getLoggers = lazy(function () {
+    const { FastArchiveAppendable } = require("./FastArchiveAppendable") as typeof import("./FastArchiveAppendable");
+    if (!FastArchiveAppendable) {
+        setImmediate(() => {
+            getLoggers.reset();
+        });
+        return undefined;
     }
-}
-let logPending: LogObj[] = [];
-function logQueue(log: LogObj) {
-    if (logPending.length === 0) {
-        void Promise.resolve().finally(logClearQueue);
+    return {
+        logLogs: new FastArchiveAppendable<LogDatum>("logs-log/"),
+        warnLogs: new FastArchiveAppendable<LogDatum>("logs-warn/"),
+        infoLogs: new FastArchiveAppendable<LogDatum>("logs-info/"),
+        errorLogs: new FastArchiveAppendable<LogDatum>("logs-error/"),
+    };
+});
+const getNotifyErrors = lazy(function () {
+    const { notifyWatchersOfError: notifyErrors } = require("./errorNotifications/ErrorNotificationController") as typeof import("./errorNotifications/ErrorNotificationController");
+    if (typeof notifyErrors !== "function") {
+        setImmediate(() => {
+            getNotifyErrors.reset();
+        });
+        return undefined;
     }
-    logPending.push(log);
-}
-function logClearQueue() {
-    let commit = logPending;
-    logPending = [];
-    logBase(commit).catch(e => {
-        process.stdout.write(`Error writing to disk logs: ${e.stack || e}\n\t${JSON.stringify(commit[0])}\n`);
-    });
-}
+    return notifyErrors;
+});
 
+// NOTE: If any message (first param) starts with this, we don't log it to the disk. VERY useful for multi-line logging where it wouldn't make sense in the logs
+// NOTE: This is visible, otherwise it's easy to accidentally copy it, and not know why the text is behaving strangely (not === other seemingly equal text, etc).
+// NOTE: Also hardcoded in measure.ts (in socket-function)
+export const noDiskLogPrefix = "█ ";
 
 let globalContextParts: (() => { [key: string]: unknown })[] = [];
 export function addGlobalContext(fnc: () => { [key: string]: unknown }) {
@@ -92,267 +76,113 @@ export function addGlobalContext(fnc: () => { [key: string]: unknown }) {
     globalContextParts.push(fnc);
 }
 
+let startupDone = false;
+void Promise.resolve().then(() => {
+    startupDone = true;
+});
 
+let logLimitLookup: {
+    resetTime: number;
+    counts: Map<string, number>;
+} | undefined;
 
-export type LogFile = {
-    startTime: number;
-    endTime: number;
-    path: string;
-    name: string;
-
-    // Only set in getLogFiles
-    size: number;
-};
+const LIMIT_PERIOD = timeInMinute * 15;
+const LIMIT_THRESHOLD = 1000;
 
-export async function getLogFiles(): Promise<LogFile[]> {
-    let files = await fs.promises.readdir(folder);
-    let paths = files.map(file => folder + file);
-    paths = paths.filter(x => x.endsWith(".log") || x.endsWith(".zip"));
-    let objs = paths.map(decodeLogFileName);
-    for (let obj of objs) {
-        try {
-            let stat = await fs.promises.stat(obj.path);
-            obj.size = stat.size;
-        } catch { }
-    }
-    return objs;
-}
-export async function getLogBuffer(path: string): Promise<Buffer | undefined> {
-    if (!path.startsWith(folder)) throw new Error(`Path must start with ${folder}`);
-    let buffer: Buffer | undefined;
+const logDiskDontShim = logDisk;
+/** NOTE: Calling this directly means we lose __FILE__ tracking. But... that's probably fine... */
+export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknown[]) {
+    if (!isNode()) return;
     try {
-        buffer = await fs.promises.readFile(path);
-    } catch { }
-    return buffer;
-}
-export function parseLogBuffer(buffer: Buffer): LogObj[] {
-    let time = Date.now();
-    console.log(`Parsing buffer ${buffer.length}`);
-    let pos = 0;
-    let logs: LogObj[] = [];
-    const newLine = "\n".charCodeAt(0);
-    let lastLoggedPos = 0;
-    while (pos < buffer.length) {
-        let end = buffer.indexOf(newLine, pos);
-        if (end === -1) {
-            end = buffer.length;
-        }
-        let line = buffer.slice(pos, end).toString("utf8");
-        try {
-            logs.push(JSON.parse(line));
-        } catch { }
-        pos = end + 1;
-        if (pos - lastLoggedPos > 1_000_000) {
-            console.log(`Parsed ${pos} of ${buffer.length}`);
-            lastLoggedPos = pos;
-        }
-    }
-    time = Date.now() - time;
-    console.log(`Parsed ${pos} of ${buffer.length} ${formatNumber(time)}`);
-    return logs;
-}
+        if (args.length === 0) return;
+        let logType = args.find(x => typeof x === "string") as string | undefined;
+        if (!logType) return;
+        if (logType.startsWith(noDiskLogPrefix)) return;
 
-function safeCopyObject<T>(obj: T): T {
-    try {
-        const maxFields = 500;
-        let fieldCount = 0;
-        const seen = new WeakSet();
+        let logObj = packageLogObj(type, args);
 
-        function copy(value: unknown): unknown {
-            // Handle primitives
-            if (!canHaveChildren(value)) {
-                return value;
-            }
+        if (logObj.__FILE__) {
+            logType = String(logObj.__FILE__);
+        }
 
-            // Check for circular references
-            if (seen.has(value as object)) {
-                return null;
+        if (logLimitLookup) {
+            if (logObj.time > logLimitLookup.resetTime) {
+                logLimitLookup = undefined;
             }
-            seen.add(value as object);
+        }
+        if (!logLimitLookup) {
+            logLimitLookup = {
+                resetTime: logObj.time + LIMIT_PERIOD,
+                counts: new Map(),
+            };
+        }
 
-            // Check if we've hit the field limit
-            if (fieldCount >= maxFields) {
-                return null;
-            }
+        let count = logLimitLookup.counts.get(logType) || 0;
+        count++;
+        logLimitLookup.counts.set(logType, count);
+        if (count > LIMIT_THRESHOLD) {
+            let timeUntilReset = logLimitLookup.resetTime - logObj.time;
+            process.stdout.write(`Log type hit limit, not writing log type to disk for ~${formatTime(timeUntilReset)}: ${logType}\n`);
+            return;
+        }
+        if (count === LIMIT_THRESHOLD) {
+            logObj[LOG_LIMIT_FLAG] = true;
+        }
 
-            if (Array.isArray(value)) {
-                const result: unknown[] = [];
-                for (const item of value) {
-                    fieldCount++;
-                    if (fieldCount >= maxFields) break;
-                    result.push(copy(item));
-                }
-                seen.delete(value);
-                return result;
-            }
+        let loggers = startupDone ? getLoggers() : undefined;
+        if (!loggers) {
+            getLoggers.reset();
+            setImmediate(() => {
+                logDiskDontShim(type, ...args);
+            });
+            return;
+        }
+        const { logLogs, warnLogs, infoLogs, errorLogs } = loggers;
+        if (type === "log") {
+            logLogs.append(logObj);
+        } else if (type === "warn") {
+            warnLogs.append(logObj);
+        } else if (type === "info") {
+            infoLogs.append(logObj);
+        } else {
+            errorLogs.append(logObj);
+        }
 
-            // Handle plain objects
-            const result: Record<string, unknown> = {};
-            for (const key of Object.keys(value as object)) {
-                fieldCount++;
-                if (fieldCount >= maxFields) break;
-                result[key] = copy((value as Record<string, unknown>)[key]);
-            }
-            seen.delete(value);
-            return result;
+        if (type === "warn" || type === "error") {
+            // Dropping notifies is fine, as long as they get added to the logs, we'll see them eventually...
+            void getNotifyErrors()?.(logObj);
         }
 
-        return copy(obj) as any;
     } catch (e: any) {
-        return { errorCopying: e.messsage } as any;
+        process.stdout.write(`Error writing to disk logs: ${e.stack || e}\n\t${String(args[0])}\n`);
     }
 }
-let __baseThreadId = nextId();
-function packageLogObj(args: unknown[]): LogObj {
-    let logObj: LogObj = {
-        time: Date.now(),
-        __baseThreadId,
+
+let lastLogTime = 0;
+
+function packageLogObj(type: string, args: unknown[]): LogDatum {
+    let now = Date.now();
+    if (now < lastLogTime) {
+        now = addEpsilons(lastLogTime, 1);
+    }
+    lastLogTime = now;
+    let logObj: LogDatum = {
+        time: now,
+        __LOG_TYPE: type,
     };
     for (let part of globalContextParts) {
-        Object.assign(logObj, safeCopyObject(part()));
+        Object.assign(logObj, partialCopyObject(part(), 500));
     }
-    args = args.map(safeCopyObject);
+    args = args.map(x => partialCopyObject(x, 500));
+    let stringCount = 0;
     for (let i = 0; i < args.length; i++) {
         let param = args[i];
         if (canHaveChildren(param)) {
             Object.assign(logObj, param);
         } else {
-            logObj["param" + i] = param;
+            logObj["param" + stringCount] = param;
+            stringCount++;
         }
     }
     return logObj;
-}
-
-
-function decodeLogFileName(path: string): LogFile {
-    let name = path.split("/").pop()!;
-    let withoutExt = name.split(".").slice(0, -1).join(".");
-    let [start, end] = withoutExt.split("-").map(Number);
-    return {
-        startTime: start,
-        endTime: end,
-        name,
-        path,
-        size: 0,
-    };
-}
-
-// NOTE: Very little delay, so that during shutdown/crashes we don't lose too much before
-//  the crash. One of the biggest reasons for logs is to diagnose crashes, so this is important!
-// NOTE: Not async, so we can properly measure the synchronous overhead of logging (otherwise
-//  we are just measuring disk latency...)
-const logBase = batchFunction({ delay: 0 }, function logBase(logObjList: LogObj[][]) {
-    let logs = logObjList.flat();
-
-    let byLogPath = new Map<string, LogObj[]>();
-    for (let log of logs) {
-        function createLogFileName(logObj: LogObj): LogFile {
-            let start = Math.floor(logObj.time / LOG_FILE_DURATION) * LOG_FILE_DURATION;
-            let startTime = start;
-            let endTime = start + LOG_FILE_DURATION;
-            let name = startTime + "-" + endTime + ".log";
-            let path = folder + name;
-            let logFile: LogFile = { startTime, endTime, name, path, size: 0, };
-            return logFile;
-        }
-        let logFile = createLogFileName(log);
-        let list = byLogPath.get(logFile.path);
-        if (!list) {
-            list = [];
-            byLogPath.set(logFile.path, list);
-        }
-        list.push(log);
-    }
-
-    for (let [path, logList] of byLogPath) {
-        // Apparently, anything more than this and our writes might not be atomic
-        const WRITE_ATOMIC_LIMIT = 4096;
-        let lines = logList.map(v => Buffer.from(JSON.stringify(v) + "\n"));
-
-        // Group lines into WRITE_ATOMIC_LIMIT byte chunks
-        let chunks: Buffer[][] = [];
-        let currentChunk: Buffer[] = [];
-        let currentSize = 0;
-        for (let line of lines) {
-            if (currentSize + line.length > WRITE_ATOMIC_LIMIT && currentChunk.length > 0) {
-                chunks.push(currentChunk);
-                currentChunk = [];
-                currentSize = 0;
-            }
-            currentChunk.push(line);
-            currentSize += line.length;
-        }
-        if (currentChunk.length > 0) {
-            chunks.push(currentChunk);
-        }
-
-        for (let chunk of chunks) {
-            void logStreamWrites(path, Buffer.concat(chunk));
-        }
-    }
-});
-const logStreamWrites = runInSerial(async function (path: string, chunk: Buffer) {
-    await fs.promises.appendFile(path, chunk);
-});
-
-
-if (isNode()) {
-    runInfinitePoll(timeInHour, async function compressLogs() {
-        // Maintain our size restrictions
-        let logFiles = await fs.promises.readdir(folder);
-        let objs: { time: number; bytes: number; path: string; }[] = [];
-        for (let file of logFiles) {
-            let path = folder + file;
-            let stat = await fs.promises.stat(path);
-            objs.push({ time: stat.mtimeMs, bytes: stat.size, path });
-        }
-        let { remove } = await SIZE_LIMIT.limit(objs);
-        for (let file of remove) {
-            await fs.promises.unlink(file.path);
-        }
-    });
-    // Wait a random time, so we hopefully don't synchronize with any other services on this machine
-    runInfinitePoll(timeInHour * (1 + Math.random()), async function compressOldLogs() {
-        let logFiles = await fs.promises.readdir(folder);
-        let compressTime = Date.now() - LOG_FILE_DURATION * 2;
-        let filesCompressed = 0;
-        for (let file of logFiles) {
-            if (!file.endsWith(".log")) continue;
-            if (filesCompressed === 0) {
-                console.log("Compressing old logs");
-            }
-            filesCompressed++;
-            let path = folder + file;
-            if (decodeLogFileName(path).endTime > compressTime) continue;
-            let basePath = path.split(".").slice(0, -1).join(".");
-            let buffer = await fs.promises.readFile(path);
-            let beforeSize = buffer.length;
-            buffer = await Zip.gzip(buffer);
-            console.log(`Compressed ${file} from ${formatNumber(beforeSize)}B to ${formatNumber(buffer.length)}B`);
-            let tempPath = basePath + Math.random() + ".temp";
-            await fs.promises.writeFile(tempPath, buffer);
-            await fs.promises.rename(tempPath, basePath + ".zip");
-            await fs.promises.unlink(path);
-        }
-        if (filesCompressed > 0) {
-            console.log(`Compressed ${filesCompressed} old log files`);
-        }
-    });
-}
-
-/*
-Append Benchmarks
-    10 processes
-        Windows = 100K/S
-        Linux Digital Ocean = 1M/S
-        Linux PI SD Card = 300K/S
-        Linux PI USB = 300K/S
-    1 process
-        Windows = 40K/S
-        Linux Digital Ocean = 685K/S
-        Linux PI = 200K/S
-
-rm test.txt
-for i in {0..9}; do node -e 'const fs=require("fs");const id='$i';let i=0;const start=Date.now();while(Date.now()-start<5000){fs.appendFileSync("test.txt", `${id},${i++}\n`)}' & done; wait
-node -e 'const fs=require("fs");const seqs=new Map();fs.readFileSync("test.txt","utf8").trim().split("\n").forEach((l,i)=>{const[id,seq]=l.split(",").map(Number);if(!seqs.has(id))seqs.set(id,{last:-1,errs:0});const s=seqs.get(id);if(seq!==s.last+1){console.error(`Error for id ${id} at line ${i}: ${s.last}->${seq}`);s.errs++}s.last=seq});seqs.forEach((v,id)=>console.log(`ID ${id}: final seq ${v.last}, ${v.errs} gaps`))'
-*/
+}

package/src/diagnostics/logs/diskShimConsoleLogs.ts

@@ -1,24 +1,13 @@
 import { canHaveChildren } from "socket-function/src/types";
 import { isNode } from "typesafecss";
 import { red, yellow } from "socket-function/src/formatting/logColors";
+import * as diskLogger from "./diskLogger";
 
-let addLocalLogModule: (typeof addLocalLogPromise extends Promise<infer T> ? T : never) | undefined;
-const addLocalLogPromise = import("../errorLogs/ErrorLogController").then(x => {
-    (addLocalLogModule as any) = x;
-    return x;
-});
 
-let diskLoggerModule: typeof diskLoggerPromise extends Promise<infer T> ? T : never;
-const diskLoggerPromise = import("./diskLogger").then(x => {
-    (diskLoggerModule as any) = x;
-    return x;
-});
 
-
-let shimmed = false;
 export function shimConsoleLogs() {
-    if (shimmed) return;
-    shimmed = true;
+    if ((globalThis as any).shimmedConsoleLogs) return;
+    (globalThis as any).shimmedConsoleLogs = true;
     if (isNode()) {
         process.on("unhandledRejection", async (reason: any, promise) => {
             console.error(`Uncaught promise rejection: ${String(reason.stack || reason)}`);
@@ -36,26 +25,14 @@ export function shimConsoleLogs() {
                     args.length > 0
                     && String(args[0]).trim().length > 0
                 ) {
-                    if (typeof diskLoggerModule?.logDisk === "function") {
-                        // Don't call it directly, so we don't get extra line debug context added to this call
-                        //  (as it wouldn't be useful, as we really want the caller)
-                        let stopDoubleShim = diskLoggerModule.logDisk;
-                        stopDoubleShim(...args, { type: fncName });
-                    }
+                    // Assign to a variable, so it isn't shimmed by injectFileLocationToConsole
+                    const logDiskThatIsntShimmed = diskLogger.logDisk;
+                    logDiskThatIsntShimmed(fncName, ...args);
                 }
 
-                let fileObj = args.find(x => canHaveChildren(x) && x["__FILE__"]);
-
                 // Filter out objects added by injectFileLocationToConsole
                 args = args.filter(x => !(canHaveChildren(x) && x["__FILE__"]));
 
-                if (typeof addLocalLogModule?.addLocalLog === "function") {
-                    if (fncName === "error" || fncName === "warn") {
-                        // ALSO, track the logs in a file, for error notifications, etc
-                        addLocalLogModule.addLocalLog({ message: args.join(" | ") + " | " + fileObj?.["__FILE__"], time: Date.now() }, fncName);
-                    }
-                }
-
                 if (
                     // Make warns / errors use better colors
                     (fncName === "warn" || fncName === "error")