querysub 0.2.0

package/src/diagnostics/logs/ObjectDisplay.tsx

@@ -0,0 +1,165 @@
+import { css } from "typesafecss";
+import { qreact } from "../../4-dom/qreact";
+import { canHaveChildren } from "socket-function/src/types";
+import { parseAnsiColors, rgbToHsl } from "./ansiFormat";
+import { ShowMore } from "../../library-components/ShowMore";
+import { formatValue } from "../../5-diagnostics/GenericFormat";
+import { Button } from "../../library-components/Button";
+import { MeasureHeightCSS } from "../../library-components/MeasureHeightCSS";
+
+module.hotreload = true;
+module.hotreload = true;
+
+export class ObjectDisplay extends qreact.Component<{
+    value: unknown;
+    excludedFields?: string[];
+    onClickKey?: (path: string[]) => void;
+    onClickValue?: (path: string[], value: unknown) => void;
+    maxHeight?: number;
+}> {
+    state = {
+        expanded: false,
+    };
+    render() {
+        const maxHeight = this.props.maxHeight ?? 60;
+        let value = this.props.value;
+        let exclude = this.props.excludedFields;
+        let expanded = this.state.expanded;
+        let { onClickKey, onClickValue } = this.props;
+
+        const renderValue = (value: unknown, path: string[]): preact.ComponentChild => {
+            if (path.length === 0 && (typeof value === "string" && !value.includes("\x1b") || typeof value === "number" || typeof value === "boolean")) {
+                return <span
+                    className={css.pad2(4, 2)
+                        + (onClickValue && css.pointer.background("hsla(210, 70%, 90%, 0.5)", "hover"))
+                    }
+                    onClick={() => onClickValue?.([], value)}
+                >
+                    {formatValue(value)}
+                </span>;
+            }
+            if (!canHaveChildren(value)) return <PrimitiveDisplay value={value} />;
+
+            let isArray = Array.isArray(value);
+            let startEnd = isArray ? "[]" : "{}";
+            let spacingCss = css.marginRight(0);
+            let entries = Object.entries(value);
+            entries = entries.filter(([key]) => {
+                let nestedPath = [...path, key];
+                let pathFlat = nestedPath.join(".");
+                return !exclude?.includes(pathFlat);
+            });
+            return <>
+                <span className={spacingCss}>{startEnd[0]}</span>
+                {entries.map(([key, value], index, list) => {
+                    let nestedPath = [...path, key];
+                    return <div className={
+                        (
+                            expanded && css.marginLeft(20)
+                            || css.display("inline")
+                        )
+                        + spacingCss
+                    }>
+                        {!isArray && <>
+                            <span
+                                className={
+                                    css.boldStyle.hslcolor(210, 70, 50)
+                                        .pad2(4, 2)
+                                    + (onClickKey && css.pointer.background("hsla(210, 70%, 90%, 0.5)", "hover"))
+                                }
+                                onClick={() => onClickKey?.(nestedPath)}
+                            >
+                                {key}
+                            </span>
+                            <span className={spacingCss}>:</span>
+                        </>}
+                        <span
+                            className={
+                                css.pad2(4, 2) +
+                                (onClickValue && css.pointer.background("hsla(210, 70%, 90%, 0.5)", "hover"))
+                            }
+                            onClick={() => onClickValue?.(nestedPath, value)}
+                        >
+                            {renderValue(value, nestedPath)}
+                        </span>
+                        {index !== list.length - 1 && <span>,</span>}
+                    </div>;
+                })}
+                <span className={""}>{startEnd[1]}</span>
+            </>;
+        };
+
+        if (!canHaveChildren(value)) {
+            return <ShowMore maxHeight={maxHeight} className={css.maxWidth("50vw")}>
+                <span className={css.maxHeight("70vh").overflowAuto}>
+                    <div class={css.relative}>
+                        {renderValue(value, [])}
+                        <MeasureHeightCSS />
+                    </div>
+                </span>
+            </ShowMore>;
+        }
+        return <div className={
+            css.hbox(6).alignItems("start")
+            + (!expanded && css.maxHeight(maxHeight).overflowHidden)
+            + (expanded && css.maxHeight("70vh").overflowAuto)
+        }>
+            {canHaveChildren(value) && <Button
+                className={
+                    css.hsl(270, 50, 50).background("hsl(270, 50%, 70%)", "hover")
+                        .width(70)
+                        .height(maxHeight)
+                        .flexShrink0
+                        .center
+                }
+                onClick={() => this.state.expanded = !this.state.expanded}
+            >
+                {expanded ? "Collapse" : "Expand"}
+            </Button>}
+            <span className={css.maxWidth("50vw").maxHeight("50vh").pad2(2).overflowAuto}>
+                <div className={css.relative}>
+                    {renderValue(value, [])}
+                    <MeasureHeightCSS />
+                </div>
+            </span>
+        </div>;
+    }
+}
+
+class PrimitiveDisplay extends qreact.Component<{
+    value: unknown;
+}> {
+    render() {
+        let value = this.props.value;
+        if (typeof value === "number") {
+            return <span className={css.boldStyle.hslcolor(265, 70, 60)}>{value}</span>;
+        }
+        if (typeof value === "boolean") {
+            return <span className={css.boldStyle.hslcolor(207, 70, 60)}>{value}</span>;
+        }
+
+        if (typeof value === "string") {
+            let colors = parseAnsiColors(value);
+            if (colors.length === 1 && !colors[0].color) {
+                return <span className={css.boldStyle.hslcolor(30, 70, 50)}>"{value}"</span>;
+            }
+            return (
+                <span className={css.display("inline-flex").rowGap(4).columnGap(1).wrap}>
+                    {colors.map(({ color, text }) => {
+                        let hue = color ? rgbToHsl(color).h : undefined;
+                        return <span
+                            className={
+                                (hue !== undefined && css.boldStyle.pad2(4, 1).hsla(hue, 70, 50, 0.5) || "")
+                                + (hue === undefined && css.boldStyle.hslcolor(30, 70, 50))
+                            }
+                        >{text}</span>;
+                    })}
+                </span>
+            );
+        }
+
+        return (
+            <span className={css.boldStyle.hslcolor(0, 70, 70)}>{value}</span>
+        );
+    }
+}

package/src/diagnostics/logs/ansiFormat.ts

@@ -0,0 +1,108 @@
+module.allowclient = true;
+export interface ColorGroup {
+    text: string;
+    color?: {
+        r: number;
+        g: number;
+        b: number;
+    };
+}
+
+const basicToRGB: Record<number, { r: number; g: number; b: number }> = {
+    30: { r: 0, g: 0, b: 0 },
+    31: { r: 205, g: 49, b: 49 },
+    32: { r: 85, g: 188, b: 64 },
+    33: { r: 229, g: 229, b: 16 },
+    34: { r: 36, g: 114, b: 200 },
+    35: { r: 188, g: 63, b: 188 },
+    36: { r: 17, g: 168, b: 205 },
+    37: { r: 229, g: 229, b: 229 },
+    90: { r: 102, g: 102, b: 102 },
+    91: { r: 241, g: 76, b: 76 },
+    92: { r: 35, g: 209, b: 139 },
+    93: { r: 245, g: 245, b: 67 },
+    94: { r: 59, g: 142, b: 234 },
+    95: { r: 214, g: 112, b: 214 },
+    96: { r: 41, g: 184, b: 219 },
+    97: { r: 255, g: 255, b: 255 },
+};
+
+export function parseAnsiColors(input: string): ColorGroup[] {
+    const result: ColorGroup[] = [];
+    let currentIndex = 0;
+    const regex = /\x1b\[(38;2;(\d+);(\d+);(\d+)m|(\d+)m)/g;
+    let match;
+
+    while ((match = regex.exec(input)) !== null) {
+        // Add any text before this color code
+        if (match.index > currentIndex) {
+            const text = input.slice(currentIndex, match.index);
+            if (text) {
+                result.push({ text });
+            }
+        }
+
+        // Find the end of this colored section
+        const colorStart = match.index + match[0].length;
+        const nextColorIndex = input.indexOf("\x1b", colorStart);
+        const textEnd = nextColorIndex === -1 ? input.length : nextColorIndex;
+        const text = input.slice(colorStart, textEnd);
+
+        if (text) {
+            const [, fullColor, r, g, b, basicCode] = match;
+            if (r && g && b) {
+                result.push({
+                    text,
+                    color: {
+                        r: parseInt(r),
+                        g: parseInt(g),
+                        b: parseInt(b)
+                    }
+                });
+            } else if (basicCode) {
+                const code = parseInt(basicCode);
+                const rgbColor = basicToRGB[code];
+                result.push({
+                    text,
+                    color: rgbColor
+                });
+            }
+        }
+
+        currentIndex = textEnd;
+    }
+
+    // Add any remaining text after the last color code
+    if (currentIndex < input.length) {
+        const text = input.slice(currentIndex);
+        if (text) {
+            result.push({ text });
+        }
+    }
+
+    return result;
+}
+
+
+export function rgbToHsl(color: { r: number; g: number; b: number }) {
+    let { r, g, b } = color;
+    r /= 255;
+    g /= 255;
+    b /= 255;
+    let max = Math.max(r, g, b);
+    let min = Math.min(r, g, b);
+    let h = 0;
+    let s = 0;
+    let l = (max + min) / 2;
+    if (max !== min) {
+        let d = max - min;
+        s = l > 0.5 ? d / (2 - max - min) : d / (max + min);
+        switch (max) {
+            case r: h = (g - b) / d + (g < b ? 6 : 0); break;
+            case g: h = (b - r) / d + 2; break;
+            case b: h = (r - g) / d + 4; break;
+        }
+        h *= 60;
+    }
+    return { h, s, l };
+}

package/src/diagnostics/logs/diskLogGlobalContext.ts

@@ -0,0 +1,38 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { lazy } from "socket-function/src/caching";
+import { getExternalIP } from "socket-function/src/networking";
+import { decodeNodeId } from "../../-a-auth/certs";
+import { getOwnNodeId } from "../../-f-node-discovery/NodeDiscovery";
+import { logErrors } from "../../errors";
+import { addGlobalContext } from "./diskLogger";
+import child_process from "child_process";
+
+export function addBuiltInContext() {
+    addGlobalContext(() => {
+        let nodeId = getOwnNodeId();
+        let nodeParts = decodeNodeId(nodeId);
+        return {
+            __machineId: nodeParts?.machineId,
+            __mountId: SocketFunction.mountedNodeId,
+            __threadId: nodeParts?.threadId,
+            __port: nodeParts?.port,
+            __nodeId: nodeId,
+            __entry: process.argv[1],
+            __hostName: getHostname(),
+            __accountName: getAccountName(),
+            __externalIP: cachedExternalIP(),
+            __os: process.platform,
+            __pid: process.pid,
+        };
+    });
+    const getHostname = lazy(() => child_process.execSync("hostname").toString().trim());
+    const getAccountName = lazy(() => child_process.execSync("whoami").toString().trim());
+    const cachedExternalIP = lazy(() => {
+        logErrors((async () => {
+            let ip = await getExternalIP();
+            cachedExternalIP.set(ip);
+        })());
+        return "";
+    });
+
+}

package/src/diagnostics/logs/diskLogger.ts

@@ -0,0 +1,305 @@
+
+import { batchFunction, runInfinitePoll } from "socket-function/src/batching";
+import { timeInDay, timeInHour } from "socket-function/src/misc";
+import { getStorageDir, getStorageFolder } from "../../fs";
+import fs from "fs";
+import { canHaveChildren } from "socket-function/src/types";
+import { SizeLimiter } from "../SizeLimiter";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { isNode } from "typesafecss";
+import { logGitHashes } from "./logGitHashes";
+
+if (isNode()) {
+    // Delayed setup, as we depend on diskLogger early, and we don't want to force high level
+    //  modules to be required before their level
+    setImmediate((async () => {
+        await import("./DiskLoggerPage");
+        const { addBuiltInContext } = await import("./diskLogGlobalContext");
+        addBuiltInContext();
+        await logGitHashes();
+    }));
+}
+
+// TODO: If we run into size problems with our logs, we might want to use compression?
+//  - We want chunks to be < 4096, but... we might still be able to compress a bit?
+
+let folder = getStorageFolder("disklogs");
+
+let SIZE_LIMIT = new SizeLimiter({
+    diskRoot: getStorageDir(),
+
+    maxBytes: 1024 * 1024 * 1024 * 1,
+    minBytes: 1024 * 1024 * 10,
+    maxDiskFraction: 0.02,
+    maxTotalDiskFraction: 0.95,
+    maxFiles: 1000,
+});
+
+const LOG_FILE_DURATION = timeInDay;
+
+export type LogObj = {
+    [key: string]: unknown;
+    time: number;
+};
+
+export const noDiskLogPrefix = "\u200C";
+
+export const diskLog = logDisk;
+export function logDisk(...args: unknown[]) {
+    if (!isNode()) return;
+    try {
+        if (args.length === 0) return;
+        // Move the first string argument to the front
+        if (args.length > 0 && typeof args[0] !== "string" && args.some(x => typeof x === "string")) {
+            let strIndex = args.findIndex(x => typeof x === "string");
+            let str = args[strIndex];
+            args.splice(strIndex, 1);
+            args.unshift(str);
+        }
+        if (args.length > 0 && typeof args[0] === "string" && args[0].trim().length === 0) return;
+        if (args.length > 0 && typeof args[0] === "string" && args[0].startsWith(noDiskLogPrefix)) return;
+        let logObj = packageLogObj(args);
+        logQueue(logObj);
+    } catch (e: any) {
+        process.stdout.write(`Error writing to disk logs: ${e.stack || e}\n\t${String(args[0])}\n`);
+    }
+}
+let logPending: LogObj[] = [];
+function logQueue(log: LogObj) {
+    if (logPending.length === 0) {
+        void Promise.resolve().finally(logClearQueue);
+    }
+    logPending.push(log);
+}
+function logClearQueue() {
+    let commit = logPending;
+    logPending = [];
+    logBase(commit).catch(e => {
+        process.stdout.write(`Error writing to disk logs: ${e.stack || e}\n\t${JSON.stringify(commit[0])}\n`);
+    });
+}
+
+
+let globalContextParts: (() => { [key: string]: unknown })[] = [];
+export function addGlobalContext(fnc: () => { [key: string]: unknown }) {
+    // If the contents are identical, don't add it. This stops hot reloading from breaking logging,
+    //  by making the global context unmanageablely large
+    if (globalContextParts.some(v => v.toString() === fnc.toString())) return;
+    globalContextParts.push(fnc);
+}
+
+
+
+export type LogFile = {
+    startTime: number;
+    endTime: number;
+    path: string;
+    name: string;
+
+    // Only set in getLogFiles
+    size: number;
+};
+
+export async function getLogFiles(): Promise<LogFile[]> {
+    let files = await fs.promises.readdir(folder);
+    let paths = files.map(file => folder + file);
+    let objs = paths.map(decodeLogFileName);
+    for (let obj of objs) {
+        try {
+            let stat = await fs.promises.stat(obj.path);
+            obj.size = stat.size;
+        } catch { }
+    }
+    return objs;
+}
+export async function getLogBuffer(path: string): Promise<Buffer | undefined> {
+    if (!path.startsWith(folder)) throw new Error(`Path must start with ${folder}`);
+    let buffer: Buffer | undefined;
+    try {
+        buffer = await fs.promises.readFile(path);
+    } catch { }
+    return buffer;
+}
+export function parseLogBuffer(buffer: Buffer): LogObj[] {
+    let pos = 0;
+    let logs: LogObj[] = [];
+    const newLine = "\n".charCodeAt(0);
+    while (pos < buffer.length) {
+        let end = buffer.indexOf(newLine, pos);
+        if (end === -1) {
+            end = buffer.length;
+        }
+        let line = buffer.slice(pos, end).toString("utf8");
+        try {
+            logs.push(JSON.parse(line));
+        } catch { }
+        pos = end + 1;
+    }
+    return logs;
+}
+
+function safeCopyObject<T>(obj: T): T {
+    const maxFields = 20;
+    let fieldCount = 0;
+    const seen = new WeakSet();
+
+    function copy(value: unknown): unknown {
+        // Handle primitives
+        if (!canHaveChildren(value)) {
+            return value;
+        }
+
+        // Check for circular references
+        if (seen.has(value as object)) {
+            return null;
+        }
+        seen.add(value as object);
+
+        // Check if we've hit the field limit
+        if (fieldCount >= maxFields) {
+            return null;
+        }
+
+        if (Array.isArray(value)) {
+            const result: unknown[] = [];
+            for (const item of value) {
+                fieldCount++;
+                if (fieldCount >= maxFields) break;
+                result.push(copy(item));
+            }
+            seen.delete(value);
+            return result;
+        }
+
+        // Handle plain objects
+        const result: Record<string, unknown> = {};
+        for (const key of Object.keys(value as object)) {
+            fieldCount++;
+            if (fieldCount >= maxFields) break;
+            result[key] = copy((value as Record<string, unknown>)[key]);
+        }
+        seen.delete(value);
+        return result;
+    }
+
+    return copy(obj) as any;
+}
+function packageLogObj(args: unknown[]): LogObj {
+    let logObj: LogObj = {
+        time: Date.now(),
+    };
+    for (let part of globalContextParts) {
+        Object.assign(logObj, safeCopyObject(part()));
+    }
+    args = args.map(safeCopyObject);
+    for (let i = 0; i < args.length; i++) {
+        let param = args[i];
+        if (canHaveChildren(param)) {
+            Object.assign(logObj, param);
+        } else {
+            logObj["param" + i] = param;
+        }
+    }
+    return logObj;
+}
+
+
+function decodeLogFileName(path: string): LogFile {
+    let name = path.split("/").pop()!;
+    let [start, end] = name.split(".log")[0].split("-").map(Number);
+    return {
+        startTime: start,
+        endTime: end,
+        name,
+        path,
+        size: 0,
+    };
+}
+
+// NOTE: Very little delay, so that during shutdown/crashes we don't lose too much before
+//  the crash. One of the biggest reasons for logs is to diagnose crashes, so this is important!
+const logBase = batchFunction({ delay: 0 }, async function logBase(logObjList: LogObj[][]) {
+    let logs = logObjList.flat();
+
+    let byLogPath = new Map<string, LogObj[]>();
+    for (let log of logs) {
+        let logFile = getLogFileName(log);
+        let list = byLogPath.get(logFile.path);
+        if (!list) {
+            list = [];
+            byLogPath.set(logFile.path, list);
+        }
+        list.push(log);
+    }
+
+    for (let [path, logList] of byLogPath) {
+        // Apparently, anything more than this and our writes might not be atomic
+        const WRITE_ATOMIC_LIMIT = 4096;
+        let lines = logList.map(v => Buffer.from(JSON.stringify(v) + "\n"));
+
+        // Group lines into WRITE_ATOMIC_LIMIT byte chunks
+        let chunks: Buffer[][] = [];
+        let currentChunk: Buffer[] = [];
+        let currentSize = 0;
+        for (let line of lines) {
+            if (currentSize + line.length > WRITE_ATOMIC_LIMIT && currentChunk.length > 0) {
+                chunks.push(currentChunk);
+                currentChunk = [];
+                currentSize = 0;
+            }
+            currentChunk.push(line);
+            currentSize += line.length;
+        }
+        if (currentChunk.length > 0) {
+            chunks.push(currentChunk);
+        }
+
+        for (let chunk of chunks) {
+            await fs.promises.appendFile(path, Buffer.concat(chunk));
+        }
+    }
+});
+
+function getLogFileName(logObj: LogObj): LogFile {
+    let start = Math.floor(logObj.time / LOG_FILE_DURATION) * LOG_FILE_DURATION;
+    let startTime = start;
+    let endTime = start + LOG_FILE_DURATION;
+    let name = startTime + "-" + endTime + ".log";
+    let path = folder + name;
+    let logFile: LogFile = { startTime, endTime, name, path, size: 0, };
+    return logFile;
+}
+
+if (isNode()) {
+    runInfinitePoll(timeInHour, async function logClearOldLogs() {
+        // Maintain our size restrictions
+        let logFiles = await fs.promises.readdir(folder);
+        let objs: { time: number; bytes: number; path: string; }[] = [];
+        for (let file of logFiles) {
+            let path = folder + file;
+            let stat = await fs.promises.stat(path);
+            objs.push({ time: stat.mtimeMs, bytes: stat.size, path });
+        }
+        let { remove } = await SIZE_LIMIT.limit(objs);
+        for (let file of remove) {
+            await fs.promises.unlink(file.path);
+        }
+    });
+}
+
+/*
+Append Benchmarks
+    10 processes
+        Windows = 100K/S
+        Linux Digital Ocean = 1M/S
+        Linux PI SD Card = 300K/S
+        Linux PI USB = 300K/S
+    1 process
+        Windows = 40K/S
+        Linux Digital Ocean = 685K/S
+        Linux PI = 200K/S
+
+rm test.txt
+for i in {0..9}; do node -e 'const fs=require("fs");const id='$i';let i=0;const start=Date.now();while(Date.now()-start<5000){fs.appendFileSync("test.txt", `${id},${i++}\n`)}' & done; wait
+node -e 'const fs=require("fs");const seqs=new Map();fs.readFileSync("test.txt","utf8").trim().split("\n").forEach((l,i)=>{const[id,seq]=l.split(",").map(Number);if(!seqs.has(id))seqs.set(id,{last:-1,errs:0});const s=seqs.get(id);if(seq!==s.last+1){console.error(`Error for id ${id} at line ${i}: ${s.last}->${seq}`);s.errs++}s.last=seq});seqs.forEach((v,id)=>console.log(`ID ${id}: final seq ${v.last}, ${v.errs} gaps`))'
+*/

package/src/diagnostics/logs/diskShimConsoleLogs.ts

@@ -0,0 +1,32 @@
+import { canHaveChildren } from "socket-function/src/types";
+import { logDisk } from "./diskLogger";
+
+let shimmed = false;
+export function shimConsoleLogs() {
+    if (shimmed) return;
+    shimmed = true;
+    function hookConsoleFnc(fncName: "log" | "warn" | "error" | "info") {
+        let console = globalThis.console;
+        let originalFnc = console[fncName];
+        console[fncName] = (...args: any[]) => {
+            if (
+                args.length > 0
+                && String(args[0]).trim().length > 0
+            ) {
+                if (typeof logDisk === "function") {
+                    // Don't call it directly, so we don't get extra line debug context added to this call
+                    //  (as it wouldn't be useful, as we really want the caller)
+                    let stopDoubleShim = logDisk;
+                    stopDoubleShim(...args, { type: fncName });
+                }
+            }
+            // Filter out objects added by injectFileLocationToConsole
+            args = args.filter(x => !(canHaveChildren(x) && x["__FILE__"]));
+            return originalFnc(...args);
+        };
+    }
+    hookConsoleFnc("log");
+    hookConsoleFnc("info");
+    hookConsoleFnc("warn");
+    hookConsoleFnc("error");
+}