querysub 0.312.0 → 0.313.0

package/src/diagnostics/logs/DiskLoggerPage.tsx

@@ -1,613 +0,0 @@
-import { css, isNode } from "typesafecss";
-
-import { SocketFunction } from "socket-function/SocketFunction";
-import { getSourceVSCodeLink, qreact } from "../../4-dom/qreact";
-import { assertIsManagementUser } from "../managementPages";
-import { LogFile, LogObj, getLogBuffer, getLogFiles, parseLogBuffer } from "./diskLogger";
-import { Querysub, t } from "../../4-querysub/Querysub";
-import { URLParam } from "../../library-components/URLParam";
-import { getSyncedController } from "../../library-components/SyncedController";
-import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
-import { formatDateTime, formatNiceDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
-import { Anchor } from "../../library-components/ATag";
-import { cache, cacheLimited, cacheShallowConfigArgEqual, lazy } from "socket-function/src/caching";
-import { InputLabel, InputLabelURL } from "../../library-components/InputLabel";
-import { DropdownSelector } from "../../library-components/DropdownSelector";
-import { Table, TableType } from "../../5-diagnostics/Table";
-import { LogType } from "../errorLogs/ErrorLogCore";
-import { canHaveChildren } from "socket-function/src/types";
-import { measureBlock, measureWrap } from "socket-function/src/profiling/measure";
-import { binarySearchBasic, list, sort, timeInDay, timeInHour } from "socket-function/src/misc";
-import { NodeViewerController, nodeViewerController } from "../NodeViewer";
-import { red } from "socket-function/src/formatting/logColors";
-import { errorMessage, formatValue, genericFormat, warnMessage } from "../../5-diagnostics/GenericFormat";
-import { hslToRGB } from "socket-function/src/formatting/colors";
-import { InputPicker } from "../../library-components/InputPicker";
-import { ObjectDisplay } from "./ObjectDisplay";
-import { parseAnsiColors, rgbToHsl } from "./ansiFormat";
-import { ShowMore } from "../../library-components/ShowMore";
-import { getNodeIdLocation } from "socket-function/src/nodeCache";
-import { decodeNodeId, encodeNodeId, getMachineId } from "../../-a-auth/certs";
-import { Button } from "../../library-components/Button";
-import { TimeRangeSelector } from "../../library-components/TimeRangeSelector";
-import { BrowserLargeFileCache, cacheCalls } from "./BrowserLargeFileCache";
-import { Zip } from "../../zip";
-
-// TODO: Realtime log mode, by reading from the previous length forward, to add buffers
-//  to what we already read.
-//  - ALSO, correctly handle cut off messages? Or not, if the writes are atomic
-//      it might be fine...
-
-// TODO: Allow showing context for logs, showing filter logs PLUS +/- a certain number of lines
-//  - Also grouping them, so we can see which are part of which.
-//  - Maybe only showing +/- within a time range as well, and maybe only the same file
-
-// TODO: Parse objects in strings and display them nicely
-//  - I don't know if we even want to add filter support, but... making them more readable
-//      should be easy enough, and might help with reading objects in error messages?
-
-// TODO: Track line locations from diskLog
-//      - Probably by injecting ids? Although how would they be persistent? Hmm... Maybe line numbers
-//          is fine? I think we map the typescript (I hope we do), so they should be correct?
-
-// TODO: Use a binary format, but faster parsing.
-//  - If we could batch it, it would probably be very fast. We ARE just dealing with KVPs though,
-//      so... maybe it wouldn't be that much faster...
-
-module.hotreload = true;
-module.hotreload = true;
-
-export let selectedNodeId = new URLParam("nodeId", "");
-
-export let filterURL = new URLParam("filter", "");
-let sortOldestFirst = new URLParam("oldestFirst", false);
-let selectedFields = new URLParam("selectedFields", "");
-
-let startTimeURL = new URLParam("startTime", undefined as number | undefined);
-let endTimeURL = new URLParam("endTime", undefined as number | undefined);
-
-let inspectTimeURL = new URLParam("inspectTime", "");
-
-
-export class DiskLoggerPage extends qreact.Component {
-    state = t.state({
-        downloadFilesHash: t.string,
-    });
-    render() {
-        let selectedNodesIds = selectedNodeId.value.split("|").filter(x => x);
-        const nodeController = nodeViewerController(getBrowserUrlNode());
-        let nodeIds = nodeController.getControllerNodeIdList(DiskLoggerController) || [];
-
-        if (selectedNodesIds.length === 0) {
-            // Group by machineId
-            let grouped = new Map<string, { nodeId: string; entryPoint: string }[]>();
-            for (let { nodeId, entryPoint } of nodeIds) {
-                let obj = decodeNodeId(nodeId);
-                if (!obj) continue;
-                let { machineId } = obj;
-                let list = grouped.get(machineId);
-                if (!list) {
-                    grouped.set(machineId, list = []);
-                }
-                list.push({ nodeId, entryPoint });
-            }
-            return (
-                <div class={css.pad2(10).vbox(10)}>
-                    <h1>Node Ids ({nodeIds.length})</h1>
-                    <div class={css.hbox(40).wrap}>
-                        {Array.from(grouped.entries()).map(([machineId, list]) =>
-
-                            <div class={css.vbox(10).wrap.maxWidth("30vw").pad2(5).hsla(0, 0, 0, 0.1)}>
-                                <div class={css.fontSize(30)}>{getMachineId(machineId)}</div>
-                                <div class={css.hbox(20, 5).wrap}>
-                                    {list.map(({ nodeId, entryPoint }) =>
-                                        <Anchor
-                                            class={css.hbox(10)}
-                                            values={[selectedNodeId.getOverride(nodeId)]}
-                                        >
-                                            {nodeId} ({entryPoint.replaceAll("\\", "/").split("/").pop()})
-                                        </Anchor>
-                                    )}
-                                </div>
-                            </div>
-                        )}
-                    </div>
-                </div>
-            );
-        }
-
-        let startTime = Number(startTimeURL.value) || (Date.now() - timeInDay);
-        let endTime = Number(endTimeURL.value) || (Date.now() + timeInDay);
-        const controller = diskLoggerController(getBrowserUrlNode());
-        let allFiles: (LogFile & { nodeId: string })[] = [];
-        let loadingCount = 0;
-
-        for (let nodeId of selectedNodesIds) {
-            try {
-                let newFiles = controller.getRemoteLogFiles(nodeId);
-                if (newFiles) {
-                    allFiles.push(...newFiles.map(x => ({ ...x, nodeId })));
-                } else {
-                    loadingCount++;
-                }
-            } catch (e) {
-                console.log("Error reading files", nodeId, e);
-            }
-        }
-        //files = files.filter(file => file.startTime <= endTime && file.endTime >= startTime);
-        // File to fix some broken files
-        allFiles = allFiles.filter(file => file.startTime >= +new Date("1980"));
-        let buffers: Buffer[] = [];
-
-        let firstTime = Number.MAX_SAFE_INTEGER;
-        let lastTime = Number.MIN_SAFE_INTEGER;
-
-        let filesHash = JSON.stringify(allFiles.map(x => x.path));
-        let fileSizeSum = 0;
-        let fileCount = 0;
-
-        let selectedFiles: (LogFile & { nodeId: string })[] = [];
-
-        for (let file of allFiles) {
-            try {
-                if (file.startTime < firstTime) firstTime = file.startTime;
-                if (file.endTime > lastTime) lastTime = file.endTime;
-                if (file.startTime > endTime) continue;
-                if (file.endTime < startTime) continue;
-                selectedFiles.push(file);
-                fileCount++;
-                fileSizeSum += file.size;
-                if (filesHash !== this.state.downloadFilesHash) {
-                    loadingCount++;
-                    continue;
-                }
-                let buffer = controller.getRemoteLogBuffer(file.nodeId, file.path);
-                if (!buffer) {
-                    loadingCount++;
-                    continue;
-                }
-                buffer = unzipCached(buffer);
-                buffers.push(buffer);
-            } catch (e) {
-                console.log("Error reading buffer", file, e);
-            }
-        }
-        if (firstTime === Number.MAX_SAFE_INTEGER) firstTime = startTime;
-        if (lastTime === Number.MIN_SAFE_INTEGER) lastTime = endTime;
-
-        let logs: LogObj[] = loadingCount ? [] : parseLogBufferCached(JSON.stringify(allFiles), buffers);
-
-        const defaultFields = ["time", "type", "actions", "param0"];
-        let selectedValues = selectedFields.value.split("|").filter(x => x);
-        let fullSelectedFields = defaultFields.concat(selectedValues);
-
-        let { table, allFields, filteredCount, totalCount } = processLogs({
-            logs,
-            startTime, endTime,
-            order: sortOldestFirst.value ? "oldestFirst" : "newestFirst",
-            filter: filterURL.value,
-            selectedFields: fullSelectedFields,
-            inspectTime: inspectTimeURL.value,
-        });
-
-        function addPath(path: string[]) {
-            let newSelected = Array.from(new Set([...selectedValues, path.join(".")])).join("|");
-            selectedFields.value = newSelected;
-        }
-
-
-        for (let column of Object.values(table.columns)) {
-            if (!column) continue;
-            if (!column.formatter) {
-                column.formatter = (value: unknown) => <ObjectDisplay
-                    value={value}
-                    excludedFields={fullSelectedFields}
-                    onClickValue={(path, value) => {
-                        if (!canHaveChildren(value)) {
-                            // Add to search as well
-                            if (filterURL.value.trim()) {
-                                filterURL.value += " & ";
-                            }
-                            filterURL.value += String(value);
-                        }
-                    }}
-                />;
-            }
-        }
-
-        table.columns["actions"] = {
-            formatter(value, context) {
-                let file = (context as any)?.row.__FILE__;
-                if (!file) return undefined;
-                let line = (context as any)?.row.__LINE__;
-                return (
-                    <div>
-                        <button onClick={() => {
-                            let path = `vscode://file/${file}`;
-                            if (typeof line === "number") {
-                                path += `:${line + 1}`;
-                            }
-                            window.open(path);
-                        }}>Open In VS Code</button>
-                    </div>
-                );
-            },
-        };
-        table.columns["time"] = {
-            formatter(value, context) {
-                return (
-                    <div
-                        className={css.button + (css.background("hsla(0, 0%, 100%, 0.2)", "hover"))}
-                        onClick={() => {
-                            inspectTimeURL.value = String(value) + "|" + 10 + "|" + 10;
-                            filterURL.value = "";
-                        }}
-                    >
-                        {genericFormat(value)}
-                    </div>
-                );
-            },
-        };
-
-        table.columns["type"] = {
-            formatter(value, context) {
-                let inner = (() => {
-                    if (value === "error") return <span className={errorMessage}>{value}</span>;
-                    if (value === "warn") return <span className={warnMessage}>{value}</span>;
-                    if (value === "info" || value === "log") return <span className={css.hsl(210, 50, 50).pad2(6, 4).color("white")}>{value}</span>;
-                    return formatValue(value, undefined, context);
-                })();
-                return <div><div className={css.width(100).hbox0.center}>{inner}</div></div>;
-            }
-        };
-        table.columns["param0"] = {
-            title: "Message",
-            formatter(value, context) {
-                return <ObjectDisplay value={value} />;
-            },
-        };
-
-        table.columns["remaining"] = {
-            title: "Object",
-            formatter(value, context) {
-                return <ObjectDisplay
-                    value={context?.row}
-                    onClickKey={path => addPath(path)}
-                    onClickValue={(path, value) => {
-                        addPath(path);
-                        if (!canHaveChildren(value)) {
-                            // Add to search as well
-                            if (filterURL.value.trim()) {
-                                filterURL.value += " & ";
-                            }
-                            filterURL.value += String(value);
-                        }
-                    }}
-                />;
-            },
-        };
-
-        let [inspectTime, inspectBeforeCount, inspectAfterCount] = inspectTimeURL.value.split("|").map(x => Number(x));
-        if (inspectTime) {
-            startTime = inspectTime;
-            endTime = inspectTime;
-        }
-
-        return (
-            <div class={css.pad2(10).vbox(10)}>
-                <Anchor values={[selectedNodeId.getOverride("")]}><h2>Back</h2></Anchor>
-                <InputPicker
-                    picked={selectedNodesIds}
-                    addPicked={x => selectedNodeId.value = [...selectedNodesIds, x].join("|")}
-                    removePicked={x => selectedNodeId.value = selectedNodesIds.filter(y => y !== x).join("|")}
-                    options={nodeIds.map(({ nodeId, entryPoint }) => ({ value: nodeId, label: `${nodeId} (${entryPoint.replaceAll("\\", "/").split("/").pop()})` }))}
-                />
-                <div className={css.fontSize(28).boldStyle}>
-                    {selectedNodesIds.length} nodes, {fileCount}/{allFiles.length} files, filtered logs {formatNumber(filteredCount)} / {formatNumber(totalCount)}
-                </div>
-                <ShowMore maxHeight={40}>
-                    <div class={css.vbox(4)}>
-                        {selectedFiles.map(file =>
-                            <div class={css.hbox(4)}>
-                                <span>{formatNumber(file.size)}B</span>
-                                <span>{formatDateTime(file.startTime)} - {formatDateTime(file.endTime)}</span>
-                                <span>({file.path})</span>
-                            </div>
-                        )}
-                    </div>
-                </ShowMore>
-                {filesHash !== this.state.downloadFilesHash && <div>
-                    <button onClick={() => {
-                        this.state.downloadFilesHash = filesHash;
-                    }}>
-                        Download Files ({formatNumber(fileSizeSum)}B)
-                    </button>
-                </div>
-                    || loadingCount > 0 && <h1 className={css.hsl(210, 75, 75).pad2(10, 2)}>Downloading {loadingCount} file(s)</h1>
-                }
-                {!inspectTime && <>
-                    <TimeRangeSelector
-                        start={startTimeURL} end={endTimeURL}
-                        defaultStart={startTime} defaultEnd={endTime}
-                        firstTime={firstTime} lastTime={lastTime}
-                    />
-                    <div class={css.hbox(20)}>
-                        <Button onClick={() => { startTimeURL.reset(); endTimeURL.reset(); }}>Reset Time Filter</Button>
-                        <div class={css.hbox(10)}>
-                            <InputLabelURL label="Start Time" url={startTimeURL} isDatetime valueDefault={startTime} />
-                            <span>({formatNiceDateTime(startTime)})</span>
-                        </div>
-                        <div class={css.hbox(10)}>
-                            <InputLabelURL label="End Time" url={endTimeURL} isDatetime valueDefault={endTime} />
-                            <span>({formatNiceDateTime(endTime)})</span>
-                        </div>
-                    </div>
-                    <div className={css.hbox(20)}>
-                        <h2>{formatTime(Date.now() - startTime)} AGO</h2> to <h2>{formatTime(Date.now() - endTime)} AGO</h2>
-                    </div>
-                </> ||
-                    <div class={css.hbox(20)}>
-                        <h2>{formatTime(Date.now() - inspectTime)} AGO</h2>
-                        <h2>{formatNiceDateTime(inspectTime)}</h2>
-                        <InputLabel
-                            label="Inspect Time"
-                            value={inspectTime}
-                            isDatetime
-                            onChange={x => inspectTimeURL.value = x + "|" + inspectBeforeCount + "|" + inspectAfterCount}
-                        />
-                        <button onClick={() => {
-                            inspectTimeURL.value = "";
-                        }}>
-                            Reset Inspect
-                        </button>
-                    </div>
-                }
-
-                <InputLabelURL fillWidth label="Filter: ['|' to match any conditions, '&' to match all, '!' for negation]" url={filterURL} />
-                <InputLabelURL label="Sort Oldest First" checkbox url={sortOldestFirst} />
-                <InputPicker
-                    label="Selected Fields"
-                    picked={selectedValues}
-                    options={Array.from(allFields.keys()).map(x => ({ value: x, label: x }))}
-                    addPicked={x => selectedFields.value = [...selectedValues, x].join("|")}
-                    removePicked={x => selectedFields.value = selectedValues.filter(y => y !== x).join("|")}
-                />
-                {!!inspectTime && (
-                    <InputLabel
-                        label="Before Count"
-                        value={inspectBeforeCount}
-                        onChangeValue={x => inspectTimeURL.value = inspectTime + "|" + x + "|" + inspectAfterCount}
-                        number
-                    />
-                )}
-                <Table {...table} initialLimit={30} />
-                {!!inspectTime && (
-                    <InputLabel
-                        label="After Count"
-                        value={inspectAfterCount}
-                        onChangeValue={x => inspectTimeURL.value = inspectTime + "|" + inspectBeforeCount + "|" + x}
-                        number
-                    />
-                )}
-            </div>
-        );
-    }
-}
-
-const unzipCached = cacheLimited(100, Zip.gunzipSync);
-
-
-let remainingCacheSize = 1024 * 1024 * 1024 * 2;
-let logBufferCache = new Map<string, {
-    size: number;
-    logs: LogObj[];
-    lastAccess: number;
-}>();
-
-const parseLogBufferCached = measureWrap(function parseLogBufferCached(hash: string, buffers: Buffer[]): LogObj[] {
-    let cached = logBufferCache.get(hash);
-    if (cached) {
-        cached.lastAccess = Date.now();
-        return cached.logs;
-    }
-    console.log(`Parsing buffers ${buffers.length}`);
-    let logs = buffers.map(x => parseLogBuffer(x)).flat();
-    let size = buffers.map(x => x.length).reduce((a, b) => a + b, 0);
-    remainingCacheSize -= size;
-    while (remainingCacheSize < 0 && logBufferCache.size > 0) {
-        let oldest = [...logBufferCache.entries()].sort((a, b) => a[1].lastAccess - b[1].lastAccess)[0];
-        logBufferCache.delete(oldest[0]);
-        remainingCacheSize += oldest[1].size;
-    }
-    logBufferCache.set(hash, {
-        size,
-        logs,
-        lastAccess: Date.now()
-    });
-    console.log(`Parsed buffers ${buffers.length} into logs ${logs.length}`);
-    return logs;
-});
-
-
-
-const processLogs = cacheShallowConfigArgEqual((config: {
-    logs: LogObj[];
-    startTime: number; endTime: number;
-    order: "oldestFirst" | "newestFirst";
-    filter: string;
-    selectedFields: string[];
-    inspectTime: string;
-}): {
-    table: TableType<LogObj>;
-    allFields: Map<string, number>;
-    totalCount: number;
-    filteredCount: number;
-} => {
-    return measureBlock(function processLogs() {
-        console.log(`Processing logs ${config.logs.length}`);
-        let { logs, order, filter, startTime, endTime, inspectTime, selectedFields } = config;
-        logs = logs.slice();
-        let totalCount = logs.length;
-
-        let logMatchesFilter: (log: LogObj) => boolean = () => true;
-
-        if (filter) {
-            filter = filter.toLowerCase();
-            let filterParts = filter.split("|").map(x => x.trim()).map(x => x.split("&").map(x => x.trim()));
-            // Extract any negation
-            let negationQueries: string[] = [];
-            filterParts = filterParts.map(filter => filter.filter(x => {
-                if (x.startsWith("!")) {
-                    negationQueries.push(x.slice(1));
-                    return false;
-                }
-                return true;
-            })).filter(x => x.length > 0);
-            function logMatches(filter: string, log: LogObj) {
-                return JSON.stringify(log).toLowerCase().includes(filter);
-            }
-            logMatchesFilter = function logMatchesFilter(log: LogObj) {
-                return (
-                    (filterParts.length === 0 || filterParts.some(filter => filter.every(filter => logMatches(filter, log))))
-                    && !negationQueries.some(filter => logMatches(filter, log))
-                );
-            };
-        }
-
-        function applySort() {
-            measureBlock(function filterSort() {
-                if (order === "oldestFirst") {
-                    sort(logs, x => x.time);
-                } else {
-                    sort(logs, x => -x.time);
-                }
-            });
-        }
-
-
-        if (inspectTime) {
-            applySort();
-            // time|before|after
-            let [time, before, after] = inspectTime.split("|").map(x => Number(x));
-            applySort();
-            measureBlock(function filterLogs() {
-                // Binary search to find the closest index
-                let sortOrder = order === "oldestFirst" ? 1 : -1;
-                let index = binarySearchBasic(logs, x => x.time * sortOrder, time * sortOrder);
-                let startIndex: number;
-                let endIndex: number;
-
-                let newLogs: LogObj[] = [];
-                if (index < 0) {
-                    index = ~index;
-                    startIndex = index - 1;
-                    endIndex = index;
-                } else {
-                    startIndex = index - 1;
-                    endIndex = index + 1;
-                    if (logMatchesFilter(logs[index])) {
-                        newLogs.push(logs[index]);
-                    }
-                }
-
-                while (before > 0 && startIndex > 0) {
-                    startIndex--;
-                    let logBefore = logs[startIndex];
-                    if (logMatchesFilter(logBefore)) {
-                        newLogs.unshift(logBefore);
-                        before--;
-                    }
-                }
-
-                while (after > 0 && endIndex < logs.length) {
-                    let logAfter = logs[endIndex];
-                    if (logMatchesFilter(logAfter)) {
-                        newLogs.push(logAfter);
-                        after--;
-                    }
-                    endIndex++;
-                }
-
-                logs = newLogs;
-            });
-        } else {
-            measureBlock(function filterLogs() {
-                logs = logs.filter(log =>
-                    startTime <= log.time && log.time <= endTime &&
-                    logMatchesFilter(log)
-                );
-            });
-            applySort();
-        }
-
-        let filteredCount = logs.length;
-
-        let fields = new Map<string, number>();
-        for (let log of logs) {
-            for (let key in log) {
-                fields.set(key, (fields.get(key) || 0) + 1);
-            }
-        }
-
-        let table: TableType<LogObj> = {
-            rows: logs,
-            columns: {},
-        };
-
-        for (let key of selectedFields) {
-            table.columns[key] = {};
-        }
-
-        console.log(`Processed logs ${logs.length}`);
-        return { table, allFields: fields, totalCount, filteredCount };
-    });
-}, 10);
-
-
-
-
-class DiskLoggerControllerBase {
-    // Have to forward it, as backend nodes don't have real domains, so HTTPS won't
-    //  work from the browser for them.
-    public async getRemoteLogFiles(nodeId: string): Promise<LogFile[]> {
-        return await DiskLoggerController.nodes[nodeId].getLogFiles();
-    }
-    public async getRemoteLogBuffer(nodeId: string, path: string): Promise<Buffer | undefined> {
-        return await DiskLoggerController.nodes[nodeId].getLogBuffer(path);
-    }
-
-    public async getLogFiles(): Promise<LogFile[]> {
-        return await getLogFiles();
-    }
-    public async getLogBuffer(path: string): Promise<Buffer | undefined> {
-        // Always compress it. If it is in progress it can't be compressed on disk, but
-        //  the compression ratio is so high it's worth it to compress it dynamically now,
-        //  to speed up the network transfer.
-        let buffer = await getLogBuffer(path);
-        if (buffer && path.endsWith(".log")) {
-            buffer = await Zip.gzip(buffer);
-        }
-        return buffer;
-    }
-}
-
-export const DiskLoggerController = SocketFunction.register(
-    "DiskLoggerController-f76a6fdf-3bd5-4bd4-a183-55a8be0a5a32",
-    new DiskLoggerControllerBase(),
-    () => ({
-        getRemoteLogFiles: {},
-        getRemoteLogBuffer: {
-            clientHooks: [cacheCalls]
-        },
-
-        getLogFiles: {},
-        getLogBuffer: {},
-    }),
-    () => ({
-        hooks: [assertIsManagementUser],
-    }),
-    {
-    }
-);
-
-let diskLoggerController = getSyncedController(DiskLoggerController);