querysub 0.2.0

package/src/diagnostics/logs/DiskLoggerPage.tsx

@@ -0,0 +1,572 @@
+import { css, isNode } from "typesafecss";
+
+import { SocketFunction } from "socket-function/SocketFunction";
+import { getSourceVSCodeLink, qreact } from "../../4-dom/qreact";
+import { assertIsManagementUser } from "../managementPages";
+import { LogFile, LogObj, getLogBuffer, getLogFiles, parseLogBuffer } from "./diskLogger";
+import { t } from "../../4-querysub/Querysub";
+import { URLParam } from "../../library-components/URLParam";
+import { getSyncedController } from "../../library-components/SyncedController";
+import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
+import { formatDateTime, formatNiceDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { Anchor } from "../../library-components/ATag";
+import { cache, cacheLimited, cacheShallowConfigArgEqual } from "socket-function/src/caching";
+import { InputLabel, InputLabelURL } from "../../library-components/InputLabel";
+import { DropdownSelector } from "../../library-components/DropdownSelector";
+import { Table, TableType } from "../../5-diagnostics/Table";
+import { LogType } from "../errorLogs/ErrorLogCore";
+import { canHaveChildren } from "socket-function/src/types";
+import { measureBlock } from "socket-function/src/profiling/measure";
+import { binarySearchBasic, list, sort, timeInDay, timeInHour } from "socket-function/src/misc";
+import { NodeViewerController } from "../NodeViewer";
+import { red } from "socket-function/src/formatting/logColors";
+import { errorMessage, formatValue, genericFormat, warnMessage } from "../../5-diagnostics/GenericFormat";
+import { hslToRGB } from "socket-function/src/formatting/colors";
+import { InputPicker } from "../../library-components/InputPicker";
+import { ObjectDisplay } from "./ObjectDisplay";
+import { parseAnsiColors, rgbToHsl } from "./ansiFormat";
+import { ShowMore } from "../../library-components/ShowMore";
+import { getNodeIdLocation } from "socket-function/src/nodeCache";
+import { decodeNodeId, encodeNodeId, getMachineId } from "../../-a-auth/certs";
+import { Button } from "../../library-components/Button";
+import { TimeRangeSelector } from "../../library-components/TimeRangeSelector";
+
+// TODO: Realtime log mode, by reading from the previous length forward, to add buffers
+//  to what we already read.
+//  - ALSO, correctly handle cut off messages? Or not, if the writes are atomic
+//      it might be fine...
+
+// TODO: Allow showing context for logs, showing filter logs PLUS +/- a certain number of lines
+//  - Also grouping them, so we can see which are part of which.
+//  - Maybe only showing +/- within a time range as well, and maybe only the same file
+
+// TODO: Parse objects in strings and display them nicely
+//  - I don't know if we even want to add filter support, but... making them more readable
+//      should be easy enough, and might help with reading objects in error messages?
+
+// TODO: Track line locations from diskLog
+//      - Probably by injecting ids? Although how would they be persistent? Hmm... Maybe line numbers
+//          is fine? I think we map the typescript (I hope we do), so they should be correct?
+
+module.hotreload = true;
+module.hotreload = true;
+
+let selectedNodeId = new URLParam("nodeId", "");
+
+let filterURL = new URLParam("filter", "");
+let sortOldestFirst = new URLParam("oldestFirst", false);
+let selectedFields = new URLParam("selectedFields", "");
+
+let startTimeURL = new URLParam("startTime", undefined as number | undefined);
+let endTimeURL = new URLParam("endTime", undefined as number | undefined);
+
+let inspectTimeURL = new URLParam("inspectTime", "");
+
+export class DiskLoggerPage extends qreact.Component {
+    state = t.state({
+        x: t.number
+    });
+    render() {
+        let selectedNodesIds = selectedNodeId.value.split("|").filter(x => x);
+        const nodeController = nodeViewerController(getBrowserUrlNode());
+        let nodeIds = nodeController.getControllerNodeIdList(DiskLoggerController) || [];
+
+        if (selectedNodesIds.length === 0) {
+            // Group by machineId
+            let grouped = new Map<string, string[]>();
+            for (let nodeId of nodeIds) {
+                let obj = decodeNodeId(nodeId);
+                if (!obj) continue;
+                let { machineId } = obj;
+                let list = grouped.get(machineId);
+                if (!list) {
+                    grouped.set(machineId, list = []);
+                }
+                list.push(nodeId);
+            }
+            return (
+                <div class={css.pad2(10).vbox(10)}>
+                    <h1>Node Ids ({nodeIds.length})</h1>
+                    <div class={css.hbox(40).wrap}>
+                        {Array.from(grouped.entries()).map(([machineId, list]) =>
+
+                            <div class={css.vbox(10).wrap.maxWidth("30vw").pad2(5).hsla(0, 0, 0, 0.1)}>
+                                <div class={css.fontSize(30)}>{getMachineId(machineId)}</div>
+                                <div class={css.hbox(20, 5).wrap}>
+                                    {list.map(nodeId =>
+                                        <Anchor
+                                            class={css.hbox(10)}
+                                            values={[selectedNodeId.getOverride(nodeId)]}
+                                        >
+                                            {nodeId}
+                                        </Anchor>
+                                    )}
+                                </div>
+                            </div>
+                        )}
+                    </div>
+                </div>
+            );
+        }
+
+        let startTime = Number(startTimeURL.value) || (Date.now() - timeInDay);
+        let endTime = Number(endTimeURL.value) || (Date.now() + timeInDay);
+        const controller = diskLoggerController(getBrowserUrlNode());
+        let files: (LogFile & { nodeId: string })[] = [];
+        let loadingCount = 0;
+
+        for (let nodeId of selectedNodesIds) {
+            try {
+                let newFiles = controller.getRemoteLogFiles(nodeId);
+                if (newFiles) {
+                    files.push(...newFiles.map(x => ({ ...x, nodeId })));
+                } else {
+                    loadingCount++;
+                }
+            } catch (e) {
+                console.log("Error reading files", nodeId, e);
+            }
+        }
+        let allFiles = files;
+        files = files.filter(file => file.startTime <= endTime && file.endTime >= startTime);
+        let buffers: Buffer[] = [];
+
+        let firstTime = Number.MAX_SAFE_INTEGER;
+        let lastTime = Number.MIN_SAFE_INTEGER;
+
+        for (let file of files) {
+            try {
+                if (file.startTime < firstTime) firstTime = file.startTime;
+                if (file.endTime > lastTime) lastTime = file.endTime;
+                let buffer = controller.getRemoteLogBuffer(file.nodeId, file.path);
+                if (buffer) {
+                    buffers.push(buffer);
+                } else {
+                    loadingCount++;
+                }
+            } catch (e) {
+                console.log("Error reading buffer", file, e);
+            }
+        }
+        if (firstTime === Number.MAX_SAFE_INTEGER) firstTime = startTime;
+        if (lastTime === Number.MIN_SAFE_INTEGER) lastTime = endTime;
+
+        let logs: LogObj[] = loadingCount ? [] : parseLogBufferCached(JSON.stringify(files), buffers);
+
+        const defaultFields = ["time", "type", "actions", "param0"];
+        let selectedValues = selectedFields.value.split("|").filter(x => x);
+        let fullSelectedFields = defaultFields.concat(selectedValues);
+
+        let { table, allFields, filteredCount, totalCount } = processLogs({
+            logs,
+            startTime, endTime,
+            order: sortOldestFirst.value ? "oldestFirst" : "newestFirst",
+            filter: filterURL.value,
+            selectedFields: fullSelectedFields,
+            inspectTime: inspectTimeURL.value,
+        });
+
+        function addPath(path: string[]) {
+            let newSelected = Array.from(new Set([...selectedValues, path.join(".")])).join("|");
+            selectedFields.value = newSelected;
+        }
+
+
+        for (let column of Object.values(table.columns)) {
+            if (!column) continue;
+            if (!column.formatter) {
+                column.formatter = (value: unknown) => <ObjectDisplay
+                    value={value}
+                    excludedFields={fullSelectedFields}
+                    onClickValue={(path, value) => {
+                        if (!canHaveChildren(value)) {
+                            // Add to search as well
+                            if (filterURL.value.trim()) {
+                                filterURL.value += " | ";
+                            }
+                            filterURL.value += String(value);
+                        }
+                    }}
+                />;
+            }
+        }
+
+        table.columns["actions"] = {
+            formatter(value, context) {
+                let file = (context as any)?.row.__FILE__;
+                if (!file) return undefined;
+                let line = (context as any)?.row.__LINE__;
+                return (
+                    <div>
+                        <button onClick={() => {
+                            let path = `vscode://file/${file}`;
+                            if (typeof line === "number") {
+                                path += `:${line + 1}`;
+                            }
+                            window.open(path);
+                        }}>Open In VS Code</button>
+                    </div>
+                );
+            },
+        };
+        table.columns["time"] = {
+            formatter(value, context) {
+                return (
+                    <div
+                        className={css.button + (css.background("hsla(0, 0%, 100%, 0.2)", "hover"))}
+                        onClick={() => {
+                            inspectTimeURL.value = String(value) + "|" + 10 + "|" + 10;
+                            filterURL.value = "";
+                        }}
+                    >
+                        {genericFormat(value)}
+                    </div>
+                );
+            },
+        };
+
+        table.columns["type"] = {
+            formatter(value, context) {
+                let inner = (() => {
+                    if (value === "error") return <span className={errorMessage}>{value}</span>;
+                    if (value === "warn") return <span className={warnMessage}>{value}</span>;
+                    if (value === "info" || value === "log") return <span className={css.hsl(210, 50, 50).pad2(6, 4).color("white")}>{value}</span>;
+                    return formatValue(value, undefined, context);
+                })();
+                return <div><div className={css.width(100).hbox0.center}>{inner}</div></div>;
+            }
+        };
+        table.columns["param0"] = {
+            title: "Message",
+            formatter(value, context) {
+                return <ObjectDisplay value={value} />;
+            },
+        };
+
+        table.columns["remaining"] = {
+            formatter(value, context) {
+                return <ObjectDisplay
+                    value={context?.row}
+                    excludedFields={fullSelectedFields}
+                    onClickKey={path => addPath(path)}
+                    onClickValue={(path, value) => {
+                        addPath(path);
+                        if (!canHaveChildren(value)) {
+                            // Add to search as well
+                            if (filterURL.value.trim()) {
+                                filterURL.value += " | ";
+                            }
+                            filterURL.value += String(value);
+                        }
+                    }}
+                />;
+            },
+        };
+
+        let [inspectTime, inspectBeforeCount, inspectAfterCount] = inspectTimeURL.value.split("|").map(x => Number(x));
+        if (inspectTime) {
+            startTime = inspectTime;
+            endTime = inspectTime;
+        }
+
+        return (
+            <div class={css.pad2(10).vbox(10)}>
+                <Anchor values={[selectedNodeId.getOverride("")]}><h2>Back</h2></Anchor>
+                <InputPicker
+                    picked={selectedNodesIds}
+                    addPicked={x => selectedNodeId.value = [...selectedNodesIds, x].join("|")}
+                    removePicked={x => selectedNodeId.value = selectedNodesIds.filter(y => y !== x).join("|")}
+                    options={nodeIds.map(x => ({ value: x, label: x }))}
+                />
+                {loadingCount > 0 && <h1>Loading {loadingCount} files</h1>}
+                <div className={css.fontSize(28).boldStyle}>
+                    {selectedNodesIds.length} nodes, {files.length} files, filtered logs {formatNumber(filteredCount)} / {formatNumber(totalCount)}
+                </div>
+                <ShowMore maxHeight={40}>
+                    <div class={css.vbox(4)}>
+                        {files.map(file =>
+                            <div class={css.hbox(4)}>
+                                <span>{formatNumber(file.size)}B</span>
+                                <span>{formatDateTime(file.startTime)} - {formatDateTime(file.endTime)}</span>
+                                <span>({file.path})</span>
+                            </div>
+                        )}
+                    </div>
+                </ShowMore>
+                {!inspectTime && <>
+                    <TimeRangeSelector
+                        start={startTimeURL} end={endTimeURL}
+                        defaultStart={startTime} defaultEnd={endTime}
+                        firstTime={firstTime} lastTime={lastTime}
+                    />
+                    <div class={css.hbox(20)}>
+                        <Button onClick={() => { startTimeURL.reset(); endTimeURL.reset(); }}>Reset Time Filter</Button>
+                        <div class={css.hbox(10)}>
+                            <InputLabelURL label="Start Time" url={startTimeURL} isDatetime valueDefault={startTime} />
+                            <span>({formatNiceDateTime(startTime)})</span>
+                        </div>
+                        <div class={css.hbox(10)}>
+                            <InputLabelURL label="End Time" url={endTimeURL} isDatetime valueDefault={endTime} />
+                            <span>({formatNiceDateTime(endTime)})</span>
+                        </div>
+                    </div>
+                    <div className={css.hbox(20)}>
+                        <h2>{formatTime(Date.now() - startTime)} AGO</h2> to <h2>{formatTime(Date.now() - endTime)} AGO</h2>
+                    </div>
+                </> ||
+                    <div class={css.hbox(20)}>
+                        <h2>{formatTime(Date.now() - inspectTime)} AGO</h2>
+                        <h2>{formatNiceDateTime(inspectTime)}</h2>
+                        <InputLabel
+                            label="Inspect Time"
+                            value={inspectTime}
+                            isDatetime
+                            onChange={x => inspectTimeURL.value = x + "|" + inspectBeforeCount + "|" + inspectAfterCount}
+                        />
+                        <button onClick={() => {
+                            inspectTimeURL.value = "";
+                        }}>
+                            Reset Inspect
+                        </button>
+                    </div>
+                }
+
+                <InputLabelURL fillWidth label="Filter: ['|' to match any conditions, '&' to match all, '!' for negation]" url={filterURL} />
+                <InputLabelURL label="Sort Oldest First" checkbox url={sortOldestFirst} />
+                <InputPicker
+                    label="Selected Fields"
+                    picked={selectedValues}
+                    options={Array.from(allFields.keys()).map(x => ({ value: x, label: x }))}
+                    addPicked={x => selectedFields.value = [...selectedValues, x].join("|")}
+                    removePicked={x => selectedFields.value = selectedValues.filter(y => y !== x).join("|")}
+                />
+                {!!inspectTime && (
+                    <InputLabel
+                        label="Before Count"
+                        value={inspectBeforeCount}
+                        onChangeValue={x => inspectTimeURL.value = inspectTime + "|" + x + "|" + inspectAfterCount}
+                        number
+                    />
+                )}
+                <Table {...table} initialLimit={30} />
+                {!!inspectTime && (
+                    <InputLabel
+                        label="After Count"
+                        value={inspectAfterCount}
+                        onChangeValue={x => inspectTimeURL.value = inspectTime + "|" + inspectBeforeCount + "|" + x}
+                        number
+                    />
+                )}
+            </div>
+        );
+    }
+}
+
+
+let remainingCacheSize = 1024 * 1024 * 1024 * 2;
+let logBufferCache = new Map<string, {
+    size: number;
+    logs: LogObj[];
+    lastAccess: number;
+}>();
+function parseLogBufferCached(hash: string, buffers: Buffer[]): LogObj[] {
+    let cached = logBufferCache.get(hash);
+    if (cached) {
+        cached.lastAccess = Date.now();
+        return cached.logs;
+    }
+    let logs = buffers.map(x => parseLogBuffer(x)).flat();
+    let size = buffers.map(x => x.length).reduce((a, b) => a + b, 0);
+    remainingCacheSize -= size;
+    while (remainingCacheSize < 0 && logBufferCache.size > 0) {
+        let oldest = [...logBufferCache.entries()].sort((a, b) => a[1].lastAccess - b[1].lastAccess)[0];
+        logBufferCache.delete(oldest[0]);
+        remainingCacheSize += oldest[1].size;
+    }
+    logBufferCache.set(hash, {
+        size,
+        logs,
+        lastAccess: Date.now()
+    });
+    return logs;
+}
+
+
+
+const processLogs = cacheShallowConfigArgEqual((config: {
+    logs: LogObj[];
+    startTime: number; endTime: number;
+    order: "oldestFirst" | "newestFirst";
+    filter: string;
+    selectedFields: string[];
+    inspectTime: string;
+}): {
+    table: TableType<LogObj>;
+    allFields: Map<string, number>;
+    totalCount: number;
+    filteredCount: number;
+} => {
+    return measureBlock(function processLogs() {
+        let { logs, order, filter, startTime, endTime, inspectTime, selectedFields } = config;
+        logs = logs.slice();
+        let totalCount = logs.length;
+
+        let logMatchesFilter: (log: LogObj) => boolean = () => true;
+
+        if (filter) {
+            filter = filter.toLowerCase();
+            let filterParts = filter.split("|").map(x => x.trim()).map(x => x.split("&").map(x => x.trim()));
+            // Extract any negation
+            let negationQueries: string[] = [];
+            filterParts = filterParts.map(filter => filter.filter(x => {
+                if (x.startsWith("!")) {
+                    negationQueries.push(x.slice(1));
+                    return false;
+                }
+                return true;
+            })).filter(x => x.length > 0);
+            function logMatches(filter: string, log: LogObj) {
+                for (let [key, value] of Object.entries(log)) {
+                    if (key.toLowerCase().includes(filter)) return true;
+                    if (canHaveChildren(value) && JSON.stringify(value).toLowerCase().includes(filter)) return true;
+                    if (String(value).toLowerCase().includes(filter)) return true;
+                }
+                return false;
+            }
+            logMatchesFilter = function logMatchesFilter(log: LogObj) {
+                return (
+                    (filterParts.length === 0 || filterParts.some(filter => filter.every(filter => logMatches(filter, log))))
+                    && !negationQueries.some(filter => logMatches(filter, log))
+                );
+            };
+        }
+
+        function applySort() {
+            measureBlock(function filterSort() {
+                if (order === "oldestFirst") {
+                    sort(logs, x => x.time);
+                } else {
+                    sort(logs, x => -x.time);
+                }
+            });
+        }
+
+
+        if (inspectTime) {
+            applySort();
+            // time|before|after
+            let [time, before, after] = inspectTime.split("|").map(x => Number(x));
+            applySort();
+            measureBlock(function filterLogs() {
+                // Binary search to find the closest index
+                let sortOrder = order === "oldestFirst" ? 1 : -1;
+                let index = binarySearchBasic(logs, x => x.time * sortOrder, time * sortOrder);
+                let startIndex: number;
+                let endIndex: number;
+
+                let newLogs: LogObj[] = [];
+                if (index < 0) {
+                    index = ~index;
+                    startIndex = index - 1;
+                    endIndex = index;
+                } else {
+                    startIndex = index - 1;
+                    endIndex = index + 1;
+                    if (logMatchesFilter(logs[index])) {
+                        newLogs.push(logs[index]);
+                    }
+                }
+
+                while (before > 0 && startIndex > 0) {
+                    startIndex--;
+                    let logBefore = logs[startIndex];
+                    if (logMatchesFilter(logBefore)) {
+                        newLogs.unshift(logBefore);
+                        before--;
+                    }
+                }
+
+                while (after > 0 && endIndex < logs.length) {
+                    let logAfter = logs[endIndex];
+                    if (logMatchesFilter(logAfter)) {
+                        newLogs.push(logAfter);
+                        after--;
+                    }
+                    endIndex++;
+                }
+
+                logs = newLogs;
+            });
+        } else {
+            measureBlock(function filterLogs() {
+                logs = logs.filter(log =>
+                    startTime <= log.time && log.time <= endTime &&
+                    logMatchesFilter(log)
+                );
+            });
+            applySort();
+        }
+
+        let filteredCount = logs.length;
+
+        let fields = new Map<string, number>();
+        for (let log of logs) {
+            for (let key in log) {
+                fields.set(key, (fields.get(key) || 0) + 1);
+            }
+        }
+
+        let table: TableType<LogObj> = {
+            rows: logs,
+            columns: {},
+        };
+
+        for (let key of selectedFields) {
+            table.columns[key] = {};
+        }
+
+        return { table, allFields: fields, totalCount, filteredCount };
+    });
+}, 10);
+
+
+
+
+class DiskLoggerControllerBase {
+    // Have to forward it, as backend nodes don't have real domains, so HTTPS won't
+    //  work from the browser for them.
+    public async getRemoteLogFiles(nodeId: string): Promise<LogFile[]> {
+        return await DiskLoggerController.nodes[nodeId].getLogFiles();
+    }
+    public async getRemoteLogBuffer(nodeId: string, path: string): Promise<Buffer | undefined> {
+        return await DiskLoggerController.nodes[nodeId].getLogBuffer(path);
+    }
+
+    public async getLogFiles(): Promise<LogFile[]> {
+        return await getLogFiles();
+    }
+    public async getLogBuffer(path: string): Promise<Buffer | undefined> {
+        return await getLogBuffer(path);
+    }
+}
+
+export const DiskLoggerController = SocketFunction.register(
+    "DiskLoggerController-f76a6fdf-3bd5-4bd4-a183-55a8be0a5a32",
+    new DiskLoggerControllerBase(),
+    () => ({
+        getRemoteLogFiles: {},
+        getRemoteLogBuffer: {},
+
+        getLogFiles: {},
+        getLogBuffer: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    }),
+    {
+    }
+);
+
+let diskLoggerController = getSyncedController(DiskLoggerController);
+
+let nodeViewerController = getSyncedController(NodeViewerController);