querysub 0.153.0 → 0.154.0

package/src/diagnostics/misc-pages/RequireAuditPage.tsx

@@ -0,0 +1,218 @@
+import preact from "preact"; import { qreact } from "../../4-dom/qreact";
+import { Querysub } from "../../4-querysub/Querysub";
+import { css } from "../../4-dom/css";
+import { isDefined } from "../../misc";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { getPathStr1 } from "../../path";
+import { URLParam, createURLSync } from "../../library-components/URLParam";
+import { Input } from "../../library-components/Input";
+import { ButtonSelector } from "../../library-components/ButtonSelector";
+
+type ModuleFolderEntry = {
+    path: string;
+    children: ModuleTreeEntry[];
+    sum: number;
+    count: number;
+};
+type ModuleTreeEntry = {
+    module: NodeJS.Module;
+} | ModuleFolderEntry;
+
+const filterURL = createURLSync<string>("filter", "");
+const expandedURL = createURLSync<string>("expanded", "");
+
+const valueType = new URLParam("requireauditvalue", "size" as "size" | "evaltime");
+export class RequireAuditPage extends qreact.Component {
+    getModuleValue(module: NodeJS.Module) {
+        if (valueType.value === "evaltime") return (module.evalEndTime ?? 0) - (module.evalStartTime ?? 0);
+        return module.size || 0;
+    }
+    formatValue(value: number) {
+        if (valueType.value === "evaltime") return formatTime(value);
+        return formatNumber(value) + "B";
+    }
+
+    getFontSize(size: number, total: number) {
+        if (size / total > 0.20) return 30;
+        if (size / total > 0.02) return 20;
+        return 12;
+        return (
+            Math.max((size / total) ** 0.5 * 50, 10)
+        );
+    }
+    renderModule(module: NodeJS.Module, depth: number) {
+        return (
+            <>
+                <div>{module.filename}</div>
+            </>
+        );
+    }
+
+    renderTreeLevel(entry: ModuleTreeEntry[], root = true, depth = 0, config: {
+        totalSum: number;
+        maxSum: number;
+        expanded: string;
+    }): preact.ComponentChild {
+        const renderLine = (entry: ModuleTreeEntry, contents: preact.ComponentChild) => {
+            let size = "module" in entry ? this.getModuleValue(entry.module) : entry.sum;
+            let path = "module" in entry ? entry.module.filename : entry.path;
+            let expanded = config.expanded.includes(getPathStr1(path));
+            const isTreeNode = !("module" in entry);
+            return (
+                <div class={
+                    css.hbox(10).marginLeft(depth * 20).relative.fillWidth
+                        .fontSize(this.getFontSize(size, config.totalSum))
+                    + (isTreeNode && css.button)
+                }
+                    onClick={(e) => {
+                        if (!isTreeNode) return;
+                        if (expanded) {
+                            expandedURL.value = config.expanded.replace(getPathStr1(entry.path), "");
+                        } else {
+                            expandedURL.value += getPathStr1(entry.path);
+                        }
+                    }}
+                >
+                    <div
+                        class={
+                            css.absolute.pos(0, 2).height("calc(100% - 4px)" as "100%").width((size / config.maxSum) * 100 + "%" as "100%")
+                                .hsl(isTreeNode ? 180 : 210, 75, 50).opacity(0.5)
+                                .borderRadius(2)
+                                .zIndex(-1)
+                        }
+                    />
+                    {contents}
+                    <span class={css.opacity(0.4)}>
+                        {
+                            "module" in entry
+                                ? <>({this.formatValue(this.getModuleValue(entry.module))}) / {(size / config.totalSum * 100).toFixed(1)}%</>
+                                : <>({formatNumber(entry.count)} / {this.formatValue(entry.sum)}) / {(size / config.totalSum * 100).toFixed(1)}%</>
+                        }
+                    </span>
+                </div>
+            );
+        };
+        if (entry.length === 1) {
+            let singleEntry = entry[0];
+            if ("module" in singleEntry) {
+                return renderLine(singleEntry, this.renderModule(singleEntry.module, depth));
+            } else {
+                return this.renderTreeLevel(singleEntry.children, root, depth, config);
+            }
+        }
+        return (
+            <div class={css.vbox(0).fillWidth}>
+                {entry.map(entry => {
+                    if ("module" in entry) return renderLine(entry, this.renderModule(entry.module, depth));
+                    let expanded = config.expanded.includes(getPathStr1(entry.path));
+                    return (
+                        <div
+                            className={css.vbox(0).fillWidth
+                                + (expanded && css.paddingBottom(10))
+                            }
+                        >
+                            {renderLine(entry,
+                                <span>{entry.path}</span>
+                            )}
+                            {expanded && this.renderTreeLevel(entry.children, false, depth + 1, config)}
+                        </div>
+                    );
+                })}
+            </div>
+        );
+    }
+    render() {
+        const self = this;
+
+        // NOTE: We might need to re-render periodically, in order to catch new requires that are added.
+        //  However... we want to avoid this, as rendering when the user is typing can cause issues.
+        // Querysub.timeDelayed(5000);
+
+        let rootModules: ModuleFolderEntry[] = [];
+        let modulesList = Object.values(require.cache).filter(isDefined);
+        let filter = filterURL.value.toLowerCase();
+        if (filter) {
+            modulesList = modulesList.filter(module => module.filename.toLowerCase().includes(filter));
+        }
+
+        let entryByPath = new Map<string, ModuleFolderEntry>();
+        function getParentPath(path: string) {
+            return path.split("/").slice(0, -1).join("/");
+        }
+        function getTreeEntry(path: string) {
+            let entry = entryByPath.get(path);
+            if (!entry) {
+                entry = { children: [], path, sum: 0, count: 0, };
+                entryByPath.set(path, entry);
+                let parent = getParentPath(path);
+                if (parent) {
+                    getTreeEntry(parent).children.push(entry);
+                } else {
+                    rootModules.push(entry);
+                }
+            }
+            return entry;
+        }
+        for (let module of modulesList) {
+            getTreeEntry(getParentPath(module.filename)).children.push({ module });
+        }
+
+        function calculateSizes(entry: ModuleFolderEntry) {
+            entry.sum = 0;
+            entry.count = 0;
+            for (let child of entry.children) {
+                if ("module" in child) {
+                    entry.sum += self.getModuleValue(child.module);
+                    entry.count++;
+                } else {
+                    calculateSizes(child);
+                    entry.sum += child.sum;
+                    entry.count += child.count;
+                }
+            }
+        }
+        for (let rootModule of rootModules) {
+            calculateSizes(rootModule);
+        }
+
+        while (rootModules.length === 1) {
+            rootModules = rootModules[0].children as ModuleFolderEntry[] || [];
+        }
+
+        let totalCount = rootModules.reduce((sum, entry) => sum + entry.count, 0);
+        let totalSum = rootModules.reduce((sum, entry) => sum + entry.sum, 0);
+        let maxSum = Math.max(...rootModules.map(entry => entry.sum));
+
+        return (
+            <div class={css.pad(20).vbox(20).fillWidth.overflowY("auto").overflowX("hidden")}>
+                <div class={css.hbox(40)}>
+                    <h1>
+                        {totalCount} / {this.formatValue(totalSum)}
+                    </h1>
+                    <ButtonSelector
+                        options={[
+                            { value: "size", title: "Size" },
+                            { value: "evaltime", title: "Eval Time (Includes Child Time)" },
+                        ]}
+                        value={valueType.value}
+                        onChange={value => valueType.value = value as any}
+                    />
+                </div>
+                <Input
+                    flavor="large"
+                    focusOnMount
+                    placeholder={"Filter"}
+                    className={css.width("60vw")}
+                    value={filterURL.value}
+                    hot
+                    onChange={e => filterURL.value = e.currentTarget.value}
+                />
+                {this.renderTreeLevel(rootModules, undefined, undefined, {
+                    totalSum,
+                    maxSum,
+                    expanded: expandedURL.value,
+                })}
+            </div>
+        );
+    }
+}

package/src/diagnostics/misc-pages/SnapshotViewer.tsx

@@ -0,0 +1,206 @@
+module.allowclient = true;
+
+import { ArchiveSnapshotOverview, ArchiveSnapshotRead, getSnapshot, getSnapshotList, loadSnapshot, saveSnapshot } from "../../0-path-value-core/archiveLocks/archiveSnapshots";
+import { qreact } from "../../4-dom/qreact";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { ValueAuditController } from "../../5-diagnostics/memoryValueAudit";
+import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
+import { Table } from "../../5-diagnostics/Table";
+import { css } from "typesafecss";
+import { t } from "../../4-querysub/Querysub";
+import { ATag, Anchor } from "../../library-components/ATag";
+import { URLParam } from "../../library-components/URLParam";
+import { formatDateTime, formatNumber } from "socket-function/src/formatting/format";
+import { Button } from "../../library-components/Button";
+import { showModal } from "../../5-diagnostics/Modal";
+import { FullscreenModal } from "../../5-diagnostics/FullscreenModal";
+import { InputLabel } from "../../library-components/InputLabel";
+import { green, magenta, red } from "socket-function/src/formatting/logColors";
+import { archiveViewerHistoryPaths } from "./ArchiveViewer";
+import { assertIsManagementUser, managementPageURL } from "../../diagnostics/managementPages";
+import { getSyncedController } from "../../library-components/SyncedController";
+
+// TODO: Create a transaction viewer, so we can view transactions in the recycle bin and data,
+//  to see why various files were moved.
+//  - This can help debug bad archives, who accidentally delete the entire database, create new files, etc.
+//      - Which apparently... we have? Although maybe we fixed it?
+
+const selectedSnapshot = new URLParam("selectedSnapshot", "");
+export class SnapshotViewer extends qreact.Component {
+    state = t.state({
+        // file => true
+        expanded: t.lookup(t.boolean)
+    });
+    render() {
+        let controller = SnapshotViewerSynced(getBrowserUrlNode());
+        let snapshotList = controller.getSnapshotList();
+        if (!snapshotList) return <div>Loading...</div>;
+        let file = selectedSnapshot.value;
+        if (!file) {
+            return (
+                <div class={css.pad2(10)}>
+                    <Table
+                        rows={snapshotList}
+                        columns={{
+                            time: {},
+                            fileCount: {},
+                            valueCount: {},
+                            byteCount: {},
+                            file: {
+                                title: "Actions",
+                                formatter: (file, context) => {
+                                    return <div class={css.hbox(10)}>
+                                        <ATag values={[{ param: selectedSnapshot, value: file }]}>
+                                            View
+                                        </ATag>
+                                        {context?.row?.file === "live" &&
+                                            <Button onClick={async () => {
+                                                await SnapshotViewerController.nodes[getBrowserUrlNode()].saveLiveSnapshot();
+                                                console.log("Saved live snapshot.");
+                                            }}>
+                                                Save
+                                            </Button>
+                                        }
+                                    </div>;
+                                }
+                            }
+                        }}
+                    />
+                </div>
+            );
+        }
+
+
+        const entry = snapshotList.find(x => x.file === file);
+        let snapshotInfo = controller.getSnapshot(file);
+        if (!entry) return <div>Snapshot not found, {file}</div>;
+        return (
+            <div class={css.pad2(10).vbox(10)}>
+                <div class={css.hbox(10)}>
+                    <Anchor values={[{ param: selectedSnapshot, value: "" }]}>
+                        Back
+                    </Anchor>
+                    <h2>Snapshot: {file}</h2>
+                </div>
+                {entry &&
+                    <div class={css.vbox(2)}>
+                        <div>Time: {formatDateTime(entry.time)}</div>
+                        <div>File Count: {formatNumber(entry.fileCount)}</div>
+                        <div>Value Count: {formatNumber(entry.valueCount)}</div>
+                        <div>Byte Count: {formatNumber(entry.byteCount)}</div>
+                    </div>
+                }
+                <Button onClick={async () => {
+                    let didConfirm = await seriousConfirm({
+                        message: `Restoring will break all servers. This won't break your data, but you will have to restart all servers before the system will be stable again. Before then data will be random, and invalid writes might be allowed. It is recommend to run this with only 1 querysub server running, on a developers machine. Consider writing a database iterate script instead of restoring (similar to how gc and join work).`,
+                    });
+                    if (!didConfirm) {
+                        console.log(red("Aborted restore"));
+                        return;
+                    }
+                    console.log(magenta("Restoring snapshot..."));
+                    await SnapshotViewerController.nodes[getBrowserUrlNode()].loadSnapshot({
+                        overview: entry
+                    });
+                    console.log(green("Restored snapshot"));
+                    window.location.reload();
+                }}>
+                    Restore
+                </Button>
+
+                {snapshotInfo && <Anchor values={[
+                    { param: archiveViewerHistoryPaths, value: snapshotInfo.files.map(x => x.file) },
+                    { param: managementPageURL, value: "ArchiveViewer" },
+                ]}>
+                    View in Archive Viewer
+                </Anchor>}
+                {snapshotInfo &&
+                    <Table
+                        rows={snapshotInfo.files}
+                        columns={{
+                            //file: {},
+                            state: {},
+                            time: {},
+                            valueCount: {},
+                            byteCount: {},
+                            source: {},
+                        }}
+                    />
+                }
+            </div>
+        );
+    }
+}
+
+async function seriousConfirm(config: {
+    message: string;
+}): Promise<boolean> {
+    return await new Promise<boolean>((resolve, reject) => {
+        let modal = showModal({
+            content: <FullscreenModal onCancel={() => {
+                modal.close();
+                resolve(false);
+            }}>
+                <div>{config.message}</div>
+                <InputLabel
+                    label={<pre>
+                        Kill the PathValueServer first.
+                        Type "confirm" and press enter to continue.
+                        Then watch querysub logs and wait until "finished loading snapshot" is logged.
+                        Then kill querysub, and start all the processes normally.
+                    </pre>}
+                    onChangeValue={value => {
+                        if (value === "confirm") {
+                            modal.close();
+                            resolve(true);
+                        }
+                    }}
+                />
+            </FullscreenModal>
+        });
+    });
+}
+
+class SnapshotViewerControllerBase {
+    public async getSnapshotList(): Promise<ArchiveSnapshotOverview[]> {
+        return await getSnapshotList();
+    }
+
+    public async loadSnapshot(config: {
+        overview: ArchiveSnapshotOverview,
+    }): Promise<void> {
+        await loadSnapshot({
+            overview: config.overview,
+            async audit(nodeId) {
+                await ValueAuditController.nodes[nodeId].diskAuditNow();
+            },
+        });
+    }
+
+    public async getSnapshot(snapshotFile: string | "live"): Promise<ArchiveSnapshotRead> {
+        return await getSnapshot(snapshotFile);
+    }
+
+    public async saveLiveSnapshot() {
+        let files = await getSnapshot("live");
+        await saveSnapshot({ files: files.files.map(x => x.file) });
+    }
+}
+
+export const SnapshotViewerController = SocketFunction.register(
+    "SnapshotViewerController-a78ae3cf-5024-49da-b71a-f064a5d4b5f4",
+    new SnapshotViewerControllerBase(),
+    () => ({
+        getSnapshotList: {},
+        loadSnapshot: {},
+        getSnapshot: {},
+        saveLiveSnapshot: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    }),
+    {
+        noAutoExpose: true,
+    }
+);
+const SnapshotViewerSynced = getSyncedController(SnapshotViewerController);