querysub 0.437.0 → 0.439.0

package/src/diagnostics/listenOnDebugger.ts

@@ -1,72 +1,72 @@
-import { isNode, list } from "socket-function/src/misc";
-import inspector from "inspector";
-import fs from "fs";
-import { getSubFolder } from "../fs";
-import { SocketFunction } from "socket-function/SocketFunction";
-import debugbreak from "debugbreak";
-import http from "http";
-import { lazy } from "socket-function/src/caching";
-
-let devToolsUrl = "";
-
-const listenOnDebugger = lazy(function listenOnDebugger() {
-    if (devToolsUrl) return;
-    if (!isNode()) return;
-    // IMPORTANT! NEVER use the built-in port. This adds a tiny bit of security. Not much,
-    //  as port scanning is easier (there are only 65K ports!), but... at least port scanning
-    //  is loud, so if a site port scans all of it's users it will probably get flagged by
-    //  chrome fairly quickly as being malicious.
-    let url = baseGetInspectorUrl();
-
-    devToolsUrl = `devtools://devtools/bundled/inspector.html?experiments=true&v8only=true&${url.replace("://", "=")}`;
-    devToolsUrl = devToolsUrl.replace("devtools://devtools/bundled", "https://notdevtools.com/devtools");
-    console.log(`Debugger listening on ${devToolsUrl}`);
-
-    const ext = ".url";
-    const debuggerFolder = getSubFolder("debugger");
-    let otherDebuggers = fs.readdirSync(debuggerFolder);
-    otherDebuggers = otherDebuggers.filter(x => x.endsWith(ext));
-    // Delete all files for processes that no longer exist
-    for (let other of otherDebuggers) {
-        let pid = +other.split("-")[0];
-        let ppid = +other.split("-")[1];
-        if (!doesProcessExist(pid) || !doesProcessExist(ppid)) {
-            try {
-                fs.unlinkSync(debuggerFolder + other);
-            } catch { }
-        }
-    }
-});
-export function getDebuggerUrl() {
-    // NOTE: Very unfortunate, as this means that if a developer is debugging a server for a bit,
-    //  the inspector is left open. It's kind of hard to tell when they are done though.
-    //  - This is still somewhat secure, as only localhost can connect, but... having local network
-    //      accessing turning into remote execution access isn't great (especially for developers,
-    //      who could then get pwned just by clicking on a link).
-    // TODO: The security of this can be improved, see NodeViewer.tsx:getExternalInspectURL
-    listenOnDebugger();
-    return devToolsUrl;
-}
-function doesProcessExist(pid: number) {
-    try {
-        process.kill(pid, 0);
-        return true;
-    } catch {
-        return false;
-    }
-}
-function baseGetInspectorUrl() {
-    function getNextPort() {
-        return 49152 + ~~((65535 - 49152) * Math.random());
-    }
-
-    while (true) {
-        let url = inspector.url();
-        if (url) return url;
-        try {
-            inspector.open(getNextPort());
-        } catch {
-            continue;
-        }
-    }
+import { isNode, list } from "socket-function/src/misc";
+import inspector from "inspector";
+import fs from "fs";
+import { getSubFolder } from "../fs";
+import { SocketFunction } from "socket-function/SocketFunction";
+import debugbreak from "debugbreak";
+import http from "http";
+import { lazy } from "socket-function/src/caching";
+
+let devToolsUrl = "";
+
+const listenOnDebugger = lazy(function listenOnDebugger() {
+    if (devToolsUrl) return;
+    if (!isNode()) return;
+    // IMPORTANT! NEVER use the built-in port. This adds a tiny bit of security. Not much,
+    //  as port scanning is easier (there are only 65K ports!), but... at least port scanning
+    //  is loud, so if a site port scans all of it's users it will probably get flagged by
+    //  chrome fairly quickly as being malicious.
+    let url = baseGetInspectorUrl();
+
+    devToolsUrl = `devtools://devtools/bundled/inspector.html?experiments=true&v8only=true&${url.replace("://", "=")}`;
+    devToolsUrl = devToolsUrl.replace("devtools://devtools/bundled", "https://notdevtools.com/devtools");
+    console.log(`Debugger listening on ${devToolsUrl}`);
+
+    const ext = ".url";
+    const debuggerFolder = getSubFolder("debugger");
+    let otherDebuggers = fs.readdirSync(debuggerFolder);
+    otherDebuggers = otherDebuggers.filter(x => x.endsWith(ext));
+    // Delete all files for processes that no longer exist
+    for (let other of otherDebuggers) {
+        let pid = +other.split("-")[0];
+        let ppid = +other.split("-")[1];
+        if (!doesProcessExist(pid) || !doesProcessExist(ppid)) {
+            try {
+                fs.unlinkSync(debuggerFolder + other);
+            } catch { }
+        }
+    }
+});
+export function getDebuggerUrl() {
+    // NOTE: Very unfortunate, as this means that if a developer is debugging a server for a bit,
+    //  the inspector is left open. It's kind of hard to tell when they are done though.
+    //  - This is still somewhat secure, as only localhost can connect, but... having local network
+    //      accessing turning into remote execution access isn't great (especially for developers,
+    //      who could then get pwned just by clicking on a link).
+    // TODO: The security of this can be improved, see NodeViewer.tsx:getExternalInspectURL
+    listenOnDebugger();
+    return devToolsUrl;
+}
+function doesProcessExist(pid: number) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    } catch {
+        return false;
+    }
+}
+function baseGetInspectorUrl() {
+    function getNextPort() {
+        return 49152 + ~~((65535 - 49152) * Math.random());
+    }
+
+    while (true) {
+        let url = inspector.url();
+        if (url) return url;
+        try {
+            inspector.open(getNextPort());
+        } catch {
+            continue;
+        }
+    }
 }

package/src/diagnostics/logs/IndexedLogs/BufferUnitIndex.ts

@@ -276,7 +276,7 @@ export class BufferUnitIndex {
                 // Only inaccurate if unique units approaches or exceeds 65k
                 return uniqueCount * 2;
             }
-        }, `collectUniqueUnits`);
+        }, `estimatedUniqueUnits`);
 
         // Step 3: Calculate hash table size and mask
         const { hashTableCapacity, mask } = measureBlock(() => {

package/src/diagnostics/logs/diskLogger.ts

@@ -21,6 +21,11 @@ if (isNode()) {
     }));
 }
 
+let loggingToDiskDisabled = false;
+export function disableLoggingToDisk() {
+    loggingToDiskDisabled = true;
+}
+
 // NOTE: When logging we spread objects. If we encounter strings, we set the field `param${index}`
 export type LogDatum = Record<string, unknown> & {
     time: number;
@@ -109,6 +114,7 @@ const logDiskDontShim = logDisk;
 /** @deprecated, Don't call this directly, call console info instead, which our shim will prevent from logging to the console, but it will still call logDisk. */
 export function logDisk(type: "log" | "warn" | "info" | "error", ...args: unknown[]) {
     if (!isNode()) return;
+    if (loggingToDiskDisabled) return;
     try {
         if (args.length === 0) return;
         let logType = args.find(x => typeof x === "string") as string | undefined;

package/src/diagnostics/misc-pages/SnapshotViewer.tsx

@@ -1,6 +1,6 @@
 module.allowclient = true;
 
-import { ArchiveSnapshotOverview, ArchiveSnapshotRead, getSnapshot, getSnapshotList, loadSnapshot, saveSnapshot } from "../../0-path-value-core/archiveLocks/archiveSnapshots";
+import { ArchiveSnapshotOverview, ArchiveSnapshotRead, downloadSnapshot, getSnapshot, getSnapshotList, loadSnapshot, saveSnapshot, uploadSnapshot } from "../../0-path-value-core/archiveLocks/archiveSnapshots";
 import { qreact } from "../../4-dom/qreact";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
@@ -30,6 +30,8 @@ export class SnapshotViewer extends qreact.Component {
         // file => true
         expanded: t.lookup(t.boolean),
         saving: t.boolean(false),
+        uploading: t.boolean(false),
+        downloading: t.lookup(t.boolean),
     });
     render() {
         let controller = SnapshotViewerSynced(getBrowserUrlNode());
@@ -40,6 +42,11 @@ export class SnapshotViewer extends qreact.Component {
             return (
                 <div class={css.pad2(10).vbox(10)}>
                     <h1>Snapshots are taken before every change. "zombie" means the file exists, but it has no confirm (so it will be ignored).</h1>
+                    <div class={css.hbox(10)}>
+                        <Button onClick={() => triggerSnapshotUpload(this)}>
+                            {this.state.uploading && "Uploading..." || "Upload Snapshot"}
+                        </Button>
+                    </div>
                     <Table
                         rows={snapshotList}
                         columns={{
@@ -68,6 +75,9 @@ export class SnapshotViewer extends qreact.Component {
                                                 {this.state.saving ? "Saving..." : "Save"}
                                             </Button>
                                         }
+                                        <Button onClick={() => downloadSnapshotFile(this, file)}>
+                                            {this.state.downloading[file] && "Downloading..." || "Download"}
+                                        </Button>
                                     </div>;
                                 }
                             }
@@ -142,6 +152,61 @@ export class SnapshotViewer extends qreact.Component {
     }
 }
 
+async function downloadSnapshotFile(component: SnapshotViewer, file: string) {
+    if (component.state.downloading[file]) return;
+    Querysub.localCommit(() => {
+        component.state.downloading[file] = true;
+    });
+    try {
+        let buffer = await SnapshotViewerController.nodes[getBrowserUrlNode()].downloadSnapshot(file);
+        let blob = new Blob([buffer]);
+        let url = URL.createObjectURL(blob);
+        let anchor = document.createElement("a");
+        anchor.href = url;
+        anchor.download = file + ".cborx";
+        if (file === "live") {
+            anchor.download = `${formatDateTime(Date.now())}.cborx`;
+        }
+        document.body.appendChild(anchor);
+        anchor.click();
+        document.body.removeChild(anchor);
+        URL.revokeObjectURL(url);
+    } catch (e: any) {
+        console.error(red(`Failed to download snapshot ${file}: ${e.stack}`));
+    } finally {
+        Querysub.localCommit(() => {
+            delete component.state.downloading[file];
+        });
+    }
+}
+
+async function triggerSnapshotUpload(component: SnapshotViewer) {
+    if (component.state.uploading) return;
+    let input = document.createElement("input");
+    input.type = "file";
+    input.accept = ".cborx";
+    input.onchange = async () => {
+        let selected = input.files?.[0];
+        if (!selected) return;
+        Querysub.localCommit(() => {
+            component.state.uploading = true;
+        });
+        try {
+            let arrayBuffer = await selected.arrayBuffer();
+            let buffer = Buffer.from(arrayBuffer);
+            await SnapshotViewerController.nodes[getBrowserUrlNode()].uploadSnapshot(buffer);
+            console.log(green("Uploaded snapshot"));
+        } catch (e: any) {
+            console.error(red(`Failed to upload snapshot: ${e.stack}`));
+        } finally {
+            Querysub.localCommit(() => {
+                component.state.uploading = false;
+            });
+        }
+    };
+    input.click();
+}
+
 async function seriousConfirm(config: {
     message: string;
 }): Promise<boolean> {
@@ -192,6 +257,14 @@ class SnapshotViewerControllerBase {
         let files = await getSnapshot("live");
         await saveSnapshot({ files: files.files.map(x => x.file) });
     }
+
+    public async downloadSnapshot(snapshotFile: string | "live"): Promise<Buffer> {
+        return await downloadSnapshot(snapshotFile);
+    }
+
+    public async uploadSnapshot(buffer: Buffer): Promise<ArchiveSnapshotOverview> {
+        return await uploadSnapshot(buffer);
+    }
 }
 
 export const SnapshotViewerController = SocketFunction.register(
@@ -202,6 +275,9 @@ export const SnapshotViewerController = SocketFunction.register(
         loadSnapshot: {},
         getSnapshot: {},
         saveLiveSnapshot: {},
+        downloadSnapshot: {},
+        uploadSnapshot: {},
+        example: {}
     }),
     () => ({
         hooks: [assertIsManagementUser],
@@ -217,5 +293,6 @@ const SnapshotViewerSynced = getSyncedController(SnapshotViewerController, {
     },
     writes: {
         saveLiveSnapshot: ["snapshots"],
+        uploadSnapshot: ["snapshots"],
     }
 });

package/src/diagnostics/periodic.ts

@@ -1,112 +1,112 @@
-import { delay, runInfinitePoll, runInfinitePollCallAtStart, shutdownPolling } from "socket-function/src/batching";
-import { isNode, timeInMinute } from "socket-function/src/misc";
-import { logErrors, timeoutToError } from "../errors";
-import debugbreak from "debugbreak";
-import { nodeDiscoveryShutdown } from "../-f-node-discovery/NodeDiscovery";
-import { authorityStorage } from "../0-path-value-core/pathValueCore";
-import { red } from "socket-function/src/formatting/logColors";
-
-// Import querysub, so all the hooks we use will exist at some point
-setImmediate(async () => {
-    await import("../4-querysub/QuerysubController");
-});
-
-let periodicFncs: (() => void)[] = [];
-export function registerPeriodic(fnc: () => void) {
-    periodicFncs.push(fnc);
-}
-
-let shutdownHandlers: (() => Promise<void>)[] = [];
-// NOTE: Won't be always called, but at least makes it possible to gracefully shutdown.
-export function registerShutdownHandler(fnc: () => Promise<void>) {
-    shutdownHandlers.push(fnc);
-}
-let preshutdownHandlers: (() => Promise<void>)[] = [];
-export function registerPreshutdownHandler(fnc: () => Promise<void>) {
-    preshutdownHandlers.push(fnc);
-}
-
-function logAll() {
-    for (let fnc of periodicFncs) {
-        fnc();
-    }
-}
-
-logErrors(runInfinitePollCallAtStart(timeInMinute * 5, logAll));
-
-let shuttingDown = false;
-export async function shutdown() {
-    if (shuttingDown) {
-        return;
-    }
-    console.log(red("Starting shutdown"));
-    shuttingDown = true;
-    const { authorityStorage } = await import("../0-path-value-core/pathValueCore");
-    try {
-        await Promise.allSettled([
-            ...preshutdownHandlers,
-        ].map(fnc => timeoutToError(timeInMinute, fnc(), () => new Error(`Preshutdown handler ${fnc.name} timed out`))));
-    } catch (e) {
-        console.log(`Error on preshutdown handlers`, e);
-    }
-    try {
-        await Promise.allSettled([
-            function authorityStorageShutdown() { return authorityStorage.onShutdown(); },
-            nodeDiscoveryShutdown,
-            shutdownPolling,
-            ...shutdownHandlers,
-        ].map(fnc => timeoutToError(timeInMinute, fnc(), () => new Error(`Shutdown handler ${fnc.name} timed out`))));
-    } catch (e) {
-        console.log("Error on shutdown", e);
-    }
-    // Wait to allow any logged errors to hopefully be written somewhere?
-    await delay(2000);
-    process.exit();
-}
-
-// IMPORTANT! Yarn detaches the processes, so they keep running when you ctrl+c, even though the shell shows back up. We can fix this by using `node -r ./node_modules/typenode/index.js ./test.ts`. However, it's probably fine, as we still run the shutdown code, it's just that the manager doesn't know if we've shutdown or not.
-if (isNode()) {
-    let lineBuffer = "";
-    process.stdin.on("data", data => {
-        lineBuffer += data.toString();
-        let lines = lineBuffer.split("\r");
-        lineBuffer = lines.pop()!;
-        for (let line of lines) {
-            if (line === "SHUTDOWN_NOW_MULTIRUN") {
-                logErrors(shutdown());
-            }
-        }
-
-        if (data.toString().includes("\r")) {
-            logAll();
-        }
-    });
-
-
-    // NOTE: This extra code is required to actual capture ctrl+c
-    if (process.platform === "win32") {
-        var rl = require("readline").createInterface({
-            input: process.stdin,
-            output: process.stdout
-        });
-
-        rl.on("SIGINT", function () {
-            process.emit("SIGINT");
-        });
-    }
-    let killCount = 0;
-    function doShutdown() {
-        killCount++;
-        console.log(`Caught interrupt signal. Attempt number ${killCount}`);
-        if (killCount >= 3) {
-            console.log("Force exit");
-            process.exit();
-        }
-
-        logErrors(shutdown());
-    }
-    process.on("SIGINT", doShutdown);
-
-}
-(globalThis as any).logAll = logAll;
+import { delay, runInfinitePoll, runInfinitePollCallAtStart, shutdownPolling } from "socket-function/src/batching";
+import { isNode, timeInMinute } from "socket-function/src/misc";
+import { logErrors, timeoutToError } from "../errors";
+import debugbreak from "debugbreak";
+import { nodeDiscoveryShutdown } from "../-f-node-discovery/NodeDiscovery";
+import { authorityStorage } from "../0-path-value-core/pathValueCore";
+import { red } from "socket-function/src/formatting/logColors";
+
+// Import querysub, so all the hooks we use will exist at some point
+setImmediate(async () => {
+    await import("../4-querysub/QuerysubController");
+});
+
+let periodicFncs: (() => void)[] = [];
+export function registerPeriodic(fnc: () => void) {
+    periodicFncs.push(fnc);
+}
+
+let shutdownHandlers: (() => Promise<void>)[] = [];
+// NOTE: Won't be always called, but at least makes it possible to gracefully shutdown.
+export function registerShutdownHandler(fnc: () => Promise<void>) {
+    shutdownHandlers.push(fnc);
+}
+let preshutdownHandlers: (() => Promise<void>)[] = [];
+export function registerPreshutdownHandler(fnc: () => Promise<void>) {
+    preshutdownHandlers.push(fnc);
+}
+
+function logAll() {
+    for (let fnc of periodicFncs) {
+        fnc();
+    }
+}
+
+logErrors(runInfinitePollCallAtStart(timeInMinute * 5, logAll));
+
+let shuttingDown = false;
+export async function shutdown() {
+    if (shuttingDown) {
+        return;
+    }
+    console.log(red("Starting shutdown"));
+    shuttingDown = true;
+    const { authorityStorage } = await import("../0-path-value-core/pathValueCore");
+    try {
+        await Promise.allSettled([
+            ...preshutdownHandlers,
+        ].map(fnc => timeoutToError(timeInMinute, fnc(), () => new Error(`Preshutdown handler ${fnc.name} timed out`))));
+    } catch (e) {
+        console.log(`Error on preshutdown handlers`, e);
+    }
+    try {
+        await Promise.allSettled([
+            function authorityStorageShutdown() { return authorityStorage.onShutdown(); },
+            nodeDiscoveryShutdown,
+            shutdownPolling,
+            ...shutdownHandlers,
+        ].map(fnc => timeoutToError(timeInMinute, fnc(), () => new Error(`Shutdown handler ${fnc.name} timed out`))));
+    } catch (e) {
+        console.log("Error on shutdown", e);
+    }
+    // Wait to allow any logged errors to hopefully be written somewhere?
+    await delay(2000);
+    process.exit();
+}
+
+// IMPORTANT! Yarn detaches the processes, so they keep running when you ctrl+c, even though the shell shows back up. We can fix this by using `node -r ./node_modules/typenode/index.js ./test.ts`. However, it's probably fine, as we still run the shutdown code, it's just that the manager doesn't know if we've shutdown or not.
+if (isNode()) {
+    let lineBuffer = "";
+    process.stdin.on("data", data => {
+        lineBuffer += data.toString();
+        let lines = lineBuffer.split("\r");
+        lineBuffer = lines.pop()!;
+        for (let line of lines) {
+            if (line === "SHUTDOWN_NOW_MULTIRUN") {
+                logErrors(shutdown());
+            }
+        }
+
+        if (data.toString().includes("\r")) {
+            logAll();
+        }
+    });
+
+
+    // NOTE: This extra code is required to actual capture ctrl+c
+    if (process.platform === "win32") {
+        var rl = require("readline").createInterface({
+            input: process.stdin,
+            output: process.stdout
+        });
+
+        rl.on("SIGINT", function () {
+            process.emit("SIGINT");
+        });
+    }
+    let killCount = 0;
+    function doShutdown() {
+        killCount++;
+        console.log(`Caught interrupt signal. Attempt number ${killCount}`);
+        if (killCount >= 3) {
+            console.log("Force exit");
+            process.exit();
+        }
+
+        logErrors(shutdown());
+    }
+    process.on("SIGINT", doShutdown);
+
+}
+(globalThis as any).logAll = logAll;
 (globalThis as any).logNow = logAll;