querysub 0.432.0 → 0.436.0

package/src/diagnostics/listenOnDebugger.ts

@@ -1,72 +1,72 @@
-import { isNode, list } from "socket-function/src/misc";
-import inspector from "inspector";
-import fs from "fs";
-import { getSubFolder } from "../fs";
-import { SocketFunction } from "socket-function/SocketFunction";
-import debugbreak from "debugbreak";
-import http from "http";
-import { lazy } from "socket-function/src/caching";
-
-let devToolsUrl = "";
-
-const listenOnDebugger = lazy(function listenOnDebugger() {
-    if (devToolsUrl) return;
-    if (!isNode()) return;
-    // IMPORTANT! NEVER use the built-in port. This adds a tiny bit of security. Not much,
-    //  as port scanning is easier (there are only 65K ports!), but... at least port scanning
-    //  is loud, so if a site port scans all of it's users it will probably get flagged by
-    //  chrome fairly quickly as being malicious.
-    let url = baseGetInspectorUrl();
-
-    devToolsUrl = `devtools://devtools/bundled/inspector.html?experiments=true&v8only=true&${url.replace("://", "=")}`;
-    devToolsUrl = devToolsUrl.replace("devtools://devtools/bundled", "https://notdevtools.com/devtools");
-    console.log(`Debugger listening on ${devToolsUrl}`);
-
-    const ext = ".url";
-    const debuggerFolder = getSubFolder("debugger");
-    let otherDebuggers = fs.readdirSync(debuggerFolder);
-    otherDebuggers = otherDebuggers.filter(x => x.endsWith(ext));
-    // Delete all files for processes that no longer exist
-    for (let other of otherDebuggers) {
-        let pid = +other.split("-")[0];
-        let ppid = +other.split("-")[1];
-        if (!doesProcessExist(pid) || !doesProcessExist(ppid)) {
-            try {
-                fs.unlinkSync(debuggerFolder + other);
-            } catch { }
-        }
-    }
-});
-export function getDebuggerUrl() {
-    // NOTE: Very unfortunate, as this means that if a developer is debugging a server for a bit,
-    //  the inspector is left open. It's kind of hard to tell when they are done though.
-    //  - This is still somewhat secure, as only localhost can connect, but... having local network
-    //      accessing turning into remote execution access isn't great (especially for developers,
-    //      who could then get pwned just by clicking on a link).
-    // TODO: The security of this can be improved, see NodeViewer.tsx:getExternalInspectURL
-    listenOnDebugger();
-    return devToolsUrl;
-}
-function doesProcessExist(pid: number) {
-    try {
-        process.kill(pid, 0);
-        return true;
-    } catch {
-        return false;
-    }
-}
-function baseGetInspectorUrl() {
-    function getNextPort() {
-        return 49152 + ~~((65535 - 49152) * Math.random());
-    }
-
-    while (true) {
-        let url = inspector.url();
-        if (url) return url;
-        try {
-            inspector.open(getNextPort());
-        } catch {
-            continue;
-        }
-    }
+import { isNode, list } from "socket-function/src/misc";
+import inspector from "inspector";
+import fs from "fs";
+import { getSubFolder } from "../fs";
+import { SocketFunction } from "socket-function/SocketFunction";
+import debugbreak from "debugbreak";
+import http from "http";
+import { lazy } from "socket-function/src/caching";
+
+let devToolsUrl = "";
+
+const listenOnDebugger = lazy(function listenOnDebugger() {
+    if (devToolsUrl) return;
+    if (!isNode()) return;
+    // IMPORTANT! NEVER use the built-in port. This adds a tiny bit of security. Not much,
+    //  as port scanning is easier (there are only 65K ports!), but... at least port scanning
+    //  is loud, so if a site port scans all of it's users it will probably get flagged by
+    //  chrome fairly quickly as being malicious.
+    let url = baseGetInspectorUrl();
+
+    devToolsUrl = `devtools://devtools/bundled/inspector.html?experiments=true&v8only=true&${url.replace("://", "=")}`;
+    devToolsUrl = devToolsUrl.replace("devtools://devtools/bundled", "https://notdevtools.com/devtools");
+    console.log(`Debugger listening on ${devToolsUrl}`);
+
+    const ext = ".url";
+    const debuggerFolder = getSubFolder("debugger");
+    let otherDebuggers = fs.readdirSync(debuggerFolder);
+    otherDebuggers = otherDebuggers.filter(x => x.endsWith(ext));
+    // Delete all files for processes that no longer exist
+    for (let other of otherDebuggers) {
+        let pid = +other.split("-")[0];
+        let ppid = +other.split("-")[1];
+        if (!doesProcessExist(pid) || !doesProcessExist(ppid)) {
+            try {
+                fs.unlinkSync(debuggerFolder + other);
+            } catch { }
+        }
+    }
+});
+export function getDebuggerUrl() {
+    // NOTE: Very unfortunate, as this means that if a developer is debugging a server for a bit,
+    //  the inspector is left open. It's kind of hard to tell when they are done though.
+    //  - This is still somewhat secure, as only localhost can connect, but... having local network
+    //      accessing turning into remote execution access isn't great (especially for developers,
+    //      who could then get pwned just by clicking on a link).
+    // TODO: The security of this can be improved, see NodeViewer.tsx:getExternalInspectURL
+    listenOnDebugger();
+    return devToolsUrl;
+}
+function doesProcessExist(pid: number) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    } catch {
+        return false;
+    }
+}
+function baseGetInspectorUrl() {
+    function getNextPort() {
+        return 49152 + ~~((65535 - 49152) * Math.random());
+    }
+
+    while (true) {
+        let url = inspector.url();
+        if (url) return url;
+        try {
+            inspector.open(getNextPort());
+        } catch {
+            continue;
+        }
+    }
 }

package/src/diagnostics/periodic.ts

@@ -1,112 +1,112 @@
-import { delay, runInfinitePoll, runInfinitePollCallAtStart, shutdownPolling } from "socket-function/src/batching";
-import { isNode, timeInMinute } from "socket-function/src/misc";
-import { logErrors, timeoutToError } from "../errors";
-import debugbreak from "debugbreak";
-import { nodeDiscoveryShutdown } from "../-f-node-discovery/NodeDiscovery";
-import { authorityStorage } from "../0-path-value-core/pathValueCore";
-import { red } from "socket-function/src/formatting/logColors";
-
-// Import querysub, so all the hooks we use will exist at some point
-setImmediate(async () => {
-    await import("../4-querysub/QuerysubController");
-});
-
-let periodicFncs: (() => void)[] = [];
-export function registerPeriodic(fnc: () => void) {
-    periodicFncs.push(fnc);
-}
-
-let shutdownHandlers: (() => Promise<void>)[] = [];
-// NOTE: Won't be always called, but at least makes it possible to gracefully shutdown.
-export function registerShutdownHandler(fnc: () => Promise<void>) {
-    shutdownHandlers.push(fnc);
-}
-let preshutdownHandlers: (() => Promise<void>)[] = [];
-export function registerPreshutdownHandler(fnc: () => Promise<void>) {
-    preshutdownHandlers.push(fnc);
-}
-
-function logAll() {
-    for (let fnc of periodicFncs) {
-        fnc();
-    }
-}
-
-logErrors(runInfinitePollCallAtStart(timeInMinute * 5, logAll));
-
-let shuttingDown = false;
-export async function shutdown() {
-    if (shuttingDown) {
-        return;
-    }
-    console.log(red("Starting shutdown"));
-    shuttingDown = true;
-    const { authorityStorage } = await import("../0-path-value-core/pathValueCore");
-    try {
-        await Promise.allSettled([
-            ...preshutdownHandlers,
-        ].map(fnc => timeoutToError(timeInMinute, fnc(), () => new Error(`Preshutdown handler ${fnc.name} timed out`))));
-    } catch (e) {
-        console.log(`Error on preshutdown handlers`, e);
-    }
-    try {
-        await Promise.allSettled([
-            function authorityStorageShutdown() { return authorityStorage.onShutdown(); },
-            nodeDiscoveryShutdown,
-            shutdownPolling,
-            ...shutdownHandlers,
-        ].map(fnc => timeoutToError(timeInMinute, fnc(), () => new Error(`Shutdown handler ${fnc.name} timed out`))));
-    } catch (e) {
-        console.log("Error on shutdown", e);
-    }
-    // Wait to allow any logged errors to hopefully be written somewhere?
-    await delay(2000);
-    process.exit();
-}
-
-// IMPORTANT! Yarn detaches the processes, so they keep running when you ctrl+c, even though the shell shows back up. We can fix this by using `node -r ./node_modules/typenode/index.js ./test.ts`. However, it's probably fine, as we still run the shutdown code, it's just that the manager doesn't know if we've shutdown or not.
-if (isNode()) {
-    let lineBuffer = "";
-    process.stdin.on("data", data => {
-        lineBuffer += data.toString();
-        let lines = lineBuffer.split("\r");
-        lineBuffer = lines.pop()!;
-        for (let line of lines) {
-            if (line === "SHUTDOWN_NOW_MULTIRUN") {
-                logErrors(shutdown());
-            }
-        }
-
-        if (data.toString().includes("\r")) {
-            logAll();
-        }
-    });
-
-
-    // NOTE: This extra code is required to actual capture ctrl+c
-    if (process.platform === "win32") {
-        var rl = require("readline").createInterface({
-            input: process.stdin,
-            output: process.stdout
-        });
-
-        rl.on("SIGINT", function () {
-            process.emit("SIGINT");
-        });
-    }
-    let killCount = 0;
-    function doShutdown() {
-        killCount++;
-        console.log(`Caught interrupt signal. Attempt number ${killCount}`);
-        if (killCount >= 3) {
-            console.log("Force exit");
-            process.exit();
-        }
-
-        logErrors(shutdown());
-    }
-    process.on("SIGINT", doShutdown);
-
-}
-(globalThis as any).logAll = logAll;
+import { delay, runInfinitePoll, runInfinitePollCallAtStart, shutdownPolling } from "socket-function/src/batching";
+import { isNode, timeInMinute } from "socket-function/src/misc";
+import { logErrors, timeoutToError } from "../errors";
+import debugbreak from "debugbreak";
+import { nodeDiscoveryShutdown } from "../-f-node-discovery/NodeDiscovery";
+import { authorityStorage } from "../0-path-value-core/pathValueCore";
+import { red } from "socket-function/src/formatting/logColors";
+
+// Import querysub, so all the hooks we use will exist at some point
+setImmediate(async () => {
+    await import("../4-querysub/QuerysubController");
+});
+
+let periodicFncs: (() => void)[] = [];
+export function registerPeriodic(fnc: () => void) {
+    periodicFncs.push(fnc);
+}
+
+let shutdownHandlers: (() => Promise<void>)[] = [];
+// NOTE: Won't be always called, but at least makes it possible to gracefully shutdown.
+export function registerShutdownHandler(fnc: () => Promise<void>) {
+    shutdownHandlers.push(fnc);
+}
+let preshutdownHandlers: (() => Promise<void>)[] = [];
+export function registerPreshutdownHandler(fnc: () => Promise<void>) {
+    preshutdownHandlers.push(fnc);
+}
+
+function logAll() {
+    for (let fnc of periodicFncs) {
+        fnc();
+    }
+}
+
+logErrors(runInfinitePollCallAtStart(timeInMinute * 5, logAll));
+
+let shuttingDown = false;
+export async function shutdown() {
+    if (shuttingDown) {
+        return;
+    }
+    console.log(red("Starting shutdown"));
+    shuttingDown = true;
+    const { authorityStorage } = await import("../0-path-value-core/pathValueCore");
+    try {
+        await Promise.allSettled([
+            ...preshutdownHandlers,
+        ].map(fnc => timeoutToError(timeInMinute, fnc(), () => new Error(`Preshutdown handler ${fnc.name} timed out`))));
+    } catch (e) {
+        console.log(`Error on preshutdown handlers`, e);
+    }
+    try {
+        await Promise.allSettled([
+            function authorityStorageShutdown() { return authorityStorage.onShutdown(); },
+            nodeDiscoveryShutdown,
+            shutdownPolling,
+            ...shutdownHandlers,
+        ].map(fnc => timeoutToError(timeInMinute, fnc(), () => new Error(`Shutdown handler ${fnc.name} timed out`))));
+    } catch (e) {
+        console.log("Error on shutdown", e);
+    }
+    // Wait to allow any logged errors to hopefully be written somewhere?
+    await delay(2000);
+    process.exit();
+}
+
+// IMPORTANT! Yarn detaches the processes, so they keep running when you ctrl+c, even though the shell shows back up. We can fix this by using `node -r ./node_modules/typenode/index.js ./test.ts`. However, it's probably fine, as we still run the shutdown code, it's just that the manager doesn't know if we've shutdown or not.
+if (isNode()) {
+    let lineBuffer = "";
+    process.stdin.on("data", data => {
+        lineBuffer += data.toString();
+        let lines = lineBuffer.split("\r");
+        lineBuffer = lines.pop()!;
+        for (let line of lines) {
+            if (line === "SHUTDOWN_NOW_MULTIRUN") {
+                logErrors(shutdown());
+            }
+        }
+
+        if (data.toString().includes("\r")) {
+            logAll();
+        }
+    });
+
+
+    // NOTE: This extra code is required to actual capture ctrl+c
+    if (process.platform === "win32") {
+        var rl = require("readline").createInterface({
+            input: process.stdin,
+            output: process.stdout
+        });
+
+        rl.on("SIGINT", function () {
+            process.emit("SIGINT");
+        });
+    }
+    let killCount = 0;
+    function doShutdown() {
+        killCount++;
+        console.log(`Caught interrupt signal. Attempt number ${killCount}`);
+        if (killCount >= 3) {
+            console.log("Force exit");
+            process.exit();
+        }
+
+        logErrors(shutdown());
+    }
+    process.on("SIGINT", doShutdown);
+
+}
+(globalThis as any).logAll = logAll;
 (globalThis as any).logNow = logAll;

package/src/diagnostics/trackResources.ts

@@ -1,92 +1,92 @@
-import { runInfinitePoll } from "socket-function/src/batching";
-import { lazy } from "socket-function/src/caching";
-import { formatNumber } from "socket-function/src/formatting/format";
-import { blue } from "socket-function/src/formatting/logColors";
-import { isNode } from "socket-function/src/misc";
-import { registerPeriodic } from "./periodic";
-import { registerMeasureInfo } from "socket-function/src/profiling/measure";
-import { logNodeStateStats } from "../-0-hooks/hooks";
-
-let resources: {
-    name: string;
-    getCount: () => number;
-}[] = [];
-
-export function registerResource<T extends Set<unknown> | Map<unknown, unknown> | unknown[]>(name: string, data: T): T {
-    let getCount: () => number;
-    if (Array.isArray(data)) {
-        getCount = () => data.length;
-    } else {
-        getCount = () => data.size;
-    }
-    resources.push({ name, getCount });
-    return data;
-}
-export function registerMapArrayResource<T extends Map<unknown, unknown[]>>(name: string, data: T): T {
-    let getCount = () => {
-        let sum = 0;
-        for (let value of data.values()) {
-            sum += value.length;
-        }
-        return sum;
-    };
-    resources.push({ name, getCount });
-    return data;
-}
-export function registerDynamicResource(name: string, getCount: () => number) {
-    resources.push({ name, getCount });
-}
-
-function getHeapSize() {
-    if (isNode()) {
-        let usage = process.memoryUsage();
-        // Not really the heap size, but it is more what we are looking for
-        return usage.rss;
-    } else {
-        return (window.performance as any).memory.totalJSHeapSize;
-    }
-}
-function getUsedHeapSize() {
-    if (isNode()) {
-        let usage = process.memoryUsage();
-        return usage.heapUsed + usage.arrayBuffers;
-    } else {
-        if (!window.performance) return 0;
-        return (window.performance as any).memory.usedJSHeapSize;
-    }
-}
-function getBufferUsage() {
-    if (isNode()) {
-        let usage = process.memoryUsage();
-        return usage.arrayBuffers;
-    } else {
-        return 0;
-    }
-}
-
-registerDynamicResource("Heap", getUsedHeapSize);
-registerDynamicResource("Buffers", getBufferUsage);
-registerDynamicResource("Used Memory", getHeapSize);
-
-function logResourcesNow() {
-    let resourcesWithCounts = resources.map(resource => ({ ...resource, count: resource.getCount() }));
-    resourcesWithCounts.sort((a, b) => b.count - a.count);
-    if (!isNode()) {
-        console.log(`Memory usage ${blue(formatNumber(getUsedHeapSize()) + "B")}/${blue(formatNumber(getHeapSize()) + "B")} at ${new Date().toISOString()}`);
-        for (let resource of resourcesWithCounts) {
-            if (resource.count === 0) continue;
-            console.log(`  ${resource.name}: ${formatNumber(resource.count)}`);
-        }
-    } else {
-        for (let resource of resourcesWithCounts) {
-            if (resource.count === 0) continue;
-            logNodeStateStats(resource.name, formatNumber, resource.count);
-        }
-    }
-}
-
-registerMeasureInfo(() => {
-    return `MEM ${formatNumber(getUsedHeapSize())}B+${formatNumber(getBufferUsage())}B/${formatNumber(getHeapSize())}B `;
-});
-
+import { runInfinitePoll } from "socket-function/src/batching";
+import { lazy } from "socket-function/src/caching";
+import { formatNumber } from "socket-function/src/formatting/format";
+import { blue } from "socket-function/src/formatting/logColors";
+import { isNode } from "socket-function/src/misc";
+import { registerPeriodic } from "./periodic";
+import { registerMeasureInfo } from "socket-function/src/profiling/measure";
+import { logNodeStateStats } from "../-0-hooks/hooks";
+
+let resources: {
+    name: string;
+    getCount: () => number;
+}[] = [];
+
+export function registerResource<T extends Set<unknown> | Map<unknown, unknown> | unknown[]>(name: string, data: T): T {
+    let getCount: () => number;
+    if (Array.isArray(data)) {
+        getCount = () => data.length;
+    } else {
+        getCount = () => data.size;
+    }
+    resources.push({ name, getCount });
+    return data;
+}
+export function registerMapArrayResource<T extends Map<unknown, unknown[]>>(name: string, data: T): T {
+    let getCount = () => {
+        let sum = 0;
+        for (let value of data.values()) {
+            sum += value.length;
+        }
+        return sum;
+    };
+    resources.push({ name, getCount });
+    return data;
+}
+export function registerDynamicResource(name: string, getCount: () => number) {
+    resources.push({ name, getCount });
+}
+
+function getHeapSize() {
+    if (isNode()) {
+        let usage = process.memoryUsage();
+        // Not really the heap size, but it is more what we are looking for
+        return usage.rss;
+    } else {
+        return (window.performance as any).memory.totalJSHeapSize;
+    }
+}
+function getUsedHeapSize() {
+    if (isNode()) {
+        let usage = process.memoryUsage();
+        return usage.heapUsed + usage.arrayBuffers;
+    } else {
+        if (!window.performance) return 0;
+        return (window.performance as any).memory.usedJSHeapSize;
+    }
+}
+function getBufferUsage() {
+    if (isNode()) {
+        let usage = process.memoryUsage();
+        return usage.arrayBuffers;
+    } else {
+        return 0;
+    }
+}
+
+registerDynamicResource("Heap", getUsedHeapSize);
+registerDynamicResource("Buffers", getBufferUsage);
+registerDynamicResource("Used Memory", getHeapSize);
+
+function logResourcesNow() {
+    let resourcesWithCounts = resources.map(resource => ({ ...resource, count: resource.getCount() }));
+    resourcesWithCounts.sort((a, b) => b.count - a.count);
+    if (!isNode()) {
+        console.log(`Memory usage ${blue(formatNumber(getUsedHeapSize()) + "B")}/${blue(formatNumber(getHeapSize()) + "B")} at ${new Date().toISOString()}`);
+        for (let resource of resourcesWithCounts) {
+            if (resource.count === 0) continue;
+            console.log(`  ${resource.name}: ${formatNumber(resource.count)}`);
+        }
+    } else {
+        for (let resource of resourcesWithCounts) {
+            if (resource.count === 0) continue;
+            logNodeStateStats(resource.name, formatNumber, resource.count);
+        }
+    }
+}
+
+registerMeasureInfo(() => {
+    return `MEM ${formatNumber(getUsedHeapSize())}B+${formatNumber(getBufferUsage())}B/${formatNumber(getHeapSize())}B `;
+});
+
 registerPeriodic(logResourcesNow);