querysub 0.2.0

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -0,0 +1,543 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { getArchives } from "../-a-archives/archives";
+import { getDomain, isDevDebugbreak, isNoNetwork } from "../config";
+import { measureWrap } from "socket-function/src/profiling/measure";
+import { isNode, sha256Hash, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { errorToUndefined, errorToUndefinedSilent, ignoreErrors, logErrors, timeoutToUndefined, timeoutToUndefinedSilent } from "../errors";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { getNodeId, getNodeIdDomain, getNodeIdFromLocation } from "socket-function/src/nodeCache";
+import { lazy } from "socket-function/src/caching";
+import { shuffle } from "../misc/random";
+import { blue, green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
+import debugbreak from "debugbreak";
+import { getArchivesBackblaze, getBackblazePath } from "../-a-archives/archivesBackBlaze";
+import { PromiseObj } from "../promise";
+import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
+import { isClient, isServer } from "../config2";
+import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
+import { decodeNodeId, decodeNodeIdAssert, getMachineId } from "../-a-auth/certs";
+import { getPublicIP } from "../misc/networking";
+
+import dns from "dns/promises";
+import { isDefined } from "../misc";
+import { diskLog } from "../diagnostics/logs/diskLogger";
+
+let HEARTBEAT_INTERVAL = timeInMinute * 5;
+// Interval which we check other heartbeats
+let CHECK_INTERVAL = HEARTBEAT_INTERVAL;
+// If the heartbeat is older than thing, it fails the dead check
+let DEAD_THRESHOLD = HEARTBEAT_INTERVAL * 2;
+// If we find another node is dead this number of checks, we remove it from storage, and tell
+//  all other nodes to remove from their memory.
+let DEAD_CHECK_COUNT = 4;
+// If we find we are unable to write our heartbeat, we have to kill our own process. Otherwise it may
+//  be too out of sync, and might commit unverified data to the disk.
+let SUICIDE_HEARTBEAT_THRESHOLD = timeInMinute * 15;
+
+let CLIENTSIDE_POLL_RATE = timeInMinute * 5;
+
+let MEMORY_AUDIT_RATE = timeInMinute * 5;
+let MEMORY_AUDIT_COUNT = 3;
+
+let shutdown = false;
+
+const archives = lazy(() => getArchives("nodes/"));
+
+let logging = true;
+export function enableNodeDiscoveryLogging() {
+    logging = true;
+}
+export function isNodeDiscoveryLogging() {
+    return logging;
+}
+
+export const getOurNodeId = getOwnNodeId;
+export const getOurMachineId = getOwnNodeId;
+export const getOurNodeIdAssert = getOwnNodeIdAssert;
+
+export const SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE = "SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE";
+export function getOwnNodeId(): string {
+    let nodeId = SocketFunction.mountedNodeId;
+    if (!nodeId) {
+        return SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE;
+    }
+    return nodeId;
+}
+export function getOwnNodeIdAssert(): string {
+    let nodeId = SocketFunction.mountedNodeId;
+    if (!nodeId) {
+        throw new Error(`Node must be mounted before nodeId is accessed`);
+    }
+    return nodeId;
+}
+
+export const getOwnThreadId = lazy(() => {
+    return decodeNodeIdAssert(getOwnNodeIdAssert()).threadId;
+});
+export const getOwnMachineId = lazy(() => {
+    return decodeNodeIdAssert(getOwnNodeIdAssert()).machineId;
+});
+
+export function isOwnNodeId(nodeId: string): boolean {
+    if (
+        nodeId === getOwnNodeId()
+        || nodeId === SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE
+    ) {
+        return true;
+    }
+    // If it's 127.0.0.1, and on the same port as us, it is us.
+    let obj = decodeNodeId(nodeId);
+    if (obj && obj.domain === "127-0-0-1." + getDomain() && obj.port === decodeNodeId(getOwnNodeId())?.port) {
+        return true;
+    }
+
+    return false;
+}
+
+let nodeOverrides: string[] | undefined;
+let beforeGetNodeAllId = async () => { };
+export async function getAllNodeIds() {
+    if (nodeOverrides) {
+        return nodeOverrides;
+    }
+    await beforeGetNodeAllId();
+    return Array.from(allNodeIds2);
+}
+
+export async function syncNodesNow() {
+    await syncArchives();
+}
+
+/** NOTE: You should try to use watchDeltaNodeIds instead of this. */
+export function watchNodeIds(callback: (nodeIds: string[]) => void) {
+    function call() {
+        callback(Array.from(allNodeIds2));
+    }
+    nodeIdWatchers.push(call);
+    call();
+}
+export function watchDeltaNodeIds(callback: (config: { newNodeIds: string[]; removedNodeIds: string[] }) => void) {
+    let prevNodeIds: string[] = [];
+    watchNodeIds(nodeIds => {
+        let newNodeIds = nodeIds.filter(nodeId => !prevNodeIds.includes(nodeId));
+        let removedNodeIds = prevNodeIds.filter(nodeId => !nodeIds.includes(nodeId));
+        if (newNodeIds.length > 0 || removedNodeIds.length > 0) {
+            callback({ newNodeIds, removedNodeIds });
+        }
+        prevNodeIds = nodeIds;
+    });
+}
+
+let allNodeIds2 = new Set<string>();
+function getAllNodesHash() {
+    // Always include ourselves in the hash
+    let sortedNodes = Array.from(allNodeIds2);
+    sortedNodes.sort();
+    return sha256Hash(JSON.stringify(sortedNodes));
+}
+function addNodeId(nodeId: string) {
+    addNodeIdBase(nodeId);
+    if (isNode() && isDevDebugbreak()) {
+        let obj = decodeNodeId(nodeId);
+        if (obj) {
+            let localNodeId = getNodeId("127-0-0-1." + getDomain(), obj.port);
+            addNodeIdBase(localNodeId);
+        }
+    }
+}
+function addNodeIdBase(nodeId: string) {
+    if (allNodeIds2.has(nodeId)) return;
+    if (logging) {
+        console.log(blue(`Discovered node ${nodeId}`));
+    }
+    allNodeIds2.add(nodeId);
+    onNodesChanged();
+}
+function removeNodeIds(nodeIds: string[]) {
+    diskLog("removeNodeIds", { nodeIds });
+    nodeIds = nodeIds.filter(nodeId => allNodeIds2.has(nodeId));
+    if (nodeIds.length === 0) return;
+    for (let nodeId of nodeIds) {
+        if (logging) {
+            console.log(red(`Removed node directly ${nodeId}`));
+        }
+        allNodeIds2.delete(nodeId);
+    }
+    onNodesChanged();
+}
+function setNodeIds(nodeIds: string[]) {
+    diskLog("setNodeIds", { nodeIds });
+    // Also try all localhost ports, if we are in dev mode
+    if (isNode() && isDevDebugbreak()) {
+        let ports = new Set(nodeIds.map(nodeId => decodeNodeId(nodeId)?.port).filter(isDefined));
+        for (let port of ports) {
+            let localNodeId = getNodeId("127-0-0-1." + getDomain(), port);
+            nodeIds.push(localNodeId);
+        }
+        nodeIds = Array.from(new Set(nodeIds));
+    }
+    let newNodeIds = nodeIds.filter(nodeId => !allNodeIds2.has(nodeId));
+    let newIds = new Set(nodeIds);
+    let removedNodeIds = Array.from(allNodeIds2).filter(nodeId => !newIds.has(nodeId));
+    if (newNodeIds.length === 0 && removedNodeIds.length === 0) return;
+    if (logging) {
+        for (let nodeId of newNodeIds) {
+            console.log(blue(`Discovered node ${nodeId}`));
+        }
+        for (let nodeId of removedNodeIds) {
+            console.log(red(`Removed node from setNodeIds ${nodeId}`));
+        }
+    }
+    allNodeIds2 = new Set(nodeIds);
+    onNodesChanged();
+}
+let nodeIdWatchers: (() => void)[] = [];
+function onNodesChanged() {
+    for (let watcher of nodeIdWatchers) {
+        try {
+            watcher();
+        } catch (e: any) {
+            console.log(red(`Ignoring error from callback ${e.stack}`));
+        }
+    }
+}
+
+let rootDiscoveryNodeId = "";
+export function configRootDiscoveryLocation(config: {
+    domain: string;
+    port: number;
+}) {
+    rootDiscoveryNodeId = getNodeId(config.domain, config.port);
+}
+/** NOTE: Can also be called serverside, if configRootDiscoveryLocation is called (otherwise can always be called clientside). */
+export function getBrowserUrlNode() {
+    if (!isClient()) throw new Error(`getBrowserUrlNode can only be called when isClient()`);
+    let rootLocation = rootDiscoveryNodeId;
+    if (!rootLocation && !isNode()) {
+        rootLocation = getNodeIdFromLocation();
+    }
+    if (!rootLocation) {
+        throw new Error(`configRootDiscoveryLocation wasn't called, and ---client appears to have been used (isNode() and isClient()). Call configRootDiscoveryLocation with the domain and port of the server you wish to use (ex, querysub.com:1111)`);
+    }
+    return rootLocation;
+}
+
+let seenNodesForEvilDNS = new Set<string>();
+async function syncArchives() {
+    if (isServer()) {
+        // Make sure we are present
+        await writeHeartbeat();
+        let nodeIds = await archives().find("");
+        console.log(green(`Syncing archives`), { nodeIds });
+        setNodeIds(nodeIds);
+    } else {
+        if (isNoNetwork() || !isNode()) {
+            // NOTE: If no network, our trust source might be different, so we can't talk to regular nodes,
+            //  and instead have to only talk to HTTP nodes
+            setNodeIds([getBrowserUrlNode()]);
+        } else {
+            // If on the network, NetworkTrust2 should sync the trusted machines from backblaze, so we should be
+            //  able to talk to any nodes.
+            //  - If they user is using --client they only want to talk to querysub nodes. There might be multiple,
+            //      which cloudflare will proxy, HOWEVER, it is more efficient to directly access the node list, which
+            //      will be better for load balancing and updating on failure than the cloudflare proxying... probably.
+            setNodeIds(await NodeDiscoveryController.nodes[getBrowserUrlNode()].getAllNodeIds());
+        }
+    }
+}
+
+async function runHeartbeatAuditLoop() {
+    await onNodeDiscoveryReady();
+    let deadCount = new Map<string, number>();
+    await runInfinitePollCallAtStart(CHECK_INTERVAL, async () => {
+        if (shutdown) return;
+        // Wait a bit longer, to try to prevent all nodes from synchronizing their audit times.
+        await delay(HEARTBEAT_INTERVAL * Math.random() * 0.1);
+        //console.log(magenta(`Auditing node list`));
+
+        let deadTime = Date.now() - DEAD_THRESHOLD;
+        let nodeIds = await archives().find("");
+        // Should be unusual, but... if we find new nodes, add them, and let everyone else know?
+        for (let nodeId of nodeIds) {
+            if (!allNodeIds2.has(nodeId)) {
+                console.log(yellow(`Found unexpected node, adding it and broadcasting it, ${nodeId}`));
+                addNodeId(nodeId);
+                for (let nodeId of allNodeIds2) {
+                    if (isOwnNodeId(nodeId)) continue;
+                    ignoreErrors(NodeDiscoveryController.nodes[nodeId].addNode(nodeId));
+                }
+            }
+        }
+        let removedNodeIds: string[] = [];
+        for (let nodeId of nodeIds) {
+            let lastTime = Number((await archives().get(nodeId))?.toString()) || 0;
+            if (lastTime < deadTime) {
+                // Increment the dead count
+                let count = deadCount.get(nodeId) || 0;
+                count++;
+                deadCount.set(nodeId, count);
+                if (count >= DEAD_CHECK_COUNT) {
+                    removedNodeIds.push(nodeId);
+                    console.log(yellow(`Node ${nodeId} was found to be dead, removing from node list. Last heartbeat at ${formatDateTime(lastTime)}, dead threshold at ${formatDateTime(deadTime)}`));
+                    await archives().del(nodeId);
+                    deadCount.delete(nodeId);
+                } else {
+                    console.log(yellow(`Node ${nodeId} was found to be dead, last heartbeat at ${formatDateTime(lastTime)} < dead threshold at ${formatDateTime(deadTime)}, dead count ${count}/${DEAD_CHECK_COUNT}. Total nodes seen ${nodeIds.length}`));
+                }
+            } else {
+                deadCount.delete(nodeId);
+                diskLog("Read node heartbeat", { nodeId, lastTime });
+            }
+        }
+
+        removeNodeIds(removedNodeIds);
+
+        // Broadcast to all nodes the removal
+        for (let nodeId of allNodeIds2) {
+            if (isOwnNodeId(nodeId)) continue;
+            ignoreErrors(NodeDiscoveryController.nodes[nodeId].removeNodes(removedNodeIds));
+        }
+    });
+}
+
+async function isOutOfSync() {
+    let auditNodes = shuffle(Array.from(allNodeIds2), Date.now()).slice(0, MEMORY_AUDIT_COUNT);
+    for (let nodeId of auditNodes) {
+        if (nodeId === getOwnNodeId()) continue;
+
+        let stableWait = timeInSecond * 30;
+        while (true) {
+            let hash = await errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].getAllNodesHash());
+            if (!hash) break;
+            let ownHash = getAllNodesHash();
+            if (hash === ownHash) break;
+            await delay(stableWait);
+            let hash2 = await errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].getAllNodesHash());
+            if (!hash2) break;
+            if (hash === hash2) {
+                console.log(yellow(`Node discovery is out of sync, syncing with disk. Told by node ${nodeId}, our hash of ${ownHash} is wrong and should be ${hash2}`));
+                return true;
+            }
+            // Otherwise, the hash changed, so... we maybe we just caught it while it was changing.
+            //  Wait less time, to try to catch up with the rate of change
+            stableWait /= 2;
+            if (stableWait < timeInSecond) {
+                // It is changing so fast we better just sync with the disk
+                console.log(yellow(`Node discovery is changing fast and likely out of sync, resyncing with disk. Told by node ${nodeId}`));
+                return true;
+            }
+        }
+    }
+}
+
+async function runMemoryAuditLoop() {
+    await onNodeDiscoveryReady();
+    runInfinitePoll(MEMORY_AUDIT_RATE, async () => {
+        // In retrospec, reading all nodes every 5 minutes is fine. It costs about 0.035 USD
+        //  per month to do this, which... is a lot less than the server to run this will cost!
+        //if (await isOutOfSync()) {
+        await syncArchives();
+        //}
+    });
+}
+
+async function writeHeartbeat() {
+    if (shutdown) return;
+    let now = Date.now();
+    console.log(green(`Writing heartbeat ${formatDateTime(now)}`));
+    await archives().set(getOwnNodeId(), Buffer.from(now + ""));
+}
+
+async function runMainSyncLoops(discoveryReady: PromiseObj<void>) {
+    await syncArchives();
+
+    discoveryReady.resolve();
+
+    // We can't heartbeat until we mount
+    await SocketFunction.mountPromise;
+
+    // We have to write before we call NodeDiscoveryController.addNode
+    await writeHeartbeat();
+    await syncArchives();
+    let selfNodeId = SocketFunction.mountedNodeId;
+    if (!allNodeIds2.has(selfNodeId)) {
+        throw new Error(`Failed sanity check, our node id didn't appear in archives after we just wrote it? Missing ${SocketFunction.mountedNodeId}`);
+    }
+    // NOTE: Our first broadcast is special
+    await Promise.allSettled(Array.from(allNodeIds2).map(async nodeId => {
+        if (isOwnNodeId(nodeId)) return;
+        // Ignore errors, but wait a bit, so hopefully 99.99% of the time we can be certain
+        //  all other nodes know our node id at this point.
+        await timeoutToUndefinedSilent(timeInSecond * 5, errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId())));
+    }));
+
+    await runInfinitePoll(HEARTBEAT_INTERVAL, async function nodeDiscoverHeartbeat() {
+        // If we waited too long, other nodes might think we are dead. In which case, we SHOULD terminate.
+        if (!isNoNetwork()) {
+            // FIRST, verify we didn't delay too long (to make sure we kill any nodes that were disconnected
+            //  from the internet for too long)
+            let lastTime = Number((await archives().get(selfNodeId))?.toString());
+            let suicideThreshold = Date.now() - SUICIDE_HEARTBEAT_THRESHOLD;
+            if (!lastTime || lastTime < suicideThreshold) {
+                if (!lastTime) {
+                    console.error(red(`Self node was removed due to not heartbeating. Terminating self process, as it likely has very stale data.`));
+                } else {
+                    console.error(red(`Self node was has very old heartbeat. Terminating self process, as it likely has very stale data.`));
+                }
+                process.exit();
+            }
+        } else {
+            // We DO have to re-broadcast, otherwise no one will know we exist
+            for (let nodeId of allNodeIds2) {
+                if (isOwnNodeId(nodeId)) continue;
+                // Ignore errors, as nodes die often, and we only want to log once - when we actually remove the node id.
+                ignoreErrors(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId()));
+            }
+        }
+
+        await writeHeartbeat();
+    });
+}
+
+export async function onNodeDiscoveryReady() {
+    // NOTE: This isn't really early enough, especially in the browser, because a lot of calls are created
+    //  immediately on render. But... it should be good enough for now.
+    await waitForFirstTimeSync();
+    await getAllNodeIds();
+}
+
+if (isServer()) {
+    let discoveryReady = new PromiseObj<void>();
+    beforeGetNodeAllId = async () => {
+        await discoveryReady.promise;
+    };
+    setImmediate(() => {
+        logErrors(runHeartbeatAuditLoop());
+        logErrors(runMemoryAuditLoop());
+        // NOTE: We used to wait until we mounted, but... we should be able to find nodes
+        //  before we mount, right? (And what if we never mount?)
+        runMainSyncLoops(discoveryReady).catch(e => {
+            discoveryReady.reject(e);
+            logErrors(Promise.reject(e));
+        });
+        // // Wait until we mount
+        // void SocketFunction.mountPromise.finally(async () => {
+
+        // });
+    });
+} else {
+    setImmediate(() => {
+        if (isNode()) {
+            // Just get the archives, syncing again if we haven't synced in a while
+            let lastGetTime = 0;
+            beforeGetNodeAllId = async () => {
+                let lastGetThreshold = lastGetTime + CLIENTSIDE_POLL_RATE;
+                if (Date.now() > lastGetThreshold) {
+                    lastGetTime = Date.now();
+                    await syncArchives();
+                }
+            };
+        } else {
+            let discoveryReady = new PromiseObj<void>();
+            let browserNodeId = getBrowserUrlNode();
+            let nodes = [getBrowserUrlNode()];
+            logErrors((async function checkForLocalHost() {
+                // If there is a server at the localhost domain, AND, it's the same as the current server...
+                //  use that domain instead. This is just an optimization, but makes a big difference
+                //  when using management tools to scan the DB (otherwise the entire DB has to go through
+                //  our router, which can easily cap it at 15MB/s, because rogers is terrible).
+                //  - Only if isNoNetwork, otherwise it causes unnecessary lag.
+                let isNoNetwork = await NodeDiscoveryController.nodes[browserNodeId].isNoNetwork();
+                if (isNoNetwork && !location.search.includes("nolocalhost")) {
+                    let url = new URL("https://" + browserNodeId);
+                    url.hostname = "127-0-0-1." + url.hostname;
+                    let localhostNode = url.host;
+                    // NOTE: The timeout isn't that important, as this is only if no network, and only on page load.
+                    //  - A low timeout here causes failure to use the localhost ip sometimes. WHICH, often breaks
+                    //      authentication (as we are likely not authenticated for our public IP).
+                    let localhostNodeId = await timeoutToUndefinedSilent(1000, NodeDiscoveryController.nodes[localhostNode].getNodeId());
+                    let httpServerNodeId = await NodeDiscoveryController.nodes[browserNodeId].getNodeId();
+                    if (localhostNodeId === httpServerNodeId) {
+                        nodes[0] = localhostNode;
+                        rootDiscoveryNodeId = localhostNode;
+                    }
+                }
+
+                allNodeIds2 = new Set(nodes);
+                discoveryReady.resolve();
+            })());
+
+            // Block until we finish checking for local host
+            beforeGetNodeAllId = async () => {
+                await discoveryReady.promise;
+            };
+
+            // NOTE: We run into TLS issues (as in, our servers use self signed certs), if we try to talk to just
+            //  any node, so... we better just talk to the edge node
+            //  - We COULD probably just use some special domain (maybe JUST the machine domain?), with limited wildcard
+            //      certs (I think we can only wildcard a single depth anyways), and A records for the machines too...
+            //      but... having all traffic route through an edge node is probably better anyways...
+            nodeOverrides = nodes;
+        }
+    });
+}
+
+
+/** Called on shutdown, to completely remove this node from discovery. */
+export async function nodeDiscoveryShutdown() {
+    console.log(red(`Shutting down node discovery`));
+    shutdown = true;
+    if (isServer()) {
+        await archives().del(getOwnNodeId());
+    }
+    // Don't bother to remove the node from ourself, as we will be shutting down shortly
+    //  (but do bother to tell other nodes!)
+    await Promise.allSettled(Array.from(allNodeIds2).map(async nodeId => {
+        if (isOwnNodeId(nodeId)) return;
+        await timeoutToUndefinedSilent(timeInSecond * 5, errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].removeNodes([getOwnNodeId()])));
+    }));
+}
+
+
+class NodeDiscoveryControllerBase {
+    public async addNode(nodeId: string) {
+        addNodeId(nodeId);
+    }
+    public async removeNodes(nodeIds: string[]) {
+        let callerId = SocketFunction.getCaller().nodeId;
+        console.log(`remote removeNode request`, { callerId, nodeIds });
+        removeNodeIds(nodeIds);
+    }
+    public async getAllNodesHash(): Promise<string> {
+        return getAllNodesHash();
+    }
+
+    public async getAllNodeIds(): Promise<string[]> {
+        return Array.from(allNodeIds2);
+    }
+    public async getNodeId() {
+        console.log(`Returning node id ${SocketFunction.mountedNodeId}`);
+        return SocketFunction.mountedNodeId;
+    }
+    public async isNoNetwork() {
+        return isNoNetwork();
+    }
+}
+const NodeDiscoveryController = SocketFunction.register(
+    "NodeDiscoveryController-7991037e-fd9e-4085-b1db-52035487e72c",
+    new NodeDiscoveryControllerBase(),
+    () => ({
+        addNode: { hooks: [requiresNetworkTrustHook] },
+        removeNodes: { hooks: [requiresNetworkTrustHook] },
+        getAllNodesHash: { hooks: [requiresNetworkTrustHook] },
+        // Skip client hooks, so we don't block on authentication (IdentityController), as some of these functions
+        //  are needed for authentication to finish!
+        getAllNodeIds: { noClientHooks: true },
+        getNodeId: { noClientHooks: true },
+        isNoNetwork: { noClientHooks: true },
+    }),
+    () => ({
+
+    })
+);

package/src/-g-core-values/NodeCapabilities.ts

@@ -0,0 +1,134 @@
+/*
+    Find nodes that implement a specific controller.
+*/
+
+import { SocketFunction } from "socket-function/SocketFunction";
+import { SocketRegistered } from "socket-function/SocketFunctionTypes";
+import { errorToUndefined, errorToUndefinedSilent, timeoutToUndefinedSilent } from "../errors";
+import { getAllNodeIds } from "../-f-node-discovery/NodeDiscovery";
+import { green, red, yellow } from "socket-function/src/formatting/logColors";
+import { shuffle } from "../misc/random";
+import { delay } from "socket-function/src/batching";
+import debugbreak from "debugbreak";
+import { formatTime } from "socket-function/src/formatting/format";
+import type { DebugFunctionShardInfo } from "../3-path-functions/PathFunctionRunner";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { isNoNetwork } from "../config";
+import { getDebuggerUrl } from "../diagnostics/listenOnDebugger";
+import { hackDevtoolsWebsocketForward } from "./oneTimeForward";
+
+let loadTime = Date.now();
+
+// NOTE: If this becomes slow (because we are just trying all servers), we could start to store capabilities
+//  in PersistedStorage. However, I don't think it will be a problem for quite some time.
+export async function getControllerNodeId(
+    controller: SocketRegistered<{}>,
+    quiet = false,
+    retryCount = 1,
+    initialTime = Date.now(),
+): Promise<string | undefined> {
+    let nodeIdsToTest = await getAllNodeIds();
+    // Shuffle, so we aren't always using the same node!
+    nodeIdsToTest = shuffle(nodeIdsToTest, Date.now());
+    for (let nodeId of nodeIdsToTest) {
+        if (await doesNodeExposeController(nodeId, controller)) {
+            if (!quiet) {
+                let duration = Date.now() - initialTime;
+                console.log(green(`Resolved capability ${controller._classGuid} with node ${nodeId}, in ${formatTime(duration)}`));
+            }
+            return nodeId;
+        }
+    }
+    if (retryCount > 0) {
+        await delay(2000);
+        return getControllerNodeId(controller, quiet, retryCount - 1, initialTime);
+    }
+    if (!quiet) {
+        console.warn(yellow(`Could not find a node that exposes controller ${controller._classGuid}, tried: ${JSON.stringify(nodeIdsToTest)}`));
+    }
+    return undefined;
+}
+
+export async function getControllerNodeIdList(
+    controller: SocketRegistered<{}>,
+): Promise<string[]> {
+    let nodeIdsToTest = await getAllNodeIds();
+    let passedNodeIds = new Set<string>();
+    for (let nodeId of nodeIdsToTest) {
+        if (await doesNodeExposeController(nodeId, controller)) {
+            passedNodeIds.add(nodeId);
+        }
+    }
+    return Array.from(passedNodeIds);
+}
+
+export async function doesNodeExposeController(reconnectNodeId: string, controller: SocketRegistered<{}>): Promise<boolean> {
+    let exposedControllers = await timeoutToUndefinedSilent(10_000, NodeCapabilitiesController.nodes[reconnectNodeId].getExposedControllers());
+    return !!exposedControllers?.includes(controller._classGuid);
+}
+
+const startupTime = Date.now();
+class NodeCapabilitiesControllerBase {
+    public async getExposedControllers() {
+        return Array.from(SocketFunction.exposedClasses);
+    }
+    public async getEntryPoint() {
+        return process.argv[1];
+    }
+    public async getStartupTime() {
+        return startupTime;
+    }
+    public async getMemoryUsage() {
+        return process.memoryUsage();
+    }
+    public async getFunctionRunnerShards() {
+        return getFunctionRunnerShards();
+    }
+
+    public async getInspectURL() {
+        return getDebuggerUrl();
+    }
+
+    public async exposeExternalDebugPortOnce(forExternalIP: string) {
+        // https://notdevtools.com/devtools/inspector.html?experiments=true&v8only=true&ws=127.0.0.1:62448/22895aed-f8da-4dfb-8d50-e432a9c2d827
+        let debugURL = await this.getInspectURL();
+
+        const outerUrl = new URL(debugURL);
+        const wsParam = outerUrl.searchParams.get("ws");
+        if (!wsParam) throw new Error("No ws parameter found");
+
+        // Add ws:// protocol to make it parseable by URL
+        const innerUrl = new URL(`ws://${wsParam}`);
+        const port = innerUrl.port;
+        if (!port) throw new Error("No port found");
+
+        const portNumber = parseInt(port, 10);
+        if (isNaN(portNumber)) throw new Error("Invalid port number");
+
+        let { externalPort, cancel } = await hackDevtoolsWebsocketForward({ internalPort: portNumber, externalIP: forExternalIP });
+        lastExposed?.();
+        lastExposed = cancel;
+        return { externalPort, internalPort: portNumber, internalInspectURL: debugURL };
+    }
+}
+let lastExposed: (() => void) | undefined;
+
+
+let getFunctionRunnerShards = () => [] as DebugFunctionShardInfo[];
+export function set_debug_getFunctionRunnerShards(fnc: () => DebugFunctionShardInfo[]) {
+    getFunctionRunnerShards = fnc;
+}
+
+export const NodeCapabilitiesController = SocketFunction.register(
+    "NodeCapabilitiesController-399b7067-75c3-4d92-8be3-8470bde32d3c",
+    new NodeCapabilitiesControllerBase(),
+    () => ({
+        getExposedControllers: {},
+        getEntryPoint: {},
+        getStartupTime: {},
+        getMemoryUsage: {},
+        getFunctionRunnerShards: {},
+        getInspectURL: { hooks: [requiresNetworkTrustHook] },
+        exposeExternalDebugPortOnce: { hooks: [requiresNetworkTrustHook] },
+    })
+);