querysub 0.433.0 → 0.436.0

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -1,641 +1,641 @@
-import { SocketFunction } from "socket-function/SocketFunction";
-import { getArchives } from "../-a-archives/archives";
-import { getDomain, isDevDebugbreak, isNoNetwork, isPublic } from "../config";
-import { measureBlock } from "socket-function/src/profiling/measure";
-import { isNode, keyByArray, sha256Hash, throttleFunction, timeInMinute, timeInSecond } from "socket-function/src/misc";
-import { errorToUndefinedSilent, ignoreErrors, logErrors, timeoutToError, timeoutToUndefinedSilent } from "../errors";
-import { ensureWeAreTrusted, requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
-import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { getCallFactory, getCreateCallFactory, getNodeId, getNodeIdFromLocation, getNodeIdLocation } from "socket-function/src/nodeCache";
-import { cache, lazy } from "socket-function/src/caching";
-import { shuffle } from "../misc/random";
-import { blue, green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
-import { PromiseObj } from "../promise";
-import { formatDateTime } from "socket-function/src/formatting/format";
-import { isClient, isServer } from "../config2";
-import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
-import { decodeNodeId, decodeNodeIdAssert } from "../-a-auth/certs";
-
-import { isDefined } from "../misc";
-import { getBootedEdgeNode } from "../-0-hooks/hooks";
-import { EdgeNodeConfig } from "../4-deploy/edgeNodes";
-import * as certs from "../-a-auth/certs";
-import { logDisk } from "../diagnostics/logs/diskLogger";
-import { MaybePromise } from "socket-function/src/types";
-import { getPathStr2 } from "../path";
-
-let HEARTBEAT_INTERVAL = timeInMinute * 15;
-// Interval which we check other heartbeats
-let CHECK_INTERVAL = HEARTBEAT_INTERVAL;
-// If the heartbeat is older than thing, it fails the dead check
-let DEAD_THRESHOLD = HEARTBEAT_INTERVAL * 2;
-// If we find another node is dead this number of checks, we remove it from storage, and tell
-//  all other nodes to remove from their memory.
-let DEAD_CHECK_COUNT = 4;
-// If we find we are unable to write our heartbeat, we have to kill our own process. Otherwise it may
-//  be too out of sync, and might commit unverified data to the disk.
-let SUICIDE_HEARTBEAT_THRESHOLD = HEARTBEAT_INTERVAL * 4;
-
-let CLIENTSIDE_POLL_RATE = timeInMinute * 30;
-
-// We can't poll backblaze too often. One a minute starts to cost around 10 cents per month.
-//  So once a second would cost 6 USD per month, per service... which starts to get expensive.
-let DISK_AUDIT_RATE = timeInMinute * 15;
-// We CAN poll our API frequently. The overhead for calls should be far less than 1ms, and
-//  the bandwidth should also be effectively nothing. Maybe 2.5GB per month if we send a
-//  request per second, and each request is 1000 bytes (which as we use websockets, it
-//  probably is less than that). Which is around 2.5 cents on digital ocean IF we go over
-//  our 1TB/month allowance.
-let API_AUDIT_RATE = timeInSecond * 30;
-let API_AUDIT_COUNT = 12;
-
-
-let DEAD_NODE_POLL_COOLDOWN = timeInMinute * 5;
-
-let shutdown = false;
-
-const archives = lazy(() => getArchives("nodes/"));
-
-let logging = true;
-export function enableNodeDiscoveryLogging() {
-    logging = true;
-}
-export function isNodeDiscoveryLogging() {
-    return logging;
-}
-
-
-function getAlternateNodeIds(nodeId: string): MaybePromise<string[] | undefined> {
-    let machineId = certs.getMachineId(nodeId);
-    if (machineId === getOwnMachineId()) {
-        let decoded = decodeNodeId(nodeId);
-        if (decoded) {
-            return [
-                "127-0-0-1." + decoded.domain + ":" + decoded.port
-            ];
-        }
-    }
-    return undefined;
-}
-SocketFunction.GET_ALTERNATE_NODE_IDS = getAlternateNodeIds;
-
-export const getOurNodeId = getOwnNodeId;
-export const getOurNodeIdAssert = getOwnNodeIdAssert;
-
-export const SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE = "SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE";
-export function getOwnNodeId(): string {
-    let nodeId = SocketFunction.mountedNodeId;
-    if (!nodeId) {
-        return SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE;
-    }
-    return nodeId;
-}
-export function getMountNodeId(): string | undefined | "" {
-    return SocketFunction.mountedNodeId;
-}
-export function getOwnNodeIdAssert(): string {
-    let nodeId = SocketFunction.mountedNodeId;
-    if (!nodeId) {
-        throw new Error(`Node must be mounted before nodeId is accessed`);
-    }
-    return nodeId;
-}
-
-export const getOwnThreadId = lazy(() => {
-    return certs.getOwnThreadId();
-});
-export const getOwnMachineId = lazy(() => {
-    return certs.getOwnMachineId();
-});
-
-export function isOwnNodeId(nodeId: string): boolean {
-    if (
-        nodeId === getOwnNodeId()
-        || nodeId === SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE
-    ) {
-        return true;
-    }
-    // If it's 127.0.0.1, and on the same port as us, it is us.
-    let obj = decodeNodeId(nodeId);
-    if (obj && obj.domain === "127-0-0-1." + getDomain() && obj.port === decodeNodeId(getOwnNodeId())?.port) {
-        return true;
-    }
-
-    return false;
-}
-
-export function isNodeIdOnOwnMachineId(nodeId: string): boolean {
-    return certs.getMachineId(nodeId) === getOwnMachineId() || decodeNodeId(nodeId)?.machineId === "127-0-0-1";
-}
-
-export function areNodeIdsEqual(lhs: string, rhs: string): boolean {
-    if (lhs === rhs) return true;
-    return isNodeIdOnOwnMachineId(lhs) && isNodeIdOnOwnMachineId(rhs) && getNodeIdLocation(rhs)?.port === getNodeIdLocation(lhs)?.port;
-}
-
-let nodeOverrides: string[] | undefined;
-let beforeGetNodeAllId = async () => { };
-export async function getAllNodeIds() {
-    await beforeGetNodeAllId();
-
-    // NOTE: We always wait for the time sync before returning any node ids. Because if we try to talk
-    //  to remote nodes with a massively out of sync time, they will likely reject our messages!
-    let promise = waitForFirstTimeSync();
-    if (promise) {
-        await measureBlock(async () => {
-            await promise;
-        }, "waitForFirstTimeSync");
-    }
-
-    if (nodeOverrides) {
-        return nodeOverrides;
-    }
-    return Array.from(allNodeIds2);
-}
-
-export async function syncNodesNow() {
-    await syncArchives();
-}
-
-/** NOTE: You should try to use watchDeltaNodeIds instead of this. */
-export function watchNodeIds(callback: (nodeIds: string[]) => void) {
-    function call() {
-        callback(Array.from(allNodeIds2));
-    }
-    nodeIdWatchers.push(call);
-    call();
-}
-export function watchDeltaNodeIds(callback: (config: { newNodeIds: string[]; removedNodeIds: string[] }) => void) {
-    let prevNodeIds: string[] = [];
-    watchNodeIds(nodeIds => {
-        let newNodeIds = nodeIds.filter(nodeId => !prevNodeIds.includes(nodeId));
-        let removedNodeIds = prevNodeIds.filter(nodeId => !nodeIds.includes(nodeId));
-        if (newNodeIds.length > 0 || removedNodeIds.length > 0) {
-            callback({ newNodeIds, removedNodeIds });
-        }
-        prevNodeIds = nodeIds;
-    });
-}
-
-let allNodeIds2 = new Set<string>();
-function getAllNodesHash() {
-    // Always include ourselves in the hash
-    let sortedNodes = Array.from(allNodeIds2);
-    sortedNodes.sort();
-    return sha256Hash(JSON.stringify(sortedNodes));
-}
-function addNodeId(nodeId: string) {
-    addNodeIdBase(nodeId);
-}
-function addNodeIdBase(nodeId: string) {
-    if (allNodeIds2.has(nodeId)) return;
-    if (logging) {
-        console.log(blue(`Discovered node ${nodeId}`));
-    }
-    allNodeIds2.add(nodeId);
-    onNodesChanged();
-}
-async function setNodeIds(nodeIds: string[]) {
-    nodeIds = nodeIds.filter(x => x !== SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE);
-
-    console.info("setNodeIds", { nodeIds });
-    let newNodeIds = nodeIds.filter(nodeId => !allNodeIds2.has(nodeId));
-    let newIds = new Set(nodeIds);
-    let removedNodeIds = Array.from(allNodeIds2).filter(nodeId => !newIds.has(nodeId));
-    if (newNodeIds.length === 0 && removedNodeIds.length === 0) return;
-    if (logging) {
-        for (let nodeId of newNodeIds) {
-            console.log(blue(`Discovered node ${nodeId}`));
-        }
-        for (let nodeId of removedNodeIds) {
-            console.log(red(`Removed node from setNodeIds ${nodeId}`));
-        }
-    }
-    allNodeIds2 = new Set(nodeIds);
-    onNodesChanged();
-}
-let nodeIdWatchers: (() => void)[] = [];
-function onNodesChanged() {
-    for (let watcher of nodeIdWatchers) {
-        try {
-            watcher();
-        } catch (e: any) {
-            console.log(red(`Ignoring error from callback ${e.stack}`));
-        }
-    }
-}
-
-let rootDiscoveryNodeId = "";
-export function configRootDiscoveryLocation(config: {
-    domain: string;
-    port: number;
-}) {
-    rootDiscoveryNodeId = getNodeId(config.domain, config.port);
-}
-
-export function updateRootDiscoveryLocation(edgeNode: EdgeNodeConfig) {
-    let prevRootId = getBrowserUrlNode();
-    if (!isClient()) {
-        throw new Error(`updateRootDiscoveryLocation should only be called in the browser. If not a client, you should be able to discovery new nodes automatically`);
-    }
-    let nodeId = edgeNode.host;
-    let nodeCache = Object.entries(require.cache);
-    for (let [path, module] of nodeCache) {
-        if (!module) continue;
-        if (!path.includes(prevRootId)) continue;
-        let newPath = path.replace(prevRootId, nodeId);
-        // Remove the old module
-        delete require.cache[path];
-        // Add the new module
-        require.cache[newPath] = module;
-        module.filename = module.filename.replace(prevRootId, nodeId);
-        module.id = module.id.replace(prevRootId, nodeId);
-        if (module.original) {
-            module.original.filename = module.original.filename.replace(prevRootId, nodeId);
-            module.original.originalId = module.original.originalId.replace(prevRootId, nodeId);
-            for (let [key, value] of Object.entries(module.original.requests)) {
-                if (key.includes(prevRootId)) {
-                    let newKey = key.replace(prevRootId, nodeId);
-                    delete module.original.requests[key];
-                    module.original.requests[newKey] = value;
-                    key = newKey;
-                }
-                if (value.includes(prevRootId)) {
-                    value = value.replace(prevRootId, nodeId);
-                    module.original.requests[key] = value;
-                }
-            }
-        }
-    }
-    globalThis.BOOTED_EDGE_NODE = edgeNode;
-    rootDiscoveryNodeId = nodeId;
-    addNodeId(nodeId);
-}
-(globalThis as any).updateRootDiscoveryLocation = updateRootDiscoveryLocation;
-
-/** NOTE: Can also be called serverside, if configRootDiscoveryLocation is called (otherwise can always be called clientside). */
-export function getBrowserUrlNode() {
-    if (!isClient()) throw new Error(`getBrowserUrlNode can only be called when isClient()`);
-    let rootLocation = rootDiscoveryNodeId;
-    if (!rootLocation && !isNode()) {
-        rootLocation = getNodeIdFromLocation();
-    }
-    if (!rootLocation) {
-        throw new Error(`configRootDiscoveryLocation wasn't called, and ---client appears to have been used (isNode() and isClient()). Call configRootDiscoveryLocation with the domain and port of the server you wish to use (ex, querysub.com:1111)`);
-    }
-    return rootLocation;
-}
-
-export async function triggerNodeChange() {
-    await syncNodesNow();
-    await Promise.allSettled(Array.from(allNodeIds2).map(async nodeId => {
-        if (isOwnNodeId(nodeId)) return;
-        await timeoutToUndefinedSilent(timeInSecond * 5, errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId())));
-    }));
-}
-
-
-async function clearDeadThreadsFromArchives() {
-    let nodes = await archives().find("");
-
-    function getPortHash(nodeId: string) {
-        let obj = decodeNodeId(nodeId);
-        if (!obj) return undefined;
-        return getPathStr2(obj.machineId, obj.port + "");
-    }
-    let byPortHash = keyByArray(nodes, getPortHash);
-    for (let [portHash, nodeIds] of Array.from(byPortHash)) {
-        if (!portHash) continue;
-        let aliveNodeId = "";
-        await Promise.all(nodeIds.map(async nodeId => {
-            let alive = await errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].isAlive());
-            if (alive) {
-                aliveNodeId = nodeId;
-            }
-        }));
-        if (aliveNodeId) {
-            let deadThreads = nodeIds.filter(nodeId => nodeId !== aliveNodeId);
-            await Promise.all(deadThreads.map(async deadNodeId => {
-                console.log(`Removing dead thread. We contacted a node on the same port and same machine (${aliveNodeId}), which means the port has been reused by another thread, which proves that the old thread has died, as otherwise the new thread would not be able to use it.`);
-                await archives().del(deadNodeId);
-            }));
-        }
-    }
-
-    return nodes;
-
-}
-
-async function syncArchives() {
-    if (isServer()) {
-        // Make sure we are present
-        await writeHeartbeat();
-        let nodeIds = await archives().find("");
-        console.info(`Synced node ids from archives`, { nodeIds });
-        await setNodeIds(nodeIds);
-    } else {
-        if (isNoNetwork() || !isNode()) {
-            // NOTE: If no network, our trust source might be different, so we can't talk to regular nodes,
-            //  and instead have to only talk to HTTP nodes
-            await setNodeIds([getBrowserUrlNode()]);
-        } else {
-            // If on the network, NetworkTrust2 should sync the trusted machines from backblaze, so we should be
-            //  able to talk to any nodes.
-            //  - If they user is using --client they only want to talk to querysub nodes. There might be multiple,
-            //      which cloudflare will proxy, HOWEVER, it is more efficient to directly access the node list, which
-            //      will be better for load balancing and updating on failure than the cloudflare proxying... probably.
-            await setNodeIds(await NodeDiscoveryController.nodes[getBrowserUrlNode()].getAllNodeIds());
-        }
-    }
-}
-
-async function runHeartbeatAuditLoop() {
-    await getAllNodeIds();
-    let deadCount = new Map<string, number>();
-    // 90% of the normal interval, so we don't run at the same time as the other audit
-    await runInfinitePollCallAtStart(CHECK_INTERVAL * 0.9, async () => {
-        if (shutdown) return;
-        // Wait a bit longer, to try to prevent all nodes from synchronizing their audit times.
-        console.log(magenta(`Auditing node list`));
-        await delay(CHECK_INTERVAL * Math.random() * 0.1);
-        //console.log(magenta(`Auditing node list`));
-
-        let deadTime = Date.now() - DEAD_THRESHOLD;
-
-        let nodeIds = await clearDeadThreadsFromArchives();
-        // We spent the money checking the node list, so we might as well update it
-        await setNodeIds(nodeIds);
-
-        let pendingDeadCount = 0;
-
-        let removedNodeIds: string[] = [];
-        for (let nodeId of nodeIds) {
-            let lastTime = Number((await archives().get(nodeId))?.toString()) || 0;
-            if (lastTime < deadTime) {
-                // Increment the dead count
-                let count = deadCount.get(nodeId) || 0;
-                count++;
-                deadCount.set(nodeId, count);
-                if (count >= DEAD_CHECK_COUNT) {
-                    removedNodeIds.push(nodeId);
-                    console.log(yellow(`Node ${nodeId} was found to be dead, removing from node list. Last heartbeat at ${formatDateTime(lastTime)}, dead threshold at ${formatDateTime(deadTime)}`));
-                    await archives().del(nodeId);
-                    deadCount.delete(nodeId);
-                } else {
-                    console.log(yellow(`Node ${nodeId} was found to be dead, last heartbeat at ${formatDateTime(lastTime)} < dead threshold at ${formatDateTime(deadTime)}, dead count ${count}/${DEAD_CHECK_COUNT}. Total nodes seen ${nodeIds.length}`));
-                    pendingDeadCount++;
-                }
-            } else {
-                deadCount.delete(nodeId);
-                console.info("Read node heartbeat", { nodeId, lastTime });
-            }
-        }
-        if (pendingDeadCount) {
-            console.log(blue(`Pending dead nodes ${pendingDeadCount}/${nodeIds.length}`));
-        }
-
-        if (removedNodeIds.length > 0) {
-            console.log(blue(`Removed ${removedNodeIds.length}/${nodeIds.length} nodes from node list`), { removedNodeIds });
-            await syncArchives();
-            await tellEveryoneNodesChanges(`removedNodeIds ${removedNodeIds.join("|")}`);
-        }
-    });
-}
-
-// NOTE: Not just dead nodes, but also nodes that are unreachable, such as developer nodes.
-let deadNodes = new Map<string, number>();
-async function fastMemorySync() {
-    let aliveNodes = new Set(Array.from(allNodeIds2).filter(x => !isOwnNodeId(x)));
-    let deadThreshold = Date.now() - DEAD_NODE_POLL_COOLDOWN;
-    for (let [nodeId, time] of Array.from(deadNodes)) {
-        if (time < deadThreshold) {
-            deadNodes.delete(nodeId);
-        } else {
-            aliveNodes.delete(nodeId);
-        }
-    }
-    let checkNodes = shuffle(Array.from(aliveNodes), Date.now()).slice(0, API_AUDIT_COUNT);
-    let otherNodesAll = await Promise.all(
-        checkNodes.map(async nodeId => {
-            let nodes = await timeoutToUndefinedSilent(200, NodeDiscoveryController.nodes[nodeId].getAllNodeIds());
-            if (!nodes) {
-                deadNodes.set(nodeId, Date.now());
-            }
-            return nodes || [];
-        })
-    );
-    let otherNodes = Array.from(new Set(otherNodesAll.flat()));
-    // This would log WAY too much, because we poll a lot, because we want to minimize downtime
-    //console.log(magenta(`Fast memory sync at ${formatVeryNiceDateTime(Date.now())}, nodes found ${otherNodes.length}`), getDebuggerUrl());
-
-    // If they are missing nodes that's fine. We constantly have extra nodes, and have to function correctly
-    //  with extra nodes. However, if we are missing nodes, we'd prefer to have them quickly, so we should
-    //  sync now.
-    let missingNodes = otherNodes.filter(nodeId => !allNodeIds2.has(nodeId));
-    if (missingNodes.length > 0) {
-        console.log(yellow(`Node list is missing nodes, resyncing node`), { missingNodes, otherNodes });
-        await syncArchives();
-    }
-}
-
-async function runMemoryAuditLoop() {
-    await getAllNodeIds();
-    runInfinitePoll(DISK_AUDIT_RATE, syncArchives);
-    runInfinitePoll(API_AUDIT_RATE, fastMemorySync);
-}
-
-async function writeHeartbeat() {
-    if (shutdown) return;
-    let now = Date.now();
-    let nodeId = getMountNodeId();
-    console.log(green(`Writing heartbeat ${formatDateTime(now)} for self (${nodeId})`));
-    if (!nodeId) return;
-    await archives().set(nodeId, Buffer.from(now + ""));
-}
-
-async function runMainSyncLoops() {
-    await syncArchives();
-
-    discoveryReady.resolve();
-
-    // We can't heartbeat until we mount
-    await SocketFunction.mountPromise;
-
-    // We have to write before we call NodeDiscoveryController.addNode
-    await writeHeartbeat();
-    await syncArchives();
-    let selfNodeId = SocketFunction.mountedNodeId;
-    if (!allNodeIds2.has(selfNodeId)) {
-        throw new Error(`Failed sanity check, our node id didn't appear in archives after we just wrote it? Missing ${SocketFunction.mountedNodeId}`);
-    }
-    // NOTE: Our first broadcast is special
-    await Promise.allSettled(Array.from(allNodeIds2).map(async nodeId => {
-        if (isOwnNodeId(nodeId)) return;
-        // Ignore errors, but wait a bit, so hopefully 99.99% of the time we can be certain
-        //  all other nodes know our node id at this point.
-        await timeoutToUndefinedSilent(timeInSecond * 5, errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId())));
-    }));
-
-    nodeBroadcasted.resolve();
-
-    console.log(magenta(`Node discovery is loaded`));
-
-    await runInfinitePollCallAtStart(HEARTBEAT_INTERVAL, async function nodeDiscoverHeartbeat() {
-        // If we waited too long, other nodes might think we are dead. In which case, we SHOULD terminate.
-        if (!isNoNetwork()) {
-            // FIRST, verify we didn't delay too long (to make sure we kill any nodes that were disconnected
-            //  from the internet for too long)
-            let lastTime = Number((await archives().get(selfNodeId))?.toString());
-            let suicideThreshold = Date.now() - SUICIDE_HEARTBEAT_THRESHOLD;
-            if (!lastTime || lastTime < suicideThreshold) {
-                if (!lastTime) {
-                    console.error(red(`Self node was removed due to not heartbeating. Terminating self process, as it likely has very stale data.`));
-                } else {
-                    console.error(red(`Self node was has very old heartbeat. Terminating self process, as it likely has very stale data.`));
-                }
-                process.exit();
-            }
-        } else {
-            // We DO have to re-broadcast, otherwise no one will know we exist
-            for (let nodeId of allNodeIds2) {
-                if (isOwnNodeId(nodeId)) continue;
-                // Ignore errors, as nodes die often, and we only want to log once - when we actually remove the node id.
-                ignoreErrors(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId()));
-            }
-        }
-
-        await writeHeartbeat();
-    });
-}
-
-let discoveryReady = new PromiseObj<void>();
-let nodeBroadcasted = new PromiseObj<void>();
-beforeGetNodeAllId = async () => {
-    await discoveryReady.promise;
-};
-export async function onNodeDiscoveryReady() {
-    await getAllNodeIds();
-}
-
-export async function onNodeBroadcasted() {
-    await nodeBroadcasted.promise;
-}
-if (isServer()) {
-    setImmediate(async () => {
-
-        logErrors(runHeartbeatAuditLoop());
-        logErrors(runMemoryAuditLoop());
-        // NOTE: We used to wait until we mounted, but... we should be able to find nodes
-        //  before we mount, right? (And what if we never mount?)
-        runMainSyncLoops().catch(e => {
-            discoveryReady.reject(e);
-            logErrors(Promise.reject(e));
-        });
-    });
-} else {
-
-    if (isNode()) {
-        discoveryReady.resolve();
-        nodeBroadcasted.resolve();
-        // Just get the archives, syncing again if we haven't synced in a while
-        let lastGetTime = 0;
-        beforeGetNodeAllId = async () => {
-            let lastGetThreshold = lastGetTime + CLIENTSIDE_POLL_RATE;
-            if (Date.now() > lastGetThreshold) {
-                lastGetTime = Date.now();
-                await syncArchives();
-            }
-        };
-    } else {
-        setImmediate(() => {
-            let edgeNode = getBootedEdgeNode();
-            if (!edgeNode) {
-                throw new Error(`No edge node set during edgeBootstrap? This should be impossible.`);
-            }
-            let nodes = [edgeNode.host];
-            allNodeIds2 = new Set(nodes);
-            discoveryReady.resolve();
-            nodeBroadcasted.resolve();
-
-            // NOTE: We run into TLS issues (as in, our servers use self signed certs), if we try to talk to just
-            //  any node, so... we better just talk to the edge node
-            //  - We COULD probably just use some special domain (maybe JUST the machine domain?), with limited wildcard
-            //      certs (I think we can only wildcard a single depth anyways), and A records for the machines too...
-            //      but... having all traffic route through an edge node is probably better anyways...
-            nodeOverrides = nodes;
-        });
-    }
-}
-
-
-export async function forceRemoveNode(nodeId: string) {
-    await archives().del(nodeId);
-    void tellEveryoneNodesChanges(`forceRemoveNode ${nodeId}`);
-}
-
-
-/** Called on shutdown, to completely remove this node from discovery. */
-export async function nodeDiscoveryShutdown() {
-    console.log(red(`Shutting down node discovery`));
-    shutdown = true;
-    if (isServer()) {
-        await archives().del(getOwnNodeId());
-    }
-    void tellEveryoneNodesChanges("nodeDiscoveryShutdown");
-}
-const tellEveryoneNodesChanges = throttleFunction(1000, function tellEveryoneNodesChanges(reason: string) {
-    if (isClient()) return;
-    console.log(red(`Telling everyone nodes changed`));
-    for (let nodeId of allNodeIds2) {
-        if (isOwnNodeId(nodeId)) continue;
-        ignoreErrors(NodeDiscoveryController.nodes[nodeId].resyncNodes(reason));
-    }
-});
-
-
-class NodeDiscoveryControllerBase {
-    public async isAlive() {
-        return true;
-    }
-    public async addNode(nodeId: string) {
-        console.log(magenta(`Received addNode`), { nodeId });
-        addNodeId(nodeId);
-        return true;
-    }
-    public async resyncNodes(reason: string) {
-        let caller = SocketFunction.getCaller();
-        console.log(magenta(`Received resyncNodes from ${caller.nodeId}, reason = ${reason}`));
-        await syncArchives();
-    }
-    public async getAllNodesHash(): Promise<string> {
-        return getAllNodesHash();
-    }
-
-    public async getAllNodeIds(): Promise<string[]> {
-        return Array.from(allNodeIds2);
-    }
-    public async getNodeId() {
-        return SocketFunction.mountedNodeId;
-    }
-    public async isNoNetwork() {
-        return isNoNetwork();
-    }
-}
-const NodeDiscoveryController = SocketFunction.register(
-    "NodeDiscoveryController-7991037e-fd9e-4085-b1db-52035487e72c",
-    new NodeDiscoveryControllerBase(),
-    () => ({
-        isAlive: {},
-        addNode: { hooks: [requiresNetworkTrustHook] },
-        resyncNodes: { hooks: [requiresNetworkTrustHook] },
-        getAllNodesHash: { hooks: [requiresNetworkTrustHook] },
-        // Skip client hooks, so we don't block on authentication (IdentityController), as some of these functions
-        //  are needed for authentication to finish!
-        getAllNodeIds: { noClientHooks: true, noDefaultHooks: true },
-        getNodeId: { noClientHooks: true, noDefaultHooks: true },
-        isNoNetwork: { noClientHooks: true, noDefaultHooks: true },
-    }),
-    () => ({
-
-    })
+import { SocketFunction } from "socket-function/SocketFunction";
+import { getArchives } from "../-a-archives/archives";
+import { getDomain, isDevDebugbreak, isNoNetwork, isPublic } from "../config";
+import { measureBlock } from "socket-function/src/profiling/measure";
+import { isNode, keyByArray, sha256Hash, throttleFunction, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { errorToUndefinedSilent, ignoreErrors, logErrors, timeoutToError, timeoutToUndefinedSilent } from "../errors";
+import { ensureWeAreTrusted, requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { getCallFactory, getCreateCallFactory, getNodeId, getNodeIdFromLocation, getNodeIdLocation } from "socket-function/src/nodeCache";
+import { cache, lazy } from "socket-function/src/caching";
+import { shuffle } from "../misc/random";
+import { blue, green, magenta, red, yellow } from "socket-function/src/formatting/logColors";
+import { PromiseObj } from "../promise";
+import { formatDateTime } from "socket-function/src/formatting/format";
+import { isClient, isServer } from "../config2";
+import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
+import { decodeNodeId, decodeNodeIdAssert } from "../-a-auth/certs";
+
+import { isDefined } from "../misc";
+import { getBootedEdgeNode } from "../-0-hooks/hooks";
+import { EdgeNodeConfig } from "../4-deploy/edgeNodes";
+import * as certs from "../-a-auth/certs";
+import { logDisk } from "../diagnostics/logs/diskLogger";
+import { MaybePromise } from "socket-function/src/types";
+import { getPathStr2 } from "../path";
+
+let HEARTBEAT_INTERVAL = timeInMinute * 15;
+// Interval which we check other heartbeats
+let CHECK_INTERVAL = HEARTBEAT_INTERVAL;
+// If the heartbeat is older than thing, it fails the dead check
+let DEAD_THRESHOLD = HEARTBEAT_INTERVAL * 2;
+// If we find another node is dead this number of checks, we remove it from storage, and tell
+//  all other nodes to remove from their memory.
+let DEAD_CHECK_COUNT = 4;
+// If we find we are unable to write our heartbeat, we have to kill our own process. Otherwise it may
+//  be too out of sync, and might commit unverified data to the disk.
+let SUICIDE_HEARTBEAT_THRESHOLD = HEARTBEAT_INTERVAL * 4;
+
+let CLIENTSIDE_POLL_RATE = timeInMinute * 30;
+
+// We can't poll backblaze too often. One a minute starts to cost around 10 cents per month.
+//  So once a second would cost 6 USD per month, per service... which starts to get expensive.
+let DISK_AUDIT_RATE = timeInMinute * 15;
+// We CAN poll our API frequently. The overhead for calls should be far less than 1ms, and
+//  the bandwidth should also be effectively nothing. Maybe 2.5GB per month if we send a
+//  request per second, and each request is 1000 bytes (which as we use websockets, it
+//  probably is less than that). Which is around 2.5 cents on digital ocean IF we go over
+//  our 1TB/month allowance.
+let API_AUDIT_RATE = timeInSecond * 30;
+let API_AUDIT_COUNT = 12;
+
+
+let DEAD_NODE_POLL_COOLDOWN = timeInMinute * 5;
+
+let shutdown = false;
+
+const archives = lazy(() => getArchives("nodes/"));
+
+let logging = true;
+export function enableNodeDiscoveryLogging() {
+    logging = true;
+}
+export function isNodeDiscoveryLogging() {
+    return logging;
+}
+
+
+function getAlternateNodeIds(nodeId: string): MaybePromise<string[] | undefined> {
+    let machineId = certs.getMachineId(nodeId);
+    if (machineId === getOwnMachineId()) {
+        let decoded = decodeNodeId(nodeId);
+        if (decoded) {
+            return [
+                "127-0-0-1." + decoded.domain + ":" + decoded.port
+            ];
+        }
+    }
+    return undefined;
+}
+SocketFunction.GET_ALTERNATE_NODE_IDS = getAlternateNodeIds;
+
+export const getOurNodeId = getOwnNodeId;
+export const getOurNodeIdAssert = getOwnNodeIdAssert;
+
+export const SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE = "SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE";
+export function getOwnNodeId(): string {
+    let nodeId = SocketFunction.mountedNodeId;
+    if (!nodeId) {
+        return SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE;
+    }
+    return nodeId;
+}
+export function getMountNodeId(): string | undefined | "" {
+    return SocketFunction.mountedNodeId;
+}
+export function getOwnNodeIdAssert(): string {
+    let nodeId = SocketFunction.mountedNodeId;
+    if (!nodeId) {
+        throw new Error(`Node must be mounted before nodeId is accessed`);
+    }
+    return nodeId;
+}
+
+export const getOwnThreadId = lazy(() => {
+    return certs.getOwnThreadId();
+});
+export const getOwnMachineId = lazy(() => {
+    return certs.getOwnMachineId();
+});
+
+export function isOwnNodeId(nodeId: string): boolean {
+    if (
+        nodeId === getOwnNodeId()
+        || nodeId === SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE
+    ) {
+        return true;
+    }
+    // If it's 127.0.0.1, and on the same port as us, it is us.
+    let obj = decodeNodeId(nodeId);
+    if (obj && obj.domain === "127-0-0-1." + getDomain() && obj.port === decodeNodeId(getOwnNodeId())?.port) {
+        return true;
+    }
+
+    return false;
+}
+
+export function isNodeIdOnOwnMachineId(nodeId: string): boolean {
+    return certs.getMachineId(nodeId) === getOwnMachineId() || decodeNodeId(nodeId)?.machineId === "127-0-0-1";
+}
+
+export function areNodeIdsEqual(lhs: string, rhs: string): boolean {
+    if (lhs === rhs) return true;
+    return isNodeIdOnOwnMachineId(lhs) && isNodeIdOnOwnMachineId(rhs) && getNodeIdLocation(rhs)?.port === getNodeIdLocation(lhs)?.port;
+}
+
+let nodeOverrides: string[] | undefined;
+let beforeGetNodeAllId = async () => { };
+export async function getAllNodeIds() {
+    await beforeGetNodeAllId();
+
+    // NOTE: We always wait for the time sync before returning any node ids. Because if we try to talk
+    //  to remote nodes with a massively out of sync time, they will likely reject our messages!
+    let promise = waitForFirstTimeSync();
+    if (promise) {
+        await measureBlock(async () => {
+            await promise;
+        }, "waitForFirstTimeSync");
+    }
+
+    if (nodeOverrides) {
+        return nodeOverrides;
+    }
+    return Array.from(allNodeIds2);
+}
+
+export async function syncNodesNow() {
+    await syncArchives();
+}
+
+/** NOTE: You should try to use watchDeltaNodeIds instead of this. */
+export function watchNodeIds(callback: (nodeIds: string[]) => void) {
+    function call() {
+        callback(Array.from(allNodeIds2));
+    }
+    nodeIdWatchers.push(call);
+    call();
+}
+export function watchDeltaNodeIds(callback: (config: { newNodeIds: string[]; removedNodeIds: string[] }) => void) {
+    let prevNodeIds: string[] = [];
+    watchNodeIds(nodeIds => {
+        let newNodeIds = nodeIds.filter(nodeId => !prevNodeIds.includes(nodeId));
+        let removedNodeIds = prevNodeIds.filter(nodeId => !nodeIds.includes(nodeId));
+        if (newNodeIds.length > 0 || removedNodeIds.length > 0) {
+            callback({ newNodeIds, removedNodeIds });
+        }
+        prevNodeIds = nodeIds;
+    });
+}
+
+let allNodeIds2 = new Set<string>();
+function getAllNodesHash() {
+    // Always include ourselves in the hash
+    let sortedNodes = Array.from(allNodeIds2);
+    sortedNodes.sort();
+    return sha256Hash(JSON.stringify(sortedNodes));
+}
+function addNodeId(nodeId: string) {
+    addNodeIdBase(nodeId);
+}
+function addNodeIdBase(nodeId: string) {
+    if (allNodeIds2.has(nodeId)) return;
+    if (logging) {
+        console.log(blue(`Discovered node ${nodeId}`));
+    }
+    allNodeIds2.add(nodeId);
+    onNodesChanged();
+}
+async function setNodeIds(nodeIds: string[]) {
+    nodeIds = nodeIds.filter(x => x !== SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE);
+
+    console.info("setNodeIds", { nodeIds });
+    let newNodeIds = nodeIds.filter(nodeId => !allNodeIds2.has(nodeId));
+    let newIds = new Set(nodeIds);
+    let removedNodeIds = Array.from(allNodeIds2).filter(nodeId => !newIds.has(nodeId));
+    if (newNodeIds.length === 0 && removedNodeIds.length === 0) return;
+    if (logging) {
+        for (let nodeId of newNodeIds) {
+            console.log(blue(`Discovered node ${nodeId}`));
+        }
+        for (let nodeId of removedNodeIds) {
+            console.log(red(`Removed node from setNodeIds ${nodeId}`));
+        }
+    }
+    allNodeIds2 = new Set(nodeIds);
+    onNodesChanged();
+}
+let nodeIdWatchers: (() => void)[] = [];
+function onNodesChanged() {
+    for (let watcher of nodeIdWatchers) {
+        try {
+            watcher();
+        } catch (e: any) {
+            console.log(red(`Ignoring error from callback ${e.stack}`));
+        }
+    }
+}
+
+let rootDiscoveryNodeId = "";
+export function configRootDiscoveryLocation(config: {
+    domain: string;
+    port: number;
+}) {
+    rootDiscoveryNodeId = getNodeId(config.domain, config.port);
+}
+
+export function updateRootDiscoveryLocation(edgeNode: EdgeNodeConfig) {
+    let prevRootId = getBrowserUrlNode();
+    if (!isClient()) {
+        throw new Error(`updateRootDiscoveryLocation should only be called in the browser. If not a client, you should be able to discovery new nodes automatically`);
+    }
+    let nodeId = edgeNode.host;
+    let nodeCache = Object.entries(require.cache);
+    for (let [path, module] of nodeCache) {
+        if (!module) continue;
+        if (!path.includes(prevRootId)) continue;
+        let newPath = path.replace(prevRootId, nodeId);
+        // Remove the old module
+        delete require.cache[path];
+        // Add the new module
+        require.cache[newPath] = module;
+        module.filename = module.filename.replace(prevRootId, nodeId);
+        module.id = module.id.replace(prevRootId, nodeId);
+        if (module.original) {
+            module.original.filename = module.original.filename.replace(prevRootId, nodeId);
+            module.original.originalId = module.original.originalId.replace(prevRootId, nodeId);
+            for (let [key, value] of Object.entries(module.original.requests)) {
+                if (key.includes(prevRootId)) {
+                    let newKey = key.replace(prevRootId, nodeId);
+                    delete module.original.requests[key];
+                    module.original.requests[newKey] = value;
+                    key = newKey;
+                }
+                if (value.includes(prevRootId)) {
+                    value = value.replace(prevRootId, nodeId);
+                    module.original.requests[key] = value;
+                }
+            }
+        }
+    }
+    globalThis.BOOTED_EDGE_NODE = edgeNode;
+    rootDiscoveryNodeId = nodeId;
+    addNodeId(nodeId);
+}
+(globalThis as any).updateRootDiscoveryLocation = updateRootDiscoveryLocation;
+
+/** NOTE: Can also be called serverside, if configRootDiscoveryLocation is called (otherwise can always be called clientside). */
+export function getBrowserUrlNode() {
+    if (!isClient()) throw new Error(`getBrowserUrlNode can only be called when isClient()`);
+    let rootLocation = rootDiscoveryNodeId;
+    if (!rootLocation && !isNode()) {
+        rootLocation = getNodeIdFromLocation();
+    }
+    if (!rootLocation) {
+        throw new Error(`configRootDiscoveryLocation wasn't called, and ---client appears to have been used (isNode() and isClient()). Call configRootDiscoveryLocation with the domain and port of the server you wish to use (ex, querysub.com:1111)`);
+    }
+    return rootLocation;
+}
+
+export async function triggerNodeChange() {
+    await syncNodesNow();
+    await Promise.allSettled(Array.from(allNodeIds2).map(async nodeId => {
+        if (isOwnNodeId(nodeId)) return;
+        await timeoutToUndefinedSilent(timeInSecond * 5, errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId())));
+    }));
+}
+
+
+async function clearDeadThreadsFromArchives() {
+    let nodes = await archives().find("");
+
+    function getPortHash(nodeId: string) {
+        let obj = decodeNodeId(nodeId);
+        if (!obj) return undefined;
+        return getPathStr2(obj.machineId, obj.port + "");
+    }
+    let byPortHash = keyByArray(nodes, getPortHash);
+    for (let [portHash, nodeIds] of Array.from(byPortHash)) {
+        if (!portHash) continue;
+        let aliveNodeId = "";
+        await Promise.all(nodeIds.map(async nodeId => {
+            let alive = await errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].isAlive());
+            if (alive) {
+                aliveNodeId = nodeId;
+            }
+        }));
+        if (aliveNodeId) {
+            let deadThreads = nodeIds.filter(nodeId => nodeId !== aliveNodeId);
+            await Promise.all(deadThreads.map(async deadNodeId => {
+                console.log(`Removing dead thread. We contacted a node on the same port and same machine (${aliveNodeId}), which means the port has been reused by another thread, which proves that the old thread has died, as otherwise the new thread would not be able to use it.`);
+                await archives().del(deadNodeId);
+            }));
+        }
+    }
+
+    return nodes;
+
+}
+
+async function syncArchives() {
+    if (isServer()) {
+        // Make sure we are present
+        await writeHeartbeat();
+        let nodeIds = await archives().find("");
+        console.info(`Synced node ids from archives`, { nodeIds });
+        await setNodeIds(nodeIds);
+    } else {
+        if (isNoNetwork() || !isNode()) {
+            // NOTE: If no network, our trust source might be different, so we can't talk to regular nodes,
+            //  and instead have to only talk to HTTP nodes
+            await setNodeIds([getBrowserUrlNode()]);
+        } else {
+            // If on the network, NetworkTrust2 should sync the trusted machines from backblaze, so we should be
+            //  able to talk to any nodes.
+            //  - If they user is using --client they only want to talk to querysub nodes. There might be multiple,
+            //      which cloudflare will proxy, HOWEVER, it is more efficient to directly access the node list, which
+            //      will be better for load balancing and updating on failure than the cloudflare proxying... probably.
+            await setNodeIds(await NodeDiscoveryController.nodes[getBrowserUrlNode()].getAllNodeIds());
+        }
+    }
+}
+
+async function runHeartbeatAuditLoop() {
+    await getAllNodeIds();
+    let deadCount = new Map<string, number>();
+    // 90% of the normal interval, so we don't run at the same time as the other audit
+    await runInfinitePollCallAtStart(CHECK_INTERVAL * 0.9, async () => {
+        if (shutdown) return;
+        // Wait a bit longer, to try to prevent all nodes from synchronizing their audit times.
+        console.log(magenta(`Auditing node list`));
+        await delay(CHECK_INTERVAL * Math.random() * 0.1);
+        //console.log(magenta(`Auditing node list`));
+
+        let deadTime = Date.now() - DEAD_THRESHOLD;
+
+        let nodeIds = await clearDeadThreadsFromArchives();
+        // We spent the money checking the node list, so we might as well update it
+        await setNodeIds(nodeIds);
+
+        let pendingDeadCount = 0;
+
+        let removedNodeIds: string[] = [];
+        for (let nodeId of nodeIds) {
+            let lastTime = Number((await archives().get(nodeId))?.toString()) || 0;
+            if (lastTime < deadTime) {
+                // Increment the dead count
+                let count = deadCount.get(nodeId) || 0;
+                count++;
+                deadCount.set(nodeId, count);
+                if (count >= DEAD_CHECK_COUNT) {
+                    removedNodeIds.push(nodeId);
+                    console.log(yellow(`Node ${nodeId} was found to be dead, removing from node list. Last heartbeat at ${formatDateTime(lastTime)}, dead threshold at ${formatDateTime(deadTime)}`));
+                    await archives().del(nodeId);
+                    deadCount.delete(nodeId);
+                } else {
+                    console.log(yellow(`Node ${nodeId} was found to be dead, last heartbeat at ${formatDateTime(lastTime)} < dead threshold at ${formatDateTime(deadTime)}, dead count ${count}/${DEAD_CHECK_COUNT}. Total nodes seen ${nodeIds.length}`));
+                    pendingDeadCount++;
+                }
+            } else {
+                deadCount.delete(nodeId);
+                console.info("Read node heartbeat", { nodeId, lastTime });
+            }
+        }
+        if (pendingDeadCount) {
+            console.log(blue(`Pending dead nodes ${pendingDeadCount}/${nodeIds.length}`));
+        }
+
+        if (removedNodeIds.length > 0) {
+            console.log(blue(`Removed ${removedNodeIds.length}/${nodeIds.length} nodes from node list`), { removedNodeIds });
+            await syncArchives();
+            await tellEveryoneNodesChanges(`removedNodeIds ${removedNodeIds.join("|")}`);
+        }
+    });
+}
+
+// NOTE: Not just dead nodes, but also nodes that are unreachable, such as developer nodes.
+let deadNodes = new Map<string, number>();
+async function fastMemorySync() {
+    let aliveNodes = new Set(Array.from(allNodeIds2).filter(x => !isOwnNodeId(x)));
+    let deadThreshold = Date.now() - DEAD_NODE_POLL_COOLDOWN;
+    for (let [nodeId, time] of Array.from(deadNodes)) {
+        if (time < deadThreshold) {
+            deadNodes.delete(nodeId);
+        } else {
+            aliveNodes.delete(nodeId);
+        }
+    }
+    let checkNodes = shuffle(Array.from(aliveNodes), Date.now()).slice(0, API_AUDIT_COUNT);
+    let otherNodesAll = await Promise.all(
+        checkNodes.map(async nodeId => {
+            let nodes = await timeoutToUndefinedSilent(200, NodeDiscoveryController.nodes[nodeId].getAllNodeIds());
+            if (!nodes) {
+                deadNodes.set(nodeId, Date.now());
+            }
+            return nodes || [];
+        })
+    );
+    let otherNodes = Array.from(new Set(otherNodesAll.flat()));
+    // This would log WAY too much, because we poll a lot, because we want to minimize downtime
+    //console.log(magenta(`Fast memory sync at ${formatVeryNiceDateTime(Date.now())}, nodes found ${otherNodes.length}`), getDebuggerUrl());
+
+    // If they are missing nodes that's fine. We constantly have extra nodes, and have to function correctly
+    //  with extra nodes. However, if we are missing nodes, we'd prefer to have them quickly, so we should
+    //  sync now.
+    let missingNodes = otherNodes.filter(nodeId => !allNodeIds2.has(nodeId));
+    if (missingNodes.length > 0) {
+        console.log(yellow(`Node list is missing nodes, resyncing node`), { missingNodes, otherNodes });
+        await syncArchives();
+    }
+}
+
+async function runMemoryAuditLoop() {
+    await getAllNodeIds();
+    runInfinitePoll(DISK_AUDIT_RATE, syncArchives);
+    runInfinitePoll(API_AUDIT_RATE, fastMemorySync);
+}
+
+async function writeHeartbeat() {
+    if (shutdown) return;
+    let now = Date.now();
+    let nodeId = getMountNodeId();
+    console.log(green(`Writing heartbeat ${formatDateTime(now)} for self (${nodeId})`));
+    if (!nodeId) return;
+    await archives().set(nodeId, Buffer.from(now + ""));
+}
+
+async function runMainSyncLoops() {
+    await syncArchives();
+
+    discoveryReady.resolve();
+
+    // We can't heartbeat until we mount
+    await SocketFunction.mountPromise;
+
+    // We have to write before we call NodeDiscoveryController.addNode
+    await writeHeartbeat();
+    await syncArchives();
+    let selfNodeId = SocketFunction.mountedNodeId;
+    if (!allNodeIds2.has(selfNodeId)) {
+        throw new Error(`Failed sanity check, our node id didn't appear in archives after we just wrote it? Missing ${SocketFunction.mountedNodeId}`);
+    }
+    // NOTE: Our first broadcast is special
+    await Promise.allSettled(Array.from(allNodeIds2).map(async nodeId => {
+        if (isOwnNodeId(nodeId)) return;
+        // Ignore errors, but wait a bit, so hopefully 99.99% of the time we can be certain
+        //  all other nodes know our node id at this point.
+        await timeoutToUndefinedSilent(timeInSecond * 5, errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId())));
+    }));
+
+    nodeBroadcasted.resolve();
+
+    console.log(magenta(`Node discovery is loaded`));
+
+    await runInfinitePollCallAtStart(HEARTBEAT_INTERVAL, async function nodeDiscoverHeartbeat() {
+        // If we waited too long, other nodes might think we are dead. In which case, we SHOULD terminate.
+        if (!isNoNetwork()) {
+            // FIRST, verify we didn't delay too long (to make sure we kill any nodes that were disconnected
+            //  from the internet for too long)
+            let lastTime = Number((await archives().get(selfNodeId))?.toString());
+            let suicideThreshold = Date.now() - SUICIDE_HEARTBEAT_THRESHOLD;
+            if (!lastTime || lastTime < suicideThreshold) {
+                if (!lastTime) {
+                    console.error(red(`Self node was removed due to not heartbeating. Terminating self process, as it likely has very stale data.`));
+                } else {
+                    console.error(red(`Self node was has very old heartbeat. Terminating self process, as it likely has very stale data.`));
+                }
+                process.exit();
+            }
+        } else {
+            // We DO have to re-broadcast, otherwise no one will know we exist
+            for (let nodeId of allNodeIds2) {
+                if (isOwnNodeId(nodeId)) continue;
+                // Ignore errors, as nodes die often, and we only want to log once - when we actually remove the node id.
+                ignoreErrors(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId()));
+            }
+        }
+
+        await writeHeartbeat();
+    });
+}
+
+let discoveryReady = new PromiseObj<void>();
+let nodeBroadcasted = new PromiseObj<void>();
+beforeGetNodeAllId = async () => {
+    await discoveryReady.promise;
+};
+export async function onNodeDiscoveryReady() {
+    await getAllNodeIds();
+}
+
+export async function onNodeBroadcasted() {
+    await nodeBroadcasted.promise;
+}
+if (isServer()) {
+    setImmediate(async () => {
+
+        logErrors(runHeartbeatAuditLoop());
+        logErrors(runMemoryAuditLoop());
+        // NOTE: We used to wait until we mounted, but... we should be able to find nodes
+        //  before we mount, right? (And what if we never mount?)
+        runMainSyncLoops().catch(e => {
+            discoveryReady.reject(e);
+            logErrors(Promise.reject(e));
+        });
+    });
+} else {
+
+    if (isNode()) {
+        discoveryReady.resolve();
+        nodeBroadcasted.resolve();
+        // Just get the archives, syncing again if we haven't synced in a while
+        let lastGetTime = 0;
+        beforeGetNodeAllId = async () => {
+            let lastGetThreshold = lastGetTime + CLIENTSIDE_POLL_RATE;
+            if (Date.now() > lastGetThreshold) {
+                lastGetTime = Date.now();
+                await syncArchives();
+            }
+        };
+    } else {
+        setImmediate(() => {
+            let edgeNode = getBootedEdgeNode();
+            if (!edgeNode) {
+                throw new Error(`No edge node set during edgeBootstrap? This should be impossible.`);
+            }
+            let nodes = [edgeNode.host];
+            allNodeIds2 = new Set(nodes);
+            discoveryReady.resolve();
+            nodeBroadcasted.resolve();
+
+            // NOTE: We run into TLS issues (as in, our servers use self signed certs), if we try to talk to just
+            //  any node, so... we better just talk to the edge node
+            //  - We COULD probably just use some special domain (maybe JUST the machine domain?), with limited wildcard
+            //      certs (I think we can only wildcard a single depth anyways), and A records for the machines too...
+            //      but... having all traffic route through an edge node is probably better anyways...
+            nodeOverrides = nodes;
+        });
+    }
+}
+
+
+export async function forceRemoveNode(nodeId: string) {
+    await archives().del(nodeId);
+    void tellEveryoneNodesChanges(`forceRemoveNode ${nodeId}`);
+}
+
+
+/** Called on shutdown, to completely remove this node from discovery. */
+export async function nodeDiscoveryShutdown() {
+    console.log(red(`Shutting down node discovery`));
+    shutdown = true;
+    if (isServer()) {
+        await archives().del(getOwnNodeId());
+    }
+    void tellEveryoneNodesChanges("nodeDiscoveryShutdown");
+}
+const tellEveryoneNodesChanges = throttleFunction(1000, function tellEveryoneNodesChanges(reason: string) {
+    if (isClient()) return;
+    console.log(red(`Telling everyone nodes changed`));
+    for (let nodeId of allNodeIds2) {
+        if (isOwnNodeId(nodeId)) continue;
+        ignoreErrors(NodeDiscoveryController.nodes[nodeId].resyncNodes(reason));
+    }
+});
+
+
+class NodeDiscoveryControllerBase {
+    public async isAlive() {
+        return true;
+    }
+    public async addNode(nodeId: string) {
+        console.log(magenta(`Received addNode`), { nodeId });
+        addNodeId(nodeId);
+        return true;
+    }
+    public async resyncNodes(reason: string) {
+        let caller = SocketFunction.getCaller();
+        console.log(magenta(`Received resyncNodes from ${caller.nodeId}, reason = ${reason}`));
+        await syncArchives();
+    }
+    public async getAllNodesHash(): Promise<string> {
+        return getAllNodesHash();
+    }
+
+    public async getAllNodeIds(): Promise<string[]> {
+        return Array.from(allNodeIds2);
+    }
+    public async getNodeId() {
+        return SocketFunction.mountedNodeId;
+    }
+    public async isNoNetwork() {
+        return isNoNetwork();
+    }
+}
+const NodeDiscoveryController = SocketFunction.register(
+    "NodeDiscoveryController-7991037e-fd9e-4085-b1db-52035487e72c",
+    new NodeDiscoveryControllerBase(),
+    () => ({
+        isAlive: {},
+        addNode: { hooks: [requiresNetworkTrustHook] },
+        resyncNodes: { hooks: [requiresNetworkTrustHook] },
+        getAllNodesHash: { hooks: [requiresNetworkTrustHook] },
+        // Skip client hooks, so we don't block on authentication (IdentityController), as some of these functions
+        //  are needed for authentication to finish!
+        getAllNodeIds: { noClientHooks: true, noDefaultHooks: true },
+        getNodeId: { noClientHooks: true, noDefaultHooks: true },
+        isNoNetwork: { noClientHooks: true, noDefaultHooks: true },
+    }),
+    () => ({
+
+    })
 );