querysub 0.514.0 → 0.516.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.514.0",
+  "version": "0.516.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -423,7 +423,7 @@ async function fastMemorySync() {
     let checkNodes = shuffle(Array.from(aliveNodes), Date.now()).slice(0, API_AUDIT_COUNT);
     let perPeerNodes = await Promise.all(
         checkNodes.map(async peerNodeId => {
-            let nodes = await timeoutToUndefinedSilent(200, NodeDiscoveryController.nodes[peerNodeId].getAllNodeIds());
+            let nodes = await timeoutToUndefinedSilent(2000, NodeDiscoveryController.nodes[peerNodeId].getAllNodeIds());
             if (!nodes) {
                 deadNodes.set(peerNodeId, Date.now());
             }

package/src/3-path-functions/PathFunctionRunner.ts

@@ -350,6 +350,12 @@ export class PathFunctionRunner {
     public static DEBUG_WATCHES_THRESHOLD = 5;
     public static MAX_WATCH_LOOPS = 1000;
 
+    public static hostControllers = lazy(() => {
+        SocketFunction.expose(FunctionPreloadController);
+        SocketFunction.expose(FunctionCaptureController);
+        SocketFunction.expose(FunctionRunnerInfoController);
+    });
+
     constructor(private config: {
         domainName: string;
         shardRange: { startFraction: number, endFraction: number };
@@ -358,9 +364,7 @@ export class PathFunctionRunner {
         // The networks we listen to calls on. Unset is equivalent to ["default"].
         networks?: string[];
     }) {
-        SocketFunction.expose(FunctionPreloadController);
-        SocketFunction.expose(FunctionCaptureController);
-        SocketFunction.expose(FunctionRunnerInfoController);
+        PathFunctionRunner.hostControllers();
         debugFunctionRunnerShards.push({
             domainName: config.domainName,
             shardRange: config.shardRange,

package/src/3-path-functions/PathFunctionRunnerMain.ts

@@ -35,6 +35,7 @@ async function main() {
     PathFunctionRunner.DEBUG_CALLS = true;
     // debugCoreMode();
 
+    PathFunctionRunner.hostControllers();
     await Querysub.hostService("PathFunctionRunnerMain");
 
     // Use a fairly high stick time (the default is 10s), because having wait to sync data is very slow,

package/src/4-deploy/edgeBootstrap.ts

@@ -354,7 +354,161 @@ async function edgeNodeFunction(config: {
             return true;
         });
 
-        // If they are using an IP domain, that's the only node we accept (we still need to pick the node
+        // Probes ALL the given nodes (even private / non-live ones), writing latencies to globalThis.EDGE_NODE_STATS (so the edge node dropdown can show them), and returns a latency-weighted random pick among `pickableHosts` (undefined if none respond).
+        async function probeAndPick(probeNodes: EdgeNodeConfig[], pickableHosts: Set<string>): Promise<EdgeNodeConfig | undefined> {
+            // Having this much lower latency than another node means we pick it 100% of the time (the weight scales linearly, from 1 at the best latency down to 0 at the best latency + this)
+            const LATENCY_WEIGHT_WINDOW = 1000;
+            // How many probes are in flight at once. The first batch fires simultaneously, so anything that responds more than LATENCY_WEIGHT_WINDOW after the first response has a latency at least that much higher than it, and so would never have been picked anyway.
+            const PROBE_BATCH_SIZE = 5;
+
+            let stats = probeNodes.map(node => ({
+                host: node.host,
+                nodeId: node.nodeId,
+                public: node.public,
+                live: node.gitHash === edgeIndex.liveHash,
+                latency: undefined as number | undefined,
+                alive: undefined as boolean | undefined,
+                finished: false,
+            }));
+            let statsObj = { nodes: stats, pickedHost: "", autoPickedHost: "" };
+            globalThis.EDGE_NODE_STATS = statsObj;
+
+            let responded: { node: EdgeNodeConfig; latency: number }[] = [];
+            let queue = probeNodes.slice();
+            let outstanding = 0;
+            let timerStarted = false;
+            let decideResolve = () => { };
+            let decidePromise = new Promise<void>(resolve => { decideResolve = resolve; });
+
+            function launchProbes() {
+                while (outstanding < PROBE_BATCH_SIZE && queue.length > 0) {
+                    let node = queue.shift();
+                    if (!node) break;
+                    void launchProbe(node);
+                }
+            }
+            async function launchProbe(node: EdgeNodeConfig) {
+                outstanding++;
+                let start = Date.now();
+                let alive = false;
+                try {
+                    alive = await isNodeAlive(node);
+                } catch {
+                    alive = false;
+                }
+                outstanding--;
+                let latency = Date.now() - start;
+                let stat = stats.find(x => x.host === node.host);
+                if (stat) {
+                    stat.finished = true;
+                    stat.alive = alive;
+                    stat.latency = latency;
+                }
+                if (alive && pickableHosts.has(node.host)) {
+                    responded.push({ node, latency });
+                    if (!timerStarted) {
+                        timerStarted = true;
+                        // Once the first node responds we decide shortly after (anything slower than the window would never be picked anyway). The probing keeps going in the background though, purely for the stats.
+                        setTimeout(decideResolve, LATENCY_WEIGHT_WINDOW);
+                    }
+                }
+                if (outstanding === 0 && queue.length === 0) {
+                    decideResolve();
+                }
+                launchProbes();
+            }
+            launchProbes();
+            if (outstanding === 0) return undefined;
+            await decidePromise;
+
+            if (responded.length === 0) return undefined;
+            let bestLatency = Math.min(...responded.map(x => x.latency));
+            let weighted = responded.map(x => ({ node: x.node, weight: Math.max(0, 1 - (x.latency - bestLatency) / LATENCY_WEIGHT_WINDOW) }));
+            let totalWeight = weighted.reduce((sum, x) => sum + x.weight, 0);
+            let remaining = Math.random() * totalWeight;
+            let picked = responded[0].node;
+            for (let obj of weighted) {
+                remaining -= obj.weight;
+                if (remaining <= 0) {
+                    picked = obj.node;
+                    break;
+                }
+            }
+            statsObj.autoPickedHost = picked.host;
+            return picked;
+        }
+
+        // What the automatic pick chooses from: public nodes on the live hash (falling back to the newest hash), best candidates first. Never a private node by default (they can still be picked explicitly, via the url override / the edge node dropdown).
+        let autoCandidates = edgeNodes.filter(x => x.public);
+        {
+            let liveNodes = autoCandidates.filter(x => x.gitHash === edgeIndex.liveHash);
+            if (liveNodes.length === 0 && !liveHashForced && autoCandidates.length > 0) {
+                let latestHash = autoCandidates[0].gitHash;
+                console.warn(`Could not find any live nodes (${edgeIndex.liveHash}), falling back to latest hash: ${latestHash}`);
+                liveNodes = autoCandidates.filter(x => x.gitHash === latestHash);
+            }
+            autoCandidates = liveNodes;
+        }
+        autoCandidates = shuffle(autoCandidates);
+        // Deprioritize (but don't exclude, in case there is nothing else) nodes that are scheduled to shut down, and nodes that just booted (they might still be warming up, and if they crash on startup we would follow them down). This only orders the probing (the best candidates go in the first batch), the actual pick is weighted by latency.
+        const EDGE_MIN_UP_TIME = 60_000;
+        const edgePickPenalty = (node: EdgeNodeConfig) => {
+            let penalty = 0;
+            if (node.scheduledShutdownTime) penalty += 2;
+            if (Date.now() - node.bootTime < EDGE_MIN_UP_TIME) penalty += 1;
+            return penalty;
+        };
+        autoCandidates.sort((a, b) => edgePickPenalty(a) - edgePickPenalty(b));
+
+        // Probe the pickable candidates first (the first batch sets the decision window), then everything else purely for the dropdown's stats
+        let pickableHosts = new Set(autoCandidates.map(x => x.host));
+        let probeOrder = [...autoCandidates, ...edgeNodes.filter(x => !pickableHosts.has(x.host))];
+        // Started once, even when the pick is overridden (so the dropdown still gets latencies AND what we would have picked automatically)
+        let probePromise: Promise<EdgeNodeConfig | undefined> | undefined;
+        function ensureProbeStarted() {
+            if (!probePromise) {
+                probePromise = probeAndPick(probeOrder, pickableHosts);
+            }
+            return probePromise;
+        }
+
+        // The url override wins over everything (including the IP domain default). If the host isn't in the list, it is ignored.
+        {
+            let overrideHost = new URL(document.location.href).searchParams.get("edgenode");
+            if (overrideHost) {
+                let overrideNode = edgeNodes.find(x => x.host === overrideHost);
+                if (overrideNode) {
+                    // We wait longer than the regular decision window for a forcefully selected node, as we really want to obey the override. If it still doesn't respond, we fall back to the automatic pick (which the dropdown shows a warning about).
+                    const FORCED_PROBE_TIMEOUT = 5000;
+                    console.log(`MATCH url edgenode override: ${getNodeDebugString(overrideNode)}`);
+                    void ensureProbeStarted();
+                    let statsObj = globalThis.EDGE_NODE_STATS;
+                    if (statsObj) {
+                        statsObj.requestedHost = overrideNode.host;
+                    }
+                    let alive = false;
+                    try {
+                        alive = await Promise.race([
+                            isNodeAlive(overrideNode),
+                            new Promise<boolean>(resolve => setTimeout(() => resolve(false), FORCED_PROBE_TIMEOUT)),
+                        ]);
+                    } catch {
+                        alive = false;
+                    }
+                    if (alive) {
+                        if (statsObj) {
+                            statsObj.pickedHost = overrideNode.host;
+                        }
+                        return overrideNode;
+                    }
+                    console.warn(`The url edgenode override ${overrideNode.host} did not respond within ${FORCED_PROBE_TIMEOUT}ms, falling back to the automatic pick`);
+                } else {
+                    console.warn(`The url edgenode override ${JSON.stringify(overrideHost)} is not in the edge node list, ignoring it`);
+                }
+            }
+        }
+
+        // If they are using an IP domain, that's the default node (we still need to pick the node
         //  though, as we need to know the entryPaths to import)
         {
             let curHost = document.location.host;
@@ -365,6 +519,10 @@ async function edgeNodeFunction(config: {
             let exactNode = edgeNodes.find(x => x.host === curHost);
             if (exactNode) {
                 console.log(`MATCH exact host: ${getNodeDebugString(exactNode)}`);
+                void ensureProbeStarted();
+                if (globalThis.EDGE_NODE_STATS) {
+                    globalThis.EDGE_NODE_STATS.pickedHost = exactNode.host;
+                }
                 return exactNode;
             }
 
@@ -374,69 +532,21 @@ async function edgeNodeFunction(config: {
             }
         }
 
-        // I guess... only allow private nodes, if they specify the exact host (which would match above).
-        edgeNodes = edgeNodes.filter(x => x.public);
-        if (edgeNodes.length === 0) {
+        if (edgeNodes.filter(x => x.public).length === 0) {
             throw new Error(`No public nodes found`);
         }
-
-        let liveNodes = edgeNodes.filter(x => x.gitHash === edgeIndex.liveHash);
-        if (liveNodes.length === 0) {
-            if (liveHashForced) {
-                throw new Error(`Could not find any live nodes (${edgeIndex.liveHash}), and the live hash is forced.`);
-            }
-            let latestHash = edgeNodes[0].gitHash;
-            console.warn(`Could not find any live nodes (${edgeIndex.liveHash}), falling back to latest hash: ${latestHash}`);
-            liveNodes = edgeNodes.filter(x => x.gitHash === latestHash);
+        if (autoCandidates.length === 0) {
+            throw new Error(`Could not find any live nodes (${edgeIndex.liveHash}), and the live hash is forced.`);
         }
-        edgeNodes = liveNodes;
-
-        // TODO: Instead of randomly shuffling, use the node's current load when picking a node.
-        //  All our future traffic (such as syncing) goes through the edge node we use, so it preferrable
-        //  to pick a node with low utilization.
-        edgeNodes = shuffle(edgeNodes);
-
-        // Deprioritize (but don't exclude, in case there is nothing else) nodes that are scheduled to shut down, and nodes that just booted (they might still be warming up, and if they crash on startup we would follow them down).
-        const EDGE_MIN_UP_TIME = 60_000;
-        const edgePickPenalty = (node: EdgeNodeConfig) => {
-            let penalty = 0;
-            if (node.scheduledShutdownTime) penalty += 2;
-            if (Date.now() - node.bootTime < EDGE_MIN_UP_TIME) penalty += 1;
-            return penalty;
-        };
-        edgeNodes.sort((a, b) => edgePickPenalty(a) - edgePickPenalty(b));
-
-        // We check multiple at a time, so a node being unresponsive doesn't cause lag
-        let PARALLEL_FACTOR = 3;
-        for (let i = 0; i < edgeNodes.length; i += PARALLEL_FACTOR) {
-            let node = await new Promise<EdgeNodeConfig | undefined>(resolve => {
-                let finished = 0;
-                for (let j = 0; j < PARALLEL_FACTOR; j++) {
-                    ((async () => {
-                        try {
-                            let node = edgeNodes[i + j];
-                            if (!node) return;
-                            let alive = await isNodeAlive(node);
-                            if (alive) {
-                                resolve(node);
-                            }
-                        }
-                        catch { }
-                        finally {
-                            finished++;
-                            if (finished >= PARALLEL_FACTOR) {
-                                resolve(undefined);
-                            }
-                        }
-                        return undefined;
-                    })()).catch(() => { });
-                }
-            });
-            if (node) {
-                return node;
-            }
+        let picked = await ensureProbeStarted();
+        if (!picked) {
+            throw new Error(`No alive nodes found`);
+        }
+        if (globalThis.EDGE_NODE_STATS) {
+            globalThis.EDGE_NODE_STATS.pickedHost = picked.host;
         }
-        throw new Error(`No alive nodes found`);
+        console.log(`PICKED edge node by latency: ${getNodeDebugString(picked)}`);
+        return picked;
     }
 
     function shuffle<T>(array: T[]): T[] {

package/src/4-deploy/edgeNodes.ts

@@ -79,6 +79,30 @@ export type EdgeNodeConfig = {
     // The function runner index at registration time, so clients have it immediately at startup (they refresh it by polling QuerysubController.getFunctionRunnerIndex).
     functionRunnerIndex?: FunctionRunnerIndex;
 };
+
+export type EdgeNodeStat = {
+    host: string;
+    nodeId: string;
+    public: boolean;
+    live: boolean;
+    // Only set once the probe finishes
+    latency?: number;
+    alive?: boolean;
+    finished: boolean;
+};
+export type EdgeNodeStats = {
+    nodes: EdgeNodeStat[];
+    /** The node we actually booted from */
+    pickedHost: string;
+    /** What the picking algorithm chose (or would have chosen, when overridden) on its own. Empty until its probe decides. */
+    autoPickedHost: string;
+    /** Set when a url override requested a specific host, even if we couldn't connect to it (compare with pickedHost to detect that) */
+    requestedHost?: string;
+};
+declare global {
+    // Written by the edge bootstrapper as it probes the edge nodes, so the edge node dropdown can show what there was to pick from (and their latencies)
+    var EDGE_NODE_STATS: EdgeNodeStats | undefined;
+}
 let registeredEdgeNode: { host: string; entryPaths: string[] } | boolean | undefined;
 export async function registerEdgeNode(config: {
     host: string;

package/src/4-querysub/FunctionRunnerTracking.ts

@@ -7,20 +7,21 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { timeInMinute, timeInSecond, sort } from "socket-function/src/misc";
 import { lazy } from "socket-function/src/caching";
-import { runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { delay, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { isClient } from "../config2";
 import { t } from "../2-proxy/schema2";
 import { createLocalSchema } from "./schemaHelpers";
 import { Querysub } from "./Querysub";
-import { getAllNodeIds, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { getAllNodeIds, getOwnNodeId, watchDeltaNodeIds } from "../-f-node-discovery/NodeDiscovery";
 import { FunctionRunnerInfoController, FunctionStatsSummary } from "../3-path-functions/PathFunctionRunner";
 import { URLParam } from "../library-components/URLParam";
-import { timeoutToUndefinedSilent } from "../errors";
+import { logErrors, timeoutToUndefined, timeoutToUndefinedSilent } from "../errors";
 import { spreadCallsOverTime } from "../misc";
-import { formatTime } from "socket-function/src/formatting/format";
-import type { DebugFunctionShardInfo } from "../3-path-functions/PathFunctionRunner";
+import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
+import type { DebugFunctionShardInfo, FunctionRunnerInfo } from "../3-path-functions/PathFunctionRunner";
 import type { EdgeNodeConfig } from "../4-deploy/edgeNodes";
 import { isPublic } from "../config";
+import { blue, green, yellow } from "socket-function/src/formatting/logColors";
 
 const POLL_INTERVAL = timeInMinute;
 const INDEX_POLL_INTERVAL = timeInMinute * 5;
@@ -144,47 +145,24 @@ export const startFunctionRunnerTracking = lazy(async () => {
     console.log(`Starting function runner tracking (polling every ${formatTime(POLL_INTERVAL)})`);
     await runInfinitePollCallAtStart(POLL_INTERVAL, pollFunctionRunners);
     finishedStartup = true;
+    // Pull newly discovered nodes immediately, instead of waiting up to a full poll interval to find out they are function runners
+    watchDeltaNodeIds(({ newNodeIds }) => {
+        for (let nodeId of newNodeIds) {
+            if (polledNodeIds.has(nodeId)) continue;
+            console.log(blue(`[${formatDateTime(Date.now())}] New node ${nodeId} discovered, immediately checking if it is a function runner`), { nodeId });
+            logErrors(pollFunctionRunner(nodeId));
+        }
+    });
 });
 
 let finishedStartup = false;
+// Every node we have already pulled (runner or not), so the new-node watcher only pulls genuinely new nodes
+let polledNodeIds = new Set<string>();
 async function pollFunctionRunners() {
-    let now = Date.now();
     let nodeIds = await getAllNodeIds();
-    await spreadCallsOverTime(nodeIds, finishedStartup ? POLL_INTERVAL : 0, async nodeId => {
-        let start = Date.now();
-        // Only function runners expose this, so a failed call just means the node isn't a runner
-        let runnerInfo = await timeoutToUndefinedSilent(POLL_INTERVAL, FunctionRunnerInfoController.nodes[nodeId].getRunnerInfo({
-            writeNodeId: getOwnNodeId(),
-        }));
-        if (!runnerInfo) return;
-        let latency = Date.now() - start;
-        if (runnerInfo.shards.length === 0) return;
-
-        let networks = new Set<string>();
-        for (let shard of runnerInfo.shards) {
-            for (let network of shard.networks) {
-                networks.add(network);
-            }
-        }
-
-        let prev = nodeInfos.get(nodeId);
-        let sampleCount = Math.min(prev?.latencySampleCount || 0, LATENCY_SAMPLE_LIMIT);
-        nodeInfos.set(nodeId, {
-            nodeId,
-            entryPoint: runnerInfo.entryPoint,
-            startupTime: runnerInfo.startupTime,
-            lastSeen: now,
-            isPublic: runnerInfo.shards.some(x => x.isPublic),
-            networks: Array.from(networks),
-            shards: runnerInfo.shards,
-            averageLatency: ((prev?.averageLatency || 0) * sampleCount + latency) / (sampleCount + 1),
-            latencySampleCount: sampleCount + 1,
-            scheduledShutdownTime: runnerInfo.scheduledShutdownTime,
-        });
-        nodeTimings.set(nodeId, runnerInfo.timings);
-        watchRunnerDisconnect(nodeId);
-    });
+    await spreadCallsOverTime(nodeIds, finishedStartup ? POLL_INTERVAL : 0, pollFunctionRunner);
 
+    let now = Date.now();
     for (let [nodeId, info] of Array.from(nodeInfos)) {
         if (info.lastSeen < now - NODE_EXPIRY_TIME) {
             nodeInfos.delete(nodeId);
@@ -197,6 +175,59 @@ async function pollFunctionRunners() {
     }
 }
 
+async function pollFunctionRunner(nodeId: string, logWarnings?: "logWarnings" | number) {
+    // Wait a bit so the node has time to set itself up, also to stagger the polling a little bit more.
+    await delay(2000 + Math.random() * 3000);
+    polledNodeIds.add(nodeId);
+    let start = Date.now();
+    // Only function runners expose this, so a failed call just means the node isn't a runner
+    let runnerInfoPromise = FunctionRunnerInfoController.nodes[nodeId].getRunnerInfo({
+        writeNodeId: getOwnNodeId(),
+    }) as Promise<FunctionRunnerInfo | undefined>;
+    if (logWarnings !== "logWarnings") {
+        runnerInfoPromise = timeoutToUndefinedSilent(POLL_INTERVAL, runnerInfoPromise);
+    } else {
+        runnerInfoPromise = timeoutToUndefined(POLL_INTERVAL, runnerInfoPromise);
+    }
+    let runnerInfo = await runnerInfoPromise;
+    if (!runnerInfo || !runnerInfo.shards.length) {
+        if (nodeInfos.has(nodeId) || nodeId.includes("bfbtchfql2fgrelem62qe")) {
+            runnerInfoPromise.catch(e => console.error("bfbtchfql2fgrelem62qe", e.stack));
+            console.log(yellow(`[${formatDateTime(Date.now())}] Function runner ${nodeId} disconnected, removing it from the runner index`), { nodeId });
+            nodeInfos.delete(nodeId);
+        }
+        return;
+    }
+    let latency = Date.now() - start;
+
+    let networks = new Set<string>();
+    for (let shard of runnerInfo.shards) {
+        for (let network of shard.networks) {
+            networks.add(network);
+        }
+    }
+
+    let prev = nodeInfos.get(nodeId);
+    let sampleCount = Math.min(prev?.latencySampleCount || 0, LATENCY_SAMPLE_LIMIT);
+    if (!nodeInfos.has(nodeId)) {
+        console.log(green(`[${formatDateTime(Date.now())}] Function runner ${nodeId} discovered, adding it to the runner index`), { nodeId });
+    }
+    nodeInfos.set(nodeId, {
+        nodeId,
+        entryPoint: runnerInfo.entryPoint,
+        startupTime: runnerInfo.startupTime,
+        lastSeen: Date.now(),
+        isPublic: runnerInfo.shards.some(x => x.isPublic),
+        networks: Array.from(networks),
+        shards: runnerInfo.shards,
+        averageLatency: ((prev?.averageLatency || 0) * sampleCount + latency) / (sampleCount + 1),
+        latencySampleCount: sampleCount + 1,
+        scheduledShutdownTime: runnerInfo.scheduledShutdownTime,
+    });
+    nodeTimings.set(nodeId, runnerInfo.timings);
+    watchRunnerDisconnect(nodeId);
+}
+
 // Remove disconnected runners immediately, otherwise we would keep recommending their networks for up to NODE_EXPIRY_TIME after every runner on them is gone. If the runner comes back, the poll re-adds it (and re-registers this watch).
 let disconnectWatchedNodes = new Set<string>();
 function watchRunnerDisconnect(nodeId: string) {
@@ -206,7 +237,7 @@ function watchRunnerDisconnect(nodeId: string) {
         disconnectWatchedNodes.delete(nodeId);
         if (nodeInfos.delete(nodeId)) {
             nodeTimings.delete(nodeId);
-            console.log(`Function runner ${nodeId} disconnected, removing it from the runner index immediately`);
+            console.log(yellow(`[${formatDateTime(Date.now())}] Function runner ${nodeId} disconnected, removing it from the runner index immediately`), { nodeId });
         }
     }, "iKnowThatServerNodeIdsMayReconnect_andIHandleReconnections");
 }

package/src/4-querysub/Querysub.ts

@@ -53,6 +53,7 @@ import * as typesafecss from "typesafecss";
 import "../library-components/urlResetGroups";
 import { createLocalSchema } from "./schemaHelpers";
 
+setTimeout(() => import("./FunctionRunnerTracking"));
 
 
 typesafecss.setMeasureBlock(measureBlock);

package/src/config.ts

@@ -181,7 +181,7 @@ export function getAuthorityPrefix() {
 
 export function isPublic() {
     if (!isNode()) {
-        return !location.hostname.startsWith("127-0-0-1.");
+        return !(globalThis.BOOTED_EDGE_NODE?.host || location.hostname).startsWith("127-0-0-1.");
     }
     return !!yargObj.public;
 }

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -149,7 +149,7 @@ export class ServiceDetailPage extends qreact.Component {
         let modal: { close: () => void } | undefined;
         modal = showModal({
             content: <div className={css.fixed.pos(0, 0).size("100vw", "100vh").hsla(0, 0, 0, 0.5).display("flex").alignItems("center").justifyContent("center")}>
-                <div className={css.vbox(12).pad2(20).hsl(0, 0, 98).bord2(0, 0, 40) + " keepModalsOpen"}>
+                <div className={css.vbox(12).pad2(20).hsl(0, 0, 12).colorhsl(0, 0, 90).bord2(0, 0, 40) + " keepModalsOpen"}>
                     <div className={css.boldStyle.fontSize(16)}>⚡ Force deploy now?</div>
                     <div>This config becomes live immediately. The old instances are shut down right away, with no overlap.</div>
                     <div className={css.hbox(10)}>

package/src/deployManager/components/ServicesListPage.tsx

@@ -1,16 +1,16 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { qreact } from "../../4-dom/qreact";
-import { DEFAULT_OVERLAP_TIME, MachineServiceController, ServiceConfig } from "../machineSchema";
+import { DEFAULT_OVERLAP_TIME, MachineServiceController, ServiceConfig, getLiveServiceParameters } from "../machineSchema";
 import { css } from "typesafecss";
 import { t } from "../../2-proxy/schema2";
 import { Querysub } from "../../4-querysub/Querysub";
 import { currentViewParam, selectedServiceIdParam } from "../urlParams";
-import { formatNiceDateTime, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
-import { sort, timeInMinute } from "socket-function/src/misc";
+import { formatDateTimeDetailed, formatNiceDateTime, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
+import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { cache } from "socket-function/src/caching";
 import { Anchor } from "../../library-components/ATag";
 import { isPublic } from "../../config";
-import { UpdateButtons, UpdateServiceButtons } from "./deployButtons";
+import { PendingDeployInfo, UpdateButtons, UpdateServiceButtons } from "./deployButtons";
 import { isDefined } from "../../misc";
 import { formatDateJSX } from "../../misc/formatJSX";
 import { Tools } from "./Tools";
@@ -26,6 +26,8 @@ export class ServicesListPage extends qreact.Component {
 
         if (!serviceList) return <div>Loading services...</div>;
 
+        let now = Querysub.nowDelayed(timeInSecond);
+
         let services = serviceList.map(serviceId => [serviceId, controller.getServiceConfig(serviceId)] as const);
         sort(services, x => -(x[1]?.info.lastUpdatedTime || Date.now()) - (x[1]?.parameters.deploy && (Number.MAX_SAFE_INTEGER / 2) || 0));
 
@@ -140,10 +142,32 @@ export class ServicesListPage extends qreact.Component {
                                 <div className={css.vbox(4)}>
                                     <div>Updated {formatDateJSX(config.info.lastUpdatedTime)} AGO</div>
                                     {config.parameters.overlapTime && <div className={css.colorhsl(210, 50, 50).fontSize(14).fontWeight("bold")}>Deploy overlap: {formatTime(config.parameters.overlapTime)}</div> || undefined}
+                                    {(() => {
+                                        let releaseTime = config.parameters.releaseTime || 0;
+                                        if (!releaseTime || releaseTime <= now || !config.oldParameters) return undefined;
+                                        let live = getLiveServiceParameters(config) as Record<string, unknown>;
+                                        let next = config.parameters as Record<string, unknown>;
+                                        let diffLines: string[] = [];
+                                        for (let key of new Set([...Object.keys(live), ...Object.keys(next)])) {
+                                            if (key === "releaseTime") continue;
+                                            let oldValue = JSON.stringify(live[key]);
+                                            let newValue = JSON.stringify(next[key]);
+                                            if (oldValue !== newValue) {
+                                                diffLines.push(`${key}: ${oldValue} -> ${newValue}`);
+                                            }
+                                        }
+                                        return <div
+                                            className={css.colorhsl(35, 80, 45).fontSize(14).fontWeight("bold")}
+                                            title={`Deploys at ${formatDateTimeDetailed(releaseTime)}\n\n${diffLines.join("\n")}`}
+                                        >
+                                            🕐 Deploys in {formatTime(releaseTime - now)}
+                                        </div>;
+                                    })()}
                                 </div>
                             </div>
                         </Anchor>
                         <UpdateServiceButtons service={config} />
+                        <PendingDeployInfo service={config} />
                     </div>
                         ;
                 })}

package/src/deployManager/components/deployButtons.tsx

@@ -5,7 +5,8 @@ import { Querysub } from "../../4-querysub/Querysub";
 import { DEFAULT_OVERLAP_TIME, MachineServiceController, MachineInfo, ServiceConfig } from "../machineSchema";
 import { showFullscreenModal } from "../../5-diagnostics/FullscreenModal";
 import { css } from "../../4-dom/css";
-import { formatTime } from "socket-function/src/formatting/format";
+import { formatDateTimeDetailed, formatTime } from "socket-function/src/formatting/format";
+import { timeInSecond } from "socket-function/src/misc";
 import { formatDateJSX } from "../../misc/formatJSX";
 import { MachineController } from "../machineController";
 import { deepCloneJSON } from "socket-function/src/misc";
@@ -192,6 +193,36 @@ export class UpdateServiceButtons extends qreact.Component<{
     }
 }
 
+/** Shows a service's scheduled (not yet live) deploy, in the same style as the update buttons (but blue, and not clickable): when it goes live, and the new vs currently live commit. */
+export class PendingDeployInfo extends qreact.Component<{
+    service: ServiceConfig;
+}> {
+    render() {
+        let service = this.props.service;
+        let releaseTime = service.parameters.releaseTime || 0;
+        let now = Querysub.nowDelayed(timeInSecond);
+        const oldParameters = service.oldParameters;
+        if (!releaseTime || releaseTime <= now || !oldParameters) return undefined;
+
+        return <div
+            className={css.pad2(12, 8).bord2(210, 60, 40).fontWeight("bold").vbox(2).alignItems("center").alignSelf("stretch").hsl(210, 70, 90)}
+            title={formatDateTimeDetailed(releaseTime)}
+        >
+            <div>
+                {bigEmoji("🕐", -4)} <span>Deploys in {formatTime(releaseTime - now)}</span>
+            </div>
+            <div className={css.hbox(5)}>
+                <b>New</b>
+                <RenderGitRefInfo gitRef={service.parameters.gitRef} />
+            </div>
+            <div className={css.hbox(5)}>
+                <b>Current</b>
+                <RenderGitRefInfo gitRef={oldParameters.gitRef} />
+            </div>
+        </div>;
+    }
+}
+
 export class DeployMachineButtons extends qreact.Component<{
     machines: MachineInfo[];
 }> {

package/src/library-components/EdgeNodeSelector.tsx

@@ -0,0 +1,78 @@
+import { css } from "typesafecss";
+import { qreact } from "../4-dom/qreact";
+import { Querysub } from "../4-querysub/Querysub";
+import { timeInSecond } from "socket-function/src/misc";
+import { formatTime } from "socket-function/src/formatting/format";
+import { isCurrentUserSuperUser } from "../user-implementation/userData";
+import type { EdgeNodeConfig, EdgeNodeStat } from "../4-deploy/edgeNodes";
+
+const EDGE_NODE_URL_PARAM = "edgenode";
+
+/** A dropdown showing all the edge nodes the bootstrapper had to pick from (with their probed latencies), which one we booted from, and allowing forcing a specific edge node. Forcing sets a url parameter and refreshes the page, as the edge node is used by the bootstrapper, so it can only apply on a fresh page load. Super users only. */
+export class EdgeNodeSelector extends qreact.Component<{}> {
+    render() {
+        if (!isCurrentUserSuperUser()) return undefined;
+        // The stats are a plain global the bootstrapper updates as its probes finish, so we re-read them every second
+        Querysub.nowDelayed(timeInSecond);
+        let stats = globalThis.EDGE_NODE_STATS;
+        const booted = (globalThis as any).BOOTED_EDGE_NODE as EdgeNodeConfig | undefined;
+        if (!booted) return undefined;
+        let forced = new URL(document.location.href).searchParams.get(EDGE_NODE_URL_PARAM) || "";
+
+        const setEdgeNode = (host: string) => {
+            let url = new URL(document.location.href);
+            if (host) {
+                url.searchParams.set(EDGE_NODE_URL_PARAM, host);
+            } else {
+                url.searchParams.delete(EDGE_NODE_URL_PARAM);
+            }
+            document.location.href = url.toString();
+        };
+
+        let nodes = stats?.nodes || [];
+        if (!nodes.some(x => x.host === booted.host)) {
+            nodes = [...nodes, { host: booted.host, nodeId: booted.nodeId, public: booted.public, live: true, finished: false }];
+        }
+        let autoStat = nodes.find(x => x.host === stats?.autoPickedHost);
+
+        return <div className={css.hbox(6)}>
+            <span className={css.opacity(0.7)}>Edge</span>
+            <select
+                value={booted.host}
+                className={css.pad2(4, 2).hsl(0, 0, 16).colorhsl(0, 0, 90).bord2(0, 0, 30)}
+                onChange={e => {
+                    setEdgeNode(e.currentTarget.value);
+                }}
+            >
+                {nodes.map(node => <option value={node.host}>{formatStatLabel(node)}</option>)}
+            </select>
+            {stats?.requestedHost && stats.requestedHost !== booted.host && <span className={css.pad2(6, 2).bord2(0, 85, 55).colorhsl(0, 85, 55).boldStyle}>
+                ⚠️ couldn't connect to {stats.requestedHost}, using {booted.host}
+            </span>}
+            {forced && <span
+                className={css.button.pad2(6, 2).hsl(35, 60, 22).colorhsl(0, 0, 90).bord2(35, 60, 40)}
+                title="The edge node is forced. Click to restore the automatically picked edge node (refreshes the page)."
+                onMouseDown={() => setEdgeNode("")}
+            >
+                restore {autoStat && formatStatLabel(autoStat) || stats?.autoPickedHost || "auto"}
+            </span>}
+        </div>;
+    }
+}
+
+function formatStatLabel(stat: EdgeNodeStat): string {
+    let label = stat.host;
+    if (stat.finished) {
+        label += ` (${stat.alive && `${formatTime(stat.latency || 0)} latency` || "dead"}`;
+    } else {
+        label += ` (checking...`;
+    }
+    if (!stat.public) {
+        label += `, private`;
+    }
+    if (!stat.live) {
+        label += `, old version`;
+    }
+    label += `)`;
+    return label;
+}

package/src/library-components/NetworkSelector.tsx

@@ -2,11 +2,12 @@ import { css } from "typesafecss";
 import { qreact } from "../4-dom/qreact";
 import { formatTime } from "socket-function/src/formatting/format";
 import { isCurrentUserSuperUser } from "../user-implementation/userData";
-import { forcedNetworkURL, getAutoSelectedNetwork, getNetworkSelectionInfos, getSelectedNetwork } from "../4-querysub/FunctionRunnerTracking";
+import { forcedNetworkURL, getAutoSelectedNetwork, getNetworkSelectionInfos, getSelectedNetwork, NetworkSelectionInfo } from "../4-querysub/FunctionRunnerTracking";
 
-/** A dropdown showing all the function runner networks, which one our function calls are being put on, and allowing forcing a specific network (and unforcing it, going back to the automatic selection). */
+/** A dropdown showing all the function runner networks, which one our function calls are being put on, and allowing forcing a specific network (and unforcing it, going back to the automatic selection). Super users only. */
 export class NetworkSelector extends qreact.Component<{}> {
     render() {
+        if (!isCurrentUserSuperUser()) return undefined;
         let infos = getNetworkSelectionInfos();
         let forced = forcedNetworkURL.value;
         let selected = getSelectedNetwork({ noThrow: true });
@@ -37,35 +38,22 @@ export class NetworkSelector extends qreact.Component<{}> {
             >
                 {networks.map(network => {
                     let info = infos.find(x => x.network === network);
-                    let label = network;
-                    if (info) {
-                        label += ` (${info.nodeCount} runner${info.nodeCount !== 1 && "s" || ""}, ${formatTime(info.averageLatency)} latency`;
-                        if (info.averageCallTime) {
-                            label += `, ${formatTime(info.averageCallTime)} avg call`;
-                        }
-                        if (!info.isPublic) {
-                            label += `, non-public`;
-                        }
-                        if (info.dying) {
-                            label += `, shutting down`;
-                        }
-                        label += `)`;
-                    }
                     let disabled = !!info && !info.isPublic && !isSuperUser;
-                    return <option value={network} disabled={disabled}>{label}</option>;
+                    return <option value={network} disabled={disabled}>{networkLabel(network, info)}</option>;
                 })}
             </select>
             {forced && <span
                 className={css.button.pad2(6, 2).hsl(35, 60, 22).colorhsl(0, 0, 90).bord2(35, 60, 40)}
-                title="The network is forced. Click to reset back to the automatically picked network."
+                title="The network is forced. Click to restore the automatically picked network."
                 onMouseDown={() => forcedNetworkURL.value = ""}
             >
-                auto: {auto}
+                restore {auto && networkLabel(auto, infos.find(x => x.network === auto)) || "auto"}
             </span>}
             {(() => {
                 let selectedInfo = infos.find(x => x.network === selected);
-                if (!selectedInfo || selectedInfo.nodeCount === 0) {
-                    return <span className={css.pad2(6, 2).bord2(0, 85, 55).colorhsl(0, 85, 55).boldStyle}>⛔ no function runners on this network</span>;
+                // The selection infos only contain networks that have at least one runner, so a missing entry means the selected network doesn't exist at all
+                if (!selectedInfo) {
+                    return <span className={css.pad2(6, 2).bord2(0, 85, 55).colorhsl(0, 85, 55).boldStyle}>⛔ the network {JSON.stringify(selected)} does not exist (no function runner is on it)</span>;
                 }
                 if (selectedInfo.dying) {
                     return <span className={css.colorhsl(35, 85, 55)}>⚠️ this network is shutting down</span>;
@@ -75,3 +63,21 @@ export class NetworkSelector extends qreact.Component<{}> {
         </div>;
     }
 }
+
+function networkLabel(network: string, info: NetworkSelectionInfo | undefined): string {
+    let label = network;
+    if (info) {
+        label += ` (${info.nodeCount} runner${info.nodeCount !== 1 && "s" || ""}, ${formatTime(info.averageLatency)} latency`;
+        if (info.averageCallTime) {
+            label += `, ${formatTime(info.averageCallTime)} avg call`;
+        }
+        if (!info.isPublic) {
+            label += `, non-public`;
+        }
+        if (info.dying) {
+            label += `, shutting down`;
+        }
+        label += `)`;
+    }
+    return label;
+}