querysub 0.508.0 → 0.509.0

package/src/3-path-functions/PathFunctionRunner.ts

@@ -1,7 +1,7 @@
 import { batchFunction, delay, runInfinitePoll } from "socket-function/src/batching";
-import { cache } from "socket-function/src/caching";
+import { cache, lazy } from "socket-function/src/caching";
 import { blue, magenta, yellow } from "socket-function/src/formatting/logColors";
-import { timeInHour, timeInSecond } from "socket-function/src/misc";
+import { timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { registerDynamicResource } from "../diagnostics/trackResources";
 import { errorToUndefined, logErrors } from "../errors";
@@ -16,6 +16,8 @@ import { parseArgs } from "./PathFunctionHelpers";
 import { FunctionMetadata, PERMISSIONS_FUNCTION_ID, addRoutingPrefixForDeploy, getAllDevelopmentModulesIds, getDevelopmentModule, getExportPath, getModuleRelativePath, getSchemaObject } from "./syncSchema";
 import { formatTime } from "socket-function/src/formatting/format";
 import { getControllerNodeIdList, set_debug_getFunctionRunnerShards } from "../-g-core-values/NodeCapabilities";
+import { getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { getScheduledShutdownTime } from "../-g-core-values/scheduledShutdown";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { getDomain, isPublic, DEFAULT_NETWORK } from "../config";
@@ -37,8 +39,8 @@ export const functionSchema = rawSchema<{
             [ModuleId: string]: {
                 Data: unknown;
                 Sources: { [FunctionId: string]: FunctionSpec | undefined; };
-                Calls: { [CallId: string]: CallSpec | undefined; };
-                Results: { [CallId: string]: FunctionResult | undefined; };
+                Calls: { [Network: string]: { [CallId: string]: CallSpec | undefined; }; };
+                Results: { [Network: string]: { [CallId: string]: FunctionResult | undefined; }; };
             };
         };
     };
@@ -53,9 +55,14 @@ setImmediate(() => {
 });
 
 export function commitCall(call: CallSpec) {
+    if (!call.network) {
+        throw new Error(`Call has no network, so it would never run (no FunctionRunner would pick it up). Call: ${debugCallSpec(call)}`);
+    }
+    call.writeNodeId = getOwnNodeId();
     let domainName = call.DomainName;
     let moduleId = call.ModuleId;
     let callId = call.CallId;
+    let network = call.network;
     proxyWatcher.writeOnly({
         canWrite: true,
         eventWrite: true,
@@ -63,7 +70,7 @@ export function commitCall(call: CallSpec) {
         // NOTE: We write the call in the present, it is callSpec.runAtTime that is in the past. No one cares
         //  when the call was added, only when it wants to run at.
         watchFunction: function writeCall() {
-            functionSchema()[domainName].PathFunctionRunner[moduleId].Calls[callId] = atomicObjectWrite(call);
+            functionSchema()[domainName].PathFunctionRunner[moduleId].Calls[network][callId] = atomicObjectWrite(call);
         },
     });
 }
@@ -107,8 +114,11 @@ export interface CallSpec {
     // Not just used for debugging, also used to add special proxy-related warnings.
     fromProxy?: string;
 
-    // The network the call is on ("default" if unset). Only FunctionRunners on this network will run the call.
-    network?: string;
+    // Only FunctionRunners on this network will run the call. If no FunctionRunner is on this network, the call never runs.
+    network: string;
+
+    // The node that committed the call. FunctionRunners record call timings tagged with this, so the node that wrote a call can pull the timings of its own calls.
+    writeNodeId?: string;
 }
 export function debugCallSpec(spec: CallSpec): string {
     return `${spec.DomainName}/${spec.ModuleId}/${spec.FunctionId}`;
@@ -126,6 +136,121 @@ export type FunctionResult = ({
     totalTime: number;
 };
 
+export type FunctionStatsSummary = {
+    count: number;
+    totalTime: number;
+    minTime: number;
+    maxTime: number;
+};
+const TIMINGS_KEPT_TIME = timeInHour;
+const TIMINGS_BUCKET_SIZE = timeInMinute * 5;
+// Kept PER writing node: how long a call takes varies by the node it was called from, so each node only wants the timings of its own calls. Bucketed (bucket start time => stats) so old data can age out without keeping per-call data; only summaries ever leave this file.
+let functionTimings = new Map<string, Map<string, Map<number, FunctionStatsSummary>>>();
+// Writing nodes get new nodeIds when they restart, so without this their old timings would be kept forever.
+const startTimingsSweep = lazy(() => {
+    runInfinitePoll(TIMINGS_KEPT_TIME, function sweepExpiredTimings() {
+        let expiry = Date.now() - TIMINGS_KEPT_TIME;
+        for (let [writeNodeId, networks] of functionTimings) {
+            for (let [network, buckets] of networks) {
+                for (let start of buckets.keys()) {
+                    if (start < expiry) {
+                        buckets.delete(start);
+                    }
+                }
+                if (buckets.size === 0) {
+                    networks.delete(network);
+                }
+            }
+            if (networks.size === 0) {
+                functionTimings.delete(writeNodeId);
+            }
+        }
+    });
+});
+function recordFunctionTiming(config: {
+    writeNodeId: string | undefined;
+    network: string;
+    durationMs: number;
+}) {
+    if (!config.writeNodeId) return;
+    startTimingsSweep();
+    let networks = functionTimings.get(config.writeNodeId);
+    if (!networks) {
+        networks = new Map();
+        functionTimings.set(config.writeNodeId, networks);
+    }
+    let buckets = networks.get(config.network);
+    if (!buckets) {
+        buckets = new Map();
+        networks.set(config.network, buckets);
+    }
+    let now = Date.now();
+    let bucketStart = Math.floor(now / TIMINGS_BUCKET_SIZE) * TIMINGS_BUCKET_SIZE;
+    let bucket = buckets.get(bucketStart);
+    if (!bucket) {
+        bucket = { count: 0, totalTime: 0, minTime: config.durationMs, maxTime: config.durationMs };
+        buckets.set(bucketStart, bucket);
+    }
+    bucket.count++;
+    bucket.totalTime += config.durationMs;
+    bucket.minTime = Math.min(bucket.minTime, config.durationMs);
+    bucket.maxTime = Math.max(bucket.maxTime, config.durationMs);
+}
+
+const runnerStartupTime = Date.now();
+export type FunctionRunnerInfo = {
+    entryPoint: string;
+    startupTime: number;
+    scheduledShutdownTime?: number;
+    shards: DebugFunctionShardInfo[];
+    // network => stats over the last hour of the calling node's own calls
+    timings: { [network: string]: FunctionStatsSummary };
+};
+
+// Everything the function runner tracking poll needs, in a single cheap call: what we run (networks/shards), our lifecycle (startup/shutdown), and how long the caller's own calls took. Only exposed by function runners, so a failed call doubles as "not a runner".
+class FunctionRunnerInfoControllerBase {
+    public async getRunnerInfo(config: {
+        writeNodeId: string;
+    }): Promise<FunctionRunnerInfo> {
+        let expiry = Date.now() - TIMINGS_KEPT_TIME;
+        let timings: FunctionRunnerInfo["timings"] = {};
+        for (let [network, buckets] of functionTimings.get(config.writeNodeId) || []) {
+            let summary: FunctionStatsSummary = { count: 0, totalTime: 0, minTime: 0, maxTime: 0 };
+            for (let [bucketStart, bucket] of buckets) {
+                if (bucketStart < expiry) continue;
+                if (summary.count === 0) {
+                    summary.minTime = bucket.minTime;
+                }
+                summary.count += bucket.count;
+                summary.totalTime += bucket.totalTime;
+                summary.minTime = Math.min(summary.minTime, bucket.minTime);
+                summary.maxTime = Math.max(summary.maxTime, bucket.maxTime);
+            }
+            timings[network] = summary;
+        }
+        return {
+            entryPoint: process.argv[1],
+            startupTime: runnerStartupTime,
+            scheduledShutdownTime: getScheduledShutdownTime(),
+            shards: debugFunctionRunnerShards,
+            timings,
+        };
+    }
+}
+export const FunctionRunnerInfoController = SocketFunction.register(
+    "FunctionRunnerInfoController-3f7a1c2e-8d94-4b6a-9e51-c0d72fa8b143",
+    new FunctionRunnerInfoControllerBase(),
+    () => ({
+        getRunnerInfo: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    }),
+    {
+        noAutoExpose: true,
+    }
+);
+
 let currentCallSpec: { spec: CallSpec; fnc: FunctionSpec; } | undefined;
 export function getCurrentCall() {
     if (!currentCallSpec) {
@@ -195,6 +320,9 @@ export type DebugFunctionShardInfo = {
     domainName: string;
     shardRange: { startFraction: number, endFraction: number };
     secondaryShardRange?: { startFraction: number, endFraction: number };
+    // The networks this runner listens to calls on
+    networks: string[];
+    isPublic: boolean;
 };
 let debugFunctionRunnerShards: DebugFunctionShardInfo[] = [];
 export function debug_getFunctionRunnerShards() {
@@ -232,10 +360,13 @@ export class PathFunctionRunner {
     }) {
         SocketFunction.expose(FunctionPreloadController);
         SocketFunction.expose(FunctionCaptureController);
+        SocketFunction.expose(FunctionRunnerInfoController);
         debugFunctionRunnerShards.push({
             domainName: config.domainName,
             shardRange: config.shardRange,
             secondaryShardRange: config.secondaryShardRange,
+            networks: config.networks && config.networks.length > 0 && config.networks || [DEFAULT_NETWORK],
+            isPublic: isPublic(),
         });
         logErrors(this.startWatching());
     }
@@ -269,9 +400,8 @@ export class PathFunctionRunner {
 
         let watchModuleCalls = cache((moduleId: string) => {
             for (let network of networks) {
-                let networkConfig = network === DEFAULT_NETWORK && fullFraction || { ...fullFraction, network };
-                Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Calls, networkConfig);
-                Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Results, networkConfig);
+                Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Calls[network], fullFraction);
+                Querysub.keys(functionSchema()[domainName].PathFunctionRunner[moduleId].Results[network], fullFraction);
             }
         });
 
@@ -326,7 +456,10 @@ export class PathFunctionRunner {
                         if (getPathIndex(path, 1) !== "PathFunctionRunner") continue;
                         if (getPathIndex(path, 0) !== domainName) continue;
 
-                        const callId = getPathIndex(path, 4);
+                        const network = getPathIndex(path, 4);
+                        if (!network) continue;
+                        if (!networks.includes(network)) continue;
+                        const callId = getPathIndex(path, 5);
                         if (!callId) continue;
                         let moduleId = getPathIndex(path, 2);
                         if (!moduleId) continue;
@@ -334,10 +467,10 @@ export class PathFunctionRunner {
 
                         let moduleData = data()[moduleId];
 
-                        let result = atomicObjectRead(moduleData.Results[callId]);
+                        let result = atomicObjectRead(moduleData.Results[network][callId]);
                         if (result) continue;
 
-                        let callData = atomicObjectRead(moduleData.Calls[callId]);
+                        let callData = atomicObjectRead(moduleData.Calls[network][callId]);
                         if (!callData) continue;
 
                         // Ignore invalid data
@@ -363,7 +496,7 @@ export class PathFunctionRunner {
                         // If we haven't synced the result, we can't know if it is ready or not
                         //  (we'll run anyways once it is synced). Most other parts are immutable,
                         //  but the result is by definition not, so... it is one of the values we check!
-                        if (!isSynced(moduleData.Results[callId])) continue;
+                        if (!isSynced(moduleData.Results[network][callId])) continue;
 
                         if (runningCalls.has(callId)) {
                             console.log(`Skipping run of already running call ${callId} at ${debugTime(callData.runAtTime)}`);
@@ -401,7 +534,7 @@ export class PathFunctionRunner {
                             // NOTE: If this starts but never runs, it means the call's network didn't match ours, which either means it was a dev call and so not relevant for us, or we are dev and it's a production call, so not relevant for us.
                             //  - The network is set by --network when running serverside, or, clientside, by --network set on the command line of the querysub server.
                             console.log(`QUEUING ${getDebugName(callData, functionSpec, true)}`);
-                            let resultsPath = getProxyPath(() => moduleData.Results[callId]);
+                            let resultsPath = getProxyPath(() => moduleData.Results[network][callId]);
                             let history = authorityStorage.getValuePlusHistory(resultsPath);
                             console.log(`    History: ${history.length}`);
                             for (let { valid, time, canGCValue } of history) {
@@ -551,7 +684,7 @@ export class PathFunctionRunner {
             skipPermissions = PermissionsChecker.skipPermissionsChecks.bind(PermissionsChecker);
         }
 
-        let callNetwork = callSpec.network || DEFAULT_NETWORK;
+        let callNetwork = callSpec.network;
         let ourNetworks = this.config.networks && this.config.networks.length > 0 && this.config.networks || [DEFAULT_NETWORK];
         if (!ourNetworks.includes(callNetwork)) {
             return;
@@ -577,7 +710,7 @@ export class PathFunctionRunner {
                         let syncedModule = skipPermissions(() =>
                             functionSchema()[callSpec.DomainName].PathFunctionRunner[callSpec.ModuleId]
                         );
-                        return !!atomicObjectRead(syncedModule.Results[callSpec.CallId]);
+                        return !!atomicObjectRead(syncedModule.Results[callNetwork][callSpec.CallId]);
                     },
                 });
             }
@@ -740,7 +873,7 @@ export class PathFunctionRunner {
                         //  know figure out the dependencies for the call, which makes function call prediction
                         //  possible.
                         skipPermissions(() => {
-                            let result = doProxyOptions({ forceReadLatest: true }, () => atomicObjectRead(syncedModule.Results[callSpec.CallId]));
+                            let result = doProxyOptions({ forceReadLatest: true }, () => atomicObjectRead(syncedModule.Results[callNetwork][callSpec.CallId]));
                             if (result) {
                                 nooped = true;
                                 // It is important to NOOP if it has already been written. Otherwise there could be multiple
@@ -757,7 +890,7 @@ export class PathFunctionRunner {
                             let currentTimeTaken = Date.now() - startTime;
                             stats.timeTaken = currentTimeTaken;
                             let totalTime = Date.now() - stats.firstQueuedTime;
-                            syncedModule.Results[callSpec.CallId] = atomicObjectWrite({
+                            syncedModule.Results[callNetwork][callSpec.CallId] = atomicObjectWrite({
                                 lastInternalLoopCount: runCount,
                                 outerLoopCount: stats.outerLoopCount,
                                 totalInternalLoopCount: stats.totalInternalLoopCount,
@@ -765,7 +898,7 @@ export class PathFunctionRunner {
                                 evalTime,
                                 totalTime,
                             });
-                            proxyWatcher.setEventPath(() => syncedModule.Results[callSpec.CallId]);
+                            proxyWatcher.setEventPath(() => syncedModule.Results[callNetwork][callSpec.CallId]);
                         });
 
                         let watcher = proxyWatcher.getTriggeredWatcher();
@@ -786,7 +919,7 @@ export class PathFunctionRunner {
                         // Capture only runs that actually authored the Result (not nooped/partial runs),
                         //  so a replayed call maps to exactly one recorded output state.
                         if (isCapturing()) {
-                            let resultsPathStr = getProxyPath(() => functionSchema()[callSpec.DomainName].PathFunctionRunner[callSpec.ModuleId].Results[callSpec.CallId]);
+                            let resultsPathStr = getProxyPath(() => functionSchema()[callSpec.DomainName].PathFunctionRunner[callSpec.ModuleId].Results[callNetwork][callSpec.CallId]);
                             let resultWrite = writes.find(w => w.path === resultsPathStr);
                             if (resultWrite) {
                                 recordCapturedCall({
@@ -871,6 +1004,13 @@ export class PathFunctionRunner {
             wallTime, syncTime, evalTime,
             loops: runCount,
         });
+        if (!nooped) {
+            recordFunctionTiming({
+                writeNodeId: callSpec.writeNodeId,
+                network: callSpec.network,
+                durationMs: Date.now() - callSpec.runAtTime.time,
+            });
+        }
         PathFunctionRunner.RUN_FINISH_COUNT++;
     }
 
@@ -908,11 +1048,12 @@ export class PathFunctionRunner {
             eventWrite: true,
             doNotStoreWritesAsPredictions: true,
             watchFunction() {
+                let network = callSpec.network;
                 let syncedModule = functionSchema()[callSpec.DomainName].PathFunctionRunner[callSpec.ModuleId];
-                // Don't clobber the value. I'm pretty sure event writes still read values, so we should still do a synchronization to see if there is an existing result. 
-                let result = doProxyOptions({ forceReadLatest: true }, () => atomicObjectRead(syncedModule.Results[callSpec.CallId]));
+                // Don't clobber the value. I'm pretty sure event writes still read values, so we should still do a synchronization to see if there is an existing result.
+                let result = doProxyOptions({ forceReadLatest: true }, () => atomicObjectRead(syncedModule.Results[network][callSpec.CallId]));
                 if (result) return;
-                syncedModule.Results[callSpec.CallId] = atomicObjectWrite({
+                syncedModule.Results[network][callSpec.CallId] = atomicObjectWrite({
                     error,
                     lastInternalLoopCount: runCount,
                     outerLoopCount,

package/src/3-path-functions/functionReplay.ts

@@ -17,6 +17,8 @@ export function makeReplayCallSpec(record: CaptureRecord): CallSpec {
         callerMachineId: record.callerMachineId,
         callerIP: record.callerIP,
         runAtTime: record.runAtTime,
+        // Replays run locally and are never committed, so the network is irrelevant
+        network: "",
     };
 }
 

package/src/3-path-functions/syncSchema.ts

@@ -579,7 +579,6 @@ export function getCallIdOverride(config: {
     functionId: string;
     callId: string;
     args: unknown[];
-    network?: string;
 }): string {
     try {
         if (querysub) {
@@ -594,7 +593,6 @@ export function getCallIdOverride(config: {
             remappedPrefix: def.prefix,
             originalKey: config.callId,
             routeKey: def.getKey(...config.args),
-            network: config.network,
         });
     } catch (e: any) {
         console.error(`Error getting call id override for ${config.moduleId}.${config.functionId}, falling back to original call id`, { error: e.stack });

package/src/4-deploy/edgeBootstrap.ts

@@ -396,6 +396,16 @@ async function edgeNodeFunction(config: {
         //  to pick a node with low utilization.
         edgeNodes = shuffle(edgeNodes);
 
+        // Deprioritize (but don't exclude, in case there is nothing else) nodes that are scheduled to shut down, and nodes that just booted (they might still be warming up, and if they crash on startup we would follow them down).
+        const EDGE_MIN_UP_TIME = 60_000;
+        const edgePickPenalty = (node: EdgeNodeConfig) => {
+            let penalty = 0;
+            if (node.scheduledShutdownTime) penalty += 2;
+            if (Date.now() - node.bootTime < EDGE_MIN_UP_TIME) penalty += 1;
+            return penalty;
+        };
+        edgeNodes.sort((a, b) => edgePickPenalty(a) - edgePickPenalty(b));
+
         // We check multiple at a time, so a node being unresponsive doesn't cause lag
         let PARALLEL_FACTOR = 3;
         for (let i = 0; i < edgeNodes.length; i += PARALLEL_FACTOR) {

package/src/4-deploy/edgeClientWatcher.tsx

@@ -42,11 +42,9 @@ export function startEdgeNotifier() {
     });
     void (async () => {
         await delay(1);
-        let { watchOnRollingUpdate } = await import("../deployManager/machineController");
-        watchOnRollingUpdate({
-            callback: (time) => {
-                void notifyClients(`forcedupdate-at-${time}`, time);
-            }
+        let { watchScheduledShutdown } = await import("../-g-core-values/scheduledShutdown");
+        watchScheduledShutdown((time) => {
+            void notifyClients(`forcedupdate-at-${time}`, time);
         });
     })();
 }
@@ -191,7 +189,7 @@ const EdgeNotifierClientController = SocketFunction.register(
 
 if (!isNode()) {
     setImmediate(() => {
-        void Querysub.optionalStartupWait().finally(async () => {
+        void Querysub.startupWait().finally(async () => {
             curHash = await EdgeNotifierController.nodes[SocketFunction.browserNodeId()].watchUpdates();
             void runRefreshLoop();
         });

package/src/4-deploy/edgeNodes.ts

@@ -26,6 +26,7 @@ import { startEdgeNotifier } from "./edgeClientWatcher";
 import { getGitRefLive, getGitURLLive } from "./git";
 import { DeployProgress } from "./deployFunctions";
 import { PromiseObj } from "../promise";
+import type { FunctionRunnerIndex } from "../4-querysub/FunctionRunnerTracking";
 
 const UPDATE_POLL_INTERVAL = timeInMinute * 15;
 const DEAD_NODE_COUNT_THRESHOLD = 15;
@@ -73,6 +74,10 @@ export type EdgeNodeConfig = {
     bootTime: number;
     entryPaths: string[];
     public: boolean;
+    // Set once this node has been told it will shut down (scheduled switchover). Clients deprioritize these nodes when picking an edge node.
+    scheduledShutdownTime?: number;
+    // The function runner index at registration time, so clients have it immediately at startup (they refresh it by polling QuerysubController.getFunctionRunnerIndex).
+    functionRunnerIndex?: FunctionRunnerIndex;
 };
 let registeredEdgeNode: { host: string; entryPaths: string[] } | boolean | undefined;
 export async function registerEdgeNode(config: {
@@ -100,19 +105,20 @@ export async function registerEdgeNode(config: {
         }));
         await updateEdgeNodesFile();
     });
-    const { watchOnRollingUpdate } = await import("../deployManager/machineController");
-    watchOnRollingUpdate({
-        callback: async (time) => {
+    const { watchScheduledShutdown } = await import("../-g-core-values/scheduledShutdown");
+    watchScheduledShutdown((time) => {
+        void (async () => {
+            // Immediately publish our shutdown time, so clients can deprioritize us when picking an edge node.
+            await publishOwnScheduledShutdown(time);
             let duration = time - Date.now();
-            // Wait a bit to remove ourself from the edge node list
-            void delay(duration * 0.2).finally(async () => {
-                console.log(magenta(`Removing node from edge node list due to rolling update at ${new Date().toLocaleString()}`));
-                await Promise.all(Array.from(registeredNodePaths).map(async nodePath => {
-                    await edgeNodeStorage.del(nodePath);
-                }));
-                await updateEdgeNodesFile();
-            });
-        },
+            // Wait a bit to fully remove ourself from the edge node list
+            await delay(Math.max(0, duration * 0.2));
+            console.log(magenta(`Removing node from edge node list due to scheduled shutdown at ${new Date(time).toLocaleString()}`));
+            await Promise.all(Array.from(registeredNodePaths).map(async nodePath => {
+                await edgeNodeStorage.del(nodePath);
+            }));
+            await updateEdgeNodesFile();
+        })();
     });
 
     await waitForFirstTimeSync();
@@ -154,6 +160,7 @@ export async function registerEdgeNode(config: {
             bootTime: Date.now(),
             entryPaths: config.entryPaths,
             public: isPublic(),
+            functionRunnerIndex: await getInitialFunctionRunnerIndex(),
         };
 
         await edgeNodeStorage.set(getNextNodePath(), Buffer.from(JSON.stringify(edgeNodeConfig)));
@@ -204,6 +211,7 @@ const loadEntryPointsByHash = runInSerial(async function loadEntryPointsByHash(c
         bootTime: Date.now(),
         entryPaths,
         public: isPublic(),
+        functionRunnerIndex: await getInitialFunctionRunnerIndex(),
     };
 
     await edgeNodeStorage.set(getNextNodePath(), Buffer.from(JSON.stringify(edgeNodeConfig)));
@@ -214,6 +222,25 @@ const loadEntryPointsByHash = runInSerial(async function loadEntryPointsByHash(c
     onEdgeNodesChanged();
 });
 
+async function getInitialFunctionRunnerIndex(): Promise<FunctionRunnerIndex | undefined> {
+    const { startFunctionRunnerTracking, getFunctionRunnerIndex } = await import("../4-querysub/FunctionRunnerTracking");
+    await startFunctionRunnerTracking();
+    return getFunctionRunnerIndex();
+}
+
+async function publishOwnScheduledShutdown(time: number) {
+    for (let nodePath of Array.from(registeredNodePaths)) {
+        let buffer = await edgeNodeStorage.get(nodePath);
+        if (!buffer) continue;
+        let config: EdgeNodeConfig = JSON.parse(buffer.toString());
+        if (config.scheduledShutdownTime === time) continue;
+        config.scheduledShutdownTime = time;
+        await edgeNodeStorage.set(nodePath, Buffer.from(JSON.stringify(config)));
+        getEdgeNodeConfig.clearKey(nodePath);
+    }
+    await updateEdgeNodesFile();
+}
+
 export const getEdgeNodeConfigURL = lazy(async () => {
     let { getURL } = await hostArchives({
         archives: edgeNodeStorage,
@@ -291,6 +318,19 @@ async function updateEdgeNodesFile() {
         diff = true;
         console.log(`Detected live hash changed. Updating ${edgeNodeIndexFile}`, { liveHash, prevHash: prevEdgeNodeIndex.liveHash });
     }
+    if (!diff) {
+        // Node configs are otherwise immutable, but the scheduled shutdown time is written into them after the fact, and clients need it to avoid picking dying edge nodes.
+        let prevShutdownTimes = new Map(prevEdgeNodeIndex.edgeNodes.map(x => [x.nodeId, x.scheduledShutdownTime]));
+        for (let nodeFile of edgeNodeFiles) {
+            let config = await getEdgeNodeConfig(nodeFile);
+            if (!config) continue;
+            if (prevShutdownTimes.get(config.nodeId) !== config.scheduledShutdownTime) {
+                diff = true;
+                console.log(`Detected scheduled shutdown time changed for ${config.nodeId}. Updating ${edgeNodeIndexFile}`);
+                break;
+            }
+        }
+    }
     if (!diff) return;