querysub 0.508.0 → 0.510.0

package/src/deployManager/machineApplyMainCode.ts

@@ -4,7 +4,7 @@ import { measureWrap } from "socket-function/src/profiling/measure";
 import { getOwnMachineId } from "sliftutils/misc/https/certs";
 import { forceRemoveNode, getOurNodeId, getOurNodeIdAssert } from "../-f-node-discovery/NodeDiscovery";
 import { Querysub } from "../4-querysub/Querysub";
-import { MACHINE_RESYNC_INTERVAL, MachineServiceControllerBase, MachineInfo, ServiceConfig, serviceConfigs, SERVICE_FOLDER, machineInfos, SERVICE_NODE_FILE_NAME, getEffectiveServiceConfigs, recordLaunch } from "./machineSchema";
+import { MACHINE_RESYNC_INTERVAL, MachineServiceControllerBase, MachineInfo, ServiceConfig, ServiceSwitchover, serviceConfigs, pendingServiceConfigs, serviceSwitchovers, SERVICE_FOLDER, machineInfos, SERVICE_NODE_FILE_NAME, getEffectiveServiceConfigs, recordLaunch, DEFAULT_OVERLAP_TIME } from "./machineSchema";
 import { runPromise } from "../functional/runCommand";
 import { getExternalIP } from "socket-function/src/networking";
 import { errorToUndefined, errorToUndefinedSilent } from "../errors";
@@ -21,7 +21,7 @@ import { lazy } from "socket-function/src/caching";
 import { getGitRefLive, getGitURLLive, setGitRef, rebuildGitFolder, nuclearReclone } from "../4-deploy/git";
 import { blue, green, magenta, red } from "socket-function/src/formatting/logColors";
 import { shutdown } from "../diagnostics/periodic";
-import { onServiceConfigChange, triggerRollingUpdate } from "./machineController";
+import { onServiceConfigChange, scheduleNodeShutdown } from "./machineController";
 import { PromiseObj } from "../promise";
 import path from "path";
 import { fsExistsAsync } from "../fs";
@@ -132,6 +132,10 @@ const SCREEN_SUFFIX = "-dply";
 export function getScreenName(config: { serviceKey: string; index: number }): string {
     return `${config.serviceKey}-${config.index}${SCREEN_SUFFIX}`.replace(/[^a-zA-Z0-9\-_]/g, "_");
 }
+// The new instances started during a switchover overlap. They cover the service while the canonical screens are restarted with the new config at switch time, and are killed shortly after.
+export function getNextScreenName(config: { serviceKey: string; index: number }): string {
+    return getScreenName({ serviceKey: config.serviceKey + "-next", index: config.index });
+}
 
 export async function streamScreenOutput(config: {
     key: string;
@@ -286,42 +290,22 @@ done`);
     })();
 }
 
-export async function getRollingServiceInfo(config: {
-    serviceKey: string;
-}): Promise<{ rollingScreenName: string; originalScreenName: string; expiresAt: number }[]> {
-    let rollingInfo: { rollingScreenName: string; originalScreenName: string; expiresAt: number }[] = [];
-
-    for (let [screenName, rolling] of rollingKeepScreenAlive) {
-        if (screenName.startsWith(config.serviceKey + "-")) {
-            rollingInfo.push({
-                rollingScreenName: rolling.rollingScreenName,
-                originalScreenName: rolling.originalScreenName,
-                expiresAt: rolling.pinnedTime + rolling.pinnedDuration,
-            });
-        }
-    }
-
-    return rollingInfo;
-}
-
-export async function killRollingServicesForService(config: {
-    serviceKey: string;
-}): Promise<void> {
-    let toDelete: string[] = [];
-
-    for (let [screenName, rolling] of rollingKeepScreenAlive) {
-        if (screenName.startsWith(config.serviceKey + "-")) {
-            console.log(red(`Explicitly killing rolling screen ${rolling.rollingScreenName} for service ${config.serviceKey}`));
-            await killScreen({
-                screenName: rolling.rollingScreenName,
-            });
-            rollingScreens.delete(rolling.rollingScreenName);
-            toDelete.push(screenName);
-        }
+// screenName => the shutdown time we last successfully told the service's node about
+let notifiedShutdowns = new Map<string, number>();
+async function notifyServiceShutdown(screenName: string, time: number) {
+    if (notifiedShutdowns.get(screenName) === time) return;
+    let nodeIdFile = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
+    if (!await fsExistsAsync(nodeIdFile)) {
+        console.log(`No nodeId file found for ${screenName}, cannot notify it of its scheduled shutdown yet (will retry on the next resync)`);
+        return;
     }
-
-    for (let screenName of toDelete) {
-        rollingKeepScreenAlive.delete(screenName);
+    let nodeId = await fs.promises.readFile(nodeIdFile, "utf8");
+    try {
+        console.log(green(`Notifying ${screenName} (${nodeId}) that it is scheduled to shut down at ${new Date(time).toLocaleString()}`));
+        await scheduleNodeShutdown({ nodeId, time });
+        notifiedShutdowns.set(screenName, time);
+    } catch (e: any) {
+        console.warn(`Error notifying ${screenName} (${nodeId}) of its scheduled shutdown (will retry on the next resync): ${e.stack}`);
     }
 }
 
@@ -451,71 +435,13 @@ async function removeOldNodeId(screenName: string) {
 }
 
 
-// originalName => info
-let rollingKeepScreenAlive = new Map<string, {
-    rollingScreenName: string;
-    originalScreenName: string;
-    pinnedDuration: number;
-    pinnedTime: number;
-}>();
-// rollingScreenName => originalScreenName
-let rollingScreens = new Map<string, string>();
-
 const runScreenCommand = measureWrap(async function runScreenCommand(config: {
     screenName: string;
     command: string;
-    rollingWindow: number | undefined;
 }) {
     let prefix = getTmuxPrefix();
     let screenName = config.screenName;
 
-    if (config.rollingWindow !== undefined) {
-        let screens = await getScreenState();
-        let rollingObj = rollingKeepScreenAlive.get(screenName);
-        if (rollingObj) {
-            rollingObj.pinnedDuration = config.rollingWindow;
-        }
-        let existingScreen = screens.find(x => x.screenName === screenName);
-        if (existingScreen && !rollingObj && await isScreenRunningProcess(existingScreen.pid)) {
-            let nodeIdPath = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
-            let rollingFinalTime = Date.now() + config.rollingWindow;
-            if (await fsExistsAsync(nodeIdPath)) {
-                let nodeId = await fs.promises.readFile(nodeIdPath, "utf8");
-                // REMOVE the nodeId file, so we the node isn't terminated!
-                await fs.promises.unlink(nodeIdPath);
-                try {
-                    console.log(green(`Triggering rolling update notification to ${nodeId} for ${screenName} at ${new Date().toLocaleString()}`));
-                    await triggerRollingUpdate({
-                        nodeId,
-                        time: rollingFinalTime,
-                    });
-                } catch (e: any) {
-                    console.warn(`Error triggering rolling update notification to ${nodeId} for ${screenName}: ${e.stack}`);
-                }
-            } else {
-                console.log(red(`No nodeId file found for ${screenName}, not triggering rolling update notification`));
-            }
-            // Stop the pipe-pane, and delete the pipe file
-            try {
-                await runPromise(`${prefix}tmux pipe-pane -O -t ${screenName}`);
-                await fs.promises.unlink(os.homedir() + "/" + SERVICE_FOLDER + screenName + "/pipe.txt");
-            } catch (e: any) {
-                console.warn(`Error stopping pipe-pane for ${screenName}: ${e.stack}`);
-            }
-
-            let rollingScreenName = screenName + "-rolling" + SCREEN_SUFFIX;
-            console.log(green(`Renaming screen ${screenName} to ${rollingScreenName} for rolling interval ${config.rollingWindow} at ${new Date().toLocaleString()}`));
-            await runPromise(`${prefix}tmux rename-session -t ${screenName} ${rollingScreenName}`);
-            rollingScreens.set(rollingScreenName, screenName);
-            rollingKeepScreenAlive.set(screenName, {
-                rollingScreenName,
-                originalScreenName: screenName,
-                pinnedDuration: config.rollingWindow,
-                pinnedTime: Date.now(),
-            });
-        }
-    }
-
     try {
         // Throw if it already exists
         await runPromise(`${prefix}tmux new -s ${screenName} -d`);
@@ -547,7 +473,6 @@ const runScreenCommand = measureWrap(async function runScreenCommand(config: {
                 await runScreenCommand({
                     screenName,
                     command: config.command,
-                    rollingWindow: config.rollingWindow,
                 });
                 return;
             }
@@ -660,44 +585,90 @@ const ensureGitSynced = measureWrap(async function ensureGitSynced(config: {
 let launchesPerService = new Map<string, number>();
 let lastLaunchedTimePerService = new Map<string, number>();
 
-async function quickIsOutdated() {
-    let machineId = getOwnMachineId(getDomain());
-    // If any rolling are ready to restart, we are outdated
-    for (let [rollingScreenName, rollingInfo] of rollingKeepScreenAlive) {
-        if (rollingInfo.pinnedTime + rollingInfo.pinnedDuration < Date.now()) {
-            console.log(red(`Rolling screen ${rollingScreenName} is finished, doing a full resync`));
-            return true;
+// Promotes any switchovers that have reached their switch time (the pending config becomes live), and returns the switchovers still in flight.
+async function processSwitchovers(): Promise<Map<string, { switchover: ServiceSwitchover; pending: ServiceConfig }>> {
+    let active = new Map<string, { switchover: ServiceSwitchover; pending: ServiceConfig }>();
+    let switchovers = await serviceSwitchovers.values();
+    for (let switchover of switchovers) {
+        let pending = await pendingServiceConfigs.get(switchover.serviceId);
+        if (!pending) {
+            console.warn(`Switchover for ${switchover.serviceId} has no pending config, removing it`);
+            await serviceSwitchovers.delete(switchover.serviceId);
+            continue;
+        }
+        if (Date.now() >= switchover.switchTime) {
+            // Multiple apply nodes may race to promote, which is fine, as they all write the same pending config.
+            console.log(green(`Promoting pending config for ${switchover.serviceId} (switchover scheduled for ${new Date(switchover.switchTime).toLocaleString()})`));
+            await serviceConfigs.set(switchover.serviceId, pending);
+            await pendingServiceConfigs.delete(switchover.serviceId);
+            await serviceSwitchovers.delete(switchover.serviceId);
+            continue;
         }
+        let overlapStart = switchover.switchTime - switchover.overlapTime;
+        console.log(magenta(`Switchover in flight for ${switchover.serviceId} (${pending.parameters.key}): new instances start at ${new Date(overlapStart).toLocaleString()}${Date.now() >= overlapStart && " (overlap running)" || ""}, old instances shut down at ${new Date(switchover.switchTime).toLocaleString()}`));
+        active.set(switchover.serviceId, { switchover, pending });
     }
-    let configs = await getEffectiveServiceConfigs();
-    let relevantConfigs = configs.filter(config => config.machineIds.includes(machineId)).filter(x => x.parameters.deploy);
-    let screens = await getScreenState();
+    return active;
+}
+
+// Starts (or verifies) the "-next" instance of a service during a switchover overlap, running the pending config. It covers the service while the canonical screens restart with the new config at switch time, and is killed by the unused-screen cleanup once the switchover promotes.
+async function ensureNextInstance(config: {
+    pending: ServiceConfig;
+    index: number;
+    screenNamesUsed: Set<string>;
+    screenStateMap: Map<string, { screenName: string; isProcessRunning: boolean; pid: string }>;
+}) {
+    let { pending, index } = config;
+    let nextScreenName = getNextScreenName({ serviceKey: pending.parameters.key, index });
+    config.screenNamesUsed.add(nextScreenName);
+
     let root = os.homedir() + "/" + SERVICE_FOLDER;
-    for (let config of relevantConfigs) {
-        let newParametersString = JSON.stringify(config.parameters);
-        let matchedCount = config.machineIds.filter(id => id === machineId).length;
-        for (let i = 0; i < matchedCount; i++) {
-            let screenName = getScreenName({
-                serviceKey: config.parameters.key,
-                index: i,
-            });
-            let screen = screens.find(x => x.screenName === screenName);
-            if (!screen?.isProcessRunning) return true;
-
-            let folder = root + screenName + "/";
-            await fs.promises.mkdir(folder, { recursive: true });
-            let parameterPath = folder + "/parameters.json";
-            if (!await fsExistsAsync(parameterPath)) return true;
-            let prevParameters = await fs.promises.readFile(parameterPath, "utf8");
-            if (prevParameters !== newParametersString) return true;
+    let folder = root + nextScreenName + "/";
+    let gitFolder = folder + "git/";
+    await fs.promises.mkdir(gitFolder, { recursive: true });
+    if (pending.parameters.gitRef) {
+        let prevGitRef = "";
+        try {
+            prevGitRef = await getGitRefLive(gitFolder);
+        } catch { }
+        await ensureGitSynced({
+            gitFolder,
+            repoUrl: pending.parameters.repoUrl,
+            gitRef: pending.parameters.gitRef,
+        });
+        let afterGitRef = await getGitRefLive(gitFolder);
+        let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
+        if (afterGitRef !== prevGitRef || nodeModulesMissing) {
+            console.log(green(`Yarn installing for ${magenta(nextScreenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
+            await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
         }
     }
-    return false;
+
+    let parameterPath = folder + "/parameters.json";
+    let prevParameters = "";
+    if (await fsExistsAsync(parameterPath)) {
+        prevParameters = await fs.promises.readFile(parameterPath, "utf8");
+    }
+    let newParametersString = JSON.stringify(pending.parameters);
+    let sameParameters = prevParameters === newParametersString;
+    let screenIsRunning = config.screenStateMap.get(nextScreenName)?.isProcessRunning;
+    if (sameParameters && screenIsRunning) {
+        console.log(green(`Verified switchover instance ${magenta(nextScreenName)} is running`));
+        return;
+    }
+
+    console.log(`Starting switchover instance ${magenta(nextScreenName)}, with ${newParametersString}, isRunning = ${screenIsRunning}, sameParameters = ${sameParameters}`);
+    await fs.promises.writeFile(parameterPath, newParametersString);
+    await runScreenCommand({
+        screenName: nextScreenName,
+        command: pending.parameters.command,
+    });
 }
 
 const resyncServicesBase = runInSerial(measureWrap(async function resyncServices() {
     console.log(magenta("Resyncing services"));
     let machineId = getOwnMachineId(getDomain());
+    let activeSwitchovers = await processSwitchovers();
     let configs = await getEffectiveServiceConfigs();
     let relevantConfigs = configs.filter(config => config.machineIds.includes(machineId)).filter(x => x.parameters.deploy);
 
@@ -726,6 +697,23 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
             try {
                 let folder = root + screenName + "/";
                 screenNamesUsed.add(screenName);
+
+                let switchoverObj = activeSwitchovers.get(config.serviceId);
+                if (switchoverObj) {
+                    // Tell the running instance when it will be shut down, so it (and everything talking to it) can wind down gracefully.
+                    await notifyServiceShutdown(screenName, switchoverObj.switchover.switchTime);
+                    let overlapStart = switchoverObj.switchover.switchTime - switchoverObj.switchover.overlapTime;
+                    let pendingMatchedCount = switchoverObj.pending.machineIds.filter(id => id === machineId).length;
+                    if (Date.now() >= overlapStart && switchoverObj.pending.parameters.deploy && i < pendingMatchedCount) {
+                        await ensureNextInstance({
+                            pending: switchoverObj.pending,
+                            index: i,
+                            screenNamesUsed,
+                            screenStateMap,
+                        });
+                    }
+                }
+
                 let gitFolder = folder + "git/";
                 await fs.promises.mkdir(gitFolder, { recursive: true });
                 if (config.parameters.gitRef) {
@@ -790,7 +778,6 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                 await runScreenCommand({
                     screenName,
                     command: config.parameters.command,
-                    rollingWindow: config.parameters.rollingWindow,
                 });
                 await delay(2000);
                 let newScreens = await getScreenState(false);
@@ -824,39 +811,13 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
 
     for (let { screenName } of screenState) {
         if (screenNamesUsed.has(screenName)) continue;
-        let rollingScreenRemainder = rollingScreens.get(screenName);
-        if (rollingScreenRemainder) {
-            let rollingInfo = rollingKeepScreenAlive.get(rollingScreenRemainder);
-            if (!rollingInfo) {
-                rollingScreens.delete(screenName);
-            }
-            if (rollingInfo && rollingInfo.pinnedTime + rollingInfo.pinnedDuration < Date.now()) {
-                console.log(green(`Killing rolling screen ${screenName} because it's expired at ${new Date().toLocaleString()}`));
-                rollingScreens.delete(screenName);
-                rollingKeepScreenAlive.delete(rollingScreenRemainder);
-                rollingInfo = undefined;
-            }
-            if (rollingInfo) {
-                console.log(green(`Skipping killing rolling screen ${screenName} because it's still alive at ${new Date().toLocaleString()}. Keeping it alive until ${new Date(rollingInfo.pinnedTime + rollingInfo.pinnedDuration).toLocaleString()}. Was pinned at ${new Date(rollingInfo.pinnedTime).toLocaleString()}, rolling duration is ${formatTime(rollingInfo.pinnedDuration)}`));
-                continue;
-            }
-        }
-
-        let rollingInfo = rollingKeepScreenAlive.get(screenName);
-        if (rollingInfo) {
-            console.log(red(`Service ${screenName} was disabled, killing associated rolling screen ${rollingInfo.rollingScreenName}`));
-            await killScreen({
-                screenName: rollingInfo.rollingScreenName,
-            });
-            rollingScreens.delete(rollingInfo.rollingScreenName);
-            rollingKeepScreenAlive.delete(screenName);
-        }
-
         await killScreen({
             screenName,
         });
     }
 
+    scheduleSwitchoverResync(activeSwitchovers);
+
     await machineInfos.set(machineId, machineInfo);
     console.log(`${magenta(`Resynced ${relevantConfigs.length} services`)}:\n${JSON.stringify(machineInfo, null, 2)}`);
 
@@ -871,6 +832,31 @@ async function resyncServices() {
     }
 }
 
+// The regular resync poll is too coarse for switchover boundaries (the overlap start and the switch itself), so schedule an extra resync at the next boundary.
+let switchoverResyncTimer: ReturnType<typeof setTimeout> | undefined;
+function scheduleSwitchoverResync(activeSwitchovers: Map<string, { switchover: ServiceSwitchover; pending: ServiceConfig }>) {
+    let now = Date.now();
+    let next: number | undefined;
+    for (let { switchover } of activeSwitchovers.values()) {
+        for (let boundary of [switchover.switchTime - switchover.overlapTime, switchover.switchTime]) {
+            if (boundary <= now) continue;
+            if (next === undefined || boundary < next) {
+                next = boundary;
+            }
+        }
+    }
+    if (switchoverResyncTimer) {
+        clearTimeout(switchoverResyncTimer);
+        switchoverResyncTimer = undefined;
+    }
+    if (next === undefined) return;
+    console.log(`Scheduling a resync for the next switchover boundary at ${new Date(next).toLocaleString()}`);
+    switchoverResyncTimer = setTimeout(() => {
+        switchoverResyncTimer = undefined;
+        void resyncServices();
+    }, next - now + 1000);
+}
+
 
 async function isPIDRunning(pid: string) {
     try {

package/src/deployManager/machineController.ts

@@ -5,14 +5,14 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { timeInMinute } from "socket-function/src/misc";
 import { assertIsManagementUser } from "../diagnostics/managementPages";
 import { isNode } from "typesafecss";
-import { streamScreenOutput, getRollingServiceInfo, killRollingServicesForService } from "./machineApplyMainCode";
+import { streamScreenOutput } from "./machineApplyMainCode";
 import { Querysub } from "../4-querysub/Querysub";
 import { getPathStr2 } from "../path";
 import { getGitURLLive, setGitRef } from "../4-deploy/git";
 import os from "os";
 import { runPromise } from "../functional/runCommand";
 import { getSyncedController } from "../library-components/SyncedController";
-import { logErrors } from "../errors";
+import { setScheduledShutdownTime } from "../-g-core-values/scheduledShutdown";
 
 const POLL_INTERVAL = timeInMinute * 15;
 
@@ -38,33 +38,21 @@ export function onServiceConfigChange(callback: () => Promise<void>): () => void
 }
 
 
-let rollingUpdateWatchers = new Set<(time: number) => void>();
-export function watchOnRollingUpdate(config: {
-    callback: (time: number) => void;
-}) {
-    rollingUpdateWatchers.add(config.callback);
-}
-// TODO: PathValueServer should use this to remove itself from the node pool, and then after 50% of the time until time tell it's clients to find a new server to get values from
-// TODO: FunctionRunner needs to reshard when it sees this (somehow?)
-export async function triggerRollingUpdate(config: {
+/** Tells a service's node that it will be shut down at the given time (due to a scheduled switchover). */
+export async function scheduleNodeShutdown(config: {
     nodeId: string;
     time: number;
 }) {
-    await OnServiceChange.nodes[config.nodeId].onRollingUpdate(config.time);
+    await OnServiceChange.nodes[config.nodeId].scheduleShutdown(config.time);
 }
 class OnServiceChangeBase {
     public async onServiceConfigChange() {
         await triggerServiceConfigChangeCallbacks();
     }
 
-    public async onRollingUpdate(time: number) {
-        for (let callback of rollingUpdateWatchers) {
-            try {
-                callback(time);
-            } catch (e) {
-                logErrors(e);
-            }
-        }
+    public async scheduleShutdown(time: number) {
+        console.log(`Received scheduled shutdown notification (service switchover): shutting down at ${new Date(time).toLocaleString()}`);
+        setScheduledShutdownTime(time);
     }
 }
 export const OnServiceChange = SocketFunction.register(
@@ -72,7 +60,7 @@ export const OnServiceChange = SocketFunction.register(
     () => new OnServiceChangeBase(),
     () => ({
         onServiceConfigChange: {},
-        onRollingUpdate: {},
+        scheduleShutdown: {},
     }),
     () => ({
         hooks: [requiresNetworkTrustHook],
@@ -140,36 +128,6 @@ class MachineControllerBase {
         await runPromise("bash machine-startup.sh", { cwd: os.homedir(), detach: true });
     }
 
-    public async getRollingServiceInfo(config: {
-        serviceKey: string;
-    }): Promise<{ rollingScreenName: string; originalScreenName: string; expiresAt: number }[]> {
-        return await getRollingServiceInfo(config);
-    }
-
-    public async getRollingServiceInfoFromBrowser(config: {
-        machineNodeId: string;
-        serviceKey: string;
-    }): Promise<{ rollingScreenName: string; originalScreenName: string; expiresAt: number }[]> {
-        return await MachineController(config.machineNodeId).getRollingServiceInfo.promise({
-            serviceKey: config.serviceKey,
-        });
-    }
-
-    public async killRollingServices(config: {
-        serviceKey: string;
-    }): Promise<void> {
-        await killRollingServicesForService(config);
-    }
-
-    public async killRollingServicesFromBrowser(config: {
-        machineNodeId: string;
-        serviceKey: string;
-    }): Promise<void> {
-        await MachineController(config.machineNodeId).killRollingServices.promise({
-            serviceKey: config.serviceKey,
-        });
-    }
-
 }
 
 let forwardedCallbacks = new Map<string, string>();
@@ -182,23 +140,16 @@ export const MachineController = getSyncedController(SocketFunction.register(
         watchOtherScreenOutput: {},
         deployMachineFromBrowser: {},
         deployMachine: {},
-        getRollingServiceInfo: {},
-        getRollingServiceInfoFromBrowser: {},
-        killRollingServices: {},
-        killRollingServicesFromBrowser: {},
     }),
     () => ({
         hooks: [assertIsManagementUser],
     }),
 ), {
     writes: {
-        deployMachineFromBrowser: ["MachineInfo", "RollingInfo"],
-        deployMachine: ["MachineInfo", "RollingInfo"],
-        killRollingServicesFromBrowser: ["RollingInfo"],
-    },
-    reads: {
-        getRollingServiceInfoFromBrowser: ["RollingInfo"],
+        deployMachineFromBrowser: ["MachineInfo"],
+        deployMachine: ["MachineInfo"],
     },
+    reads: {},
 });
 
 let callbacks = new Map<string, (data: string) => Promise<void>>();

package/src/deployManager/machineSchema.ts

@@ -86,8 +86,8 @@ export type ServiceConfig = {
         /** Not set by default, so we can setup the configuration before deploying it (or so we can undeploy easily without deleting it) */
         deploy?: boolean;
 
-        /** Keep the instances of the service alive for this long after we start a new one. Only keeps the oldest alive, so multiple updates won't break things (or update the oldest one, so if the oldest one is bad, set this to 0 to make sure it dies immediately). */
-        rollingWindow?: number;
+        /** How long the new and old instances run simultaneously during a switchover. The new instances start at switchTime - overlapTime, and the old instances are shut down at switchTime. This is the default; each switchover can override it. */
+        overlapTime?: number;
     };
     info: {
         title: string;
@@ -95,12 +95,26 @@ export type ServiceConfig = {
         lastUpdatedTime: number;
     };
 };
+
+export const DEFAULT_OVERLAP_TIME = timeInMinute * 10;
+
+export type ServiceSwitchover = {
+    serviceId: string;
+    /** When the old instances are shut down and the pending config becomes live */
+    switchTime: number;
+    /** The new instances start at switchTime - overlapTime */
+    overlapTime: number;
+    scheduledAt: number;
+};
 export type MachineConfig = {
     machineId: string;
     disabled: boolean;
 };
 export const machineInfos = archiveJSONT<MachineInfo>(() => nestArchives("machines/machine-heartbeats/", getArchivesBackblaze(getDomain())));
 export const serviceConfigs = archiveJSONT<ServiceConfig>(() => nestArchives("machines/service-configs/", getArchivesBackblaze(getDomain())));
+// Edits in the UI save here (still on disk), and only become live when a switchover is deployed / scheduled
+export const pendingServiceConfigs = archiveJSONT<ServiceConfig>(() => nestArchives("machines/service-configs-pending/", getArchivesBackblaze(getDomain())));
+export const serviceSwitchovers = archiveJSONT<ServiceSwitchover>(() => nestArchives("machines/service-switchovers/", getArchivesBackblaze(getDomain())));
 export const machineConfigs = archiveJSONT<MachineConfig>(() => nestArchives("machines/machine-configs/", getArchivesBackblaze(getDomain())));
 
 export type LaunchRecord = {
@@ -309,8 +323,63 @@ export class MachineServiceControllerBase {
 
         // Remove the service config
         await serviceConfigs.delete(serviceId);
+        await pendingServiceConfigs.delete(serviceId);
+        await serviceSwitchovers.delete(serviceId);
         await this.notifyMachines([], serviceConfig.machineIds);
     }
+
+    /** Saves edits WITHOUT deploying them. They only take effect once a switchover is scheduled (scheduleSwitchover). */
+    public async setPendingServiceConfigs(configs: ServiceConfig[]) {
+        for (let config of configs) {
+            let serviceId = config.serviceId;
+            config.info.lastUpdatedTime = Date.now();
+            let serviceConfig = await serviceConfigs.get(serviceId);
+            if (!serviceConfig) {
+                throw new Error(`Service ${serviceId} does not exist`);
+            }
+            await pendingServiceConfigs.set(serviceId, config);
+        }
+    }
+    public async getPendingServiceConfig(serviceId: string): Promise<ServiceConfig | undefined> {
+        return await pendingServiceConfigs.get(serviceId);
+    }
+    public async discardPendingServiceConfig(serviceId: string) {
+        await pendingServiceConfigs.delete(serviceId);
+        await serviceSwitchovers.delete(serviceId);
+    }
+
+    /** Deploys the pending config: the new instances start at switchTime - overlapTime, the old instances are told they will shut down at switchTime, and at switchTime the pending config becomes live. */
+    public async scheduleSwitchover(config: {
+        serviceId: string;
+        switchTime: number;
+        overlapTime: number;
+    }) {
+        let pending = await pendingServiceConfigs.get(config.serviceId);
+        if (!pending) {
+            throw new Error(`No pending config for service ${config.serviceId}. Save changes first, then deploy them.`);
+        }
+        let live = await serviceConfigs.get(config.serviceId);
+        if (!live) {
+            throw new Error(`Service ${config.serviceId} does not exist`);
+        }
+        await serviceSwitchovers.set(config.serviceId, {
+            serviceId: config.serviceId,
+            switchTime: config.switchTime,
+            overlapTime: config.overlapTime,
+            scheduledAt: Date.now(),
+        });
+        await this.notifyMachines([...new Set([...live.machineIds, ...pending.machineIds])], []);
+    }
+    public async cancelSwitchover(serviceId: string) {
+        let live = await serviceConfigs.get(serviceId);
+        await serviceSwitchovers.delete(serviceId);
+        if (live) {
+            await this.notifyMachines(live.machineIds, []);
+        }
+    }
+    public async getServiceSwitchover(serviceId: string): Promise<ServiceSwitchover | undefined> {
+        return await serviceSwitchovers.get(serviceId);
+    }
     public async getServiceConfigType(): Promise<string> {
         return extractType(module, "ServiceConfig");
     }
@@ -538,6 +607,12 @@ export const MachineServiceController = getSyncedController(
             setMachineConfig: {},
             addServiceConfig: {},
             setServiceConfigs: {},
+            setPendingServiceConfigs: {},
+            getPendingServiceConfig: {},
+            discardPendingServiceConfig: {},
+            scheduleSwitchover: {},
+            cancelSwitchover: {},
+            getServiceSwitchover: {},
             getServiceConfigType: {},
             deleteServiceConfig: {},
             getGitInfo: {},
@@ -562,7 +637,11 @@ export const MachineServiceController = getSyncedController(
             setMachineConfig: ["MachineConfig", "MachineConfigList"],
             addServiceConfig: ["ServiceConfig", "ServiceConfigList"],
             setServiceConfigs: ["ServiceConfig"],
-            deleteServiceConfig: ["ServiceConfig", "ServiceConfigList"],
+            setPendingServiceConfigs: ["PendingServiceConfig"],
+            discardPendingServiceConfig: ["PendingServiceConfig", "ServiceSwitchover"],
+            scheduleSwitchover: ["ServiceSwitchover"],
+            cancelSwitchover: ["ServiceSwitchover"],
+            deleteServiceConfig: ["ServiceConfig", "ServiceConfigList", "PendingServiceConfig", "ServiceSwitchover"],
             commitPushService: ["gitInfo", "ServiceConfig"],
             commitPushAndPublishQuerysub: ["gitInfo", "ServiceConfig"],
             deployFunctions: ["gitInfo"],
@@ -574,6 +653,8 @@ export const MachineServiceController = getSyncedController(
             getMachineConfig: ["MachineConfig"],
             getServiceList: ["ServiceConfigList"],
             getServiceConfig: ["ServiceConfig"],
+            getPendingServiceConfig: ["PendingServiceConfig"],
+            getServiceSwitchover: ["ServiceSwitchover"],
             getGitInfo: ["gitInfo"],
             getPendingFunctions: ["gitInfo"],
             getLiveFunctions: ["gitInfo"],