querysub 0.569.0 → 0.570.0

package/appSecrets.ts

@@ -23,6 +23,8 @@ export const getCloudflareCreds = lazy(async (): Promise<{ key: string; email: s
 
 /** Serves the secret keys sliftutils' getSecret expects, reading them the way querysub already does: backblaze creds off disk (getBackblazePath), cloudflare creds from the keys archives bucket (with its local file fallback). */
 export async function getAppSecret(key: string): Promise<string | undefined> {
+    // HACK: This is used by the storage server. Which we want to shut down immediately. That way the next instance can start up. The sword server already looks for notifications about when it's going to be shutting down and it flushes to disk early in this case, so it should be fairly safe to hard kill it.
+    doImmediateShutdown();
     // Import query sub so our logs go to our actual log server. Otherwise we won't know if we get any errors. 
     Querysub;
     if (key.startsWith("backblaze.json.")) {
@@ -39,3 +41,21 @@ export async function getAppSecret(key: string): Promise<string | undefined> {
     }
     return undefined;
 }
+
+let doImmediateShutdown = lazy(() => {
+    // NOTE: This extra code is required to actual capture ctrl+c
+    if (process.platform === "win32") {
+        var rl = require("readline").createInterface({
+            input: process.stdin,
+            output: process.stdout
+        });
+
+        rl.on("SIGINT", function () {
+            process.emit("SIGINT");
+        });
+    }
+    function doShutdown() {
+        process.exit();
+    }
+    process.on("SIGINT", doShutdown);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.569.0",
+  "version": "0.570.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/deployManager/machineApplyMainCode.ts

@@ -26,7 +26,7 @@ import { PromiseObj } from "../promise";
 import path from "path";
 import { fsExistsAsync } from "../fs";
 import { ALIVE_WINDOW_FOREVER, ParametersTimelineEntry, syncParametersTimelineFiles } from "./parametersTimeline";
-import { getScreenName, getFutureScreenName, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, ensureProcessRecord, runScreenCommand, killScreen, streamProcessOutput, readServiceNodeId, removeServiceNode, takeoverScreen, scheduleTakeover } from "./processManager";
+import { getScreenName, getFutureScreenName, RETIRING_MARKER, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, ensureProcessRecord, runScreenCommand, killScreen, streamProcessOutput, readServiceNodeId, removeServiceNode, takeoverScreen, withScreenLock } from "./processManager";
 import { ProcessRecord, syncProcessRecords, writeProcessRecord, listProcessRecords, findRunningRecord, getProcessLogPath } from "./processLogs";
 
 
@@ -220,116 +220,210 @@ const ensureGitSynced = measureWrap(async function ensureGitSynced(config: {
 let launchesPerService = new Map<string, number>();
 let lastLaunchedTimePerService = new Map<string, number>();
 
-const RELEASE_PREP_LEAD_TIME = timeInSecond * 30;
+const LAUNCH_VERIFY_DELAY = 2000;
 
-// Syncs the canonical folder to the new parameters (git + yarn install + parameters.json) and creates the future screen, which just echos when it will start. Idempotent.
-type FutureConfig = {
+/** One tmux session that should exist right now, and the configuration it should be running. The whole resync is derived from this list: converge each of these, kill everything else. */
+type DesiredScreen = {
+    /** The session that should exist. For an incoming version that is the "-future" name, until it takes the canonical name over. */
+    screenName: string;
+    /** The slot this belongs to - its own name, or the one it is replacing */
     canonicalScreenName: string;
-    next: ServiceParameters;
-    releaseTime: number;
-    /** When the old process's overlap runs out, so the takeover can be scheduled the moment the new one starts */
-    killTime: number;
+    /** The instance folder, which is always the canonical screen's folder (an incoming version runs out of the same one) */
+    folder: string;
     serviceId: string;
+    serviceKey: string;
     index: number;
     machineId: string;
-    screenNamesUsed: Set<string>;
-    screenStateMap: Map<string, { screenName: string; isProcessRunning: boolean; pid: string }>;
+    /** The parameters it runs, with template variables already resolved */
+    parameters: ServiceParameters;
+    aliveWindow: [number, number];
+    /** The version being replaced during an overlap: it must keep running and must not be touched, because the folder already belongs to its replacement. */
+    outgoing?: boolean;
+    /** On an incoming version, when it takes the canonical name over. Absent means take over as soon as it is up. */
+    takeoverAt?: number;
 };
 
-async function ensureFuturePrepared(config: FutureConfig) {
-    let { canonicalScreenName, next, releaseTime } = config;
-    let futureScreenName = getFutureScreenName(canonicalScreenName);
-    config.screenNamesUsed.add(futureScreenName);
-    let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
-    let gitFolder = folder + "git/";
-    await fs.promises.mkdir(gitFolder, { recursive: true });
-    if (next.gitRef) {
-        let prevGitRef = "";
-        try {
-            prevGitRef = await getGitRefLive(gitFolder);
-        } catch { }
-        await ensureGitSynced({
-            gitFolder,
-            repoUrl: next.repoUrl,
-            gitRef: next.gitRef,
-        });
-        let afterGitRef = await getGitRefLive(gitFolder);
-        let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
-        if (afterGitRef !== prevGitRef || nodeModulesMissing) {
-            console.log(green(`Yarn installing for ${magenta(futureScreenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
-            await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
+/** Every screen that should exist on this machine, worked out purely from the configs and the clock. */
+function getDesiredScreens(configs: ServiceConfig[], machineId: string, root: string): DesiredScreen[] {
+    let desired: DesiredScreen[] = [];
+    let now = Date.now();
+    for (let config of configs) {
+        let releaseTime = config.parameters.releaseTime || 0;
+        // The overlap length always comes from the NEWEST parameters, even though it governs how long the OLD instances outlive the release
+        let overlapTime = config.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME;
+        let killTime = releaseTime + overlapTime;
+        // Only in flight while there is an old version to keep alive, and only until its overlap runs out
+        let releaseInFlight = !!releaseTime && !!config.oldParameters && now < killTime;
+
+        const add = (parameters: ServiceParameters, extra: Partial<DesiredScreen>) => {
+            if (!parameters.deploy) return;
+            let targets = getMachineTargets(parameters).filter(target => target.machineId === machineId);
+            for (let index = 0; index < targets.length; index++) {
+                let canonicalScreenName = getScreenName({ serviceKey: parameters.key, index });
+                desired.push({
+                    screenName: canonicalScreenName,
+                    canonicalScreenName,
+                    folder: root + canonicalScreenName + "/",
+                    serviceId: config.serviceId,
+                    serviceKey: parameters.key,
+                    index,
+                    machineId,
+                    // Each instance runs the command with its own entry's variables resolved, so the parameters written to disk and compared for restarts carry the resolved command
+                    parameters: { ...parameters, command: applyCommandTemplate(parameters.command, targets[index].variables) },
+                    aliveWindow: [parameters.releaseTime || 0, ALIVE_WINDOW_FOREVER],
+                    ...extra,
+                });
+            }
+        };
+
+        if (releaseInFlight && config.oldParameters) {
+            // The old version keeps running under the canonical name until its overlap runs out
+            add(config.oldParameters, { outgoing: true, aliveWindow: [config.oldParameters.releaseTime || 0, killTime] });
+            // The new version runs alongside it from the release, and takes the canonical name at killTime
+            if (now >= releaseTime) {
+                add(config.parameters, { takeoverAt: killTime, aliveWindow: [releaseTime, ALIVE_WINDOW_FOREVER] });
+            }
+        } else {
+            add(getLiveServiceParameters(config), {});
         }
     }
-    await fs.promises.writeFile(folder + "parameters.json", JSON.stringify(next));
-    if (!config.screenStateMap.has(futureScreenName)) {
-        let prefix = getTmuxPrefix();
-        await runPromise(`${prefix}tmux new -s ${futureScreenName} -d`);
-        await runPromise(`${prefix}tmux send-keys -t ${futureScreenName} 'echo "Waiting to start the new version at ${new Date(releaseTime).toISOString()} (in ${Math.round((releaseTime - Date.now()) / timeInSecond)} seconds)"' Enter`);
-        console.log(magenta(`Prepared future screen ${futureScreenName}, starting at ${new Date(releaseTime).toLocaleString()}`));
+    // An incoming version lives under the future name until it takes over
+    for (let screen of desired) {
+        if (screen.takeoverAt === undefined) continue;
+        screen.screenName = getFutureScreenName(screen.canonicalScreenName);
     }
+    return desired;
 }
 
-// Starts (or verifies) the future screen actually running the new version, in the canonical screen's folder. Idempotent.
-async function ensureFutureStarted(config: FutureConfig) {
-    let { canonicalScreenName, next } = config;
-    let futureScreenName = getFutureScreenName(canonicalScreenName);
-    let existing = config.screenStateMap.get(futureScreenName);
-    if (existing?.isProcessRunning) {
-        config.screenNamesUsed.add(futureScreenName);
-        // Started before we restarted, so record it rather than only ever recording processes we launched ourselves
-        await ensureProcessRecord({
-            screenName: futureScreenName,
-            folder: os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/",
-            panePid: existing.pid,
-            record: {
-                serviceId: config.serviceId,
-                serviceKey: config.next.key,
-                index: config.index,
-                machineId: config.machineId,
-                parameters: config.next,
-            },
+/** Puts one screen into the state its configuration describes: the folder synced, the process running what it should be, and the records and timeline saying so. Everything about one screen happens under its lock, so a resync and a scheduled takeover can never interleave. */
+async function ensureScreen(config: {
+    desired: DesiredScreen;
+    screenStateMap: Map<string, { screenName: string; isProcessRunning: boolean; pid: string }>;
+    machineInfo: MachineInfo;
+}): Promise<void> {
+    let { desired, screenStateMap, machineInfo } = config;
+    let { screenName, canonicalScreenName, folder, parameters, serviceId } = desired;
+
+    let launchCount = launchesPerService.get(serviceId) || 0;
+    let lastLaunchedTime = lastLaunchedTimePerService.get(serviceId) || 0;
+    machineInfo.services[serviceId] = {
+        lastLaunchedTime,
+        errorFromLastRun: "",
+        totalTimesLaunched: launchCount,
+        nodeId: "",
+    };
+
+    try {
+        await withScreenLock(canonicalScreenName, async () => {
+            let screen = screenStateMap.get(screenName);
+
+            if (desired.outgoing) {
+                // Its replacement owns the folder now, so the only thing to do is keep its record straight and let it know when it goes down
+                if (screen?.isProcessRunning) {
+                    await ensureProcessRecord({ screenName, folder, panePid: screen.pid, record: desired });
+                    void (async () => {
+                        let running = await findRunningRecord(folder, screenName);
+                        let nodeId = running?.nodeId || await readServiceNodeId(folder);
+                        if (nodeId) {
+                            await notifyNodeShutdown(screenName, nodeId, desired.aliveWindow[1]);
+                        }
+                    })();
+                }
+                return;
+            }
+
+            let gitFolder = folder + "git/";
+            await fs.promises.mkdir(gitFolder, { recursive: true });
+            if (parameters.gitRef) {
+                let prevGitRef = "";
+                try {
+                    prevGitRef = await getGitRefLive(gitFolder);
+                } catch { }
+                await ensureGitSynced({ gitFolder, repoUrl: parameters.repoUrl, gitRef: parameters.gitRef });
+                let afterGitRef = await getGitRefLive(gitFolder);
+                // Reinstall when the ref changed OR node_modules is missing. A recovery re-clone can land on the same commit, so a ref-only check would leave the service with no node_modules.
+                let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
+                if (afterGitRef !== prevGitRef || nodeModulesMissing) {
+                    console.log(green(`Yarn installing for ${magenta(screenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
+                    await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
+                }
+            }
+
+            let parameterPath = folder + "parameters.json";
+            let prevParameters = "";
+            if (await fsExistsAsync(parameterPath)) {
+                prevParameters = await fs.promises.readFile(parameterPath, "utf8");
+            }
+            let newParametersString = JSON.stringify(parameters);
+
+            let nodePathId = folder + SERVICE_NODE_FILE_NAME;
+            if (await fsExistsAsync(nodePathId)) {
+                machineInfo.services[serviceId].nodeId = await fs.promises.readFile(nodePathId, "utf8");
+            }
+
+            if (sameRestartParameters(prevParameters, parameters) && screen?.isProcessRunning) {
+                if (prevParameters !== newParametersString) {
+                    await fs.promises.writeFile(parameterPath, newParametersString);
+                }
+                // We may not have launched this one (we restarted, or it predates process records), so make sure it is recorded before leaving it alone
+                await ensureProcessRecord({ screenName, folder, panePid: screen.pid, record: desired });
+                await syncParametersTimelineFiles(folder, [{ pid: parseInt(screen.pid) || undefined, aliveWindow: desired.aliveWindow, parameters }]);
+                console.log(green(`Verified ${magenta(screenName)} is running`));
+                return;
+            }
+
+            console.log(`Resyncing ${magenta(screenName)}, with ${newParametersString}, isRunning = ${screen?.isProcessRunning}`);
+            await fs.promises.writeFile(parameterPath, newParametersString);
+            void recordLaunch({
+                serviceId,
+                serviceKey: desired.serviceKey,
+                screenName: canonicalScreenName,
+                machineId: desired.machineId,
+                reason: screen?.isProcessRunning && "update" || "crashed",
+                time: Date.now(),
+            });
+
+            // Every launch goes into the future name and renames over the canonical one, releases and plain restarts alike: the replacement is up before anything touches the old process, and it always gets a brand new pane - which is what makes it a distinct process with its own log.
+            let launchScreenName = getFutureScreenName(canonicalScreenName);
+            let launchId = await runScreenCommand({
+                screenName: launchScreenName,
+                folder,
+                command: parameters.command,
+                record: desired,
+                // Scheduled the instant the command is sent. Without a time the old process goes as soon as this one is verified.
+                takeover: desired.takeoverAt !== undefined && { canonicalScreenName, killTime: desired.takeoverAt } || undefined,
+            });
+
+            await delay(LAUNCH_VERIFY_DELAY);
+            let launched = (await getScreenState(false)).find(x => x.screenName === launchScreenName);
+            let isRunning = launched && await isScreenRunningProcess(launched.pid);
+            if (!launched || !isRunning) {
+                let prefix = getTmuxPrefix();
+                let logs = await runPromise(`${prefix}tmux capture-pane -t ${launchScreenName} -p`);
+                throw new Error(`${logs}\n\nService ${magenta(launchScreenName)} is not running after starting. Trying again in ${formatTime(MACHINE_RESYNC_INTERVAL)}, or on next change. Last logs above.`);
+            }
+            console.log(green(`${magenta(launchScreenName)} is verified to be running (process ${launchId})`));
+            if (desired.takeoverAt === undefined) {
+                // Nothing is holding the canonical name open for an overlap, so take it now
+                await takeoverScreen({ canonicalScreenName, folder });
+                screenStateMap.set(canonicalScreenName, { ...launched, screenName: canonicalScreenName });
+            }
+            await syncParametersTimelineFiles(folder, [{ pid: parseInt(launched.pid) || undefined, aliveWindow: desired.aliveWindow, parameters }]);
+
+            launchCount++;
+            lastLaunchedTime = Date.now();
         });
-        console.log(green(`Verified future instance ${magenta(futureScreenName)} is running`));
-        return;
+    } catch (e: any) {
+        // Counted on failure too, as a service that keeps crashing is only visible as a launch count climbing faster than everyone else's
+        launchCount++;
+        lastLaunchedTime = Date.now();
+        console.error(`Error resyncing ${magenta(screenName)} (${serviceId}): ${e.stack}`);
+        machineInfo.services[serviceId].errorFromLastRun = e.stack;
     }
-    await ensureFuturePrepared(config);
-    let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
-    console.log(green(`Starting future instance ${magenta(futureScreenName)} (release time reached)`));
-    await runScreenCommand({
-        screenName: futureScreenName,
-        folder,
-        command: next.command,
-        record: {
-            serviceId: config.serviceId,
-            serviceKey: next.key,
-            index: config.index,
-            machineId: config.machineId,
-            parameters: next,
-        },
-    });
-    // The old process's overlap is counted from here, not from whenever a resync next comes around
-    scheduleTakeover({ canonicalScreenName, folder, killTime: config.killTime });
-}
-
-/** The loop's route into the takeover: the timer scheduled at launch normally gets there first, and this is what catches up if we restarted and lost it. The takeover itself is shared, and locked, so the two can't interleave. */
-async function takeoverFutureScreen(config: {
-    canonicalScreenName: string;
-    screenNamesUsed: Set<string>;
-    screenStateMap: Map<string, { screenName: string; isProcessRunning: boolean; pid: string }>;
-}) {
-    let { canonicalScreenName, screenStateMap } = config;
-    let futureScreenName = getFutureScreenName(canonicalScreenName);
-    let future = screenStateMap.get(futureScreenName);
-    if (!future) return;
-    // The renames make both the future and the retiring names vanish from the caller's screen list, so the unused-screen sweep must not try to kill them
-    config.screenNamesUsed.add(futureScreenName);
-    let tookOver = await takeoverScreen({
-        canonicalScreenName,
-        folder: os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/",
-    });
-    if (!tookOver) return;
-    screenStateMap.set(canonicalScreenName, { ...future, screenName: canonicalScreenName });
-    screenStateMap.delete(futureScreenName);
+    machineInfo.services[serviceId].totalTimesLaunched = launchCount;
+    machineInfo.services[serviceId].lastLaunchedTime = lastLaunchedTime;
+    lastLaunchedTimePerService.set(serviceId, lastLaunchedTime);
+    launchesPerService.set(serviceId, launchCount);
 }
 
 function sameRestartParameters(prevParametersJSON: string, next: ServiceParameters): boolean {
@@ -363,308 +457,29 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
         },
         now: Date.now(),
     });
-    let screenNamesUsed = new Set<string>();
-    let upcomingReleases: { releaseTime: number; overlapTime: number }[] = [];
     let root = os.homedir() + "/" + SERVICE_FOLDER;
+    let desiredScreens = getDesiredScreens(relevantConfigs, machineId, root);
 
-    let recordStates = relevantConfigs.map(record => {
-        let releaseTime = record.parameters.releaseTime || 0;
-        // The overlap length always comes from the NEWEST parameters, even though it governs how long the OLD instances outlive the release
-        let overlapTime = record.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME;
-        let killTime = releaseTime + overlapTime;
-        // The release dance only exists while there are old parameters to keep alive; past killTime this is just a normal config (the takeover in the canonical loop below is what gets it there)
-        let releaseInFlight = !!releaseTime && !!record.oldParameters && Date.now() < killTime;
-        let isPrepTime = releaseInFlight && Date.now() >= releaseTime - RELEASE_PREP_LEAD_TIME;
-        let isOverlapTime = releaseInFlight && Date.now() >= releaseTime;
-        // The canonical screens keep running the OLD parameters (completely untouched) until killTime; the new version runs in "-future" screens in the SAME folders from releaseTime, and takes over the canonical screen names at killTime
-        let config = { ...record, parameters: releaseInFlight && record.oldParameters || getLiveServiceParameters(record) };
-        return { record, config, releaseTime, overlapTime, killTime, releaseInFlight, isPrepTime, isOverlapTime };
-    });
-
-    // Fire every instance folder's git fast-path check (HEAD ref + dirty scan) in parallel up front, so the serial loop below just awaits the shared promises instead of paying for one working-tree traversal at a time
+    // Fire every folder's git fast-path check (HEAD ref + dirty scan) up front, so converging awaits shared promises instead of paying for one working-tree traversal at a time
     gitFastPathChecks.clear();
-    for (let { record, config, isPrepTime } of recordStates) {
-        const prewarm = (parameters: ServiceParameters) => {
-            if (!parameters.gitRef) return;
-            let targets = getMachineTargets(parameters).filter(target => target.machineId === machineId);
-            for (let i = 0; i < targets.length; i++) {
-                let folder = root + getScreenName({ serviceKey: parameters.key, index: i }) + "/";
-                void checkGitSyncedAndClean(folder + "git/", parameters.gitRef);
-            }
-        };
-        if (config.parameters.deploy) {
-            prewarm(config.parameters);
-        }
-        if (isPrepTime && record.parameters.deploy) {
-            prewarm(record.parameters);
-        }
+    for (let desired of desiredScreens) {
+        if (desired.outgoing || !desired.parameters.gitRef) continue;
+        void checkGitSyncedAndClean(desired.folder + "git/", desired.parameters.gitRef);
     }
 
-    // Services run concurrently: each has its own folders, its own screens and its own records, and yarn install takes a network mutex, so there is nothing for them to serialize behind. One slow service (a git sync, a yarn install, a screen taking its full shutdown grace) no longer delays every service after it.
-    await Promise.all(recordStates.map(async ({ record, config, releaseTime, overlapTime, killTime, releaseInFlight, isPrepTime, isOverlapTime }) => {
-        if (releaseInFlight) {
-            upcomingReleases.push({ releaseTime, overlapTime });
-            console.log(magenta(`Release in flight for ${record.serviceId} (${record.parameters.key}): the new version starts at ${new Date(releaseTime).toLocaleString()}${isOverlapTime && " (overlap running)" || ""}, the old instances are killed at ${new Date(killTime).toLocaleString()}`));
-        }
-
-        if (isPrepTime && record.parameters.deploy) {
-            let nextTargets = getMachineTargets(record.parameters).filter(target => target.machineId === machineId);
-            for (let i = 0; i < nextTargets.length; i++) {
-                let next = { ...record.parameters, command: applyCommandTemplate(record.parameters.command, nextTargets[i].variables) };
-                let canonicalScreenName = getScreenName({ serviceKey: record.parameters.key, index: i });
-                try {
-                    if (isOverlapTime) {
-                        await ensureFutureStarted({ canonicalScreenName, next, releaseTime, killTime, serviceId: record.serviceId, index: i, machineId, screenNamesUsed, screenStateMap });
-                    } else {
-                        await ensureFuturePrepared({ canonicalScreenName, next, releaseTime, killTime, serviceId: record.serviceId, index: i, machineId, screenNamesUsed, screenStateMap });
-                    }
-                    // The canonical loop below writes the timeline, but it is skipped entirely when the OLD parameters aren't deployed (a release that turns deploy on). The new instance would then run with no timeline file saying when it takes over, so it is written here instead. The old parameters aren't running, so the future entry is the whole timeline.
-                    if (!config.parameters.deploy) {
-                        let futureScreen = screenStateMap.get(getFutureScreenName(canonicalScreenName));
-                        await syncParametersTimelineFiles(root + canonicalScreenName + "/", [{
-                            pid: futureScreen?.isProcessRunning && parseInt(futureScreen.pid) || undefined,
-                            aliveWindow: [releaseTime, ALIVE_WINDOW_FOREVER],
-                            parameters: next,
-                        }]);
-                    }
-                } catch (e: any) {
-                    console.error(`Error preparing/starting the future instance for ${record.serviceId} (${record.parameters.key}): ${e.stack}`);
-                }
-            }
-        }
-        if (!config.parameters.deploy) return;
-        let matchedTargets = getMachineTargets(config.parameters).filter(target => target.machineId === machineId);
-        for (let i = 0; i < matchedTargets.length; i++) {
-            // Each instance runs the command with its own entry's template variables resolved, so the per-instance parameters (written to parameters.json and compared for restarts) carry the resolved command.
-            let instanceParameters = { ...config.parameters, command: applyCommandTemplate(config.parameters.command, matchedTargets[i].variables) };
-            let screenName = getScreenName({
-                serviceKey: config.parameters.key,
-                index: i,
-            });
-
-            let launchCount = launchesPerService.get(config.serviceId) || 0;
-            let lastLaunchedTime = lastLaunchedTimePerService.get(config.serviceId) || 0;
-            machineInfo.services[config.serviceId] = {
-                lastLaunchedTime,
-                errorFromLastRun: "",
-                totalTimesLaunched: launchCount,
-                nodeId: "",
-            };
-            try {
-                let folder = root + screenName + "/";
-                screenNamesUsed.add(screenName);
-
-                // Set once the future screen has become the canonical one, so the timeline stops listing it as a separate upcoming version
-                let tookOverFuture = false;
-
-                const syncTimeline = async (pid: string | undefined) => {
-                    // The windows overlap: a version runs from its releaseTime until the NEXT version's releaseTime + overlapTime (the overlap is when both run)
-                    let aliveWindow: [number, number] = [config.parameters.releaseTime || 0, releaseInFlight ? killTime : ALIVE_WINDOW_FOREVER];
-                    if (tookOverFuture) {
-                        // The canonical screen IS the new version now, so it starts at the release and has no scheduled end - the old parameters' window died with the old process
-                        aliveWindow = [releaseTime, ALIVE_WINDOW_FOREVER];
-                    }
-                    let entries: ParametersTimelineEntry[] = [{
-                        pid: parseInt(pid || "") || undefined,
-                        aliveWindow,
-                        parameters: instanceParameters,
-                    }];
-                    if (releaseInFlight && !tookOverFuture) {
-                        let futureTargets = getMachineTargets(record.parameters).filter(target => target.machineId === machineId);
-                        if (i < futureTargets.length) {
-                            let futureScreen = screenStateMap.get(getFutureScreenName(screenName));
-                            entries.push({
-                                pid: futureScreen?.isProcessRunning && parseInt(futureScreen.pid) || undefined,
-                                aliveWindow: [releaseTime, ALIVE_WINDOW_FOREVER],
-                                parameters: { ...record.parameters, command: applyCommandTemplate(record.parameters.command, futureTargets[i].variables) },
-                            });
-                        }
-                    }
-                    await syncParametersTimelineFiles(folder, entries);
-                };
-
-                if (releaseInFlight) {
-                    // Tell the running instance when it will be shut down, so it (and everything talking to it) can wind down gracefully. Fire and forget: it is a courtesy to the process, and nothing about taking it down may wait on a call to the process we are taking down. The node comes off its own record, which was taken before any replacement could overwrite the folder's file.
-                    void (async () => {
-                        let running = await findRunningRecord(folder, screenName);
-                        let nodeId = running?.nodeId || await readServiceNodeId(folder);
-                        if (nodeId) {
-                            await notifyNodeShutdown(screenName, nodeId, killTime);
-                        }
-                    })();
-                } else {
-                    // A finished release's future screen takes over here: kill the old screen, rename the future one — the new process itself is untouched, so the normal logic below just verifies it
-                    await takeoverFutureScreen({ canonicalScreenName: screenName, screenNamesUsed, screenStateMap });
-                }
-                if (isPrepTime) {
-                    // From prep time on, the canonical (old) screen is left completely untouched — the folder holds the NEW code and parameters by now, so the normal compare/sync logic below must not run against the old process
-                    if (screenStateMap.get(screenName)?.isProcessRunning) {
-                        // The old instance keeps running untouched through the overlap, but it still needs a record - it may have been launched before we restarted
-                        let panePid = screenStateMap.get(screenName)?.pid;
-                        if (panePid) {
-                            await ensureProcessRecord({
-                                screenName,
-                                folder,
-                                panePid,
-                                record: {
-                                    serviceId: config.serviceId,
-                                    serviceKey: config.parameters.key,
-                                    index: i,
-                                    machineId,
-                                    parameters: instanceParameters,
-                                },
-                            });
-                        }
-                        await syncTimeline(screenStateMap.get(screenName)?.pid);
-                        let nodePathId = folder + SERVICE_NODE_FILE_NAME;
-                        if (await fsExistsAsync(nodePathId)) {
-                            machineInfo.services[config.serviceId].nodeId = await fs.promises.readFile(nodePathId, "utf8");
-                        }
-                        console.log(green(`Verified ${magenta(screenName)} is running (release in flight, leaving it untouched until ${new Date(killTime).toLocaleString()})`));
-                        continue;
-                    }
-                    // The old instance is dead, so there is nothing to keep alive through the overlap — take the new version over immediately
-                    console.log(magenta(`Old instance ${screenName} is not running during a release, taking the new version over immediately`));
-                    await takeoverFutureScreen({ canonicalScreenName: screenName, screenNamesUsed, screenStateMap });
-                    tookOverFuture = true;
-                    let nextTargets = getMachineTargets(record.parameters).filter(target => target.machineId === machineId);
-                    if (record.parameters.deploy && record.parameters.key === config.parameters.key && i < nextTargets.length) {
-                        instanceParameters = { ...record.parameters, command: applyCommandTemplate(record.parameters.command, nextTargets[i].variables) };
-                    }
-                }
-
-                let gitFolder = folder + "git/";
-                await fs.promises.mkdir(gitFolder, { recursive: true });
-                if (instanceParameters.gitRef) {
-                    let prevGitRef = "";
-                    try {
-                        prevGitRef = await getGitRefLive(gitFolder);
-                    } catch { }
-                    await ensureGitSynced({
-                        gitFolder: gitFolder,
-                        repoUrl: instanceParameters.repoUrl,
-                        gitRef: instanceParameters.gitRef,
-                    });
-                    let afterGitRef = await getGitRefLive(gitFolder);
-                    // Reinstall when the ref changed OR node_modules is missing. The latter is the real fix for the "Cannot find module 'ws'" crash: a recovery re-clone can land on the same commit, so a ref-only check would skip the install and leave the service with no node_modules. Restoring node_modules also self-heals an already-running process, since a failed `require` is never cached and the next reconnect re-resolves it.
-                    let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
-                    if (afterGitRef !== prevGitRef || nodeModulesMissing) {
-                        console.log(green(`Yarn installing for ${magenta(screenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
-                        await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
-                    }
-                }
-                let parameterPath = folder + "/parameters.json";
-                let prevParameters = "";
-                if (await fsExistsAsync(parameterPath)) {
-                    prevParameters = await fs.promises.readFile(parameterPath, "utf8");
-                }
-                let newParametersString = JSON.stringify(instanceParameters);
-
-                let sameParameters = sameRestartParameters(prevParameters, instanceParameters);
-                let screenIsRunning = screenStateMap.get(screenName)?.isProcessRunning;
-
-                let nodePathId = folder + SERVICE_NODE_FILE_NAME;
-                if (await fsExistsAsync(nodePathId)) {
-                    let nodeId = await fs.promises.readFile(nodePathId, "utf8");
-                    machineInfo.services[config.serviceId].nodeId = nodeId;
-                }
-
-                if (sameParameters && screenIsRunning) {
-                    if (prevParameters !== newParametersString) {
-                        await fs.promises.writeFile(parameterPath, newParametersString);
-                    }
-                    // We did not launch this process (we restarted, or it predates process records), so make sure it is recorded before we leave it alone
-                    let panePid = screenStateMap.get(screenName)?.pid;
-                    if (panePid) {
-                        await ensureProcessRecord({
-                            screenName,
-                            folder,
-                            panePid,
-                            record: {
-                                serviceId: config.serviceId,
-                                serviceKey: config.parameters.key,
-                                index: i,
-                                machineId,
-                                parameters: instanceParameters,
-                            },
-                        });
-                    }
-                    await syncTimeline(screenStateMap.get(screenName)?.pid);
-                    console.log(green(`Verified ${magenta(screenName)} is running`));
-                    continue;
-                }
-
-                console.log(`Resyncing service ${magenta(screenName)}, with ${newParametersString}, isRunning = ${screenIsRunning}, sameParameters = ${sameParameters}`);
-
-                await fs.promises.writeFile(parameterPath, newParametersString);
-
-                let launchReason: "crashed" | "update";
-                if (!sameParameters) {
-                    launchReason = "update";
-                } else {
-                    launchReason = "crashed";
-                }
-                void recordLaunch({
-                    serviceId: config.serviceId,
-                    serviceKey: config.parameters.key,
-                    screenName,
-                    machineId,
-                    reason: launchReason,
-                    time: Date.now(),
-                });
-
-                // Every launch goes through the future screen and a rename, releases and plain restarts alike. The new process is up before the old one is touched, so nothing waits on a shutdown, and the pane it runs in is always new - which is what makes it a distinct process with its own log.
-                let launchScreenName = getFutureScreenName(screenName);
-                let launchId = await runScreenCommand({
-                    screenName: launchScreenName,
-                    folder,
-                    command: instanceParameters.command,
-                    record: {
-                        serviceId: config.serviceId,
-                        serviceKey: config.parameters.key,
-                        index: i,
-                        machineId,
-                        parameters: instanceParameters,
-                    },
-                });
-                await delay(2000);
-                let newScreens = await getScreenState(false);
-                let foundScreen = newScreens.find(x => x.screenName === launchScreenName);
-                if (!foundScreen) {
-                    console.error(`Just created screen ${launchScreenName}, but it's not in the list of screens!`);
-                }
-                let isRunning = foundScreen && await isScreenRunningProcess(foundScreen.pid);
-                if (!isRunning) {
-                    let prefix = getTmuxPrefix();
-                    let logs = await runPromise(`${prefix}tmux capture-pane -t ${launchScreenName} -p`);
-                    throw new Error(`${logs}\n\nService ${magenta(launchScreenName)} is not running after starting. Trying again in ${formatTime(MACHINE_RESYNC_INTERVAL)}, or on next change. Last logs above.`);
-                }
-                // Up and verified, so the old process can go now
-                if (foundScreen) {
-                    screenStateMap.set(launchScreenName, foundScreen);
-                }
-                await takeoverFutureScreen({ canonicalScreenName: screenName, screenNamesUsed, screenStateMap });
-                console.log(green(`Service ${magenta(screenName)} is verified to be running after starting (process ${launchId})`));
-                await syncTimeline(foundScreen?.pid);
-
-                launchCount++;
-                lastLaunchedTime = Date.now();
-            } catch (e: any) {
-                // Increase on failures too, as this is the only way to detect if a service is repeatedly crashing. If this is really high for one service, but not others... it's crashing a lot.
-                launchCount++;
-                lastLaunchedTime = Date.now();
-                console.error(`Error resyncing service ${magenta(config.serviceId)} (${config.parameters?.key}): ${e.stack}`);
-                machineInfo.services[config.serviceId].errorFromLastRun = e.stack;
-            }
-            machineInfo.services[config.serviceId].totalTimesLaunched = launchCount;
-            machineInfo.services[config.serviceId].lastLaunchedTime = lastLaunchedTime;
-            lastLaunchedTimePerService.set(config.serviceId, lastLaunchedTime);
-            launchesPerService.set(config.serviceId, launchCount);
-        }
-    }));
+    // Every screen converges concurrently. Each has its own folder, its own records and its own lock, and yarn install takes a network mutex, so there is nothing to serialize behind - one slow git sync or shutdown no longer delays everything after it.
+    await Promise.all(desiredScreens.map(desired => ensureScreen({ desired, screenStateMap, machineInfo })));
 
+    // Both names count as wanted: a screen that took over during this pass answers to the canonical one now, and one that is about to take over still answers to its future name
+    let wantedNames = new Set<string>();
+    for (let desired of desiredScreens) {
+        wantedNames.add(desired.screenName);
+        wantedNames.add(desired.canonicalScreenName);
+    }
     for (let { screenName } of screenState) {
-        if (screenNamesUsed.has(screenName)) continue;
+        if (wantedNames.has(screenName)) continue;
+        // A screen that is winding down after a takeover is already being killed on its own timer
+        if (screenName.includes(RETIRING_MARKER)) continue;
         // Detached, so a screen that takes its full grace period to go down doesn't hold up the end of the resync
         void (async () => {
             // Nothing is replacing this one, so its registration and the file it wrote both go
@@ -677,7 +492,8 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
         })();
     }
 
-    scheduleReleaseResync(upcomingReleases);
+    // The takeover is scheduled the moment the replacement starts, so this only exists to catch up after a restart loses those timers
+    scheduleReleaseResync(desiredScreens);
 
     await machineInfos.set(machineId, machineInfo);
     console.log(`${magenta(`Resynced ${relevantConfigs.length} services`)}:\n${JSON.stringify(machineInfo, null, 2)}`);
@@ -693,17 +509,16 @@ async function resyncServices() {
     }
 }
 
-// The regular resync poll is too coarse for release boundaries (the future-screen prep, the release itself, and the kill time), so schedule an extra resync at the next boundary. The boundaries carry no logic of their own — each resync just re-evaluates the current state.
+// The regular resync poll is too coarse for a takeover boundary, so schedule an extra resync at the next one. It carries no logic of its own — the resync just re-evaluates the current state, which is what recovers a takeover whose timer we lost by restarting.
 let cancelReleaseResync: (() => void) | undefined;
-function scheduleReleaseResync(upcomingReleases: { releaseTime: number; overlapTime: number }[]) {
+function scheduleReleaseResync(desiredScreens: DesiredScreen[]) {
     let now = Date.now();
     let next: number | undefined;
-    for (let release of upcomingReleases) {
-        for (let boundary of [release.releaseTime - RELEASE_PREP_LEAD_TIME, release.releaseTime, release.releaseTime + release.overlapTime]) {
-            if (boundary <= now) continue;
-            if (next === undefined || boundary < next) {
-                next = boundary;
-            }
+    for (let desired of desiredScreens) {
+        let boundary = desired.takeoverAt;
+        if (boundary === undefined || boundary <= now) continue;
+        if (next === undefined || boundary < next) {
+            next = boundary;
         }
     }
     if (cancelReleaseResync) {

package/src/deployManager/processManager.ts

@@ -25,9 +25,11 @@ const SCREEN_SUFFIX = "-dply";
 export function getScreenName(config: { serviceKey: string; index: number }): string {
     return `${config.serviceKey}-${config.index}${SCREEN_SUFFIX}`.replace(/[^a-zA-Z0-9\-_]/g, "_");
 }
+/** Marks a session that has already been handed off and is winding down on its own timer. */
+export const RETIRING_MARKER = "-retiring-";
 /** The name an outgoing screen is moved to so the new process can take the canonical name at once, instead of anything waiting for the old process to wind down. Keeps the suffix, so it is still a screen we know about. */
 export function getRetiringScreenName(canonicalScreenName: string, at: number): string {
-    return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + `-retiring-${at}` + SCREEN_SUFFIX;
+    return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + RETIRING_MARKER + at + SCREEN_SUFFIX;
 }
 // The new version's screen during a release, in the SAME folder as the canonical screen: created (just echoing when it will start) shortly before releaseTime, started at releaseTime, and renamed to the canonical screen name once the old screen is killed at releaseTime + overlapTime.
 export function getFutureScreenName(canonicalScreenName: string): string {
@@ -215,6 +217,8 @@ export const runScreenCommand = measureWrap(async function runScreenCommand(conf
     folder?: string;
     // Identifies the process this launch creates, so its log and its configuration are stored against it
     record: Omit<ProcessRecord, "launchId" | "folder" | "screenName" | "startTime">;
+    // The screen this launch replaces, and when that one's overlap runs out. The timer is set the instant the command is sent, so the overlap is measured from the new process actually starting.
+    takeover?: { canonicalScreenName: string; killTime: number };
 }): Promise<string> {
     let prefix = getTmuxPrefix();
     let screenName = config.screenName;
@@ -259,6 +263,14 @@ ${config.command}
 `;
     await fs.promises.writeFile(folder + "command.sh", command);
     await runPromise(`${prefix}tmux send-keys -t ${screenName} 'bash ../command.sh' Enter`);
+    // The command is running as of this line, so the overlap starts counting here - nothing between this and the timer.
+    if (config.takeover) {
+        scheduleTakeover({
+            canonicalScreenName: config.takeover.canonicalScreenName,
+            folder,
+            killTime: config.takeover.killTime,
+        });
+    }
 
     await setupPipePane({ screenName, folder, launchId });
     return launchId;