querysub 0.568.0 → 0.570.0

package/appSecrets.ts

@@ -23,6 +23,8 @@ export const getCloudflareCreds = lazy(async (): Promise<{ key: string; email: s
 
 /** Serves the secret keys sliftutils' getSecret expects, reading them the way querysub already does: backblaze creds off disk (getBackblazePath), cloudflare creds from the keys archives bucket (with its local file fallback). */
 export async function getAppSecret(key: string): Promise<string | undefined> {
+    // HACK: This is used by the storage server. Which we want to shut down immediately. That way the next instance can start up. The sword server already looks for notifications about when it's going to be shutting down and it flushes to disk early in this case, so it should be fairly safe to hard kill it.
+    doImmediateShutdown();
     // Import query sub so our logs go to our actual log server. Otherwise we won't know if we get any errors. 
     Querysub;
     if (key.startsWith("backblaze.json.")) {
@@ -39,3 +41,21 @@ export async function getAppSecret(key: string): Promise<string | undefined> {
     }
     return undefined;
 }
+
+let doImmediateShutdown = lazy(() => {
+    // NOTE: This extra code is required to actual capture ctrl+c
+    if (process.platform === "win32") {
+        var rl = require("readline").createInterface({
+            input: process.stdin,
+            output: process.stdout
+        });
+
+        rl.on("SIGINT", function () {
+            process.emit("SIGINT");
+        });
+    }
+    function doShutdown() {
+        process.exit();
+    }
+    process.on("SIGINT", doShutdown);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.568.0",
+  "version": "0.570.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -82,28 +82,6 @@ export class ServiceDetailPage extends qreact.Component {
         });
     }
 
-    private confirmForceDeployNow(deployConfig: ServiceConfig) {
-        let modal: { close: () => void } | undefined;
-        modal = showModal({
-            content: <div className={css.fixed.pos(0, 0).size("100vw", "100vh").hsla(0, 0, 0, 0.5).display("flex").alignItems("center").justifyContent("center")}>
-                <div className={css.vbox(12).pad2(20).hsl(0, 0, 12).colorhsl(0, 0, 90).bord2(0, 0, 40) + " keepModalsOpen"}>
-                    <div className={css.boldStyle.fontSize(16)}>🚀 Deploy?</div>
-                    <div>{deployConfig.parameters.overlapTime
-                        ? `This config goes live now. The old instances keep running for ${formatTime(deployConfig.parameters.overlapTime)} before shutting down.`
-                        : "This config goes live now. The old instances are shut down as soon as the new ones are up, so anything still connected to them is dropped."}</div>
-                    <div className={css.hbox(10)}>
-                        <Button hue={0} onClick={() => {
-                            modal && modal.close();
-                            deployConfig.parameters.releaseTime = Date.now();
-                            this.deployConfig(deployConfig);
-                        }}>🚀 Deploy</Button>
-                        <Button onClick={() => modal && modal.close()}>Cancel</Button>
-                    </div>
-                </div>
-            </div>,
-        });
-    }
-
     // Shows the `${name}` variables found in the command, and lets each picked machine entry set its own values for them (which converts machineIds to the { machineId, variables } form).
     private renderTemplateVariables(config: ServiceConfig) {
         let templateVariables = getCommandTemplateVariables(config.parameters.command);
@@ -601,7 +579,9 @@ export class ServiceDetailPage extends qreact.Component {
                             disabled={this.state.isDeploying}
                             title="Changes the scheduled deploy to go live immediately"
                             onClick={() => {
-                                this.confirmForceDeployNow(deepCloneJSON(stripReleaseState(originalConfig)));
+                                let deployConfig = deepCloneJSON(stripReleaseState(originalConfig));
+                                deployConfig.parameters.releaseTime = Date.now();
+                                this.deployConfig(deployConfig);
                             }}>
                             ⚡ Force Schedule Now
                         </button>
@@ -656,7 +636,7 @@ export class ServiceDetailPage extends qreact.Component {
                             disabled={this.state.isDeploying}
                             title="Deploys now; the old instances keep running for the overlap before shutting down"
                             onClick={() => {
-                                this.confirmForceDeployNow(makeDeployConfig());
+                                this.deployConfig(makeDeployConfig());
                             }}>
                             {this.state.isDeploying && "⏳ Deploying..." || "🚀 Deploy"}
                         </button>
@@ -665,7 +645,7 @@ export class ServiceDetailPage extends qreact.Component {
                             disabled={this.state.isDeploying}
                             title="Deploys now AND shuts the old instances down as soon as the new ones are up (overlap of 0)"
                             onClick={() => {
-                                this.confirmForceDeployNow(makeDeployConfig(0));
+                                this.deployConfig(makeDeployConfig(0));
                             }}>
                             Deploy, No Overlap
                         </button>

package/src/deployManager/components/deployButtons.tsx

@@ -101,32 +101,6 @@ export class UpdateButtons extends qreact.Component<{
             }
         });
     }
-    private confirmDeployAllNow(outdatedServices: ServiceConfig[], latestRef: string, noOverlap: boolean) {
-        let modal: { close: () => void } | undefined;
-        modal = showFullscreenModal({
-            content: <div className={css.vbox(12).pad2(20)}>
-                <div className={css.boldStyle.fontSize(16)}>⚡ Deploy all {outdatedServices.length} services now?</div>
-                <div>{noOverlap
-                    ? "The old instances are shut down as soon as the new ones are up (overlap of 0), so anything still connected to them is dropped."
-                    : "The old instances keep running for their configured overlap before shutting down, so connections have somewhere to go while clients move across."}</div>
-                <div className={css.hbox(10)}>
-                    <button
-                        className={css.pad2(12, 8).button + (noOverlap && css.bord2(0, 90, 30).hsl(0, 85, 65).colorhsl(0, 0, 100).fontWeight("bold") || css.bord2(45, 80, 35).hsl(45, 85, 82))}
-                        onClick={() => {
-                            modal && modal.close();
-                            this.deployAll(outdatedServices, latestRef, noOverlap);
-                        }}>
-                        ⚡ Deploy Now
-                    </button>
-                    <button
-                        className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 100)}
-                        onClick={() => modal && modal.close()}>
-                        Cancel
-                    </button>
-                </div>
-            </div>
-        });
-    }
     render() {
         let controller = MachineServiceController(SocketFunction.browserNodeId());
         const gitInfo = controller.getGitInfo();
@@ -185,7 +159,7 @@ export class UpdateButtons extends qreact.Component<{
                     disabled={this.state.isDeploying}
                     title="Deploys now; the old instances keep running for their configured overlap before shutting down"
                     onClick={() => {
-                        this.confirmDeployAllNow(outdatedServices, gitInfo.latestRef, false);
+                        this.deployAll(outdatedServices, gitInfo.latestRef, false);
                     }}
                 >
                     <div>
@@ -205,7 +179,7 @@ export class UpdateButtons extends qreact.Component<{
                     disabled={this.state.isDeploying}
                     title="Deploys now AND shuts the old instances down as soon as the new ones are up (overlap of 0)"
                     onClick={() => {
-                        this.confirmDeployAllNow(outdatedServices, gitInfo.latestRef, true);
+                        this.deployAll(outdatedServices, gitInfo.latestRef, true);
                     }}
                 >
                     <div>

package/src/deployManager/machineApplyMainCode.ts

@@ -26,8 +26,8 @@ import { PromiseObj } from "../promise";
 import path from "path";
 import { fsExistsAsync } from "../fs";
 import { ALIVE_WINDOW_FOREVER, ParametersTimelineEntry, syncParametersTimelineFiles } from "./parametersTimeline";
-import { getScreenName, getFutureScreenName, getRetiringScreenName, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, ensureProcessRecord, runScreenCommand, killScreen, streamProcessOutput } from "./processManager";
-import { ProcessRecord, syncProcessRecords, writeProcessRecord, listProcessRecords, getProcessLogPath } from "./processLogs";
+import { getScreenName, getFutureScreenName, RETIRING_MARKER, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, ensureProcessRecord, runScreenCommand, killScreen, streamProcessOutput, readServiceNodeId, removeServiceNode, takeoverScreen, withScreenLock } from "./processManager";
+import { ProcessRecord, syncProcessRecords, writeProcessRecord, listProcessRecords, findRunningRecord, getProcessLogPath } from "./processLogs";
 
 
 
@@ -137,17 +137,6 @@ async function notifyNodeShutdown(screenName: string, nodeId: string, time: numb
         console.warn(`Error notifying ${screenName} (${nodeId}) of its scheduled shutdown (will retry on the next resync): ${e.stack}`);
     }
 }
-async function notifyServiceShutdown(screenName: string, time: number) {
-    if (notifiedShutdowns.get(screenName) === time) return;
-    let nodeIdFile = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
-    if (!await fsExistsAsync(nodeIdFile)) {
-        console.log(`No nodeId file found for ${screenName}, cannot notify it of its scheduled shutdown yet (will retry on the next resync)`);
-        return;
-    }
-    let nodeId = await fs.promises.readFile(nodeIdFile, "utf8");
-    await notifyNodeShutdown(screenName, nodeId, time);
-}
-
 
 
 
@@ -231,140 +220,210 @@ const ensureGitSynced = measureWrap(async function ensureGitSynced(config: {
 let launchesPerService = new Map<string, number>();
 let lastLaunchedTimePerService = new Map<string, number>();
 
-const RELEASE_PREP_LEAD_TIME = timeInSecond * 30;
-
-// In-memory only: the old process's nodeId, captured just before the future instance starts (the new process overwrites the shared folder's nodeId file), so the old node can be force-removed when its screen is finally killed. Lost on apply restart, which is fine — node discovery times dead nodes out anyway.
-let takeoverOldNodeIds = new Map<string, string>();
+const LAUNCH_VERIFY_DELAY = 2000;
 
-// Syncs the canonical folder to the new parameters (git + yarn install + parameters.json) and creates the future screen, which just echos when it will start. Idempotent.
-type FutureConfig = {
+/** One tmux session that should exist right now, and the configuration it should be running. The whole resync is derived from this list: converge each of these, kill everything else. */
+type DesiredScreen = {
+    /** The session that should exist. For an incoming version that is the "-future" name, until it takes the canonical name over. */
+    screenName: string;
+    /** The slot this belongs to - its own name, or the one it is replacing */
     canonicalScreenName: string;
-    next: ServiceParameters;
-    releaseTime: number;
+    /** The instance folder, which is always the canonical screen's folder (an incoming version runs out of the same one) */
+    folder: string;
     serviceId: string;
+    serviceKey: string;
     index: number;
     machineId: string;
-    screenNamesUsed: Set<string>;
-    screenStateMap: Map<string, { screenName: string; isProcessRunning: boolean; pid: string }>;
+    /** The parameters it runs, with template variables already resolved */
+    parameters: ServiceParameters;
+    aliveWindow: [number, number];
+    /** The version being replaced during an overlap: it must keep running and must not be touched, because the folder already belongs to its replacement. */
+    outgoing?: boolean;
+    /** On an incoming version, when it takes the canonical name over. Absent means take over as soon as it is up. */
+    takeoverAt?: number;
 };
 
-async function ensureFuturePrepared(config: FutureConfig) {
-    let { canonicalScreenName, next, releaseTime } = config;
-    let futureScreenName = getFutureScreenName(canonicalScreenName);
-    config.screenNamesUsed.add(futureScreenName);
-    let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
-    let gitFolder = folder + "git/";
-    await fs.promises.mkdir(gitFolder, { recursive: true });
-    if (next.gitRef) {
-        let prevGitRef = "";
-        try {
-            prevGitRef = await getGitRefLive(gitFolder);
-        } catch { }
-        await ensureGitSynced({
-            gitFolder,
-            repoUrl: next.repoUrl,
-            gitRef: next.gitRef,
-        });
-        let afterGitRef = await getGitRefLive(gitFolder);
-        let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
-        if (afterGitRef !== prevGitRef || nodeModulesMissing) {
-            console.log(green(`Yarn installing for ${magenta(futureScreenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
-            await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
+/** Every screen that should exist on this machine, worked out purely from the configs and the clock. */
+function getDesiredScreens(configs: ServiceConfig[], machineId: string, root: string): DesiredScreen[] {
+    let desired: DesiredScreen[] = [];
+    let now = Date.now();
+    for (let config of configs) {
+        let releaseTime = config.parameters.releaseTime || 0;
+        // The overlap length always comes from the NEWEST parameters, even though it governs how long the OLD instances outlive the release
+        let overlapTime = config.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME;
+        let killTime = releaseTime + overlapTime;
+        // Only in flight while there is an old version to keep alive, and only until its overlap runs out
+        let releaseInFlight = !!releaseTime && !!config.oldParameters && now < killTime;
+
+        const add = (parameters: ServiceParameters, extra: Partial<DesiredScreen>) => {
+            if (!parameters.deploy) return;
+            let targets = getMachineTargets(parameters).filter(target => target.machineId === machineId);
+            for (let index = 0; index < targets.length; index++) {
+                let canonicalScreenName = getScreenName({ serviceKey: parameters.key, index });
+                desired.push({
+                    screenName: canonicalScreenName,
+                    canonicalScreenName,
+                    folder: root + canonicalScreenName + "/",
+                    serviceId: config.serviceId,
+                    serviceKey: parameters.key,
+                    index,
+                    machineId,
+                    // Each instance runs the command with its own entry's variables resolved, so the parameters written to disk and compared for restarts carry the resolved command
+                    parameters: { ...parameters, command: applyCommandTemplate(parameters.command, targets[index].variables) },
+                    aliveWindow: [parameters.releaseTime || 0, ALIVE_WINDOW_FOREVER],
+                    ...extra,
+                });
+            }
+        };
+
+        if (releaseInFlight && config.oldParameters) {
+            // The old version keeps running under the canonical name until its overlap runs out
+            add(config.oldParameters, { outgoing: true, aliveWindow: [config.oldParameters.releaseTime || 0, killTime] });
+            // The new version runs alongside it from the release, and takes the canonical name at killTime
+            if (now >= releaseTime) {
+                add(config.parameters, { takeoverAt: killTime, aliveWindow: [releaseTime, ALIVE_WINDOW_FOREVER] });
+            }
+        } else {
+            add(getLiveServiceParameters(config), {});
         }
     }
-    await fs.promises.writeFile(folder + "parameters.json", JSON.stringify(next));
-    if (!config.screenStateMap.has(futureScreenName)) {
-        let prefix = getTmuxPrefix();
-        await runPromise(`${prefix}tmux new -s ${futureScreenName} -d`);
-        await runPromise(`${prefix}tmux send-keys -t ${futureScreenName} 'echo "Waiting to start the new version at ${new Date(releaseTime).toISOString()} (in ${Math.round((releaseTime - Date.now()) / timeInSecond)} seconds)"' Enter`);
-        console.log(magenta(`Prepared future screen ${futureScreenName}, starting at ${new Date(releaseTime).toLocaleString()}`));
+    // An incoming version lives under the future name until it takes over
+    for (let screen of desired) {
+        if (screen.takeoverAt === undefined) continue;
+        screen.screenName = getFutureScreenName(screen.canonicalScreenName);
     }
+    return desired;
 }
 
-// Starts (or verifies) the future screen actually running the new version, in the canonical screen's folder. Idempotent.
-async function ensureFutureStarted(config: FutureConfig) {
-    let { canonicalScreenName, next } = config;
-    let futureScreenName = getFutureScreenName(canonicalScreenName);
-    let existing = config.screenStateMap.get(futureScreenName);
-    if (existing?.isProcessRunning) {
-        config.screenNamesUsed.add(futureScreenName);
-        // Started before we restarted, so record it rather than only ever recording processes we launched ourselves
-        await ensureProcessRecord({
-            screenName: futureScreenName,
-            folder: os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/",
-            panePid: existing.pid,
-            record: {
-                serviceId: config.serviceId,
-                serviceKey: config.next.key,
-                index: config.index,
-                machineId: config.machineId,
-                parameters: config.next,
-            },
-        });
-        console.log(green(`Verified future instance ${magenta(futureScreenName)} is running`));
-        return;
-    }
-    await ensureFuturePrepared(config);
-    let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
-    let nodeIdFile = folder + SERVICE_NODE_FILE_NAME;
-    // Only the FIRST start can trust the file to hold the old process's nodeId — on a future-instance crash + restart it may already hold the future's own nodeId
-    if (!takeoverOldNodeIds.has(canonicalScreenName) && await fsExistsAsync(nodeIdFile)) {
-        takeoverOldNodeIds.set(canonicalScreenName, await fs.promises.readFile(nodeIdFile, "utf8"));
-    }
-    console.log(green(`Starting future instance ${magenta(futureScreenName)} (release time reached)`));
-    await runScreenCommand({
-        screenName: futureScreenName,
-        folder,
-        command: next.command,
-        record: {
-            serviceId: config.serviceId,
-            serviceKey: next.key,
-            index: config.index,
-            machineId: config.machineId,
-            parameters: next,
-        },
-    });
-}
-
-// Kills the old canonical screen (if any) and renames the future screen to the canonical name. The rename leaves the new process completely untouched, so the takeover is seamless. No-op when there is no future screen.
-async function takeoverFutureScreen(config: {
-    canonicalScreenName: string;
-    screenNamesUsed: Set<string>;
+/** Puts one screen into the state its configuration describes: the folder synced, the process running what it should be, and the records and timeline saying so. Everything about one screen happens under its lock, so a resync and a scheduled takeover can never interleave. */
+async function ensureScreen(config: {
+    desired: DesiredScreen;
     screenStateMap: Map<string, { screenName: string; isProcessRunning: boolean; pid: string }>;
-}) {
-    let { canonicalScreenName, screenStateMap } = config;
-    let futureScreenName = getFutureScreenName(canonicalScreenName);
-    let future = screenStateMap.get(futureScreenName);
-    if (!future) return;
-    // The rename makes the future name vanish, but the caller's original screen list still has it — mark it used so the unused-screen cleanup doesn't try to kill a nonexistent session
-    config.screenNamesUsed.add(futureScreenName);
-    if (screenStateMap.has(canonicalScreenName)) {
-        // The old screen is moved aside rather than waited on: the canonical name is free immediately, and its shutdown (Ctrl+C, then the hard kill) runs on its own without holding up this service or the next one.
-        let retiringScreenName = getRetiringScreenName(canonicalScreenName, Date.now());
-        let prefix = getTmuxPrefix();
-        await runPromise(`${prefix}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
-        screenStateMap.delete(canonicalScreenName);
-        // Kept out of the unused-screen sweep, which would kill it a second time while it is already winding down
-        config.screenNamesUsed.add(retiringScreenName);
-        void killScreen({ screenName: retiringScreenName, skipNodeIdRemoval: true });
-        let oldNodeId = takeoverOldNodeIds.get(canonicalScreenName);
-        if (oldNodeId) {
-            void forceRemoveNode(oldNodeId);
-        }
-    }
-    takeoverOldNodeIds.delete(canonicalScreenName);
-    let prefix = getTmuxPrefix();
-    console.log(green(`Renaming future screen ${magenta(futureScreenName)} to ${magenta(canonicalScreenName)} (takeover complete)`));
-    await runPromise(`${prefix}tmux rename-session -t ${futureScreenName} ${canonicalScreenName}`);
-    screenStateMap.set(canonicalScreenName, { ...future, screenName: canonicalScreenName });
-    screenStateMap.delete(futureScreenName);
-    // Nothing to do to the logs: the future process has been writing to its own file all along, and keeps writing to it under the new session name. Its record still names the future session though, so point it at the name it now runs under.
-    let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
-    for (let record of await listProcessRecords()) {
-        if (record.folder !== folder || record.screenName !== futureScreenName || record.deadTime !== undefined) continue;
-        record.screenName = canonicalScreenName;
-        await writeProcessRecord(record);
+    machineInfo: MachineInfo;
+}): Promise<void> {
+    let { desired, screenStateMap, machineInfo } = config;
+    let { screenName, canonicalScreenName, folder, parameters, serviceId } = desired;
+
+    let launchCount = launchesPerService.get(serviceId) || 0;
+    let lastLaunchedTime = lastLaunchedTimePerService.get(serviceId) || 0;
+    machineInfo.services[serviceId] = {
+        lastLaunchedTime,
+        errorFromLastRun: "",
+        totalTimesLaunched: launchCount,
+        nodeId: "",
+    };
+
+    try {
+        await withScreenLock(canonicalScreenName, async () => {
+            let screen = screenStateMap.get(screenName);
+
+            if (desired.outgoing) {
+                // Its replacement owns the folder now, so the only thing to do is keep its record straight and let it know when it goes down
+                if (screen?.isProcessRunning) {
+                    await ensureProcessRecord({ screenName, folder, panePid: screen.pid, record: desired });
+                    void (async () => {
+                        let running = await findRunningRecord(folder, screenName);
+                        let nodeId = running?.nodeId || await readServiceNodeId(folder);
+                        if (nodeId) {
+                            await notifyNodeShutdown(screenName, nodeId, desired.aliveWindow[1]);
+                        }
+                    })();
+                }
+                return;
+            }
+
+            let gitFolder = folder + "git/";
+            await fs.promises.mkdir(gitFolder, { recursive: true });
+            if (parameters.gitRef) {
+                let prevGitRef = "";
+                try {
+                    prevGitRef = await getGitRefLive(gitFolder);
+                } catch { }
+                await ensureGitSynced({ gitFolder, repoUrl: parameters.repoUrl, gitRef: parameters.gitRef });
+                let afterGitRef = await getGitRefLive(gitFolder);
+                // Reinstall when the ref changed OR node_modules is missing. A recovery re-clone can land on the same commit, so a ref-only check would leave the service with no node_modules.
+                let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
+                if (afterGitRef !== prevGitRef || nodeModulesMissing) {
+                    console.log(green(`Yarn installing for ${magenta(screenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
+                    await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
+                }
+            }
+
+            let parameterPath = folder + "parameters.json";
+            let prevParameters = "";
+            if (await fsExistsAsync(parameterPath)) {
+                prevParameters = await fs.promises.readFile(parameterPath, "utf8");
+            }
+            let newParametersString = JSON.stringify(parameters);
+
+            let nodePathId = folder + SERVICE_NODE_FILE_NAME;
+            if (await fsExistsAsync(nodePathId)) {
+                machineInfo.services[serviceId].nodeId = await fs.promises.readFile(nodePathId, "utf8");
+            }
+
+            if (sameRestartParameters(prevParameters, parameters) && screen?.isProcessRunning) {
+                if (prevParameters !== newParametersString) {
+                    await fs.promises.writeFile(parameterPath, newParametersString);
+                }
+                // We may not have launched this one (we restarted, or it predates process records), so make sure it is recorded before leaving it alone
+                await ensureProcessRecord({ screenName, folder, panePid: screen.pid, record: desired });
+                await syncParametersTimelineFiles(folder, [{ pid: parseInt(screen.pid) || undefined, aliveWindow: desired.aliveWindow, parameters }]);
+                console.log(green(`Verified ${magenta(screenName)} is running`));
+                return;
+            }
+
+            console.log(`Resyncing ${magenta(screenName)}, with ${newParametersString}, isRunning = ${screen?.isProcessRunning}`);
+            await fs.promises.writeFile(parameterPath, newParametersString);
+            void recordLaunch({
+                serviceId,
+                serviceKey: desired.serviceKey,
+                screenName: canonicalScreenName,
+                machineId: desired.machineId,
+                reason: screen?.isProcessRunning && "update" || "crashed",
+                time: Date.now(),
+            });
+
+            // Every launch goes into the future name and renames over the canonical one, releases and plain restarts alike: the replacement is up before anything touches the old process, and it always gets a brand new pane - which is what makes it a distinct process with its own log.
+            let launchScreenName = getFutureScreenName(canonicalScreenName);
+            let launchId = await runScreenCommand({
+                screenName: launchScreenName,
+                folder,
+                command: parameters.command,
+                record: desired,
+                // Scheduled the instant the command is sent. Without a time the old process goes as soon as this one is verified.
+                takeover: desired.takeoverAt !== undefined && { canonicalScreenName, killTime: desired.takeoverAt } || undefined,
+            });
+
+            await delay(LAUNCH_VERIFY_DELAY);
+            let launched = (await getScreenState(false)).find(x => x.screenName === launchScreenName);
+            let isRunning = launched && await isScreenRunningProcess(launched.pid);
+            if (!launched || !isRunning) {
+                let prefix = getTmuxPrefix();
+                let logs = await runPromise(`${prefix}tmux capture-pane -t ${launchScreenName} -p`);
+                throw new Error(`${logs}\n\nService ${magenta(launchScreenName)} is not running after starting. Trying again in ${formatTime(MACHINE_RESYNC_INTERVAL)}, or on next change. Last logs above.`);
+            }
+            console.log(green(`${magenta(launchScreenName)} is verified to be running (process ${launchId})`));
+            if (desired.takeoverAt === undefined) {
+                // Nothing is holding the canonical name open for an overlap, so take it now
+                await takeoverScreen({ canonicalScreenName, folder });
+                screenStateMap.set(canonicalScreenName, { ...launched, screenName: canonicalScreenName });
+            }
+            await syncParametersTimelineFiles(folder, [{ pid: parseInt(launched.pid) || undefined, aliveWindow: desired.aliveWindow, parameters }]);
+
+            launchCount++;
+            lastLaunchedTime = Date.now();
+        });
+    } catch (e: any) {
+        // Counted on failure too, as a service that keeps crashing is only visible as a launch count climbing faster than everyone else's
+        launchCount++;
+        lastLaunchedTime = Date.now();
+        console.error(`Error resyncing ${magenta(screenName)} (${serviceId}): ${e.stack}`);
+        machineInfo.services[serviceId].errorFromLastRun = e.stack;
     }
+    machineInfo.services[serviceId].totalTimesLaunched = launchCount;
+    machineInfo.services[serviceId].lastLaunchedTime = lastLaunchedTime;
+    lastLaunchedTimePerService.set(serviceId, lastLaunchedTime);
+    launchesPerService.set(serviceId, launchCount);
 }
 
 function sameRestartParameters(prevParametersJSON: string, next: ServiceParameters): boolean {
@@ -398,316 +457,43 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
         },
         now: Date.now(),
     });
-    let screenNamesUsed = new Set<string>();
-    let upcomingReleases: { releaseTime: number; overlapTime: number }[] = [];
     let root = os.homedir() + "/" + SERVICE_FOLDER;
+    let desiredScreens = getDesiredScreens(relevantConfigs, machineId, root);
 
-    let recordStates = relevantConfigs.map(record => {
-        let releaseTime = record.parameters.releaseTime || 0;
-        // The overlap length always comes from the NEWEST parameters, even though it governs how long the OLD instances outlive the release
-        let overlapTime = record.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME;
-        let killTime = releaseTime + overlapTime;
-        // The release dance only exists while there are old parameters to keep alive; past killTime this is just a normal config (the takeover in the canonical loop below is what gets it there)
-        let releaseInFlight = !!releaseTime && !!record.oldParameters && Date.now() < killTime;
-        let isPrepTime = releaseInFlight && Date.now() >= releaseTime - RELEASE_PREP_LEAD_TIME;
-        let isOverlapTime = releaseInFlight && Date.now() >= releaseTime;
-        // The canonical screens keep running the OLD parameters (completely untouched) until killTime; the new version runs in "-future" screens in the SAME folders from releaseTime, and takes over the canonical screen names at killTime
-        let config = { ...record, parameters: releaseInFlight && record.oldParameters || getLiveServiceParameters(record) };
-        return { record, config, releaseTime, overlapTime, killTime, releaseInFlight, isPrepTime, isOverlapTime };
-    });
-
-    // Fire every instance folder's git fast-path check (HEAD ref + dirty scan) in parallel up front, so the serial loop below just awaits the shared promises instead of paying for one working-tree traversal at a time
+    // Fire every folder's git fast-path check (HEAD ref + dirty scan) up front, so converging awaits shared promises instead of paying for one working-tree traversal at a time
     gitFastPathChecks.clear();
-    for (let { record, config, isPrepTime } of recordStates) {
-        const prewarm = (parameters: ServiceParameters) => {
-            if (!parameters.gitRef) return;
-            let targets = getMachineTargets(parameters).filter(target => target.machineId === machineId);
-            for (let i = 0; i < targets.length; i++) {
-                let folder = root + getScreenName({ serviceKey: parameters.key, index: i }) + "/";
-                void checkGitSyncedAndClean(folder + "git/", parameters.gitRef);
-            }
-        };
-        if (config.parameters.deploy) {
-            prewarm(config.parameters);
-        }
-        if (isPrepTime && record.parameters.deploy) {
-            prewarm(record.parameters);
-        }
+    for (let desired of desiredScreens) {
+        if (desired.outgoing || !desired.parameters.gitRef) continue;
+        void checkGitSyncedAndClean(desired.folder + "git/", desired.parameters.gitRef);
     }
 
-    // Services run concurrently: each has its own folders, its own screens and its own records, and yarn install takes a network mutex, so there is nothing for them to serialize behind. One slow service (a git sync, a yarn install, a screen taking its full shutdown grace) no longer delays every service after it.
-    await Promise.all(recordStates.map(async ({ record, config, releaseTime, overlapTime, killTime, releaseInFlight, isPrepTime, isOverlapTime }) => {
-        if (releaseInFlight) {
-            upcomingReleases.push({ releaseTime, overlapTime });
-            console.log(magenta(`Release in flight for ${record.serviceId} (${record.parameters.key}): the new version starts at ${new Date(releaseTime).toLocaleString()}${isOverlapTime && " (overlap running)" || ""}, the old instances are killed at ${new Date(killTime).toLocaleString()}`));
-        }
-
-        if (isPrepTime && record.parameters.deploy) {
-            let nextTargets = getMachineTargets(record.parameters).filter(target => target.machineId === machineId);
-            for (let i = 0; i < nextTargets.length; i++) {
-                let next = { ...record.parameters, command: applyCommandTemplate(record.parameters.command, nextTargets[i].variables) };
-                let canonicalScreenName = getScreenName({ serviceKey: record.parameters.key, index: i });
-                try {
-                    if (isOverlapTime) {
-                        await ensureFutureStarted({ canonicalScreenName, next, releaseTime, serviceId: record.serviceId, index: i, machineId, screenNamesUsed, screenStateMap });
-                    } else {
-                        await ensureFuturePrepared({ canonicalScreenName, next, releaseTime, serviceId: record.serviceId, index: i, machineId, screenNamesUsed, screenStateMap });
-                    }
-                    // The canonical loop below writes the timeline, but it is skipped entirely when the OLD parameters aren't deployed (a release that turns deploy on). The new instance would then run with no timeline file saying when it takes over, so it is written here instead. The old parameters aren't running, so the future entry is the whole timeline.
-                    if (!config.parameters.deploy) {
-                        let futureScreen = screenStateMap.get(getFutureScreenName(canonicalScreenName));
-                        await syncParametersTimelineFiles(root + canonicalScreenName + "/", [{
-                            pid: futureScreen?.isProcessRunning && parseInt(futureScreen.pid) || undefined,
-                            aliveWindow: [releaseTime, ALIVE_WINDOW_FOREVER],
-                            parameters: next,
-                        }]);
-                    }
-                } catch (e: any) {
-                    console.error(`Error preparing/starting the future instance for ${record.serviceId} (${record.parameters.key}): ${e.stack}`);
-                }
-            }
-        }
-        if (!config.parameters.deploy) return;
-        let matchedTargets = getMachineTargets(config.parameters).filter(target => target.machineId === machineId);
-        for (let i = 0; i < matchedTargets.length; i++) {
-            // Each instance runs the command with its own entry's template variables resolved, so the per-instance parameters (written to parameters.json and compared for restarts) carry the resolved command.
-            let instanceParameters = { ...config.parameters, command: applyCommandTemplate(config.parameters.command, matchedTargets[i].variables) };
-            let screenName = getScreenName({
-                serviceKey: config.parameters.key,
-                index: i,
-            });
-
-            let launchCount = launchesPerService.get(config.serviceId) || 0;
-            let lastLaunchedTime = lastLaunchedTimePerService.get(config.serviceId) || 0;
-            machineInfo.services[config.serviceId] = {
-                lastLaunchedTime,
-                errorFromLastRun: "",
-                totalTimesLaunched: launchCount,
-                nodeId: "",
-            };
-            try {
-                let folder = root + screenName + "/";
-                screenNamesUsed.add(screenName);
-
-                // Set once the future screen has become the canonical one, so the timeline stops listing it as a separate upcoming version
-                let tookOverFuture = false;
-
-                const syncTimeline = async (pid: string | undefined) => {
-                    // The windows overlap: a version runs from its releaseTime until the NEXT version's releaseTime + overlapTime (the overlap is when both run)
-                    let aliveWindow: [number, number] = [config.parameters.releaseTime || 0, releaseInFlight ? killTime : ALIVE_WINDOW_FOREVER];
-                    if (tookOverFuture) {
-                        // The canonical screen IS the new version now, so it starts at the release and has no scheduled end - the old parameters' window died with the old process
-                        aliveWindow = [releaseTime, ALIVE_WINDOW_FOREVER];
-                    }
-                    let entries: ParametersTimelineEntry[] = [{
-                        pid: parseInt(pid || "") || undefined,
-                        aliveWindow,
-                        parameters: instanceParameters,
-                    }];
-                    if (releaseInFlight && !tookOverFuture) {
-                        let futureTargets = getMachineTargets(record.parameters).filter(target => target.machineId === machineId);
-                        if (i < futureTargets.length) {
-                            let futureScreen = screenStateMap.get(getFutureScreenName(screenName));
-                            entries.push({
-                                pid: futureScreen?.isProcessRunning && parseInt(futureScreen.pid) || undefined,
-                                aliveWindow: [releaseTime, ALIVE_WINDOW_FOREVER],
-                                parameters: { ...record.parameters, command: applyCommandTemplate(record.parameters.command, futureTargets[i].variables) },
-                            });
-                        }
-                    }
-                    await syncParametersTimelineFiles(folder, entries);
-                };
-
-                if (releaseInFlight) {
-                    // Tell the running instance when it will be shut down, so it (and everything talking to it) can wind down gracefully. Fire and forget: it is a courtesy to the process, and nothing about taking it down may wait on a call to the process we are taking down. Once the future instance may have started, the folder's nodeId file can belong to the NEW process, so from then on only the stashed old nodeId is safe to notify.
-                    if (!isOverlapTime) {
-                        void notifyServiceShutdown(screenName, killTime);
-                    } else {
-                        let oldNodeId = takeoverOldNodeIds.get(screenName);
-                        if (oldNodeId) {
-                            void notifyNodeShutdown(screenName, oldNodeId, killTime);
-                        }
-                    }
-                } else {
-                    // A finished release's future screen takes over here: kill the old screen, rename the future one — the new process itself is untouched, so the normal logic below just verifies it
-                    await takeoverFutureScreen({ canonicalScreenName: screenName, screenNamesUsed, screenStateMap });
-                }
-                if (isPrepTime) {
-                    // From prep time on, the canonical (old) screen is left completely untouched — the folder holds the NEW code and parameters by now, so the normal compare/sync logic below must not run against the old process
-                    if (screenStateMap.get(screenName)?.isProcessRunning) {
-                        // The old instance keeps running untouched through the overlap, but it still needs a record - it may have been launched before we restarted
-                        let panePid = screenStateMap.get(screenName)?.pid;
-                        if (panePid) {
-                            await ensureProcessRecord({
-                                screenName,
-                                folder,
-                                panePid,
-                                record: {
-                                    serviceId: config.serviceId,
-                                    serviceKey: config.parameters.key,
-                                    index: i,
-                                    machineId,
-                                    parameters: instanceParameters,
-                                },
-                            });
-                        }
-                        await syncTimeline(screenStateMap.get(screenName)?.pid);
-                        let nodePathId = folder + SERVICE_NODE_FILE_NAME;
-                        if (await fsExistsAsync(nodePathId)) {
-                            machineInfo.services[config.serviceId].nodeId = await fs.promises.readFile(nodePathId, "utf8");
-                        }
-                        console.log(green(`Verified ${magenta(screenName)} is running (release in flight, leaving it untouched until ${new Date(killTime).toLocaleString()})`));
-                        continue;
-                    }
-                    // The old instance is dead, so there is nothing to keep alive through the overlap — take the new version over immediately
-                    console.log(magenta(`Old instance ${screenName} is not running during a release, taking the new version over immediately`));
-                    await takeoverFutureScreen({ canonicalScreenName: screenName, screenNamesUsed, screenStateMap });
-                    tookOverFuture = true;
-                    let nextTargets = getMachineTargets(record.parameters).filter(target => target.machineId === machineId);
-                    if (record.parameters.deploy && record.parameters.key === config.parameters.key && i < nextTargets.length) {
-                        instanceParameters = { ...record.parameters, command: applyCommandTemplate(record.parameters.command, nextTargets[i].variables) };
-                    }
-                }
-
-                let gitFolder = folder + "git/";
-                await fs.promises.mkdir(gitFolder, { recursive: true });
-                if (instanceParameters.gitRef) {
-                    let prevGitRef = "";
-                    try {
-                        prevGitRef = await getGitRefLive(gitFolder);
-                    } catch { }
-                    await ensureGitSynced({
-                        gitFolder: gitFolder,
-                        repoUrl: instanceParameters.repoUrl,
-                        gitRef: instanceParameters.gitRef,
-                    });
-                    let afterGitRef = await getGitRefLive(gitFolder);
-                    // Reinstall when the ref changed OR node_modules is missing. The latter is the real fix for the "Cannot find module 'ws'" crash: a recovery re-clone can land on the same commit, so a ref-only check would skip the install and leave the service with no node_modules. Restoring node_modules also self-heals an already-running process, since a failed `require` is never cached and the next reconnect re-resolves it.
-                    let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
-                    if (afterGitRef !== prevGitRef || nodeModulesMissing) {
-                        console.log(green(`Yarn installing for ${magenta(screenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
-                        await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
-                    }
-                }
-                let parameterPath = folder + "/parameters.json";
-                let prevParameters = "";
-                if (await fsExistsAsync(parameterPath)) {
-                    prevParameters = await fs.promises.readFile(parameterPath, "utf8");
-                }
-                let newParametersString = JSON.stringify(instanceParameters);
-
-                let sameParameters = sameRestartParameters(prevParameters, instanceParameters);
-                let screenIsRunning = screenStateMap.get(screenName)?.isProcessRunning;
-
-                let nodePathId = folder + SERVICE_NODE_FILE_NAME;
-                if (await fsExistsAsync(nodePathId)) {
-                    let nodeId = await fs.promises.readFile(nodePathId, "utf8");
-                    machineInfo.services[config.serviceId].nodeId = nodeId;
-                }
-
-                if (sameParameters && screenIsRunning) {
-                    if (prevParameters !== newParametersString) {
-                        await fs.promises.writeFile(parameterPath, newParametersString);
-                    }
-                    // We did not launch this process (we restarted, or it predates process records), so make sure it is recorded before we leave it alone
-                    let panePid = screenStateMap.get(screenName)?.pid;
-                    if (panePid) {
-                        await ensureProcessRecord({
-                            screenName,
-                            folder,
-                            panePid,
-                            record: {
-                                serviceId: config.serviceId,
-                                serviceKey: config.parameters.key,
-                                index: i,
-                                machineId,
-                                parameters: instanceParameters,
-                            },
-                        });
-                    }
-                    await syncTimeline(screenStateMap.get(screenName)?.pid);
-                    console.log(green(`Verified ${magenta(screenName)} is running`));
-                    continue;
-                }
-
-                console.log(`Resyncing service ${magenta(screenName)}, with ${newParametersString}, isRunning = ${screenIsRunning}, sameParameters = ${sameParameters}`);
-
-                await fs.promises.writeFile(parameterPath, newParametersString);
-
-                let launchReason: "crashed" | "update";
-                if (!sameParameters) {
-                    launchReason = "update";
-                } else {
-                    launchReason = "crashed";
-                }
-                void recordLaunch({
-                    serviceId: config.serviceId,
-                    serviceKey: config.parameters.key,
-                    screenName,
-                    machineId,
-                    reason: launchReason,
-                    time: Date.now(),
-                });
-
-                // Every launch goes through the future screen and a rename, releases and plain restarts alike. The new process is up before the old one is touched, so nothing waits on a shutdown, and the pane it runs in is always new - which is what makes it a distinct process with its own log.
-                let launchScreenName = getFutureScreenName(screenName);
-                let launchId = await runScreenCommand({
-                    screenName: launchScreenName,
-                    folder,
-                    command: instanceParameters.command,
-                    record: {
-                        serviceId: config.serviceId,
-                        serviceKey: config.parameters.key,
-                        index: i,
-                        machineId,
-                        parameters: instanceParameters,
-                    },
-                });
-                await delay(2000);
-                let newScreens = await getScreenState(false);
-                let foundScreen = newScreens.find(x => x.screenName === launchScreenName);
-                if (!foundScreen) {
-                    console.error(`Just created screen ${launchScreenName}, but it's not in the list of screens!`);
-                }
-                let isRunning = foundScreen && await isScreenRunningProcess(foundScreen.pid);
-                if (!isRunning) {
-                    let prefix = getTmuxPrefix();
-                    let logs = await runPromise(`${prefix}tmux capture-pane -t ${launchScreenName} -p`);
-                    throw new Error(`${logs}\n\nService ${magenta(launchScreenName)} is not running after starting. Trying again in ${formatTime(MACHINE_RESYNC_INTERVAL)}, or on next change. Last logs above.`);
-                }
-                // Up and verified, so the old process can go now
-                if (foundScreen) {
-                    screenStateMap.set(launchScreenName, foundScreen);
-                }
-                await takeoverFutureScreen({ canonicalScreenName: screenName, screenNamesUsed, screenStateMap });
-                console.log(green(`Service ${magenta(screenName)} is verified to be running after starting (process ${launchId})`));
-                await syncTimeline(foundScreen?.pid);
-
-                launchCount++;
-                lastLaunchedTime = Date.now();
-            } catch (e: any) {
-                // Increase on failures too, as this is the only way to detect if a service is repeatedly crashing. If this is really high for one service, but not others... it's crashing a lot.
-                launchCount++;
-                lastLaunchedTime = Date.now();
-                console.error(`Error resyncing service ${magenta(config.serviceId)} (${config.parameters?.key}): ${e.stack}`);
-                machineInfo.services[config.serviceId].errorFromLastRun = e.stack;
-            }
-            machineInfo.services[config.serviceId].totalTimesLaunched = launchCount;
-            machineInfo.services[config.serviceId].lastLaunchedTime = lastLaunchedTime;
-            lastLaunchedTimePerService.set(config.serviceId, lastLaunchedTime);
-            launchesPerService.set(config.serviceId, launchCount);
-        }
-    }));
+    // Every screen converges concurrently. Each has its own folder, its own records and its own lock, and yarn install takes a network mutex, so there is nothing to serialize behind - one slow git sync or shutdown no longer delays everything after it.
+    await Promise.all(desiredScreens.map(desired => ensureScreen({ desired, screenStateMap, machineInfo })));
 
+    // Both names count as wanted: a screen that took over during this pass answers to the canonical one now, and one that is about to take over still answers to its future name
+    let wantedNames = new Set<string>();
+    for (let desired of desiredScreens) {
+        wantedNames.add(desired.screenName);
+        wantedNames.add(desired.canonicalScreenName);
+    }
     for (let { screenName } of screenState) {
-        if (screenNamesUsed.has(screenName)) continue;
+        if (wantedNames.has(screenName)) continue;
+        // A screen that is winding down after a takeover is already being killed on its own timer
+        if (screenName.includes(RETIRING_MARKER)) continue;
         // Detached, so a screen that takes its full grace period to go down doesn't hold up the end of the resync
-        void killScreen({
-            screenName,
-        });
+        void (async () => {
+            // Nothing is replacing this one, so its registration and the file it wrote both go
+            let folder = root + screenName + "/";
+            let nodeId = (await findRunningRecord(folder, screenName))?.nodeId || await readServiceNodeId(folder);
+            if (nodeId) {
+                await removeServiceNode({ folder, nodeId, ownsFile: true });
+            }
+            await killScreen({ screenName });
+        })();
     }
 
-    scheduleReleaseResync(upcomingReleases);
+    // The takeover is scheduled the moment the replacement starts, so this only exists to catch up after a restart loses those timers
+    scheduleReleaseResync(desiredScreens);
 
     await machineInfos.set(machineId, machineInfo);
     console.log(`${magenta(`Resynced ${relevantConfigs.length} services`)}:\n${JSON.stringify(machineInfo, null, 2)}`);
@@ -723,17 +509,16 @@ async function resyncServices() {
     }
 }
 
-// The regular resync poll is too coarse for release boundaries (the future-screen prep, the release itself, and the kill time), so schedule an extra resync at the next boundary. The boundaries carry no logic of their own — each resync just re-evaluates the current state.
+// The regular resync poll is too coarse for a takeover boundary, so schedule an extra resync at the next one. It carries no logic of its own — the resync just re-evaluates the current state, which is what recovers a takeover whose timer we lost by restarting.
 let cancelReleaseResync: (() => void) | undefined;
-function scheduleReleaseResync(upcomingReleases: { releaseTime: number; overlapTime: number }[]) {
+function scheduleReleaseResync(desiredScreens: DesiredScreen[]) {
     let now = Date.now();
     let next: number | undefined;
-    for (let release of upcomingReleases) {
-        for (let boundary of [release.releaseTime - RELEASE_PREP_LEAD_TIME, release.releaseTime, release.releaseTime + release.overlapTime]) {
-            if (boundary <= now) continue;
-            if (next === undefined || boundary < next) {
-                next = boundary;
-            }
+    for (let desired of desiredScreens) {
+        let boundary = desired.takeoverAt;
+        if (boundary === undefined || boundary <= now) continue;
+        if (next === undefined || boundary < next) {
+            next = boundary;
         }
     }
     if (cancelReleaseResync) {

package/src/deployManager/processLogs.ts

@@ -30,6 +30,8 @@ export type ProcessRecord = {
     machineId: string;
     /** The start time the OS reports for the pid, which is also half of the launchId */
     startTime: number;
+    /** The node this process registered as, read out of the folder before any replacement is started - after that the file belongs to the replacement. Kept here so taking this process down needs nothing but its own record. */
+    nodeId?: string;
     /** When we first noticed the process was gone. Absent while it is running. */
     deadTime?: number;
     pid?: number;
@@ -99,6 +101,13 @@ while true; do
 done`;
 }
 
+/** The record of whatever process is currently running in a screen, if we have one. */
+export async function findRunningRecord(folder: string, screenName: string): Promise<ProcessRecord | undefined> {
+    return (await listProcessRecords()).find(x =>
+        x.folder === folder && x.screenName === screenName && x.deadTime === undefined
+    );
+}
+
 export async function writeProcessRecord(record: ProcessRecord): Promise<void> {
     await fs.promises.mkdir(path.join(record.folder, PROCESS_FOLDER), { recursive: true });
     await fs.promises.writeFile(getRecordPath(record.folder, record.launchId), JSON.stringify(record));

package/src/deployManager/processManager.ts

@@ -13,7 +13,8 @@ import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
 import { fsExistsAsync } from "../fs";
 import { PromiseObj } from "../promise";
 import { SERVICE_FOLDER, SERVICE_NODE_FILE_NAME } from "./machineSchema";
-import { ProcessRecord, createLaunchId, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, listProcessRecords, writeProcessRecord } from "./processLogs";
+import { ProcessRecord, createLaunchId, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, listProcessRecords, findRunningRecord, writeProcessRecord } from "./processLogs";
+import { setPreciseTimeout } from "../misc";
 
 // Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
 
@@ -24,9 +25,11 @@ const SCREEN_SUFFIX = "-dply";
 export function getScreenName(config: { serviceKey: string; index: number }): string {
     return `${config.serviceKey}-${config.index}${SCREEN_SUFFIX}`.replace(/[^a-zA-Z0-9\-_]/g, "_");
 }
+/** Marks a session that has already been handed off and is winding down on its own timer. */
+export const RETIRING_MARKER = "-retiring-";
 /** The name an outgoing screen is moved to so the new process can take the canonical name at once, instead of anything waiting for the old process to wind down. Keeps the suffix, so it is still a screen we know about. */
 export function getRetiringScreenName(canonicalScreenName: string, at: number): string {
-    return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + `-retiring-${at}` + SCREEN_SUFFIX;
+    return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + RETIRING_MARKER + at + SCREEN_SUFFIX;
 }
 // The new version's screen during a release, in the SAME folder as the canonical screen: created (just echoing when it will start) shortly before releaseTime, started at releaseTime, and renamed to the canonical screen name once the old screen is killed at releaseTime + overlapTime.
 export function getFutureScreenName(canonicalScreenName: string): string {
@@ -34,16 +37,47 @@ export function getFutureScreenName(canonicalScreenName: string): string {
 }
 
 
-async function removeOldNodeId(screenName: string) {
-    let nodeIdFile = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
-    if (await fsExistsAsync(nodeIdFile)) {
-        let nodeId = await fs.promises.readFile(nodeIdFile, "utf8");
-        console.log(green(`Removing node if for dead service on ${nodeIdFile}, node id ${nodeId}`));
-        await fs.promises.unlink(nodeIdFile);
-        await forceRemoveNode(nodeId);
+// Everything that touches a service's screens - the resync loop, and the timer that retires the old process at its kill time - runs under this, keyed by the canonical screen name. They fire at similar times by design, and a takeover is a rename dance that must not interleave with another one.
+const screenLocks = new Map<string, Promise<unknown>>();
+export function withScreenLock<T>(canonicalScreenName: string, run: () => Promise<T>): Promise<T> {
+    let previous = screenLocks.get(canonicalScreenName) || Promise.resolve();
+    let next = previous.then(run, run);
+    // Failures release the lock rather than poisoning every later operation on this screen
+    screenLocks.set(canonicalScreenName, next.then(() => { }, () => { }));
+    return next;
+}
+
+/** Whether a session exists, without listing every session and its panes - the takeover already knows the exact names it cares about. */
+async function screenExists(screenName: string): Promise<boolean> {
+    let prefix = getTmuxPrefix();
+    try {
+        await runPromise(`${prefix}tmux has-session -t ${screenName}`, { quiet: true });
+        return true;
+    } catch {
+        // has-session exits non-zero for "no such session", which is an answer, not a failure
+        return false;
     }
 }
 
+/** The node the process in this folder registered as. Read it BEFORE starting a replacement in the same folder - once the replacement starts, the file is the replacement's. */
+export async function readServiceNodeId(folder: string): Promise<string | undefined> {
+    let nodeIdFile = path.join(folder, SERVICE_NODE_FILE_NAME);
+    if (!await fsExistsAsync(nodeIdFile)) return undefined;
+    return await fs.promises.readFile(nodeIdFile, "utf8");
+}
+
+/** Drops a process's node registration. The file is only removed when it is still that process's - a replacement has its own registration in the same file. */
+export async function removeServiceNode(config: { folder: string; nodeId: string; ownsFile: boolean }): Promise<void> {
+    console.log(green(`Removing the node registration ${config.nodeId} for the process in ${config.folder}`));
+    if (config.ownsFile) {
+        let nodeIdFile = path.join(config.folder, SERVICE_NODE_FILE_NAME);
+        if (await fsExistsAsync(nodeIdFile)) {
+            await fs.promises.unlink(nodeIdFile);
+        }
+    }
+    await forceRemoveNode(config.nodeId);
+}
+
 export const getTmuxPrefix = lazy(() => {
     if (os.platform() === "win32") {
         return "C:/cygwin64/bin/";
@@ -183,6 +217,8 @@ export const runScreenCommand = measureWrap(async function runScreenCommand(conf
     folder?: string;
     // Identifies the process this launch creates, so its log and its configuration are stored against it
     record: Omit<ProcessRecord, "launchId" | "folder" | "screenName" | "startTime">;
+    // The screen this launch replaces, and when that one's overlap runs out. The timer is set the instant the command is sent, so the overlap is measured from the new process actually starting.
+    takeover?: { canonicalScreenName: string; killTime: number };
 }): Promise<string> {
     let prefix = getTmuxPrefix();
     let screenName = config.screenName;
@@ -195,9 +231,17 @@ export const runScreenCommand = measureWrap(async function runScreenCommand(conf
     }
     await runPromise(`${prefix}tmux new -s ${screenName} -d`);
 
-    await removeOldNodeId(screenName);
     let folder = config.folder || os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
 
+    // Read the outgoing process's node id onto its own record BEFORE this launch overwrites the file. Whoever takes that process down then has everything it needs on the record, instead of us having to remember not to touch a file that stopped being its own.
+    let outgoingNodeId = await readServiceNodeId(folder);
+    if (outgoingNodeId) {
+        for (let record of await listProcessRecords()) {
+            if (record.folder !== folder || record.deadTime !== undefined || record.nodeId) continue;
+            await writeProcessRecord({ ...record, nodeId: outgoingNodeId });
+        }
+    }
+
     // The pane we are about to run the command in identifies the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream has to trust our bookkeeping about what is still running. The session was just created, so this pid is always new.
     let panePid = (await getScreenState(false)).find(x => x.screenName === screenName)?.pid;
     if (!panePid) {
@@ -219,6 +263,14 @@ ${config.command}
 `;
     await fs.promises.writeFile(folder + "command.sh", command);
     await runPromise(`${prefix}tmux send-keys -t ${screenName} 'bash ../command.sh' Enter`);
+    // The command is running as of this line, so the overlap starts counting here - nothing between this and the timer.
+    if (config.takeover) {
+        scheduleTakeover({
+            canonicalScreenName: config.takeover.canonicalScreenName,
+            folder,
+            killTime: config.takeover.killTime,
+        });
+    }
 
     await setupPipePane({ screenName, folder, launchId });
     return launchId;
@@ -267,10 +319,60 @@ async function setupPipePane(config: { screenName: string; folder: string; launc
     await runPromise(`${prefix}tmux pipe-pane -t ${config.screenName} 'bash ${pipeScript}'`);
 }
 
+/** Retires whatever is running under the canonical name and renames the future screen onto it. Everything it needs is derived from the two names, so it costs two `has-session` checks - the kill time is when this runs, not when a resync gets around to it. Returns whether there was a future screen to take over. */
+export async function takeoverScreen(config: { canonicalScreenName: string; folder: string }): Promise<boolean> {
+    let { canonicalScreenName, folder } = config;
+    return await withScreenLock(canonicalScreenName, async () => {
+        let futureScreenName = getFutureScreenName(canonicalScreenName);
+        if (!await screenExists(futureScreenName)) return false;
+        let prefix = getTmuxPrefix();
+        if (await screenExists(canonicalScreenName)) {
+            let retiringScreenName = getRetiringScreenName(canonicalScreenName, Date.now());
+            // Its record carries the node it registered as, taken before the replacement started - the file in the folder belongs to the replacement now, so it is left alone
+            let outgoing = await findRunningRecord(folder, canonicalScreenName);
+            await runPromise(`${prefix}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
+            if (outgoing) {
+                await writeProcessRecord({ ...outgoing, screenName: retiringScreenName });
+                if (outgoing.nodeId) {
+                    void removeServiceNode({ folder, nodeId: outgoing.nodeId, ownsFile: false });
+                }
+            }
+            // Detached: the canonical name is already free, so nothing waits on the old process winding down
+            void killScreen({ screenName: retiringScreenName });
+        }
+        console.log(green(`Renaming future screen ${futureScreenName} to ${canonicalScreenName} (takeover complete)`));
+        await runPromise(`${prefix}tmux rename-session -t ${futureScreenName} ${canonicalScreenName}`);
+        let incoming = await findRunningRecord(folder, futureScreenName);
+        if (incoming) {
+            await writeProcessRecord({ ...incoming, screenName: canonicalScreenName });
+        }
+        return true;
+    });
+}
+
+// canonical screen name -> the takeover we already have scheduled for it
+const scheduledTakeovers = new Map<string, { time: number; cancel: () => void }>();
+
+/** Schedules the takeover for the moment the old process's overlap runs out. The resync loop still does the same takeover whenever it notices the time has passed, which is what recovers this if we restart and lose the timer. */
+export function scheduleTakeover(config: { canonicalScreenName: string; folder: string; killTime: number }): void {
+    let { canonicalScreenName, folder, killTime } = config;
+    let existing = scheduledTakeovers.get(canonicalScreenName);
+    if (existing?.time === killTime) return;
+    existing?.cancel();
+    console.log(green(`Scheduling the takeover of ${canonicalScreenName} for ${new Date(killTime).toISOString()}`));
+    let cancel = setPreciseTimeout({
+        time: killTime,
+        callback: () => {
+            scheduledTakeovers.delete(canonicalScreenName);
+            void takeoverScreen({ canonicalScreenName, folder });
+        },
+    });
+    scheduledTakeovers.set(canonicalScreenName, { time: killTime, cancel });
+}
+
+// Kills a screen and nothing else. Which node the process registered as, and whether that registration should be dropped, is the deploy logic's business - it reads the node id off the process's record, which was taken before any replacement could overwrite the file.
 export const killScreen = measureWrap(async function killScreen(config: {
     screenName: string;
-    // During a takeover the folder's nodeId file already belongs to the NEW process, so the old screen's kill must not remove it
-    skipNodeIdRemoval?: boolean;
 }) {
     console.log(red(`Killing screen ${config.screenName} (Ctrl+C, then killing the session in ${formatTime(SHUTDOWN_GRACE_TIME)})`));
     let prefix = getTmuxPrefix();
@@ -278,9 +380,6 @@ export const killScreen = measureWrap(async function killScreen(config: {
     await runPromise(`${prefix}tmux send-keys -t ${config.screenName} 'C-c' Enter`);
     await delay(SHUTDOWN_GRACE_TIME);
     await runPromise(`${prefix}tmux kill-session -t ${config.screenName}`);
-    if (!config.skipNodeIdRemoval) {
-        await removeOldNodeId(config.screenName);
-    }
 });
 
 /** Streams one process's log: everything already in it, then everything appended after. One process, one file - nothing here has to reason about which process a byte came from. */