querysub 0.570.0 → 0.572.0

package/appSecrets.ts

@@ -55,6 +55,7 @@ let doImmediateShutdown = lazy(() => {
         });
     }
     function doShutdown() {
+        console.log("SIGINT received, shutting down immediately");
         process.exit();
     }
     process.on("SIGINT", doShutdown);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.570.0",
+  "version": "0.572.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/deployManager/components/ProcessesView.tsx

@@ -32,14 +32,15 @@ class ProcessOutput extends qreact.Component<{ record: ProcessRecord; machineNod
     private unmounted = false;
     componentDidMount() {
         // Props are synchronized state, so the plain values the async work needs are read here
-        let { folder, launchId } = this.props.record;
+        let { folder, pid, startTime } = this.props.record;
         let nodeId = this.props.machineNodeId;
         let callbackId = this.callbackId;
         this.watching = (async () => {
             await watchProcessOutput({
                 nodeId,
                 folder,
-                launchId,
+                pid,
+                startTime,
                 callbackId,
                 onData: async (data: string) => {
                     Querysub.commit(() => {
@@ -153,9 +154,9 @@ class MachineProcesses extends qreact.Component<{ machineId: string; applyNodeId
                     {this.state.showHistorical && `Hide ${historical.length} historical` || `Show ${historical.length} historical`}
                 </Button>}
             </div>
-            {running.map(record => <ProcessRow key={record.launchId} record={record} machineNodeId={this.props.applyNodeId} />)}
+            {running.map(record => <ProcessRow key={`${record.pid}-${record.startTime}`} record={record} machineNodeId={this.props.applyNodeId} />)}
             {this.state.showHistorical && historical.map(record =>
-                <ProcessRow key={record.launchId} record={record} machineNodeId={this.props.applyNodeId} />
+                <ProcessRow key={`${record.pid}-${record.startTime}`} record={record} machineNodeId={this.props.applyNodeId} />
             )}
         </div>;
     }

package/src/deployManager/machineApplyMainCode.ts

@@ -10,7 +10,7 @@ import { getExternalIP } from "socket-function/src/networking";
 import { errorToUndefined, errorToUndefinedSilent } from "../errors";
 import { getDomain } from "../config";
 import { formatTime } from "socket-function/src/formatting/format";
-import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { sort, timeInMinute, timeInSecond, watchSlowPromise } from "socket-function/src/misc";
 import { isDefined, setPreciseTimeout } from "../misc";
 import { logLoadTime } from "../logModuleLoadTimes";
 import { delay, retryFunctional, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
@@ -26,8 +26,8 @@ import { PromiseObj } from "../promise";
 import path from "path";
 import { fsExistsAsync } from "../fs";
 import { ALIVE_WINDOW_FOREVER, ParametersTimelineEntry, syncParametersTimelineFiles } from "./parametersTimeline";
-import { getScreenName, getFutureScreenName, RETIRING_MARKER, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, ensureProcessRecord, runScreenCommand, killScreen, streamProcessOutput, readServiceNodeId, removeServiceNode, takeoverScreen, withScreenLock } from "./processManager";
-import { ProcessRecord, syncProcessRecords, writeProcessRecord, listProcessRecords, findRunningRecord, getProcessLogPath } from "./processLogs";
+import { getScreenName, getFutureScreenName, isRetiring, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, getScreenPanePid, ensureProcessRecord, runScreenCommand, killScreen, readServiceNodeId, removeServiceNode, takeoverScreenLocked, scheduleTakeover, killScreenNow, withScreenLock } from "./processManager";
+import { ProcessRecord, syncProcessRecords, listProcessRecords } from "./processLogs";
 
 
 
@@ -222,87 +222,65 @@ let lastLaunchedTimePerService = new Map<string, number>();
 
 const LAUNCH_VERIFY_DELAY = 2000;
 
-/** One tmux session that should exist right now, and the configuration it should be running. The whole resync is derived from this list: converge each of these, kill everything else. */
-type DesiredScreen = {
-    /** The session that should exist. For an incoming version that is the "-future" name, until it takes the canonical name over. */
-    screenName: string;
-    /** The slot this belongs to - its own name, or the one it is replacing */
-    canonicalScreenName: string;
-    /** The instance folder, which is always the canonical screen's folder (an incoming version runs out of the same one) */
-    folder: string;
-    serviceId: string;
-    serviceKey: string;
-    index: number;
-    machineId: string;
-    /** The parameters it runs, with template variables already resolved */
-    parameters: ServiceParameters;
-    aliveWindow: [number, number];
-    /** The version being replaced during an overlap: it must keep running and must not be touched, because the folder already belongs to its replacement. */
-    outgoing?: boolean;
-    /** On an incoming version, when it takes the canonical name over. Absent means take over as soon as it is up. */
-    takeoverAt?: number;
-};
-
-/** Every screen that should exist on this machine, worked out purely from the configs and the clock. */
-function getDesiredScreens(configs: ServiceConfig[], machineId: string, root: string): DesiredScreen[] {
-    let desired: DesiredScreen[] = [];
-    let now = Date.now();
+/** Every instance of every service that belongs to this machine, with its config already resolved for that instance - the command has this entry's template variables substituted, and a version with no entry at this index is marked as deploying nothing here. */
+function getMachineInstances(configs: ServiceConfig[], machineId: string): { config: ServiceConfig; index: number }[] {
+    let instances: { config: ServiceConfig; index: number }[] = [];
     for (let config of configs) {
-        let releaseTime = config.parameters.releaseTime || 0;
-        // The overlap length always comes from the NEWEST parameters, even though it governs how long the OLD instances outlive the release
-        let overlapTime = config.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME;
-        let killTime = releaseTime + overlapTime;
-        // Only in flight while there is an old version to keep alive, and only until its overlap runs out
-        let releaseInFlight = !!releaseTime && !!config.oldParameters && now < killTime;
-
-        const add = (parameters: ServiceParameters, extra: Partial<DesiredScreen>) => {
-            if (!parameters.deploy) return;
-            let targets = getMachineTargets(parameters).filter(target => target.machineId === machineId);
-            for (let index = 0; index < targets.length; index++) {
-                let canonicalScreenName = getScreenName({ serviceKey: parameters.key, index });
-                desired.push({
-                    screenName: canonicalScreenName,
-                    canonicalScreenName,
-                    folder: root + canonicalScreenName + "/",
-                    serviceId: config.serviceId,
-                    serviceKey: parameters.key,
-                    index,
-                    machineId,
-                    // Each instance runs the command with its own entry's variables resolved, so the parameters written to disk and compared for restarts carry the resolved command
-                    parameters: { ...parameters, command: applyCommandTemplate(parameters.command, targets[index].variables) },
-                    aliveWindow: [parameters.releaseTime || 0, ALIVE_WINDOW_FOREVER],
-                    ...extra,
-                });
-            }
-        };
-
-        if (releaseInFlight && config.oldParameters) {
-            // The old version keeps running under the canonical name until its overlap runs out
-            add(config.oldParameters, { outgoing: true, aliveWindow: [config.oldParameters.releaseTime || 0, killTime] });
-            // The new version runs alongside it from the release, and takes the canonical name at killTime
-            if (now >= releaseTime) {
-                add(config.parameters, { takeoverAt: killTime, aliveWindow: [releaseTime, ALIVE_WINDOW_FOREVER] });
-            }
-        } else {
-            add(getLiveServiceParameters(config), {});
+        // A release can add or remove instances, so the count is whichever version asks for the most
+        let count = 0;
+        for (let parameters of [config.parameters, config.oldParameters]) {
+            if (!parameters) continue;
+            count = Math.max(count, getMachineTargets(parameters).filter(x => x.machineId === machineId).length);
+        }
+        for (let index = 0; index < count; index++) {
+            const resolve = (parameters: ServiceParameters) => {
+                let target = getMachineTargets(parameters).filter(x => x.machineId === machineId)[index];
+                return {
+                    ...parameters,
+                    command: target && applyCommandTemplate(parameters.command, target.variables) || parameters.command,
+                    deploy: !!parameters.deploy && !!target,
+                };
+            };
+            instances.push({
+                index,
+                config: {
+                    ...config,
+                    parameters: resolve(config.parameters),
+                    oldParameters: config.oldParameters && resolve(config.oldParameters),
+                },
+            });
         }
     }
-    // An incoming version lives under the future name until it takes over
-    for (let screen of desired) {
-        if (screen.takeoverAt === undefined) continue;
-        screen.screenName = getFutureScreenName(screen.canonicalScreenName);
-    }
-    return desired;
+    return instances;
 }
 
-/** Puts one screen into the state its configuration describes: the folder synced, the process running what it should be, and the records and timeline saying so. Everything about one screen happens under its lock, so a resync and a scheduled takeover can never interleave. */
+/** Puts one instance of a service into the state its configuration describes: the folder synced, the process running what it should be, and the records and timeline saying so. Returns the screens it is using, so anything nobody claimed can be killed. Everything about one screen happens under its lock, so a resync and a scheduled takeover can never interleave. */
 async function ensureScreen(config: {
-    desired: DesiredScreen;
-    screenStateMap: Map<string, { screenName: string; isProcessRunning: boolean; pid: string }>;
+    config: ServiceConfig;
+    index: number;
     machineInfo: MachineInfo;
-}): Promise<void> {
-    let { desired, screenStateMap, machineInfo } = config;
-    let { screenName, canonicalScreenName, folder, parameters, serviceId } = desired;
+}): Promise<string[]> {
+    let { machineInfo, index } = config;
+    let machineId = machineInfo.machineId;
+    let serviceId = config.config.serviceId;
+    let serviceConfig = config.config;
+
+    let releaseTime = serviceConfig.parameters.releaseTime || 0;
+    // The overlap length always comes from the NEWEST parameters, even though it governs how long the OLD instances outlive the release
+    let killTime = releaseTime + (serviceConfig.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME);
+    // Only in flight while there is an old version to keep alive, and only until its overlap runs out
+    let releaseInFlight = !!releaseTime && !!serviceConfig.oldParameters && Date.now() < killTime;
+
+    // What should be running under the canonical name now, and the version waiting to take it over
+    let current: ServiceParameters | undefined = releaseInFlight && serviceConfig.oldParameters || getLiveServiceParameters(serviceConfig);
+    if (!current?.deploy) current = undefined;
+    let incoming: ServiceParameters | undefined = releaseInFlight && Date.now() >= releaseTime && serviceConfig.parameters || undefined;
+    if (!incoming?.deploy) incoming = undefined;
+    let canonicalScreenName = getScreenName({ serviceKey: (current || incoming || serviceConfig.parameters).key, index });
+    let futureScreenName = getFutureScreenName(canonicalScreenName);
+    let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
+    // Returned even when the work below throws, so a transient failure never makes the sweep kill a screen we still want
+    let usedScreenNames = [canonicalScreenName, futureScreenName];
 
     let launchCount = launchesPerService.get(serviceId) || 0;
     let lastLaunchedTime = lastLaunchedTimePerService.get(serviceId) || 0;
@@ -314,46 +292,70 @@ async function ensureScreen(config: {
     };
 
     try {
-        await withScreenLock(canonicalScreenName, async () => {
-            let screen = screenStateMap.get(screenName);
-
-            if (desired.outgoing) {
-                // Its replacement owns the folder now, so the only thing to do is keep its record straight and let it know when it goes down
-                if (screen?.isProcessRunning) {
-                    await ensureProcessRecord({ screenName, folder, panePid: screen.pid, record: desired });
-                    void (async () => {
-                        let running = await findRunningRecord(folder, screenName);
-                        let nodeId = running?.nodeId || await readServiceNodeId(folder);
-                        if (nodeId) {
-                            await notifyNodeShutdown(screenName, nodeId, desired.aliveWindow[1]);
-                        }
-                    })();
-                }
-                return;
+        await watchSlowPromise(`ensureScreen|${canonicalScreenName}`, withScreenLock(canonicalScreenName, async () => {
+            /*  Three facts, each moving ? -> known -> correct as we go:
+                    EXISTS   - which of the two screens are there
+                    VERSION  - which parameters they were started with
+                    RUNNING  - whether a process is actually alive in them
+                Every step below says where those three stand, so it is always clear what we are still assuming.
+                START:  EXISTS ?  VERSION ?  RUNNING ?
+            */
+            let parameters = incoming || current;
+            let screenName = incoming && futureScreenName || canonicalScreenName;
+            let takeoverAt = incoming && killTime || undefined;
+
+            // A screen we do not expect holds something we never asked for, so it goes at once - nothing here is worth winding down. Each name is judged on its own: a release that turns deploy on has an incoming version and nothing that should be running under the canonical name yet.
+            // AFTER: EXISTS correct in the "nothing extra" sense. A screen that SHOULD exist may still be missing, which is RUNNING's problem below.
+            if (!incoming) {
+                await killScreenNow(futureScreenName, "there is no release in flight, so nothing should be waiting to take over");
             }
-
-            let gitFolder = folder + "git/";
-            await fs.promises.mkdir(gitFolder, { recursive: true });
-            if (parameters.gitRef) {
-                let prevGitRef = "";
-                try {
-                    prevGitRef = await getGitRefLive(gitFolder);
-                } catch { }
-                await ensureGitSynced({ gitFolder, repoUrl: parameters.repoUrl, gitRef: parameters.gitRef });
-                let afterGitRef = await getGitRefLive(gitFolder);
-                // Reinstall when the ref changed OR node_modules is missing. A recovery re-clone can land on the same commit, so a ref-only check would leave the service with no node_modules.
-                let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
-                if (afterGitRef !== prevGitRef || nodeModulesMissing) {
-                    console.log(green(`Yarn installing for ${magenta(screenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
-                    await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
-                }
+            if (!current) {
+                await killScreenNow(canonicalScreenName, "nothing is supposed to be serving this name yet");
             }
+            if (!current && !incoming) return;
 
+            // Both screens run out of one folder, so its parameters.json is the ONE version they were last started from - it cannot tell the two screens apart.
+            // AFTER: VERSION known. Anything it does not match, including a version we no longer recognize, is handled by starting the right one further down - which keeps whatever is up serving until the replacement is verified, instead of killing it now and serving nothing while we sync and start.
             let parameterPath = folder + "parameters.json";
             let prevParameters = "";
             if (await fsExistsAsync(parameterPath)) {
                 prevParameters = await fs.promises.readFile(parameterPath, "utf8");
             }
+
+            // AFTER: RUNNING known for every name that can still have anything in it. A screen can exist with nothing alive in it, and one that should exist can be missing entirely - both read as not up, and both are fixed the same way. Without an incoming version the future name was just killed, so there is nothing left to ask about it.
+            let canonicalPid = await getScreenPanePid(canonicalScreenName);
+            let canonicalUp = !!canonicalPid && await isScreenRunningProcess(canonicalPid);
+            let futurePid = incoming && await getScreenPanePid(futureScreenName) || undefined;
+            let futureUp = !!futurePid && await isScreenRunningProcess(futurePid);
+
+            // SMOOTH PATH: this is what a deploy looks like from the canonical name's side. The version being replaced is already correct - it keeps running untouched until its overlap ends, so this name is settled and only the future one is left to converge. It needs its record kept straight, and to hear when it goes down.
+            // AFTER (canonical, during a release): EXISTS correct   VERSION correct   RUNNING correct
+            if (incoming && current && !canonicalUp) {
+                console.log(red(`Nothing is running as ${canonicalScreenName}, which should be serving until ${new Date(killTime).toISOString()} while its replacement starts`));
+            }
+            if (incoming && current && canonicalUp && canonicalPid) {
+                await ensureProcessRecord({
+                    screenName: canonicalScreenName,
+                    folder,
+                    panePid: canonicalPid,
+                    record: { serviceId, serviceKey: current.key, index, machineId, parameters: current },
+                });
+                void (async () => {
+                    let nodeId = await readServiceNodeId(folder);
+                    if (nodeId) {
+                        await notifyNodeShutdown(canonicalScreenName, nodeId, killTime);
+                    }
+                })();
+            }
+
+            // From here on this is about ONE screen - the future one during a release, the canonical one otherwise. Everything else is already correct.
+            // STATE: EXISTS correct   VERSION known   RUNNING known
+            if (!parameters) return;
+            let panePid = incoming && futurePid || canonicalPid;
+            let isUp = incoming && futureUp || canonicalUp;
+            let aliveWindow: [number, number] = [parameters.releaseTime || 0, ALIVE_WINDOW_FOREVER];
+            let record = { serviceId, serviceKey: parameters.key, index, machineId, parameters };
+
             let newParametersString = JSON.stringify(parameters);
 
             let nodePathId = folder + SERVICE_NODE_FILE_NAME;
@@ -361,69 +363,112 @@ async function ensureScreen(config: {
                 machineInfo.services[serviceId].nodeId = await fs.promises.readFile(nodePathId, "utf8");
             }
 
-            if (sameRestartParameters(prevParameters, parameters) && screen?.isProcessRunning) {
+            // Already running this exact version, so nothing needs starting - this is the state we are trying to reach
+            // AFTER: EXISTS correct   VERSION correct   RUNNING correct, and the takeover below settles which name it answers to
+            if (isUp && panePid && sameRestartParameters(prevParameters, parameters)) {
                 if (prevParameters !== newParametersString) {
                     await fs.promises.writeFile(parameterPath, newParametersString);
                 }
-                // We may not have launched this one (we restarted, or it predates process records), so make sure it is recorded before leaving it alone
-                await ensureProcessRecord({ screenName, folder, panePid: screen.pid, record: desired });
-                await syncParametersTimelineFiles(folder, [{ pid: parseInt(screen.pid) || undefined, aliveWindow: desired.aliveWindow, parameters }]);
+                // We may not have started this one (we restarted, or it predates process records), so make sure it is recorded before leaving it alone
+                await ensureProcessRecord({ screenName, folder, panePid, record });
+                await syncParametersTimelineFiles(folder, [{ pid: parseInt(panePid) || undefined, aliveWindow, parameters }]);
+                if (takeoverAt !== undefined) {
+                    if (!canonicalUp) {
+                        // The version this one is waiting to replace is gone, so there is no overlap left to serve - take the name now instead of sitting under the future one
+                        console.log(red(`Nothing is running as ${canonicalScreenName}, so ${screenName} takes it over now instead of waiting until ${new Date(takeoverAt).toISOString()}`));
+                        await takeoverScreenLocked({ canonicalScreenName, folder });
+                    } else {
+                        // SMOOTH PATH: both versions are up, the old one serving and the new one waiting. A restart loses the timer that retires the old one, so it is put back every time we see it still waiting.
+                        scheduleTakeover({ canonicalScreenName, folder, killTime: takeoverAt });
+                    }
+                }
                 console.log(green(`Verified ${magenta(screenName)} is running`));
                 return;
             }
 
-            console.log(`Resyncing ${magenta(screenName)}, with ${newParametersString}, isRunning = ${screen?.isProcessRunning}`);
+            // RUNNING is wrong - either nothing is alive, or what is alive is the wrong version. Starting is the only thing that fixes it.
+            if (!isUp) {
+                // Nothing should ever stop on its own: either it crashed, or it never came up
+                console.log(red(`Nothing is running as ${screenName}, starting it: ${newParametersString}`));
+            } else {
+                // SMOOTH PATH: the previous version is up and this replaces it with a new one, which is exactly what a deploy is
+                console.log(green(`Starting a new version as ${magenta(screenName)}, replacing the one running there: ${newParametersString}`));
+            }
+
+            // Only now does the folder have to hold this version, which is why the git and yarn work sits behind the check above rather than running on every resync
+            let gitFolder = folder + "git/";
+            await fs.promises.mkdir(gitFolder, { recursive: true });
+            if (parameters.gitRef) {
+                let prevGitRef = "";
+                try {
+                    prevGitRef = await getGitRefLive(gitFolder);
+                } catch { }
+                await ensureGitSynced({ gitFolder, repoUrl: parameters.repoUrl, gitRef: parameters.gitRef });
+                let afterGitRef = await getGitRefLive(gitFolder);
+                // Reinstall when the ref changed OR node_modules is missing. A recovery re-clone can land on the same commit, so a ref-only check would leave the service with no node_modules.
+                let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
+                if (afterGitRef !== prevGitRef || nodeModulesMissing) {
+                    console.log(green(`Yarn installing for ${magenta(screenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
+                    await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
+                }
+            }
+
             await fs.promises.writeFile(parameterPath, newParametersString);
             void recordLaunch({
                 serviceId,
-                serviceKey: desired.serviceKey,
+                serviceKey: parameters.key,
                 screenName: canonicalScreenName,
-                machineId: desired.machineId,
-                reason: screen?.isProcessRunning && "update" || "crashed",
+                machineId,
+                reason: isUp && "update" || "crashed",
                 time: Date.now(),
             });
 
-            // Every launch goes into the future name and renames over the canonical one, releases and plain restarts alike: the replacement is up before anything touches the old process, and it always gets a brand new pane - which is what makes it a distinct process with its own log.
-            let launchScreenName = getFutureScreenName(canonicalScreenName);
-            let launchId = await runScreenCommand({
+            // The node the version being replaced registered as, read before the launch overwrites the file with the new process's
+            let replacedNodeId = await readServiceNodeId(folder);
+
+            // Always started under the future name and renamed over the canonical one, releases and plain restarts alike: it is up before anything touches what it replaces, and it always gets a brand new pane - which is what makes it a distinct process with its own log.
+            let launchScreenName = futureScreenName;
+            let launchedPid = await runScreenCommand({
                 screenName: launchScreenName,
                 folder,
                 command: parameters.command,
-                record: desired,
-                // Scheduled the instant the command is sent. Without a time the old process goes as soon as this one is verified.
-                takeover: desired.takeoverAt !== undefined && { canonicalScreenName, killTime: desired.takeoverAt } || undefined,
+                record,
             });
+            // SMOOTH PATH: the command is running as of the line above, so the overlap the version it replaces gets is measured from here
+            if (takeoverAt !== undefined) {
+                scheduleTakeover({ canonicalScreenName, folder, killTime: takeoverAt, replacedNodeId });
+            }
 
             await delay(LAUNCH_VERIFY_DELAY);
-            let launched = (await getScreenState(false)).find(x => x.screenName === launchScreenName);
-            let isRunning = launched && await isScreenRunningProcess(launched.pid);
-            if (!launched || !isRunning) {
+            let launchedPanePid = await getScreenPanePid(launchScreenName);
+            if (!launchedPanePid || !await isScreenRunningProcess(launchedPanePid)) {
                 let prefix = getTmuxPrefix();
                 let logs = await runPromise(`${prefix}tmux capture-pane -t ${launchScreenName} -p`);
                 throw new Error(`${logs}\n\nService ${magenta(launchScreenName)} is not running after starting. Trying again in ${formatTime(MACHINE_RESYNC_INTERVAL)}, or on next change. Last logs above.`);
             }
-            console.log(green(`${magenta(launchScreenName)} is verified to be running (process ${launchId})`));
-            if (desired.takeoverAt === undefined) {
-                // Nothing is holding the canonical name open for an overlap, so take it now
-                await takeoverScreen({ canonicalScreenName, folder });
-                screenStateMap.set(canonicalScreenName, { ...launched, screenName: canonicalScreenName });
+            console.log(green(`${magenta(launchScreenName)} is verified to be running (pid ${launchedPid})`));
+            // Take the canonical name now unless something is still serving under it - waiting out an overlap that nothing is serving would leave the name empty for no reason. SMOOTH PATH during a release is neither: the version being replaced is still up, so this one waits under the future name until its timer fires.
+            // AFTER: EXISTS correct   VERSION correct   RUNNING correct, under the canonical name, or under the future one with the version it is replacing still serving
+            if (takeoverAt === undefined || !canonicalUp) {
+                await takeoverScreenLocked({ canonicalScreenName, folder, replacedNodeId });
             }
-            await syncParametersTimelineFiles(folder, [{ pid: parseInt(launched.pid) || undefined, aliveWindow: desired.aliveWindow, parameters }]);
+            await syncParametersTimelineFiles(folder, [{ pid: parseInt(launchedPanePid) || undefined, aliveWindow, parameters }]);
 
             launchCount++;
             lastLaunchedTime = Date.now();
-        });
+        }));
     } catch (e: any) {
         // Counted on failure too, as a service that keeps crashing is only visible as a launch count climbing faster than everyone else's
         launchCount++;
         lastLaunchedTime = Date.now();
-        console.error(`Error resyncing ${magenta(screenName)} (${serviceId}): ${e.stack}`);
+        console.error(`Error resyncing ${magenta(canonicalScreenName)} (${serviceId}): ${e.stack}`);
         machineInfo.services[serviceId].errorFromLastRun = e.stack;
     }
     machineInfo.services[serviceId].totalTimesLaunched = launchCount;
     machineInfo.services[serviceId].lastLaunchedTime = lastLaunchedTime;
     lastLaunchedTimePerService.set(serviceId, lastLaunchedTime);
     launchesPerService.set(serviceId, launchCount);
+    return usedScreenNames;
 }
 
 function sameRestartParameters(prevParametersJSON: string, next: ServiceParameters): boolean {
@@ -445,46 +490,36 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
 
     let machineInfo = await getLiveMachineInfo();
 
+    // The whole machine's screens, which is what the record maintenance and the kill sweep are about. Converging a screen asks about that one screen itself.
     let screenState = await getScreenState();
-    let screenStateMap = new Map(screenState.map(s => [s.screenName, s]));
-    // A process is alive while a pane with its pid still exists AND that pid still reports the start time it was named after - a reused pid is a different process
-    let panePids = new Set(screenState.map(x => x.pid));
+    // A process is alive while its pane still exists, still reports the start time it was named after (a reused pid is a different process), and still has something running in it - a pane whose command exited is a dead process, not a live one
+    let running = new Set(screenState.filter(x => x.isProcessRunning).map(x => x.pid));
     await syncProcessRecords({
         isAlive: async record => {
-            let pid = String(record.pid);
-            if (!panePids.has(pid)) return false;
-            return await getProcessStartTime(pid) === record.startTime;
+            if (!running.has(record.pid)) return false;
+            return await getProcessStartTime(record.pid) === record.startTime;
         },
         now: Date.now(),
     });
     let root = os.homedir() + "/" + SERVICE_FOLDER;
-    let desiredScreens = getDesiredScreens(relevantConfigs, machineId, root);
+    let instances = getMachineInstances(relevantConfigs, machineId);
 
-    // Fire every folder's git fast-path check (HEAD ref + dirty scan) up front, so converging awaits shared promises instead of paying for one working-tree traversal at a time
+    // The fast-path results are memoized per resync, so a folder shared by two instances is only traversed once
     gitFastPathChecks.clear();
-    for (let desired of desiredScreens) {
-        if (desired.outgoing || !desired.parameters.gitRef) continue;
-        void checkGitSyncedAndClean(desired.folder + "git/", desired.parameters.gitRef);
-    }
 
-    // Every screen converges concurrently. Each has its own folder, its own records and its own lock, and yarn install takes a network mutex, so there is nothing to serialize behind - one slow git sync or shutdown no longer delays everything after it.
-    await Promise.all(desiredScreens.map(desired => ensureScreen({ desired, screenStateMap, machineInfo })));
+    // Every instance converges concurrently. Each has its own folder, its own records and its own lock, and yarn install takes a network mutex, so there is nothing to serialize behind - one slow git sync or shutdown no longer delays everything after it.
+    let usedScreenNames = new Set((await Promise.all(instances.map(instance => ensureScreen({ ...instance, machineInfo })))).flat());
 
-    // Both names count as wanted: a screen that took over during this pass answers to the canonical one now, and one that is about to take over still answers to its future name
-    let wantedNames = new Set<string>();
-    for (let desired of desiredScreens) {
-        wantedNames.add(desired.screenName);
-        wantedNames.add(desired.canonicalScreenName);
-    }
+    // Anything no instance claimed is not supposed to exist
     for (let { screenName } of screenState) {
-        if (wantedNames.has(screenName)) continue;
-        // A screen that is winding down after a takeover is already being killed on its own timer
-        if (screenName.includes(RETIRING_MARKER)) continue;
+        if (usedScreenNames.has(screenName)) continue;
+        // Already being taken down. One left over from a restart is not, so it still gets cleaned up here.
+        if (isRetiring(screenName)) continue;
         // Detached, so a screen that takes its full grace period to go down doesn't hold up the end of the resync
         void (async () => {
             // Nothing is replacing this one, so its registration and the file it wrote both go
             let folder = root + screenName + "/";
-            let nodeId = (await findRunningRecord(folder, screenName))?.nodeId || await readServiceNodeId(folder);
+            let nodeId = await readServiceNodeId(folder);
             if (nodeId) {
                 await removeServiceNode({ folder, nodeId, ownsFile: true });
             }
@@ -492,9 +527,6 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
         })();
     }
 
-    // The takeover is scheduled the moment the replacement starts, so this only exists to catch up after a restart loses those timers
-    scheduleReleaseResync(desiredScreens);
-
     await machineInfos.set(machineId, machineInfo);
     console.log(`${magenta(`Resynced ${relevantConfigs.length} services`)}:\n${JSON.stringify(machineInfo, null, 2)}`);
 
@@ -509,34 +541,6 @@ async function resyncServices() {
     }
 }
 
-// The regular resync poll is too coarse for a takeover boundary, so schedule an extra resync at the next one. It carries no logic of its own — the resync just re-evaluates the current state, which is what recovers a takeover whose timer we lost by restarting.
-let cancelReleaseResync: (() => void) | undefined;
-function scheduleReleaseResync(desiredScreens: DesiredScreen[]) {
-    let now = Date.now();
-    let next: number | undefined;
-    for (let desired of desiredScreens) {
-        let boundary = desired.takeoverAt;
-        if (boundary === undefined || boundary <= now) continue;
-        if (next === undefined || boundary < next) {
-            next = boundary;
-        }
-    }
-    if (cancelReleaseResync) {
-        cancelReleaseResync();
-        cancelReleaseResync = undefined;
-    }
-    if (next === undefined) return;
-    console.log(`Scheduling a resync for the next release boundary at ${new Date(next).toLocaleString()}`);
-    cancelReleaseResync = setPreciseTimeout({
-        time: next,
-        callback: () => {
-            cancelReleaseResync = undefined;
-            void resyncServices();
-        },
-    });
-}
-
-
 async function isPIDRunning(pid: string) {
     try {
         // Use process.kill with signal 0 to test if process exists without actually killing it

package/src/deployManager/machineController.ts

@@ -83,16 +83,18 @@ class MachineControllerBase {
     }
     public async streamProcessOutput(config: {
         folder: string;
-        launchId: string;
+        pid: string;
+        startTime: number;
         callbackId: string;
     }): Promise<void> {
         let caller = SocketFunction.getCaller();
         await streamProcessOutput({
             folder: config.folder,
-            launchId: config.launchId,
+            pid: config.pid,
+            startTime: config.startTime,
             onData: async (data) => {
                 await MachineControllerClient.nodes[caller.nodeId].onScreenOutput({
-                    launchId: config.launchId,
+                    pid: config.pid,
                     data,
                     callbackId: config.callbackId,
                 });
@@ -106,14 +108,16 @@ class MachineControllerBase {
     public async watchOtherProcessOutput(config: {
         nodeId: string;
         folder: string;
-        launchId: string;
+        pid: string;
+        startTime: number;
         callbackId: string;
     }) {
         let caller = SocketFunction.getCaller();
         forwardedCallbacks.set(config.callbackId, caller.nodeId);
         await MachineController(config.nodeId).streamProcessOutput.promise({
             folder: config.folder,
-            launchId: config.launchId,
+            pid: config.pid,
+            startTime: config.startTime,
             callbackId: config.callbackId,
         });
     }
@@ -170,7 +174,8 @@ let callbacks = new Map<string, (data: string) => Promise<void>>();
 export async function watchProcessOutput(config: {
     nodeId: string;
     folder: string;
-    launchId: string;
+    pid: string;
+    startTime: number;
     callbackId: string;
     onData: (data: string) => Promise<void>;
 }) {
@@ -179,7 +184,8 @@ export async function watchProcessOutput(config: {
     await MachineController(SocketFunction.browserNodeId()).watchOtherProcessOutput.promise({
         nodeId: config.nodeId,
         folder: config.folder,
-        launchId: config.launchId,
+        pid: config.pid,
+        startTime: config.startTime,
         callbackId,
     });
 }
@@ -192,14 +198,14 @@ export async function stopWatchingProcessOutput(config: {
 }
 class MachineControllerClientBase {
     public async onScreenOutput(config: {
-        launchId: string;
+        pid: string;
         data: string;
         callbackId: string;
     }): Promise<void> {
         let forwardToNodeId = forwardedCallbacks.get(config.callbackId);
         if (forwardToNodeId) {
             await MachineControllerClient.nodes[forwardToNodeId].onScreenOutput({
-                launchId: config.launchId,
+                pid: config.pid,
                 data: config.data,
                 callbackId: config.callbackId,
             });

package/src/deployManager/processLogs.ts

@@ -1,6 +1,7 @@
 import os from "os";
 import fs from "fs";
 import path from "path";
+import { fsExistsAsync } from "../fs";
 import { SERVICE_FOLDER, ServiceParameters } from "./machineSchema";
 
 // One log file per process, appended to until it passes LOG_SIZE_LIMIT, then emptied wholesale. Watchers see the file shrink and re-seek to the start. A watcher attaching in the moment right after a reset sees an empty log, which at this size is rare enough to be worth how much simpler this is than rolling files.
@@ -16,53 +17,43 @@ const LOG_SUFFIX = ".log";
 const PIPE_SCRIPT_SUFFIX = ".pipe.sh";
 const TAIL_SCRIPT_SUFFIX = ".tail.sh";
 
-/** One launch of one service instance: the process, the configuration it was launched with, and its output. Written when the process is launched, and updated only to record its pid and then its death. */
+/** One process: the configuration it was started with, and its output. A process is named by its pid and the start time the OS reports for that pid - the pid alone gets reused, and both halves can be re-checked against the machine at any time instead of trusted from our own bookkeeping. */
 export type ProcessRecord = {
-    launchId: string;
-    /** The instance folder this process runs out of. A release's future process shares its canonical instance's folder, so this is what owns the logs, not the tmux session name. */
+    pid: string;
+    /** The start time the OS reports for the pid */
+    startTime: number;
+    /** The instance folder this process runs out of. A replacement shares the folder with the process it replaces, so this is what owns the logs, not the tmux session name. */
     folder: string;
-    /** The tmux session, which a takeover renames from the future name to the canonical one */
+    /** The tmux session, which a takeover renames */
     screenName: string;
     serviceId: string;
     serviceKey: string;
     /** Which instance of the service this is, on this machine */
     index: number;
     machineId: string;
-    /** The start time the OS reports for the pid, which is also half of the launchId */
-    startTime: number;
-    /** The node this process registered as, read out of the folder before any replacement is started - after that the file belongs to the replacement. Kept here so taking this process down needs nothing but its own record. */
+    /** The node this process registered as, read out of the folder before any replacement is started - after that the file belongs to the replacement. */
     nodeId?: string;
     /** When we first noticed the process was gone. Absent while it is running. */
     deadTime?: number;
-    pid?: number;
-    /** The parameters it was launched with, with template variables already resolved */
+    /** The parameters it was started with, with template variables already resolved */
     parameters: ServiceParameters;
 };
 
-export function getProcessLogPath(folder: string, launchId: string): string {
-    return path.join(folder, PROCESS_FOLDER, launchId + LOG_SUFFIX);
-}
-function getRecordPath(folder: string, launchId: string): string {
-    return path.join(folder, PROCESS_FOLDER, launchId + RECORD_SUFFIX);
+// A process's files all share this name, so its whole footprint is found, and deleted, together
+function getProcessFileName(pid: string, startTime: number): string {
+    return `${pid}-${startTime}`;
 }
-// The scripts live beside the log so a process's entire footprint is one set of files, deleted together
-export function getPipeScriptPath(folder: string, launchId: string): string {
-    return path.join(folder, PROCESS_FOLDER, launchId + PIPE_SCRIPT_SUFFIX);
+export function getProcessLogPath(folder: string, pid: string, startTime: number): string {
+    return path.join(folder, PROCESS_FOLDER, getProcessFileName(pid, startTime) + LOG_SUFFIX);
 }
-export function getTailScriptPath(folder: string, launchId: string): string {
-    return path.join(folder, PROCESS_FOLDER, launchId + TAIL_SCRIPT_SUFFIX);
+export function getPipeScriptPath(folder: string, pid: string, startTime: number): string {
+    return path.join(folder, PROCESS_FOLDER, getProcessFileName(pid, startTime) + PIPE_SCRIPT_SUFFIX);
 }
-
-/** A process identifies itself: its pid, plus the start time the OS reports for it. The pid alone is reused, but a pid running since a specific second is one process, and both halves can be re-checked against the machine at any time instead of trusted from our own bookkeeping. */
-export function createLaunchId(pid: string, startTime: number): string {
-    return `${pid}-${startTime}`;
+export function getTailScriptPath(folder: string, pid: string, startTime: number): string {
+    return path.join(folder, PROCESS_FOLDER, getProcessFileName(pid, startTime) + TAIL_SCRIPT_SUFFIX);
 }
-export function parseLaunchId(launchId: string): { pid: string; startTime: number } {
-    let separator = launchId.lastIndexOf("-");
-    return {
-        pid: launchId.slice(0, separator),
-        startTime: parseInt(launchId.slice(separator + 1)) || 0,
-    };
+function getRecordPath(folder: string, pid: string, startTime: number): string {
+    return path.join(folder, PROCESS_FOLDER, getProcessFileName(pid, startTime) + RECORD_SUFFIX);
 }
 
 /** The shell a screen's output is piped through: appends every line, and empties the file once it grows past the limit. */
@@ -101,16 +92,16 @@ while true; do
 done`;
 }
 
-/** The record of whatever process is currently running in a screen, if we have one. */
-export async function findRunningRecord(folder: string, screenName: string): Promise<ProcessRecord | undefined> {
-    return (await listProcessRecords()).find(x =>
-        x.folder === folder && x.screenName === screenName && x.deadTime === undefined
-    );
+/** One process's record, read straight from the file its pid and start time name. */
+export async function readProcessRecord(folder: string, pid: string, startTime: number): Promise<ProcessRecord | undefined> {
+    let file = getRecordPath(folder, pid, startTime);
+    if (!await fsExistsAsync(file)) return undefined;
+    return JSON.parse(await fs.promises.readFile(file, "utf8")) as ProcessRecord;
 }
 
 export async function writeProcessRecord(record: ProcessRecord): Promise<void> {
     await fs.promises.mkdir(path.join(record.folder, PROCESS_FOLDER), { recursive: true });
-    await fs.promises.writeFile(getRecordPath(record.folder, record.launchId), JSON.stringify(record));
+    await fs.promises.writeFile(getRecordPath(record.folder, record.pid, record.startTime), JSON.stringify(record));
 }
 
 async function readFolderRecords(folder: string): Promise<ProcessRecord[]> {
@@ -147,10 +138,10 @@ export async function listProcessRecords(): Promise<ProcessRecord[]> {
 
 async function removeProcessRecord(record: ProcessRecord): Promise<void> {
     let files = [
-        getRecordPath(record.folder, record.launchId),
-        getProcessLogPath(record.folder, record.launchId),
-        getPipeScriptPath(record.folder, record.launchId),
-        getTailScriptPath(record.folder, record.launchId),
+        getRecordPath(record.folder, record.pid, record.startTime),
+        getProcessLogPath(record.folder, record.pid, record.startTime),
+        getPipeScriptPath(record.folder, record.pid, record.startTime),
+        getTailScriptPath(record.folder, record.pid, record.startTime),
     ];
     for (let file of files) {
         try {
@@ -169,11 +160,11 @@ export async function syncProcessRecords(config: {
     for (let record of await listProcessRecords()) {
         if (record.deadTime === undefined && !await config.isAlive(record)) {
             record.deadTime = config.now;
-            console.log(`Process ${record.launchId} (${record.screenName}) is no longer running, marking it dead. Its log is deleted at ${new Date(record.deadTime + DEAD_PROCESS_RETENTION).toISOString()}`);
+            console.log(`Process ${record.pid} (${record.screenName}) is no longer running, marking it dead. Its log is deleted at ${new Date(record.deadTime + DEAD_PROCESS_RETENTION).toISOString()}`);
             await writeProcessRecord(record);
         }
         if (record.deadTime !== undefined && config.now - record.deadTime > DEAD_PROCESS_RETENTION) {
-            console.log(`Deleting the log of process ${record.launchId} (${record.screenName}), dead since ${new Date(record.deadTime).toISOString()}`);
+            console.log(`Deleting the log of process ${record.pid} (${record.screenName}), dead since ${new Date(record.deadTime).toISOString()}`);
             await removeProcessRecord(record);
         }
     }

package/src/deployManager/processManager.ts

@@ -2,7 +2,7 @@ import os from "os";
 import fs from "fs";
 import path from "path";
 import { spawn, ChildProcess } from "child_process";
-import { lazy } from "socket-function/src/caching";
+import { cache, lazy } from "socket-function/src/caching";
 import { measureWrap } from "socket-function/src/profiling/measure";
 import { delay, runInSerial } from "socket-function/src/batching";
 import { timeInSecond } from "socket-function/src/misc";
@@ -13,7 +13,7 @@ import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
 import { fsExistsAsync } from "../fs";
 import { PromiseObj } from "../promise";
 import { SERVICE_FOLDER, SERVICE_NODE_FILE_NAME } from "./machineSchema";
-import { ProcessRecord, createLaunchId, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, listProcessRecords, findRunningRecord, writeProcessRecord } from "./processLogs";
+import { ProcessRecord, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, readProcessRecord, writeProcessRecord } from "./processLogs";
 import { setPreciseTimeout } from "../misc";
 
 // Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
@@ -38,25 +38,29 @@ export function getFutureScreenName(canonicalScreenName: string): string {
 
 
 // Everything that touches a service's screens - the resync loop, and the timer that retires the old process at its kill time - runs under this, keyed by the canonical screen name. They fire at similar times by design, and a takeover is a rename dance that must not interleave with another one.
-const screenLocks = new Map<string, Promise<unknown>>();
-export function withScreenLock<T>(canonicalScreenName: string, run: () => Promise<T>): Promise<T> {
-    let previous = screenLocks.get(canonicalScreenName) || Promise.resolve();
-    let next = previous.then(run, run);
-    // Failures release the lock rather than poisoning every later operation on this screen
-    screenLocks.set(canonicalScreenName, next.then(() => { }, () => { }));
-    return next;
+const getScreenLock = cache((canonicalScreenName: string) => runInSerial(async (run: () => Promise<unknown>) => await run()));
+export async function withScreenLock<T>(canonicalScreenName: string, run: () => Promise<T>): Promise<T> {
+    return await getScreenLock(canonicalScreenName)(run) as T;
 }
 
-/** Whether a session exists, without listing every session and its panes - the takeover already knows the exact names it cares about. */
-async function screenExists(screenName: string): Promise<boolean> {
+/** The pid of a screen's pane, or undefined when there is no such session. Asks about the one screen instead of listing every session on the machine. */
+export async function getScreenPanePid(screenName: string): Promise<string | undefined> {
     let prefix = getTmuxPrefix();
+    let output: string;
     try {
-        await runPromise(`${prefix}tmux has-session -t ${screenName}`, { quiet: true });
-        return true;
+        output = await runPromise(`${prefix}tmux list-panes -t ${screenName} -F "#{pane_pid}"`, { quiet: true });
     } catch {
-        // has-session exits non-zero for "no such session", which is an answer, not a failure
-        return false;
+        // Exits non-zero for "no such session", which is an answer, not a failure
+        return undefined;
     }
+    return output.trim().split("\n")[0].trim() || undefined;
+}
+
+/** Whether a screen exists AND has a process running in it - the only sense in which a service is up. */
+export async function isScreenUp(screenName: string): Promise<boolean> {
+    let panePid = await getScreenPanePid(screenName);
+    if (!panePid) return false;
+    return await isScreenRunningProcess(panePid);
 }
 
 /** The node the process in this folder registered as. Read it BEFORE starting a replacement in the same folder - once the replacement starts, the file is the replacement's. */
@@ -215,104 +219,76 @@ export const runScreenCommand = measureWrap(async function runScreenCommand(conf
     command: string;
     // Defaults to the folder derived from screenName; a future screen passes its canonical screen's folder
     folder?: string;
-    // Identifies the process this launch creates, so its log and its configuration are stored against it
-    record: Omit<ProcessRecord, "launchId" | "folder" | "screenName" | "startTime">;
-    // The screen this launch replaces, and when that one's overlap runs out. The timer is set the instant the command is sent, so the overlap is measured from the new process actually starting.
-    takeover?: { canonicalScreenName: string; killTime: number };
+    // Identifies the process this start creates, so its log and its configuration are stored against it
+    record: Omit<ProcessRecord, "pid" | "startTime" | "folder" | "screenName">;
 }): Promise<string> {
     let prefix = getTmuxPrefix();
     let screenName = config.screenName;
 
-    // A launch is ALWAYS a brand new session. Callers launch into the "-future" name and rename it over the canonical one once the new process is up, so starting a replacement never waits on the old process going away. Nothing is killed here.
-    // A leftover session under this name is from a launch that died before its takeover, so it is dead weight rather than something we are replacing.
-    if ((await getScreenState(false)).some(x => x.screenName === screenName)) {
-        console.log(red(`Removing the leftover session ${screenName} from a launch that never completed`));
+    // Always a brand new session, so the pane is always new - and the pane's pid is what makes this a distinct process with its own log. A session already under this name is debris from a start that never got taken over.
+    if (await getScreenPanePid(screenName)) {
+        console.log(red(`Removing the leftover session ${screenName} from a start that never completed`));
         await runPromise(`${prefix}tmux kill-session -t ${screenName}`);
     }
     await runPromise(`${prefix}tmux new -s ${screenName} -d`);
 
     let folder = config.folder || os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
 
-    // Read the outgoing process's node id onto its own record BEFORE this launch overwrites the file. Whoever takes that process down then has everything it needs on the record, instead of us having to remember not to touch a file that stopped being its own.
-    let outgoingNodeId = await readServiceNodeId(folder);
-    if (outgoingNodeId) {
-        for (let record of await listProcessRecords()) {
-            if (record.folder !== folder || record.deadTime !== undefined || record.nodeId) continue;
-            await writeProcessRecord({ ...record, nodeId: outgoingNodeId });
-        }
-    }
-
-    // The pane we are about to run the command in identifies the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream has to trust our bookkeeping about what is still running. The session was just created, so this pid is always new.
-    let panePid = (await getScreenState(false)).find(x => x.screenName === screenName)?.pid;
-    if (!panePid) {
+    // The pane we are about to run the command in names the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream trusts our bookkeeping about what is still running.
+    let pid = await getScreenPanePid(screenName);
+    if (!pid) {
         throw new Error(`Screen ${screenName} does not exist after creating it, so there is no process to run the command in`);
     }
-    let startTime = await getProcessStartTime(panePid);
-    let launchId = createLaunchId(panePid, startTime);
-    await writeProcessRecord({
-        ...config.record,
-        launchId,
-        folder,
-        screenName,
-        pid: parseInt(panePid) || undefined,
-        startTime,
-    });
+    let startTime = await getProcessStartTime(pid);
+    await writeProcessRecord({ ...config.record, pid, startTime, folder, screenName });
+    // The pipe is attached before the command runs, so the log holds everything the process ever printed
+    await setupPipePane({ screenName, folder, pid, startTime });
     await runPromise(`${prefix}tmux send-keys -t ${screenName} 'cd ${folder}git' Enter`);
     let command = `#!/bin/bash
 ${config.command}
 `;
     await fs.promises.writeFile(folder + "command.sh", command);
+    // Sending the command is the LAST thing this does, so the caller's next line runs the instant the process starts
     await runPromise(`${prefix}tmux send-keys -t ${screenName} 'bash ../command.sh' Enter`);
-    // The command is running as of this line, so the overlap starts counting here - nothing between this and the timer.
-    if (config.takeover) {
-        scheduleTakeover({
-            canonicalScreenName: config.takeover.canonicalScreenName,
-            folder,
-            killTime: config.takeover.killTime,
-        });
-    }
-
-    await setupPipePane({ screenName, folder, launchId });
-    return launchId;
+    return pid;
 });
 
-/** Links an already-running screen to a process record, creating one when this process has never been recorded - it was launched before we restarted, or before process records existed at all. Idempotent: a process we already know about is left alone, so the pipe is only attached the first time we adopt it.
+/** Links an already-running screen to a process record, creating one when this process has never been recorded - it was started before we restarted, or before process records existed at all. Idempotent: a process we already know about is left alone, so the pipe is only attached the first time we adopt it.
  *
- * The screen identifies itself (pane pid + the start time the OS reports for it), so adoption is just naming what is already there - there is nothing to reconcile and nothing to guess. */
+ * The screen names the process itself (pane pid + the start time the OS reports for it), so adoption is just writing down what is already there. */
 export async function ensureProcessRecord(config: {
     screenName: string;
     folder: string;
     panePid: string;
-    record: Omit<ProcessRecord, "launchId" | "folder" | "screenName" | "startTime" | "pid">;
+    record: Omit<ProcessRecord, "pid" | "startTime" | "folder" | "screenName">;
 }): Promise<void> {
-    let startTime = await getProcessStartTime(config.panePid);
+    let pid = config.panePid;
+    let startTime = await getProcessStartTime(pid);
     if (!startTime) return;
-    let launchId = createLaunchId(config.panePid, startTime);
-    let existing = (await listProcessRecords()).find(x => x.launchId === launchId && x.folder === config.folder);
+    let existing = await readProcessRecord(config.folder, pid, startTime);
     if (existing) {
         // A takeover renames the session, so the record follows the name it now runs under
         if (existing.screenName === config.screenName) return;
         await writeProcessRecord({ ...existing, screenName: config.screenName });
         return;
     }
-    console.log(`Adopting already-running process ${launchId} on screen ${config.screenName}, which has no record yet`);
+    console.log(`Adopting already-running process ${pid} on screen ${config.screenName}, which has no record yet`);
     await writeProcessRecord({
         ...config.record,
-        launchId,
+        pid,
+        startTime,
         folder: config.folder,
         screenName: config.screenName,
-        pid: parseInt(config.panePid) || undefined,
-        startTime,
     });
     // Its output was going to whatever the previous incarnation pointed at (or nowhere), so point it at its own log from here on
-    await setupPipePane({ screenName: config.screenName, folder: config.folder, launchId });
+    await setupPipePane({ screenName: config.screenName, folder: config.folder, pid, startTime });
 }
 
-// Points the screen's pipe-pane at this launch's own log file.
-async function setupPipePane(config: { screenName: string; folder: string; launchId: string }) {
+// Points the screen's pipe-pane at this process's own log file.
+async function setupPipePane(config: { screenName: string; folder: string; pid: string; startTime: number }) {
     let prefix = getTmuxPrefix();
-    let logPath = getProcessLogPath(config.folder, config.launchId);
-    let pipeScript = getPipeScriptPath(config.folder, config.launchId);
+    let logPath = getProcessLogPath(config.folder, config.pid, config.startTime);
+    let pipeScript = getPipeScriptPath(config.folder, config.pid, config.startTime);
     await fs.promises.mkdir(path.dirname(logPath), { recursive: true });
     await fs.promises.writeFile(pipeScript, getLogPipeScript(logPath));
     await runPromise(`chmod +x ${pipeScript}`);
@@ -320,42 +296,69 @@ async function setupPipePane(config: { screenName: string; folder: string; launc
 }
 
 /** Retires whatever is running under the canonical name and renames the future screen onto it. Everything it needs is derived from the two names, so it costs two `has-session` checks - the kill time is when this runs, not when a resync gets around to it. Returns whether there was a future screen to take over. */
-export async function takeoverScreen(config: { canonicalScreenName: string; folder: string }): Promise<boolean> {
-    let { canonicalScreenName, folder } = config;
-    return await withScreenLock(canonicalScreenName, async () => {
-        let futureScreenName = getFutureScreenName(canonicalScreenName);
-        if (!await screenExists(futureScreenName)) return false;
-        let prefix = getTmuxPrefix();
-        if (await screenExists(canonicalScreenName)) {
-            let retiringScreenName = getRetiringScreenName(canonicalScreenName, Date.now());
-            // Its record carries the node it registered as, taken before the replacement started - the file in the folder belongs to the replacement now, so it is left alone
-            let outgoing = await findRunningRecord(folder, canonicalScreenName);
-            await runPromise(`${prefix}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
-            if (outgoing) {
-                await writeProcessRecord({ ...outgoing, screenName: retiringScreenName });
-                if (outgoing.nodeId) {
-                    void removeServiceNode({ folder, nodeId: outgoing.nodeId, ownsFile: false });
-                }
-            }
-            // Detached: the canonical name is already free, so nothing waits on the old process winding down
-            void killScreen({ screenName: retiringScreenName });
-        }
-        console.log(green(`Renaming future screen ${futureScreenName} to ${canonicalScreenName} (takeover complete)`));
-        await runPromise(`${prefix}tmux rename-session -t ${futureScreenName} ${canonicalScreenName}`);
-        let incoming = await findRunningRecord(folder, futureScreenName);
-        if (incoming) {
-            await writeProcessRecord({ ...incoming, screenName: canonicalScreenName });
+export async function takeoverScreen(config: {
+    canonicalScreenName: string;
+    folder: string;
+    /** The node the version being replaced registered as, read before the replacement overwrote the folder's file. */
+    replacedNodeId?: string;
+}): Promise<boolean> {
+    return await withScreenLock(config.canonicalScreenName, () => takeoverScreenLocked(config));
+}
+
+/** The takeover itself. Callers already holding this screen's lock use it directly - taking the lock again from inside it would wait on themselves. */
+export async function takeoverScreenLocked(config: {
+    canonicalScreenName: string;
+    folder: string;
+    replacedNodeId?: string;
+}): Promise<boolean> {
+    let { canonicalScreenName, folder, replacedNodeId } = config;
+    let futureScreenName = getFutureScreenName(canonicalScreenName);
+    let futurePid = await getScreenPanePid(futureScreenName);
+    if (!futurePid) return false;
+    let prefix = getTmuxPrefix();
+
+    let outgoingPid = await getScreenPanePid(canonicalScreenName);
+    if (outgoingPid) {
+        // Renamed out of the way immediately, so the canonical name is free at once and the old process winds down on its own
+        let retiringScreenName = getRetiringScreenName(canonicalScreenName, Date.now());
+        await runPromise(`${prefix}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
+        await renameProcessScreen({ folder, panePid: outgoingPid, screenName: retiringScreenName });
+        if (replacedNodeId) {
+            // The file in the folder belongs to the replacement now, so only the registration goes
+            void removeServiceNode({ folder, nodeId: replacedNodeId, ownsFile: false });
         }
-        return true;
-    });
+        retiringScreens.add(retiringScreenName);
+        void killScreen({ screenName: retiringScreenName }).finally(() => {
+            retiringScreens.delete(retiringScreenName);
+        });
+    }
+    console.log(green(`Renaming ${futureScreenName} to ${canonicalScreenName} (takeover complete)`));
+    await runPromise(`${prefix}tmux rename-session -t ${futureScreenName} ${canonicalScreenName}`);
+    await renameProcessScreen({ folder, panePid: futurePid, screenName: canonicalScreenName });
+    return true;
+}
+
+/** Points a process's record at the session name it now runs under. The pid is the key, so this reads exactly one file. */
+async function renameProcessScreen(config: { folder: string; panePid: string; screenName: string }): Promise<void> {
+    let startTime = await getProcessStartTime(config.panePid);
+    if (!startTime) return;
+    let record = await readProcessRecord(config.folder, config.panePid, startTime);
+    if (!record || record.screenName === config.screenName) return;
+    await writeProcessRecord({ ...record, screenName: config.screenName });
 }
 
 // canonical screen name -> the takeover we already have scheduled for it
 const scheduledTakeovers = new Map<string, { time: number; cancel: () => void }>();
 
 /** Schedules the takeover for the moment the old process's overlap runs out. The resync loop still does the same takeover whenever it notices the time has passed, which is what recovers this if we restart and lose the timer. */
-export function scheduleTakeover(config: { canonicalScreenName: string; folder: string; killTime: number }): void {
-    let { canonicalScreenName, folder, killTime } = config;
+export function scheduleTakeover(config: {
+    canonicalScreenName: string;
+    folder: string;
+    killTime: number;
+    /** The node the version being replaced registered as, read before the replacement overwrote the folder's file. */
+    replacedNodeId?: string;
+}): void {
+    let { canonicalScreenName, folder, killTime, replacedNodeId } = config;
     let existing = scheduledTakeovers.get(canonicalScreenName);
     if (existing?.time === killTime) return;
     existing?.cancel();
@@ -364,12 +367,25 @@ export function scheduleTakeover(config: { canonicalScreenName: string; folder:
         time: killTime,
         callback: () => {
             scheduledTakeovers.delete(canonicalScreenName);
-            void takeoverScreen({ canonicalScreenName, folder });
+            void takeoverScreen({ canonicalScreenName, folder, replacedNodeId });
         },
     });
     scheduledTakeovers.set(canonicalScreenName, { time: killTime, cancel });
 }
 
+// The screens whose shutdown is already running. Nothing else should start a second one, but a screen left behind by a restart is NOT in here, so the unused-screen sweep still cleans it up.
+const retiringScreens = new Set<string>();
+export function isRetiring(screenName: string): boolean {
+    return retiringScreens.has(screenName);
+}
+
+/** Ends a session at once, with no notice - for a screen that should not exist at all, so there is nothing to wind down gracefully. Finding one is always a surprise, so the reason it was not supposed to be there is logged. */
+export async function killScreenNow(screenName: string, reason: string): Promise<void> {
+    if (!await getScreenPanePid(screenName)) return;
+    console.log(red(`Unexpected screen ${screenName}: ${reason}. Killing it immediately.`));
+    await runPromise(`${getTmuxPrefix()}tmux kill-session -t ${screenName}`);
+}
+
 // Kills a screen and nothing else. Which node the process registered as, and whether that registration should be dropped, is the deploy logic's business - it reads the node id off the process's record, which was taken before any replacement could overwrite the file.
 export const killScreen = measureWrap(async function killScreen(config: {
     screenName: string;
@@ -385,10 +401,11 @@ export const killScreen = measureWrap(async function killScreen(config: {
 /** Streams one process's log: everything already in it, then everything appended after. One process, one file - nothing here has to reason about which process a byte came from. */
 export async function streamProcessOutput(config: {
     folder: string;
-    launchId: string;
+    pid: string;
+    startTime: number;
     onData: (data: string) => Promise<void>;
 }) {
-    let logPath = getProcessLogPath(config.folder, config.launchId);
+    let logPath = getProcessLogPath(config.folder, config.pid, config.startTime);
     let serialOnData = runInSerial(config.onData);
     let stopped = false;
     let childProcess: ChildProcess | undefined;
@@ -407,14 +424,14 @@ export async function streamProcessOutput(config: {
     const onDataWrapped = async (data: string) => {
         pendingDataCalls++;
         if (pendingDataCalls > MAX_PENDING_CALLS) {
-            console.error(`Too many queued onData calls for ${config.launchId}, stopping stream.`);
+            console.error(`Too many queued onData calls for ${config.pid}, stopping stream.`);
             await stop();
             return;
         }
         try {
             await serialOnData(data);
         } catch (e: any) {
-            console.log(`Callback for stream output ${config.launchId} failed. It probably just disconnected, almost certainly not an error: ${e.message}`);
+            console.log(`Callback for stream output ${config.pid} failed. It probably just disconnected, almost certainly not an error: ${e.message}`);
             await stop();
         } finally {
             pendingDataCalls--;
@@ -422,7 +439,7 @@ export async function streamProcessOutput(config: {
     };
 
     try {
-        let tailScript = getTailScriptPath(config.folder, config.launchId);
+        let tailScript = getTailScriptPath(config.folder, config.pid, config.startTime);
         await fs.promises.writeFile(tailScript, getLogTailScript(logPath));
         await runPromise(`chmod +x ${tailScript}`);