querysub 0.571.0 → 0.573.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.571.0",
+  "version": "0.573.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/deployManager/machineApplyMainCode.ts

@@ -10,7 +10,7 @@ import { getExternalIP } from "socket-function/src/networking";
 import { errorToUndefined, errorToUndefinedSilent } from "../errors";
 import { getDomain } from "../config";
 import { formatTime } from "socket-function/src/formatting/format";
-import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { sort, timeInMinute, timeInSecond, watchSlowPromise } from "socket-function/src/misc";
 import { isDefined, setPreciseTimeout } from "../misc";
 import { logLoadTime } from "../logModuleLoadTimes";
 import { delay, retryFunctional, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
@@ -26,7 +26,7 @@ import { PromiseObj } from "../promise";
 import path from "path";
 import { fsExistsAsync } from "../fs";
 import { ALIVE_WINDOW_FOREVER, ParametersTimelineEntry, syncParametersTimelineFiles } from "./parametersTimeline";
-import { getScreenName, getFutureScreenName, isRetiring, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, getScreenPanePid, ensureProcessRecord, runScreenCommand, killScreen, readServiceNodeId, removeServiceNode, takeoverScreen, scheduleTakeover, withScreenLock } from "./processManager";
+import { getScreenName, getFutureScreenName, isRetiring, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, getScreenPanePid, ensureProcessRecord, runScreenCommand, killScreen, readServiceNodeId, removeServiceNode, takeoverScreenLocked, scheduleTakeover, killScreenNow, withScreenLock } from "./processManager";
 import { ProcessRecord, syncProcessRecords, listProcessRecords } from "./processLogs";
 
 
@@ -221,87 +221,68 @@ let launchesPerService = new Map<string, number>();
 let lastLaunchedTimePerService = new Map<string, number>();
 
 const LAUNCH_VERIFY_DELAY = 2000;
+// How far a re-read start time may drift from the recorded one and still be the same process. Well under any interval a pid could be reused in, and wide enough for sources that only resolve to the second.
+const START_TIME_TOLERANCE = timeInSecond * 5;
 
-/** One tmux session that should exist right now, and the configuration it should be running. The whole resync is derived from this list: converge each of these, kill everything else. */
-type DesiredScreen = {
-    /** The session that should exist. For an incoming version that is the "-future" name, until it takes the canonical name over. */
-    screenName: string;
-    /** The slot this belongs to - its own name, or the one it is replacing */
-    canonicalScreenName: string;
-    /** The instance folder, which is always the canonical screen's folder (an incoming version runs out of the same one) */
-    folder: string;
-    serviceId: string;
-    serviceKey: string;
-    index: number;
-    machineId: string;
-    /** The parameters it runs, with template variables already resolved */
-    parameters: ServiceParameters;
-    aliveWindow: [number, number];
-    /** The version being replaced during an overlap: it must keep running and must not be touched, because the folder already belongs to its replacement. */
-    outgoing?: boolean;
-    /** On an incoming version, when it takes the canonical name over. Absent means take over as soon as it is up. */
-    takeoverAt?: number;
-};
-
-/** Every screen that should exist on this machine, worked out purely from the configs and the clock. */
-function getDesiredScreens(configs: ServiceConfig[], machineId: string, root: string): DesiredScreen[] {
-    let desired: DesiredScreen[] = [];
-    let now = Date.now();
+/** Every instance of every service that belongs to this machine, with its config already resolved for that instance - the command has this entry's template variables substituted, and a version with no entry at this index is marked as deploying nothing here. */
+function getMachineInstances(configs: ServiceConfig[], machineId: string): { config: ServiceConfig; index: number }[] {
+    let instances: { config: ServiceConfig; index: number }[] = [];
     for (let config of configs) {
-        let releaseTime = config.parameters.releaseTime || 0;
-        // The overlap length always comes from the NEWEST parameters, even though it governs how long the OLD instances outlive the release
-        let overlapTime = config.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME;
-        let killTime = releaseTime + overlapTime;
-        // Only in flight while there is an old version to keep alive, and only until its overlap runs out
-        let releaseInFlight = !!releaseTime && !!config.oldParameters && now < killTime;
-
-        const add = (parameters: ServiceParameters, extra: Partial<DesiredScreen>) => {
-            if (!parameters.deploy) return;
-            let targets = getMachineTargets(parameters).filter(target => target.machineId === machineId);
-            for (let index = 0; index < targets.length; index++) {
-                let canonicalScreenName = getScreenName({ serviceKey: parameters.key, index });
-                desired.push({
-                    screenName: canonicalScreenName,
-                    canonicalScreenName,
-                    folder: root + canonicalScreenName + "/",
-                    serviceId: config.serviceId,
-                    serviceKey: parameters.key,
-                    index,
-                    machineId,
-                    // Each instance runs the command with its own entry's variables resolved, so the parameters written to disk and compared for restarts carry the resolved command
-                    parameters: { ...parameters, command: applyCommandTemplate(parameters.command, targets[index].variables) },
-                    aliveWindow: [parameters.releaseTime || 0, ALIVE_WINDOW_FOREVER],
-                    ...extra,
-                });
-            }
-        };
-
-        if (releaseInFlight && config.oldParameters) {
-            // The old version keeps running under the canonical name until its overlap runs out
-            add(config.oldParameters, { outgoing: true, aliveWindow: [config.oldParameters.releaseTime || 0, killTime] });
-            // The new version runs alongside it from the release, and takes the canonical name at killTime
-            if (now >= releaseTime) {
-                add(config.parameters, { takeoverAt: killTime, aliveWindow: [releaseTime, ALIVE_WINDOW_FOREVER] });
-            }
-        } else {
-            add(getLiveServiceParameters(config), {});
+        // A release can add or remove instances, so the count is whichever version asks for the most
+        let count = 0;
+        for (let parameters of [config.parameters, config.oldParameters]) {
+            if (!parameters) continue;
+            count = Math.max(count, getMachineTargets(parameters).filter(x => x.machineId === machineId).length);
+        }
+        for (let index = 0; index < count; index++) {
+            const resolve = (parameters: ServiceParameters) => {
+                let target = getMachineTargets(parameters).filter(x => x.machineId === machineId)[index];
+                return {
+                    ...parameters,
+                    command: target && applyCommandTemplate(parameters.command, target.variables) || parameters.command,
+                    deploy: !!parameters.deploy && !!target,
+                };
+            };
+            instances.push({
+                index,
+                config: {
+                    ...config,
+                    parameters: resolve(config.parameters),
+                    oldParameters: config.oldParameters && resolve(config.oldParameters),
+                },
+            });
         }
     }
-    // An incoming version lives under the future name until it takes over
-    for (let screen of desired) {
-        if (screen.takeoverAt === undefined) continue;
-        screen.screenName = getFutureScreenName(screen.canonicalScreenName);
-    }
-    return desired;
+    return instances;
 }
 
-/** Puts one screen into the state its configuration describes: the folder synced, the process running what it should be, and the records and timeline saying so. Everything about one screen happens under its lock, so a resync and a scheduled takeover can never interleave. */
+/** Puts one instance of a service into the state its configuration describes: the folder synced, the process running what it should be, and the records and timeline saying so. Returns the screens it is using, so anything nobody claimed can be killed. Everything about one screen happens under its lock, so a resync and a scheduled takeover can never interleave. */
 async function ensureScreen(config: {
-    desired: DesiredScreen;
+    config: ServiceConfig;
+    index: number;
     machineInfo: MachineInfo;
-}): Promise<void> {
-    let { desired, machineInfo } = config;
-    let { screenName, canonicalScreenName, folder, parameters, serviceId } = desired;
+}): Promise<string[]> {
+    let { machineInfo, index } = config;
+    let machineId = machineInfo.machineId;
+    let serviceId = config.config.serviceId;
+    let serviceConfig = config.config;
+
+    let releaseTime = serviceConfig.parameters.releaseTime || 0;
+    // The overlap length always comes from the NEWEST parameters, even though it governs how long the OLD instances outlive the release
+    let killTime = releaseTime + (serviceConfig.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME);
+    // Only in flight while there is an old version to keep alive, and only until its overlap runs out
+    let releaseInFlight = !!releaseTime && !!serviceConfig.oldParameters && Date.now() < killTime;
+
+    // What should be running under the canonical name now, and the version waiting to take it over
+    let current: ServiceParameters | undefined = releaseInFlight && serviceConfig.oldParameters || getLiveServiceParameters(serviceConfig);
+    if (!current?.deploy) current = undefined;
+    let incoming: ServiceParameters | undefined = releaseInFlight && Date.now() >= releaseTime && serviceConfig.parameters || undefined;
+    if (!incoming?.deploy) incoming = undefined;
+    let canonicalScreenName = getScreenName({ serviceKey: (current || incoming || serviceConfig.parameters).key, index });
+    let futureScreenName = getFutureScreenName(canonicalScreenName);
+    let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
+    // Returned even when the work below throws, so a transient failure never makes the sweep kill a screen we still want
+    let usedScreenNames = [canonicalScreenName, futureScreenName];
 
     let launchCount = launchesPerService.get(serviceId) || 0;
     let lastLaunchedTime = lastLaunchedTimePerService.get(serviceId) || 0;
@@ -313,47 +294,70 @@ async function ensureScreen(config: {
     };
 
     try {
-        await withScreenLock(canonicalScreenName, async () => {
-            // Asked of tmux here, not handed in: this decides everything from the names it was given
-            let panePid = await getScreenPanePid(screenName);
-            let isUp = !!panePid && await isScreenRunningProcess(panePid);
-
-            if (desired.outgoing) {
-                // Its replacement owns the folder now, so the only thing to do is keep its record straight and let it know when it goes down
-                if (panePid && isUp) {
-                    await ensureProcessRecord({ screenName, folder, panePid, record: desired });
-                    void (async () => {
-                        let nodeId = await readServiceNodeId(folder);
-                        if (nodeId) {
-                            await notifyNodeShutdown(screenName, nodeId, desired.aliveWindow[1]);
-                        }
-                    })();
-                }
-                return;
+        await watchSlowPromise(`ensureScreen|${canonicalScreenName}`, withScreenLock(canonicalScreenName, async () => {
+            /*  Three facts, each moving ? -> known -> correct as we go:
+                    EXISTS   - which of the two screens are there
+                    VERSION  - which parameters they were started with
+                    RUNNING  - whether a process is actually alive in them
+                Every step below says where those three stand, so it is always clear what we are still assuming.
+                START:  EXISTS ?  VERSION ?  RUNNING ?
+            */
+            let parameters = incoming || current;
+            let screenName = incoming && futureScreenName || canonicalScreenName;
+            let takeoverAt = incoming && killTime || undefined;
+
+            // A screen we do not expect holds something we never asked for, so it goes at once - nothing here is worth winding down. Each name is judged on its own: a release that turns deploy on has an incoming version and nothing that should be running under the canonical name yet.
+            // AFTER: EXISTS correct in the "nothing extra" sense. A screen that SHOULD exist may still be missing, which is RUNNING's problem below.
+            if (!incoming) {
+                await killScreenNow(futureScreenName, "there is no release in flight, so nothing should be waiting to take over");
             }
-
-            let gitFolder = folder + "git/";
-            await fs.promises.mkdir(gitFolder, { recursive: true });
-            if (parameters.gitRef) {
-                let prevGitRef = "";
-                try {
-                    prevGitRef = await getGitRefLive(gitFolder);
-                } catch { }
-                await ensureGitSynced({ gitFolder, repoUrl: parameters.repoUrl, gitRef: parameters.gitRef });
-                let afterGitRef = await getGitRefLive(gitFolder);
-                // Reinstall when the ref changed OR node_modules is missing. A recovery re-clone can land on the same commit, so a ref-only check would leave the service with no node_modules.
-                let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
-                if (afterGitRef !== prevGitRef || nodeModulesMissing) {
-                    console.log(green(`Yarn installing for ${magenta(screenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
-                    await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
-                }
+            if (!current) {
+                await killScreenNow(canonicalScreenName, "nothing is supposed to be serving this name yet");
             }
+            if (!current && !incoming) return;
 
+            // Both screens run out of one folder, so its parameters.json is the ONE version they were last started from - it cannot tell the two screens apart.
+            // AFTER: VERSION known. Anything it does not match, including a version we no longer recognize, is handled by starting the right one further down - which keeps whatever is up serving until the replacement is verified, instead of killing it now and serving nothing while we sync and start.
             let parameterPath = folder + "parameters.json";
             let prevParameters = "";
             if (await fsExistsAsync(parameterPath)) {
                 prevParameters = await fs.promises.readFile(parameterPath, "utf8");
             }
+
+            // AFTER: RUNNING known for every name that can still have anything in it. A screen can exist with nothing alive in it, and one that should exist can be missing entirely - both read as not up, and both are fixed the same way. Without an incoming version the future name was just killed, so there is nothing left to ask about it.
+            let canonicalPid = await getScreenPanePid(canonicalScreenName);
+            let canonicalUp = !!canonicalPid && await isScreenRunningProcess(canonicalPid);
+            let futurePid = incoming && await getScreenPanePid(futureScreenName) || undefined;
+            let futureUp = !!futurePid && await isScreenRunningProcess(futurePid);
+
+            // SMOOTH PATH: this is what a deploy looks like from the canonical name's side. The version being replaced is already correct - it keeps running untouched until its overlap ends, so this name is settled and only the future one is left to converge. It needs its record kept straight, and to hear when it goes down.
+            // AFTER (canonical, during a release): EXISTS correct   VERSION correct   RUNNING correct
+            if (incoming && current && !canonicalUp) {
+                console.log(red(`Nothing is running as ${canonicalScreenName}, which should be serving until ${new Date(killTime).toISOString()} while its replacement starts`));
+            }
+            if (incoming && current && canonicalUp && canonicalPid) {
+                await ensureProcessRecord({
+                    screenName: canonicalScreenName,
+                    folder,
+                    panePid: canonicalPid,
+                    record: { serviceId, serviceKey: current.key, index, machineId, parameters: current },
+                });
+                void (async () => {
+                    let nodeId = await readServiceNodeId(folder);
+                    if (nodeId) {
+                        await notifyNodeShutdown(canonicalScreenName, nodeId, killTime);
+                    }
+                })();
+            }
+
+            // From here on this is about ONE screen - the future one during a release, the canonical one otherwise. Everything else is already correct.
+            // STATE: EXISTS correct   VERSION known   RUNNING known
+            if (!parameters) return;
+            let panePid = incoming && futurePid || canonicalPid;
+            let isUp = incoming && futureUp || canonicalUp;
+            let aliveWindow: [number, number] = [parameters.releaseTime || 0, ALIVE_WINDOW_FOREVER];
+            let record = { serviceId, serviceKey: parameters.key, index, machineId, parameters };
+
             let newParametersString = JSON.stringify(parameters);
 
             let nodePathId = folder + SERVICE_NODE_FILE_NAME;
@@ -361,28 +365,62 @@ async function ensureScreen(config: {
                 machineInfo.services[serviceId].nodeId = await fs.promises.readFile(nodePathId, "utf8");
             }
 
+            // Already running this exact version, so nothing needs starting - this is the state we are trying to reach
+            // AFTER: EXISTS correct   VERSION correct   RUNNING correct, and the takeover below settles which name it answers to
             if (isUp && panePid && sameRestartParameters(prevParameters, parameters)) {
                 if (prevParameters !== newParametersString) {
                     await fs.promises.writeFile(parameterPath, newParametersString);
                 }
-                // We may not have launched this one (we restarted, or it predates process records), so make sure it is recorded before leaving it alone
-                await ensureProcessRecord({ screenName, folder, panePid, record: desired });
-                await syncParametersTimelineFiles(folder, [{ pid: parseInt(panePid) || undefined, aliveWindow: desired.aliveWindow, parameters }]);
-                // A restart loses the timer that retires the version this one replaces, so it is put back every time we see it still waiting
-                if (desired.takeoverAt !== undefined) {
-                    scheduleTakeover({ canonicalScreenName, folder, killTime: desired.takeoverAt });
+                // We may not have started this one (we restarted, or it predates process records), so make sure it is recorded before leaving it alone
+                await ensureProcessRecord({ screenName, folder, panePid, record });
+                await syncParametersTimelineFiles(folder, [{ pid: parseInt(panePid) || undefined, aliveWindow, parameters }]);
+                if (takeoverAt !== undefined) {
+                    if (!canonicalUp) {
+                        // The version this one is waiting to replace is gone, so there is no overlap left to serve - take the name now instead of sitting under the future one
+                        console.log(red(`Nothing is running as ${canonicalScreenName}, so ${screenName} takes it over now instead of waiting until ${new Date(takeoverAt).toISOString()}`));
+                        await takeoverScreenLocked({ canonicalScreenName, folder });
+                    } else {
+                        // SMOOTH PATH: both versions are up, the old one serving and the new one waiting. A restart loses the timer that retires the old one, so it is put back every time we see it still waiting.
+                        scheduleTakeover({ canonicalScreenName, folder, killTime: takeoverAt });
+                    }
                 }
                 console.log(green(`Verified ${magenta(screenName)} is running`));
                 return;
             }
 
-            console.log(`Resyncing ${magenta(screenName)}, with ${newParametersString}, isUp = ${isUp}`);
+            // RUNNING is wrong - either nothing is alive, or what is alive is the wrong version. Starting is the only thing that fixes it.
+            if (!isUp) {
+                // Nothing should ever stop on its own: either it crashed, or it never came up
+                console.log(red(`Nothing is running as ${screenName}, starting it: ${newParametersString}`));
+            } else {
+                // SMOOTH PATH: the previous version is up and this replaces it with a new one, which is exactly what a deploy is
+                console.log(green(`Starting a new version as ${magenta(screenName)}, replacing the one running there: ${newParametersString}`));
+            }
+
+            // Only now does the folder have to hold this version, which is why the git and yarn work sits behind the check above rather than running on every resync
+            let gitFolder = folder + "git/";
+            await fs.promises.mkdir(gitFolder, { recursive: true });
+            if (parameters.gitRef) {
+                let prevGitRef = "";
+                try {
+                    prevGitRef = await getGitRefLive(gitFolder);
+                } catch { }
+                await ensureGitSynced({ gitFolder, repoUrl: parameters.repoUrl, gitRef: parameters.gitRef });
+                let afterGitRef = await getGitRefLive(gitFolder);
+                // Reinstall when the ref changed OR node_modules is missing. A recovery re-clone can land on the same commit, so a ref-only check would leave the service with no node_modules.
+                let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
+                if (afterGitRef !== prevGitRef || nodeModulesMissing) {
+                    console.log(green(`Yarn installing for ${magenta(screenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
+                    await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
+                }
+            }
+
             await fs.promises.writeFile(parameterPath, newParametersString);
             void recordLaunch({
                 serviceId,
-                serviceKey: desired.serviceKey,
+                serviceKey: parameters.key,
                 screenName: canonicalScreenName,
-                machineId: desired.machineId,
+                machineId,
                 reason: isUp && "update" || "crashed",
                 time: Date.now(),
             });
@@ -390,17 +428,17 @@ async function ensureScreen(config: {
             // The node the version being replaced registered as, read before the launch overwrites the file with the new process's
             let replacedNodeId = await readServiceNodeId(folder);
 
-            // Every launch goes into the future name and renames over the canonical one, releases and plain restarts alike: the replacement is up before anything touches the old process, and it always gets a brand new pane - which is what makes it a distinct process with its own log.
-            let launchScreenName = getFutureScreenName(canonicalScreenName);
+            // Always started under the future name and renamed over the canonical one, releases and plain restarts alike: it is up before anything touches what it replaces, and it always gets a brand new pane - which is what makes it a distinct process with its own log.
+            let launchScreenName = futureScreenName;
             let launchedPid = await runScreenCommand({
                 screenName: launchScreenName,
                 folder,
                 command: parameters.command,
-                record: desired,
+                record,
             });
-            // The command is running as of the line above, so the overlap is measured from here
-            if (desired.takeoverAt !== undefined) {
-                scheduleTakeover({ canonicalScreenName, folder, killTime: desired.takeoverAt, replacedNodeId });
+            // SMOOTH PATH: the command is running as of the line above, so the overlap the version it replaces gets is measured from here
+            if (takeoverAt !== undefined) {
+                scheduleTakeover({ canonicalScreenName, folder, killTime: takeoverAt, replacedNodeId });
             }
 
             await delay(LAUNCH_VERIFY_DELAY);
@@ -411,26 +449,28 @@ async function ensureScreen(config: {
                 throw new Error(`${logs}\n\nService ${magenta(launchScreenName)} is not running after starting. Trying again in ${formatTime(MACHINE_RESYNC_INTERVAL)}, or on next change. Last logs above.`);
             }
             console.log(green(`${magenta(launchScreenName)} is verified to be running (pid ${launchedPid})`));
-            if (desired.takeoverAt === undefined) {
-                // Nothing is holding the canonical name open for an overlap, so take it now
-                await takeoverScreen({ canonicalScreenName, folder, replacedNodeId });
+            // Take the canonical name now unless something is still serving under it - waiting out an overlap that nothing is serving would leave the name empty for no reason. SMOOTH PATH during a release is neither: the version being replaced is still up, so this one waits under the future name until its timer fires.
+            // AFTER: EXISTS correct   VERSION correct   RUNNING correct, under the canonical name, or under the future one with the version it is replacing still serving
+            if (takeoverAt === undefined || !canonicalUp) {
+                await takeoverScreenLocked({ canonicalScreenName, folder, replacedNodeId });
             }
-            await syncParametersTimelineFiles(folder, [{ pid: parseInt(launchedPanePid) || undefined, aliveWindow: desired.aliveWindow, parameters }]);
+            await syncParametersTimelineFiles(folder, [{ pid: parseInt(launchedPanePid) || undefined, aliveWindow, parameters }]);
 
             launchCount++;
             lastLaunchedTime = Date.now();
-        });
+        }));
     } catch (e: any) {
         // Counted on failure too, as a service that keeps crashing is only visible as a launch count climbing faster than everyone else's
         launchCount++;
         lastLaunchedTime = Date.now();
-        console.error(`Error resyncing ${magenta(screenName)} (${serviceId}): ${e.stack}`);
+        console.error(`Error resyncing ${magenta(canonicalScreenName)} (${serviceId}): ${e.stack}`);
         machineInfo.services[serviceId].errorFromLastRun = e.stack;
     }
     machineInfo.services[serviceId].totalTimesLaunched = launchCount;
     machineInfo.services[serviceId].lastLaunchedTime = lastLaunchedTime;
     lastLaunchedTimePerService.set(serviceId, lastLaunchedTime);
     launchesPerService.set(serviceId, launchCount);
+    return usedScreenNames;
 }
 
 function sameRestartParameters(prevParametersJSON: string, next: ServiceParameters): boolean {
@@ -459,27 +499,23 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
     await syncProcessRecords({
         isAlive: async record => {
             if (!running.has(record.pid)) return false;
-            return await getProcessStartTime(record.pid) === record.startTime;
+            // Compared with a tolerance rather than exactly: the sources we read a start time from only resolve to the second, and they do not all round the same way, so an exact match would call a live process dead
+            return Math.abs(await getProcessStartTime(record.pid) - record.startTime) <= START_TIME_TOLERANCE;
         },
         now: Date.now(),
     });
     let root = os.homedir() + "/" + SERVICE_FOLDER;
-    let desiredScreens = getDesiredScreens(relevantConfigs, machineId, root);
+    let instances = getMachineInstances(relevantConfigs, machineId);
 
-    // The fast-path results are memoized per resync, so a folder two desired screens share is only traversed once
+    // The fast-path results are memoized per resync, so a folder shared by two instances is only traversed once
     gitFastPathChecks.clear();
 
-    // Every screen converges concurrently. Each has its own folder, its own records and its own lock, and yarn install takes a network mutex, so there is nothing to serialize behind - one slow git sync or shutdown no longer delays everything after it.
-    await Promise.all(desiredScreens.map(desired => ensureScreen({ desired, machineInfo })));
+    // Every instance converges concurrently. Each has its own folder, its own records and its own lock, and yarn install takes a network mutex, so there is nothing to serialize behind - one slow git sync or shutdown no longer delays everything after it.
+    let usedScreenNames = new Set((await Promise.all(instances.map(instance => ensureScreen({ ...instance, machineInfo })))).flat());
 
-    // Both names count as wanted: a screen that took over during this pass answers to the canonical one now, and one that is about to take over still answers to its future name
-    let wantedNames = new Set<string>();
-    for (let desired of desiredScreens) {
-        wantedNames.add(desired.screenName);
-        wantedNames.add(desired.canonicalScreenName);
-    }
+    // Anything no instance claimed is not supposed to exist
     for (let { screenName } of screenState) {
-        if (wantedNames.has(screenName)) continue;
+        if (usedScreenNames.has(screenName)) continue;
         // Already being taken down. One left over from a restart is not, so it still gets cleaned up here.
         if (isRetiring(screenName)) continue;
         // Detached, so a screen that takes its full grace period to go down doesn't hold up the end of the resync
@@ -494,9 +530,6 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
         })();
     }
 
-    // The takeover is scheduled the moment the replacement starts, so this only exists to catch up after a restart loses those timers
-    scheduleReleaseResync(desiredScreens);
-
     await machineInfos.set(machineId, machineInfo);
     console.log(`${magenta(`Resynced ${relevantConfigs.length} services`)}:\n${JSON.stringify(machineInfo, null, 2)}`);
 
@@ -511,34 +544,6 @@ async function resyncServices() {
     }
 }
 
-// The regular resync poll is too coarse for a takeover boundary, so schedule an extra resync at the next one. It carries no logic of its own — the resync just re-evaluates the current state, which is what recovers a takeover whose timer we lost by restarting.
-let cancelReleaseResync: (() => void) | undefined;
-function scheduleReleaseResync(desiredScreens: DesiredScreen[]) {
-    let now = Date.now();
-    let next: number | undefined;
-    for (let desired of desiredScreens) {
-        let boundary = desired.takeoverAt;
-        if (boundary === undefined || boundary <= now) continue;
-        if (next === undefined || boundary < next) {
-            next = boundary;
-        }
-    }
-    if (cancelReleaseResync) {
-        cancelReleaseResync();
-        cancelReleaseResync = undefined;
-    }
-    if (next === undefined) return;
-    console.log(`Scheduling a resync for the next release boundary at ${new Date(next).toLocaleString()}`);
-    cancelReleaseResync = setPreciseTimeout({
-        time: next,
-        callback: () => {
-            cancelReleaseResync = undefined;
-            void resyncServices();
-        },
-    });
-}
-
-
 async function isPIDRunning(pid: string) {
     try {
         // Use process.kill with signal 0 to test if process exists without actually killing it

package/src/deployManager/processLogs.ts

@@ -99,6 +99,11 @@ export async function readProcessRecord(folder: string, pid: string, startTime:
     return JSON.parse(await fs.promises.readFile(file, "utf8")) as ProcessRecord;
 }
 
+/** The live record for a pid in this folder. The pid is what identifies a process, so a start time that reads back a second off - the sources only resolve to the second - must not make us miss the record and write a duplicate. */
+export async function findProcessRecordByPid(folder: string, pid: string): Promise<ProcessRecord | undefined> {
+    return (await readFolderRecords(folder)).find(x => x.pid === pid && x.deadTime === undefined);
+}
+
 export async function writeProcessRecord(record: ProcessRecord): Promise<void> {
     await fs.promises.mkdir(path.join(record.folder, PROCESS_FOLDER), { recursive: true });
     await fs.promises.writeFile(getRecordPath(record.folder, record.pid, record.startTime), JSON.stringify(record));

package/src/deployManager/processManager.ts

@@ -13,7 +13,7 @@ import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
 import { fsExistsAsync } from "../fs";
 import { PromiseObj } from "../promise";
 import { SERVICE_FOLDER, SERVICE_NODE_FILE_NAME } from "./machineSchema";
-import { ProcessRecord, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, readProcessRecord, writeProcessRecord } from "./processLogs";
+import { ProcessRecord, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, findProcessRecordByPid, writeProcessRecord } from "./processLogs";
 import { setPreciseTimeout } from "../misc";
 
 // Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
@@ -163,11 +163,23 @@ const getAllParentPids = measureWrap(async function getAllParentPids(): Promise<
     let obj = textTableLineToObj(table) as { PID: string; PPID: string }[];
     return new Set(obj.map(x => x.PPID));
 });
-/** The start time the OS reports for a pid, as an epoch time in milliseconds like every other time we store. Asked of the machine rather than remembered, so it stays true across our restarts and catches a reused pid. Returns 0 when the process is gone. */
+/** The start time the OS reports for a pid, as an epoch time in milliseconds like every other time we store. Asked of the machine rather than remembered, so it stays true across our restarts and catches a reused pid. Returns 0 when the process is gone, or when neither source can answer. */
 export const getProcessStartTime = measureWrap(async function getProcessStartTime(pid: string): Promise<number> {
-    // ps only resolves to the second, so this is second-accurate - which is plenty to tell two processes on the same pid apart
-    let started = await runPromise(`date -d "$(ps -o lstart= -p ${pid})" +%s`, { quiet: true });
-    return (parseInt(started.trim()) || 0) * 1000;
+    // /proc/<pid> is created when the process is, so its ctime IS the start time - no date parsing, and nothing to get wrong about locales or ps output formats
+    try {
+        let started = await runPromise(`stat -c %Y /proc/${pid}`, { quiet: true });
+        let seconds = parseInt(started.trim()) || 0;
+        if (seconds) return seconds * 1000;
+    } catch {
+        // No procfs (or the process is gone), so fall through to ps
+    }
+    try {
+        // Second-accurate, which is plenty to tell two processes on the same pid apart
+        let started = await runPromise(`date -d "$(ps -o lstart= -p ${pid})" +%s`, { quiet: true });
+        return (parseInt(started.trim()) || 0) * 1000;
+    } catch {
+        return 0;
+    }
 });
 
 export const isScreenRunningProcess = measureWrap(async function isScreenRunningProcess(pid: string): Promise<boolean> {
@@ -263,9 +275,9 @@ export async function ensureProcessRecord(config: {
     record: Omit<ProcessRecord, "pid" | "startTime" | "folder" | "screenName">;
 }): Promise<void> {
     let pid = config.panePid;
+    // A start time we could not read is still recorded: a process with no record at all is invisible everywhere, which is worse than one whose identity is only its pid
     let startTime = await getProcessStartTime(pid);
-    if (!startTime) return;
-    let existing = await readProcessRecord(config.folder, pid, startTime);
+    let existing = await findProcessRecordByPid(config.folder, pid);
     if (existing) {
         // A takeover renames the session, so the record follows the name it now runs under
         if (existing.screenName === config.screenName) return;
@@ -301,40 +313,46 @@ export async function takeoverScreen(config: {
     folder: string;
     /** The node the version being replaced registered as, read before the replacement overwrote the folder's file. */
     replacedNodeId?: string;
+}): Promise<boolean> {
+    return await withScreenLock(config.canonicalScreenName, () => takeoverScreenLocked(config));
+}
+
+/** The takeover itself. Callers already holding this screen's lock use it directly - taking the lock again from inside it would wait on themselves. */
+export async function takeoverScreenLocked(config: {
+    canonicalScreenName: string;
+    folder: string;
+    replacedNodeId?: string;
 }): Promise<boolean> {
     let { canonicalScreenName, folder, replacedNodeId } = config;
-    return await withScreenLock(canonicalScreenName, async () => {
-        let futurePid = await getScreenPanePid(getFutureScreenName(canonicalScreenName));
-        if (!futurePid) return false;
-        let futureScreenName = getFutureScreenName(canonicalScreenName);
-        let prefix = getTmuxPrefix();
-        let outgoingPid = await getScreenPanePid(canonicalScreenName);
-        if (outgoingPid) {
-            // Moved aside rather than waited on, so the canonical name is free at once and the old process winds down on its own
-            let retiringScreenName = getRetiringScreenName(canonicalScreenName, Date.now());
-            await runPromise(`${prefix}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
-            await renameProcessScreen({ folder, panePid: outgoingPid, screenName: retiringScreenName });
-            if (replacedNodeId) {
-                // The file in the folder belongs to the replacement now, so only the registration goes
-                void removeServiceNode({ folder, nodeId: replacedNodeId, ownsFile: false });
-            }
-            retiringScreens.add(retiringScreenName);
-            void killScreen({ screenName: retiringScreenName }).finally(() => {
-                retiringScreens.delete(retiringScreenName);
-            });
+    let futureScreenName = getFutureScreenName(canonicalScreenName);
+    let futurePid = await getScreenPanePid(futureScreenName);
+    if (!futurePid) return false;
+    let prefix = getTmuxPrefix();
+
+    let outgoingPid = await getScreenPanePid(canonicalScreenName);
+    if (outgoingPid) {
+        // Renamed out of the way immediately, so the canonical name is free at once and the old process winds down on its own
+        let retiringScreenName = getRetiringScreenName(canonicalScreenName, Date.now());
+        await runPromise(`${prefix}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
+        await renameProcessScreen({ folder, panePid: outgoingPid, screenName: retiringScreenName });
+        if (replacedNodeId) {
+            // The file in the folder belongs to the replacement now, so only the registration goes
+            void removeServiceNode({ folder, nodeId: replacedNodeId, ownsFile: false });
         }
-        console.log(green(`Renaming ${futureScreenName} to ${canonicalScreenName} (takeover complete)`));
-        await runPromise(`${prefix}tmux rename-session -t ${futureScreenName} ${canonicalScreenName}`);
-        await renameProcessScreen({ folder, panePid: futurePid, screenName: canonicalScreenName });
-        return true;
-    });
+        retiringScreens.add(retiringScreenName);
+        void killScreen({ screenName: retiringScreenName }).finally(() => {
+            retiringScreens.delete(retiringScreenName);
+        });
+    }
+    console.log(green(`Renaming ${futureScreenName} to ${canonicalScreenName} (takeover complete)`));
+    await runPromise(`${prefix}tmux rename-session -t ${futureScreenName} ${canonicalScreenName}`);
+    await renameProcessScreen({ folder, panePid: futurePid, screenName: canonicalScreenName });
+    return true;
 }
 
 /** Points a process's record at the session name it now runs under. The pid is the key, so this reads exactly one file. */
 async function renameProcessScreen(config: { folder: string; panePid: string; screenName: string }): Promise<void> {
-    let startTime = await getProcessStartTime(config.panePid);
-    if (!startTime) return;
-    let record = await readProcessRecord(config.folder, config.panePid, startTime);
+    let record = await findProcessRecordByPid(config.folder, config.panePid);
     if (!record || record.screenName === config.screenName) return;
     await writeProcessRecord({ ...record, screenName: config.screenName });
 }
@@ -371,6 +389,13 @@ export function isRetiring(screenName: string): boolean {
     return retiringScreens.has(screenName);
 }
 
+/** Ends a session at once, with no notice - for a screen that should not exist at all, so there is nothing to wind down gracefully. Finding one is always a surprise, so the reason it was not supposed to be there is logged. */
+export async function killScreenNow(screenName: string, reason: string): Promise<void> {
+    if (!await getScreenPanePid(screenName)) return;
+    console.log(red(`Unexpected screen ${screenName}: ${reason}. Killing it immediately.`));
+    await runPromise(`${getTmuxPrefix()}tmux kill-session -t ${screenName}`);
+}
+
 // Kills a screen and nothing else. Which node the process registered as, and whether that registration should be dropped, is the deploy logic's business - it reads the node id off the process's record, which was taken before any replacement could overwrite the file.
 export const killScreen = measureWrap(async function killScreen(config: {
     screenName: string;