querysub 0.572.0 → 0.574.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.572.0",
+  "version": "0.574.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/deployManager/machineApplyMainCode.ts

@@ -26,7 +26,7 @@ import { PromiseObj } from "../promise";
 import path from "path";
 import { fsExistsAsync } from "../fs";
 import { ALIVE_WINDOW_FOREVER, ParametersTimelineEntry, syncParametersTimelineFiles } from "./parametersTimeline";
-import { getScreenName, getFutureScreenName, isRetiring, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, getScreenPanePid, ensureProcessRecord, runScreenCommand, killScreen, readServiceNodeId, removeServiceNode, takeoverScreenLocked, scheduleTakeover, killScreenNow, withScreenLock } from "./processManager";
+import { getScreenName, getRetiringKillTime, retireScreen, retireCurrentScreen, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, getScreenPanePid, ensureProcessRecord, runScreenCommand, killScreen, readServiceNodeId, removeServiceNode, killScreenNow, withScreenLock } from "./processManager";
 import { ProcessRecord, syncProcessRecords, listProcessRecords } from "./processLogs";
 
 
@@ -221,6 +221,8 @@ let launchesPerService = new Map<string, number>();
 let lastLaunchedTimePerService = new Map<string, number>();
 
 const LAUNCH_VERIFY_DELAY = 2000;
+// How far a re-read start time may drift from the recorded one and still be the same process. Well under any interval a pid could be reused in, and wide enough for sources that only resolve to the second.
+const START_TIME_TOLERANCE = timeInSecond * 5;
 
 /** Every instance of every service that belongs to this machine, with its config already resolved for that instance - the command has this entry's template variables substituted, and a version with no entry at this index is marked as deploying nothing here. */
 function getMachineInstances(configs: ServiceConfig[], machineId: string): { config: ServiceConfig; index: number }[] {
@@ -254,7 +256,7 @@ function getMachineInstances(configs: ServiceConfig[], machineId: string): { con
     return instances;
 }
 
-/** Puts one instance of a service into the state its configuration describes: the folder synced, the process running what it should be, and the records and timeline saying so. Returns the screens it is using, so anything nobody claimed can be killed. Everything about one screen happens under its lock, so a resync and a scheduled takeover can never interleave. */
+/** Puts one instance of a service into the state its configuration describes: the folder synced, the process running what it should be, and the records and timeline saying so. Returns the screen it is using, so anything nobody claimed can be killed. Everything about the screen happens under its lock, so two passes can never interleave. */
 async function ensureScreen(config: {
     config: ServiceConfig;
     index: number;
@@ -265,22 +267,16 @@ async function ensureScreen(config: {
     let serviceId = config.config.serviceId;
     let serviceConfig = config.config;
 
-    let releaseTime = serviceConfig.parameters.releaseTime || 0;
-    // The overlap length always comes from the NEWEST parameters, even though it governs how long the OLD instances outlive the release
-    let killTime = releaseTime + (serviceConfig.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME);
-    // Only in flight while there is an old version to keep alive, and only until its overlap runs out
-    let releaseInFlight = !!releaseTime && !!serviceConfig.oldParameters && Date.now() < killTime;
-
-    // What should be running under the canonical name now, and the version waiting to take it over
-    let current: ServiceParameters | undefined = releaseInFlight && serviceConfig.oldParameters || getLiveServiceParameters(serviceConfig);
-    if (!current?.deploy) current = undefined;
-    let incoming: ServiceParameters | undefined = releaseInFlight && Date.now() >= releaseTime && serviceConfig.parameters || undefined;
-    if (!incoming?.deploy) incoming = undefined;
-    let canonicalScreenName = getScreenName({ serviceKey: (current || incoming || serviceConfig.parameters).key, index });
-    let futureScreenName = getFutureScreenName(canonicalScreenName);
-    let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
+    // A version being replaced is renamed aside and given this long before it is taken down, which is the only thing an overlap is
+    let killTime = Date.now() + (serviceConfig.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME);
+
+    // The screen name means one thing: the version that should be running. There is no second name to reason about - anything it replaced is already renamed aside and carries its own deadline.
+    let parameters: ServiceParameters | undefined = getLiveServiceParameters(serviceConfig);
+    if (!parameters.deploy) parameters = undefined;
+    let screenName = getScreenName({ serviceKey: (parameters || serviceConfig.parameters).key, index });
+    let folder = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
     // Returned even when the work below throws, so a transient failure never makes the sweep kill a screen we still want
-    let usedScreenNames = [canonicalScreenName, futureScreenName];
+    let usedScreenNames = [screenName];
 
     let launchCount = launchesPerService.get(serviceId) || 0;
     let lastLaunchedTime = lastLaunchedTimePerService.get(serviceId) || 0;
@@ -292,108 +288,88 @@ async function ensureScreen(config: {
     };
 
     try {
-        await watchSlowPromise(`ensureScreen|${canonicalScreenName}`, withScreenLock(canonicalScreenName, async () => {
+        await watchSlowPromise(`ensureScreen|${screenName}`, withScreenLock(screenName, async () => {
             /*  Three facts, each moving ? -> known -> correct as we go:
-                    EXISTS   - which of the two screens are there
-                    VERSION  - which parameters they were started with
-                    RUNNING  - whether a process is actually alive in them
+                    EXISTS   - whether the screen is there
+                    VERSION  - which parameters it was started with
+                    RUNNING  - whether a process is actually alive in it
                 Every step below says where those three stand, so it is always clear what we are still assuming.
                 START:  EXISTS ?  VERSION ?  RUNNING ?
             */
-            let parameters = incoming || current;
-            let screenName = incoming && futureScreenName || canonicalScreenName;
-            let takeoverAt = incoming && killTime || undefined;
 
-            // A screen we do not expect holds something we never asked for, so it goes at once - nothing here is worth winding down. Each name is judged on its own: a release that turns deploy on has an incoming version and nothing that should be running under the canonical name yet.
             // AFTER: EXISTS correct in the "nothing extra" sense. A screen that SHOULD exist may still be missing, which is RUNNING's problem below.
-            if (!incoming) {
-                await killScreenNow(futureScreenName, "there is no release in flight, so nothing should be waiting to take over");
-            }
-            if (!current) {
-                await killScreenNow(canonicalScreenName, "nothing is supposed to be serving this name yet");
+            if (!parameters) {
+                await killScreenNow(screenName, "this instance deploys nothing here");
+                return;
             }
-            if (!current && !incoming) return;
 
-            // Both screens run out of one folder, so its parameters.json is the ONE version they were last started from - it cannot tell the two screens apart.
-            // AFTER: VERSION known. Anything it does not match, including a version we no longer recognize, is handled by starting the right one further down - which keeps whatever is up serving until the replacement is verified, instead of killing it now and serving nothing while we sync and start.
+            // The folder holds the version the screen was last started from
+            // AFTER: VERSION known
             let parameterPath = folder + "parameters.json";
             let prevParameters = "";
             if (await fsExistsAsync(parameterPath)) {
                 prevParameters = await fs.promises.readFile(parameterPath, "utf8");
             }
 
-            // AFTER: RUNNING known for every name that can still have anything in it. A screen can exist with nothing alive in it, and one that should exist can be missing entirely - both read as not up, and both are fixed the same way. Without an incoming version the future name was just killed, so there is nothing left to ask about it.
-            let canonicalPid = await getScreenPanePid(canonicalScreenName);
-            let canonicalUp = !!canonicalPid && await isScreenRunningProcess(canonicalPid);
-            let futurePid = incoming && await getScreenPanePid(futureScreenName) || undefined;
-            let futureUp = !!futurePid && await isScreenRunningProcess(futurePid);
-
-            // SMOOTH PATH: this is what a deploy looks like from the canonical name's side. The version being replaced is already correct - it keeps running untouched until its overlap ends, so this name is settled and only the future one is left to converge. It needs its record kept straight, and to hear when it goes down.
-            // AFTER (canonical, during a release): EXISTS correct   VERSION correct   RUNNING correct
-            if (incoming && current && !canonicalUp) {
-                console.log(red(`Nothing is running as ${canonicalScreenName}, which should be serving until ${new Date(killTime).toISOString()} while its replacement starts`));
-            }
-            if (incoming && current && canonicalUp && canonicalPid) {
-                await ensureProcessRecord({
-                    screenName: canonicalScreenName,
-                    folder,
-                    panePid: canonicalPid,
-                    record: { serviceId, serviceKey: current.key, index, machineId, parameters: current },
-                });
-                void (async () => {
-                    let nodeId = await readServiceNodeId(folder);
-                    if (nodeId) {
-                        await notifyNodeShutdown(canonicalScreenName, nodeId, killTime);
-                    }
-                })();
-            }
-
-            // From here on this is about ONE screen - the future one during a release, the canonical one otherwise. Everything else is already correct.
-            // STATE: EXISTS correct   VERSION known   RUNNING known
-            if (!parameters) return;
-            let panePid = incoming && futurePid || canonicalPid;
-            let isUp = incoming && futureUp || canonicalUp;
-            let aliveWindow: [number, number] = [parameters.releaseTime || 0, ALIVE_WINDOW_FOREVER];
-            let record = { serviceId, serviceKey: parameters.key, index, machineId, parameters };
+            // AFTER: RUNNING known. A screen can exist with nothing alive in it, and one that should exist can be missing entirely - both read as not up, and both are fixed the same way.
+            let panePid = await getScreenPanePid(screenName);
+            let isUp = !!panePid && await isScreenRunningProcess(panePid);
 
             let newParametersString = JSON.stringify(parameters);
+            let record = { serviceId, serviceKey: parameters.key, index, machineId, parameters };
 
             let nodePathId = folder + SERVICE_NODE_FILE_NAME;
             if (await fsExistsAsync(nodePathId)) {
                 machineInfo.services[serviceId].nodeId = await fs.promises.readFile(nodePathId, "utf8");
             }
 
-            // Already running this exact version, so nothing needs starting - this is the state we are trying to reach
-            // AFTER: EXISTS correct   VERSION correct   RUNNING correct, and the takeover below settles which name it answers to
-            if (isUp && panePid && sameRestartParameters(prevParameters, parameters)) {
+            // SMOOTH PATH: what is running is not what should be, so it is renamed aside with its own deadline and keeps serving out its overlap there. The name is free from here, which is what makes a replacement and a first start the same thing.
+            if (isUp && panePid && !sameRestartParameters(prevParameters, parameters)) {
+                console.log(green(`Replacing what is running as ${magenta(screenName)}: ${newParametersString}`));
+                // Recorded against the parameters it was actually started with, not the ones replacing it, so its history says what it really ran
+                let outgoingParameters: ServiceParameters | undefined;
+                try {
+                    outgoingParameters = JSON.parse(prevParameters) as ServiceParameters;
+                } catch {
+                    // Never started by us, or the file is unreadable - there is nothing to record it against
+                }
+                if (outgoingParameters) {
+                    await ensureProcessRecord({
+                        screenName,
+                        folder,
+                        panePid,
+                        record: { serviceId, serviceKey: outgoingParameters.key, index, machineId, parameters: outgoingParameters },
+                    });
+                }
+                let outgoingNodeId = await readServiceNodeId(folder);
+                if (outgoingNodeId) {
+                    void notifyNodeShutdown(screenName, outgoingNodeId, killTime);
+                }
+                await retireCurrentScreen({ canonicalScreenName: screenName, folder, killTime, nodeId: outgoingNodeId });
+                isUp = false;
+                panePid = undefined;
+            }
+
+            // A screen with nothing alive in it is just debris in the way of starting one
+            if (!isUp && panePid) {
+                await killScreenNow(screenName, "the screen is there but nothing is running in it");
+                panePid = undefined;
+            }
+
+            // AFTER: EXISTS correct   VERSION correct   RUNNING correct - nothing to start
+            if (isUp && panePid) {
                 if (prevParameters !== newParametersString) {
                     await fs.promises.writeFile(parameterPath, newParametersString);
                 }
                 // We may not have started this one (we restarted, or it predates process records), so make sure it is recorded before leaving it alone
                 await ensureProcessRecord({ screenName, folder, panePid, record });
-                await syncParametersTimelineFiles(folder, [{ pid: parseInt(panePid) || undefined, aliveWindow, parameters }]);
-                if (takeoverAt !== undefined) {
-                    if (!canonicalUp) {
-                        // The version this one is waiting to replace is gone, so there is no overlap left to serve - take the name now instead of sitting under the future one
-                        console.log(red(`Nothing is running as ${canonicalScreenName}, so ${screenName} takes it over now instead of waiting until ${new Date(takeoverAt).toISOString()}`));
-                        await takeoverScreenLocked({ canonicalScreenName, folder });
-                    } else {
-                        // SMOOTH PATH: both versions are up, the old one serving and the new one waiting. A restart loses the timer that retires the old one, so it is put back every time we see it still waiting.
-                        scheduleTakeover({ canonicalScreenName, folder, killTime: takeoverAt });
-                    }
-                }
+                await syncParametersTimelineFiles(folder, [{ pid: parseInt(panePid) || undefined, aliveWindow: [parameters.releaseTime || 0, ALIVE_WINDOW_FOREVER], parameters }]);
                 console.log(green(`Verified ${magenta(screenName)} is running`));
                 return;
             }
 
-            // RUNNING is wrong - either nothing is alive, or what is alive is the wrong version. Starting is the only thing that fixes it.
-            if (!isUp) {
-                // Nothing should ever stop on its own: either it crashed, or it never came up
-                console.log(red(`Nothing is running as ${screenName}, starting it: ${newParametersString}`));
-            } else {
-                // SMOOTH PATH: the previous version is up and this replaces it with a new one, which is exactly what a deploy is
-                console.log(green(`Starting a new version as ${magenta(screenName)}, replacing the one running there: ${newParametersString}`));
-            }
+            // The name is free, so a first start and a replacement are the same from here
+            console.log(red(`Nothing is running as ${screenName}, starting it: ${newParametersString}`));
 
             // Only now does the folder have to hold this version, which is why the git and yarn work sits behind the check above rather than running on every resync
             let gitFolder = folder + "git/";
@@ -417,42 +393,24 @@ async function ensureScreen(config: {
             void recordLaunch({
                 serviceId,
                 serviceKey: parameters.key,
-                screenName: canonicalScreenName,
+                screenName,
                 machineId,
-                reason: isUp && "update" || "crashed",
+                reason: prevParameters && "update" || "crashed",
                 time: Date.now(),
             });
 
-            // The node the version being replaced registered as, read before the launch overwrites the file with the new process's
-            let replacedNodeId = await readServiceNodeId(folder);
-
-            // Always started under the future name and renamed over the canonical one, releases and plain restarts alike: it is up before anything touches what it replaces, and it always gets a brand new pane - which is what makes it a distinct process with its own log.
-            let launchScreenName = futureScreenName;
-            let launchedPid = await runScreenCommand({
-                screenName: launchScreenName,
-                folder,
-                command: parameters.command,
-                record,
-            });
-            // SMOOTH PATH: the command is running as of the line above, so the overlap the version it replaces gets is measured from here
-            if (takeoverAt !== undefined) {
-                scheduleTakeover({ canonicalScreenName, folder, killTime: takeoverAt, replacedNodeId });
-            }
+            let startedPid = await runScreenCommand({ screenName, folder, command: parameters.command, record });
 
             await delay(LAUNCH_VERIFY_DELAY);
-            let launchedPanePid = await getScreenPanePid(launchScreenName);
-            if (!launchedPanePid || !await isScreenRunningProcess(launchedPanePid)) {
+            let startedPanePid = await getScreenPanePid(screenName);
+            if (!startedPanePid || !await isScreenRunningProcess(startedPanePid)) {
                 let prefix = getTmuxPrefix();
-                let logs = await runPromise(`${prefix}tmux capture-pane -t ${launchScreenName} -p`);
-                throw new Error(`${logs}\n\nService ${magenta(launchScreenName)} is not running after starting. Trying again in ${formatTime(MACHINE_RESYNC_INTERVAL)}, or on next change. Last logs above.`);
-            }
-            console.log(green(`${magenta(launchScreenName)} is verified to be running (pid ${launchedPid})`));
-            // Take the canonical name now unless something is still serving under it - waiting out an overlap that nothing is serving would leave the name empty for no reason. SMOOTH PATH during a release is neither: the version being replaced is still up, so this one waits under the future name until its timer fires.
-            // AFTER: EXISTS correct   VERSION correct   RUNNING correct, under the canonical name, or under the future one with the version it is replacing still serving
-            if (takeoverAt === undefined || !canonicalUp) {
-                await takeoverScreenLocked({ canonicalScreenName, folder, replacedNodeId });
+                let logs = await runPromise(`${prefix}tmux capture-pane -t ${screenName} -p`);
+                throw new Error(`${logs}\n\nService ${magenta(screenName)} is not running after starting. Trying again in ${formatTime(MACHINE_RESYNC_INTERVAL)}, or on next change. Last logs above.`);
             }
-            await syncParametersTimelineFiles(folder, [{ pid: parseInt(launchedPanePid) || undefined, aliveWindow, parameters }]);
+            // AFTER: EXISTS correct   VERSION correct   RUNNING correct
+            console.log(green(`${magenta(screenName)} is verified to be running (pid ${startedPid})`));
+            await syncParametersTimelineFiles(folder, [{ pid: parseInt(startedPanePid) || undefined, aliveWindow: [parameters.releaseTime || 0, ALIVE_WINDOW_FOREVER], parameters }]);
 
             launchCount++;
             lastLaunchedTime = Date.now();
@@ -461,7 +419,7 @@ async function ensureScreen(config: {
         // Counted on failure too, as a service that keeps crashing is only visible as a launch count climbing faster than everyone else's
         launchCount++;
         lastLaunchedTime = Date.now();
-        console.error(`Error resyncing ${magenta(canonicalScreenName)} (${serviceId}): ${e.stack}`);
+        console.error(`Error resyncing ${magenta(screenName)} (${serviceId}): ${e.stack}`);
         machineInfo.services[serviceId].errorFromLastRun = e.stack;
     }
     machineInfo.services[serviceId].totalTimesLaunched = launchCount;
@@ -497,7 +455,8 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
     await syncProcessRecords({
         isAlive: async record => {
             if (!running.has(record.pid)) return false;
-            return await getProcessStartTime(record.pid) === record.startTime;
+            // Compared with a tolerance rather than exactly: the sources we read a start time from only resolve to the second, and they do not all round the same way, so an exact match would call a live process dead
+            return Math.abs(await getProcessStartTime(record.pid) - record.startTime) <= START_TIME_TOLERANCE;
         },
         now: Date.now(),
     });
@@ -513,8 +472,11 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
     // Anything no instance claimed is not supposed to exist
     for (let { screenName } of screenState) {
         if (usedScreenNames.has(screenName)) continue;
-        // Already being taken down. One left over from a restart is not, so it still gets cleaned up here.
-        if (isRetiring(screenName)) continue;
+        // A retiring screen is not unclaimed - it is a version that was replaced, and its name says when it is due to go. Cached per name, so seeing it on every resync only ever starts one, and a restart picks it up again from the name alone.
+        if (getRetiringKillTime(screenName) !== undefined) {
+            void retireScreen(screenName);
+            continue;
+        }
         // Detached, so a screen that takes its full grace period to go down doesn't hold up the end of the resync
         void (async () => {
             // Nothing is replacing this one, so its registration and the file it wrote both go

package/src/deployManager/processLogs.ts

@@ -99,6 +99,11 @@ export async function readProcessRecord(folder: string, pid: string, startTime:
     return JSON.parse(await fs.promises.readFile(file, "utf8")) as ProcessRecord;
 }
 
+/** The live record for a pid in this folder. The pid is what identifies a process, so a start time that reads back a second off - the sources only resolve to the second - must not make us miss the record and write a duplicate. */
+export async function findProcessRecordByPid(folder: string, pid: string): Promise<ProcessRecord | undefined> {
+    return (await readFolderRecords(folder)).find(x => x.pid === pid && x.deadTime === undefined);
+}
+
 export async function writeProcessRecord(record: ProcessRecord): Promise<void> {
     await fs.promises.mkdir(path.join(record.folder, PROCESS_FOLDER), { recursive: true });
     await fs.promises.writeFile(getRecordPath(record.folder, record.pid, record.startTime), JSON.stringify(record));

package/src/deployManager/processManager.ts

@@ -13,7 +13,7 @@ import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
 import { fsExistsAsync } from "../fs";
 import { PromiseObj } from "../promise";
 import { SERVICE_FOLDER, SERVICE_NODE_FILE_NAME } from "./machineSchema";
-import { ProcessRecord, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, readProcessRecord, writeProcessRecord } from "./processLogs";
+import { ProcessRecord, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, findProcessRecordByPid, writeProcessRecord } from "./processLogs";
 import { setPreciseTimeout } from "../misc";
 
 // Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
@@ -25,17 +25,35 @@ const SCREEN_SUFFIX = "-dply";
 export function getScreenName(config: { serviceKey: string; index: number }): string {
     return `${config.serviceKey}-${config.index}${SCREEN_SUFFIX}`.replace(/[^a-zA-Z0-9\-_]/g, "_");
 }
-/** Marks a session that has already been handed off and is winding down on its own timer. */
-export const RETIRING_MARKER = "-retiring-";
-/** The name an outgoing screen is moved to so the new process can take the canonical name at once, instead of anything waiting for the old process to wind down. Keeps the suffix, so it is still a screen we know about. */
-export function getRetiringScreenName(canonicalScreenName: string, at: number): string {
-    return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + RETIRING_MARKER + at + SCREEN_SUFFIX;
+const RETIRING_MARKER = "-retiring-";
+/** A version being replaced is renamed to this, which frees the canonical name for its replacement at once. The name carries WHEN it is to be killed, so nothing has to remember: any pass that sees the screen can read its own deadline off it, and a restart loses nothing. */
+export function getRetiringScreenName(canonicalScreenName: string, killTime: number): string {
+    return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + RETIRING_MARKER + killTime + SCREEN_SUFFIX;
 }
-// The new version's screen during a release, in the SAME folder as the canonical screen: created (just echoing when it will start) shortly before releaseTime, started at releaseTime, and renamed to the canonical screen name once the old screen is killed at releaseTime + overlapTime.
-export function getFutureScreenName(canonicalScreenName: string): string {
-    return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + "-future" + SCREEN_SUFFIX;
+/** When a retiring screen is due to be killed, or undefined when it is not a retiring screen at all. */
+export function getRetiringKillTime(screenName: string): number | undefined {
+    let marker = screenName.lastIndexOf(RETIRING_MARKER);
+    if (marker === -1) return undefined;
+    return parseInt(screenName.slice(marker + RETIRING_MARKER.length, -SCREEN_SUFFIX.length)) || undefined;
 }
 
+/** Takes a retiring screen down at the time its name says. Cached per screen, so seeing the same screen on every resync only ever starts one of these, and it drops itself from the cache once the screen is gone. */
+export const retireScreen = cache(async (screenName: string): Promise<void> => {
+    try {
+        let killTime = getRetiringKillTime(screenName);
+        if (killTime === undefined) return;
+        let dueAt = killTime;
+        if (dueAt > Date.now()) {
+            console.log(green(`${screenName} is retiring, and is due to be killed at ${new Date(dueAt).toISOString()}`));
+            await new Promise<void>(resolve => setPreciseTimeout({ time: dueAt, callback: () => resolve() }));
+        }
+        await killScreen({ screenName });
+    } finally {
+        // Dropped either way: a failure should be retried by the next resync rather than remembered as done
+        retireScreen.clear(screenName);
+    }
+});
+
 
 // Everything that touches a service's screens - the resync loop, and the timer that retires the old process at its kill time - runs under this, keyed by the canonical screen name. They fire at similar times by design, and a takeover is a rename dance that must not interleave with another one.
 const getScreenLock = cache((canonicalScreenName: string) => runInSerial(async (run: () => Promise<unknown>) => await run()));
@@ -163,11 +181,23 @@ const getAllParentPids = measureWrap(async function getAllParentPids(): Promise<
     let obj = textTableLineToObj(table) as { PID: string; PPID: string }[];
     return new Set(obj.map(x => x.PPID));
 });
-/** The start time the OS reports for a pid, as an epoch time in milliseconds like every other time we store. Asked of the machine rather than remembered, so it stays true across our restarts and catches a reused pid. Returns 0 when the process is gone. */
+/** The start time the OS reports for a pid, as an epoch time in milliseconds like every other time we store. Asked of the machine rather than remembered, so it stays true across our restarts and catches a reused pid. Returns 0 when the process is gone, or when neither source can answer. */
 export const getProcessStartTime = measureWrap(async function getProcessStartTime(pid: string): Promise<number> {
-    // ps only resolves to the second, so this is second-accurate - which is plenty to tell two processes on the same pid apart
-    let started = await runPromise(`date -d "$(ps -o lstart= -p ${pid})" +%s`, { quiet: true });
-    return (parseInt(started.trim()) || 0) * 1000;
+    // /proc/<pid> is created when the process is, so its ctime IS the start time - no date parsing, and nothing to get wrong about locales or ps output formats
+    try {
+        let started = await runPromise(`stat -c %Y /proc/${pid}`, { quiet: true });
+        let seconds = parseInt(started.trim()) || 0;
+        if (seconds) return seconds * 1000;
+    } catch {
+        // No procfs (or the process is gone), so fall through to ps
+    }
+    try {
+        // Second-accurate, which is plenty to tell two processes on the same pid apart
+        let started = await runPromise(`date -d "$(ps -o lstart= -p ${pid})" +%s`, { quiet: true });
+        return (parseInt(started.trim()) || 0) * 1000;
+    } catch {
+        return 0;
+    }
 });
 
 export const isScreenRunningProcess = measureWrap(async function isScreenRunningProcess(pid: string): Promise<boolean> {
@@ -263,9 +293,9 @@ export async function ensureProcessRecord(config: {
     record: Omit<ProcessRecord, "pid" | "startTime" | "folder" | "screenName">;
 }): Promise<void> {
     let pid = config.panePid;
+    // A start time we could not read is still recorded: a process with no record at all is invisible everywhere, which is worse than one whose identity is only its pid
     let startTime = await getProcessStartTime(pid);
-    if (!startTime) return;
-    let existing = await readProcessRecord(config.folder, pid, startTime);
+    let existing = await findProcessRecordByPid(config.folder, pid);
     if (existing) {
         // A takeover renames the session, so the record follows the name it now runs under
         if (existing.screenName === config.screenName) return;
@@ -295,88 +325,28 @@ async function setupPipePane(config: { screenName: string; folder: string; pid:
     await runPromise(`${prefix}tmux pipe-pane -t ${config.screenName} 'bash ${pipeScript}'`);
 }
 
-/** Retires whatever is running under the canonical name and renames the future screen onto it. Everything it needs is derived from the two names, so it costs two `has-session` checks - the kill time is when this runs, not when a resync gets around to it. Returns whether there was a future screen to take over. */
-export async function takeoverScreen(config: {
-    canonicalScreenName: string;
-    folder: string;
-    /** The node the version being replaced registered as, read before the replacement overwrote the folder's file. */
-    replacedNodeId?: string;
-}): Promise<boolean> {
-    return await withScreenLock(config.canonicalScreenName, () => takeoverScreenLocked(config));
-}
-
-/** The takeover itself. Callers already holding this screen's lock use it directly - taking the lock again from inside it would wait on themselves. */
-export async function takeoverScreenLocked(config: {
+/** Moves whatever is running under this name aside so its replacement can have the name at once, and gives it until killTime before it is taken down. Does nothing when the name is free. */
+export async function retireCurrentScreen(config: {
     canonicalScreenName: string;
     folder: string;
-    replacedNodeId?: string;
-}): Promise<boolean> {
-    let { canonicalScreenName, folder, replacedNodeId } = config;
-    let futureScreenName = getFutureScreenName(canonicalScreenName);
-    let futurePid = await getScreenPanePid(futureScreenName);
-    if (!futurePid) return false;
-    let prefix = getTmuxPrefix();
-
+    killTime: number;
+    /** The node it registered as, read before its replacement overwrites the folder's file. */
+    nodeId?: string;
+}): Promise<void> {
+    let { canonicalScreenName, folder, killTime, nodeId } = config;
     let outgoingPid = await getScreenPanePid(canonicalScreenName);
-    if (outgoingPid) {
-        // Renamed out of the way immediately, so the canonical name is free at once and the old process winds down on its own
-        let retiringScreenName = getRetiringScreenName(canonicalScreenName, Date.now());
-        await runPromise(`${prefix}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
-        await renameProcessScreen({ folder, panePid: outgoingPid, screenName: retiringScreenName });
-        if (replacedNodeId) {
-            // The file in the folder belongs to the replacement now, so only the registration goes
-            void removeServiceNode({ folder, nodeId: replacedNodeId, ownsFile: false });
-        }
-        retiringScreens.add(retiringScreenName);
-        void killScreen({ screenName: retiringScreenName }).finally(() => {
-            retiringScreens.delete(retiringScreenName);
-        });
+    if (!outgoingPid) return;
+    let retiringScreenName = getRetiringScreenName(canonicalScreenName, killTime);
+    console.log(green(`Retiring ${canonicalScreenName} as ${retiringScreenName}`));
+    await runPromise(`${getTmuxPrefix()}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
+    let record = await findProcessRecordByPid(folder, outgoingPid);
+    if (record) {
+        await writeProcessRecord({ ...record, screenName: retiringScreenName });
+    }
+    if (nodeId) {
+        // The file in the folder belongs to its replacement now, so only the registration goes
+        void removeServiceNode({ folder, nodeId, ownsFile: false });
     }
-    console.log(green(`Renaming ${futureScreenName} to ${canonicalScreenName} (takeover complete)`));
-    await runPromise(`${prefix}tmux rename-session -t ${futureScreenName} ${canonicalScreenName}`);
-    await renameProcessScreen({ folder, panePid: futurePid, screenName: canonicalScreenName });
-    return true;
-}
-
-/** Points a process's record at the session name it now runs under. The pid is the key, so this reads exactly one file. */
-async function renameProcessScreen(config: { folder: string; panePid: string; screenName: string }): Promise<void> {
-    let startTime = await getProcessStartTime(config.panePid);
-    if (!startTime) return;
-    let record = await readProcessRecord(config.folder, config.panePid, startTime);
-    if (!record || record.screenName === config.screenName) return;
-    await writeProcessRecord({ ...record, screenName: config.screenName });
-}
-
-// canonical screen name -> the takeover we already have scheduled for it
-const scheduledTakeovers = new Map<string, { time: number; cancel: () => void }>();
-
-/** Schedules the takeover for the moment the old process's overlap runs out. The resync loop still does the same takeover whenever it notices the time has passed, which is what recovers this if we restart and lose the timer. */
-export function scheduleTakeover(config: {
-    canonicalScreenName: string;
-    folder: string;
-    killTime: number;
-    /** The node the version being replaced registered as, read before the replacement overwrote the folder's file. */
-    replacedNodeId?: string;
-}): void {
-    let { canonicalScreenName, folder, killTime, replacedNodeId } = config;
-    let existing = scheduledTakeovers.get(canonicalScreenName);
-    if (existing?.time === killTime) return;
-    existing?.cancel();
-    console.log(green(`Scheduling the takeover of ${canonicalScreenName} for ${new Date(killTime).toISOString()}`));
-    let cancel = setPreciseTimeout({
-        time: killTime,
-        callback: () => {
-            scheduledTakeovers.delete(canonicalScreenName);
-            void takeoverScreen({ canonicalScreenName, folder, replacedNodeId });
-        },
-    });
-    scheduledTakeovers.set(canonicalScreenName, { time: killTime, cancel });
-}
-
-// The screens whose shutdown is already running. Nothing else should start a second one, but a screen left behind by a restart is NOT in here, so the unused-screen sweep still cleans it up.
-const retiringScreens = new Set<string>();
-export function isRetiring(screenName: string): boolean {
-    return retiringScreens.has(screenName);
 }
 
 /** Ends a session at once, with no notice - for a screen that should not exist at all, so there is nothing to wind down gracefully. Finding one is always a surprise, so the reason it was not supposed to be there is logged. */