querysub 0.569.0 → 0.571.0

package/src/deployManager/machineController.ts

@@ -83,16 +83,18 @@ class MachineControllerBase {
     }
     public async streamProcessOutput(config: {
         folder: string;
-        launchId: string;
+        pid: string;
+        startTime: number;
         callbackId: string;
     }): Promise<void> {
         let caller = SocketFunction.getCaller();
         await streamProcessOutput({
             folder: config.folder,
-            launchId: config.launchId,
+            pid: config.pid,
+            startTime: config.startTime,
             onData: async (data) => {
                 await MachineControllerClient.nodes[caller.nodeId].onScreenOutput({
-                    launchId: config.launchId,
+                    pid: config.pid,
                     data,
                     callbackId: config.callbackId,
                 });
@@ -106,14 +108,16 @@ class MachineControllerBase {
     public async watchOtherProcessOutput(config: {
         nodeId: string;
         folder: string;
-        launchId: string;
+        pid: string;
+        startTime: number;
         callbackId: string;
     }) {
         let caller = SocketFunction.getCaller();
         forwardedCallbacks.set(config.callbackId, caller.nodeId);
         await MachineController(config.nodeId).streamProcessOutput.promise({
             folder: config.folder,
-            launchId: config.launchId,
+            pid: config.pid,
+            startTime: config.startTime,
             callbackId: config.callbackId,
         });
     }
@@ -170,7 +174,8 @@ let callbacks = new Map<string, (data: string) => Promise<void>>();
 export async function watchProcessOutput(config: {
     nodeId: string;
     folder: string;
-    launchId: string;
+    pid: string;
+    startTime: number;
     callbackId: string;
     onData: (data: string) => Promise<void>;
 }) {
@@ -179,7 +184,8 @@ export async function watchProcessOutput(config: {
     await MachineController(SocketFunction.browserNodeId()).watchOtherProcessOutput.promise({
         nodeId: config.nodeId,
         folder: config.folder,
-        launchId: config.launchId,
+        pid: config.pid,
+        startTime: config.startTime,
         callbackId,
     });
 }
@@ -192,14 +198,14 @@ export async function stopWatchingProcessOutput(config: {
 }
 class MachineControllerClientBase {
     public async onScreenOutput(config: {
-        launchId: string;
+        pid: string;
         data: string;
         callbackId: string;
     }): Promise<void> {
         let forwardToNodeId = forwardedCallbacks.get(config.callbackId);
         if (forwardToNodeId) {
             await MachineControllerClient.nodes[forwardToNodeId].onScreenOutput({
-                launchId: config.launchId,
+                pid: config.pid,
                 data: config.data,
                 callbackId: config.callbackId,
             });

package/src/deployManager/processLogs.ts

@@ -1,6 +1,7 @@
 import os from "os";
 import fs from "fs";
 import path from "path";
+import { fsExistsAsync } from "../fs";
 import { SERVICE_FOLDER, ServiceParameters } from "./machineSchema";
 
 // One log file per process, appended to until it passes LOG_SIZE_LIMIT, then emptied wholesale. Watchers see the file shrink and re-seek to the start. A watcher attaching in the moment right after a reset sees an empty log, which at this size is rare enough to be worth how much simpler this is than rolling files.
@@ -16,53 +17,43 @@ const LOG_SUFFIX = ".log";
 const PIPE_SCRIPT_SUFFIX = ".pipe.sh";
 const TAIL_SCRIPT_SUFFIX = ".tail.sh";
 
-/** One launch of one service instance: the process, the configuration it was launched with, and its output. Written when the process is launched, and updated only to record its pid and then its death. */
+/** One process: the configuration it was started with, and its output. A process is named by its pid and the start time the OS reports for that pid - the pid alone gets reused, and both halves can be re-checked against the machine at any time instead of trusted from our own bookkeeping. */
 export type ProcessRecord = {
-    launchId: string;
-    /** The instance folder this process runs out of. A release's future process shares its canonical instance's folder, so this is what owns the logs, not the tmux session name. */
+    pid: string;
+    /** The start time the OS reports for the pid */
+    startTime: number;
+    /** The instance folder this process runs out of. A replacement shares the folder with the process it replaces, so this is what owns the logs, not the tmux session name. */
     folder: string;
-    /** The tmux session, which a takeover renames from the future name to the canonical one */
+    /** The tmux session, which a takeover renames */
     screenName: string;
     serviceId: string;
     serviceKey: string;
     /** Which instance of the service this is, on this machine */
     index: number;
     machineId: string;
-    /** The start time the OS reports for the pid, which is also half of the launchId */
-    startTime: number;
-    /** The node this process registered as, read out of the folder before any replacement is started - after that the file belongs to the replacement. Kept here so taking this process down needs nothing but its own record. */
+    /** The node this process registered as, read out of the folder before any replacement is started - after that the file belongs to the replacement. */
     nodeId?: string;
     /** When we first noticed the process was gone. Absent while it is running. */
     deadTime?: number;
-    pid?: number;
-    /** The parameters it was launched with, with template variables already resolved */
+    /** The parameters it was started with, with template variables already resolved */
     parameters: ServiceParameters;
 };
 
-export function getProcessLogPath(folder: string, launchId: string): string {
-    return path.join(folder, PROCESS_FOLDER, launchId + LOG_SUFFIX);
-}
-function getRecordPath(folder: string, launchId: string): string {
-    return path.join(folder, PROCESS_FOLDER, launchId + RECORD_SUFFIX);
+// A process's files all share this name, so its whole footprint is found, and deleted, together
+function getProcessFileName(pid: string, startTime: number): string {
+    return `${pid}-${startTime}`;
 }
-// The scripts live beside the log so a process's entire footprint is one set of files, deleted together
-export function getPipeScriptPath(folder: string, launchId: string): string {
-    return path.join(folder, PROCESS_FOLDER, launchId + PIPE_SCRIPT_SUFFIX);
+export function getProcessLogPath(folder: string, pid: string, startTime: number): string {
+    return path.join(folder, PROCESS_FOLDER, getProcessFileName(pid, startTime) + LOG_SUFFIX);
 }
-export function getTailScriptPath(folder: string, launchId: string): string {
-    return path.join(folder, PROCESS_FOLDER, launchId + TAIL_SCRIPT_SUFFIX);
+export function getPipeScriptPath(folder: string, pid: string, startTime: number): string {
+    return path.join(folder, PROCESS_FOLDER, getProcessFileName(pid, startTime) + PIPE_SCRIPT_SUFFIX);
 }
-
-/** A process identifies itself: its pid, plus the start time the OS reports for it. The pid alone is reused, but a pid running since a specific second is one process, and both halves can be re-checked against the machine at any time instead of trusted from our own bookkeeping. */
-export function createLaunchId(pid: string, startTime: number): string {
-    return `${pid}-${startTime}`;
+export function getTailScriptPath(folder: string, pid: string, startTime: number): string {
+    return path.join(folder, PROCESS_FOLDER, getProcessFileName(pid, startTime) + TAIL_SCRIPT_SUFFIX);
 }
-export function parseLaunchId(launchId: string): { pid: string; startTime: number } {
-    let separator = launchId.lastIndexOf("-");
-    return {
-        pid: launchId.slice(0, separator),
-        startTime: parseInt(launchId.slice(separator + 1)) || 0,
-    };
+function getRecordPath(folder: string, pid: string, startTime: number): string {
+    return path.join(folder, PROCESS_FOLDER, getProcessFileName(pid, startTime) + RECORD_SUFFIX);
 }
 
 /** The shell a screen's output is piped through: appends every line, and empties the file once it grows past the limit. */
@@ -101,16 +92,16 @@ while true; do
 done`;
 }
 
-/** The record of whatever process is currently running in a screen, if we have one. */
-export async function findRunningRecord(folder: string, screenName: string): Promise<ProcessRecord | undefined> {
-    return (await listProcessRecords()).find(x =>
-        x.folder === folder && x.screenName === screenName && x.deadTime === undefined
-    );
+/** One process's record, read straight from the file its pid and start time name. */
+export async function readProcessRecord(folder: string, pid: string, startTime: number): Promise<ProcessRecord | undefined> {
+    let file = getRecordPath(folder, pid, startTime);
+    if (!await fsExistsAsync(file)) return undefined;
+    return JSON.parse(await fs.promises.readFile(file, "utf8")) as ProcessRecord;
 }
 
 export async function writeProcessRecord(record: ProcessRecord): Promise<void> {
     await fs.promises.mkdir(path.join(record.folder, PROCESS_FOLDER), { recursive: true });
-    await fs.promises.writeFile(getRecordPath(record.folder, record.launchId), JSON.stringify(record));
+    await fs.promises.writeFile(getRecordPath(record.folder, record.pid, record.startTime), JSON.stringify(record));
 }
 
 async function readFolderRecords(folder: string): Promise<ProcessRecord[]> {
@@ -147,10 +138,10 @@ export async function listProcessRecords(): Promise<ProcessRecord[]> {
 
 async function removeProcessRecord(record: ProcessRecord): Promise<void> {
     let files = [
-        getRecordPath(record.folder, record.launchId),
-        getProcessLogPath(record.folder, record.launchId),
-        getPipeScriptPath(record.folder, record.launchId),
-        getTailScriptPath(record.folder, record.launchId),
+        getRecordPath(record.folder, record.pid, record.startTime),
+        getProcessLogPath(record.folder, record.pid, record.startTime),
+        getPipeScriptPath(record.folder, record.pid, record.startTime),
+        getTailScriptPath(record.folder, record.pid, record.startTime),
     ];
     for (let file of files) {
         try {
@@ -169,11 +160,11 @@ export async function syncProcessRecords(config: {
     for (let record of await listProcessRecords()) {
         if (record.deadTime === undefined && !await config.isAlive(record)) {
             record.deadTime = config.now;
-            console.log(`Process ${record.launchId} (${record.screenName}) is no longer running, marking it dead. Its log is deleted at ${new Date(record.deadTime + DEAD_PROCESS_RETENTION).toISOString()}`);
+            console.log(`Process ${record.pid} (${record.screenName}) is no longer running, marking it dead. Its log is deleted at ${new Date(record.deadTime + DEAD_PROCESS_RETENTION).toISOString()}`);
             await writeProcessRecord(record);
         }
         if (record.deadTime !== undefined && config.now - record.deadTime > DEAD_PROCESS_RETENTION) {
-            console.log(`Deleting the log of process ${record.launchId} (${record.screenName}), dead since ${new Date(record.deadTime).toISOString()}`);
+            console.log(`Deleting the log of process ${record.pid} (${record.screenName}), dead since ${new Date(record.deadTime).toISOString()}`);
             await removeProcessRecord(record);
         }
     }

package/src/deployManager/processManager.ts

@@ -2,7 +2,7 @@ import os from "os";
 import fs from "fs";
 import path from "path";
 import { spawn, ChildProcess } from "child_process";
-import { lazy } from "socket-function/src/caching";
+import { cache, lazy } from "socket-function/src/caching";
 import { measureWrap } from "socket-function/src/profiling/measure";
 import { delay, runInSerial } from "socket-function/src/batching";
 import { timeInSecond } from "socket-function/src/misc";
@@ -13,7 +13,7 @@ import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
 import { fsExistsAsync } from "../fs";
 import { PromiseObj } from "../promise";
 import { SERVICE_FOLDER, SERVICE_NODE_FILE_NAME } from "./machineSchema";
-import { ProcessRecord, createLaunchId, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, listProcessRecords, findRunningRecord, writeProcessRecord } from "./processLogs";
+import { ProcessRecord, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, readProcessRecord, writeProcessRecord } from "./processLogs";
 import { setPreciseTimeout } from "../misc";
 
 // Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
@@ -25,9 +25,11 @@ const SCREEN_SUFFIX = "-dply";
 export function getScreenName(config: { serviceKey: string; index: number }): string {
     return `${config.serviceKey}-${config.index}${SCREEN_SUFFIX}`.replace(/[^a-zA-Z0-9\-_]/g, "_");
 }
+/** Marks a session that has already been handed off and is winding down on its own timer. */
+export const RETIRING_MARKER = "-retiring-";
 /** The name an outgoing screen is moved to so the new process can take the canonical name at once, instead of anything waiting for the old process to wind down. Keeps the suffix, so it is still a screen we know about. */
 export function getRetiringScreenName(canonicalScreenName: string, at: number): string {
-    return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + `-retiring-${at}` + SCREEN_SUFFIX;
+    return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + RETIRING_MARKER + at + SCREEN_SUFFIX;
 }
 // The new version's screen during a release, in the SAME folder as the canonical screen: created (just echoing when it will start) shortly before releaseTime, started at releaseTime, and renamed to the canonical screen name once the old screen is killed at releaseTime + overlapTime.
 export function getFutureScreenName(canonicalScreenName: string): string {
@@ -36,25 +38,29 @@ export function getFutureScreenName(canonicalScreenName: string): string {
 
 
 // Everything that touches a service's screens - the resync loop, and the timer that retires the old process at its kill time - runs under this, keyed by the canonical screen name. They fire at similar times by design, and a takeover is a rename dance that must not interleave with another one.
-const screenLocks = new Map<string, Promise<unknown>>();
-export function withScreenLock<T>(canonicalScreenName: string, run: () => Promise<T>): Promise<T> {
-    let previous = screenLocks.get(canonicalScreenName) || Promise.resolve();
-    let next = previous.then(run, run);
-    // Failures release the lock rather than poisoning every later operation on this screen
-    screenLocks.set(canonicalScreenName, next.then(() => { }, () => { }));
-    return next;
+const getScreenLock = cache((canonicalScreenName: string) => runInSerial(async (run: () => Promise<unknown>) => await run()));
+export async function withScreenLock<T>(canonicalScreenName: string, run: () => Promise<T>): Promise<T> {
+    return await getScreenLock(canonicalScreenName)(run) as T;
 }
 
-/** Whether a session exists, without listing every session and its panes - the takeover already knows the exact names it cares about. */
-async function screenExists(screenName: string): Promise<boolean> {
+/** The pid of a screen's pane, or undefined when there is no such session. Asks about the one screen instead of listing every session on the machine. */
+export async function getScreenPanePid(screenName: string): Promise<string | undefined> {
     let prefix = getTmuxPrefix();
+    let output: string;
     try {
-        await runPromise(`${prefix}tmux has-session -t ${screenName}`, { quiet: true });
-        return true;
+        output = await runPromise(`${prefix}tmux list-panes -t ${screenName} -F "#{pane_pid}"`, { quiet: true });
     } catch {
-        // has-session exits non-zero for "no such session", which is an answer, not a failure
-        return false;
+        // Exits non-zero for "no such session", which is an answer, not a failure
+        return undefined;
     }
+    return output.trim().split("\n")[0].trim() || undefined;
+}
+
+/** Whether a screen exists AND has a process running in it - the only sense in which a service is up. */
+export async function isScreenUp(screenName: string): Promise<boolean> {
+    let panePid = await getScreenPanePid(screenName);
+    if (!panePid) return false;
+    return await isScreenRunningProcess(panePid);
 }
 
 /** The node the process in this folder registered as. Read it BEFORE starting a replacement in the same folder - once the replacement starts, the file is the replacement's. */
@@ -213,94 +219,76 @@ export const runScreenCommand = measureWrap(async function runScreenCommand(conf
     command: string;
     // Defaults to the folder derived from screenName; a future screen passes its canonical screen's folder
     folder?: string;
-    // Identifies the process this launch creates, so its log and its configuration are stored against it
-    record: Omit<ProcessRecord, "launchId" | "folder" | "screenName" | "startTime">;
+    // Identifies the process this start creates, so its log and its configuration are stored against it
+    record: Omit<ProcessRecord, "pid" | "startTime" | "folder" | "screenName">;
 }): Promise<string> {
     let prefix = getTmuxPrefix();
     let screenName = config.screenName;
 
-    // A launch is ALWAYS a brand new session. Callers launch into the "-future" name and rename it over the canonical one once the new process is up, so starting a replacement never waits on the old process going away. Nothing is killed here.
-    // A leftover session under this name is from a launch that died before its takeover, so it is dead weight rather than something we are replacing.
-    if ((await getScreenState(false)).some(x => x.screenName === screenName)) {
-        console.log(red(`Removing the leftover session ${screenName} from a launch that never completed`));
+    // Always a brand new session, so the pane is always new - and the pane's pid is what makes this a distinct process with its own log. A session already under this name is debris from a start that never got taken over.
+    if (await getScreenPanePid(screenName)) {
+        console.log(red(`Removing the leftover session ${screenName} from a start that never completed`));
         await runPromise(`${prefix}tmux kill-session -t ${screenName}`);
     }
     await runPromise(`${prefix}tmux new -s ${screenName} -d`);
 
     let folder = config.folder || os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
 
-    // Read the outgoing process's node id onto its own record BEFORE this launch overwrites the file. Whoever takes that process down then has everything it needs on the record, instead of us having to remember not to touch a file that stopped being its own.
-    let outgoingNodeId = await readServiceNodeId(folder);
-    if (outgoingNodeId) {
-        for (let record of await listProcessRecords()) {
-            if (record.folder !== folder || record.deadTime !== undefined || record.nodeId) continue;
-            await writeProcessRecord({ ...record, nodeId: outgoingNodeId });
-        }
-    }
-
-    // The pane we are about to run the command in identifies the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream has to trust our bookkeeping about what is still running. The session was just created, so this pid is always new.
-    let panePid = (await getScreenState(false)).find(x => x.screenName === screenName)?.pid;
-    if (!panePid) {
+    // The pane we are about to run the command in names the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream trusts our bookkeeping about what is still running.
+    let pid = await getScreenPanePid(screenName);
+    if (!pid) {
         throw new Error(`Screen ${screenName} does not exist after creating it, so there is no process to run the command in`);
     }
-    let startTime = await getProcessStartTime(panePid);
-    let launchId = createLaunchId(panePid, startTime);
-    await writeProcessRecord({
-        ...config.record,
-        launchId,
-        folder,
-        screenName,
-        pid: parseInt(panePid) || undefined,
-        startTime,
-    });
+    let startTime = await getProcessStartTime(pid);
+    await writeProcessRecord({ ...config.record, pid, startTime, folder, screenName });
+    // The pipe is attached before the command runs, so the log holds everything the process ever printed
+    await setupPipePane({ screenName, folder, pid, startTime });
     await runPromise(`${prefix}tmux send-keys -t ${screenName} 'cd ${folder}git' Enter`);
     let command = `#!/bin/bash
 ${config.command}
 `;
     await fs.promises.writeFile(folder + "command.sh", command);
+    // Sending the command is the LAST thing this does, so the caller's next line runs the instant the process starts
     await runPromise(`${prefix}tmux send-keys -t ${screenName} 'bash ../command.sh' Enter`);
-
-    await setupPipePane({ screenName, folder, launchId });
-    return launchId;
+    return pid;
 });
 
-/** Links an already-running screen to a process record, creating one when this process has never been recorded - it was launched before we restarted, or before process records existed at all. Idempotent: a process we already know about is left alone, so the pipe is only attached the first time we adopt it.
+/** Links an already-running screen to a process record, creating one when this process has never been recorded - it was started before we restarted, or before process records existed at all. Idempotent: a process we already know about is left alone, so the pipe is only attached the first time we adopt it.
  *
- * The screen identifies itself (pane pid + the start time the OS reports for it), so adoption is just naming what is already there - there is nothing to reconcile and nothing to guess. */
+ * The screen names the process itself (pane pid + the start time the OS reports for it), so adoption is just writing down what is already there. */
 export async function ensureProcessRecord(config: {
     screenName: string;
     folder: string;
     panePid: string;
-    record: Omit<ProcessRecord, "launchId" | "folder" | "screenName" | "startTime" | "pid">;
+    record: Omit<ProcessRecord, "pid" | "startTime" | "folder" | "screenName">;
 }): Promise<void> {
-    let startTime = await getProcessStartTime(config.panePid);
+    let pid = config.panePid;
+    let startTime = await getProcessStartTime(pid);
     if (!startTime) return;
-    let launchId = createLaunchId(config.panePid, startTime);
-    let existing = (await listProcessRecords()).find(x => x.launchId === launchId && x.folder === config.folder);
+    let existing = await readProcessRecord(config.folder, pid, startTime);
     if (existing) {
         // A takeover renames the session, so the record follows the name it now runs under
         if (existing.screenName === config.screenName) return;
         await writeProcessRecord({ ...existing, screenName: config.screenName });
         return;
     }
-    console.log(`Adopting already-running process ${launchId} on screen ${config.screenName}, which has no record yet`);
+    console.log(`Adopting already-running process ${pid} on screen ${config.screenName}, which has no record yet`);
     await writeProcessRecord({
         ...config.record,
-        launchId,
+        pid,
+        startTime,
         folder: config.folder,
         screenName: config.screenName,
-        pid: parseInt(config.panePid) || undefined,
-        startTime,
     });
     // Its output was going to whatever the previous incarnation pointed at (or nowhere), so point it at its own log from here on
-    await setupPipePane({ screenName: config.screenName, folder: config.folder, launchId });
+    await setupPipePane({ screenName: config.screenName, folder: config.folder, pid, startTime });
 }
 
-// Points the screen's pipe-pane at this launch's own log file.
-async function setupPipePane(config: { screenName: string; folder: string; launchId: string }) {
+// Points the screen's pipe-pane at this process's own log file.
+async function setupPipePane(config: { screenName: string; folder: string; pid: string; startTime: number }) {
     let prefix = getTmuxPrefix();
-    let logPath = getProcessLogPath(config.folder, config.launchId);
-    let pipeScript = getPipeScriptPath(config.folder, config.launchId);
+    let logPath = getProcessLogPath(config.folder, config.pid, config.startTime);
+    let pipeScript = getPipeScriptPath(config.folder, config.pid, config.startTime);
     await fs.promises.mkdir(path.dirname(logPath), { recursive: true });
     await fs.promises.writeFile(pipeScript, getLogPipeScript(logPath));
     await runPromise(`chmod +x ${pipeScript}`);
@@ -308,42 +296,61 @@ async function setupPipePane(config: { screenName: string; folder: string; launc
 }
 
 /** Retires whatever is running under the canonical name and renames the future screen onto it. Everything it needs is derived from the two names, so it costs two `has-session` checks - the kill time is when this runs, not when a resync gets around to it. Returns whether there was a future screen to take over. */
-export async function takeoverScreen(config: { canonicalScreenName: string; folder: string }): Promise<boolean> {
-    let { canonicalScreenName, folder } = config;
+export async function takeoverScreen(config: {
+    canonicalScreenName: string;
+    folder: string;
+    /** The node the version being replaced registered as, read before the replacement overwrote the folder's file. */
+    replacedNodeId?: string;
+}): Promise<boolean> {
+    let { canonicalScreenName, folder, replacedNodeId } = config;
     return await withScreenLock(canonicalScreenName, async () => {
+        let futurePid = await getScreenPanePid(getFutureScreenName(canonicalScreenName));
+        if (!futurePid) return false;
         let futureScreenName = getFutureScreenName(canonicalScreenName);
-        if (!await screenExists(futureScreenName)) return false;
         let prefix = getTmuxPrefix();
-        if (await screenExists(canonicalScreenName)) {
+        let outgoingPid = await getScreenPanePid(canonicalScreenName);
+        if (outgoingPid) {
+            // Moved aside rather than waited on, so the canonical name is free at once and the old process winds down on its own
             let retiringScreenName = getRetiringScreenName(canonicalScreenName, Date.now());
-            // Its record carries the node it registered as, taken before the replacement started - the file in the folder belongs to the replacement now, so it is left alone
-            let outgoing = await findRunningRecord(folder, canonicalScreenName);
             await runPromise(`${prefix}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
-            if (outgoing) {
-                await writeProcessRecord({ ...outgoing, screenName: retiringScreenName });
-                if (outgoing.nodeId) {
-                    void removeServiceNode({ folder, nodeId: outgoing.nodeId, ownsFile: false });
-                }
+            await renameProcessScreen({ folder, panePid: outgoingPid, screenName: retiringScreenName });
+            if (replacedNodeId) {
+                // The file in the folder belongs to the replacement now, so only the registration goes
+                void removeServiceNode({ folder, nodeId: replacedNodeId, ownsFile: false });
             }
-            // Detached: the canonical name is already free, so nothing waits on the old process winding down
-            void killScreen({ screenName: retiringScreenName });
+            retiringScreens.add(retiringScreenName);
+            void killScreen({ screenName: retiringScreenName }).finally(() => {
+                retiringScreens.delete(retiringScreenName);
+            });
         }
-        console.log(green(`Renaming future screen ${futureScreenName} to ${canonicalScreenName} (takeover complete)`));
+        console.log(green(`Renaming ${futureScreenName} to ${canonicalScreenName} (takeover complete)`));
         await runPromise(`${prefix}tmux rename-session -t ${futureScreenName} ${canonicalScreenName}`);
-        let incoming = await findRunningRecord(folder, futureScreenName);
-        if (incoming) {
-            await writeProcessRecord({ ...incoming, screenName: canonicalScreenName });
-        }
+        await renameProcessScreen({ folder, panePid: futurePid, screenName: canonicalScreenName });
         return true;
     });
 }
 
+/** Points a process's record at the session name it now runs under. The pid is the key, so this reads exactly one file. */
+async function renameProcessScreen(config: { folder: string; panePid: string; screenName: string }): Promise<void> {
+    let startTime = await getProcessStartTime(config.panePid);
+    if (!startTime) return;
+    let record = await readProcessRecord(config.folder, config.panePid, startTime);
+    if (!record || record.screenName === config.screenName) return;
+    await writeProcessRecord({ ...record, screenName: config.screenName });
+}
+
 // canonical screen name -> the takeover we already have scheduled for it
 const scheduledTakeovers = new Map<string, { time: number; cancel: () => void }>();
 
 /** Schedules the takeover for the moment the old process's overlap runs out. The resync loop still does the same takeover whenever it notices the time has passed, which is what recovers this if we restart and lose the timer. */
-export function scheduleTakeover(config: { canonicalScreenName: string; folder: string; killTime: number }): void {
-    let { canonicalScreenName, folder, killTime } = config;
+export function scheduleTakeover(config: {
+    canonicalScreenName: string;
+    folder: string;
+    killTime: number;
+    /** The node the version being replaced registered as, read before the replacement overwrote the folder's file. */
+    replacedNodeId?: string;
+}): void {
+    let { canonicalScreenName, folder, killTime, replacedNodeId } = config;
     let existing = scheduledTakeovers.get(canonicalScreenName);
     if (existing?.time === killTime) return;
     existing?.cancel();
@@ -352,12 +359,18 @@ export function scheduleTakeover(config: { canonicalScreenName: string; folder:
         time: killTime,
         callback: () => {
             scheduledTakeovers.delete(canonicalScreenName);
-            void takeoverScreen({ canonicalScreenName, folder });
+            void takeoverScreen({ canonicalScreenName, folder, replacedNodeId });
         },
     });
     scheduledTakeovers.set(canonicalScreenName, { time: killTime, cancel });
 }
 
+// The screens whose shutdown is already running. Nothing else should start a second one, but a screen left behind by a restart is NOT in here, so the unused-screen sweep still cleans it up.
+const retiringScreens = new Set<string>();
+export function isRetiring(screenName: string): boolean {
+    return retiringScreens.has(screenName);
+}
+
 // Kills a screen and nothing else. Which node the process registered as, and whether that registration should be dropped, is the deploy logic's business - it reads the node id off the process's record, which was taken before any replacement could overwrite the file.
 export const killScreen = measureWrap(async function killScreen(config: {
     screenName: string;
@@ -373,10 +386,11 @@ export const killScreen = measureWrap(async function killScreen(config: {
 /** Streams one process's log: everything already in it, then everything appended after. One process, one file - nothing here has to reason about which process a byte came from. */
 export async function streamProcessOutput(config: {
     folder: string;
-    launchId: string;
+    pid: string;
+    startTime: number;
     onData: (data: string) => Promise<void>;
 }) {
-    let logPath = getProcessLogPath(config.folder, config.launchId);
+    let logPath = getProcessLogPath(config.folder, config.pid, config.startTime);
     let serialOnData = runInSerial(config.onData);
     let stopped = false;
     let childProcess: ChildProcess | undefined;
@@ -395,14 +409,14 @@ export async function streamProcessOutput(config: {
     const onDataWrapped = async (data: string) => {
         pendingDataCalls++;
         if (pendingDataCalls > MAX_PENDING_CALLS) {
-            console.error(`Too many queued onData calls for ${config.launchId}, stopping stream.`);
+            console.error(`Too many queued onData calls for ${config.pid}, stopping stream.`);
             await stop();
             return;
         }
         try {
             await serialOnData(data);
         } catch (e: any) {
-            console.log(`Callback for stream output ${config.launchId} failed. It probably just disconnected, almost certainly not an error: ${e.message}`);
+            console.log(`Callback for stream output ${config.pid} failed. It probably just disconnected, almost certainly not an error: ${e.message}`);
             await stop();
         } finally {
             pendingDataCalls--;
@@ -410,7 +424,7 @@ export async function streamProcessOutput(config: {
     };
 
     try {
-        let tailScript = getTailScriptPath(config.folder, config.launchId);
+        let tailScript = getTailScriptPath(config.folder, config.pid, config.startTime);
         await fs.promises.writeFile(tailScript, getLogTailScript(logPath));
         await runPromise(`chmod +x ${tailScript}`);