querysub 0.536.0 → 0.538.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.536.0",
+  "version": "0.538.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -71,8 +71,8 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.20",
-    "socket-function": "^1.2.25",
+    "sliftutils": "^1.7.21",
+    "socket-function": "^1.2.26",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/deployManager/machineApplyMainCode.ts

@@ -133,9 +133,9 @@ const SCREEN_SUFFIX = "-dply";
 export function getScreenName(config: { serviceKey: string; index: number }): string {
     return `${config.serviceKey}-${config.index}${SCREEN_SUFFIX}`.replace(/[^a-zA-Z0-9\-_]/g, "_");
 }
-// The old instances kept alive during a release overlap. The canonical screens restart with the new config at release time; these keep the OLD version serving until releaseTime + overlapTime, and are killed shortly after.
-export function getOldScreenName(config: { serviceKey: string; index: number }): string {
-    return getScreenName({ serviceKey: config.serviceKey + "-old", index: config.index });
+// The new version's screen during a release, in the SAME folder as the canonical screen: created (just echoing when it will start) shortly before releaseTime, started at releaseTime, and renamed to the canonical screen name once the old screen is killed at releaseTime + overlapTime.
+export function getFutureScreenName(canonicalScreenName: string): string {
+    return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + "-future" + SCREEN_SUFFIX;
 }
 
 export async function streamScreenOutput(config: {
@@ -293,14 +293,8 @@ done`);
 
 // screenName => the shutdown time we last successfully told the service's node about
 let notifiedShutdowns = new Map<string, number>();
-async function notifyServiceShutdown(screenName: string, time: number) {
+async function notifyNodeShutdown(screenName: string, nodeId: string, time: number) {
     if (notifiedShutdowns.get(screenName) === time) return;
-    let nodeIdFile = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
-    if (!await fsExistsAsync(nodeIdFile)) {
-        console.log(`No nodeId file found for ${screenName}, cannot notify it of its scheduled shutdown yet (will retry on the next resync)`);
-        return;
-    }
-    let nodeId = await fs.promises.readFile(nodeIdFile, "utf8");
     try {
         console.log(green(`Notifying ${screenName} (${nodeId}) that it is scheduled to shut down at ${new Date(time).toLocaleString()}`));
         await scheduleNodeShutdown({ nodeId, time });
@@ -309,6 +303,16 @@ async function notifyServiceShutdown(screenName: string, time: number) {
         console.warn(`Error notifying ${screenName} (${nodeId}) of its scheduled shutdown (will retry on the next resync): ${e.stack}`);
     }
 }
+async function notifyServiceShutdown(screenName: string, time: number) {
+    if (notifiedShutdowns.get(screenName) === time) return;
+    let nodeIdFile = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
+    if (!await fsExistsAsync(nodeIdFile)) {
+        console.log(`No nodeId file found for ${screenName}, cannot notify it of its scheduled shutdown yet (will retry on the next resync)`);
+        return;
+    }
+    let nodeId = await fs.promises.readFile(nodeIdFile, "utf8");
+    await notifyNodeShutdown(screenName, nodeId, time);
+}
 
 
 const getTmuxPrefix = lazy(() => {
@@ -383,6 +387,15 @@ const getLinuxChildPids = measureWrap(async function getLinuxChildPids(pid: stri
         return obj.filter(x => x.PPID === pid);
     }
 });
+// One process-table read for the whole machine — a pid is "running something" when any process has it as a parent. Used by getScreenState so a resync doesn't spawn one ps per screen (which made the release boundary ticks wait on every idle screen first).
+const getAllParentPids = measureWrap(async function getAllParentPids(): Promise<Set<string>> {
+    let prefix = getTmuxPrefix();
+    let table = os.platform() === "win32"
+        ? await runPromise(`${prefix}ps`, { quiet: true })
+        : await runPromise(`ps -eo pid,ppid,cmd`, { quiet: true });
+    let obj = textTableLineToObj(table) as { PID: string; PPID: string }[];
+    return new Set(obj.map(x => x.PPID));
+});
 const isScreenRunningProcess = measureWrap(async function isScreenRunningProcess(pid: string): Promise<boolean> {
     try {
         let bashChildPids = await getLinuxChildPids(pid);
@@ -418,9 +431,10 @@ const getScreenState = measureWrap(async function getScreenState(populateIsProce
         ;
 
     if (populateIsProcessRunning) {
-        await Promise.all(screenList.map(async x => {
-            x.isProcessRunning = await isScreenRunningProcess(x.pid);
-        }));
+        let parentPids = await getAllParentPids();
+        for (let x of screenList) {
+            x.isProcessRunning = parentPids.has(x.pid);
+        }
     }
 
     return screenList;
@@ -439,6 +453,8 @@ async function removeOldNodeId(screenName: string) {
 const runScreenCommand = measureWrap(async function runScreenCommand(config: {
     screenName: string;
     command: string;
+    // Defaults to the folder derived from screenName; a future screen passes its canonical screen's folder
+    folder?: string;
 }) {
     let prefix = getTmuxPrefix();
     let screenName = config.screenName;
@@ -474,13 +490,14 @@ const runScreenCommand = measureWrap(async function runScreenCommand(config: {
                 await runScreenCommand({
                     screenName,
                     command: config.command,
+                    folder: config.folder,
                 });
                 return;
             }
         }
     }
     await removeOldNodeId(screenName);
-    let folder = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
+    let folder = config.folder || os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
     await runPromise(`${prefix}tmux send-keys -t ${screenName} 'cd ${folder}git' Enter`);
     let command = `#!/bin/bash
 ${config.command}
@@ -519,31 +536,63 @@ done`);
 });
 const killScreen = measureWrap(async function killScreen(config: {
     screenName: string;
+    // During a takeover the folder's nodeId file already belongs to the NEW process, so the old screen's kill must not remove it
+    skipNodeIdRemoval?: boolean;
 }) {
     console.log(red(`Killing screen ${config.screenName}`));
     let prefix = getTmuxPrefix();
     // Try ctrl+c a few times first
+    let pid = (await getScreenState(false)).find(x => x.screenName === config.screenName)?.pid;
     for (let i = 0; i < 5; i++) {
-        if (!await isScreenRunningProcess(config.screenName)) {
+        if (!pid || !await isScreenRunningProcess(pid)) {
             break;
         }
         await runPromise(`${prefix}tmux send-keys -t ${config.screenName} 'C-c' Enter`);
         await delay(5000);
     }
     await runPromise(`${prefix}tmux kill-session -t ${config.screenName}`);
-    await removeOldNodeId(config.screenName);
+    if (!config.skipNodeIdRemoval) {
+        await removeOldNodeId(config.screenName);
+    }
 });
 // When a present repo fails to sync, retry a few times with backoff before deciding it is actually broken — most failures are transient network blips (ssh-keyscan / fetch), and reacting to those by clobbering the checkout is what stranded running services (they kept running but crashed on the next lazy `require("ws")` once node_modules was gone).
 const GIT_SYNC_MAX_RETRIES = 4;
 const GIT_SYNC_RETRY_MIN_DELAY = timeInSecond * 2;
 const GIT_SYNC_RETRY_MAX_DELAY = timeInSecond * 30;
 
+// Memoized per resync (the map is cleared at each resync's start), so the prewarm pass can fire every folder's check in parallel and the serial loop just awaits the shared promises. The dirty check (`git status`) is a working-tree traversal, so serializing one per instance would delay the release boundary ticks.
+let gitFastPathChecks = new Map<string, Promise<boolean>>();
+function checkGitSyncedAndClean(gitFolder: string, gitRef: string): Promise<boolean> {
+    let key = gitFolder + "|" + gitRef;
+    let cached = gitFastPathChecks.get(key);
+    if (!cached) {
+        cached = (async () => {
+            try {
+                let [currentRef, status] = await Promise.all([
+                    getGitRefLive(gitFolder),
+                    runPromise(`git status --porcelain`, { cwd: gitFolder, quiet: true }),
+                ]);
+                return currentRef === gitRef && !status.trim();
+            } catch {
+                // No usable repo yet, the full sync handles it
+                return false;
+            }
+        })();
+        gitFastPathChecks.set(key, cached);
+    }
+    return cached;
+}
+
 const ensureGitSynced = measureWrap(async function ensureGitSynced(config: {
     // Forces the checkout at gitFolder to gitRef. Escalates through increasingly destructive recoveries, but only ever destroys the working tree as an absolute last resort — a network blip must never wipe node_modules out from under a running service.
     gitFolder: string;
     repoUrl: string;
     gitRef: string;
 }) {
+    // Fast path: nothing to do when HEAD already matches (gitRef is normally an immutable commit sha) and the working tree is clean. This is what keeps idle resyncs (and the release boundary ticks queued behind them) fast — the full sync below does network fetches, and a dirty tree needs its stash + reset --hard repair.
+    if (await checkGitSyncedAndClean(config.gitFolder, config.gitRef)) {
+        return;
+    }
     let hasGit = await fsExistsAsync(config.gitFolder + ".git");
 
     // Repo present: sync to the ref, retrying transient failures with backoff. We do NOT clobber the checkout just because a network call blipped.
@@ -586,71 +635,104 @@ const ensureGitSynced = measureWrap(async function ensureGitSynced(config: {
 let launchesPerService = new Map<string, number>();
 let lastLaunchedTimePerService = new Map<string, number>();
 
-// Starts (or verifies) the "-old" instance of a service during a release overlap, running the old (no longer live) parameters. It keeps the old version serving while the new canonical instances start up, and is killed by the unused-screen cleanup afterwards (once releaseTime + overlapTime passes we stop ensuring it).
-async function ensureOldInstance(config: {
-    old: ServiceParameters;
-    aliveWindow: [number, number];
-    index: number;
+const RELEASE_PREP_LEAD_TIME = timeInSecond * 30;
+
+// In-memory only: the old process's nodeId, captured just before the future instance starts (the new process overwrites the shared folder's nodeId file), so the old node can be force-removed when its screen is finally killed. Lost on apply restart, which is fine — node discovery times dead nodes out anyway.
+let takeoverOldNodeIds = new Map<string, string>();
+
+// Syncs the canonical folder to the new parameters (git + yarn install + parameters.json) and creates the future screen, which just echos when it will start. Idempotent.
+async function ensureFuturePrepared(config: {
+    canonicalScreenName: string;
+    next: ServiceParameters;
+    releaseTime: number;
     screenNamesUsed: Set<string>;
     screenStateMap: Map<string, { screenName: string; isProcessRunning: boolean; pid: string }>;
 }) {
-    let { old, index } = config;
-    let oldScreenName = getOldScreenName({ serviceKey: old.key, index });
-    config.screenNamesUsed.add(oldScreenName);
-    let shutdownTime = config.aliveWindow[1];
-    const syncTimeline = async () => {
-        let pid = parseInt(config.screenStateMap.get(oldScreenName)?.pid || "") || undefined;
-        await syncParametersTimelineFiles(os.homedir() + "/" + SERVICE_FOLDER + oldScreenName + "/", [{ pid, aliveWindow: config.aliveWindow, parameters: old }]);
-    };
-
-    let root = os.homedir() + "/" + SERVICE_FOLDER;
-    let folder = root + oldScreenName + "/";
+    let { canonicalScreenName, next, releaseTime } = config;
+    let futureScreenName = getFutureScreenName(canonicalScreenName);
+    config.screenNamesUsed.add(futureScreenName);
+    let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
     let gitFolder = folder + "git/";
     await fs.promises.mkdir(gitFolder, { recursive: true });
-    if (old.gitRef) {
+    if (next.gitRef) {
         let prevGitRef = "";
         try {
             prevGitRef = await getGitRefLive(gitFolder);
         } catch { }
         await ensureGitSynced({
             gitFolder,
-            repoUrl: old.repoUrl,
-            gitRef: old.gitRef,
+            repoUrl: next.repoUrl,
+            gitRef: next.gitRef,
         });
         let afterGitRef = await getGitRefLive(gitFolder);
         let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
         if (afterGitRef !== prevGitRef || nodeModulesMissing) {
-            console.log(green(`Yarn installing for ${magenta(oldScreenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
+            console.log(green(`Yarn installing for ${magenta(futureScreenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing})`));
             await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
         }
     }
-
-    let parameterPath = folder + "/parameters.json";
-    let prevParameters = "";
-    if (await fsExistsAsync(parameterPath)) {
-        prevParameters = await fs.promises.readFile(parameterPath, "utf8");
+    await fs.promises.writeFile(folder + "parameters.json", JSON.stringify(next));
+    if (!config.screenStateMap.has(futureScreenName)) {
+        let prefix = getTmuxPrefix();
+        await runPromise(`${prefix}tmux new -s ${futureScreenName} -d`);
+        await runPromise(`${prefix}tmux send-keys -t ${futureScreenName} 'echo "Waiting to start the new version at ${new Date(releaseTime).toISOString()} (in ${Math.round((releaseTime - Date.now()) / timeInSecond)} seconds)"' Enter`);
+        console.log(magenta(`Prepared future screen ${futureScreenName}, starting at ${new Date(releaseTime).toLocaleString()}`));
     }
-    let newParametersString = JSON.stringify(old);
-    let sameParameters = sameRestartParameters(prevParameters, old);
-    let screenIsRunning = config.screenStateMap.get(oldScreenName)?.isProcessRunning;
-    if (sameParameters && screenIsRunning) {
-        if (prevParameters !== newParametersString) {
-            await fs.promises.writeFile(parameterPath, newParametersString);
-        }
-        await notifyServiceShutdown(oldScreenName, shutdownTime);
-        await syncTimeline();
-        console.log(green(`Verified release-overlap old instance ${magenta(oldScreenName)} is running`));
+}
+
+// Starts (or verifies) the future screen actually running the new version, in the canonical screen's folder. Idempotent.
+async function ensureFutureStarted(config: {
+    canonicalScreenName: string;
+    next: ServiceParameters;
+    releaseTime: number;
+    screenNamesUsed: Set<string>;
+    screenStateMap: Map<string, { screenName: string; isProcessRunning: boolean; pid: string }>;
+}) {
+    let { canonicalScreenName, next } = config;
+    let futureScreenName = getFutureScreenName(canonicalScreenName);
+    let existing = config.screenStateMap.get(futureScreenName);
+    if (existing?.isProcessRunning) {
+        config.screenNamesUsed.add(futureScreenName);
+        console.log(green(`Verified future instance ${magenta(futureScreenName)} is running`));
         return;
     }
+    await ensureFuturePrepared(config);
+    let folder = os.homedir() + "/" + SERVICE_FOLDER + canonicalScreenName + "/";
+    let nodeIdFile = folder + SERVICE_NODE_FILE_NAME;
+    // Only the FIRST start can trust the file to hold the old process's nodeId — on a future-instance crash + restart it may already hold the future's own nodeId
+    if (!takeoverOldNodeIds.has(canonicalScreenName) && await fsExistsAsync(nodeIdFile)) {
+        takeoverOldNodeIds.set(canonicalScreenName, await fs.promises.readFile(nodeIdFile, "utf8"));
+    }
+    console.log(green(`Starting future instance ${magenta(futureScreenName)} (release time reached)`));
+    await runScreenCommand({ screenName: futureScreenName, folder, command: next.command });
+}
 
-    console.log(`Starting release-overlap old instance ${magenta(oldScreenName)}, with ${newParametersString}, isRunning = ${screenIsRunning}, sameParameters = ${sameParameters}`);
-    await fs.promises.writeFile(parameterPath, newParametersString);
-    await runScreenCommand({
-        screenName: oldScreenName,
-        command: old.command,
-    });
-    await notifyServiceShutdown(oldScreenName, shutdownTime);
-    await syncTimeline();
+// Kills the old canonical screen (if any) and renames the future screen to the canonical name. The rename leaves the new process completely untouched, so the takeover is seamless. No-op when there is no future screen.
+async function takeoverFutureScreen(config: {
+    canonicalScreenName: string;
+    screenNamesUsed: Set<string>;
+    screenStateMap: Map<string, { screenName: string; isProcessRunning: boolean; pid: string }>;
+}) {
+    let { canonicalScreenName, screenStateMap } = config;
+    let futureScreenName = getFutureScreenName(canonicalScreenName);
+    let future = screenStateMap.get(futureScreenName);
+    if (!future) return;
+    // The rename makes the future name vanish, but the caller's original screen list still has it — mark it used so the unused-screen cleanup doesn't try to kill a nonexistent session
+    config.screenNamesUsed.add(futureScreenName);
+    if (screenStateMap.has(canonicalScreenName)) {
+        await killScreen({ screenName: canonicalScreenName, skipNodeIdRemoval: true });
+        screenStateMap.delete(canonicalScreenName);
+        let oldNodeId = takeoverOldNodeIds.get(canonicalScreenName);
+        if (oldNodeId) {
+            await forceRemoveNode(oldNodeId);
+        }
+    }
+    takeoverOldNodeIds.delete(canonicalScreenName);
+    let prefix = getTmuxPrefix();
+    console.log(green(`Renaming future screen ${magenta(futureScreenName)} to ${magenta(canonicalScreenName)} (takeover complete)`));
+    await runPromise(`${prefix}tmux rename-session -t ${futureScreenName} ${canonicalScreenName}`);
+    screenStateMap.set(canonicalScreenName, { ...future, screenName: canonicalScreenName });
+    screenStateMap.delete(futureScreenName);
 }
 
 function sameRestartParameters(prevParametersJSON: string, next: ServiceParameters): boolean {
@@ -677,32 +759,59 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
     let screenNamesUsed = new Set<string>();
     let upcomingReleases: { releaseTime: number; overlapTime: number }[] = [];
     let root = os.homedir() + "/" + SERVICE_FOLDER;
-    for (let record of relevantConfigs) {
-        let config = { ...record, parameters: getLiveServiceParameters(record) };
+
+    let recordStates = relevantConfigs.map(record => {
         let releaseTime = record.parameters.releaseTime || 0;
         // The overlap length always comes from the NEWEST parameters, even though it governs how long the OLD instances outlive the release
         let overlapTime = record.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME;
-        let isUpcomingRelease = !!releaseTime && Date.now() < releaseTime && !!record.oldParameters;
-        let isOverlapRunning = !!releaseTime && Date.now() >= releaseTime && Date.now() < releaseTime + overlapTime && !!record.oldParameters;
-        if (isUpcomingRelease || isOverlapRunning) {
+        let killTime = releaseTime + overlapTime;
+        // The release dance only exists while there are old parameters to keep alive; past killTime this is just a normal config (the takeover in the canonical loop below is what gets it there)
+        let releaseInFlight = !!releaseTime && !!record.oldParameters && Date.now() < killTime;
+        let isPrepTime = releaseInFlight && Date.now() >= releaseTime - RELEASE_PREP_LEAD_TIME;
+        let isOverlapTime = releaseInFlight && Date.now() >= releaseTime;
+        // The canonical screens keep running the OLD parameters (completely untouched) until killTime; the new version runs in "-future" screens in the SAME folders from releaseTime, and takes over the canonical screen names at killTime
+        let config = { ...record, parameters: releaseInFlight && record.oldParameters || getLiveServiceParameters(record) };
+        return { record, config, releaseTime, overlapTime, killTime, releaseInFlight, isPrepTime, isOverlapTime };
+    });
+
+    // Fire every instance folder's git fast-path check (HEAD ref + dirty scan) in parallel up front, so the serial loop below just awaits the shared promises instead of paying for one working-tree traversal at a time
+    gitFastPathChecks.clear();
+    for (let { record, config, isPrepTime } of recordStates) {
+        const prewarm = (parameters: ServiceParameters) => {
+            if (!parameters.gitRef) return;
+            let targets = getMachineTargets(parameters).filter(target => target.machineId === machineId);
+            for (let i = 0; i < targets.length; i++) {
+                let folder = root + getScreenName({ serviceKey: parameters.key, index: i }) + "/";
+                void checkGitSyncedAndClean(folder + "git/", parameters.gitRef);
+            }
+        };
+        if (config.parameters.deploy) {
+            prewarm(config.parameters);
+        }
+        if (isPrepTime && record.parameters.deploy) {
+            prewarm(record.parameters);
+        }
+    }
+
+    for (let { record, config, releaseTime, overlapTime, killTime, releaseInFlight, isPrepTime, isOverlapTime } of recordStates) {
+        if (releaseInFlight) {
             upcomingReleases.push({ releaseTime, overlapTime });
-            console.log(magenta(`Release in flight for ${record.serviceId} (${record.parameters.key}): new instances start at ${new Date(releaseTime).toLocaleString()}${isOverlapRunning && " (overlap running)" || ""}, old instances shut down at ${new Date(releaseTime + overlapTime).toLocaleString()}`));
+            console.log(magenta(`Release in flight for ${record.serviceId} (${record.parameters.key}): the new version starts at ${new Date(releaseTime).toLocaleString()}${isOverlapTime && " (overlap running)" || ""}, the old instances are killed at ${new Date(killTime).toLocaleString()}`));
         }
-        // The old-overlap instances follow the OLD parameters' machineIds, so machines being removed by the release still keep the old version alive through the overlap (their canonical screens die at release time)
-        if (isOverlapRunning && record.oldParameters?.deploy) {
-            let oldParameters = record.oldParameters;
-            let oldTargets = getMachineTargets(oldParameters).filter(target => target.machineId === machineId);
-            for (let i = 0; i < oldTargets.length; i++) {
+
+        if (isPrepTime && record.parameters.deploy) {
+            let nextTargets = getMachineTargets(record.parameters).filter(target => target.machineId === machineId);
+            for (let i = 0; i < nextTargets.length; i++) {
+                let next = { ...record.parameters, command: applyCommandTemplate(record.parameters.command, nextTargets[i].variables) };
+                let canonicalScreenName = getScreenName({ serviceKey: record.parameters.key, index: i });
                 try {
-                    await ensureOldInstance({
-                        old: { ...oldParameters, command: applyCommandTemplate(oldParameters.command, oldTargets[i].variables) },
-                        aliveWindow: [oldParameters.releaseTime || 0, releaseTime + overlapTime],
-                        index: i,
-                        screenNamesUsed,
-                        screenStateMap,
-                    });
+                    if (isOverlapTime) {
+                        await ensureFutureStarted({ canonicalScreenName, next, releaseTime, screenNamesUsed, screenStateMap });
+                    } else {
+                        await ensureFuturePrepared({ canonicalScreenName, next, releaseTime, screenNamesUsed, screenStateMap });
+                    }
                 } catch (e: any) {
-                    console.error(`Error starting release-overlap old instance for ${record.serviceId} (${record.parameters.key}): ${e.stack}`);
+                    console.error(`Error preparing/starting the future instance for ${record.serviceId} (${record.parameters.key}): ${e.stack}`);
                 }
             }
         }
@@ -728,22 +837,72 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                 let folder = root + screenName + "/";
                 screenNamesUsed.add(screenName);
 
-                if (isUpcomingRelease) {
-                    // Tell the running instance when it will be shut down, so it (and everything talking to it) can wind down gracefully.
-                    await notifyServiceShutdown(screenName, releaseTime);
+                const syncTimeline = async (pid: string | undefined) => {
+                    // The windows overlap: a version runs from its releaseTime until the NEXT version's releaseTime + overlapTime (the overlap is when both run)
+                    let entries: ParametersTimelineEntry[] = [{
+                        pid: parseInt(pid || "") || undefined,
+                        aliveWindow: [config.parameters.releaseTime || 0, releaseInFlight ? killTime : ALIVE_WINDOW_FOREVER],
+                        parameters: instanceParameters,
+                    }];
+                    if (releaseInFlight) {
+                        let futureTargets = getMachineTargets(record.parameters).filter(target => target.machineId === machineId);
+                        if (i < futureTargets.length) {
+                            let futureScreen = screenStateMap.get(getFutureScreenName(screenName));
+                            entries.push({
+                                pid: futureScreen?.isProcessRunning && parseInt(futureScreen.pid) || undefined,
+                                aliveWindow: [releaseTime, ALIVE_WINDOW_FOREVER],
+                                parameters: { ...record.parameters, command: applyCommandTemplate(record.parameters.command, futureTargets[i].variables) },
+                            });
+                        }
+                    }
+                    await syncParametersTimelineFiles(folder, entries);
+                };
+
+                if (releaseInFlight) {
+                    // Tell the running instance when it will be shut down, so it (and everything talking to it) can wind down gracefully. Once the future instance may have started, the folder's nodeId file can belong to the NEW process, so from then on only the stashed old nodeId is safe to notify.
+                    if (!isOverlapTime) {
+                        await notifyServiceShutdown(screenName, killTime);
+                    } else {
+                        let oldNodeId = takeoverOldNodeIds.get(screenName);
+                        if (oldNodeId) {
+                            await notifyNodeShutdown(screenName, oldNodeId, killTime);
+                        }
+                    }
+                } else {
+                    // A finished release's future screen takes over here: kill the old screen, rename the future one — the new process itself is untouched, so the normal logic below just verifies it
+                    await takeoverFutureScreen({ canonicalScreenName: screenName, screenNamesUsed, screenStateMap });
+                }
+                if (isPrepTime) {
+                    // From prep time on, the canonical (old) screen is left completely untouched — the folder holds the NEW code and parameters by now, so the normal compare/sync logic below must not run against the old process
+                    if (screenStateMap.get(screenName)?.isProcessRunning) {
+                        await syncTimeline(screenStateMap.get(screenName)?.pid);
+                        let nodePathId = folder + SERVICE_NODE_FILE_NAME;
+                        if (await fsExistsAsync(nodePathId)) {
+                            machineInfo.services[config.serviceId].nodeId = await fs.promises.readFile(nodePathId, "utf8");
+                        }
+                        console.log(green(`Verified ${magenta(screenName)} is running (release in flight, leaving it untouched until ${new Date(killTime).toLocaleString()})`));
+                        continue;
+                    }
+                    // The old instance is dead, so there is nothing to keep alive through the overlap — take the new version over immediately
+                    console.log(magenta(`Old instance ${screenName} is not running during a release, taking the new version over immediately`));
+                    await takeoverFutureScreen({ canonicalScreenName: screenName, screenNamesUsed, screenStateMap });
+                    let nextTargets = getMachineTargets(record.parameters).filter(target => target.machineId === machineId);
+                    if (record.parameters.deploy && record.parameters.key === config.parameters.key && i < nextTargets.length) {
+                        instanceParameters = { ...record.parameters, command: applyCommandTemplate(record.parameters.command, nextTargets[i].variables) };
+                    }
                 }
 
                 let gitFolder = folder + "git/";
                 await fs.promises.mkdir(gitFolder, { recursive: true });
-                if (config.parameters.gitRef) {
+                if (instanceParameters.gitRef) {
                     let prevGitRef = "";
                     try {
                         prevGitRef = await getGitRefLive(gitFolder);
                     } catch { }
                     await ensureGitSynced({
                         gitFolder: gitFolder,
-                        repoUrl: config.parameters.repoUrl,
-                        gitRef: config.parameters.gitRef,
+                        repoUrl: instanceParameters.repoUrl,
+                        gitRef: instanceParameters.gitRef,
                     });
                     let afterGitRef = await getGitRefLive(gitFolder);
                     // Reinstall when the ref changed OR node_modules is missing. The latter is the real fix for the "Cannot find module 'ws'" crash: a recovery re-clone can land on the same commit, so a ref-only check would skip the install and leave the service with no node_modules. Restoring node_modules also self-heals an already-running process, since a failed `require` is never cached and the next reconnect re-resolves it.
@@ -763,34 +922,6 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                 let sameParameters = sameRestartParameters(prevParameters, instanceParameters);
                 let screenIsRunning = screenStateMap.get(screenName)?.isProcessRunning;
 
-                const syncTimeline = async (pid: string | undefined) => {
-                    // The windows overlap: a version runs from its releaseTime until the NEXT version's releaseTime + overlapTime (the overlap is when both run)
-                    let entries: ParametersTimelineEntry[] = [{
-                        pid: parseInt(pid || "") || undefined,
-                        aliveWindow: [config.parameters.releaseTime || 0, isUpcomingRelease ? releaseTime + overlapTime : ALIVE_WINDOW_FOREVER],
-                        parameters: instanceParameters,
-                    }];
-                    if (isUpcomingRelease) {
-                        let futureTargets = getMachineTargets(record.parameters).filter(target => target.machineId === machineId);
-                        if (i < futureTargets.length) {
-                            entries.push({
-                                aliveWindow: [releaseTime, ALIVE_WINDOW_FOREVER],
-                                parameters: { ...record.parameters, command: applyCommandTemplate(record.parameters.command, futureTargets[i].variables) },
-                            });
-                        }
-                    }
-                    if (isOverlapRunning && record.oldParameters) {
-                        let oldTargets = getMachineTargets(record.oldParameters).filter(target => target.machineId === machineId);
-                        if (i < oldTargets.length) {
-                            entries.push({
-                                aliveWindow: [record.oldParameters.releaseTime || 0, releaseTime + overlapTime],
-                                parameters: { ...record.oldParameters, command: applyCommandTemplate(record.oldParameters.command, oldTargets[i].variables) },
-                            });
-                        }
-                    }
-                    await syncParametersTimelineFiles(folder, entries);
-                };
-
                 let nodePathId = folder + SERVICE_NODE_FILE_NAME;
                 if (await fsExistsAsync(nodePathId)) {
                     let nodeId = await fs.promises.readFile(nodePathId, "utf8");
@@ -883,13 +1014,13 @@ async function resyncServices() {
     }
 }
 
-// The regular resync poll is too coarse for release boundaries (the release itself and the overlap end), so schedule an extra resync at the next boundary.
+// The regular resync poll is too coarse for release boundaries (the future-screen prep, the release itself, and the kill time), so schedule an extra resync at the next boundary. The boundaries carry no logic of their own — each resync just re-evaluates the current state.
 let cancelReleaseResync: (() => void) | undefined;
 function scheduleReleaseResync(upcomingReleases: { releaseTime: number; overlapTime: number }[]) {
     let now = Date.now();
     let next: number | undefined;
     for (let release of upcomingReleases) {
-        for (let boundary of [release.releaseTime, release.releaseTime + release.overlapTime]) {
+        for (let boundary of [release.releaseTime - RELEASE_PREP_LEAD_TIME, release.releaseTime, release.releaseTime + release.overlapTime]) {
             if (boundary <= now) continue;
             if (next === undefined || boundary < next) {
                 next = boundary;

package/src/diagnostics/misc-pages/RoutingTablePage.tsx

@@ -23,10 +23,9 @@ import { URLParam } from "../../library-components/URLParam";
 import { mainResets } from "../../library-components/urlResetGroups";
 
 const ID_CHARS = 8;
-// Green means querysub, blue means path value, purple means function runner.
-const FUNCTION_COLOR = "hsl(280, 65%, 72%)";
+// The single place these are defined: purple means function runner (networks, function calls, querysub addCalls), blue means path value.
+const FUNCTION_RUNNER_COLOR = "hsl(280, 65%, 72%)";
 const PATHVALUE_COLOR = "hsl(210, 75%, 68%)";
-const QUERYSUB_COLOR = "hsl(140, 60%, 60%)";
 
 const PROBE_TIMEOUT_MS = 5000;
 const RANGE_BAR_WIDTH_PX = 360;
@@ -42,14 +41,13 @@ const trafficWindowParam = new URLParam<number>("rtTrafficWindow", timeInMinute
 // `perMachine` columns (the IP and machine id) only print on the first row of each machine group.
 const NODE_TABLE_COLUMNS: { key: keyof NodeRow; label: string; width: number; color?: string; perMachine?: boolean; }[] = [
     { key: "ip", label: "IP", width: 130, perMachine: true },
-    { key: "networks", label: "Networks", width: 180, color: QUERYSUB_COLOR, perMachine: true },
+    { key: "networks", label: "Networks", width: 180, color: FUNCTION_RUNNER_COLOR, perMachine: true },
     { key: "machineShort", label: "Machine", width: 96, perMachine: true },
     { key: "threadPort", label: "Thread:Port", width: 130 },
     { key: "name", label: "Name", width: 160 },
     { key: "data", label: "Data", width: 210 },
-    { key: "fn", label: "Function", width: 320, color: FUNCTION_COLOR },
+    { key: "fn", label: "Function", width: 380, color: FUNCTION_RUNNER_COLOR },
     { key: "pv", label: "Path Values", width: 250, color: PATHVALUE_COLOR },
-    { key: "qs", label: "Querysub", width: 120, color: QUERYSUB_COLOR },
 ];
 
 type NodeRow = {
@@ -63,7 +61,6 @@ type NodeRow = {
     data: string;
     fn: string;
     pv: string;
-    qs: string;
 };
 
 type NodeAuthorityInfo = {
@@ -193,7 +190,7 @@ class FunctionRunnersSection extends qreact.Component {
                 <div className={css.vbox(4).pad2(10).fillWidth.bord2(0, 0, 85).hsl(0, 0, 99)}>
                     <div className={css.hbox(10).fillWidth}>
                         <span className={css.boldStyle}>{node.nodeId}</span>
-                        <span className={css.colorhsl(210, 60, 40)}>networks: {node.networks.join(", ")}</span>
+                        <span className={css.color(FUNCTION_RUNNER_COLOR)}>networks: {node.networks.join(", ")}</span>
                         {!node.isPublic && <span className={css.colorhsl(0, 70, 35)}>(non-public)</span>}
                         <span>latency {formatTime(node.averageLatency)}</span>
                         <span>up for {formatTime(Date.now() - node.startupTime)}</span>
@@ -304,7 +301,7 @@ function machineLabelLines(config: {
         lines.push({ text: ip });
     }
     if (networks.length) {
-        lines.push({ text: networks.join(" | "), color: QUERYSUB_COLOR });
+        lines.push({ text: networks.join(" | "), color: FUNCTION_RUNNER_COLOR });
     }
     lines.push({ text: `${machineId.slice(0, ID_CHARS)}  ${threads.length} threads` });
     if (totals.dataSent + totals.dataReceived > 0) {
@@ -314,10 +311,10 @@ function machineLabelLines(config: {
         lines.push({ text: `↑${formatNumber(totals.valuesSent)}/s ↓${formatNumber(totals.valuesReceived)}/s values`, color: PATHVALUE_COLOR });
     }
     if (totals.calls) {
-        lines.push({ text: `${formatNumber(totals.calls)}/s calls`, color: FUNCTION_COLOR });
+        lines.push({ text: `${formatNumber(totals.calls)}/s calls`, color: FUNCTION_RUNNER_COLOR });
     }
     if (totals.addCalls) {
-        lines.push({ text: `${formatNumber(totals.addCalls)}/s addCalls`, color: QUERYSUB_COLOR });
+        lines.push({ text: `${formatNumber(totals.addCalls)}/s querysub addCalls`, color: FUNCTION_RUNNER_COLOR });
     }
     return lines;
 }
@@ -357,6 +354,10 @@ function buildNodeRow(config: {
         fnParts.push(`${formatNumber(traffic?.functionsExecuted ?? 0)}/s calls`);
         if (!runner.isPublic) fnParts.push("PRIVATE");
     }
+    // A slightly different source than function calls, but still function calls, so it lives in the function column
+    if (traffic?.querysubCalls) {
+        fnParts.push(`${formatNumber(traffic.querysubCalls)}/s querysub addCalls`);
+    }
 
     let pvParts: string[] = [];
     let spec = info?.spec;
@@ -367,7 +368,6 @@ function buildNodeRow(config: {
         pvParts.push(`↑${formatNumber(traffic.pathValuesSent)}/s ↓${formatNumber(traffic.pathValuesReceived)}/s values`);
     }
 
-    let qs = traffic?.querysubCalls ? `${formatNumber(traffic.querysubCalls)}/s addCalls` : "";
     return {
         nodeId,
         machineId,
@@ -379,7 +379,6 @@ function buildNodeRow(config: {
         data,
         fn: fnParts.join("  "),
         pv: pvParts.join("  "),
-        qs,
     };
 }