querysub 0.570.0 → 0.571.0

package/appSecrets.ts

@@ -55,6 +55,7 @@ let doImmediateShutdown = lazy(() => {
         });
     }
     function doShutdown() {
+        console.log("SIGINT received, shutting down immediately");
         process.exit();
     }
     process.on("SIGINT", doShutdown);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.570.0",
+  "version": "0.571.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/deployManager/components/ProcessesView.tsx

@@ -32,14 +32,15 @@ class ProcessOutput extends qreact.Component<{ record: ProcessRecord; machineNod
     private unmounted = false;
     componentDidMount() {
         // Props are synchronized state, so the plain values the async work needs are read here
-        let { folder, launchId } = this.props.record;
+        let { folder, pid, startTime } = this.props.record;
         let nodeId = this.props.machineNodeId;
         let callbackId = this.callbackId;
         this.watching = (async () => {
             await watchProcessOutput({
                 nodeId,
                 folder,
-                launchId,
+                pid,
+                startTime,
                 callbackId,
                 onData: async (data: string) => {
                     Querysub.commit(() => {
@@ -153,9 +154,9 @@ class MachineProcesses extends qreact.Component<{ machineId: string; applyNodeId
                     {this.state.showHistorical && `Hide ${historical.length} historical` || `Show ${historical.length} historical`}
                 </Button>}
             </div>
-            {running.map(record => <ProcessRow key={record.launchId} record={record} machineNodeId={this.props.applyNodeId} />)}
+            {running.map(record => <ProcessRow key={`${record.pid}-${record.startTime}`} record={record} machineNodeId={this.props.applyNodeId} />)}
             {this.state.showHistorical && historical.map(record =>
-                <ProcessRow key={record.launchId} record={record} machineNodeId={this.props.applyNodeId} />
+                <ProcessRow key={`${record.pid}-${record.startTime}`} record={record} machineNodeId={this.props.applyNodeId} />
             )}
         </div>;
     }

package/src/deployManager/machineApplyMainCode.ts

@@ -26,8 +26,8 @@ import { PromiseObj } from "../promise";
 import path from "path";
 import { fsExistsAsync } from "../fs";
 import { ALIVE_WINDOW_FOREVER, ParametersTimelineEntry, syncParametersTimelineFiles } from "./parametersTimeline";
-import { getScreenName, getFutureScreenName, RETIRING_MARKER, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, ensureProcessRecord, runScreenCommand, killScreen, streamProcessOutput, readServiceNodeId, removeServiceNode, takeoverScreen, withScreenLock } from "./processManager";
-import { ProcessRecord, syncProcessRecords, writeProcessRecord, listProcessRecords, findRunningRecord, getProcessLogPath } from "./processLogs";
+import { getScreenName, getFutureScreenName, isRetiring, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, getScreenPanePid, ensureProcessRecord, runScreenCommand, killScreen, readServiceNodeId, removeServiceNode, takeoverScreen, scheduleTakeover, withScreenLock } from "./processManager";
+import { ProcessRecord, syncProcessRecords, listProcessRecords } from "./processLogs";
 
 
 
@@ -298,10 +298,9 @@ function getDesiredScreens(configs: ServiceConfig[], machineId: string, root: st
 /** Puts one screen into the state its configuration describes: the folder synced, the process running what it should be, and the records and timeline saying so. Everything about one screen happens under its lock, so a resync and a scheduled takeover can never interleave. */
 async function ensureScreen(config: {
     desired: DesiredScreen;
-    screenStateMap: Map<string, { screenName: string; isProcessRunning: boolean; pid: string }>;
     machineInfo: MachineInfo;
 }): Promise<void> {
-    let { desired, screenStateMap, machineInfo } = config;
+    let { desired, machineInfo } = config;
     let { screenName, canonicalScreenName, folder, parameters, serviceId } = desired;
 
     let launchCount = launchesPerService.get(serviceId) || 0;
@@ -315,15 +314,16 @@ async function ensureScreen(config: {
 
     try {
         await withScreenLock(canonicalScreenName, async () => {
-            let screen = screenStateMap.get(screenName);
+            // Asked of tmux here, not handed in: this decides everything from the names it was given
+            let panePid = await getScreenPanePid(screenName);
+            let isUp = !!panePid && await isScreenRunningProcess(panePid);
 
             if (desired.outgoing) {
                 // Its replacement owns the folder now, so the only thing to do is keep its record straight and let it know when it goes down
-                if (screen?.isProcessRunning) {
-                    await ensureProcessRecord({ screenName, folder, panePid: screen.pid, record: desired });
+                if (panePid && isUp) {
+                    await ensureProcessRecord({ screenName, folder, panePid, record: desired });
                     void (async () => {
-                        let running = await findRunningRecord(folder, screenName);
-                        let nodeId = running?.nodeId || await readServiceNodeId(folder);
+                        let nodeId = await readServiceNodeId(folder);
                         if (nodeId) {
                             await notifyNodeShutdown(screenName, nodeId, desired.aliveWindow[1]);
                         }
@@ -361,54 +361,61 @@ async function ensureScreen(config: {
                 machineInfo.services[serviceId].nodeId = await fs.promises.readFile(nodePathId, "utf8");
             }
 
-            if (sameRestartParameters(prevParameters, parameters) && screen?.isProcessRunning) {
+            if (isUp && panePid && sameRestartParameters(prevParameters, parameters)) {
                 if (prevParameters !== newParametersString) {
                     await fs.promises.writeFile(parameterPath, newParametersString);
                 }
                 // We may not have launched this one (we restarted, or it predates process records), so make sure it is recorded before leaving it alone
-                await ensureProcessRecord({ screenName, folder, panePid: screen.pid, record: desired });
-                await syncParametersTimelineFiles(folder, [{ pid: parseInt(screen.pid) || undefined, aliveWindow: desired.aliveWindow, parameters }]);
+                await ensureProcessRecord({ screenName, folder, panePid, record: desired });
+                await syncParametersTimelineFiles(folder, [{ pid: parseInt(panePid) || undefined, aliveWindow: desired.aliveWindow, parameters }]);
+                // A restart loses the timer that retires the version this one replaces, so it is put back every time we see it still waiting
+                if (desired.takeoverAt !== undefined) {
+                    scheduleTakeover({ canonicalScreenName, folder, killTime: desired.takeoverAt });
+                }
                 console.log(green(`Verified ${magenta(screenName)} is running`));
                 return;
             }
 
-            console.log(`Resyncing ${magenta(screenName)}, with ${newParametersString}, isRunning = ${screen?.isProcessRunning}`);
+            console.log(`Resyncing ${magenta(screenName)}, with ${newParametersString}, isUp = ${isUp}`);
             await fs.promises.writeFile(parameterPath, newParametersString);
             void recordLaunch({
                 serviceId,
                 serviceKey: desired.serviceKey,
                 screenName: canonicalScreenName,
                 machineId: desired.machineId,
-                reason: screen?.isProcessRunning && "update" || "crashed",
+                reason: isUp && "update" || "crashed",
                 time: Date.now(),
             });
 
+            // The node the version being replaced registered as, read before the launch overwrites the file with the new process's
+            let replacedNodeId = await readServiceNodeId(folder);
+
             // Every launch goes into the future name and renames over the canonical one, releases and plain restarts alike: the replacement is up before anything touches the old process, and it always gets a brand new pane - which is what makes it a distinct process with its own log.
             let launchScreenName = getFutureScreenName(canonicalScreenName);
-            let launchId = await runScreenCommand({
+            let launchedPid = await runScreenCommand({
                 screenName: launchScreenName,
                 folder,
                 command: parameters.command,
                 record: desired,
-                // Scheduled the instant the command is sent. Without a time the old process goes as soon as this one is verified.
-                takeover: desired.takeoverAt !== undefined && { canonicalScreenName, killTime: desired.takeoverAt } || undefined,
             });
+            // The command is running as of the line above, so the overlap is measured from here
+            if (desired.takeoverAt !== undefined) {
+                scheduleTakeover({ canonicalScreenName, folder, killTime: desired.takeoverAt, replacedNodeId });
+            }
 
             await delay(LAUNCH_VERIFY_DELAY);
-            let launched = (await getScreenState(false)).find(x => x.screenName === launchScreenName);
-            let isRunning = launched && await isScreenRunningProcess(launched.pid);
-            if (!launched || !isRunning) {
+            let launchedPanePid = await getScreenPanePid(launchScreenName);
+            if (!launchedPanePid || !await isScreenRunningProcess(launchedPanePid)) {
                 let prefix = getTmuxPrefix();
                 let logs = await runPromise(`${prefix}tmux capture-pane -t ${launchScreenName} -p`);
                 throw new Error(`${logs}\n\nService ${magenta(launchScreenName)} is not running after starting. Trying again in ${formatTime(MACHINE_RESYNC_INTERVAL)}, or on next change. Last logs above.`);
             }
-            console.log(green(`${magenta(launchScreenName)} is verified to be running (process ${launchId})`));
+            console.log(green(`${magenta(launchScreenName)} is verified to be running (pid ${launchedPid})`));
             if (desired.takeoverAt === undefined) {
                 // Nothing is holding the canonical name open for an overlap, so take it now
-                await takeoverScreen({ canonicalScreenName, folder });
-                screenStateMap.set(canonicalScreenName, { ...launched, screenName: canonicalScreenName });
+                await takeoverScreen({ canonicalScreenName, folder, replacedNodeId });
             }
-            await syncParametersTimelineFiles(folder, [{ pid: parseInt(launched.pid) || undefined, aliveWindow: desired.aliveWindow, parameters }]);
+            await syncParametersTimelineFiles(folder, [{ pid: parseInt(launchedPanePid) || undefined, aliveWindow: desired.aliveWindow, parameters }]);
 
             launchCount++;
             lastLaunchedTime = Date.now();
@@ -445,30 +452,25 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
 
     let machineInfo = await getLiveMachineInfo();
 
+    // The whole machine's screens, which is what the record maintenance and the kill sweep are about. Converging a screen asks about that one screen itself.
     let screenState = await getScreenState();
-    let screenStateMap = new Map(screenState.map(s => [s.screenName, s]));
-    // A process is alive while a pane with its pid still exists AND that pid still reports the start time it was named after - a reused pid is a different process
-    let panePids = new Set(screenState.map(x => x.pid));
+    // A process is alive while its pane still exists, still reports the start time it was named after (a reused pid is a different process), and still has something running in it - a pane whose command exited is a dead process, not a live one
+    let running = new Set(screenState.filter(x => x.isProcessRunning).map(x => x.pid));
     await syncProcessRecords({
         isAlive: async record => {
-            let pid = String(record.pid);
-            if (!panePids.has(pid)) return false;
-            return await getProcessStartTime(pid) === record.startTime;
+            if (!running.has(record.pid)) return false;
+            return await getProcessStartTime(record.pid) === record.startTime;
         },
         now: Date.now(),
     });
     let root = os.homedir() + "/" + SERVICE_FOLDER;
     let desiredScreens = getDesiredScreens(relevantConfigs, machineId, root);
 
-    // Fire every folder's git fast-path check (HEAD ref + dirty scan) up front, so converging awaits shared promises instead of paying for one working-tree traversal at a time
+    // The fast-path results are memoized per resync, so a folder two desired screens share is only traversed once
     gitFastPathChecks.clear();
-    for (let desired of desiredScreens) {
-        if (desired.outgoing || !desired.parameters.gitRef) continue;
-        void checkGitSyncedAndClean(desired.folder + "git/", desired.parameters.gitRef);
-    }
 
     // Every screen converges concurrently. Each has its own folder, its own records and its own lock, and yarn install takes a network mutex, so there is nothing to serialize behind - one slow git sync or shutdown no longer delays everything after it.
-    await Promise.all(desiredScreens.map(desired => ensureScreen({ desired, screenStateMap, machineInfo })));
+    await Promise.all(desiredScreens.map(desired => ensureScreen({ desired, machineInfo })));
 
     // Both names count as wanted: a screen that took over during this pass answers to the canonical one now, and one that is about to take over still answers to its future name
     let wantedNames = new Set<string>();
@@ -478,13 +480,13 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
     }
     for (let { screenName } of screenState) {
         if (wantedNames.has(screenName)) continue;
-        // A screen that is winding down after a takeover is already being killed on its own timer
-        if (screenName.includes(RETIRING_MARKER)) continue;
+        // Already being taken down. One left over from a restart is not, so it still gets cleaned up here.
+        if (isRetiring(screenName)) continue;
         // Detached, so a screen that takes its full grace period to go down doesn't hold up the end of the resync
         void (async () => {
             // Nothing is replacing this one, so its registration and the file it wrote both go
             let folder = root + screenName + "/";
-            let nodeId = (await findRunningRecord(folder, screenName))?.nodeId || await readServiceNodeId(folder);
+            let nodeId = await readServiceNodeId(folder);
             if (nodeId) {
                 await removeServiceNode({ folder, nodeId, ownsFile: true });
             }

package/src/deployManager/machineController.ts

@@ -83,16 +83,18 @@ class MachineControllerBase {
     }
     public async streamProcessOutput(config: {
         folder: string;
-        launchId: string;
+        pid: string;
+        startTime: number;
         callbackId: string;
     }): Promise<void> {
         let caller = SocketFunction.getCaller();
         await streamProcessOutput({
             folder: config.folder,
-            launchId: config.launchId,
+            pid: config.pid,
+            startTime: config.startTime,
             onData: async (data) => {
                 await MachineControllerClient.nodes[caller.nodeId].onScreenOutput({
-                    launchId: config.launchId,
+                    pid: config.pid,
                     data,
                     callbackId: config.callbackId,
                 });
@@ -106,14 +108,16 @@ class MachineControllerBase {
     public async watchOtherProcessOutput(config: {
         nodeId: string;
         folder: string;
-        launchId: string;
+        pid: string;
+        startTime: number;
         callbackId: string;
     }) {
         let caller = SocketFunction.getCaller();
         forwardedCallbacks.set(config.callbackId, caller.nodeId);
         await MachineController(config.nodeId).streamProcessOutput.promise({
             folder: config.folder,
-            launchId: config.launchId,
+            pid: config.pid,
+            startTime: config.startTime,
             callbackId: config.callbackId,
         });
     }
@@ -170,7 +174,8 @@ let callbacks = new Map<string, (data: string) => Promise<void>>();
 export async function watchProcessOutput(config: {
     nodeId: string;
     folder: string;
-    launchId: string;
+    pid: string;
+    startTime: number;
     callbackId: string;
     onData: (data: string) => Promise<void>;
 }) {
@@ -179,7 +184,8 @@ export async function watchProcessOutput(config: {
     await MachineController(SocketFunction.browserNodeId()).watchOtherProcessOutput.promise({
         nodeId: config.nodeId,
         folder: config.folder,
-        launchId: config.launchId,
+        pid: config.pid,
+        startTime: config.startTime,
         callbackId,
     });
 }
@@ -192,14 +198,14 @@ export async function stopWatchingProcessOutput(config: {
 }
 class MachineControllerClientBase {
     public async onScreenOutput(config: {
-        launchId: string;
+        pid: string;
         data: string;
         callbackId: string;
     }): Promise<void> {
         let forwardToNodeId = forwardedCallbacks.get(config.callbackId);
         if (forwardToNodeId) {
             await MachineControllerClient.nodes[forwardToNodeId].onScreenOutput({
-                launchId: config.launchId,
+                pid: config.pid,
                 data: config.data,
                 callbackId: config.callbackId,
             });

package/src/deployManager/processLogs.ts

@@ -1,6 +1,7 @@
 import os from "os";
 import fs from "fs";
 import path from "path";
+import { fsExistsAsync } from "../fs";
 import { SERVICE_FOLDER, ServiceParameters } from "./machineSchema";
 
 // One log file per process, appended to until it passes LOG_SIZE_LIMIT, then emptied wholesale. Watchers see the file shrink and re-seek to the start. A watcher attaching in the moment right after a reset sees an empty log, which at this size is rare enough to be worth how much simpler this is than rolling files.
@@ -16,53 +17,43 @@ const LOG_SUFFIX = ".log";
 const PIPE_SCRIPT_SUFFIX = ".pipe.sh";
 const TAIL_SCRIPT_SUFFIX = ".tail.sh";
 
-/** One launch of one service instance: the process, the configuration it was launched with, and its output. Written when the process is launched, and updated only to record its pid and then its death. */
+/** One process: the configuration it was started with, and its output. A process is named by its pid and the start time the OS reports for that pid - the pid alone gets reused, and both halves can be re-checked against the machine at any time instead of trusted from our own bookkeeping. */
 export type ProcessRecord = {
-    launchId: string;
-    /** The instance folder this process runs out of. A release's future process shares its canonical instance's folder, so this is what owns the logs, not the tmux session name. */
+    pid: string;
+    /** The start time the OS reports for the pid */
+    startTime: number;
+    /** The instance folder this process runs out of. A replacement shares the folder with the process it replaces, so this is what owns the logs, not the tmux session name. */
     folder: string;
-    /** The tmux session, which a takeover renames from the future name to the canonical one */
+    /** The tmux session, which a takeover renames */
     screenName: string;
     serviceId: string;
     serviceKey: string;
     /** Which instance of the service this is, on this machine */
     index: number;
     machineId: string;
-    /** The start time the OS reports for the pid, which is also half of the launchId */
-    startTime: number;
-    /** The node this process registered as, read out of the folder before any replacement is started - after that the file belongs to the replacement. Kept here so taking this process down needs nothing but its own record. */
+    /** The node this process registered as, read out of the folder before any replacement is started - after that the file belongs to the replacement. */
     nodeId?: string;
     /** When we first noticed the process was gone. Absent while it is running. */
     deadTime?: number;
-    pid?: number;
-    /** The parameters it was launched with, with template variables already resolved */
+    /** The parameters it was started with, with template variables already resolved */
     parameters: ServiceParameters;
 };
 
-export function getProcessLogPath(folder: string, launchId: string): string {
-    return path.join(folder, PROCESS_FOLDER, launchId + LOG_SUFFIX);
-}
-function getRecordPath(folder: string, launchId: string): string {
-    return path.join(folder, PROCESS_FOLDER, launchId + RECORD_SUFFIX);
+// A process's files all share this name, so its whole footprint is found, and deleted, together
+function getProcessFileName(pid: string, startTime: number): string {
+    return `${pid}-${startTime}`;
 }
-// The scripts live beside the log so a process's entire footprint is one set of files, deleted together
-export function getPipeScriptPath(folder: string, launchId: string): string {
-    return path.join(folder, PROCESS_FOLDER, launchId + PIPE_SCRIPT_SUFFIX);
+export function getProcessLogPath(folder: string, pid: string, startTime: number): string {
+    return path.join(folder, PROCESS_FOLDER, getProcessFileName(pid, startTime) + LOG_SUFFIX);
 }
-export function getTailScriptPath(folder: string, launchId: string): string {
-    return path.join(folder, PROCESS_FOLDER, launchId + TAIL_SCRIPT_SUFFIX);
+export function getPipeScriptPath(folder: string, pid: string, startTime: number): string {
+    return path.join(folder, PROCESS_FOLDER, getProcessFileName(pid, startTime) + PIPE_SCRIPT_SUFFIX);
 }
-
-/** A process identifies itself: its pid, plus the start time the OS reports for it. The pid alone is reused, but a pid running since a specific second is one process, and both halves can be re-checked against the machine at any time instead of trusted from our own bookkeeping. */
-export function createLaunchId(pid: string, startTime: number): string {
-    return `${pid}-${startTime}`;
+export function getTailScriptPath(folder: string, pid: string, startTime: number): string {
+    return path.join(folder, PROCESS_FOLDER, getProcessFileName(pid, startTime) + TAIL_SCRIPT_SUFFIX);
 }
-export function parseLaunchId(launchId: string): { pid: string; startTime: number } {
-    let separator = launchId.lastIndexOf("-");
-    return {
-        pid: launchId.slice(0, separator),
-        startTime: parseInt(launchId.slice(separator + 1)) || 0,
-    };
+function getRecordPath(folder: string, pid: string, startTime: number): string {
+    return path.join(folder, PROCESS_FOLDER, getProcessFileName(pid, startTime) + RECORD_SUFFIX);
 }
 
 /** The shell a screen's output is piped through: appends every line, and empties the file once it grows past the limit. */
@@ -101,16 +92,16 @@ while true; do
 done`;
 }
 
-/** The record of whatever process is currently running in a screen, if we have one. */
-export async function findRunningRecord(folder: string, screenName: string): Promise<ProcessRecord | undefined> {
-    return (await listProcessRecords()).find(x =>
-        x.folder === folder && x.screenName === screenName && x.deadTime === undefined
-    );
+/** One process's record, read straight from the file its pid and start time name. */
+export async function readProcessRecord(folder: string, pid: string, startTime: number): Promise<ProcessRecord | undefined> {
+    let file = getRecordPath(folder, pid, startTime);
+    if (!await fsExistsAsync(file)) return undefined;
+    return JSON.parse(await fs.promises.readFile(file, "utf8")) as ProcessRecord;
 }
 
 export async function writeProcessRecord(record: ProcessRecord): Promise<void> {
     await fs.promises.mkdir(path.join(record.folder, PROCESS_FOLDER), { recursive: true });
-    await fs.promises.writeFile(getRecordPath(record.folder, record.launchId), JSON.stringify(record));
+    await fs.promises.writeFile(getRecordPath(record.folder, record.pid, record.startTime), JSON.stringify(record));
 }
 
 async function readFolderRecords(folder: string): Promise<ProcessRecord[]> {
@@ -147,10 +138,10 @@ export async function listProcessRecords(): Promise<ProcessRecord[]> {
 
 async function removeProcessRecord(record: ProcessRecord): Promise<void> {
     let files = [
-        getRecordPath(record.folder, record.launchId),
-        getProcessLogPath(record.folder, record.launchId),
-        getPipeScriptPath(record.folder, record.launchId),
-        getTailScriptPath(record.folder, record.launchId),
+        getRecordPath(record.folder, record.pid, record.startTime),
+        getProcessLogPath(record.folder, record.pid, record.startTime),
+        getPipeScriptPath(record.folder, record.pid, record.startTime),
+        getTailScriptPath(record.folder, record.pid, record.startTime),
     ];
     for (let file of files) {
         try {
@@ -169,11 +160,11 @@ export async function syncProcessRecords(config: {
     for (let record of await listProcessRecords()) {
         if (record.deadTime === undefined && !await config.isAlive(record)) {
             record.deadTime = config.now;
-            console.log(`Process ${record.launchId} (${record.screenName}) is no longer running, marking it dead. Its log is deleted at ${new Date(record.deadTime + DEAD_PROCESS_RETENTION).toISOString()}`);
+            console.log(`Process ${record.pid} (${record.screenName}) is no longer running, marking it dead. Its log is deleted at ${new Date(record.deadTime + DEAD_PROCESS_RETENTION).toISOString()}`);
             await writeProcessRecord(record);
         }
         if (record.deadTime !== undefined && config.now - record.deadTime > DEAD_PROCESS_RETENTION) {
-            console.log(`Deleting the log of process ${record.launchId} (${record.screenName}), dead since ${new Date(record.deadTime).toISOString()}`);
+            console.log(`Deleting the log of process ${record.pid} (${record.screenName}), dead since ${new Date(record.deadTime).toISOString()}`);
             await removeProcessRecord(record);
         }
     }

package/src/deployManager/processManager.ts

@@ -2,7 +2,7 @@ import os from "os";
 import fs from "fs";
 import path from "path";
 import { spawn, ChildProcess } from "child_process";
-import { lazy } from "socket-function/src/caching";
+import { cache, lazy } from "socket-function/src/caching";
 import { measureWrap } from "socket-function/src/profiling/measure";
 import { delay, runInSerial } from "socket-function/src/batching";
 import { timeInSecond } from "socket-function/src/misc";
@@ -13,7 +13,7 @@ import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
 import { fsExistsAsync } from "../fs";
 import { PromiseObj } from "../promise";
 import { SERVICE_FOLDER, SERVICE_NODE_FILE_NAME } from "./machineSchema";
-import { ProcessRecord, createLaunchId, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, listProcessRecords, findRunningRecord, writeProcessRecord } from "./processLogs";
+import { ProcessRecord, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, readProcessRecord, writeProcessRecord } from "./processLogs";
 import { setPreciseTimeout } from "../misc";
 
 // Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
@@ -38,25 +38,29 @@ export function getFutureScreenName(canonicalScreenName: string): string {
 
 
 // Everything that touches a service's screens - the resync loop, and the timer that retires the old process at its kill time - runs under this, keyed by the canonical screen name. They fire at similar times by design, and a takeover is a rename dance that must not interleave with another one.
-const screenLocks = new Map<string, Promise<unknown>>();
-export function withScreenLock<T>(canonicalScreenName: string, run: () => Promise<T>): Promise<T> {
-    let previous = screenLocks.get(canonicalScreenName) || Promise.resolve();
-    let next = previous.then(run, run);
-    // Failures release the lock rather than poisoning every later operation on this screen
-    screenLocks.set(canonicalScreenName, next.then(() => { }, () => { }));
-    return next;
+const getScreenLock = cache((canonicalScreenName: string) => runInSerial(async (run: () => Promise<unknown>) => await run()));
+export async function withScreenLock<T>(canonicalScreenName: string, run: () => Promise<T>): Promise<T> {
+    return await getScreenLock(canonicalScreenName)(run) as T;
 }
 
-/** Whether a session exists, without listing every session and its panes - the takeover already knows the exact names it cares about. */
-async function screenExists(screenName: string): Promise<boolean> {
+/** The pid of a screen's pane, or undefined when there is no such session. Asks about the one screen instead of listing every session on the machine. */
+export async function getScreenPanePid(screenName: string): Promise<string | undefined> {
     let prefix = getTmuxPrefix();
+    let output: string;
     try {
-        await runPromise(`${prefix}tmux has-session -t ${screenName}`, { quiet: true });
-        return true;
+        output = await runPromise(`${prefix}tmux list-panes -t ${screenName} -F "#{pane_pid}"`, { quiet: true });
     } catch {
-        // has-session exits non-zero for "no such session", which is an answer, not a failure
-        return false;
+        // Exits non-zero for "no such session", which is an answer, not a failure
+        return undefined;
     }
+    return output.trim().split("\n")[0].trim() || undefined;
+}
+
+/** Whether a screen exists AND has a process running in it - the only sense in which a service is up. */
+export async function isScreenUp(screenName: string): Promise<boolean> {
+    let panePid = await getScreenPanePid(screenName);
+    if (!panePid) return false;
+    return await isScreenRunningProcess(panePid);
 }
 
 /** The node the process in this folder registered as. Read it BEFORE starting a replacement in the same folder - once the replacement starts, the file is the replacement's. */
@@ -215,104 +219,76 @@ export const runScreenCommand = measureWrap(async function runScreenCommand(conf
     command: string;
     // Defaults to the folder derived from screenName; a future screen passes its canonical screen's folder
     folder?: string;
-    // Identifies the process this launch creates, so its log and its configuration are stored against it
-    record: Omit<ProcessRecord, "launchId" | "folder" | "screenName" | "startTime">;
-    // The screen this launch replaces, and when that one's overlap runs out. The timer is set the instant the command is sent, so the overlap is measured from the new process actually starting.
-    takeover?: { canonicalScreenName: string; killTime: number };
+    // Identifies the process this start creates, so its log and its configuration are stored against it
+    record: Omit<ProcessRecord, "pid" | "startTime" | "folder" | "screenName">;
 }): Promise<string> {
     let prefix = getTmuxPrefix();
     let screenName = config.screenName;
 
-    // A launch is ALWAYS a brand new session. Callers launch into the "-future" name and rename it over the canonical one once the new process is up, so starting a replacement never waits on the old process going away. Nothing is killed here.
-    // A leftover session under this name is from a launch that died before its takeover, so it is dead weight rather than something we are replacing.
-    if ((await getScreenState(false)).some(x => x.screenName === screenName)) {
-        console.log(red(`Removing the leftover session ${screenName} from a launch that never completed`));
+    // Always a brand new session, so the pane is always new - and the pane's pid is what makes this a distinct process with its own log. A session already under this name is debris from a start that never got taken over.
+    if (await getScreenPanePid(screenName)) {
+        console.log(red(`Removing the leftover session ${screenName} from a start that never completed`));
         await runPromise(`${prefix}tmux kill-session -t ${screenName}`);
     }
     await runPromise(`${prefix}tmux new -s ${screenName} -d`);
 
     let folder = config.folder || os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
 
-    // Read the outgoing process's node id onto its own record BEFORE this launch overwrites the file. Whoever takes that process down then has everything it needs on the record, instead of us having to remember not to touch a file that stopped being its own.
-    let outgoingNodeId = await readServiceNodeId(folder);
-    if (outgoingNodeId) {
-        for (let record of await listProcessRecords()) {
-            if (record.folder !== folder || record.deadTime !== undefined || record.nodeId) continue;
-            await writeProcessRecord({ ...record, nodeId: outgoingNodeId });
-        }
-    }
-
-    // The pane we are about to run the command in identifies the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream has to trust our bookkeeping about what is still running. The session was just created, so this pid is always new.
-    let panePid = (await getScreenState(false)).find(x => x.screenName === screenName)?.pid;
-    if (!panePid) {
+    // The pane we are about to run the command in names the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream trusts our bookkeeping about what is still running.
+    let pid = await getScreenPanePid(screenName);
+    if (!pid) {
         throw new Error(`Screen ${screenName} does not exist after creating it, so there is no process to run the command in`);
     }
-    let startTime = await getProcessStartTime(panePid);
-    let launchId = createLaunchId(panePid, startTime);
-    await writeProcessRecord({
-        ...config.record,
-        launchId,
-        folder,
-        screenName,
-        pid: parseInt(panePid) || undefined,
-        startTime,
-    });
+    let startTime = await getProcessStartTime(pid);
+    await writeProcessRecord({ ...config.record, pid, startTime, folder, screenName });
+    // The pipe is attached before the command runs, so the log holds everything the process ever printed
+    await setupPipePane({ screenName, folder, pid, startTime });
     await runPromise(`${prefix}tmux send-keys -t ${screenName} 'cd ${folder}git' Enter`);
     let command = `#!/bin/bash
 ${config.command}
 `;
     await fs.promises.writeFile(folder + "command.sh", command);
+    // Sending the command is the LAST thing this does, so the caller's next line runs the instant the process starts
     await runPromise(`${prefix}tmux send-keys -t ${screenName} 'bash ../command.sh' Enter`);
-    // The command is running as of this line, so the overlap starts counting here - nothing between this and the timer.
-    if (config.takeover) {
-        scheduleTakeover({
-            canonicalScreenName: config.takeover.canonicalScreenName,
-            folder,
-            killTime: config.takeover.killTime,
-        });
-    }
-
-    await setupPipePane({ screenName, folder, launchId });
-    return launchId;
+    return pid;
 });
 
-/** Links an already-running screen to a process record, creating one when this process has never been recorded - it was launched before we restarted, or before process records existed at all. Idempotent: a process we already know about is left alone, so the pipe is only attached the first time we adopt it.
+/** Links an already-running screen to a process record, creating one when this process has never been recorded - it was started before we restarted, or before process records existed at all. Idempotent: a process we already know about is left alone, so the pipe is only attached the first time we adopt it.
  *
- * The screen identifies itself (pane pid + the start time the OS reports for it), so adoption is just naming what is already there - there is nothing to reconcile and nothing to guess. */
+ * The screen names the process itself (pane pid + the start time the OS reports for it), so adoption is just writing down what is already there. */
 export async function ensureProcessRecord(config: {
     screenName: string;
     folder: string;
     panePid: string;
-    record: Omit<ProcessRecord, "launchId" | "folder" | "screenName" | "startTime" | "pid">;
+    record: Omit<ProcessRecord, "pid" | "startTime" | "folder" | "screenName">;
 }): Promise<void> {
-    let startTime = await getProcessStartTime(config.panePid);
+    let pid = config.panePid;
+    let startTime = await getProcessStartTime(pid);
     if (!startTime) return;
-    let launchId = createLaunchId(config.panePid, startTime);
-    let existing = (await listProcessRecords()).find(x => x.launchId === launchId && x.folder === config.folder);
+    let existing = await readProcessRecord(config.folder, pid, startTime);
     if (existing) {
         // A takeover renames the session, so the record follows the name it now runs under
         if (existing.screenName === config.screenName) return;
         await writeProcessRecord({ ...existing, screenName: config.screenName });
         return;
     }
-    console.log(`Adopting already-running process ${launchId} on screen ${config.screenName}, which has no record yet`);
+    console.log(`Adopting already-running process ${pid} on screen ${config.screenName}, which has no record yet`);
     await writeProcessRecord({
         ...config.record,
-        launchId,
+        pid,
+        startTime,
         folder: config.folder,
         screenName: config.screenName,
-        pid: parseInt(config.panePid) || undefined,
-        startTime,
     });
     // Its output was going to whatever the previous incarnation pointed at (or nowhere), so point it at its own log from here on
-    await setupPipePane({ screenName: config.screenName, folder: config.folder, launchId });
+    await setupPipePane({ screenName: config.screenName, folder: config.folder, pid, startTime });
 }
 
-// Points the screen's pipe-pane at this launch's own log file.
-async function setupPipePane(config: { screenName: string; folder: string; launchId: string }) {
+// Points the screen's pipe-pane at this process's own log file.
+async function setupPipePane(config: { screenName: string; folder: string; pid: string; startTime: number }) {
     let prefix = getTmuxPrefix();
-    let logPath = getProcessLogPath(config.folder, config.launchId);
-    let pipeScript = getPipeScriptPath(config.folder, config.launchId);
+    let logPath = getProcessLogPath(config.folder, config.pid, config.startTime);
+    let pipeScript = getPipeScriptPath(config.folder, config.pid, config.startTime);
     await fs.promises.mkdir(path.dirname(logPath), { recursive: true });
     await fs.promises.writeFile(pipeScript, getLogPipeScript(logPath));
     await runPromise(`chmod +x ${pipeScript}`);
@@ -320,42 +296,61 @@ async function setupPipePane(config: { screenName: string; folder: string; launc
 }
 
 /** Retires whatever is running under the canonical name and renames the future screen onto it. Everything it needs is derived from the two names, so it costs two `has-session` checks - the kill time is when this runs, not when a resync gets around to it. Returns whether there was a future screen to take over. */
-export async function takeoverScreen(config: { canonicalScreenName: string; folder: string }): Promise<boolean> {
-    let { canonicalScreenName, folder } = config;
+export async function takeoverScreen(config: {
+    canonicalScreenName: string;
+    folder: string;
+    /** The node the version being replaced registered as, read before the replacement overwrote the folder's file. */
+    replacedNodeId?: string;
+}): Promise<boolean> {
+    let { canonicalScreenName, folder, replacedNodeId } = config;
     return await withScreenLock(canonicalScreenName, async () => {
+        let futurePid = await getScreenPanePid(getFutureScreenName(canonicalScreenName));
+        if (!futurePid) return false;
         let futureScreenName = getFutureScreenName(canonicalScreenName);
-        if (!await screenExists(futureScreenName)) return false;
         let prefix = getTmuxPrefix();
-        if (await screenExists(canonicalScreenName)) {
+        let outgoingPid = await getScreenPanePid(canonicalScreenName);
+        if (outgoingPid) {
+            // Moved aside rather than waited on, so the canonical name is free at once and the old process winds down on its own
             let retiringScreenName = getRetiringScreenName(canonicalScreenName, Date.now());
-            // Its record carries the node it registered as, taken before the replacement started - the file in the folder belongs to the replacement now, so it is left alone
-            let outgoing = await findRunningRecord(folder, canonicalScreenName);
             await runPromise(`${prefix}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
-            if (outgoing) {
-                await writeProcessRecord({ ...outgoing, screenName: retiringScreenName });
-                if (outgoing.nodeId) {
-                    void removeServiceNode({ folder, nodeId: outgoing.nodeId, ownsFile: false });
-                }
+            await renameProcessScreen({ folder, panePid: outgoingPid, screenName: retiringScreenName });
+            if (replacedNodeId) {
+                // The file in the folder belongs to the replacement now, so only the registration goes
+                void removeServiceNode({ folder, nodeId: replacedNodeId, ownsFile: false });
             }
-            // Detached: the canonical name is already free, so nothing waits on the old process winding down
-            void killScreen({ screenName: retiringScreenName });
+            retiringScreens.add(retiringScreenName);
+            void killScreen({ screenName: retiringScreenName }).finally(() => {
+                retiringScreens.delete(retiringScreenName);
+            });
         }
-        console.log(green(`Renaming future screen ${futureScreenName} to ${canonicalScreenName} (takeover complete)`));
+        console.log(green(`Renaming ${futureScreenName} to ${canonicalScreenName} (takeover complete)`));
         await runPromise(`${prefix}tmux rename-session -t ${futureScreenName} ${canonicalScreenName}`);
-        let incoming = await findRunningRecord(folder, futureScreenName);
-        if (incoming) {
-            await writeProcessRecord({ ...incoming, screenName: canonicalScreenName });
-        }
+        await renameProcessScreen({ folder, panePid: futurePid, screenName: canonicalScreenName });
         return true;
     });
 }
 
+/** Points a process's record at the session name it now runs under. The pid is the key, so this reads exactly one file. */
+async function renameProcessScreen(config: { folder: string; panePid: string; screenName: string }): Promise<void> {
+    let startTime = await getProcessStartTime(config.panePid);
+    if (!startTime) return;
+    let record = await readProcessRecord(config.folder, config.panePid, startTime);
+    if (!record || record.screenName === config.screenName) return;
+    await writeProcessRecord({ ...record, screenName: config.screenName });
+}
+
 // canonical screen name -> the takeover we already have scheduled for it
 const scheduledTakeovers = new Map<string, { time: number; cancel: () => void }>();
 
 /** Schedules the takeover for the moment the old process's overlap runs out. The resync loop still does the same takeover whenever it notices the time has passed, which is what recovers this if we restart and lose the timer. */
-export function scheduleTakeover(config: { canonicalScreenName: string; folder: string; killTime: number }): void {
-    let { canonicalScreenName, folder, killTime } = config;
+export function scheduleTakeover(config: {
+    canonicalScreenName: string;
+    folder: string;
+    killTime: number;
+    /** The node the version being replaced registered as, read before the replacement overwrote the folder's file. */
+    replacedNodeId?: string;
+}): void {
+    let { canonicalScreenName, folder, killTime, replacedNodeId } = config;
     let existing = scheduledTakeovers.get(canonicalScreenName);
     if (existing?.time === killTime) return;
     existing?.cancel();
@@ -364,12 +359,18 @@ export function scheduleTakeover(config: { canonicalScreenName: string; folder:
         time: killTime,
         callback: () => {
             scheduledTakeovers.delete(canonicalScreenName);
-            void takeoverScreen({ canonicalScreenName, folder });
+            void takeoverScreen({ canonicalScreenName, folder, replacedNodeId });
         },
     });
     scheduledTakeovers.set(canonicalScreenName, { time: killTime, cancel });
 }
 
+// The screens whose shutdown is already running. Nothing else should start a second one, but a screen left behind by a restart is NOT in here, so the unused-screen sweep still cleans it up.
+const retiringScreens = new Set<string>();
+export function isRetiring(screenName: string): boolean {
+    return retiringScreens.has(screenName);
+}
+
 // Kills a screen and nothing else. Which node the process registered as, and whether that registration should be dropped, is the deploy logic's business - it reads the node id off the process's record, which was taken before any replacement could overwrite the file.
 export const killScreen = measureWrap(async function killScreen(config: {
     screenName: string;
@@ -385,10 +386,11 @@ export const killScreen = measureWrap(async function killScreen(config: {
 /** Streams one process's log: everything already in it, then everything appended after. One process, one file - nothing here has to reason about which process a byte came from. */
 export async function streamProcessOutput(config: {
     folder: string;
-    launchId: string;
+    pid: string;
+    startTime: number;
     onData: (data: string) => Promise<void>;
 }) {
-    let logPath = getProcessLogPath(config.folder, config.launchId);
+    let logPath = getProcessLogPath(config.folder, config.pid, config.startTime);
     let serialOnData = runInSerial(config.onData);
     let stopped = false;
     let childProcess: ChildProcess | undefined;
@@ -407,14 +409,14 @@ export async function streamProcessOutput(config: {
     const onDataWrapped = async (data: string) => {
         pendingDataCalls++;
         if (pendingDataCalls > MAX_PENDING_CALLS) {
-            console.error(`Too many queued onData calls for ${config.launchId}, stopping stream.`);
+            console.error(`Too many queued onData calls for ${config.pid}, stopping stream.`);
             await stop();
             return;
         }
         try {
             await serialOnData(data);
         } catch (e: any) {
-            console.log(`Callback for stream output ${config.launchId} failed. It probably just disconnected, almost certainly not an error: ${e.message}`);
+            console.log(`Callback for stream output ${config.pid} failed. It probably just disconnected, almost certainly not an error: ${e.message}`);
             await stop();
         } finally {
             pendingDataCalls--;
@@ -422,7 +424,7 @@ export async function streamProcessOutput(config: {
     };
 
     try {
-        let tailScript = getTailScriptPath(config.folder, config.launchId);
+        let tailScript = getTailScriptPath(config.folder, config.pid, config.startTime);
         await fs.promises.writeFile(tailScript, getLogTailScript(logPath));
         await runPromise(`chmod +x ${tailScript}`);