querysub 0.561.0 → 0.563.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.561.0",
+  "version": "0.563.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -13,7 +13,6 @@ import { InputLabel } from "../../library-components/InputLabel";
 import { Button } from "../../library-components/Button";
 import { isDefined } from "../../misc";
 import { ProcessesView } from "./ProcessesView";
-import { ResyncMachinesButton } from "./deployButtons";
 import { getPathStr2 } from "../../path";
 import { ATag, Anchor } from "../../library-components/ATag";
 import { ScrollOnMount } from "../../library-components/ScrollOnMount";
@@ -346,10 +345,9 @@ export class ServiceDetailPage extends qreact.Component {
             </div>
             {/* Machine Status */}
             {config.parameters.deploy && <div className={css.vbox(8).fillWidth}>
-                <div className={css.hbox(12).wrap.alignItems("center")}>
-                    <h3 className={css.flexGrow(1)}>Deployed Machines ({machineIds.length})</h3>
-                    <ResyncMachinesButton applyNodeIds={machineStatuses.map(x => x.machineInfo?.applyNodeId || "")} />
-                </div>
+                <h3>Deployed Machines ({machineIds.length})</h3>
+                {/* The machines are notified on every config change and poll as a fallback, so forcing a resync had no use in practice. Kept in case that changes.
+                <ResyncMachinesButton applyNodeIds={machineStatuses.map(x => x.machineInfo?.applyNodeId || "")} /> */}
                 <div className={css.vbox(4).fillWidth}>
                     {machineStatuses.map(({ machineId, variables, machineInfo, serviceInfo, isMachineDead, hasError, isDisabled, index }) => {
                         if (!machineInfo) return <div key={machineId}>Loading {machineId}...</div>;

package/src/deployManager/components/ServicesListPage.tsx

@@ -10,7 +10,8 @@ import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { cache } from "socket-function/src/caching";
 import { Anchor } from "../../library-components/ATag";
 import { isPublic } from "../../config";
-import { PendingDeployInfo, ResyncMachinesButton, UpdateButtons, UpdateServiceButtons } from "./deployButtons";
+// ResyncMachinesButton is still exported from deployButtons, for whenever forcing a resync becomes useful
+import { PendingDeployInfo, UpdateButtons, UpdateServiceButtons } from "./deployButtons";
 import { isDefined } from "../../misc";
 import { formatDateJSX } from "../../misc/formatJSX";
 import { Tools } from "./Tools";
@@ -43,9 +44,10 @@ export class ServicesListPage extends qreact.Component {
         return <div className={css.vbox(16)}>
             <div className={css.hbox(12).wrap}>
                 <h2 className={css.flexGrow(1)}>Services</h2>
+                {/* The machines are notified on every config change and poll as a fallback, so forcing a resync had no use in practice. Kept in case that changes.
                 <ResyncMachinesButton applyNodeIds={(machineConfigList || []).map(x =>
                     controller.getMachineInfo(x.machineId)?.applyNodeId || ""
-                )} />
+                )} /> */}
                 <button className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 100)}
                     onClick={() => {
                         Querysub.onCommitFinished(async () => {

package/src/deployManager/machineApplyMainCode.ts

@@ -642,8 +642,11 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                     time: Date.now(),
                 });
 
+                // Every launch goes through the future screen and a rename, releases and plain restarts alike. The new process is up before the old one is touched, so nothing waits on a shutdown, and the pane it runs in is always new - which is what makes it a distinct process with its own log.
+                let launchScreenName = getFutureScreenName(screenName);
                 let launchId = await runScreenCommand({
-                    screenName,
+                    screenName: launchScreenName,
+                    folder,
                     command: instanceParameters.command,
                     record: {
                         serviceId: config.serviceId,
@@ -655,16 +658,21 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                 });
                 await delay(2000);
                 let newScreens = await getScreenState(false);
-                let foundScreen = newScreens.find(x => x.screenName === screenName);
+                let foundScreen = newScreens.find(x => x.screenName === launchScreenName);
                 if (!foundScreen) {
-                    console.error(`Just created screen ${screenName}, but it's not in the list of screens!`);
+                    console.error(`Just created screen ${launchScreenName}, but it's not in the list of screens!`);
                 }
                 let isRunning = foundScreen && await isScreenRunningProcess(foundScreen.pid);
                 if (!isRunning) {
                     let prefix = getTmuxPrefix();
-                    let logs = await runPromise(`${prefix}tmux capture-pane -t ${screenName} -p`);
-                    throw new Error(`${logs}\n\nService ${magenta(screenName)} is not running after starting. Trying again in ${formatTime(MACHINE_RESYNC_INTERVAL)}, or on next change. Last logs above.`);
+                    let logs = await runPromise(`${prefix}tmux capture-pane -t ${launchScreenName} -p`);
+                    throw new Error(`${logs}\n\nService ${magenta(launchScreenName)} is not running after starting. Trying again in ${formatTime(MACHINE_RESYNC_INTERVAL)}, or on next change. Last logs above.`);
+                }
+                // Up and verified, so the old process can go now
+                if (foundScreen) {
+                    screenStateMap.set(launchScreenName, foundScreen);
                 }
+                await takeoverFutureScreen({ canonicalScreenName: screenName, screenNamesUsed, screenStateMap });
                 console.log(green(`Service ${magenta(screenName)} is verified to be running after starting (process ${launchId})`));
                 await syncTimeline(foundScreen?.pid);
 
@@ -765,95 +773,6 @@ async function getPPID(pid: string) {
     }
 }
 
-// Node prints this when the process has exited but the V8 inspector is still
-//  holding it open waiting for a debugger client to detach. If a debugger client
-//  drops uncleanly the process can sit on this forever, holding the screen and
-//  preventing the supervisor from spawning a fresh instance. The watcher below
-//  polls each running process's log for this phrase and sends Ctrl+C to break it out.
-const DEBUGGER_DISCONNECT_MESSAGE = "Waiting for the debugger to disconnect...";
-
-// To avoid false positives from app code that happens to log a string
-//  containing the phrase, the wedge is only reported when:
-//    1. the phrase is on its own line (start-of-string or after a newline), AND
-//    2. nothing else has been logged after it (only trailing whitespace/EOF).
-//  In a real wedge, this line is the very last thing Node writes before going
-//  silent — so any later output rules out the wedge interpretation.
-const DEBUGGER_DISCONNECT_TAIL_PATTERN = /(?:^|\r?\n)Waiting for the debugger to disconnect\.\.\.\s*$/;
-
-// How often the wedge watcher scans every running process's log.
-const DEBUGGER_WEDGE_POLL_INTERVAL = timeInSecond * 15;
-// How much of the tail of the log to read on each scan. The phrase is short
-//  and appears near the very end of the log when a process is stuck, so a few
-//  KB is plenty and keeps scans cheap.
-const DEBUGGER_WEDGE_TAIL_BYTES = 8 * 1024;
-// Suppress repeated Ctrl+C bursts to the same screen while it tears down; if
-//  the first signal didn't break the wait, give it this long before we try again.
-const DEBUGGER_WEDGE_RESIGNAL_DELAY = timeInMinute;
-// After a first match, wait this long and re-check before signaling. If the
-//  process is still alive and just happened to log a message ending in the
-//  phrase, more output will appear in the meantime and the second check fails.
-const DEBUGGER_WEDGE_RECHECK_DELAY = timeInSecond * 3;
-
-// screenName -> last time we sent Ctrl+C because of a debugger wedge.
-const lastDebuggerWedgeSignal = new Map<string, number>();
-
-async function readPipeFileTail(pipeFile: string, maxBytes: number): Promise<string> {
-    let handle: fs.promises.FileHandle | undefined;
-    try {
-        handle = await fs.promises.open(pipeFile, "r");
-        let stat = await handle.stat();
-        let start = Math.max(0, stat.size - maxBytes);
-        let length = stat.size - start;
-        if (length <= 0) return "";
-        let buf = Buffer.alloc(length);
-        await handle.read(buf, 0, length, start);
-        return buf.toString("utf8");
-    } catch {
-        // File doesn't exist yet, or got truncated mid-read — treat as empty.
-        return "";
-    } finally {
-        if (handle) {
-            try {
-                await handle.close();
-            } catch {
-                // ignore
-            }
-        }
-    }
-}
-
-async function unwedgeStuckDebuggerScreens(): Promise<void> {
-    let prefix = getTmuxPrefix();
-    let screens = await getScreenState(false);
-    let running = (await listProcessRecords()).filter(x => x.deadTime === undefined);
-    for (let { screenName } of screens) {
-        let record = running.find(x => x.screenName === screenName);
-        if (!record) continue;
-        let pipeFile = getProcessLogPath(record.folder, record.launchId);
-        let tail1 = await readPipeFileTail(pipeFile, DEBUGGER_WEDGE_TAIL_BYTES);
-        if (!DEBUGGER_DISCONNECT_TAIL_PATTERN.test(tail1)) continue;
-
-        // Confirm the process is actually wedged and not just briefly idle by
-        //  re-checking after a short delay. A live process will write more
-        //  output in this window, which moves the phrase away from the end
-        //  and fails the second match.
-        await delay(DEBUGGER_WEDGE_RECHECK_DELAY);
-        let tail2 = await readPipeFileTail(pipeFile, DEBUGGER_WEDGE_TAIL_BYTES);
-        if (!DEBUGGER_DISCONNECT_TAIL_PATTERN.test(tail2)) continue;
-
-        let last = lastDebuggerWedgeSignal.get(screenName) ?? 0;
-        if (Date.now() - last < DEBUGGER_WEDGE_RESIGNAL_DELAY) continue;
-        lastDebuggerWedgeSignal.set(screenName, Date.now());
-
-        console.warn(red(`Detected stuck "${DEBUGGER_DISCONNECT_MESSAGE}" in ${screenName} (confirmed across two checks ${DEBUGGER_WEDGE_RECHECK_DELAY}ms apart); sending Ctrl+C to unblock it.`));
-        try {
-            await runPromise(`${prefix}tmux send-keys -t ${screenName} 'C-c' Enter`);
-        } catch (e: any) {
-            console.warn(`Failed to send Ctrl+C to ${screenName}: ${e.stack ?? e}`);
-        }
-    }
-}
-
 export async function machineApplyMain() {
     let parentPID = process.argv[2];
     // Wait for the console to get shimmed
@@ -882,7 +801,6 @@ export async function machineApplyMain() {
     await Querysub.hostService("machine-apply");
     onServiceConfigChange(resyncServices);
 
-    runInfinitePoll(DEBUGGER_WEDGE_POLL_INTERVAL, unwedgeStuckDebuggerScreens);
 
     runInfinitePoll(timeInMinute * 3, async () => {
         //console.log(magenta(`Quick outdated check at ${new Date().toISOString()}`));

package/src/deployManager/processManager.ts

@@ -5,6 +5,8 @@ import { spawn, ChildProcess } from "child_process";
 import { lazy } from "socket-function/src/caching";
 import { measureWrap } from "socket-function/src/profiling/measure";
 import { delay, runInSerial } from "socket-function/src/batching";
+import { timeInSecond } from "socket-function/src/misc";
+import { formatTime } from "socket-function/src/formatting/format";
 import { red, green } from "socket-function/src/formatting/logColors";
 import { runPromise } from "../functional/runCommand";
 import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
@@ -15,6 +17,9 @@ import { ProcessRecord, createLaunchId, getLogPipeScript, getLogTailScript, getP
 
 // Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
 
+// How long a process gets between being told to shut down and having its session killed. It is a courtesy, not a negotiation - nothing waits on the process actually exiting.
+const SHUTDOWN_GRACE_TIME = timeInSecond * 15;
+
 const SCREEN_SUFFIX = "-dply";
 export function getScreenName(config: { serviceKey: string; index: number }): string {
     return `${config.serviceKey}-${config.index}${SCREEN_SUFFIX}`.replace(/[^a-zA-Z0-9\-_]/g, "_");
@@ -178,47 +183,18 @@ export const runScreenCommand = measureWrap(async function runScreenCommand(conf
     let prefix = getTmuxPrefix();
     let screenName = config.screenName;
 
-    try {
-        // Throw if it already exists
-        await runPromise(`${prefix}tmux new -s ${screenName} -d`);
-    } catch { }
-    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'echo "Updating running command at ${new Date().toISOString()}"' Enter`);
-    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'C-c' Enter`);
-    await delay(1000);
-
-
-    let screens = await getScreenState();
-    let screen = screens.find(x => x.screenName === screenName);
-    let pid = screen?.pid;
-    if (pid && await isScreenRunningProcess(pid)) {
-        // It doesn't want to die. Wait longer, but it it just won't die, kill the screen
-        console.warn(`Screen ${screenName} is not dying, giving it another 30 seconds`);
-        for (let i = 0; i < 6; i++) {
-            await delay(5);
-            if (!await isScreenRunningProcess(pid)) {
-                break;
-            }
-        }
-        if (pid && await isScreenRunningProcess(pid)) {
-            console.warn(`Screen ${screenName} is still running, killing it forcefully`);
-            await killScreen({ screenName });
-            if (pid && await isScreenRunningProcess(pid)) {
-                console.error(`I don't know what happened. The screen won't die. We can't do much else, I guess we'll just ignore it...`);
-            } else {
-                // Nested, to create the screen again.
-                return await runScreenCommand({
-                    screenName,
-                    command: config.command,
-                    folder: config.folder,
-                    record: config.record,
-                });
-            }
-        }
+    // A launch is ALWAYS a brand new session. Callers launch into the "-future" name and rename it over the canonical one once the new process is up, so starting a replacement never waits on the old process going away. Nothing is killed here.
+    // A leftover session under this name is from a launch that died before its takeover, so it is dead weight rather than something we are replacing.
+    if ((await getScreenState(false)).some(x => x.screenName === screenName)) {
+        console.log(red(`Removing the leftover session ${screenName} from a launch that never completed`));
+        await runPromise(`${prefix}tmux kill-session -t ${screenName}`);
     }
+    await runPromise(`${prefix}tmux new -s ${screenName} -d`);
+
     await removeOldNodeId(screenName);
     let folder = config.folder || os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
 
-    // The pane we are about to run the command in identifies the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream has to trust our bookkeeping about what is still running.
+    // The pane we are about to run the command in identifies the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream has to trust our bookkeeping about what is still running. The session was just created, so this pid is always new.
     let panePid = (await getScreenState(false)).find(x => x.screenName === screenName)?.pid;
     if (!panePid) {
         throw new Error(`Screen ${screenName} does not exist after creating it, so there is no process to run the command in`);
@@ -292,17 +268,11 @@ export const killScreen = measureWrap(async function killScreen(config: {
     // During a takeover the folder's nodeId file already belongs to the NEW process, so the old screen's kill must not remove it
     skipNodeIdRemoval?: boolean;
 }) {
-    console.log(red(`Killing screen ${config.screenName}`));
+    console.log(red(`Killing screen ${config.screenName} (Ctrl+C, then killing the session in ${formatTime(SHUTDOWN_GRACE_TIME)})`));
     let prefix = getTmuxPrefix();
-    // Try ctrl+c a few times first
-    let pid = (await getScreenState(false)).find(x => x.screenName === config.screenName)?.pid;
-    for (let i = 0; i < 5; i++) {
-        if (!pid || !await isScreenRunningProcess(pid)) {
-            break;
-        }
-        await runPromise(`${prefix}tmux send-keys -t ${config.screenName} 'C-c' Enter`);
-        await delay(5000);
-    }
+    // Purely a notice to the process that it is going away, so it can wind down. We never check whether it worked and we never reuse the screen - the session is killed either way.
+    await runPromise(`${prefix}tmux send-keys -t ${config.screenName} 'C-c' Enter`);
+    await delay(SHUTDOWN_GRACE_TIME);
     await runPromise(`${prefix}tmux kill-session -t ${config.screenName}`);
     if (!config.skipNodeIdRemoval) {
         await removeOldNodeId(config.screenName);