querysub 0.560.0 → 0.562.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.560.0",
+  "version": "0.562.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/deployManager/components/ProcessesView.tsx

@@ -17,6 +17,7 @@ const DEAD_COLOR = { h: 0, s: 0, l: 92 };
 const OUTPUT_BUFFER_LIMIT = 1_000_000;
 const OUTPUT_BUFFER_KEPT = 100_000;
 const OUTPUT_MAX_HEIGHT = "40vh";
+const DETAIL_FONT_SIZE = 12;
 const ANSI_SATURATION = 70;
 const ANSI_LIGHTNESS = 70;
 
@@ -113,6 +114,17 @@ class ProcessRow extends qreact.Component<{ record: ProcessRecord; machineNodeId
                     {this.state.watching && "Hide output" || "Watch output"}
                 </Button>
             </div>
+            {/* The ref and the command are both long, so they get their own line rather than pushing the summary around */}
+            <div className={css.hbox(10).wrap.fontSize(DETAIL_FONT_SIZE).colorhsl(0, 0, 30)}>
+                {record.parameters.gitRef && <div className={css.hbox(4)}>
+                    <span className={css.colorhsl(0, 0, 45)}>ref</span>
+                    <span className={css.fontFamily("monospace")}>{record.parameters.gitRef}</span>
+                </div>}
+                {record.parameters.command && <div className={css.hbox(4).flexGrow(1)}>
+                    <span className={css.colorhsl(0, 0, 45)}>command</span>
+                    <span className={css.fontFamily("monospace").whiteSpace("pre-wrap")}>{record.parameters.command}</span>
+                </div>}
+            </div>
             {this.state.watching && <ProcessOutput record={record} machineNodeId={this.props.machineNodeId} />}
         </div>;
     }

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -346,6 +346,8 @@ export class ServiceDetailPage extends qreact.Component {
             {/* Machine Status */}
             {config.parameters.deploy && <div className={css.vbox(8).fillWidth}>
                 <h3>Deployed Machines ({machineIds.length})</h3>
+                {/* The machines are notified on every config change and poll as a fallback, so forcing a resync had no use in practice. Kept in case that changes.
+                <ResyncMachinesButton applyNodeIds={machineStatuses.map(x => x.machineInfo?.applyNodeId || "")} /> */}
                 <div className={css.vbox(4).fillWidth}>
                     {machineStatuses.map(({ machineId, variables, machineInfo, serviceInfo, isMachineDead, hasError, isDisabled, index }) => {
                         if (!machineInfo) return <div key={machineId}>Loading {machineId}...</div>;

package/src/deployManager/components/ServicesListPage.tsx

@@ -10,6 +10,7 @@ import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { cache } from "socket-function/src/caching";
 import { Anchor } from "../../library-components/ATag";
 import { isPublic } from "../../config";
+// ResyncMachinesButton is still exported from deployButtons, for whenever forcing a resync becomes useful
 import { PendingDeployInfo, UpdateButtons, UpdateServiceButtons } from "./deployButtons";
 import { isDefined } from "../../misc";
 import { formatDateJSX } from "../../misc/formatJSX";
@@ -43,6 +44,10 @@ export class ServicesListPage extends qreact.Component {
         return <div className={css.vbox(16)}>
             <div className={css.hbox(12).wrap}>
                 <h2 className={css.flexGrow(1)}>Services</h2>
+                {/* The machines are notified on every config change and poll as a fallback, so forcing a resync had no use in practice. Kept in case that changes.
+                <ResyncMachinesButton applyNodeIds={(machineConfigList || []).map(x =>
+                    controller.getMachineInfo(x.machineId)?.applyNodeId || ""
+                )} /> */}
                 <button className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(0, 0, 100)}
                     onClick={() => {
                         Querysub.onCommitFinished(async () => {

package/src/deployManager/components/deployButtons.tsx

@@ -19,6 +19,37 @@ export function bigEmoji(emoji: string, topOffset = 0) {
     return <span className={css.fontSize(26).marginTop(topOffset)}>{emoji}</span>;
 }
 
+/** Makes the machines re-read the service configs immediately, instead of waiting out their fallback poll. Useful when a config was changed without the machines being notified, or a notification was missed. */
+export class ResyncMachinesButton extends qreact.Component<{ applyNodeIds: string[] }> {
+    state = t.state({
+        isResyncing: t.type(false),
+    });
+    render() {
+        let applyNodeIds = unique(this.props.applyNodeIds.filter(x => x));
+        if (!applyNodeIds.length) return undefined;
+        return <button
+            className={css.pad2(12, 8).button.bord2(0, 0, 20).hsl(200, 70, 90)}
+            disabled={this.state.isResyncing}
+            title="Tells each machine's apply code to re-read the service configs now"
+            onClick={() => {
+                this.state.isResyncing = true;
+                let controller = MachineController(SocketFunction.browserNodeId());
+                Querysub.onCommitFinished(async () => {
+                    try {
+                        await Promise.all(applyNodeIds.map(nodeId => controller.resyncMachine.promise({ nodeId })));
+                    } finally {
+                        Querysub.commit(() => {
+                            this.state.isResyncing = false;
+                        });
+                    }
+                });
+            }}
+        >
+            {this.state.isResyncing && "⏳ Resyncing..." || `🔄 Resync ${applyNodeIds.length} machine${applyNodeIds.length === 1 && "" || "s"}`}
+        </button>;
+    }
+}
+
 export class RenderGitRefInfo extends qreact.Component<{
     gitRef: string;
     isQuerysub?: boolean;

package/src/deployManager/machineApplyMainCode.ts

@@ -765,95 +765,6 @@ async function getPPID(pid: string) {
     }
 }
 
-// Node prints this when the process has exited but the V8 inspector is still
-//  holding it open waiting for a debugger client to detach. If a debugger client
-//  drops uncleanly the process can sit on this forever, holding the screen and
-//  preventing the supervisor from spawning a fresh instance. The watcher below
-//  polls each running process's log for this phrase and sends Ctrl+C to break it out.
-const DEBUGGER_DISCONNECT_MESSAGE = "Waiting for the debugger to disconnect...";
-
-// To avoid false positives from app code that happens to log a string
-//  containing the phrase, the wedge is only reported when:
-//    1. the phrase is on its own line (start-of-string or after a newline), AND
-//    2. nothing else has been logged after it (only trailing whitespace/EOF).
-//  In a real wedge, this line is the very last thing Node writes before going
-//  silent — so any later output rules out the wedge interpretation.
-const DEBUGGER_DISCONNECT_TAIL_PATTERN = /(?:^|\r?\n)Waiting for the debugger to disconnect\.\.\.\s*$/;
-
-// How often the wedge watcher scans every running process's log.
-const DEBUGGER_WEDGE_POLL_INTERVAL = timeInSecond * 15;
-// How much of the tail of the log to read on each scan. The phrase is short
-//  and appears near the very end of the log when a process is stuck, so a few
-//  KB is plenty and keeps scans cheap.
-const DEBUGGER_WEDGE_TAIL_BYTES = 8 * 1024;
-// Suppress repeated Ctrl+C bursts to the same screen while it tears down; if
-//  the first signal didn't break the wait, give it this long before we try again.
-const DEBUGGER_WEDGE_RESIGNAL_DELAY = timeInMinute;
-// After a first match, wait this long and re-check before signaling. If the
-//  process is still alive and just happened to log a message ending in the
-//  phrase, more output will appear in the meantime and the second check fails.
-const DEBUGGER_WEDGE_RECHECK_DELAY = timeInSecond * 3;
-
-// screenName -> last time we sent Ctrl+C because of a debugger wedge.
-const lastDebuggerWedgeSignal = new Map<string, number>();
-
-async function readPipeFileTail(pipeFile: string, maxBytes: number): Promise<string> {
-    let handle: fs.promises.FileHandle | undefined;
-    try {
-        handle = await fs.promises.open(pipeFile, "r");
-        let stat = await handle.stat();
-        let start = Math.max(0, stat.size - maxBytes);
-        let length = stat.size - start;
-        if (length <= 0) return "";
-        let buf = Buffer.alloc(length);
-        await handle.read(buf, 0, length, start);
-        return buf.toString("utf8");
-    } catch {
-        // File doesn't exist yet, or got truncated mid-read — treat as empty.
-        return "";
-    } finally {
-        if (handle) {
-            try {
-                await handle.close();
-            } catch {
-                // ignore
-            }
-        }
-    }
-}
-
-async function unwedgeStuckDebuggerScreens(): Promise<void> {
-    let prefix = getTmuxPrefix();
-    let screens = await getScreenState(false);
-    let running = (await listProcessRecords()).filter(x => x.deadTime === undefined);
-    for (let { screenName } of screens) {
-        let record = running.find(x => x.screenName === screenName);
-        if (!record) continue;
-        let pipeFile = getProcessLogPath(record.folder, record.launchId);
-        let tail1 = await readPipeFileTail(pipeFile, DEBUGGER_WEDGE_TAIL_BYTES);
-        if (!DEBUGGER_DISCONNECT_TAIL_PATTERN.test(tail1)) continue;
-
-        // Confirm the process is actually wedged and not just briefly idle by
-        //  re-checking after a short delay. A live process will write more
-        //  output in this window, which moves the phrase away from the end
-        //  and fails the second match.
-        await delay(DEBUGGER_WEDGE_RECHECK_DELAY);
-        let tail2 = await readPipeFileTail(pipeFile, DEBUGGER_WEDGE_TAIL_BYTES);
-        if (!DEBUGGER_DISCONNECT_TAIL_PATTERN.test(tail2)) continue;
-
-        let last = lastDebuggerWedgeSignal.get(screenName) ?? 0;
-        if (Date.now() - last < DEBUGGER_WEDGE_RESIGNAL_DELAY) continue;
-        lastDebuggerWedgeSignal.set(screenName, Date.now());
-
-        console.warn(red(`Detected stuck "${DEBUGGER_DISCONNECT_MESSAGE}" in ${screenName} (confirmed across two checks ${DEBUGGER_WEDGE_RECHECK_DELAY}ms apart); sending Ctrl+C to unblock it.`));
-        try {
-            await runPromise(`${prefix}tmux send-keys -t ${screenName} 'C-c' Enter`);
-        } catch (e: any) {
-            console.warn(`Failed to send Ctrl+C to ${screenName}: ${e.stack ?? e}`);
-        }
-    }
-}
-
 export async function machineApplyMain() {
     let parentPID = process.argv[2];
     // Wait for the console to get shimmed
@@ -882,7 +793,6 @@ export async function machineApplyMain() {
     await Querysub.hostService("machine-apply");
     onServiceConfigChange(resyncServices);
 
-    runInfinitePoll(DEBUGGER_WEDGE_POLL_INTERVAL, unwedgeStuckDebuggerScreens);
 
     runInfinitePoll(timeInMinute * 3, async () => {
         //console.log(magenta(`Quick outdated check at ${new Date().toISOString()}`));

package/src/deployManager/machineController.ts

@@ -73,6 +73,10 @@ export const OnServiceChange = SocketFunction.register(
 
 class MachineControllerBase {
     // NOTE: We don't need to worry about escaping commands here. YES, the user CAN inject code into the key. But this system is literally for running arbitrary commands, so they could just write a serviceConfig and run anything they want, on all the machines...
+    /** Tells a machine's apply code to re-read the service configs now, rather than waiting for its 15 minute fallback poll. Forwarded, as clients can't reach the machines directly. */
+    public async resyncMachine(config: { nodeId: string }): Promise<void> {
+        await OnServiceChange.nodes[config.nodeId].onServiceConfigChange();
+    }
     /** Every process this machine has a record of, running or dead. */
     public async listProcesses(): Promise<ProcessRecord[]> {
         return await listProcessRecords();
@@ -143,6 +147,7 @@ export const MachineController = getSyncedController(SocketFunction.register(
     "machine-controller-c3157d4a-580c-4e76-9dc9-072dd92e70af",
     () => new MachineControllerBase(),
     () => ({
+        resyncMachine: {},
         listProcesses: {},
         streamProcessOutput: {},
         listOtherProcesses: {},

package/src/deployManager/processManager.ts

@@ -5,6 +5,8 @@ import { spawn, ChildProcess } from "child_process";
 import { lazy } from "socket-function/src/caching";
 import { measureWrap } from "socket-function/src/profiling/measure";
 import { delay, runInSerial } from "socket-function/src/batching";
+import { timeInSecond } from "socket-function/src/misc";
+import { formatTime } from "socket-function/src/formatting/format";
 import { red, green } from "socket-function/src/formatting/logColors";
 import { runPromise } from "../functional/runCommand";
 import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
@@ -15,6 +17,9 @@ import { ProcessRecord, createLaunchId, getLogPipeScript, getLogTailScript, getP
 
 // Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
 
+// How long a process gets between being told to shut down and having its session killed. It is a courtesy, not a negotiation - nothing waits on the process actually exiting.
+const SHUTDOWN_GRACE_TIME = timeInSecond * 15;
+
 const SCREEN_SUFFIX = "-dply";
 export function getScreenName(config: { serviceKey: string; index: number }): string {
     return `${config.serviceKey}-${config.index}${SCREEN_SUFFIX}`.replace(/[^a-zA-Z0-9\-_]/g, "_");
@@ -178,47 +183,16 @@ export const runScreenCommand = measureWrap(async function runScreenCommand(conf
     let prefix = getTmuxPrefix();
     let screenName = config.screenName;
 
-    try {
-        // Throw if it already exists
-        await runPromise(`${prefix}tmux new -s ${screenName} -d`);
-    } catch { }
-    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'echo "Updating running command at ${new Date().toISOString()}"' Enter`);
-    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'C-c' Enter`);
-    await delay(1000);
-
-
-    let screens = await getScreenState();
-    let screen = screens.find(x => x.screenName === screenName);
-    let pid = screen?.pid;
-    if (pid && await isScreenRunningProcess(pid)) {
-        // It doesn't want to die. Wait longer, but it it just won't die, kill the screen
-        console.warn(`Screen ${screenName} is not dying, giving it another 30 seconds`);
-        for (let i = 0; i < 6; i++) {
-            await delay(5);
-            if (!await isScreenRunningProcess(pid)) {
-                break;
-            }
-        }
-        if (pid && await isScreenRunningProcess(pid)) {
-            console.warn(`Screen ${screenName} is still running, killing it forcefully`);
-            await killScreen({ screenName });
-            if (pid && await isScreenRunningProcess(pid)) {
-                console.error(`I don't know what happened. The screen won't die. We can't do much else, I guess we'll just ignore it...`);
-            } else {
-                // Nested, to create the screen again.
-                return await runScreenCommand({
-                    screenName,
-                    command: config.command,
-                    folder: config.folder,
-                    record: config.record,
-                });
-            }
-        }
+    // A launch is ALWAYS a new session, never a new command in the old one. Reusing the session kept the pane, and the pane's pid and start time are what identify the process - so the "new" process was indistinguishable from the one before it and appended to its log. Every path goes through this, including an immediate zero-overlap deploy: a special case for that would be a second flow to get wrong.
+    if ((await getScreenState(false)).some(x => x.screenName === screenName)) {
+        await killScreen({ screenName, skipNodeIdRemoval: true });
     }
+    await runPromise(`${prefix}tmux new -s ${screenName} -d`);
+
     await removeOldNodeId(screenName);
     let folder = config.folder || os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
 
-    // The pane we are about to run the command in identifies the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream has to trust our bookkeeping about what is still running.
+    // The pane we are about to run the command in identifies the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream has to trust our bookkeeping about what is still running. The session was just created, so this pid is always new.
     let panePid = (await getScreenState(false)).find(x => x.screenName === screenName)?.pid;
     if (!panePid) {
         throw new Error(`Screen ${screenName} does not exist after creating it, so there is no process to run the command in`);
@@ -292,17 +266,11 @@ export const killScreen = measureWrap(async function killScreen(config: {
     // During a takeover the folder's nodeId file already belongs to the NEW process, so the old screen's kill must not remove it
     skipNodeIdRemoval?: boolean;
 }) {
-    console.log(red(`Killing screen ${config.screenName}`));
+    console.log(red(`Killing screen ${config.screenName} (Ctrl+C, then killing the session in ${formatTime(SHUTDOWN_GRACE_TIME)})`));
     let prefix = getTmuxPrefix();
-    // Try ctrl+c a few times first
-    let pid = (await getScreenState(false)).find(x => x.screenName === config.screenName)?.pid;
-    for (let i = 0; i < 5; i++) {
-        if (!pid || !await isScreenRunningProcess(pid)) {
-            break;
-        }
-        await runPromise(`${prefix}tmux send-keys -t ${config.screenName} 'C-c' Enter`);
-        await delay(5000);
-    }
+    // Purely a notice to the process that it is going away, so it can wind down. We never check whether it worked and we never reuse the screen - the session is killed either way.
+    await runPromise(`${prefix}tmux send-keys -t ${config.screenName} 'C-c' Enter`);
+    await delay(SHUTDOWN_GRACE_TIME);
     await runPromise(`${prefix}tmux kill-session -t ${config.screenName}`);
     if (!config.skipNodeIdRemoval) {
         await removeOldNodeId(config.screenName);