querysub 0.574.0 → 0.576.0

package/appSecrets.ts

@@ -3,6 +3,7 @@ import { lazy } from "socket-function/src/caching";
 import { ArchivesBackblaze } from "sliftutils/storage/backblaze";
 import { STORAGE_DIR, getDomain, getBackblazePath } from "./src/misc/appPaths";
 import { Querysub } from "./src/4-querysub/Querysub";
+import { formatDateTime } from "socket-function/src/formatting/format";
 
 const keysArchives = lazy(() => new ArchivesBackblaze({ bucketName: getDomain() }));
 
@@ -55,7 +56,7 @@ let doImmediateShutdown = lazy(() => {
         });
     }
     function doShutdown() {
-        console.log("SIGINT received, shutting down immediately");
+        console.log(`SIGINT received, shutting down immediately (${formatDateTime(Date.now())})`);
         process.exit();
     }
     process.on("SIGINT", doShutdown);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.574.0",
+  "version": "0.576.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -71,7 +71,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.43",
+    "sliftutils": "^1.7.45",
     "socket-function": "^1.2.26",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/deployManager/processLogs.ts

@@ -104,6 +104,16 @@ export async function findProcessRecordByPid(folder: string, pid: string): Promi
     return (await readFolderRecords(folder)).find(x => x.pid === pid && x.deadTime === undefined);
 }
 
+/** Marks a process dead now, for a process we killed ourselves. Without this the record reads as running until the next resync happens to look, which is a long time to show something that we know is gone. */
+export async function markProcessDead(pid: string, now: number): Promise<void> {
+    for (let record of await listProcessRecords()) {
+        if (record.pid !== pid || record.deadTime !== undefined) continue;
+        record.deadTime = now;
+        console.log(`Process ${record.pid} (${record.screenName}) was killed, marking it dead. Its log is deleted at ${new Date(record.deadTime + DEAD_PROCESS_RETENTION).toISOString()}`);
+        await writeProcessRecord(record);
+    }
+}
+
 export async function writeProcessRecord(record: ProcessRecord): Promise<void> {
     await fs.promises.mkdir(path.join(record.folder, PROCESS_FOLDER), { recursive: true });
     await fs.promises.writeFile(getRecordPath(record.folder, record.pid, record.startTime), JSON.stringify(record));

package/src/deployManager/processManager.ts

@@ -13,7 +13,7 @@ import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
 import { fsExistsAsync } from "../fs";
 import { PromiseObj } from "../promise";
 import { SERVICE_FOLDER, SERVICE_NODE_FILE_NAME } from "./machineSchema";
-import { ProcessRecord, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, findProcessRecordByPid, writeProcessRecord } from "./processLogs";
+import { ProcessRecord, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, findProcessRecordByPid, writeProcessRecord, markProcessDead } from "./processLogs";
 import { setPreciseTimeout } from "../misc";
 
 // Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
@@ -256,9 +256,11 @@ export const runScreenCommand = measureWrap(async function runScreenCommand(conf
     let screenName = config.screenName;
 
     // Always a brand new session, so the pane is always new - and the pane's pid is what makes this a distinct process with its own log. A session already under this name is debris from a start that never got taken over.
-    if (await getScreenPanePid(screenName)) {
+    let leftoverPid = await getScreenPanePid(screenName);
+    if (leftoverPid) {
         console.log(red(`Removing the leftover session ${screenName} from a start that never completed`));
         await runPromise(`${prefix}tmux kill-session -t ${screenName}`);
+        await markProcessDead(leftoverPid, Date.now());
     }
     await runPromise(`${prefix}tmux new -s ${screenName} -d`);
 
@@ -339,21 +341,27 @@ export async function retireCurrentScreen(config: {
     let retiringScreenName = getRetiringScreenName(canonicalScreenName, killTime);
     console.log(green(`Retiring ${canonicalScreenName} as ${retiringScreenName}`));
     await runPromise(`${getTmuxPrefix()}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
-    let record = await findProcessRecordByPid(folder, outgoingPid);
-    if (record) {
-        await writeProcessRecord({ ...record, screenName: retiringScreenName });
-    }
+    void (async () => {
+        let record = await findProcessRecordByPid(folder, outgoingPid);
+        if (record) {
+            await writeProcessRecord({ ...record, screenName: retiringScreenName });
+        }
+    })();
     if (nodeId) {
         // The file in the folder belongs to its replacement now, so only the registration goes
         void removeServiceNode({ folder, nodeId, ownsFile: false });
     }
+    // Started here rather than waiting for a resync to notice the screen: with a killTime already past - a zero overlap - this takes it down right now
+    void retireScreen(retiringScreenName);
 }
 
 /** Ends a session at once, with no notice - for a screen that should not exist at all, so there is nothing to wind down gracefully. Finding one is always a surprise, so the reason it was not supposed to be there is logged. */
 export async function killScreenNow(screenName: string, reason: string): Promise<void> {
-    if (!await getScreenPanePid(screenName)) return;
+    let panePid = await getScreenPanePid(screenName);
+    if (!panePid) return;
     console.log(red(`Unexpected screen ${screenName}: ${reason}. Killing it immediately.`));
     await runPromise(`${getTmuxPrefix()}tmux kill-session -t ${screenName}`);
+    await markProcessDead(panePid, Date.now());
 }
 
 // Kills a screen and nothing else. Which node the process registered as, and whether that registration should be dropped, is the deploy logic's business - it reads the node id off the process's record, which was taken before any replacement could overwrite the file.
@@ -362,10 +370,15 @@ export const killScreen = measureWrap(async function killScreen(config: {
 }) {
     console.log(red(`Killing screen ${config.screenName} (Ctrl+C, then killing the session in ${formatTime(SHUTDOWN_GRACE_TIME)})`));
     let prefix = getTmuxPrefix();
+    // Taken before the grace period, as the process is free to exit on the Ctrl+C and take the pane's pid with it
+    let panePid = await getScreenPanePid(config.screenName);
     // Purely a notice to the process that it is going away, so it can wind down. We never check whether it worked and we never reuse the screen - the session is killed either way.
-    await runPromise(`${prefix}tmux send-keys -t ${config.screenName} 'C-c' Enter`);
+    void runPromise(`${prefix}tmux send-keys -t ${config.screenName} 'C-c' Enter`);
     await delay(SHUTDOWN_GRACE_TIME);
     await runPromise(`${prefix}tmux kill-session -t ${config.screenName}`);
+    if (panePid) {
+        await markProcessDead(panePid, Date.now());
+    }
 });
 
 /** Streams one process's log: everything already in it, then everything appended after. One process, one file - nothing here has to reason about which process a byte came from. */