querysub 0.611.0 → 0.613.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.611.0",
+  "version": "0.613.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -75,7 +75,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.84",
+    "sliftutils": "^1.7.86",
     "socket-function": "^1.2.30",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/-a-archives/archives2.ts

@@ -73,7 +73,7 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
     ]);
 
     return createArchives({
-        version: 20,
+        version: 21,
         sources,
     });
 }

package/src/4-deploy/edgeNodes.ts

@@ -26,6 +26,8 @@ import { getGitRefLive, getGitURLLive } from "./git";
 import { DeployProgress } from "./deployFunctions";
 import { PromiseObj } from "../promise";
 import type { FunctionRunnerIndex } from "../4-querysub/FunctionRunnerTracking";
+import { isDefined } from "sliftutils/misc/types";
+
 const UPDATE_POLL_INTERVAL = timeInMinute * 15;
 const HEARTBEAT_INTERVAL = timeInMinute;
 const EDGE_NODE_DELETE_THRESHOLD = timeInMinute * 15;
@@ -34,11 +36,8 @@ const EDGE_NODE_READ_PARALLEL = 64;
 export const edgeNodeStorage = isNodeTrue() && getArchives2BufferedPublic("edgenodes2/");
 export const edgeNodeIndexFile = "edge-nodes-index.json";
 
-let registeredNodePaths = new Set<string>();
 function getNextNodePath() {
-    let path = "node0_" + getOwnNodeId();
-    registeredNodePaths.add(path);
-    return path;
+    return "node0_" + getOwnNodeId();
 }
 
 let edgeShutdown = false;
@@ -94,6 +93,7 @@ declare global {
     var EDGE_NODE_STATS: EdgeNodeStats | undefined;
 }
 let registeredEdgeNode: { host: string; entryPaths: string[] } | boolean | undefined;
+let lastEdgeNodeConfig: EdgeNodeConfig | undefined;
 export async function registerEdgeNode(config: {
     host: string;
     entryPaths: string[];
@@ -114,9 +114,7 @@ export async function registerEdgeNode(config: {
     registerShutdownHandler(async () => {
         edgeShutdown = true;
         console.log(magenta(`Removing node from edge node list due to shutdown`));
-        await Promise.all(Array.from(registeredNodePaths).map(async nodePath => {
-            await edgeNodeStorage.del(nodePath);
-        }));
+        await edgeNodeStorage.del(getNextNodePath());
         await updateEdgeNodesFile();
     });
     const { watchScheduledShutdown } = await import("../-g-core-values/scheduledShutdown");
@@ -128,9 +126,7 @@ export async function registerEdgeNode(config: {
             // Wait a bit to fully remove ourself from the edge node list
             await delay(Math.max(0, duration * 0.2));
             console.log(magenta(`Removing node from edge node list due to scheduled shutdown at ${new Date(time).toLocaleString()}`));
-            await Promise.all(Array.from(registeredNodePaths).map(async nodePath => {
-                await edgeNodeStorage.del(nodePath);
-            }));
+            await edgeNodeStorage.del(getNextNodePath());
             await updateEdgeNodesFile();
         })();
     });
@@ -177,7 +173,7 @@ export async function registerEdgeNode(config: {
             public: isPublic(),
             functionRunnerIndex: await getInitialFunctionRunnerIndex(),
         };
-
+        lastEdgeNodeConfig = edgeNodeConfig;
         await edgeNodeStorage.set(getNextNodePath(), Buffer.from(JSON.stringify(edgeNodeConfig)));
     }
     registeredEdgeNode = {
@@ -229,6 +225,7 @@ const loadEntryPointsByHash = runInSerial(async function loadEntryPointsByHash(c
         public: isPublic(),
         functionRunnerIndex: await getInitialFunctionRunnerIndex(),
     };
+    lastEdgeNodeConfig = edgeNodeConfig;
 
     await edgeNodeStorage.set(getNextNodePath(), Buffer.from(JSON.stringify(edgeNodeConfig)));
     console.log(magenta(`Registered edge node`), edgeNodeConfig);
@@ -245,14 +242,9 @@ async function getInitialFunctionRunnerIndex(): Promise<FunctionRunnerIndex | un
 }
 
 async function publishOwnScheduledShutdown(time: number) {
-    for (let nodePath of Array.from(registeredNodePaths)) {
-        let buffer = await edgeNodeStorage.get(nodePath);
-        if (!buffer) continue;
-        let config: EdgeNodeConfig = JSON.parse(buffer.toString());
-        if (config.scheduledShutdownTime === time) continue;
-        config.scheduledShutdownTime = time;
-        await edgeNodeStorage.set(nodePath, Buffer.from(JSON.stringify(config)));
-    }
+    if (!lastEdgeNodeConfig) return;
+    lastEdgeNodeConfig.scheduledShutdownTime = time;
+    await edgeNodeStorage.set(getNextNodePath(), Buffer.from(JSON.stringify(lastEdgeNodeConfig)));
     await updateEdgeNodesFile();
 }
 
@@ -271,12 +263,10 @@ const startUpdateLoop = lazy(async () => {
 
 async function heartbeatLoop() {
     if (edgeShutdown) return;
+    if (!lastEdgeNodeConfig) return;
     let nodePath = getNextNodePath();
-    let buffer = await edgeNodeStorage.get(nodePath);
-    if (!buffer) return;
-    let config: EdgeNodeConfig = JSON.parse(buffer.toString());
-    config.heartbeatTime = Date.now();
-    await edgeNodeStorage.set(nodePath, Buffer.from(JSON.stringify(config)));
+    lastEdgeNodeConfig.heartbeatTime = Date.now();
+    await edgeNodeStorage.set(nodePath, Buffer.from(JSON.stringify(lastEdgeNodeConfig)));
     // Push the fresh heartbeat into the index so clients see it (they read only the index, not the per-node files).
     await updateEdgeNodesFile();
 }
@@ -287,7 +277,6 @@ async function updateLoop() {
 }
 
 async function updateEdgeNodesFile() {
-    // Snapshot the time BEFORE reading anything below. Staleness is judged against when we STARTED, so if the reads block we can never mass-evict live nodes just because wall-clock advanced while we were stuck.
     let now = Date.now();
 
     let prevEdgeNodeFile = await edgeNodeStorage.get(edgeNodeIndexFile);
@@ -318,7 +307,6 @@ async function updateEdgeNodesFile() {
     let edgeNodeFiles = await edgeNodeStorage.find("node", { type: "files" });
     // Reads are independent, so run them in parallel (bounded): with many files a serial pass would take files × read-latency, which could be minutes.
     let readEdgeNodeFile = runInParallel({ parallelCount: EDGE_NODE_READ_PARALLEL }, async (nodeFile: string) => {
-        // Read directly, not through a cache: heartbeatTime mutates every minute, so a cached config would report a stale heartbeat and wrongly evict a live peer.
         let buffer = await edgeNodeStorage.get(nodeFile);
         if (!buffer) return undefined;
         let config: EdgeNodeConfig = JSON.parse(buffer.toString());
@@ -330,19 +318,9 @@ async function updateEdgeNodesFile() {
         }
         return config;
     });
-    let configs = await Promise.all(edgeNodeFiles.map(readEdgeNodeFile));
-
-    let bestByNodeId = new Map<string, EdgeNodeConfig>();
-    for (let config of configs) {
-        if (!config) continue;
-        // One entry per nodeId (re-registrations can leave more than one), keeping the freshest heartbeat.
-        let existing = bestByNodeId.get(config.nodeId);
-        if (!existing || (config.heartbeatTime || config.bootTime) > (existing.heartbeatTime || existing.bootTime)) {
-            bestByNodeId.set(config.nodeId, config);
-        }
-    }
+    let configs = (await Promise.all(edgeNodeFiles.map(readEdgeNodeFile))).filter(isDefined);
 
-    let newEdgeNodes = sort(Array.from(bestByNodeId.values()), x => -(x.heartbeatTime || x.bootTime));
+    let newEdgeNodes = sort(configs, x => -(x.heartbeatTime || x.bootTime));
     let newEdgeNodeIndex: EdgeNodesIndex = {
         edgeNodes: newEdgeNodes,
         liveHash,

package/src/deployManager/machineApplyMainCode.ts

@@ -26,7 +26,7 @@ import { PromiseObj } from "../promise";
 import path from "path";
 import { fsExistsAsync } from "../fs";
 import { ALIVE_WINDOW_FOREVER, ParametersTimelineEntry, syncParametersTimelineFiles } from "./parametersTimeline";
-import { getScreenName, getRetiringKillTime, retireScreen, retireCurrentScreen, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, getScreenPanePid, ensureProcessRecord, runScreenCommand, killScreen, readServiceNodeId, removeServiceNode, killScreenNow, withScreenLock } from "./processManager";
+import { getScreenName, getRetiringKillTime, getCanonicalScreenName, retireScreen, retireScreenWithMaxKillTime, retireCurrentScreen, getTmuxPrefix, getScreenState, isScreenRunningProcess, getProcessStartTime, getScreenPanePid, ensureProcessRecord, runScreenCommand, killScreen, readServiceNodeId, removeServiceNode, killScreenNow, withScreenLock } from "./processManager";
 import { ProcessRecord, syncProcessRecords, listProcessRecords } from "./processLogs";
 
 
@@ -261,6 +261,17 @@ function getMachineInstances(configs: ServiceConfig[], machineId: string): { con
     return instances;
 }
 
+// A version being replaced is renamed aside and given until this before it is taken down, which is the only thing an overlap is. Anchored to the release, not to whenever this resync happened to run, so every machine's old process goes at the same moment however staggered the resyncs are. Uses the NEWEST parameters' overlapTime, so shortening the overlap pulls the deadline in even for versions already retiring.
+function getInstanceKillTime(parameters: ServiceParameters): number {
+    return (parameters.releaseTime || Date.now()) + (parameters.overlapTime ?? DEFAULT_OVERLAP_TIME);
+}
+/** The canonical screen name of one instance - the version that should be running there. Single source of truth for the name, so the retiring-screen sweep derives it the same way ensureScreen does. */
+function getInstanceScreenName(config: ServiceConfig, index: number): string {
+    let live = getLiveServiceParameters(config);
+    let serviceKey = (live.deploy ? live : config.parameters).key;
+    return getScreenName({ serviceKey, index });
+}
+
 /** Puts one instance of a service into the state its configuration describes: the folder synced, the process running what it should be, and the records and timeline saying so. Returns the screen it is using, so anything nobody claimed can be killed. Everything about the screen happens under its lock, so two passes can never interleave. */
 async function ensureScreen(config: {
     config: ServiceConfig;
@@ -272,13 +283,12 @@ async function ensureScreen(config: {
     let serviceId = config.config.serviceId;
     let serviceConfig = config.config;
 
-    // A version being replaced is renamed aside and given until this before it is taken down, which is the only thing an overlap is. Anchored to the release, not to whenever this resync happened to run, so every machine's old process goes at the same moment however staggered the resyncs are.
-    let killTime = (serviceConfig.parameters.releaseTime || Date.now()) + (serviceConfig.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME);
+    let killTime = getInstanceKillTime(serviceConfig.parameters);
 
     // The screen name means one thing: the version that should be running. There is no second name to reason about - anything it replaced is already renamed aside and carries its own deadline.
     let parameters: ServiceParameters | undefined = getLiveServiceParameters(serviceConfig);
     if (!parameters.deploy) parameters = undefined;
-    let screenName = getScreenName({ serviceKey: (parameters || serviceConfig.parameters).key, index });
+    let screenName = getInstanceScreenName(serviceConfig, index);
     let folder = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
     // Returned even when the work below throws, so a transient failure never makes the sweep kill a screen we still want
     let usedScreenNames = [screenName];
@@ -500,6 +510,12 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
     // The fast-path results are memoized per resync, so a folder shared by two instances is only traversed once
     gitFastPathChecks.clear();
 
+    // The kill time each instance's overlap now wants, keyed by the canonical screen name its retiring predecessor was renamed from. Lets the sweep cap an already-retiring screen's deadline against the newest overlap instead of the (possibly longer) one baked into its name.
+    let killTimeByScreen = new Map<string, number>();
+    for (let { config, index } of instances) {
+        killTimeByScreen.set(getInstanceScreenName(config, index), getInstanceKillTime(config.parameters));
+    }
+
     // Every instance converges concurrently. Each has its own folder, its own records and its own lock, and yarn install takes a network mutex, so there is nothing to serialize behind - one slow git sync or shutdown no longer delays everything after it.
     let usedScreenNames = new Set((await Promise.all(instances.map(instance => ensureScreen({ ...instance, machineInfo })))).flat());
 
@@ -508,7 +524,14 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
         if (usedScreenNames.has(screenName)) continue;
         // A retiring screen is not unclaimed - it is a version that was replaced, and its name says when it is due to go. Cached per name, so seeing it on every resync only ever starts one, and a restart picks it up again from the name alone.
         if (getRetiringKillTime(screenName) !== undefined) {
-            void retireScreen(screenName);
+            let canonicalScreenName = getCanonicalScreenName(screenName);
+            // Capped against the newest overlap when we still have that instance's config, so a shortened overlap pulls the deadline in (down to now, killing it) rather than letting it wait out the time baked into its name. Without a matching config we cannot know the new overlap, so we honor the name.
+            let maxKillTime = canonicalScreenName !== undefined ? killTimeByScreen.get(canonicalScreenName) : undefined;
+            if (maxKillTime !== undefined && canonicalScreenName !== undefined) {
+                void retireScreenWithMaxKillTime({ screenName, folder: root + canonicalScreenName + "/", maxKillTime });
+            } else {
+                void retireScreen(screenName);
+            }
             continue;
         }
         // Detached, so a screen that takes its full grace period to go down doesn't hold up the end of the resync

package/src/deployManager/processManager.ts

@@ -356,6 +356,47 @@ export async function retireCurrentScreen(config: {
     void retireScreen(retiringScreenName);
 }
 
+/** The canonical name a retiring screen was renamed from - the inverse of getRetiringScreenName. Undefined for a name that is not a retiring screen. */
+export function getCanonicalScreenName(screenName: string): string | undefined {
+    let marker = screenName.lastIndexOf(RETIRING_MARKER);
+    if (marker === -1) return undefined;
+    return screenName.slice(0, marker) + SCREEN_SUFFIX;
+}
+
+/** Schedules a retiring screen's takedown, capped at maxKillTime: a shortened overlap can leave a screen scheduled to go later than it now should, and since the deadline lives in the name, the only way to move it is to rename the screen. When the cap is earlier than the time baked into the name, this renames it to carry the earlier time (down to now, which retireScreen then acts on immediately, killing it) and re-arms against the new name. When nothing needs pulling in, it just re-arms the existing timer, so it is safe to call every sweep. */
+export async function retireScreenWithMaxKillTime(config: { screenName: string; folder: string; maxKillTime: number }): Promise<void> {
+    let { screenName, folder } = config;
+    let maxKillTime = Math.round(config.maxKillTime);
+    let currentKillTime = getRetiringKillTime(screenName);
+    if (currentKillTime === undefined) return;
+    let canonicalScreenName = getCanonicalScreenName(screenName);
+    if (canonicalScreenName !== undefined && maxKillTime < currentKillTime) {
+        let newScreenName = getRetiringScreenName(canonicalScreenName, maxKillTime);
+        if (newScreenName !== screenName) {
+            console.log(green(`Pulling in retiring ${screenName} to ${newScreenName} (kill time ${new Date(currentKillTime).toISOString()} -> ${new Date(maxKillTime).toISOString()})`));
+            let pid = await getScreenPanePid(screenName);
+            // The pending timer is waiting on the OLD, later deadline; drop it so a fresh one runs against the new name's earlier deadline.
+            retireScreen.clear(screenName);
+            try {
+                await runPromise(`${getTmuxPrefix()}tmux rename-session -t ${screenName} ${newScreenName}`);
+            } catch (e) {
+                // Another pass may have already renamed or killed it
+                console.log(`Could not rename retiring ${screenName} to ${newScreenName} (already renamed or gone?): ${(e as Error).stack ?? e}`);
+                return;
+            }
+            if (pid) {
+                let record = await findProcessRecordByPid(folder, pid);
+                if (record) {
+                    await writeProcessRecord({ ...record, screenName: newScreenName });
+                }
+            }
+            void retireScreen(newScreenName);
+            return;
+        }
+    }
+    void retireScreen(screenName);
+}
+
 /** Ends a session at once, with no notice - for a screen that should not exist at all, so there is nothing to wind down gracefully. Finding one is always a surprise, so the reason it was not supposed to be there is logged. */
 export async function killScreenNow(screenName: string, reason: string): Promise<void> {
     let panePid = await getScreenPanePid(screenName);