querysub 0.509.0 → 0.511.0

package/src/deployManager/components/ServicesListPage.tsx

@@ -48,9 +48,9 @@ export class ServicesListPage extends qreact.Component {
                             const newServiceId = `service-${Math.round(Date.now())}`;
                             const newConfig: ServiceConfig = {
                                 serviceId: newServiceId,
-                                machineIds: [],
                                 parameters: {
                                     key: newServiceId,
+                                    machineIds: [],
                                     repoUrl: gitInfo.repoUrl,
                                     gitRef: gitInfo.latestRef,
                                     command: "ping 1.1.1.1",
@@ -80,11 +80,12 @@ export class ServicesListPage extends qreact.Component {
                 {services.map(([serviceId, config]) => {
                     if (!config) return <div key={serviceId}>Config is broken? Missing value for service? Is the file corrupted?</div>;
 
-                    let disabledMachines = config.machineIds.filter(machineId => {
+                    let machineIds = config.parameters.machineIds || [];
+                    let disabledMachines = machineIds.filter(machineId => {
                         let machineConfig = getMachineConfig(machineId);
                         return machineConfig?.disabled;
                     });
-                    let enabledMachineIds = config.machineIds.filter(machineId => {
+                    let enabledMachineIds = machineIds.filter(machineId => {
                         let machineConfig = getMachineConfig(machineId);
                         return !machineConfig?.disabled;
                     });
@@ -138,7 +139,7 @@ export class ServicesListPage extends qreact.Component {
                                 </div>
                                 <div className={css.vbox(4)}>
                                     <div>Updated {formatDateJSX(config.info.lastUpdatedTime)} AGO</div>
-                                    {config.parameters.overlapTime && <div className={css.colorhsl(210, 50, 50).fontSize(14).fontWeight("bold")}>Switchover overlap: {formatTime(config.parameters.overlapTime)}</div> || undefined}
+                                    {config.parameters.overlapTime && <div className={css.colorhsl(210, 50, 50).fontSize(14).fontWeight("bold")}>Deploy overlap: {formatTime(config.parameters.overlapTime)}</div> || undefined}
                                 </div>
                             </div>
                         </Anchor>

package/src/deployManager/components/Tools.tsx

@@ -125,11 +125,14 @@ class RenameMachine extends qreact.Component {
                                 );
                                 let toUpdate: ServiceConfig[] = [];
                                 for (let config of configs.filter(isDefined)) {
-                                    if (!config.machineIds.includes(from)) continue;
-                                    let newMachineIds = config.machineIds.map(m => m === from ? to : m);
+                                    let machineIds = config.parameters.machineIds || [];
+                                    if (!machineIds.includes(from)) continue;
                                     toUpdate.push({
                                         ...config,
-                                        machineIds: newMachineIds,
+                                        parameters: {
+                                            ...config.parameters,
+                                            machineIds: machineIds.map(m => m === from ? to : m),
+                                        },
                                     });
                                 }
                                 if (toUpdate.length === 0) {

package/src/deployManager/components/deployButtons.tsx

@@ -1,6 +1,6 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { qreact } from "../../4-dom/qreact";
-import { MachineServiceController, MachineInfo, ServiceConfig } from "../machineSchema";
+import { DEFAULT_OVERLAP_TIME, MachineServiceController, MachineInfo, ServiceConfig } from "../machineSchema";
 import { showFullscreenModal } from "../../5-diagnostics/FullscreenModal";
 import { css } from "../../4-dom/css";
 import { formatTime } from "socket-function/src/formatting/format";
@@ -97,6 +97,9 @@ export class UpdateButtons extends qreact.Component<{
                     onClick={async () => {
                         for (let service of outdatedServices) {
                             service.parameters.gitRef = gitInfo.latestRef;
+                            // Release with the configured overlap, so the new instances are running before the old ones shut down
+                            let overlap = service.parameters.overlapTime || DEFAULT_OVERLAP_TIME;
+                            service.parameters.releaseTime = Date.now() + overlap;
                         }
                         await controller.setServiceConfigs.promise(outdatedServices);
                     }}
@@ -133,6 +136,9 @@ export class UpdateServiceButtons extends qreact.Component<{
                 className={buttonStyle.hsl(120, 70, 90).alignSelf("stretch")}
                 onClick={async () => {
                     this.props.service.parameters.gitRef = gitInfo.latestRef;
+                    // Release with the configured overlap, so the new instances are running before the old ones shut down
+                    let overlap = this.props.service.parameters.overlapTime || DEFAULT_OVERLAP_TIME;
+                    this.props.service.parameters.releaseTime = Date.now() + overlap;
                     await controller.setServiceConfigs.promise([this.props.service]);
                 }}
             >

package/src/deployManager/machineApplyMainCode.ts

@@ -4,14 +4,14 @@ import { measureWrap } from "socket-function/src/profiling/measure";
 import { getOwnMachineId } from "sliftutils/misc/https/certs";
 import { forceRemoveNode, getOurNodeId, getOurNodeIdAssert } from "../-f-node-discovery/NodeDiscovery";
 import { Querysub } from "../4-querysub/Querysub";
-import { MACHINE_RESYNC_INTERVAL, MachineServiceControllerBase, MachineInfo, ServiceConfig, ServiceSwitchover, serviceConfigs, pendingServiceConfigs, serviceSwitchovers, SERVICE_FOLDER, machineInfos, SERVICE_NODE_FILE_NAME, getEffectiveServiceConfigs, recordLaunch, DEFAULT_OVERLAP_TIME } from "./machineSchema";
+import { MACHINE_RESYNC_INTERVAL, MachineServiceControllerBase, MachineInfo, ServiceConfig, ServiceParameters, SERVICE_FOLDER, machineInfos, SERVICE_NODE_FILE_NAME, getEffectiveServiceConfigs, getLiveServiceParameters, restartParametersKey, recordLaunch, DEFAULT_OVERLAP_TIME } from "./machineSchema";
 import { runPromise } from "../functional/runCommand";
 import { getExternalIP } from "socket-function/src/networking";
 import { errorToUndefined, errorToUndefinedSilent } from "../errors";
 import { getDomain } from "../config";
 import { formatTime } from "socket-function/src/formatting/format";
 import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
-import { isDefined } from "../misc";
+import { isDefined, setPreciseTimeout } from "../misc";
 import { logLoadTime } from "../logModuleLoadTimes";
 import { delay, retryFunctional, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import os from "os";
@@ -132,7 +132,7 @@ const SCREEN_SUFFIX = "-dply";
 export function getScreenName(config: { serviceKey: string; index: number }): string {
     return `${config.serviceKey}-${config.index}${SCREEN_SUFFIX}`.replace(/[^a-zA-Z0-9\-_]/g, "_");
 }
-// The new instances started during a switchover overlap. They cover the service while the canonical screens are restarted with the new config at switch time, and are killed shortly after.
+// The new instances started during a release overlap. They cover the service while the canonical screens are restarted with the new config at release time, and are killed shortly after.
 export function getNextScreenName(config: { serviceKey: string; index: number }): string {
     return getScreenName({ serviceKey: config.serviceKey + "-next", index: config.index });
 }
@@ -585,56 +585,30 @@ const ensureGitSynced = measureWrap(async function ensureGitSynced(config: {
 let launchesPerService = new Map<string, number>();
 let lastLaunchedTimePerService = new Map<string, number>();
 
-// Promotes any switchovers that have reached their switch time (the pending config becomes live), and returns the switchovers still in flight.
-async function processSwitchovers(): Promise<Map<string, { switchover: ServiceSwitchover; pending: ServiceConfig }>> {
-    let active = new Map<string, { switchover: ServiceSwitchover; pending: ServiceConfig }>();
-    let switchovers = await serviceSwitchovers.values();
-    for (let switchover of switchovers) {
-        let pending = await pendingServiceConfigs.get(switchover.serviceId);
-        if (!pending) {
-            console.warn(`Switchover for ${switchover.serviceId} has no pending config, removing it`);
-            await serviceSwitchovers.delete(switchover.serviceId);
-            continue;
-        }
-        if (Date.now() >= switchover.switchTime) {
-            // Multiple apply nodes may race to promote, which is fine, as they all write the same pending config.
-            console.log(green(`Promoting pending config for ${switchover.serviceId} (switchover scheduled for ${new Date(switchover.switchTime).toLocaleString()})`));
-            await serviceConfigs.set(switchover.serviceId, pending);
-            await pendingServiceConfigs.delete(switchover.serviceId);
-            await serviceSwitchovers.delete(switchover.serviceId);
-            continue;
-        }
-        let overlapStart = switchover.switchTime - switchover.overlapTime;
-        console.log(magenta(`Switchover in flight for ${switchover.serviceId} (${pending.parameters.key}): new instances start at ${new Date(overlapStart).toLocaleString()}${Date.now() >= overlapStart && " (overlap running)" || ""}, old instances shut down at ${new Date(switchover.switchTime).toLocaleString()}`));
-        active.set(switchover.serviceId, { switchover, pending });
-    }
-    return active;
-}
-
-// Starts (or verifies) the "-next" instance of a service during a switchover overlap, running the pending config. It covers the service while the canonical screens restart with the new config at switch time, and is killed by the unused-screen cleanup once the switchover promotes.
+// Starts (or verifies) the "-next" instance of a service during a release overlap, running the new (not yet live) parameters. It covers the service while the canonical screens restart with the new parameters at release time, and is killed by the unused-screen cleanup afterwards (once the release passes we stop ensuring it).
 async function ensureNextInstance(config: {
-    pending: ServiceConfig;
+    next: ServiceParameters;
     index: number;
     screenNamesUsed: Set<string>;
     screenStateMap: Map<string, { screenName: string; isProcessRunning: boolean; pid: string }>;
 }) {
-    let { pending, index } = config;
-    let nextScreenName = getNextScreenName({ serviceKey: pending.parameters.key, index });
+    let { next, index } = config;
+    let nextScreenName = getNextScreenName({ serviceKey: next.key, index });
     config.screenNamesUsed.add(nextScreenName);
 
     let root = os.homedir() + "/" + SERVICE_FOLDER;
     let folder = root + nextScreenName + "/";
     let gitFolder = folder + "git/";
     await fs.promises.mkdir(gitFolder, { recursive: true });
-    if (pending.parameters.gitRef) {
+    if (next.gitRef) {
         let prevGitRef = "";
         try {
             prevGitRef = await getGitRefLive(gitFolder);
         } catch { }
         await ensureGitSynced({
             gitFolder,
-            repoUrl: pending.parameters.repoUrl,
-            gitRef: pending.parameters.gitRef,
+            repoUrl: next.repoUrl,
+            gitRef: next.gitRef,
         });
         let afterGitRef = await getGitRefLive(gitFolder);
         let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
@@ -649,37 +623,76 @@ async function ensureNextInstance(config: {
     if (await fsExistsAsync(parameterPath)) {
         prevParameters = await fs.promises.readFile(parameterPath, "utf8");
     }
-    let newParametersString = JSON.stringify(pending.parameters);
-    let sameParameters = prevParameters === newParametersString;
+    let newParametersString = JSON.stringify(next);
+    let sameParameters = sameRestartParameters(prevParameters, next);
     let screenIsRunning = config.screenStateMap.get(nextScreenName)?.isProcessRunning;
     if (sameParameters && screenIsRunning) {
-        console.log(green(`Verified switchover instance ${magenta(nextScreenName)} is running`));
+        if (prevParameters !== newParametersString) {
+            await fs.promises.writeFile(parameterPath, newParametersString);
+        }
+        console.log(green(`Verified release-overlap instance ${magenta(nextScreenName)} is running`));
         return;
     }
 
-    console.log(`Starting switchover instance ${magenta(nextScreenName)}, with ${newParametersString}, isRunning = ${screenIsRunning}, sameParameters = ${sameParameters}`);
+    console.log(`Starting release-overlap instance ${magenta(nextScreenName)}, with ${newParametersString}, isRunning = ${screenIsRunning}, sameParameters = ${sameParameters}`);
     await fs.promises.writeFile(parameterPath, newParametersString);
     await runScreenCommand({
         screenName: nextScreenName,
-        command: pending.parameters.command,
+        command: next.command,
     });
 }
 
+function sameRestartParameters(prevParametersJSON: string, next: ServiceParameters): boolean {
+    if (!prevParametersJSON) return false;
+    try {
+        return restartParametersKey(JSON.parse(prevParametersJSON)) === restartParametersKey(next);
+    } catch {
+        return false;
+    }
+}
+
 const resyncServicesBase = runInSerial(measureWrap(async function resyncServices() {
     console.log(magenta("Resyncing services"));
     let machineId = getOwnMachineId(getDomain());
-    let activeSwitchovers = await processSwitchovers();
-    let configs = await getEffectiveServiceConfigs();
-    let relevantConfigs = configs.filter(config => config.machineIds.includes(machineId)).filter(x => x.parameters.deploy);
+    let allConfigs = await getEffectiveServiceConfigs();
+    let relevantConfigs = allConfigs
+        .filter(config => (getLiveServiceParameters(config).machineIds || []).includes(machineId) || (config.parameters.machineIds || []).includes(machineId))
+        .filter(x => getLiveServiceParameters(x).deploy || x.parameters.deploy);
 
     let machineInfo = await getLiveMachineInfo();
 
     let screenState = await getScreenState();
     let screenStateMap = new Map(screenState.map(s => [s.screenName, s]));
     let screenNamesUsed = new Set<string>();
+    let upcomingReleases: { releaseTime: number; overlapTime: number }[] = [];
     let root = os.homedir() + "/" + SERVICE_FOLDER;
-    for (let config of relevantConfigs) {
-        let matchedCount = config.machineIds.filter(id => id === machineId).length;
+    for (let record of relevantConfigs) {
+        let config = { ...record, parameters: getLiveServiceParameters(record) };
+        let releaseTime = record.parameters.releaseTime || 0;
+        let isUpcomingRelease = !!releaseTime && Date.now() < releaseTime && !!record.oldParameters;
+        let overlapTime = record.parameters.overlapTime || DEFAULT_OVERLAP_TIME;
+        if (isUpcomingRelease) {
+            upcomingReleases.push({ releaseTime, overlapTime });
+            console.log(magenta(`Release in flight for ${record.serviceId} (${record.parameters.key}): new instances start at ${new Date(releaseTime - overlapTime).toLocaleString()}${Date.now() >= releaseTime - overlapTime && " (overlap running)" || ""}, old instances shut down at ${new Date(releaseTime).toLocaleString()}`));
+            // The release-overlap instances follow the NEW parameters' machineIds, so machines that are only in the new parameters still start their instances at overlap time (their canonical instances start once the release passes)
+            if (Date.now() >= releaseTime - overlapTime && record.parameters.deploy) {
+                let nextMatchedCount = (record.parameters.machineIds || []).filter(id => id === machineId).length;
+                for (let i = 0; i < nextMatchedCount; i++) {
+                    try {
+                        await ensureNextInstance({
+                            next: record.parameters,
+                            index: i,
+                            screenNamesUsed,
+                            screenStateMap,
+                        });
+                    } catch (e: any) {
+                        console.error(`Error starting release-overlap instance for ${record.serviceId} (${record.parameters.key}): ${e.stack}`);
+                    }
+                }
+            }
+        }
+        if (!config.parameters.deploy) continue;
+        let matchedCount = (config.parameters.machineIds || []).filter(id => id === machineId).length;
         for (let i = 0; i < matchedCount; i++) {
             let screenName = getScreenName({
                 serviceKey: config.parameters.key,
@@ -698,20 +711,9 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                 let folder = root + screenName + "/";
                 screenNamesUsed.add(screenName);
 
-                let switchoverObj = activeSwitchovers.get(config.serviceId);
-                if (switchoverObj) {
+                if (isUpcomingRelease) {
                     // Tell the running instance when it will be shut down, so it (and everything talking to it) can wind down gracefully.
-                    await notifyServiceShutdown(screenName, switchoverObj.switchover.switchTime);
-                    let overlapStart = switchoverObj.switchover.switchTime - switchoverObj.switchover.overlapTime;
-                    let pendingMatchedCount = switchoverObj.pending.machineIds.filter(id => id === machineId).length;
-                    if (Date.now() >= overlapStart && switchoverObj.pending.parameters.deploy && i < pendingMatchedCount) {
-                        await ensureNextInstance({
-                            pending: switchoverObj.pending,
-                            index: i,
-                            screenNamesUsed,
-                            screenStateMap,
-                        });
-                    }
+                    await notifyServiceShutdown(screenName, releaseTime);
                 }
 
                 let gitFolder = folder + "git/";
@@ -741,8 +743,7 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                 }
                 let newParametersString = JSON.stringify(config.parameters);
 
-
-                let sameParameters = prevParameters === newParametersString;
+                let sameParameters = sameRestartParameters(prevParameters, config.parameters);
                 let screenIsRunning = screenStateMap.get(screenName)?.isProcessRunning;
 
                 let nodePathId = folder + SERVICE_NODE_FILE_NAME;
@@ -752,6 +753,9 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                 }
 
                 if (sameParameters && screenIsRunning) {
+                    if (prevParameters !== newParametersString) {
+                        await fs.promises.writeFile(parameterPath, newParametersString);
+                    }
                     console.log(green(`Verified ${magenta(screenName)} is running`));
                     continue;
                 }
@@ -816,7 +820,7 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
         });
     }
 
-    scheduleSwitchoverResync(activeSwitchovers);
+    scheduleReleaseResync(upcomingReleases);
 
     await machineInfos.set(machineId, machineInfo);
     console.log(`${magenta(`Resynced ${relevantConfigs.length} services`)}:\n${JSON.stringify(machineInfo, null, 2)}`);
@@ -832,29 +836,32 @@ async function resyncServices() {
     }
 }
 
-// The regular resync poll is too coarse for switchover boundaries (the overlap start and the switch itself), so schedule an extra resync at the next boundary.
-let switchoverResyncTimer: ReturnType<typeof setTimeout> | undefined;
-function scheduleSwitchoverResync(activeSwitchovers: Map<string, { switchover: ServiceSwitchover; pending: ServiceConfig }>) {
+// The regular resync poll is too coarse for release boundaries (the overlap start and the release itself), so schedule an extra resync at the next boundary.
+let cancelReleaseResync: (() => void) | undefined;
+function scheduleReleaseResync(upcomingReleases: { releaseTime: number; overlapTime: number }[]) {
     let now = Date.now();
     let next: number | undefined;
-    for (let { switchover } of activeSwitchovers.values()) {
-        for (let boundary of [switchover.switchTime - switchover.overlapTime, switchover.switchTime]) {
+    for (let release of upcomingReleases) {
+        for (let boundary of [release.releaseTime - release.overlapTime, release.releaseTime]) {
             if (boundary <= now) continue;
             if (next === undefined || boundary < next) {
                 next = boundary;
             }
         }
     }
-    if (switchoverResyncTimer) {
-        clearTimeout(switchoverResyncTimer);
-        switchoverResyncTimer = undefined;
+    if (cancelReleaseResync) {
+        cancelReleaseResync();
+        cancelReleaseResync = undefined;
     }
     if (next === undefined) return;
-    console.log(`Scheduling a resync for the next switchover boundary at ${new Date(next).toLocaleString()}`);
-    switchoverResyncTimer = setTimeout(() => {
-        switchoverResyncTimer = undefined;
-        void resyncServices();
-    }, next - now + 1000);
+    console.log(`Scheduling a resync for the next release boundary at ${new Date(next).toLocaleString()}`);
+    cancelReleaseResync = setPreciseTimeout({
+        time: next,
+        callback: () => {
+            cancelReleaseResync = undefined;
+            void resyncServices();
+        },
+    });
 }
 
 

package/src/deployManager/machineController.ts

@@ -38,7 +38,7 @@ export function onServiceConfigChange(callback: () => Promise<void>): () => void
 }
 
 
-/** Tells a service's node that it will be shut down at the given time (due to a scheduled switchover). */
+/** Tells a service's node that it will be shut down at the given time (due to a scheduled deploy). */
 export async function scheduleNodeShutdown(config: {
     nodeId: string;
     time: number;
@@ -51,7 +51,7 @@ class OnServiceChangeBase {
     }
 
     public async scheduleShutdown(time: number) {
-        console.log(`Received scheduled shutdown notification (service switchover): shutting down at ${new Date(time).toLocaleString()}`);
+        console.log(`Received scheduled shutdown notification (service deploy): shutting down at ${new Date(time).toLocaleString()}`);
         setScheduledShutdownTime(time);
     }
 }