querysub 0.580.0 → 0.582.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.580.0",
+  "version": "0.582.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -71,7 +71,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.50",
+    "sliftutils": "^1.7.51",
     "socket-function": "^1.2.26",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/deployManager/components/RouteConfigView.tsx

@@ -28,7 +28,8 @@ const PAST_COLOR = { h: 0, s: 0, l: 90 };
 const FUTURE_COLOR = { h: 45, s: 80, l: 85 };
 // Windows are days or hours wide, so the countdown only has to be roughly right
 const TIME_REFRESH_INTERVAL = 60 * 1000;
-const WATCH_POLL_INTERVAL = 60 * 1000;
+// Watching exists to see a config change as it happens - a deploy or a server disagreeing with the others - so it polls far faster than the window countdowns need to update
+const WATCH_POLL_INTERVAL = 5 * 1000;
 const WATCH_ACTIVE_COLOR = { h: 195, s: 60, l: 85 };
 const WINDOW_HEADER_SIZE = 13;
 const WINDOW_TIME_SIZE = 14;
@@ -513,7 +514,8 @@ class RouteConfigGroupView extends qreact.Component<{ group: RouteConfigGroup }>
 class LiveWatchView extends qreact.Component<{ watchKey: string; watch: LiveWatch }> {
     render() {
         let { watchKey, watch } = this.props;
-        let now = Querysub.timeDelayed(TIME_REFRESH_INTERVAL);
+        // Ticks with the polling, not with the window countdowns - "updated 1 minute ago" on something refreshed seconds ago reads as broken
+        let now = Querysub.timeDelayed(WATCH_POLL_INTERVAL);
         let sources = (watch.routing?.sources || []).map(normalizeSource);
         return <div className={css.vbox(10).fillWidth.pad2(10, 8).bord2(WATCH_ACTIVE_COLOR.h, WATCH_ACTIVE_COLOR.s, WATCH_ACTIVE_COLOR.l - 25)}>
             <div className={css.hbox(8).wrap.fontSize(WINDOW_HEADER_SIZE)}>

package/src/deployManager/machineApplyMainCode.ts

@@ -220,6 +220,11 @@ const ensureGitSynced = measureWrap(async function ensureGitSynced(config: {
 let launchesPerService = new Map<string, number>();
 let lastLaunchedTimePerService = new Map<string, number>();
 
+// How often we check that everything that should be running still is. Cheap - it asks tmux and the local filesystem - so it runs often enough that a crashed process is noticed in seconds rather than minutes.
+const RESYNC_INTERVAL = timeInSecond * 30;
+// How often the configs themselves are re-read. Reading them hits backblaze, so this is the interval that has to stay conservative, and it is entirely independent of how often we check liveness.
+const CONFIG_REFRESH_INTERVAL = RESYNC_INTERVAL * 6;
+
 const LAUNCH_VERIFY_DELAY = 2000;
 // How far a re-read start time may drift from the recorded one and still be the same process. Well under any interval a pid could be reused in, and wide enough for sources that only resolve to the second.
 const START_TIME_TOLERANCE = timeInSecond * 5;
@@ -448,10 +453,29 @@ function sameRestartParameters(prevParametersJSON: string, next: ServiceParamete
     }
 }
 
+// The configs come from backblaze, and reading them is by far the most expensive thing a resync does - far too expensive to do every time we want to check that the processes are still up, which is the other, much cheaper half of a resync. So the configs are re-read on their own schedule and reused in between.
+let cachedConfigs: ServiceConfig[] | undefined;
+let cachedConfigsTime = 0;
+// Set when something tells us the configs actually changed, which no interval can predict
+let configRefreshRequested = false;
+async function getResyncConfigs(): Promise<{ configs: ServiceConfig[]; refreshed: boolean }> {
+    let force = configRefreshRequested;
+    if (!force && cachedConfigs && Date.now() - cachedConfigsTime < CONFIG_REFRESH_INTERVAL) {
+        return { configs: cachedConfigs, refreshed: false };
+    }
+    // Cleared before the read, so a change arriving DURING it is not swallowed by our own success
+    configRefreshRequested = false;
+    cachedConfigs = await getEffectiveServiceConfigs();
+    cachedConfigsTime = Date.now();
+    return { configs: cachedConfigs, refreshed: true };
+}
+
 const resyncServicesBase = runInSerial(measureWrap(async function resyncServices() {
-    console.log(magenta("Resyncing services"));
     let machineId = getOwnMachineId(getDomain());
-    let allConfigs = await getEffectiveServiceConfigs();
+    let { configs: allConfigs, refreshed } = await getResyncConfigs();
+    if (refreshed) {
+        console.log(magenta("Resyncing services, with freshly read configs"));
+    }
     let relevantConfigs = allConfigs
         .filter(config => getMachineIdList(getLiveServiceParameters(config)).includes(machineId) || getMachineIdList(config.parameters).includes(machineId) || config.oldParameters && getMachineIdList(config.oldParameters).includes(machineId))
         .filter(x => getLiveServiceParameters(x).deploy || x.parameters.deploy || x.oldParameters?.deploy);
@@ -500,11 +524,19 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
     }
 
     await machineInfos.set(machineId, machineInfo);
-    console.log(`${magenta(`Resynced ${relevantConfigs.length} services`)}:\n${JSON.stringify(machineInfo, null, 2)}`);
+    // Only on the runs that read the configs. The rest are a liveness check that happens many times a minute, and dumping the whole machine info each time buries everything that actually matters.
+    if (refreshed) {
+        console.log(`${magenta(`Resynced ${relevantConfigs.length} services`)}:\n${JSON.stringify(machineInfo, null, 2)}`);
+    }
 
     // Don't resync TOO often, so always wait a bit
     await delay(5000);
 }));
+/** A resync that re-reads the configs, however recently they were last read - for when something tells us they changed. */
+async function resyncServicesWithFreshConfigs() {
+    configRefreshRequested = true;
+    await resyncServices();
+}
 async function resyncServices() {
     try {
         await resyncServicesBase();
@@ -568,10 +600,10 @@ export async function machineApplyMain() {
 
 
     await Querysub.hostService("machine-apply");
-    onServiceConfigChange(resyncServices);
+    onServiceConfigChange(resyncServicesWithFreshConfigs);
 
 
-    runInfinitePoll(timeInMinute * 3, async () => {
+    runInfinitePoll(RESYNC_INTERVAL, async () => {
         //console.log(magenta(`Quick outdated check at ${new Date().toISOString()}`));
         // console.log(magenta("Likely outdated, resyncing now"));
         // NOTE: I'm not sure why we were hesitant to call this. It shouldn't be that slow...