querysub 0.710.0 → 0.711.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.710.0",
+  "version": "0.711.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -79,7 +79,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.149",
+    "sliftutils": "^1.8.20",
     "socket-function": "^1.3.1",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/deployManager/machineApplyMainCode.ts

@@ -9,7 +9,8 @@ import { runPromise } from "../functional/runCommand";
 import { getExternalIP } from "socket-function/src/networking";
 import { errorToUndefined, errorToUndefinedSilent } from "../errors";
 import { getDomain } from "../config";
-import { formatTime } from "socket-function/src/formatting/format";
+import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
+import { httpsRequest, HttpsResponseInfo } from "socket-function/src/https";
 import { sort, timeInMinute, timeInSecond, watchSlowPromise } from "socket-function/src/misc";
 import { isDefined, setPreciseTimeout } from "../misc";
 import { logLoadTime } from "../logModuleLoadTimes";
@@ -247,6 +248,30 @@ function checkGitSyncedAndClean(gitFolder: string, gitRef: string): Promise<bool
     return cached;
 }
 
+async function recordMissingDependency(config: { gitFolder: string; screenName: string; error: unknown }) {
+    let { gitFolder, screenName } = config;
+    let match = String((config.error as Error)?.stack || config.error).match(/(https?:\/\/\S+?\.tgz): Request failed "404 Not Found"/);
+    if (!match) return;
+    let failedUrl = new URL(match[1]);
+    let tarballUrl = NPM_REGISTRY_ORIGIN + failedUrl.pathname;
+    let outResponse: HttpsResponseInfo = { headers: {} };
+    try {
+        await httpsRequest(tarballUrl, undefined, "HEAD", false, { outResponse });
+    } catch { }
+    if (outResponse.statusCode !== 404) {
+        console.warn(`Yarn install for ${screenName} failed on ${match[1]} returning 404, but npm returned ${outResponse.statusCode} for ${tarballUrl}, so it is not missing and we are not backing off`);
+        return;
+    }
+    let failures = (missingDependencyRetries.get(gitFolder)?.failures || 0) + 1;
+    let nextAttemptTime = failures >= MISSING_DEPENDENCY_MAX_FAILURES && Number.POSITIVE_INFINITY || Date.now() + MISSING_DEPENDENCY_RETRY_DELAY;
+    missingDependencyRetries.set(gitFolder, { failures, nextAttemptTime, tarballUrl });
+    if (failures >= MISSING_DEPENDENCY_MAX_FAILURES) {
+        console.error(`Giving up yarn installing for ${screenName}, ${tarballUrl} is still missing from npm after ${failures} failed installs. Retrying only on a config change.`);
+        return;
+    }
+    console.warn(`Yarn install for ${screenName} failed because ${tarballUrl} is missing from npm (failure ${failures}/${MISSING_DEPENDENCY_MAX_FAILURES}), retrying at ${formatDateTime(nextAttemptTime)}`);
+}
+
 async function getQuerysubVersionMismatch(gitFolder: string): Promise<string> {
     let packageJson = JSON.parse(await fs.promises.readFile(gitFolder + "package.json", "utf8")) as { dependencies?: { [name: string]: string } };
     let requested = packageJson.dependencies?.querysub;
@@ -322,6 +347,11 @@ const RESYNC_INTERVAL = timeInSecond * 30;
 const CONFIG_REFRESH_INTERVAL = RESYNC_INTERVAL * 6;
 
 const LAUNCH_VERIFY_DELAY = 2000;
+
+const NPM_REGISTRY_ORIGIN = "https://registry.npmjs.org";
+const MISSING_DEPENDENCY_RETRY_DELAY = timeInMinute * 5;
+const MISSING_DEPENDENCY_MAX_FAILURES = 5;
+let missingDependencyRetries = new Map<string, { failures: number; nextAttemptTime: number; tarballUrl: string }>();
 // How far a re-read start time may drift from the recorded one and still be the same process. Well under any interval a pid could be reused in, and wide enough for sources that only resolve to the second.
 const START_TIME_TOLERANCE = timeInSecond * 5;
 
@@ -503,9 +533,22 @@ async function ensureScreen(config: {
                 let afterGitRef = await getGitRefLive(gitFolder);
                 let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
                 let querysubMismatch = await getQuerysubVersionMismatch(gitFolder);
-                if (afterGitRef !== prevGitRef || nodeModulesMissing || querysubMismatch) {
-                    console.log(green(`Yarn installing for ${magenta(screenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing}, querysubMismatch=${querysubMismatch || "none"})`));
-                    await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
+                let missingRetry = missingDependencyRetries.get(gitFolder);
+                if (afterGitRef !== prevGitRef || nodeModulesMissing || querysubMismatch || missingRetry) {
+                    if (missingRetry && Date.now() < missingRetry.nextAttemptTime) {
+                        throw new Error(`Not yarn installing for ${screenName}, ${missingRetry.tarballUrl} was missing from npm after ${missingRetry.failures} failed installs. ${missingRetry.failures >= MISSING_DEPENDENCY_MAX_FAILURES && "Gave up, waiting for a config change to retry" || `Next attempt at ${formatDateTime(missingRetry.nextAttemptTime)}`}`);
+                    }
+                    console.log(green(`Yarn installing for ${magenta(screenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing}, querysubMismatch=${querysubMismatch || "none"}, missingDependencyFailures=${missingRetry?.failures || 0})`));
+                    try {
+                        await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
+                    } catch (e) {
+                        await recordMissingDependency({ gitFolder, screenName, error: e });
+                        throw e;
+                    }
+                    if (missingRetry) {
+                        console.log(green(`Yarn install for ${magenta(screenName)} succeeded after ${missingRetry.failures} failures from ${missingRetry.tarballUrl} missing from npm`));
+                        missingDependencyRetries.delete(gitFolder);
+                    }
                 }
             }
 
@@ -654,6 +697,10 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
 /** A resync that re-reads the configs, however recently they were last read - for when something tells us they changed. */
 async function resyncServicesWithFreshConfigs() {
     configRefreshRequested = true;
+    if (missingDependencyRetries.size) {
+        console.log(`Config changed, clearing the missing dependency backoff for ${Array.from(missingDependencyRetries.keys()).join(", ")}`);
+        missingDependencyRetries.clear();
+    }
     await resyncServices();
 }
 async function resyncServices() {