querysub 0.710.0 → 0.712.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.710.0",
+  "version": "0.712.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -79,7 +79,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.149",
+    "sliftutils": "^1.8.20",
     "socket-function": "^1.3.1",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/deployManager/components/ServicesListPage.tsx

@@ -102,8 +102,8 @@ export class ServicesListPage extends qreact.Component {
                                 parameters: {
                                     key: newServiceId,
                                     machineIds: [],
-                                    repoUrl: gitInfo.repoUrl,
-                                    gitRef: gitInfo.latestRef,
+                                    repoUrl: gitInfo?.repoUrl || "",
+                                    gitRef: gitInfo?.latestRef || "",
                                     command: "ping 1.1.1.1",
                                     deploy: false,
                                     overlapTime: DEFAULT_OVERLAP_TIME,

package/src/deployManager/emergencyUpdateMain.ts

@@ -32,7 +32,7 @@ async function main() {
     let controller = new MachineServiceControllerBase();
     let commitMessage = `emergency update ${formatDateTime(Date.now())}`;
 
-    let gitInfo = await controller.getGitInfo();
+    let gitInfo = await getGitInfoAssert(controller);
     console.log(magenta(`Starting emergency update of ${gitInfo.repoUrl}`), {
         commitMessage,
         querysubUncommittedFiles: gitInfo.querysubUncommitted.length,
@@ -47,7 +47,7 @@ async function main() {
     }
 
     // Re-read, as publishing querysub updates our package.json, which then needs to be committed here
-    gitInfo = await controller.getGitInfo();
+    gitInfo = await getGitInfoAssert(controller);
     if (gitInfo.uncommitted.length > 0) {
         console.log(magenta(`Committing and pushing the service repo (${gitInfo.uncommitted.length} files changed)`));
         await controller.commitPushService(commitMessage);
@@ -56,7 +56,7 @@ async function main() {
     }
 
     // Re-read, as latestRef must include the commit we just pushed
-    gitInfo = await controller.getGitInfo();
+    gitInfo = await getGitInfoAssert(controller);
 
     let serviceIds = await controller.getServiceList();
     let configs: ServiceConfig[] = [];
@@ -92,4 +92,12 @@ async function main() {
     console.log(green(`Archives initialized`));
 }
 
+async function getGitInfoAssert(controller: MachineServiceControllerBase) {
+    let gitInfo = await controller.getGitInfo();
+    if (!gitInfo) {
+        throw new Error(`Expected git info for the emergency update, but getGitInfo returned undefined because this process is public (isPublic() is true). Run the emergency update without --public, from the repo you deploy from.`);
+    }
+    return gitInfo;
+}
+
 main().catch(logErrors).finally(() => process.exit());

package/src/deployManager/machineApplyMainCode.ts

@@ -9,7 +9,8 @@ import { runPromise } from "../functional/runCommand";
 import { getExternalIP } from "socket-function/src/networking";
 import { errorToUndefined, errorToUndefinedSilent } from "../errors";
 import { getDomain } from "../config";
-import { formatTime } from "socket-function/src/formatting/format";
+import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
+import { httpsRequest, HttpsResponseInfo } from "socket-function/src/https";
 import { sort, timeInMinute, timeInSecond, watchSlowPromise } from "socket-function/src/misc";
 import { isDefined, setPreciseTimeout } from "../misc";
 import { logLoadTime } from "../logModuleLoadTimes";
@@ -247,6 +248,30 @@ function checkGitSyncedAndClean(gitFolder: string, gitRef: string): Promise<bool
     return cached;
 }
 
+async function recordMissingDependency(config: { gitFolder: string; screenName: string; error: unknown }) {
+    let { gitFolder, screenName } = config;
+    let match = String((config.error as Error)?.stack || config.error).match(/(https?:\/\/\S+?\.tgz): Request failed "404 Not Found"/);
+    if (!match) return;
+    let failedUrl = new URL(match[1]);
+    let tarballUrl = NPM_REGISTRY_ORIGIN + failedUrl.pathname;
+    let outResponse: HttpsResponseInfo = { headers: {} };
+    try {
+        await httpsRequest(tarballUrl, undefined, "HEAD", false, { outResponse });
+    } catch { }
+    if (outResponse.statusCode !== 404) {
+        console.warn(`Yarn install for ${screenName} failed on ${match[1]} returning 404, but npm returned ${outResponse.statusCode} for ${tarballUrl}, so it is not missing and we are not backing off`);
+        return;
+    }
+    let failures = (missingDependencyRetries.get(gitFolder)?.failures || 0) + 1;
+    let nextAttemptTime = failures >= MISSING_DEPENDENCY_MAX_FAILURES && Number.POSITIVE_INFINITY || Date.now() + MISSING_DEPENDENCY_RETRY_DELAY;
+    missingDependencyRetries.set(gitFolder, { failures, nextAttemptTime, tarballUrl });
+    if (failures >= MISSING_DEPENDENCY_MAX_FAILURES) {
+        console.error(`Giving up yarn installing for ${screenName}, ${tarballUrl} is still missing from npm after ${failures} failed installs. Retrying only on a config change.`);
+        return;
+    }
+    console.warn(`Yarn install for ${screenName} failed because ${tarballUrl} is missing from npm (failure ${failures}/${MISSING_DEPENDENCY_MAX_FAILURES}), retrying at ${formatDateTime(nextAttemptTime)}`);
+}
+
 async function getQuerysubVersionMismatch(gitFolder: string): Promise<string> {
     let packageJson = JSON.parse(await fs.promises.readFile(gitFolder + "package.json", "utf8")) as { dependencies?: { [name: string]: string } };
     let requested = packageJson.dependencies?.querysub;
@@ -322,6 +347,11 @@ const RESYNC_INTERVAL = timeInSecond * 30;
 const CONFIG_REFRESH_INTERVAL = RESYNC_INTERVAL * 6;
 
 const LAUNCH_VERIFY_DELAY = 2000;
+
+const NPM_REGISTRY_ORIGIN = "https://registry.npmjs.org";
+const MISSING_DEPENDENCY_RETRY_DELAY = timeInMinute * 5;
+const MISSING_DEPENDENCY_MAX_FAILURES = 5;
+let missingDependencyRetries = new Map<string, { failures: number; nextAttemptTime: number; tarballUrl: string }>();
 // How far a re-read start time may drift from the recorded one and still be the same process. Well under any interval a pid could be reused in, and wide enough for sources that only resolve to the second.
 const START_TIME_TOLERANCE = timeInSecond * 5;
 
@@ -503,9 +533,22 @@ async function ensureScreen(config: {
                 let afterGitRef = await getGitRefLive(gitFolder);
                 let nodeModulesMissing = !await fsExistsAsync(gitFolder + "node_modules");
                 let querysubMismatch = await getQuerysubVersionMismatch(gitFolder);
-                if (afterGitRef !== prevGitRef || nodeModulesMissing || querysubMismatch) {
-                    console.log(green(`Yarn installing for ${magenta(screenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing}, querysubMismatch=${querysubMismatch || "none"})`));
-                    await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
+                let missingRetry = missingDependencyRetries.get(gitFolder);
+                if (afterGitRef !== prevGitRef || nodeModulesMissing || querysubMismatch || missingRetry) {
+                    if (missingRetry && Date.now() < missingRetry.nextAttemptTime) {
+                        throw new Error(`Not yarn installing for ${screenName}, ${missingRetry.tarballUrl} was missing from npm after ${missingRetry.failures} failed installs. ${missingRetry.failures >= MISSING_DEPENDENCY_MAX_FAILURES && "Gave up, waiting for a config change to retry" || `Next attempt at ${formatDateTime(missingRetry.nextAttemptTime)}`}`);
+                    }
+                    console.log(green(`Yarn installing for ${magenta(screenName)} (ref ${prevGitRef} -> ${afterGitRef}, nodeModulesMissing=${nodeModulesMissing}, querysubMismatch=${querysubMismatch || "none"}, missingDependencyFailures=${missingRetry?.failures || 0})`));
+                    try {
+                        await runPromise(`yarn install --mutex network`, { cwd: gitFolder });
+                    } catch (e) {
+                        await recordMissingDependency({ gitFolder, screenName, error: e });
+                        throw e;
+                    }
+                    if (missingRetry) {
+                        console.log(green(`Yarn install for ${magenta(screenName)} succeeded after ${missingRetry.failures} failures from ${missingRetry.tarballUrl} missing from npm`));
+                        missingDependencyRetries.delete(gitFolder);
+                    }
                 }
             }
 
@@ -654,6 +697,10 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
 /** A resync that re-reads the configs, however recently they were last read - for when something tells us they changed. */
 async function resyncServicesWithFreshConfigs() {
     configRefreshRequested = true;
+    if (missingDependencyRetries.size) {
+        console.log(`Config changed, clearing the missing dependency backoff for ${Array.from(missingDependencyRetries.keys()).join(", ")}`);
+        missingDependencyRetries.clear();
+    }
     await resyncServices();
 }
 async function resyncServices() {

package/src/deployManager/machineSchema.ts

@@ -1,6 +1,6 @@
 import { isNodeTrue, list, timeInDay, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { getArchives2 } from "../-a-archives/archives2";
-import { getDomain } from "../config";
+import { getDomain, isPublic } from "../config";
 import { archiveJSONT } from "../-a-archives/archivesJSONT";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { assertIsManagementUser } from "../diagnostics/managementPages";
@@ -442,6 +442,7 @@ export class MachineServiceControllerBase {
         return extractType(module, "ServiceConfig");
     }
     public async getGitInfo() {
+        if (isPublic()) return undefined;
         let repoUrl = await getGitURLLive();
         let uncommitted = await getGitUncommitted(".");
         let latestRef = await getLatestRefOnUpstreamBranch(".");