run402 4.109.0 → 4.109.1

package/sdk/dist/namespaces/deploy.js

@@ -26,6 +26,7 @@ import { assertCiDeployableSpec } from "./ci.js";
 import { ROUTE_PRICING_NETWORKS, ROUTE_HTTP_METHODS, isModernPlanMigrationDiff, normalizeDeployResolveRequest, } from "./deploy.types.js";
 import { ApiError, LocalError, NetworkError, PaymentRequired, Run402DeployError, Unauthorized, isRun402Error, isTransferFreezeError, } from "../errors.js";
 import { assertAssetMetadata, assertExifPolicy, } from "./assets-validation.js";
+import { withSiteRemovalSummaries } from "./site-removal-summary.js";
 // ─── Constants ───────────────────────────────────────────────────────────────
 const PLAN_BODY_LIMIT_BYTES = 5 * 1024 * 1024;
 const COMMIT_POLL_INITIAL_MS = 1_000;
@@ -45,6 +46,25 @@ const SECRET_KEY_RE = /^[A-Z_][A-Z0-9_]{0,127}$/;
 const APPLY_SAFE_RETRY_CODES = new Set([
     "BASE_RELEASE_CONFLICT",
 ]);
+/**
+ * Transient platform conditions the gateway marks `safe_to_retry: true`: the
+ * task running the deploy was replaced mid-commit, an image encode ran out of
+ * wall clock or queue under load, or an archive export briefly paused writes.
+ * Nothing was activated, so the whole apply re-plans and runs again — content
+ * is addressed by hash, so already-uploaded bytes are not sent twice
+ * (kychee-com/run402#628, #616). These wait longer than a release-race replan.
+ */
+const APPLY_TRANSIENT_RETRY_CODES = new Set([
+    "OPERATION_INTERRUPTED",
+    "IMAGE_ENCODE_TIMEOUT",
+    "TOO_MANY_ENCODES_QUEUED",
+    "EXPORT_CONSISTENCY_UNAVAILABLE",
+]);
+const APPLY_TRANSIENT_RETRY_BASE_DELAY_MS = 2_000;
+const APPLY_TRANSIENT_RETRY_MAX_DELAY_MS = 15_000;
+/** A read or an idempotent request lost to a 502/503/504 or a dropped connection. */
+const TRANSIENT_REQUEST_MAX_ATTEMPTS = 3;
+const TRANSIENT_REQUEST_RETRY_BASE_DELAY_MS = 500;
 const EMAIL_TRIGGER_EVENTS = new Set(["reply_received", "delivery", "bounced", "complained", "mailbox_suspended"]);
 const STATIC_ACTIVATION_FAILURE_CODES = new Set([
     "BAD_FIELD",
@@ -111,29 +131,31 @@ export class Deploy {
                 return await applyOnce(this.client, spec, opts, emit);
             }
             catch (err) {
-                if (!(err instanceof Run402DeployError))
+                if (!isRun402Error(err))
                     throw err;
                 const safeToRetry = isSafeDeployApplyRetry(err, spec);
                 if (!safeToRetry)
                     throw err;
                 if (attempt === maxAttempts) {
-                    if (maxRetries > 0) {
+                    if (maxRetries > 0 && err instanceof Run402DeployError) {
                         throw withDeployRetryMetadata(err, attempt, maxRetries, err.code);
                     }
                     throw err;
                 }
-                const delayMs = deployApplyRetryDelayMs(attempt);
+                const code = err.code ?? "UNKNOWN";
+                const delayMs = deployApplyRetryDelayMs(attempt, code);
+                const deployErr = err instanceof Run402DeployError ? err : null;
                 emit({
                     type: "deploy.retry",
                     attempt,
                     nextAttempt: attempt + 1,
                     maxAttempts,
                     delayMs,
-                    code: err.code,
-                    phase: err.phase,
-                    resource: err.resource,
-                    operationId: err.operationId,
-                    planId: err.planId,
+                    code,
+                    phase: deployErr?.phase ?? null,
+                    resource: deployErr?.resource ?? null,
+                    operationId: deployErr?.operationId ?? null,
+                    planId: deployErr?.planId ?? null,
                     message: err.message,
                 });
                 await sleep(delayMs);
@@ -842,7 +864,14 @@ function normalizeApplyMaxRetries(value) {
     return value;
 }
 function isSafeDeployApplyRetry(err, spec) {
-    return (err.safeToRetry === true &&
+    if (err.safeToRetry !== true || typeof err.code !== "string")
+        return false;
+    // Transient codes reach here from the deploy state machine (a failed
+    // operation) or as a plain API error from an upload-phase request. Nothing
+    // was activated either way, so the spec's base still holds.
+    if (APPLY_TRANSIENT_RETRY_CODES.has(err.code))
+        return true;
+    return (err instanceof Run402DeployError &&
         APPLY_SAFE_RETRY_CODES.has(err.code) &&
         isAutoRebasableSpec(spec));
 }
@@ -851,9 +880,11 @@ function isAutoRebasableSpec(spec) {
         return true;
     return "release" in spec.base && spec.base.release === "current";
 }
-function deployApplyRetryDelayMs(attempt) {
-    const exponential = APPLY_RETRY_BASE_DELAY_MS * 2 ** (attempt - 1);
-    const capped = Math.min(exponential, APPLY_RETRY_MAX_DELAY_MS);
+function deployApplyRetryDelayMs(attempt, code) {
+    const transient = APPLY_TRANSIENT_RETRY_CODES.has(code);
+    const base = transient ? APPLY_TRANSIENT_RETRY_BASE_DELAY_MS : APPLY_RETRY_BASE_DELAY_MS;
+    const max = transient ? APPLY_TRANSIENT_RETRY_MAX_DELAY_MS : APPLY_RETRY_MAX_DELAY_MS;
+    const capped = Math.min(base * 2 ** (attempt - 1), max);
     return capped + Math.floor(Math.random() * (APPLY_RETRY_JITTER_MS + 1));
 }
 function withDeployRetryMetadata(err, attempts, maxRetries, lastRetryCode) {
@@ -1599,6 +1630,7 @@ function expandCronNumberPart(part, min, max) {
     return values;
 }
 async function commitInternal(client, planId, idempotencyKey, project, requiredPlan, vault, operationId) {
+    const tries = { attempts: 0 };
     try {
         const body = {};
         if (idempotencyKey)
@@ -1622,10 +1654,16 @@ async function commitInternal(client, planId, idempotencyKey, project, requiredP
                 : {}),
             context: "committing deploy",
         });
-        return await sendCommitRecoveringLostResponse(client, send, operationId, project);
+        return await sendCommitRecoveringLostResponse(client, send, operationId, project, tries);
     }
     catch (err) {
-        throw translateDeployError(err, "commit", planId, null);
+        const translated = translateDeployError(err, "commit", planId, operationId ?? null);
+        // Retries were spent on a lost response: say how many and on what, so a
+        // caller can tell "the SDK already tried" from "the first attempt failed".
+        if (tries.attempts > 1) {
+            throw withDeployRetryMetadata(translated, tries.attempts, DEPLOY_COMMIT_MAX_ATTEMPTS - 1, translated.code);
+        }
+        throw translated;
     }
 }
 /**
@@ -1636,9 +1674,11 @@ async function commitInternal(client, planId, idempotencyKey, project, requiredP
  * operation first: once it has left `planning`/`uploading` the commit landed,
  * and the caller follows the operation; otherwise the commit never arrived and
  * is sent again. Without an operation id to read, the failure surfaces as is.
+ * `tries.attempts` counts the commits sent, for the error's retry metadata.
  */
-async function sendCommitRecoveringLostResponse(client, send, operationId, projectId) {
+async function sendCommitRecoveringLostResponse(client, send, operationId, projectId, tries) {
     for (let attempt = 1;; attempt++) {
+        tries.attempts = attempt;
         try {
             return await send();
         }
@@ -1692,14 +1732,16 @@ async function uploadMissing(client, projectId, presence, byteReaders, emit) {
             content_type: reader?.contentType,
         };
     });
-    const planRes = await client.request("/content/v1/plans", {
+    // A lost content plan is safe to repeat: it only opens upload sessions, and
+    // an abandoned one expires unused.
+    const planRes = await requestWithTransientRetry(() => client.request("/content/v1/plans", {
         method: "POST",
         headers,
         body: ciCredentials
             ? { project_id: projectId, content: contentRequest }
             : { content: contentRequest },
         context: "planning content upload",
-    });
+    }));
     const total = planRes.missing.length;
     let done = 0;
     for (const session of planRes.missing) {
@@ -1782,6 +1824,19 @@ async function commitContentPlanWithRetry(client, planId, headers) {
         }
     }
 }
+/** Repeat a read or an idempotent request lost to a 502/503/504 or a dropped connection. */
+async function requestWithTransientRetry(send) {
+    for (let attempt = 1;; attempt++) {
+        try {
+            return await send();
+        }
+        catch (err) {
+            if (!isTransientGatewayFailure(err) || attempt >= TRANSIENT_REQUEST_MAX_ATTEMPTS)
+                throw err;
+            await sleep(TRANSIENT_REQUEST_RETRY_BASE_DELAY_MS * 2 ** (attempt - 1));
+        }
+    }
+}
 function isTransientGatewayFailure(err) {
     if (!isRun402Error(err))
         return false;
@@ -2068,7 +2123,10 @@ async function pollSnapshotUntilReady(client, initial, diff, warnings, emit, pro
         if (Date.now() - start > COMMIT_POLL_BACKOFF_AFTER_MS) {
             interval = Math.min(Math.floor(interval * 1.5), COMMIT_POLL_MAX_MS);
         }
-        snapshot = await client.request(`/apply/v1/operations/${encodeURIComponent(snapshot.operation_id)}`, { headers: opHeaders, context: "polling deploy operation" });
+        // A poll lost to a 502/503/504 (a gateway rollout) says nothing about the
+        // operation; read it again rather than abandon a deploy that is running.
+        const operationId = snapshot.operation_id;
+        snapshot = await requestWithTransientRetry(() => client.request(`/apply/v1/operations/${encodeURIComponent(operationId)}`, { headers: opHeaders, context: "polling deploy operation" }));
     }
 }
 async function drainRecordedOperationEvents(client, operationId, headers, cursor, emit) {
@@ -3113,7 +3171,7 @@ function invalidRouteSpec(message, resource) {
 }
 function normalizePlanResponse(plan) {
     const raw = plan;
-    const warnings = Array.isArray(raw.warnings) ? raw.warnings : [];
+    const warnings = withSiteRemovalSummaries(Array.isArray(raw.warnings) ? raw.warnings : []);
     const missingContent = Array.isArray(raw.missing_content) ? raw.missing_content : [];
     if (raw.kind === "plan_response") {
         const diff = {
@@ -4106,14 +4164,14 @@ function makeUnreadableReader(sha256, label) {
 async function uploadInlineCas(client, projectId, bytes, contentType) {
     const sha256 = await sha256Hex(bytes);
     const headers = await apikeyHeaders(client, projectId);
-    const planRes = await client.request("/content/v1/plans", {
+    const planRes = await requestWithTransientRetry(() => client.request("/content/v1/plans", {
         method: "POST",
         headers,
         body: {
             content: [{ sha256, size: bytes.byteLength, content_type: contentType }],
         },
         context: "planning content upload",
-    });
+    }));
     if (planRes.missing.length > 0) {
         const session = planRes.missing[0];
         await uploadOne(client.fetch, session, bytes);