scrapely 1.2.16 → 1.2.17

package/dist/actor.js

@@ -33,6 +33,12 @@ const charging_client_js_1 = require("./charging_client.js");
 const proxy_configuration_js_1 = require("./proxy-configuration.js");
 const browser_configuration_js_1 = require("./browser-configuration.js");
 const utils_js_1 = require("./utils.js");
+/** "429 Run API rate limit exceeded ..." for an API error, the message otherwise. */
+function describeRunApiError(err) {
+    const status = err.statusCode;
+    const message = err?.message ?? String(err);
+    return typeof status === 'number' ? `${status} ${message}` : message;
+}
 /** Pre-init fallback; after init, {@link Actor.events} returns the EventManager emitter. */
 const fallbackEvents = new node_events_1.EventEmitter();
 let apifyClientInstance;
@@ -948,7 +954,9 @@ class Actor {
                 }
                 if (cm && typeof cm.flushPending === 'function') {
                     await Promise.race([
-                        cm.flushPending(64, flushDeadlineMs),
+                        // The flush's own deadline ends its retries a little before
+                        // the race gives up on it, so none is cut off mid-request.
+                        cm.flushPending(64, Math.max(500, flushDeadlineMs - 1000)),
                         new Promise((resolve) => setTimeout(resolve, flushDeadlineMs)),
                     ]);
                 }
@@ -979,6 +987,8 @@ class Actor {
                 Actor.setStatusMessage(message, {
                     isStatusMessageTerminal: true,
                     level: exitCode > 0 ? 'ERROR' : 'INFO',
+                    // Stop retrying when the exit stops waiting for it.
+                    retryWindowMs: timeoutMs,
                 }),
                 new Promise((_, reject) => {
                     timer = setTimeout(() => reject(new Error(`timed out after ${timeoutMs}ms`)), timeoutMs);
@@ -1034,7 +1044,9 @@ class Actor {
         if (!apiUrl || !token || !runId) {
             return {};
         }
-        const response = await fetch(`${apiUrl}/v2/actor-runs/${runId}`, {
+        // Setting the message is idempotent: a 429 (run API rate limit), 5xx or
+        // dropped connection is retried with backoff instead of failing the run.
+        const body = await (0, api_storage_client_js_1.fetchWithRetries)(`${apiUrl}/v2/actor-runs/${runId}`, {
             method: 'PUT',
             headers: {
                 'Content-Type': 'application/json',
@@ -1044,13 +1056,14 @@ class Actor {
                 statusMessage: message,
                 isStatusMessageTerminal,
             }),
-            signal: AbortSignal.timeout(10_000),
+        }, {
+            idempotent: true,
+            timeoutMs: 10_000,
+            label: 'PUT status message',
+            retryWindowMs: options?.retryWindowMs,
+        }, async (response) => (await response.json().catch(() => ({})))).catch((err) => {
+            throw new Error(`Failed to update status message: ${describeRunApiError(err)}`);
         });
-        if (!response.ok) {
-            const error = await response.text();
-            throw new Error(`Failed to update status message: ${response.status} ${error}`);
-        }
-        const body = (await response.json());
         return (body.data ?? {});
     }
     /**
@@ -1204,18 +1217,18 @@ class Actor {
         console.log(`[Scrapely] Metamorphosing to actor ${targetActorId}`);
         // Signal the server to perform the metamorph
         try {
-            const response = await fetch(`${apiUrl}/v2/actor-runs/${runId}/metamorph`, {
+            // Retried only while certainly not applied (429, limiter/HAProxy 503,
+            // refused connection): a second metamorph must never be started.
+            await (0, api_storage_client_js_1.fetchWithRetries)(`${apiUrl}/v2/actor-runs/${runId}/metamorph`, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
                     'Authorization': `Bearer ${token}`,
                 },
                 body: JSON.stringify(body),
+            }, { idempotent: false, timeoutMs: 30_000, label: 'POST metamorph' }, async () => undefined).catch((err) => {
+                throw new Error(`Metamorph request failed: ${describeRunApiError(err)}`);
             });
-            if (!response.ok) {
-                const error = await response.text();
-                throw new Error(`Metamorph request failed: ${response.status} ${error}`);
-            }
             console.log('[Scrapely] Metamorph request accepted, exiting...');
         }
         catch (err) {
@@ -1302,18 +1315,18 @@ class Actor {
         const token = config.get('token') || '';
         const runId = config.get('actorRunId') || '';
         try {
-            const response = await fetch(`${apiUrl}/v2/actor-runs/${runId}/reboot`, {
+            // Retried only while certainly not applied (429, limiter/HAProxy 503,
+            // refused connection).
+            const status = await (0, api_storage_client_js_1.fetchWithRetries)(`${apiUrl}/v2/actor-runs/${runId}/reboot`, {
                 method: 'POST',
                 headers: {
                     'Authorization': `Bearer ${token}`,
                     'Content-Type': 'application/json',
                 },
+            }, { idempotent: false, timeoutMs: 30_000, label: 'POST reboot' }, async (response) => response.status).catch((err) => {
+                throw new Error(`Reboot request failed: ${describeRunApiError(err)}`);
             });
-            if (!response.ok) {
-                const error = await response.text();
-                throw new Error(`Reboot request failed: ${response.status} ${error}`);
-            }
-            console.log(`[Scrapely] Reboot request accepted (${response.status})`);
+            console.log(`[Scrapely] Reboot request accepted (${status})`);
             // Wait briefly for the reboot signal to be processed, then exit
             const sleepMillis = options?.customAfterSleepMillis ?? Actor._actorConfig().get('metamorphAfterSleepMillis', 1000);
             await new Promise((resolve) => setTimeout(resolve, sleepMillis));

package/dist/charging.d.ts

@@ -161,10 +161,29 @@ export declare class ChargingManager {
     _chargeableWithinLimit(): Record<string, number>;
     private getChargeableWithinLimit;
     private quarantineStalePending;
+    /**
+     * The run client with scrapely-client's internal retries off (one request
+     * per call) and its timeout capped at `timeoutMs`, so the retry loops here
+     * decide when to retry and never outlive their deadline. `meta.retryAfter`
+     * receives the Retry-After of the last answer (ApifyApiError drops the
+     * headers). A client without that shape (tests, custom clients) is used as
+     * it is.
+     */
+    private runClientOnce;
+    /**
+     * Sleep before the next same-key attempt; false when none is left (attempts,
+     * the API retry window from `started`, or `deadline` would be passed).
+     * Never sooner than the server's Retry-After, and at least 1 s after any
+     * 429 (the server's minimum, also when the header is missing or malformed).
+     */
+    private backoffBeforeRetry;
+    /** Time left for one attempt before `deadline`, if any. */
+    private static attemptTimeout;
     /**
      * POST a charge intent. Returns acceptedCount on success.
      * Throws ChargeRejectedError for definitive platform rejections (no retry).
-     * Returns null when the outcome is uncertain after retries.
+     * Returns null when the outcome is uncertain after retries (the intent stays
+     * pending); `deadline` (epoch ms) ends the retries earlier, e.g. at exit.
      */
     private postChargeWithRetries;
     /** @internal Idempotency key of the charge accepted by the last _chargeExclusive call, if any. */
@@ -181,12 +200,12 @@ export declare class ChargingManager {
      * means it never applied.
      */
     _cancelUncertainCharge(eventName: string, operationId: string, reservedCount: number): Promise<boolean>;
-    _voidCharge(eventName: string, operationId: string, reservedCount: number, notFoundMeansVoided?: boolean): Promise<boolean>;
+    _voidCharge(eventName: string, operationId: string, reservedCount: number, notFoundMeansVoided?: boolean, deadline?: number): Promise<boolean>;
     /**
      * @internal Retry unconfirmed voids (caller holds the operation lock).
      * Returns false while any earlier void is still unconfirmed.
      */
-    _retryPendingVoids(): Promise<boolean>;
+    _retryPendingVoids(deadline?: number): Promise<boolean>;
     /** Clamp local reservation down when the server accepted fewer events. */
     /** @internal Used by pushDataAndCharge dual-event reconciliation. */
     applyAcceptedCount(eventName: string, reservedCount: number, acceptedCount: number, eventPrice?: number): number;

package/dist/charging.js

@@ -16,10 +16,23 @@ exports.pushCertainlyFailed = pushCertainlyFailed;
 // Modified by TeraShift GmbH. See LICENSE-APACHE and NOTICE.
 const node_async_hooks_1 = require("node:async_hooks");
 const node_crypto_1 = require("node:crypto");
+const api_storage_client_js_1 = require("./storage/api-storage-client.js");
 /** Client safe-replay window; server TTL is longer (35d). */
 exports.CHARGE_IDEMPOTENCY_SAFE_AGE_MS = 30 * 24 * 60 * 60 * 1000;
 const PENDING_MAX_ENTRIES = 1024;
-const POST_ATTEMPTS = 3;
+// Same-key attempts per charge / void. Each attempt is a single request
+// (scrapely-client's own retries are switched off for it, see runClientOnce),
+// so the retries here can honour deadlines: 429, 408, 5xx, timeouts and
+// network errors are re-sent with the same idempotency key, with exponential
+// backoff and jitter, never sooner than the server's Retry-After, within the API
+// retry window (2 minutes, SCRAPELY_API_RETRY_WINDOW_SECS) or the caller's
+// deadline. Then the charge stays pending / the void is re-queued and retried
+// before the next push and at exit, the same as the Python SDK.
+const POST_ATTEMPTS = 64;
+/** 408/429 and 5xx: the platform could not take the request now; try again later. */
+function isTransientStatus(status) {
+    return status === 408 || status === 429 || (status !== undefined && status >= 500);
+}
 class ChargeUncertainError extends Error {
     operationId;
     eventName;
@@ -487,16 +500,76 @@ class ChargingManager {
             }
         }
     }
+    /**
+     * The run client with scrapely-client's internal retries off (one request
+     * per call) and its timeout capped at `timeoutMs`, so the retry loops here
+     * decide when to retry and never outlive their deadline. `meta.retryAfter`
+     * receives the Retry-After of the last answer (ApifyApiError drops the
+     * headers). A client without that shape (tests, custom clients) is used as
+     * it is.
+     */
+    runClientOnce(timeoutMs) {
+        const meta = { retryAfter: null };
+        const run = this.apifyClient.run(this.actorRunId);
+        const http = run.httpClient;
+        if (http && typeof http === 'object' && typeof http.maxRetries === 'number') {
+            const once = Object.create(http);
+            once.maxRetries = 0;
+            if (timeoutMs !== undefined && typeof http.timeoutMillis === 'number') {
+                once.timeoutMillis = Math.max(1000, Math.min(http.timeoutMillis, timeoutMs));
+            }
+            const axios = http.axios;
+            if (axios && typeof axios.request === 'function') {
+                const request = axios.request.bind(axios);
+                once.axios = Object.assign(Object.create(axios), {
+                    request: async (cfg) => {
+                        const response = (await request(cfg));
+                        const h = response?.headers;
+                        const value = typeof h?.get === 'function' ? h.get('retry-after') : h?.['retry-after'];
+                        meta.retryAfter = value === undefined || value === null ? null : String(value);
+                        return response;
+                    },
+                });
+            }
+            run.httpClient = once;
+        }
+        return { run, meta };
+    }
+    /**
+     * Sleep before the next same-key attempt; false when none is left (attempts,
+     * the API retry window from `started`, or `deadline` would be passed).
+     * Never sooner than the server's Retry-After, and at least 1 s after any
+     * 429 (the server's minimum, also when the header is missing or malformed).
+     */
+    async backoffBeforeRetry(attempt, started, err, deadline, retryAfter = null) {
+        if (attempt + 1 >= this.postAttempts)
+            return false;
+        let delay = (0, api_storage_client_js_1.serverRetryDelayMs)(attempt, retryAfter);
+        if (httpStatusOf(err) === 429)
+            delay = Math.max(delay, 1000 + Math.random() * 1000);
+        const end = Math.min(started + (0, api_storage_client_js_1.apiRetryWindowMs)(), deadline ?? Number.POSITIVE_INFINITY);
+        if (Date.now() + delay >= end)
+            return false;
+        await new Promise((r) => setTimeout(r, delay));
+        return true;
+    }
+    /** Time left for one attempt before `deadline`, if any. */
+    static attemptTimeout(deadline) {
+        return deadline === undefined ? undefined : Math.max(0, deadline - Date.now());
+    }
     /**
      * POST a charge intent. Returns acceptedCount on success.
      * Throws ChargeRejectedError for definitive platform rejections (no retry).
-     * Returns null when the outcome is uncertain after retries.
+     * Returns null when the outcome is uncertain after retries (the intent stays
+     * pending); `deadline` (epoch ms) ends the retries earlier, e.g. at exit.
      */
-    async postChargeWithRetries(intent) {
+    async postChargeWithRetries(intent, deadline) {
         let lastError;
+        const started = Date.now();
         for (let attempt = 0; attempt < this.postAttempts; attempt++) {
+            const { run, meta } = this.runClientOnce(ChargingManager.attemptTimeout(deadline));
             try {
-                const response = await this.apifyClient.run(this.actorRunId).charge({
+                const response = await run.charge({
                     eventName: intent.eventName,
                     count: intent.count,
                     idempotencyKey: intent.idempotencyKey,
@@ -511,9 +584,8 @@ class ChargingManager {
                 if (rejected) {
                     throw rejected;
                 }
-                if (attempt + 1 < this.postAttempts) {
-                    await new Promise((r) => setTimeout(r, 250 * (attempt + 1)));
-                }
+                if (!(await this.backoffBeforeRetry(attempt, started, err, deadline, meta.retryAfter)))
+                    break;
             }
         }
         console.warn(`[Scrapely] Failed to record charge for event ${intent.eventName} ` +
@@ -542,11 +614,13 @@ class ChargingManager {
         this.pendingCharges.delete(operationId);
         return this._voidCharge(eventName, operationId, reservedCount, true);
     }
-    async _voidCharge(eventName, operationId, reservedCount, notFoundMeansVoided = false) {
+    async _voidCharge(eventName, operationId, reservedCount, notFoundMeansVoided = false, deadline) {
         if (!this.isAtHome || !this.actorRunId)
             return false;
-        const runClient = this.apifyClient.run(this.actorRunId);
+        const started = Date.now();
         for (let attempt = 0; attempt < this.postAttempts; attempt++) {
+            const { run, meta } = this.runClientOnce(ChargingManager.attemptTimeout(deadline));
+            const runClient = run;
             try {
                 await runClient.httpClient.call({
                     url: runClient._url('charge/void'),
@@ -568,10 +642,11 @@ class ChargingManager {
                 }
                 // 404: nothing to void; 409: already settled; 4xx auth: permanent.
                 // The charge stands; keep the local reservation that matches it.
-                if (status !== undefined && status >= 400 && status < 500)
+                // 408/429 (rate limited) and 5xx are not answers: re-queue below.
+                if (status !== undefined && status >= 400 && status < 500 && !isTransientStatus(status))
                     return false;
-                if (attempt + 1 < this.postAttempts)
-                    await new Promise((r) => setTimeout(r, 250 * (attempt + 1)));
+                if (!(await this.backoffBeforeRetry(attempt, started, err, deadline, meta.retryAfter)))
+                    break;
             }
         }
         // Outcome unknown: keep the reservation and retry before the next push / at exit.
@@ -583,13 +658,21 @@ class ChargingManager {
      * @internal Retry unconfirmed voids (caller holds the operation lock).
      * Returns false while any earlier void is still unconfirmed.
      */
-    async _retryPendingVoids() {
-        const pending = this.pendingVoids;
-        this.pendingVoids = [];
-        for (const v of pending) {
-            await this._voidCharge(v.eventName, v.operationId, v.reservedCount, v.notFoundMeansVoided);
+    async _retryPendingVoids(deadline) {
+        for (let round = 0;; round++) {
+            const pending = this.pendingVoids;
+            this.pendingVoids = [];
+            for (const v of pending) {
+                await this._voidCharge(v.eventName, v.operationId, v.reservedCount, v.notFoundMeansVoided, deadline);
+            }
+            if (this.pendingVoids.length === 0)
+                return true;
+            // With a deadline (exit flush): keep retrying with backoff while time is left.
+            const delay = (0, api_storage_client_js_1.serverRetryDelayMs)(round, null);
+            if (deadline === undefined || Date.now() + delay >= deadline)
+                return false;
+            await new Promise((r) => setTimeout(r, delay));
         }
-        return this.pendingVoids.length === 0;
     }
     /** Clamp local reservation down when the server accepted fewer events. */
     /** @internal Used by pushDataAndCharge dual-event reconciliation. */
@@ -624,11 +707,13 @@ class ChargingManager {
     async _flushPendingExclusive(maxIntents = 32, deadlineMs) {
         if (!this.actorRunId)
             return 0;
-        await this._retryPendingVoids();
+        const deadline = deadlineMs !== undefined ? Date.now() + Math.max(0, deadlineMs) : undefined;
+        // Voids first: an unconfirmed one means the user may be billed for items
+        // that were never stored. At exit (with a deadline) keep retrying them.
+        await this._retryPendingVoids(deadline);
         if (this.flushInFlight) {
             return this.flushInFlight;
         }
-        const deadline = deadlineMs !== undefined ? Date.now() + Math.max(0, deadlineMs) : undefined;
         this.flushInFlight = (async () => {
             let confirmedTotal = 0;
             // Drain loop: keep flushing batches until empty or deadline.
@@ -646,7 +731,7 @@ class ChargingManager {
                     if (deadline !== undefined && Date.now() >= deadline)
                         break;
                     try {
-                        const accepted = await this.postChargeWithRetries(intent);
+                        const accepted = await this.postChargeWithRetries(intent, deadline);
                         if (accepted !== null) {
                             if (accepted < intent.count)
                                 this.markChargeLimitHit();

package/dist/storage/api-storage-client.d.ts

@@ -13,18 +13,64 @@ export declare class ApiStorageClientError extends Error {
     constructor(message: string, statusCode: number);
 }
 /**
- * Whether a non-2xx answer is retried. Idempotent methods: 500/502/503/504.
- * Others (dataset push, KV PUT) only on a 503 that certainly was not
- * processed: the rate limiter refused it ("rate-limit-unavailable", before the
- * handler), or HAProxy had no backend (its own page, nothing forwarded). Any
- * other 503 may mean "outcome unknown" (e.g. a direct dataset write whose
- * commit was not confirmed), and re-sending could store the items twice.
+ * Whether a non-2xx answer is retried. 429 always: the run API limiter and
+ * dataset backpressure refuse it before anything is written. Idempotent
+ * methods: also 500/502/503/504. Others (dataset push, KV PUT) only on a 503
+ * that certainly was not processed: the rate limiter refused it
+ * ("rate-limit-unavailable", before the handler), or HAProxy had no backend
+ * (its own page, nothing forwarded). Any other 503 may mean "outcome unknown"
+ * (e.g. a direct dataset write whose commit was not confirmed), and re-sending
+ * could store the items twice.
  */
 export declare function isRetryableResponse(status: number, idempotent: boolean, errorType?: string, bodyText?: string): boolean;
-/** Exponential backoff with jitter; honours a Retry-After (seconds or HTTP date) of up to 30 s. */
-export declare function serverRetryDelayMs(serverAttempt: number, retryAfter: string | null, random?: () => number, now?: number): number;
+/** Default time budget for retrying one API call (429, 503, network errors). */
+export declare const DEFAULT_API_RETRY_WINDOW_MS = 120000;
+/**
+ * How long one API call keeps retrying before it gives up:
+ * SCRAPELY_API_RETRY_WINDOW_SECS when set, else {@link BaseApiClient.retryWindowMs}
+ * (2 minutes by default).
+ */
+export declare function apiRetryWindowMs(): number;
+/**
+ * Delay before retry number `attempt + 1`: exponential backoff (0.5 s doubling,
+ * capped at 30 s) with equal jitter, and never earlier than the server's
+ * Retry-After (seconds or HTTP date) plus a little jitter so
+ * throttled clients don't all come back in the same instant.
+ */
+export declare function serverRetryDelayMs(attempt: number, retryAfter: string | null, random?: () => number, now?: number): number;
 /** fetch() failed before the request reached the server. */
 export declare function isConnectFailure(err: unknown): boolean;
+export interface RetryingFetchOptions {
+    /** Safe to re-send when the outcome is unknown (5xx, timeout, reset). */
+    idempotent: boolean;
+    /** Per-attempt timeout. */
+    timeoutMs: number;
+    /** Retry timeouts (idempotent calls only; a timed-out write may have been applied). */
+    retryTimeouts?: boolean;
+    /** Total time budget for retries; defaults to {@link apiRetryWindowMs}. */
+    retryWindowMs?: number;
+    /** Short label for log lines, e.g. "POST /datasets/x/items". */
+    label: string;
+}
+export interface RetryingFetchInit {
+    method: string;
+    headers: Record<string, string>;
+    body?: BodyInit;
+}
+/**
+ * fetch() with the API retry policy. Retries, with exponential backoff and
+ * jitter that honours Retry-After, until the retry window (2 minutes by
+ * default) runs out:
+ * - 429 (run API rate limit, dataset backpressure): always; nothing was applied.
+ * - 503 from the limiter or HAProxy: always; nothing was forwarded.
+ * - 500/502/503/504, timeouts and dropped connections: idempotent calls only.
+ * - Refused/unresolvable connections: always; the request never left.
+ * A write (dataset push, KV PUT, metamorph) whose request may have reached the
+ * API is never re-sent: there is no idempotency key for it, so a re-send could
+ * store the same rows twice. Such failures surface to the caller instead.
+ * Non-2xx answers that are not retried throw ApiStorageClientError.
+ */
+export declare function fetchWithRetries<R>(url: string, init: RetryingFetchInit, options: RetryingFetchOptions, decode: (response: Response) => Promise<R>): Promise<R>;
 /**
  * Options for API requests (passed through from RecordOptions)
  */
@@ -44,16 +90,12 @@ interface ApiRequestOptions {
 export declare abstract class BaseApiClient {
     protected config: ApiStorageConfig;
     protected headers: Record<string, string>;
-    private static rateLimiter;
-    private static maxRetries;
-    static maxServerRetries: number;
+    static retryWindowMs: number;
     private static defaultTimeoutMs;
     constructor(config: ApiStorageConfig);
     /**
      * Sends the request and returns the successful (2xx) response, with the
-     * retry policy: 429 (rate limiter), timeouts, and the API being briefly
-     * unavailable (503, 5xx for idempotent methods, connection failures).
-     * Non-2xx answers that are not retried throw ApiStorageClientError.
+     * retry policy of {@link fetchWithRetries}.
      */
     private send;
     protected request<T>(method: string, path: string, body?: any, contentType?: string, options?: ApiRequestOptions): Promise<T>;

package/dist/storage/api-storage-client.js

@@ -1,9 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ApiStorageClientFactory = exports.ApiRequestQueueClient = exports.ApiKeyValueStoreClient = exports.ApiDatasetClient = exports.BaseApiClient = exports.ApiStorageClientError = void 0;
+exports.ApiStorageClientFactory = exports.ApiRequestQueueClient = exports.ApiKeyValueStoreClient = exports.ApiDatasetClient = exports.BaseApiClient = exports.DEFAULT_API_RETRY_WINDOW_MS = exports.ApiStorageClientError = void 0;
 exports.isRetryableResponse = isRetryableResponse;
+exports.apiRetryWindowMs = apiRetryWindowMs;
 exports.serverRetryDelayMs = serverRetryDelayMs;
 exports.isConnectFailure = isConnectFailure;
+exports.fetchWithRetries = fetchWithRetries;
 const node_crypto_1 = require("node:crypto");
 const charging_js_1 = require("../charging.js");
 const kvs_iteration_js_1 = require("./kvs-iteration.js");
@@ -94,32 +96,6 @@ class AdaptiveRateLimiter {
         }
         this.lastRequestTime = Date.now();
     }
-    /**
-     * Handle a 429 rate limit response.
-     * Parses Retry-After header and waits the appropriate time.
-     */
-    async handleRateLimited(response) {
-        const retryAfter = response.headers.get('Retry-After');
-        let waitMs = 1000; // Default 1 second
-        if (retryAfter) {
-            // Retry-After can be seconds or a date
-            const seconds = parseInt(retryAfter, 10);
-            if (!isNaN(seconds)) {
-                waitMs = seconds * 1000;
-            }
-            else {
-                // It's a date string
-                const retryDate = new Date(retryAfter).getTime();
-                if (!isNaN(retryDate)) {
-                    waitMs = Math.max(0, retryDate - Date.now());
-                }
-            }
-        }
-        // Add jitter to prevent thundering herd
-        waitMs = waitMs + Math.random() * 100;
-        console.warn(`[RateLimiter] Rate limited, waiting ${Math.round(waitMs)}ms before retry`);
-        await new Promise(resolve => setTimeout(resolve, waitMs));
-    }
 }
 /**
  * Unwrap Apify-style v2 response envelopes ({ data: ... }).
@@ -166,32 +142,62 @@ const IDEMPOTENT_METHODS = new Set(['GET', 'HEAD', 'DELETE', 'OPTIONS']);
 const NO_BACKEND_MARKER = 'No server is available to handle this request';
 const CONNECT_FAILURE_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'EHOSTUNREACH', 'ENETUNREACH', 'UND_ERR_CONNECT_TIMEOUT']);
 /**
- * Whether a non-2xx answer is retried. Idempotent methods: 500/502/503/504.
- * Others (dataset push, KV PUT) only on a 503 that certainly was not
- * processed: the rate limiter refused it ("rate-limit-unavailable", before the
- * handler), or HAProxy had no backend (its own page, nothing forwarded). Any
- * other 503 may mean "outcome unknown" (e.g. a direct dataset write whose
- * commit was not confirmed), and re-sending could store the items twice.
+ * Whether a non-2xx answer is retried. 429 always: the run API limiter and
+ * dataset backpressure refuse it before anything is written. Idempotent
+ * methods: also 500/502/503/504. Others (dataset push, KV PUT) only on a 503
+ * that certainly was not processed: the rate limiter refused it
+ * ("rate-limit-unavailable", before the handler), or HAProxy had no backend
+ * (its own page, nothing forwarded). Any other 503 may mean "outcome unknown"
+ * (e.g. a direct dataset write whose commit was not confirmed), and re-sending
+ * could store the items twice.
  */
 function isRetryableResponse(status, idempotent, errorType, bodyText) {
+    if (status === 429)
+        return true;
     if (idempotent)
         return status === 500 || status === 502 || status === 503 || status === 504;
     if (status !== 503)
         return false;
     return errorType === 'rate-limit-unavailable' || (bodyText ?? '').includes(NO_BACKEND_MARKER);
 }
-/** Exponential backoff with jitter; honours a Retry-After (seconds or HTTP date) of up to 30 s. */
-function serverRetryDelayMs(serverAttempt, retryAfter, random = Math.random, now = Date.now()) {
-    if (retryAfter !== null && retryAfter.trim() !== '') {
-        let secs = Number(retryAfter);
-        if (!Number.isFinite(secs)) {
-            const at = Date.parse(retryAfter);
-            secs = Number.isFinite(at) ? Math.max(0, (at - now) / 1000) : NaN;
-        }
-        if (Number.isFinite(secs) && secs >= 0)
-            return Math.min(secs, 30) * 1000 + random() * 100;
-    }
-    return Math.min(8000, 500 * 2 ** serverAttempt) + random() * 250;
+/** Default time budget for retrying one API call (429, 503, network errors). */
+exports.DEFAULT_API_RETRY_WINDOW_MS = 120_000;
+const MAX_BACKOFF_MS = 30_000;
+/**
+ * How long one API call keeps retrying before it gives up:
+ * SCRAPELY_API_RETRY_WINDOW_SECS when set, else {@link BaseApiClient.retryWindowMs}
+ * (2 minutes by default).
+ */
+function apiRetryWindowMs() {
+    const env = Number(process.env.SCRAPELY_API_RETRY_WINDOW_SECS);
+    if (process.env.SCRAPELY_API_RETRY_WINDOW_SECS && Number.isFinite(env) && env >= 0)
+        return env * 1000;
+    return BaseApiClient.retryWindowMs;
+}
+function parseRetryAfterMs(retryAfter, now) {
+    if (retryAfter === null || retryAfter === undefined || retryAfter.trim() === '')
+        return undefined;
+    let secs = Number(retryAfter);
+    if (!Number.isFinite(secs)) {
+        const at = Date.parse(retryAfter);
+        secs = Number.isFinite(at) ? Math.max(0, (at - now) / 1000) : NaN;
+    }
+    // Not capped: a Retry-After past the retry window ends the retries instead.
+    return Number.isFinite(secs) && secs >= 0 ? secs * 1000 : undefined;
+}
+/**
+ * Delay before retry number `attempt + 1`: exponential backoff (0.5 s doubling,
+ * capped at 30 s) with equal jitter, and never earlier than the server's
+ * Retry-After (seconds or HTTP date) plus a little jitter so
+ * throttled clients don't all come back in the same instant.
+ */
+function serverRetryDelayMs(attempt, retryAfter, random = Math.random, now = Date.now()) {
+    const exp = Math.min(MAX_BACKOFF_MS, 500 * 2 ** Math.min(attempt, 16));
+    const backoff = exp / 2 + random() * (exp / 2);
+    const ra = parseRetryAfterMs(retryAfter, now);
+    if (ra === undefined)
+        return backoff;
+    return Math.max(ra + random() * Math.min(1000, ra * 0.2 + 100), backoff);
 }
 function fetchErrorCode(err) {
     const e = err;
@@ -206,6 +212,109 @@ function describeFetchError(err) {
     return fetchErrorCode(err) ?? err?.message ?? String(err);
 }
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+// Shared by every API call of this process (storage clients, status message,
+// metamorph, reboot), so throttling seen on one slows the others too.
+const sharedRateLimiter = new AdaptiveRateLimiter();
+/**
+ * fetch() with the API retry policy. Retries, with exponential backoff and
+ * jitter that honours Retry-After, until the retry window (2 minutes by
+ * default) runs out:
+ * - 429 (run API rate limit, dataset backpressure): always; nothing was applied.
+ * - 503 from the limiter or HAProxy: always; nothing was forwarded.
+ * - 500/502/503/504, timeouts and dropped connections: idempotent calls only.
+ * - Refused/unresolvable connections: always; the request never left.
+ * A write (dataset push, KV PUT, metamorph) whose request may have reached the
+ * API is never re-sent: there is no idempotency key for it, so a re-send could
+ * store the same rows twice. Such failures surface to the caller instead.
+ * Non-2xx answers that are not retried throw ApiStorageClientError.
+ */
+async function fetchWithRetries(url, init, options, decode) {
+    const { idempotent, timeoutMs, label } = options;
+    const retryTimeouts = options.retryTimeouts !== false && idempotent;
+    const deadline = Date.now() + Math.max(0, options.retryWindowMs ?? apiRetryWindowMs());
+    let attempt = 0;
+    /** Sleeps before the next attempt; false once the retry window is used up. */
+    const backoff = async (why, retryAfter) => {
+        // Retrying sooner than asked would only be refused again: give up when
+        // the next try would land past the window.
+        const waitMs = serverRetryDelayMs(attempt, retryAfter);
+        if (Date.now() + waitMs > deadline)
+            return false;
+        attempt++;
+        console.warn(`[Scrapely] ${label} ${why}; retry ${attempt} in ${Math.round(waitMs)}ms`);
+        await sleep(waitMs);
+        return true;
+    };
+    for (;;) {
+        await sharedRateLimiter.waitForSlot();
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+        try {
+            const response = await fetch(url, { method: init.method, headers: init.headers, body: init.body, signal: controller.signal });
+            sharedRateLimiter.updateFromHeaders(response.headers);
+            if (response.ok) {
+                try {
+                    // Decoded inside the retry loop: a body cut off after
+                    // the headers is a transport failure like any other.
+                    return await decode(response);
+                }
+                catch (bodyErr) {
+                    // The request was applied (2xx); a write whose answer
+                    // timed out while being read must not be sent again.
+                    if (bodyErr.name === 'AbortError') {
+                        if (!idempotent || !retryTimeouts) {
+                            const timeoutError = new Error(`Request timed out after ${timeoutMs}ms (response body${idempotent ? '' : '; the request was applied'})`);
+                            timeoutError.isTimeout = true;
+                            throw timeoutError;
+                        }
+                        if (await backoff(`response body timed out after ${timeoutMs}ms`, null))
+                            continue;
+                        throw bodyErr;
+                    }
+                    if (idempotent && await backoff(`response body failed (${describeFetchError(bodyErr)})`, null))
+                        continue;
+                    throw bodyErr;
+                }
+            }
+            const text = await response.text().catch(() => '');
+            let errorBody;
+            try {
+                errorBody = JSON.parse(text);
+            }
+            catch {
+                errorBody = undefined;
+            }
+            if (isRetryableResponse(response.status, idempotent, errorBody?.error?.type, text)
+                && await backoff(`answered ${response.status}`, response.headers.get('retry-after'))) {
+                continue;
+            }
+            throw new ApiStorageClientError(errorBody?.error?.message || response.statusText || `API request failed: ${response.status}`, response.status);
+        }
+        catch (err) {
+            // HTTP errors (already decided above) are final.
+            if (isApiStorageClientError(err))
+                throw err;
+            if (err?.isTimeout)
+                throw err;
+            if (err.name === 'AbortError') {
+                const timeoutError = new Error(`Request timed out after ${timeoutMs}ms`);
+                timeoutError.isTimeout = true;
+                if (retryTimeouts && await backoff(`timed out after ${timeoutMs}ms`, null))
+                    continue;
+                throw timeoutError;
+            }
+            // fetch() failed without a response. A refused/unresolvable
+            // connection never reached the API, so it is safe for every
+            // method; anything else (reset mid-request) only for idempotent ones.
+            if ((idempotent || isConnectFailure(err)) && await backoff(`failed (${describeFetchError(err)})`, null))
+                continue;
+            throw err;
+        }
+        finally {
+            clearTimeout(timeoutId);
+        }
+    }
+}
 /** Map GET storage errors: rethrow 403, return undefined for 404 (triggers getOrCreate for names only). */
 function handleStorageGetError(err) {
     if (isApiStorageClientError(err)) {
@@ -266,15 +375,10 @@ function makeAsyncIterableResult(promiseFn) {
 class BaseApiClient {
     config;
     headers;
-    // Shared rate limiter instance - coordinates across all API clients
-    // This is static so it's shared across all storage clients for this run
-    static rateLimiter = new AdaptiveRateLimiter();
-    // Maximum retries for rate-limited requests
-    static maxRetries = 3;
-    // Retries while the API is briefly unavailable (503/5xx, connection
-    // refused): backoff 0.5 s doubling to 8 s, about 45 s in total, which
-    // covers a main redeploy.
-    static maxServerRetries = 8;
+    // How long one call keeps retrying 429/503/network errors before it gives
+    // up (SCRAPELY_API_RETRY_WINDOW_SECS overrides it). Long enough that a
+    // throttled run slows down instead of failing, and covers a main redeploy.
+    static retryWindowMs = exports.DEFAULT_API_RETRY_WINDOW_MS;
     // Default timeout for requests (30 seconds)
     static defaultTimeoutMs = 30000;
     constructor(config) {
@@ -287,12 +391,9 @@ class BaseApiClient {
     }
     /**
      * Sends the request and returns the successful (2xx) response, with the
-     * retry policy: 429 (rate limiter), timeouts, and the API being briefly
-     * unavailable (503, 5xx for idempotent methods, connection failures).
-     * Non-2xx answers that are not retried throw ApiStorageClientError.
+     * retry policy of {@link fetchWithRetries}.
      */
     async send(method, path, body, contentType, options, decode) {
-        const url = `${this.config.apiUrl}/v2${path}`;
         const headers = { ...this.headers };
         if (contentType) {
             headers['Content-Type'] = contentType;
@@ -300,114 +401,19 @@ class BaseApiClient {
         const timeoutMs = options?.timeoutSecs
             ? options.timeoutSecs * 1000
             : BaseApiClient.defaultTimeoutMs;
-        const shouldRetryTimeouts = options?.doNotRetryTimeouts !== true;
-        const idempotent = options?.idempotent ?? IDEMPOTENT_METHODS.has(method.toUpperCase());
-        let attempt = 0; // timeouts and 429s
-        let serverAttempt = 0; // 503/5xx and connection failures (API briefly unavailable)
-        for (;;) {
-            // Wait for rate limiter slot before making request
-            await BaseApiClient.rateLimiter.waitForSlot();
-            // Create AbortController for timeout
-            const controller = new AbortController();
-            const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
-            try {
-                const response = await fetch(url, {
-                    method,
-                    headers,
-                    // Binary bodies (KV records) go out as the raw bytes, never JSON/base64.
-                    body: body === undefined ? undefined
-                        : typeof body === 'string' || body instanceof Uint8Array ? body
-                            : JSON.stringify(body),
-                    signal: controller.signal,
-                });
-                // Update rate limiter state from response headers
-                BaseApiClient.rateLimiter.updateFromHeaders(response.headers);
-                // Handle rate limiting (429)
-                if (response.status === 429) {
-                    if (attempt < BaseApiClient.maxRetries) {
-                        attempt++;
-                        await BaseApiClient.rateLimiter.handleRateLimited(response);
-                        continue; // Retry
-                    }
-                    const error = await response.json().catch(() => ({ error: { message: 'Rate limit exceeded' } }));
-                    throw new ApiStorageClientError(error.error?.message || 'Rate limit exceeded', 429);
-                }
-                if (response.ok) {
-                    try {
-                        // Decoded inside the retry loop: a body cut off after
-                        // the headers is a transport failure like any other.
-                        return await decode(response);
-                    }
-                    catch (bodyErr) {
-                        // The request was applied (2xx); a write whose answer
-                        // timed out while being read must not be sent again.
-                        if (bodyErr.name === 'AbortError' && !idempotent) {
-                            const timeoutError = new Error(`Request timed out after ${timeoutMs}ms (response body; the request was applied)`);
-                            timeoutError.isTimeout = true;
-                            throw timeoutError;
-                        }
-                        if (idempotent && bodyErr.name !== 'AbortError' && serverAttempt < BaseApiClient.maxServerRetries) {
-                            const waitMs = serverRetryDelayMs(serverAttempt, null);
-                            serverAttempt++;
-                            console.warn(`[Scrapely] ${method} ${path} response body failed (${describeFetchError(bodyErr)}); retry ${serverAttempt}/${BaseApiClient.maxServerRetries} in ${Math.round(waitMs)}ms`);
-                            await sleep(waitMs);
-                            continue;
-                        }
-                        throw bodyErr;
-                    }
-                }
-                // API briefly unavailable (main redeploy, limiter store blip).
-                const text = await response.text().catch(() => '');
-                let errorBody;
-                try {
-                    errorBody = JSON.parse(text);
-                }
-                catch {
-                    errorBody = undefined;
-                }
-                if (isRetryableResponse(response.status, idempotent, errorBody?.error?.type, text)
-                    && serverAttempt < BaseApiClient.maxServerRetries) {
-                    const waitMs = serverRetryDelayMs(serverAttempt, response.headers.get('retry-after'));
-                    serverAttempt++;
-                    console.warn(`[Scrapely] ${method} ${path} answered ${response.status}; retry ${serverAttempt}/${BaseApiClient.maxServerRetries} in ${Math.round(waitMs)}ms`);
-                    await sleep(waitMs);
-                    continue;
-                }
-                throw new ApiStorageClientError(errorBody?.error?.message || response.statusText || `API request failed: ${response.status}`, response.status);
-            }
-            catch (err) {
-                // Handle timeout/abort errors
-                if (err.name === 'AbortError') {
-                    const timeoutError = new Error(`Request timed out after ${timeoutMs}ms`);
-                    timeoutError.isTimeout = true;
-                    // Don't retry if doNotRetryTimeouts is true
-                    if (!shouldRetryTimeouts || attempt >= BaseApiClient.maxRetries) {
-                        throw timeoutError;
-                    }
-                    attempt++;
-                    // Wait a bit before retrying
-                    await sleep(100 * attempt);
-                    continue;
-                }
-                // HTTP errors (already decided above) are final.
-                if (isApiStorageClientError(err))
-                    throw err;
-                // fetch() failed without a response. A refused/unresolvable
-                // connection never reached the API, so it is safe for every
-                // method; anything else (reset mid-request) only for idempotent ones.
-                if ((idempotent || isConnectFailure(err)) && serverAttempt < BaseApiClient.maxServerRetries) {
-                    const waitMs = serverRetryDelayMs(serverAttempt, null);
-                    serverAttempt++;
-                    console.warn(`[Scrapely] ${method} ${path} failed (${describeFetchError(err)}); retry ${serverAttempt}/${BaseApiClient.maxServerRetries} in ${Math.round(waitMs)}ms`);
-                    await sleep(waitMs);
-                    continue;
-                }
-                throw err;
-            }
-            finally {
-                clearTimeout(timeoutId);
-            }
-        }
+        return fetchWithRetries(`${this.config.apiUrl}/v2${path}`, {
+            method,
+            headers,
+            // Binary bodies (KV records) go out as the raw bytes, never JSON/base64.
+            body: body === undefined ? undefined
+                : typeof body === 'string' || body instanceof Uint8Array ? body
+                    : JSON.stringify(body),
+        }, {
+            idempotent: options?.idempotent ?? IDEMPOTENT_METHODS.has(method.toUpperCase()),
+            timeoutMs,
+            retryTimeouts: options?.doNotRetryTimeouts !== true,
+            label: `${method} ${path}`,
+        }, decode);
     }
     async request(method, path, body, contentType, options) {
         return this.send(method, path, body, contentType, options, async (response) => {
@@ -728,7 +734,10 @@ class ApiKeyValueStoreClient extends BaseApiClient {
      * @returns A public URL for accessing the record
      */
     getPublicUrl(key) {
-        return `${this.config.apiUrl}/v2/key-value-stores/${this.id}/records/${key}`;
+        // apiUrl is the container-internal API address; links handed to users
+        // must use the public base URL the platform injects.
+        const publicBase = (process.env.APIFY_API_PUBLIC_BASE_URL || process.env.ACTOR_API_PUBLIC_BASE_URL || this.config.apiUrl).replace(/\/+$/, '');
+        return `${publicBase}/v2/key-value-stores/${this.id}/records/${key}`;
     }
 }
 exports.ApiKeyValueStoreClient = ApiKeyValueStoreClient;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.16",
+    "version": "1.2.17",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",