scrapely 1.2.15 → 1.2.17

package/dist/actor.js

@@ -33,6 +33,12 @@ const charging_client_js_1 = require("./charging_client.js");
 const proxy_configuration_js_1 = require("./proxy-configuration.js");
 const browser_configuration_js_1 = require("./browser-configuration.js");
 const utils_js_1 = require("./utils.js");
+/** "429 Run API rate limit exceeded ..." for an API error, the message otherwise. */
+function describeRunApiError(err) {
+    const status = err.statusCode;
+    const message = err?.message ?? String(err);
+    return typeof status === 'number' ? `${status} ${message}` : message;
+}
 /** Pre-init fallback; after init, {@link Actor.events} returns the EventManager emitter. */
 const fallbackEvents = new node_events_1.EventEmitter();
 let apifyClientInstance;
@@ -273,6 +279,8 @@ class Actor {
             apifyClientInstance = this._apifyClient;
             apifyClientAssigned = true;
             await this.chargingManager.init();
+            // Storage clients refuse default-dataset writes past this run's limit.
+            (0, charging_js_1.setActiveChargingManager)(this.chargingManager);
             // Flush pending PPE intents before hard exit so uncertain charges can still settle.
             const gracefulSignalExit = (signal) => {
                 const doExit = () => {
@@ -517,6 +525,14 @@ class Actor {
         const { Dataset } = await import('crawlee');
         const dataset = await Dataset.open();
         if (eventName === undefined && !this.usesPushDataInterception(dataset)) {
+            // Charge-limit safety net (also for local storage): this push is not
+            // charged, so past the run's spending limit it is not stored.
+            if (this.initialized &&
+                dataset.id === this.config.get('defaultDatasetId') &&
+                this.chargingManager.isChargeLimitReached()) {
+                this.chargingManager._noteRefusedDefaultDatasetItems(Array.isArray(item) ? item.length : 1);
+                return;
+            }
             await dataset.pushData(item);
             return;
         }
@@ -938,7 +954,9 @@ class Actor {
                 }
                 if (cm && typeof cm.flushPending === 'function') {
                     await Promise.race([
-                        cm.flushPending(64, flushDeadlineMs),
+                        // The flush's own deadline ends its retries a little before
+                        // the race gives up on it, so none is cut off mid-request.
+                        cm.flushPending(64, Math.max(500, flushDeadlineMs - 1000)),
                         new Promise((resolve) => setTimeout(resolve, flushDeadlineMs)),
                     ]);
                 }
@@ -951,6 +969,7 @@ class Actor {
         await eventManager.close();
         (0, events_client_js_1.disconnectEvents)();
         (0, patch_storage_js_1.disconnectStorage)();
+        (0, charging_js_1.setActiveChargingManager)(undefined);
         // Let buffered stdout/stderr reach the log pipe before the process exits
         // (pipe writes are async in Node). Storage was flushed above, so this
         // replaces the old fixed 1 s sleep: it resolves as soon as both streams
@@ -968,6 +987,8 @@ class Actor {
                 Actor.setStatusMessage(message, {
                     isStatusMessageTerminal: true,
                     level: exitCode > 0 ? 'ERROR' : 'INFO',
+                    // Stop retrying when the exit stops waiting for it.
+                    retryWindowMs: timeoutMs,
                 }),
                 new Promise((_, reject) => {
                     timer = setTimeout(() => reject(new Error(`timed out after ${timeoutMs}ms`)), timeoutMs);
@@ -1023,7 +1044,9 @@ class Actor {
         if (!apiUrl || !token || !runId) {
             return {};
         }
-        const response = await fetch(`${apiUrl}/v2/actor-runs/${runId}`, {
+        // Setting the message is idempotent: a 429 (run API rate limit), 5xx or
+        // dropped connection is retried with backoff instead of failing the run.
+        const body = await (0, api_storage_client_js_1.fetchWithRetries)(`${apiUrl}/v2/actor-runs/${runId}`, {
             method: 'PUT',
             headers: {
                 'Content-Type': 'application/json',
@@ -1033,13 +1056,14 @@ class Actor {
                 statusMessage: message,
                 isStatusMessageTerminal,
             }),
-            signal: AbortSignal.timeout(10_000),
+        }, {
+            idempotent: true,
+            timeoutMs: 10_000,
+            label: 'PUT status message',
+            retryWindowMs: options?.retryWindowMs,
+        }, async (response) => (await response.json().catch(() => ({})))).catch((err) => {
+            throw new Error(`Failed to update status message: ${describeRunApiError(err)}`);
         });
-        if (!response.ok) {
-            const error = await response.text();
-            throw new Error(`Failed to update status message: ${response.status} ${error}`);
-        }
-        const body = (await response.json());
         return (body.data ?? {});
     }
     /**
@@ -1193,18 +1217,18 @@ class Actor {
         console.log(`[Scrapely] Metamorphosing to actor ${targetActorId}`);
         // Signal the server to perform the metamorph
         try {
-            const response = await fetch(`${apiUrl}/v2/actor-runs/${runId}/metamorph`, {
+            // Retried only while certainly not applied (429, limiter/HAProxy 503,
+            // refused connection): a second metamorph must never be started.
+            await (0, api_storage_client_js_1.fetchWithRetries)(`${apiUrl}/v2/actor-runs/${runId}/metamorph`, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
                     'Authorization': `Bearer ${token}`,
                 },
                 body: JSON.stringify(body),
+            }, { idempotent: false, timeoutMs: 30_000, label: 'POST metamorph' }, async () => undefined).catch((err) => {
+                throw new Error(`Metamorph request failed: ${describeRunApiError(err)}`);
             });
-            if (!response.ok) {
-                const error = await response.text();
-                throw new Error(`Metamorph request failed: ${response.status} ${error}`);
-            }
             console.log('[Scrapely] Metamorph request accepted, exiting...');
         }
         catch (err) {
@@ -1291,18 +1315,18 @@ class Actor {
         const token = config.get('token') || '';
         const runId = config.get('actorRunId') || '';
         try {
-            const response = await fetch(`${apiUrl}/v2/actor-runs/${runId}/reboot`, {
+            // Retried only while certainly not applied (429, limiter/HAProxy 503,
+            // refused connection).
+            const status = await (0, api_storage_client_js_1.fetchWithRetries)(`${apiUrl}/v2/actor-runs/${runId}/reboot`, {
                 method: 'POST',
                 headers: {
                     'Authorization': `Bearer ${token}`,
                     'Content-Type': 'application/json',
                 },
+            }, { idempotent: false, timeoutMs: 30_000, label: 'POST reboot' }, async (response) => response.status).catch((err) => {
+                throw new Error(`Reboot request failed: ${describeRunApiError(err)}`);
             });
-            if (!response.ok) {
-                const error = await response.text();
-                throw new Error(`Reboot request failed: ${response.status} ${error}`);
-            }
-            console.log(`[Scrapely] Reboot request accepted (${response.status})`);
+            console.log(`[Scrapely] Reboot request accepted (${status})`);
             // Wait briefly for the reboot signal to be processed, then exit
             const sleepMillis = options?.customAfterSleepMillis ?? Actor._actorConfig().get('metamorphAfterSleepMillis', 1000);
             await new Promise((resolve) => setTimeout(resolve, sleepMillis));

package/dist/charging.d.ts

@@ -29,6 +29,17 @@ export declare class ChargeRejectedError extends Error {
         reason: string;
     });
 }
+/** @internal Register the manager whose limit guards default-dataset writes. */
+export declare function setActiveChargingManager(manager: ChargingManager | undefined): void;
+/** @internal The run's default dataset ID, as the platform passes it. */
+export declare function defaultDatasetIdFromEnv(): string | undefined;
+/**
+ * @internal True when a write of `itemCount` rows to the default dataset must
+ * be dropped (pay-per-event run past its spending limit). Called by storage
+ * clients for writes that did not go through `pushDataAndCharge`
+ * (`Actor.openDataset().pushData()`, crawler `pushData`, ...).
+ */
+export declare function refuseDefaultDatasetPush(itemCount: number): boolean;
 export declare const APIFY_DEFAULT_DATASET_ITEM_EVENT = "apify-default-dataset-item";
 /** Scrapely-native synthetic event for default-dataset items. */
 export declare const SCRAPELY_DEFAULT_DATASET_ITEM_EVENT = "scrapely-default-dataset-item";
@@ -62,8 +73,32 @@ export declare class ChargingManager {
     private readonly apifyClient;
     /** Set during actor shutdown; rejects new charge operations after the final flush. */
     private _closed;
+    /** Sticky: the platform refused or capped a charge (HTTP 402 / fewer accepted) or the local gate trimmed one. */
+    private chargeLimitHit;
+    /** Default-dataset rows refused after the limit (one warning, then a count). */
+    private refusedDefaultDatasetItems;
     /** @internal Called by Actor shutdown before the final flush. */
     close(): void;
+    /**
+     * Whether this pay-per-event run has used up its spending limit.
+     *
+     * True once a charge was refused or capped by the limit (the platform's
+     * HTTP 402 or a partial acceptance, or the local budget gate), or once the
+     * remaining budget cannot pay for one event of the cheapest priced type.
+     * From then on, pushes to the default dataset that are not paid for are
+     * refused. Always false for actors without pay-per-event pricing, without
+     * a limit, or with only free events.
+     */
+    isChargeLimitReached(): boolean;
+    private markChargeLimitHit;
+    /** @internal Count rows dropped because of the limit; warn on the first one only. */
+    _noteRefusedDefaultDatasetItems(count: number): void;
+    /** @internal */
+    _markChargeLimitHit(): void;
+    /** @internal Whether every row of a push with these events is charged a positive price. */
+    _isPaidPerItem(eventName: string | undefined, isDefaultDataset: boolean): boolean;
+    /** @internal */
+    _refuseDefaultDatasetPush(itemCount: number): boolean;
     /** @internal Whether shutdown has closed this manager to new charges. */
     isClosed(): boolean;
     private readonly LOCAL_CHARGING_LOG_DATASET_NAME;
@@ -126,10 +161,29 @@ export declare class ChargingManager {
     _chargeableWithinLimit(): Record<string, number>;
     private getChargeableWithinLimit;
     private quarantineStalePending;
+    /**
+     * The run client with scrapely-client's internal retries off (one request
+     * per call) and its timeout capped at `timeoutMs`, so the retry loops here
+     * decide when to retry and never outlive their deadline. `meta.retryAfter`
+     * receives the Retry-After of the last answer (ApifyApiError drops the
+     * headers). A client without that shape (tests, custom clients) is used as
+     * it is.
+     */
+    private runClientOnce;
+    /**
+     * Sleep before the next same-key attempt; false when none is left (attempts,
+     * the API retry window from `started`, or `deadline` would be passed).
+     * Never sooner than the server's Retry-After, and at least 1 s after any
+     * 429 (the server's minimum, also when the header is missing or malformed).
+     */
+    private backoffBeforeRetry;
+    /** Time left for one attempt before `deadline`, if any. */
+    private static attemptTimeout;
     /**
      * POST a charge intent. Returns acceptedCount on success.
      * Throws ChargeRejectedError for definitive platform rejections (no retry).
-     * Returns null when the outcome is uncertain after retries.
+     * Returns null when the outcome is uncertain after retries (the intent stays
+     * pending); `deadline` (epoch ms) ends the retries earlier, e.g. at exit.
      */
     private postChargeWithRetries;
     /** @internal Idempotency key of the charge accepted by the last _chargeExclusive call, if any. */
@@ -146,12 +200,12 @@ export declare class ChargingManager {
      * means it never applied.
      */
     _cancelUncertainCharge(eventName: string, operationId: string, reservedCount: number): Promise<boolean>;
-    _voidCharge(eventName: string, operationId: string, reservedCount: number, notFoundMeansVoided?: boolean): Promise<boolean>;
+    _voidCharge(eventName: string, operationId: string, reservedCount: number, notFoundMeansVoided?: boolean, deadline?: number): Promise<boolean>;
     /**
      * @internal Retry unconfirmed voids (caller holds the operation lock).
      * Returns false while any earlier void is still unconfirmed.
      */
-    _retryPendingVoids(): Promise<boolean>;
+    _retryPendingVoids(deadline?: number): Promise<boolean>;
     /** Clamp local reservation down when the server accepted fewer events. */
     /** @internal Used by pushDataAndCharge dual-event reconciliation. */
     applyAcceptedCount(eventName: string, reservedCount: number, acceptedCount: number, eventPrice?: number): number;

package/dist/charging.js

@@ -1,6 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ChargingManager = exports.DEFAULT_DATASET_ITEM_EVENT = exports.SCRAPELY_DEFAULT_DATASET_ITEM_EVENT = exports.APIFY_DEFAULT_DATASET_ITEM_EVENT = exports.ChargeRejectedError = exports.ChargeUncertainError = exports.CHARGE_IDEMPOTENCY_SAFE_AGE_MS = void 0;
+exports.setActiveChargingManager = setActiveChargingManager;
+exports.defaultDatasetIdFromEnv = defaultDatasetIdFromEnv;
+exports.refuseDefaultDatasetPush = refuseDefaultDatasetPush;
 exports.isDefaultDatasetItemEvent = isDefaultDatasetItemEvent;
 exports.resolveDefaultDatasetItemEventName = resolveDefaultDatasetItemEventName;
 exports.hasDefaultDatasetItemEventInPricing = hasDefaultDatasetItemEventInPricing;
@@ -11,11 +14,25 @@ exports.pushCertainlyFailed = pushCertainlyFailed;
 // Derived from apify-sdk-js (src/charging.ts),
 // Copyright Apify Technologies s.r.o., licensed under the Apache License 2.0.
 // Modified by TeraShift GmbH. See LICENSE-APACHE and NOTICE.
+const node_async_hooks_1 = require("node:async_hooks");
 const node_crypto_1 = require("node:crypto");
+const api_storage_client_js_1 = require("./storage/api-storage-client.js");
 /** Client safe-replay window; server TTL is longer (35d). */
 exports.CHARGE_IDEMPOTENCY_SAFE_AGE_MS = 30 * 24 * 60 * 60 * 1000;
 const PENDING_MAX_ENTRIES = 1024;
-const POST_ATTEMPTS = 3;
+// Same-key attempts per charge / void. Each attempt is a single request
+// (scrapely-client's own retries are switched off for it, see runClientOnce),
+// so the retries here can honour deadlines: 429, 408, 5xx, timeouts and
+// network errors are re-sent with the same idempotency key, with exponential
+// backoff and jitter, never sooner than the server's Retry-After, within the API
+// retry window (2 minutes, SCRAPELY_API_RETRY_WINDOW_SECS) or the caller's
+// deadline. Then the charge stays pending / the void is re-queued and retried
+// before the next push and at exit, the same as the Python SDK.
+const POST_ATTEMPTS = 64;
+/** 408/429 and 5xx: the platform could not take the request now; try again later. */
+function isTransientStatus(status) {
+    return status === 408 || status === 429 || (status !== undefined && status >= 500);
+}
 class ChargeUncertainError extends Error {
     operationId;
     eventName;
@@ -48,6 +65,44 @@ class ChargeRejectedError extends Error {
     }
 }
 exports.ChargeRejectedError = ChargeRejectedError;
+/** True when a ChargeRejectedError means the run's spending limit is used up. */
+function isChargeLimitRejection(err) {
+    return (err.statusCode === 402 ||
+        err.reason === 'charge-limit-exceeded' ||
+        /charge-limit-exceeded/i.test(err.message));
+}
+/*
+ * Charge-limit safety net for the default dataset.
+ *
+ * Once a pay-per-event run has used up its spending limit (maxTotalChargeUsd),
+ * rows written to the run's default dataset would go out unbilled. The SDK then
+ * refuses such writes: nothing is stored, one warning is logged and
+ * `ChargingManager.isChargeLimitReached()` turns true. Pushes made by
+ * `pushDataAndCharge` are decided there and run inside `chargeGatedPush`, so
+ * the storage client does not check them twice.
+ */
+const chargeGatedPush = new node_async_hooks_1.AsyncLocalStorage();
+/** The charging manager of the running Actor; storage clients consult it. */
+let activeChargingManager;
+/** @internal Register the manager whose limit guards default-dataset writes. */
+function setActiveChargingManager(manager) {
+    activeChargingManager = manager;
+}
+/** @internal The run's default dataset ID, as the platform passes it. */
+function defaultDatasetIdFromEnv() {
+    return process.env.ACTOR_DEFAULT_DATASET_ID || process.env.APIFY_DEFAULT_DATASET_ID || undefined;
+}
+/**
+ * @internal True when a write of `itemCount` rows to the default dataset must
+ * be dropped (pay-per-event run past its spending limit). Called by storage
+ * clients for writes that did not go through `pushDataAndCharge`
+ * (`Actor.openDataset().pushData()`, crawler `pushData`, ...).
+ */
+function refuseDefaultDatasetPush(itemCount) {
+    if (chargeGatedPush.getStore())
+        return false;
+    return activeChargingManager?._refuseDefaultDatasetPush(itemCount) ?? false;
+}
 exports.APIFY_DEFAULT_DATASET_ITEM_EVENT = 'apify-default-dataset-item';
 /** Scrapely-native synthetic event for default-dataset items. */
 exports.SCRAPELY_DEFAULT_DATASET_ITEM_EVENT = 'scrapely-default-dataset-item';
@@ -95,9 +150,77 @@ class ChargingManager {
     apifyClient;
     /** Set during actor shutdown; rejects new charge operations after the final flush. */
     _closed = false;
+    /** Sticky: the platform refused or capped a charge (HTTP 402 / fewer accepted) or the local gate trimmed one. */
+    chargeLimitHit = false;
+    /** Default-dataset rows refused after the limit (one warning, then a count). */
+    refusedDefaultDatasetItems = 0;
     /** @internal Called by Actor shutdown before the final flush. */
     close() {
         this._closed = true;
+        if (this.refusedDefaultDatasetItems > 0) {
+            console.warn(`[Scrapely] ${this.refusedDefaultDatasetItems} result(s) pushed to the default dataset after the spending limit was reached were not saved`);
+        }
+    }
+    /**
+     * Whether this pay-per-event run has used up its spending limit.
+     *
+     * True once a charge was refused or capped by the limit (the platform's
+     * HTTP 402 or a partial acceptance, or the local budget gate), or once the
+     * remaining budget cannot pay for one event of the cheapest priced type.
+     * From then on, pushes to the default dataset that are not paid for are
+     * refused. Always false for actors without pay-per-event pricing, without
+     * a limit, or with only free events.
+     */
+    isChargeLimitReached() {
+        if (this.chargingState === undefined || !this.isPayPerEvent)
+            return false;
+        if (this.chargeLimitHit)
+            return true;
+        if (!Number.isFinite(this.maxTotalChargeUsd))
+            return false;
+        const prices = Object.values(this.pricingInfo)
+            .map(({ price }) => price)
+            .filter((price) => price > 0);
+        if (prices.length === 0)
+            return false;
+        // Same rounding as the budget gate, so 0.03 - 0.02 still pays a $0.01 event.
+        return this.calculateMaxChargesByPrice(Math.min(...prices)) <= 0;
+    }
+    markChargeLimitHit() {
+        if (this.isPayPerEvent)
+            this.chargeLimitHit = true;
+    }
+    /** @internal Count rows dropped because of the limit; warn on the first one only. */
+    _noteRefusedDefaultDatasetItems(count) {
+        if (count <= 0)
+            return;
+        if (this.refusedDefaultDatasetItems === 0) {
+            const limit = Number.isFinite(this.maxTotalChargeUsd) ? `$${this.maxTotalChargeUsd}` : 'the limit';
+            console.warn(`[Scrapely] The spending limit of this run (maxTotalChargeUsd = ${limit}) has been reached: ` +
+                'results pushed to the default dataset are no longer saved. The Actor should stop now ' +
+                '(check ChargeResult.eventChargeLimitReached or Actor.getChargingManager().isChargeLimitReached()).');
+        }
+        this.refusedDefaultDatasetItems += count;
+    }
+    /** @internal */
+    _markChargeLimitHit() {
+        this.markChargeLimitHit();
+    }
+    /** @internal Whether every row of a push with these events is charged a positive price. */
+    _isPaidPerItem(eventName, isDefaultDataset) {
+        const names = [];
+        if (eventName !== undefined)
+            names.push(eventName);
+        if (isDefaultDataset)
+            names.push(resolveDefaultDatasetItemEventName(this.pricingInfo));
+        return names.some((name) => (this.calculateEventPrice(name) ?? 0) > 0);
+    }
+    /** @internal */
+    _refuseDefaultDatasetPush(itemCount) {
+        if (itemCount <= 0 || !this.isChargeLimitReached())
+            return false;
+        this._noteRefusedDefaultDatasetItems(itemCount);
+        return true;
     }
     /** @internal Whether shutdown has closed this manager to new charges. */
     isClosed() {
@@ -377,16 +500,76 @@ class ChargingManager {
             }
         }
     }
+    /**
+     * The run client with scrapely-client's internal retries off (one request
+     * per call) and its timeout capped at `timeoutMs`, so the retry loops here
+     * decide when to retry and never outlive their deadline. `meta.retryAfter`
+     * receives the Retry-After of the last answer (ApifyApiError drops the
+     * headers). A client without that shape (tests, custom clients) is used as
+     * it is.
+     */
+    runClientOnce(timeoutMs) {
+        const meta = { retryAfter: null };
+        const run = this.apifyClient.run(this.actorRunId);
+        const http = run.httpClient;
+        if (http && typeof http === 'object' && typeof http.maxRetries === 'number') {
+            const once = Object.create(http);
+            once.maxRetries = 0;
+            if (timeoutMs !== undefined && typeof http.timeoutMillis === 'number') {
+                once.timeoutMillis = Math.max(1000, Math.min(http.timeoutMillis, timeoutMs));
+            }
+            const axios = http.axios;
+            if (axios && typeof axios.request === 'function') {
+                const request = axios.request.bind(axios);
+                once.axios = Object.assign(Object.create(axios), {
+                    request: async (cfg) => {
+                        const response = (await request(cfg));
+                        const h = response?.headers;
+                        const value = typeof h?.get === 'function' ? h.get('retry-after') : h?.['retry-after'];
+                        meta.retryAfter = value === undefined || value === null ? null : String(value);
+                        return response;
+                    },
+                });
+            }
+            run.httpClient = once;
+        }
+        return { run, meta };
+    }
+    /**
+     * Sleep before the next same-key attempt; false when none is left (attempts,
+     * the API retry window from `started`, or `deadline` would be passed).
+     * Never sooner than the server's Retry-After, and at least 1 s after any
+     * 429 (the server's minimum, also when the header is missing or malformed).
+     */
+    async backoffBeforeRetry(attempt, started, err, deadline, retryAfter = null) {
+        if (attempt + 1 >= this.postAttempts)
+            return false;
+        let delay = (0, api_storage_client_js_1.serverRetryDelayMs)(attempt, retryAfter);
+        if (httpStatusOf(err) === 429)
+            delay = Math.max(delay, 1000 + Math.random() * 1000);
+        const end = Math.min(started + (0, api_storage_client_js_1.apiRetryWindowMs)(), deadline ?? Number.POSITIVE_INFINITY);
+        if (Date.now() + delay >= end)
+            return false;
+        await new Promise((r) => setTimeout(r, delay));
+        return true;
+    }
+    /** Time left for one attempt before `deadline`, if any. */
+    static attemptTimeout(deadline) {
+        return deadline === undefined ? undefined : Math.max(0, deadline - Date.now());
+    }
     /**
      * POST a charge intent. Returns acceptedCount on success.
      * Throws ChargeRejectedError for definitive platform rejections (no retry).
-     * Returns null when the outcome is uncertain after retries.
+     * Returns null when the outcome is uncertain after retries (the intent stays
+     * pending); `deadline` (epoch ms) ends the retries earlier, e.g. at exit.
      */
-    async postChargeWithRetries(intent) {
+    async postChargeWithRetries(intent, deadline) {
         let lastError;
+        const started = Date.now();
         for (let attempt = 0; attempt < this.postAttempts; attempt++) {
+            const { run, meta } = this.runClientOnce(ChargingManager.attemptTimeout(deadline));
             try {
-                const response = await this.apifyClient.run(this.actorRunId).charge({
+                const response = await run.charge({
                     eventName: intent.eventName,
                     count: intent.count,
                     idempotencyKey: intent.idempotencyKey,
@@ -401,9 +584,8 @@ class ChargingManager {
                 if (rejected) {
                     throw rejected;
                 }
-                if (attempt + 1 < this.postAttempts) {
-                    await new Promise((r) => setTimeout(r, 250 * (attempt + 1)));
-                }
+                if (!(await this.backoffBeforeRetry(attempt, started, err, deadline, meta.retryAfter)))
+                    break;
             }
         }
         console.warn(`[Scrapely] Failed to record charge for event ${intent.eventName} ` +
@@ -432,11 +614,13 @@ class ChargingManager {
         this.pendingCharges.delete(operationId);
         return this._voidCharge(eventName, operationId, reservedCount, true);
     }
-    async _voidCharge(eventName, operationId, reservedCount, notFoundMeansVoided = false) {
+    async _voidCharge(eventName, operationId, reservedCount, notFoundMeansVoided = false, deadline) {
         if (!this.isAtHome || !this.actorRunId)
             return false;
-        const runClient = this.apifyClient.run(this.actorRunId);
+        const started = Date.now();
         for (let attempt = 0; attempt < this.postAttempts; attempt++) {
+            const { run, meta } = this.runClientOnce(ChargingManager.attemptTimeout(deadline));
+            const runClient = run;
             try {
                 await runClient.httpClient.call({
                     url: runClient._url('charge/void'),
@@ -458,10 +642,11 @@ class ChargingManager {
                 }
                 // 404: nothing to void; 409: already settled; 4xx auth: permanent.
                 // The charge stands; keep the local reservation that matches it.
-                if (status !== undefined && status >= 400 && status < 500)
+                // 408/429 (rate limited) and 5xx are not answers: re-queue below.
+                if (status !== undefined && status >= 400 && status < 500 && !isTransientStatus(status))
                     return false;
-                if (attempt + 1 < this.postAttempts)
-                    await new Promise((r) => setTimeout(r, 250 * (attempt + 1)));
+                if (!(await this.backoffBeforeRetry(attempt, started, err, deadline, meta.retryAfter)))
+                    break;
             }
         }
         // Outcome unknown: keep the reservation and retry before the next push / at exit.
@@ -473,13 +658,21 @@ class ChargingManager {
      * @internal Retry unconfirmed voids (caller holds the operation lock).
      * Returns false while any earlier void is still unconfirmed.
      */
-    async _retryPendingVoids() {
-        const pending = this.pendingVoids;
-        this.pendingVoids = [];
-        for (const v of pending) {
-            await this._voidCharge(v.eventName, v.operationId, v.reservedCount, v.notFoundMeansVoided);
+    async _retryPendingVoids(deadline) {
+        for (let round = 0;; round++) {
+            const pending = this.pendingVoids;
+            this.pendingVoids = [];
+            for (const v of pending) {
+                await this._voidCharge(v.eventName, v.operationId, v.reservedCount, v.notFoundMeansVoided, deadline);
+            }
+            if (this.pendingVoids.length === 0)
+                return true;
+            // With a deadline (exit flush): keep retrying with backoff while time is left.
+            const delay = (0, api_storage_client_js_1.serverRetryDelayMs)(round, null);
+            if (deadline === undefined || Date.now() + delay >= deadline)
+                return false;
+            await new Promise((r) => setTimeout(r, delay));
         }
-        return this.pendingVoids.length === 0;
     }
     /** Clamp local reservation down when the server accepted fewer events. */
     /** @internal Used by pushDataAndCharge dual-event reconciliation. */
@@ -514,11 +707,13 @@ class ChargingManager {
     async _flushPendingExclusive(maxIntents = 32, deadlineMs) {
         if (!this.actorRunId)
             return 0;
-        await this._retryPendingVoids();
+        const deadline = deadlineMs !== undefined ? Date.now() + Math.max(0, deadlineMs) : undefined;
+        // Voids first: an unconfirmed one means the user may be billed for items
+        // that were never stored. At exit (with a deadline) keep retrying them.
+        await this._retryPendingVoids(deadline);
         if (this.flushInFlight) {
             return this.flushInFlight;
         }
-        const deadline = deadlineMs !== undefined ? Date.now() + Math.max(0, deadlineMs) : undefined;
         this.flushInFlight = (async () => {
             let confirmedTotal = 0;
             // Drain loop: keep flushing batches until empty or deadline.
@@ -536,8 +731,10 @@ class ChargingManager {
                     if (deadline !== undefined && Date.now() >= deadline)
                         break;
                     try {
-                        const accepted = await this.postChargeWithRetries(intent);
+                        const accepted = await this.postChargeWithRetries(intent, deadline);
                         if (accepted !== null) {
+                            if (accepted < intent.count)
+                                this.markChargeLimitHit();
                             const eventPrice = this.getEventPricing(intent.eventName).price;
                             this.applyAcceptedCount(intent.eventName, intent.count, accepted, eventPrice);
                             this.pendingCharges.delete(intent.idempotencyKey);
@@ -546,6 +743,8 @@ class ChargingManager {
                     }
                     catch (err) {
                         if (err instanceof ChargeRejectedError) {
+                            if (isChargeLimitRejection(err))
+                                this.markChargeLimitHit();
                             const eventPrice = this.getEventPricing(intent.eventName).price;
                             this.applyAcceptedCount(intent.eventName, intent.count, 0, eventPrice);
                             this.pendingCharges.delete(intent.idempotencyKey);
@@ -613,6 +812,8 @@ class ChargingManager {
             }
             return 0;
         })();
+        if (chargedCount < count)
+            this.markChargeLimitHit();
         if (chargedCount === 0) {
             return {
                 eventChargeLimitReached: count > 0,
@@ -655,6 +856,8 @@ class ChargingManager {
             }
             catch (err) {
                 if (err instanceof ChargeRejectedError) {
+                    if (isChargeLimitRejection(err))
+                        this.markChargeLimitHit();
                     // Server definitively rejected: unwind local reservation.
                     this.pendingCharges.delete(intent.idempotencyKey);
                     chargedCount = this.applyAcceptedCount(normalizedEventName, chargedCount, 0, eventPricing.price);
@@ -670,6 +873,9 @@ class ChargingManager {
                 this.pendingCharges.delete(intent.idempotencyKey);
                 if (accepted > 0)
                     this.lastAcceptedOperationId = intent.idempotencyKey;
+                // The platform capped the charge at the run's limit.
+                if (accepted < chargedCount)
+                    this.markChargeLimitHit();
                 chargedCount = this.applyAcceptedCount(normalizedEventName, chargedCount, accepted, eventPricing.price);
             }
             else {
@@ -772,6 +978,22 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
         // A void that did not go through for an earlier failed push must land
         // before this push charges again, or one unsaved result is billed twice.
         const voidsConfirmed = (await chargingManager._retryPendingVoids?.()) !== false;
+        const requested = Array.isArray(items) ? items.length : 1;
+        // Charge-limit safety net: once the run's spending limit is used up, a
+        // default-dataset push that does not pay per row (no event, or only free
+        // events) stores nothing. Paid pushes go on below, where the budget cap
+        // stores only the rows that are actually charged.
+        if (isDefaultDataset &&
+            requested > 0 &&
+            chargingManager.isChargeLimitReached?.() &&
+            !chargingManager._isPaidPerItem(eventName, isDefaultDataset)) {
+            chargingManager._noteRefusedDefaultDatasetItems(requested);
+            return {
+                eventChargeLimitReached: true,
+                chargedCount: 0,
+                chargeableWithinLimit: chargingManager._chargeableWithinLimit?.() ?? {},
+            };
+        }
         const { limitedItems, eventsToCharge } = chargingManager.calculatePushDataLimits({
             items,
             eventName,
@@ -787,6 +1009,11 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
         }
         const itemsArray = Array.isArray(items) ? items : [items];
         const allItemsTrimmed = itemsArray.length > 0 && limitedItems.length === 0;
+        if (isDefaultDataset && limitedItems.length < itemsArray.length) {
+            // Rows beyond what the budget can pay for are dropped.
+            chargingManager._markChargeLimitHit?.();
+            chargingManager._noteRefusedDefaultDatasetItems?.(itemsArray.length - limitedItems.length);
+        }
         if (limitedItems.length === 0) {
             return {
                 eventChargeLimitReached: allItemsTrimmed,
@@ -875,6 +1102,9 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
             // Report rows charged, as Apify does for the pushed event: summing the
             // explicit and default-dataset events would count every row twice.
             chargeResult = { ...results.reduce(mergeChargeResults), chargedCount: acceptedForItems };
+            if (isDefaultDataset && acceptedForItems < limitedItems.length) {
+                chargingManager._noteRefusedDefaultDatasetItems?.(limitedItems.length - acceptedForItems);
+            }
             if (acceptedForItems <= 0) {
                 // Nothing will be stored (another event of this push was capped to
                 // zero): undo every platform charge this operation made, so no
@@ -898,7 +1128,7 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
             }
             const toPersist = limitedItems.slice(0, acceptedForItems);
             try {
-                await pushFn(Array.isArray(items) ? toPersist : toPersist[0]);
+                await chargeGatedPush.run(true, () => pushFn(Array.isArray(items) ? toPersist : toPersist[0]));
             }
             catch (pushErr) {
                 // Storage failed after the charge was accepted: a result is billed
@@ -934,7 +1164,7 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
             return chargeResult;
         }
         // No billable events — still persist the budget-limited items.
-        await pushFn(Array.isArray(items) ? limitedItems : limitedItems[0]);
+        await chargeGatedPush.run(true, () => pushFn(Array.isArray(items) ? limitedItems : limitedItems[0]));
         return {
             eventChargeLimitReached: false,
             chargedCount: 0,

package/dist/charging_client.js

@@ -17,7 +17,11 @@ function createChargingHooks(actor) {
         // Called by the client at `dataset()` time, so pricing is consulted
         // lazily — same timing as the former subclass override.
         shouldChargeDatasetPushes() {
-            return (0, charging_js_1.hasDefaultDatasetItemEventInPricing)(actor.getChargingManager().getPricingInfo().perEventPrices);
+            const chargingManager = actor.getChargingManager();
+            // Past the spending limit, route default-dataset pushes through
+            // pushDataAndCharge too, which refuses rows nobody pays for.
+            return ((0, charging_js_1.hasDefaultDatasetItemEventInPricing)(chargingManager.getPricingInfo().perEventPrices) ||
+                (chargingManager.isChargeLimitReached?.() ?? false));
         },
         async pushAndCharge({ items, eventName, pushFn }) {
             return (0, charging_js_1.pushDataAndCharge)({

package/dist/storage/api-storage-client.d.ts

@@ -13,18 +13,64 @@ export declare class ApiStorageClientError extends Error {
     constructor(message: string, statusCode: number);
 }
 /**
- * Whether a non-2xx answer is retried. Idempotent methods: 500/502/503/504.
- * Others (dataset push, KV PUT) only on a 503 that certainly was not
- * processed: the rate limiter refused it ("rate-limit-unavailable", before the
- * handler), or HAProxy had no backend (its own page, nothing forwarded). Any
- * other 503 may mean "outcome unknown" (e.g. a direct dataset write whose
- * commit was not confirmed), and re-sending could store the items twice.
+ * Whether a non-2xx answer is retried. 429 always: the run API limiter and
+ * dataset backpressure refuse it before anything is written. Idempotent
+ * methods: also 500/502/503/504. Others (dataset push, KV PUT) only on a 503
+ * that certainly was not processed: the rate limiter refused it
+ * ("rate-limit-unavailable", before the handler), or HAProxy had no backend
+ * (its own page, nothing forwarded). Any other 503 may mean "outcome unknown"
+ * (e.g. a direct dataset write whose commit was not confirmed), and re-sending
+ * could store the items twice.
  */
 export declare function isRetryableResponse(status: number, idempotent: boolean, errorType?: string, bodyText?: string): boolean;
-/** Exponential backoff with jitter; honours a Retry-After (seconds or HTTP date) of up to 30 s. */
-export declare function serverRetryDelayMs(serverAttempt: number, retryAfter: string | null, random?: () => number, now?: number): number;
+/** Default time budget for retrying one API call (429, 503, network errors). */
+export declare const DEFAULT_API_RETRY_WINDOW_MS = 120000;
+/**
+ * How long one API call keeps retrying before it gives up:
+ * SCRAPELY_API_RETRY_WINDOW_SECS when set, else {@link BaseApiClient.retryWindowMs}
+ * (2 minutes by default).
+ */
+export declare function apiRetryWindowMs(): number;
+/**
+ * Delay before retry number `attempt + 1`: exponential backoff (0.5 s doubling,
+ * capped at 30 s) with equal jitter, and never earlier than the server's
+ * Retry-After (seconds or HTTP date) plus a little jitter so
+ * throttled clients don't all come back in the same instant.
+ */
+export declare function serverRetryDelayMs(attempt: number, retryAfter: string | null, random?: () => number, now?: number): number;
 /** fetch() failed before the request reached the server. */
 export declare function isConnectFailure(err: unknown): boolean;
+export interface RetryingFetchOptions {
+    /** Safe to re-send when the outcome is unknown (5xx, timeout, reset). */
+    idempotent: boolean;
+    /** Per-attempt timeout. */
+    timeoutMs: number;
+    /** Retry timeouts (idempotent calls only; a timed-out write may have been applied). */
+    retryTimeouts?: boolean;
+    /** Total time budget for retries; defaults to {@link apiRetryWindowMs}. */
+    retryWindowMs?: number;
+    /** Short label for log lines, e.g. "POST /datasets/x/items". */
+    label: string;
+}
+export interface RetryingFetchInit {
+    method: string;
+    headers: Record<string, string>;
+    body?: BodyInit;
+}
+/**
+ * fetch() with the API retry policy. Retries, with exponential backoff and
+ * jitter that honours Retry-After, until the retry window (2 minutes by
+ * default) runs out:
+ * - 429 (run API rate limit, dataset backpressure): always; nothing was applied.
+ * - 503 from the limiter or HAProxy: always; nothing was forwarded.
+ * - 500/502/503/504, timeouts and dropped connections: idempotent calls only.
+ * - Refused/unresolvable connections: always; the request never left.
+ * A write (dataset push, KV PUT, metamorph) whose request may have reached the
+ * API is never re-sent: there is no idempotency key for it, so a re-send could
+ * store the same rows twice. Such failures surface to the caller instead.
+ * Non-2xx answers that are not retried throw ApiStorageClientError.
+ */
+export declare function fetchWithRetries<R>(url: string, init: RetryingFetchInit, options: RetryingFetchOptions, decode: (response: Response) => Promise<R>): Promise<R>;
 /**
  * Options for API requests (passed through from RecordOptions)
  */
@@ -44,16 +90,12 @@ interface ApiRequestOptions {
 export declare abstract class BaseApiClient {
     protected config: ApiStorageConfig;
     protected headers: Record<string, string>;
-    private static rateLimiter;
-    private static maxRetries;
-    static maxServerRetries: number;
+    static retryWindowMs: number;
     private static defaultTimeoutMs;
     constructor(config: ApiStorageConfig);
     /**
      * Sends the request and returns the successful (2xx) response, with the
-     * retry policy: 429 (rate limiter), timeouts, and the API being briefly
-     * unavailable (503, 5xx for idempotent methods, connection failures).
-     * Non-2xx answers that are not retried throw ApiStorageClientError.
+     * retry policy of {@link fetchWithRetries}.
      */
     private send;
     protected request<T>(method: string, path: string, body?: any, contentType?: string, options?: ApiRequestOptions): Promise<T>;

package/dist/storage/api-storage-client.js

@@ -1,10 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ApiStorageClientFactory = exports.ApiRequestQueueClient = exports.ApiKeyValueStoreClient = exports.ApiDatasetClient = exports.BaseApiClient = exports.ApiStorageClientError = void 0;
+exports.ApiStorageClientFactory = exports.ApiRequestQueueClient = exports.ApiKeyValueStoreClient = exports.ApiDatasetClient = exports.BaseApiClient = exports.DEFAULT_API_RETRY_WINDOW_MS = exports.ApiStorageClientError = void 0;
 exports.isRetryableResponse = isRetryableResponse;
+exports.apiRetryWindowMs = apiRetryWindowMs;
 exports.serverRetryDelayMs = serverRetryDelayMs;
 exports.isConnectFailure = isConnectFailure;
+exports.fetchWithRetries = fetchWithRetries;
 const node_crypto_1 = require("node:crypto");
+const charging_js_1 = require("../charging.js");
 const kvs_iteration_js_1 = require("./kvs-iteration.js");
 function buildRqQueryParams(baseClientKey, options) {
     const params = new URLSearchParams();
@@ -93,32 +96,6 @@ class AdaptiveRateLimiter {
         }
         this.lastRequestTime = Date.now();
     }
-    /**
-     * Handle a 429 rate limit response.
-     * Parses Retry-After header and waits the appropriate time.
-     */
-    async handleRateLimited(response) {
-        const retryAfter = response.headers.get('Retry-After');
-        let waitMs = 1000; // Default 1 second
-        if (retryAfter) {
-            // Retry-After can be seconds or a date
-            const seconds = parseInt(retryAfter, 10);
-            if (!isNaN(seconds)) {
-                waitMs = seconds * 1000;
-            }
-            else {
-                // It's a date string
-                const retryDate = new Date(retryAfter).getTime();
-                if (!isNaN(retryDate)) {
-                    waitMs = Math.max(0, retryDate - Date.now());
-                }
-            }
-        }
-        // Add jitter to prevent thundering herd
-        waitMs = waitMs + Math.random() * 100;
-        console.warn(`[RateLimiter] Rate limited, waiting ${Math.round(waitMs)}ms before retry`);
-        await new Promise(resolve => setTimeout(resolve, waitMs));
-    }
 }
 /**
  * Unwrap Apify-style v2 response envelopes ({ data: ... }).
@@ -165,32 +142,62 @@ const IDEMPOTENT_METHODS = new Set(['GET', 'HEAD', 'DELETE', 'OPTIONS']);
 const NO_BACKEND_MARKER = 'No server is available to handle this request';
 const CONNECT_FAILURE_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'EHOSTUNREACH', 'ENETUNREACH', 'UND_ERR_CONNECT_TIMEOUT']);
 /**
- * Whether a non-2xx answer is retried. Idempotent methods: 500/502/503/504.
- * Others (dataset push, KV PUT) only on a 503 that certainly was not
- * processed: the rate limiter refused it ("rate-limit-unavailable", before the
- * handler), or HAProxy had no backend (its own page, nothing forwarded). Any
- * other 503 may mean "outcome unknown" (e.g. a direct dataset write whose
- * commit was not confirmed), and re-sending could store the items twice.
+ * Whether a non-2xx answer is retried. 429 always: the run API limiter and
+ * dataset backpressure refuse it before anything is written. Idempotent
+ * methods: also 500/502/503/504. Others (dataset push, KV PUT) only on a 503
+ * that certainly was not processed: the rate limiter refused it
+ * ("rate-limit-unavailable", before the handler), or HAProxy had no backend
+ * (its own page, nothing forwarded). Any other 503 may mean "outcome unknown"
+ * (e.g. a direct dataset write whose commit was not confirmed), and re-sending
+ * could store the items twice.
  */
 function isRetryableResponse(status, idempotent, errorType, bodyText) {
+    if (status === 429)
+        return true;
     if (idempotent)
         return status === 500 || status === 502 || status === 503 || status === 504;
     if (status !== 503)
         return false;
     return errorType === 'rate-limit-unavailable' || (bodyText ?? '').includes(NO_BACKEND_MARKER);
 }
-/** Exponential backoff with jitter; honours a Retry-After (seconds or HTTP date) of up to 30 s. */
-function serverRetryDelayMs(serverAttempt, retryAfter, random = Math.random, now = Date.now()) {
-    if (retryAfter !== null && retryAfter.trim() !== '') {
-        let secs = Number(retryAfter);
-        if (!Number.isFinite(secs)) {
-            const at = Date.parse(retryAfter);
-            secs = Number.isFinite(at) ? Math.max(0, (at - now) / 1000) : NaN;
-        }
-        if (Number.isFinite(secs) && secs >= 0)
-            return Math.min(secs, 30) * 1000 + random() * 100;
-    }
-    return Math.min(8000, 500 * 2 ** serverAttempt) + random() * 250;
+/** Default time budget for retrying one API call (429, 503, network errors). */
+exports.DEFAULT_API_RETRY_WINDOW_MS = 120_000;
+const MAX_BACKOFF_MS = 30_000;
+/**
+ * How long one API call keeps retrying before it gives up:
+ * SCRAPELY_API_RETRY_WINDOW_SECS when set, else {@link BaseApiClient.retryWindowMs}
+ * (2 minutes by default).
+ */
+function apiRetryWindowMs() {
+    const env = Number(process.env.SCRAPELY_API_RETRY_WINDOW_SECS);
+    if (process.env.SCRAPELY_API_RETRY_WINDOW_SECS && Number.isFinite(env) && env >= 0)
+        return env * 1000;
+    return BaseApiClient.retryWindowMs;
+}
+function parseRetryAfterMs(retryAfter, now) {
+    if (retryAfter === null || retryAfter === undefined || retryAfter.trim() === '')
+        return undefined;
+    let secs = Number(retryAfter);
+    if (!Number.isFinite(secs)) {
+        const at = Date.parse(retryAfter);
+        secs = Number.isFinite(at) ? Math.max(0, (at - now) / 1000) : NaN;
+    }
+    // Not capped: a Retry-After past the retry window ends the retries instead.
+    return Number.isFinite(secs) && secs >= 0 ? secs * 1000 : undefined;
+}
+/**
+ * Delay before retry number `attempt + 1`: exponential backoff (0.5 s doubling,
+ * capped at 30 s) with equal jitter, and never earlier than the server's
+ * Retry-After (seconds or HTTP date) plus a little jitter so
+ * throttled clients don't all come back in the same instant.
+ */
+function serverRetryDelayMs(attempt, retryAfter, random = Math.random, now = Date.now()) {
+    const exp = Math.min(MAX_BACKOFF_MS, 500 * 2 ** Math.min(attempt, 16));
+    const backoff = exp / 2 + random() * (exp / 2);
+    const ra = parseRetryAfterMs(retryAfter, now);
+    if (ra === undefined)
+        return backoff;
+    return Math.max(ra + random() * Math.min(1000, ra * 0.2 + 100), backoff);
 }
 function fetchErrorCode(err) {
     const e = err;
@@ -205,6 +212,109 @@ function describeFetchError(err) {
     return fetchErrorCode(err) ?? err?.message ?? String(err);
 }
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+// Shared by every API call of this process (storage clients, status message,
+// metamorph, reboot), so throttling seen on one slows the others too.
+const sharedRateLimiter = new AdaptiveRateLimiter();
+/**
+ * fetch() with the API retry policy. Retries, with exponential backoff and
+ * jitter that honours Retry-After, until the retry window (2 minutes by
+ * default) runs out:
+ * - 429 (run API rate limit, dataset backpressure): always; nothing was applied.
+ * - 503 from the limiter or HAProxy: always; nothing was forwarded.
+ * - 500/502/503/504, timeouts and dropped connections: idempotent calls only.
+ * - Refused/unresolvable connections: always; the request never left.
+ * A write (dataset push, KV PUT, metamorph) whose request may have reached the
+ * API is never re-sent: there is no idempotency key for it, so a re-send could
+ * store the same rows twice. Such failures surface to the caller instead.
+ * Non-2xx answers that are not retried throw ApiStorageClientError.
+ */
+async function fetchWithRetries(url, init, options, decode) {
+    const { idempotent, timeoutMs, label } = options;
+    const retryTimeouts = options.retryTimeouts !== false && idempotent;
+    const deadline = Date.now() + Math.max(0, options.retryWindowMs ?? apiRetryWindowMs());
+    let attempt = 0;
+    /** Sleeps before the next attempt; false once the retry window is used up. */
+    const backoff = async (why, retryAfter) => {
+        // Retrying sooner than asked would only be refused again: give up when
+        // the next try would land past the window.
+        const waitMs = serverRetryDelayMs(attempt, retryAfter);
+        if (Date.now() + waitMs > deadline)
+            return false;
+        attempt++;
+        console.warn(`[Scrapely] ${label} ${why}; retry ${attempt} in ${Math.round(waitMs)}ms`);
+        await sleep(waitMs);
+        return true;
+    };
+    for (;;) {
+        await sharedRateLimiter.waitForSlot();
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+        try {
+            const response = await fetch(url, { method: init.method, headers: init.headers, body: init.body, signal: controller.signal });
+            sharedRateLimiter.updateFromHeaders(response.headers);
+            if (response.ok) {
+                try {
+                    // Decoded inside the retry loop: a body cut off after
+                    // the headers is a transport failure like any other.
+                    return await decode(response);
+                }
+                catch (bodyErr) {
+                    // The request was applied (2xx); a write whose answer
+                    // timed out while being read must not be sent again.
+                    if (bodyErr.name === 'AbortError') {
+                        if (!idempotent || !retryTimeouts) {
+                            const timeoutError = new Error(`Request timed out after ${timeoutMs}ms (response body${idempotent ? '' : '; the request was applied'})`);
+                            timeoutError.isTimeout = true;
+                            throw timeoutError;
+                        }
+                        if (await backoff(`response body timed out after ${timeoutMs}ms`, null))
+                            continue;
+                        throw bodyErr;
+                    }
+                    if (idempotent && await backoff(`response body failed (${describeFetchError(bodyErr)})`, null))
+                        continue;
+                    throw bodyErr;
+                }
+            }
+            const text = await response.text().catch(() => '');
+            let errorBody;
+            try {
+                errorBody = JSON.parse(text);
+            }
+            catch {
+                errorBody = undefined;
+            }
+            if (isRetryableResponse(response.status, idempotent, errorBody?.error?.type, text)
+                && await backoff(`answered ${response.status}`, response.headers.get('retry-after'))) {
+                continue;
+            }
+            throw new ApiStorageClientError(errorBody?.error?.message || response.statusText || `API request failed: ${response.status}`, response.status);
+        }
+        catch (err) {
+            // HTTP errors (already decided above) are final.
+            if (isApiStorageClientError(err))
+                throw err;
+            if (err?.isTimeout)
+                throw err;
+            if (err.name === 'AbortError') {
+                const timeoutError = new Error(`Request timed out after ${timeoutMs}ms`);
+                timeoutError.isTimeout = true;
+                if (retryTimeouts && await backoff(`timed out after ${timeoutMs}ms`, null))
+                    continue;
+                throw timeoutError;
+            }
+            // fetch() failed without a response. A refused/unresolvable
+            // connection never reached the API, so it is safe for every
+            // method; anything else (reset mid-request) only for idempotent ones.
+            if ((idempotent || isConnectFailure(err)) && await backoff(`failed (${describeFetchError(err)})`, null))
+                continue;
+            throw err;
+        }
+        finally {
+            clearTimeout(timeoutId);
+        }
+    }
+}
 /** Map GET storage errors: rethrow 403, return undefined for 404 (triggers getOrCreate for names only). */
 function handleStorageGetError(err) {
     if (isApiStorageClientError(err)) {
@@ -265,15 +375,10 @@ function makeAsyncIterableResult(promiseFn) {
 class BaseApiClient {
     config;
     headers;
-    // Shared rate limiter instance - coordinates across all API clients
-    // This is static so it's shared across all storage clients for this run
-    static rateLimiter = new AdaptiveRateLimiter();
-    // Maximum retries for rate-limited requests
-    static maxRetries = 3;
-    // Retries while the API is briefly unavailable (503/5xx, connection
-    // refused): backoff 0.5 s doubling to 8 s, about 45 s in total, which
-    // covers a main redeploy.
-    static maxServerRetries = 8;
+    // How long one call keeps retrying 429/503/network errors before it gives
+    // up (SCRAPELY_API_RETRY_WINDOW_SECS overrides it). Long enough that a
+    // throttled run slows down instead of failing, and covers a main redeploy.
+    static retryWindowMs = exports.DEFAULT_API_RETRY_WINDOW_MS;
     // Default timeout for requests (30 seconds)
     static defaultTimeoutMs = 30000;
     constructor(config) {
@@ -286,12 +391,9 @@ class BaseApiClient {
     }
     /**
      * Sends the request and returns the successful (2xx) response, with the
-     * retry policy: 429 (rate limiter), timeouts, and the API being briefly
-     * unavailable (503, 5xx for idempotent methods, connection failures).
-     * Non-2xx answers that are not retried throw ApiStorageClientError.
+     * retry policy of {@link fetchWithRetries}.
      */
     async send(method, path, body, contentType, options, decode) {
-        const url = `${this.config.apiUrl}/v2${path}`;
         const headers = { ...this.headers };
         if (contentType) {
             headers['Content-Type'] = contentType;
@@ -299,114 +401,19 @@ class BaseApiClient {
         const timeoutMs = options?.timeoutSecs
             ? options.timeoutSecs * 1000
             : BaseApiClient.defaultTimeoutMs;
-        const shouldRetryTimeouts = options?.doNotRetryTimeouts !== true;
-        const idempotent = options?.idempotent ?? IDEMPOTENT_METHODS.has(method.toUpperCase());
-        let attempt = 0; // timeouts and 429s
-        let serverAttempt = 0; // 503/5xx and connection failures (API briefly unavailable)
-        for (;;) {
-            // Wait for rate limiter slot before making request
-            await BaseApiClient.rateLimiter.waitForSlot();
-            // Create AbortController for timeout
-            const controller = new AbortController();
-            const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
-            try {
-                const response = await fetch(url, {
-                    method,
-                    headers,
-                    // Binary bodies (KV records) go out as the raw bytes, never JSON/base64.
-                    body: body === undefined ? undefined
-                        : typeof body === 'string' || body instanceof Uint8Array ? body
-                            : JSON.stringify(body),
-                    signal: controller.signal,
-                });
-                // Update rate limiter state from response headers
-                BaseApiClient.rateLimiter.updateFromHeaders(response.headers);
-                // Handle rate limiting (429)
-                if (response.status === 429) {
-                    if (attempt < BaseApiClient.maxRetries) {
-                        attempt++;
-                        await BaseApiClient.rateLimiter.handleRateLimited(response);
-                        continue; // Retry
-                    }
-                    const error = await response.json().catch(() => ({ error: { message: 'Rate limit exceeded' } }));
-                    throw new ApiStorageClientError(error.error?.message || 'Rate limit exceeded', 429);
-                }
-                if (response.ok) {
-                    try {
-                        // Decoded inside the retry loop: a body cut off after
-                        // the headers is a transport failure like any other.
-                        return await decode(response);
-                    }
-                    catch (bodyErr) {
-                        // The request was applied (2xx); a write whose answer
-                        // timed out while being read must not be sent again.
-                        if (bodyErr.name === 'AbortError' && !idempotent) {
-                            const timeoutError = new Error(`Request timed out after ${timeoutMs}ms (response body; the request was applied)`);
-                            timeoutError.isTimeout = true;
-                            throw timeoutError;
-                        }
-                        if (idempotent && bodyErr.name !== 'AbortError' && serverAttempt < BaseApiClient.maxServerRetries) {
-                            const waitMs = serverRetryDelayMs(serverAttempt, null);
-                            serverAttempt++;
-                            console.warn(`[Scrapely] ${method} ${path} response body failed (${describeFetchError(bodyErr)}); retry ${serverAttempt}/${BaseApiClient.maxServerRetries} in ${Math.round(waitMs)}ms`);
-                            await sleep(waitMs);
-                            continue;
-                        }
-                        throw bodyErr;
-                    }
-                }
-                // API briefly unavailable (main redeploy, limiter store blip).
-                const text = await response.text().catch(() => '');
-                let errorBody;
-                try {
-                    errorBody = JSON.parse(text);
-                }
-                catch {
-                    errorBody = undefined;
-                }
-                if (isRetryableResponse(response.status, idempotent, errorBody?.error?.type, text)
-                    && serverAttempt < BaseApiClient.maxServerRetries) {
-                    const waitMs = serverRetryDelayMs(serverAttempt, response.headers.get('retry-after'));
-                    serverAttempt++;
-                    console.warn(`[Scrapely] ${method} ${path} answered ${response.status}; retry ${serverAttempt}/${BaseApiClient.maxServerRetries} in ${Math.round(waitMs)}ms`);
-                    await sleep(waitMs);
-                    continue;
-                }
-                throw new ApiStorageClientError(errorBody?.error?.message || response.statusText || `API request failed: ${response.status}`, response.status);
-            }
-            catch (err) {
-                // Handle timeout/abort errors
-                if (err.name === 'AbortError') {
-                    const timeoutError = new Error(`Request timed out after ${timeoutMs}ms`);
-                    timeoutError.isTimeout = true;
-                    // Don't retry if doNotRetryTimeouts is true
-                    if (!shouldRetryTimeouts || attempt >= BaseApiClient.maxRetries) {
-                        throw timeoutError;
-                    }
-                    attempt++;
-                    // Wait a bit before retrying
-                    await sleep(100 * attempt);
-                    continue;
-                }
-                // HTTP errors (already decided above) are final.
-                if (isApiStorageClientError(err))
-                    throw err;
-                // fetch() failed without a response. A refused/unresolvable
-                // connection never reached the API, so it is safe for every
-                // method; anything else (reset mid-request) only for idempotent ones.
-                if ((idempotent || isConnectFailure(err)) && serverAttempt < BaseApiClient.maxServerRetries) {
-                    const waitMs = serverRetryDelayMs(serverAttempt, null);
-                    serverAttempt++;
-                    console.warn(`[Scrapely] ${method} ${path} failed (${describeFetchError(err)}); retry ${serverAttempt}/${BaseApiClient.maxServerRetries} in ${Math.round(waitMs)}ms`);
-                    await sleep(waitMs);
-                    continue;
-                }
-                throw err;
-            }
-            finally {
-                clearTimeout(timeoutId);
-            }
-        }
+        return fetchWithRetries(`${this.config.apiUrl}/v2${path}`, {
+            method,
+            headers,
+            // Binary bodies (KV records) go out as the raw bytes, never JSON/base64.
+            body: body === undefined ? undefined
+                : typeof body === 'string' || body instanceof Uint8Array ? body
+                    : JSON.stringify(body),
+        }, {
+            idempotent: options?.idempotent ?? IDEMPOTENT_METHODS.has(method.toUpperCase()),
+            timeoutMs,
+            retryTimeouts: options?.doNotRetryTimeouts !== true,
+            label: `${method} ${path}`,
+        }, decode);
     }
     async request(method, path, body, contentType, options) {
         return this.send(method, path, body, contentType, options, async (response) => {
@@ -559,6 +566,10 @@ class ApiDatasetClient extends BaseApiClient {
         else {
             parsed = [items];
         }
+        // Charge-limit safety net: a pay-per-event run past its spending limit
+        // must not store unbilled rows in its default dataset.
+        if (this.id === (0, charging_js_1.defaultDatasetIdFromEnv)() && (0, charging_js_1.refuseDefaultDatasetPush)(parsed.length))
+            return;
         await this.request('POST', `/datasets/${this.id}/items`, parsed);
     }
 }
@@ -723,7 +734,10 @@ class ApiKeyValueStoreClient extends BaseApiClient {
      * @returns A public URL for accessing the record
      */
     getPublicUrl(key) {
-        return `${this.config.apiUrl}/v2/key-value-stores/${this.id}/records/${key}`;
+        // apiUrl is the container-internal API address; links handed to users
+        // must use the public base URL the platform injects.
+        const publicBase = (process.env.APIFY_API_PUBLIC_BASE_URL || process.env.ACTOR_API_PUBLIC_BASE_URL || this.config.apiUrl).replace(/\/+$/, '');
+        return `${publicBase}/v2/key-value-stores/${this.id}/records/${key}`;
     }
 }
 exports.ApiKeyValueStoreClient = ApiKeyValueStoreClient;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.15",
+    "version": "1.2.17",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -22,6 +22,7 @@
     "scripts": {
         "test": "npx tsx tests/test-mvp.ts",
         "test:charging": "npx tsx --test tests/charging-pending.test.ts",
+        "test:charge-limit-guard": "npx tsx --test tests/charge-limit-guard.test.ts",
         "test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",
         "test:stdio-drain": "npx tsx --test tests/stdio-drain.test.ts",
         "test:charging-client": "npx tsx --test tests/charging-client.test.ts",