scrapely 1.2.9 → 1.2.11

package/dist/charging.d.ts

@@ -122,6 +122,8 @@ export declare class ChargingManager {
     private getEventPricing;
     private normalizeChargingEventName;
     private calculateMaxChargesByPrice;
+    /** @internal Remaining allowance per event (pushDataAndCharge's uncertain path). */
+    _chargeableWithinLimit(): Record<string, number>;
     private getChargeableWithinLimit;
     private quarantineStalePending;
     /**
@@ -137,7 +139,14 @@ export declare class ChargingManager {
      * (POST /v2/actor-runs/:runId/charge/void), then unwind the local
      * reservation. Returns true when the platform voided it.
      */
-    _voidCharge(eventName: string, operationId: string, reservedCount: number): Promise<boolean>;
+    /**
+     * @internal Cancel a charge whose outcome was never confirmed, for items that
+     * were then not stored: drop the pending intent (so the exit flush never
+     * posts it) and void it in case the original request did apply. A 404 here
+     * means it never applied.
+     */
+    _cancelUncertainCharge(eventName: string, operationId: string, reservedCount: number): Promise<boolean>;
+    _voidCharge(eventName: string, operationId: string, reservedCount: number, notFoundMeansVoided?: boolean): Promise<boolean>;
     /**
      * @internal Retry unconfirmed voids (caller holds the operation lock).
      * Returns false while any earlier void is still unconfirmed.

package/dist/charging.js

@@ -345,6 +345,10 @@ class ChargingManager {
         const roundedResult = Math.floor(Number(unroundedResult.toFixed(4)));
         return Math.max(0, roundedResult);
     }
+    /** @internal Remaining allowance per event (pushDataAndCharge's uncertain path). */
+    _chargeableWithinLimit() {
+        return this.chargingState === undefined ? {} : this.getChargeableWithinLimit();
+    }
     getChargeableWithinLimit() {
         if (Object.keys(this.pricingInfo).length === 0) {
             return {};
@@ -414,7 +418,17 @@ class ChargingManager {
      * (POST /v2/actor-runs/:runId/charge/void), then unwind the local
      * reservation. Returns true when the platform voided it.
      */
-    async _voidCharge(eventName, operationId, reservedCount) {
+    /**
+     * @internal Cancel a charge whose outcome was never confirmed, for items that
+     * were then not stored: drop the pending intent (so the exit flush never
+     * posts it) and void it in case the original request did apply. A 404 here
+     * means it never applied.
+     */
+    async _cancelUncertainCharge(eventName, operationId, reservedCount) {
+        this.pendingCharges.delete(operationId);
+        return this._voidCharge(eventName, operationId, reservedCount, true);
+    }
+    async _voidCharge(eventName, operationId, reservedCount, notFoundMeansVoided = false) {
         if (!this.isAtHome || !this.actorRunId)
             return false;
         const runClient = this.apifyClient.run(this.actorRunId);
@@ -431,6 +445,13 @@ class ChargingManager {
             }
             catch (err) {
                 const status = httpStatusOf(err);
+                // Only the void route's own answer proves it never applied; a server
+                // without the route (older deployment) also answers 404.
+                if (status === 404 && notFoundMeansVoided && apiErrorTypeOf(err) === 'charge-not-found') {
+                    // The unconfirmed charge never reached the ledger: nothing billed.
+                    this.applyAcceptedCount(eventName, reservedCount, 0);
+                    return true;
+                }
                 // 404: nothing to void; 409: already settled; 4xx auth: permanent.
                 // The charge stands; keep the local reservation that matches it.
                 if (status !== undefined && status >= 400 && status < 500)
@@ -440,7 +461,7 @@ class ChargingManager {
             }
         }
         // Outcome unknown: keep the reservation and retry before the next push / at exit.
-        this.pendingVoids.push({ eventName, operationId, reservedCount });
+        this.pendingVoids.push({ eventName, operationId, reservedCount, notFoundMeansVoided });
         console.error(`[Scrapely] Void of charge ${operationId} ('${eventName}') not confirmed yet; will retry`);
         return false;
     }
@@ -452,7 +473,7 @@ class ChargingManager {
         const pending = this.pendingVoids;
         this.pendingVoids = [];
         for (const v of pending) {
-            await this._voidCharge(v.eventName, v.operationId, v.reservedCount);
+            await this._voidCharge(v.eventName, v.operationId, v.reservedCount, v.notFoundMeansVoided);
         }
         return this.pendingVoids.length === 0;
     }
@@ -784,10 +805,30 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
             try {
                 for (const [name, chargeCount] of ordered) {
                     chargingManager._takeLastAcceptedOperationId?.();
-                    const r = await chargingManager._chargeExclusive({
-                        eventName: name,
-                        count: chargeCount,
-                    });
+                    let r;
+                    try {
+                        r = await chargingManager._chargeExclusive({
+                            eventName: name,
+                            count: chargeCount,
+                        });
+                    }
+                    catch (chargeErr) {
+                        if (!(chargeErr instanceof ChargeUncertainError))
+                            throw chargeErr;
+                        // The platform did not confirm the charge (5xx / timeouts after
+                        // retries). Never drop the items for that: the intent stays
+                        // pending with its idempotency key and is re-posted by the exit
+                        // flush, so the items are stored now and billed once.
+                        console.warn(`[Scrapely] Charge for '${name}' not confirmed yet (${chargeErr.operationId}); storing the items and retrying the charge later`);
+                        r = {
+                            eventChargeLimitReached: false,
+                            chargedCount: chargeErr.count,
+                            chargeableWithinLimit: chargingManager._chargeableWithinLimit?.() ?? {},
+                        };
+                        results.push(r);
+                        reservedInOp.push({ name, reserved: chargeErr.count, operationId: chargeErr.operationId, uncertain: true });
+                        continue;
+                    }
                     results.push(r);
                     reservedInOp.push({ name, reserved: r.chargedCount, operationId: chargingManager._takeLastAcceptedOperationId?.() });
                 }
@@ -814,7 +855,12 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
                 const [name] = ordered[i];
                 const reserved = results[i].chargedCount;
                 if (reserved > acceptedForItems) {
-                    chargingManager.applyAcceptedCount(name, reserved, acceptedForItems);
+                    // Nothing stored and a platform charge was made: leave its local
+                    // count to the void below, which releases it only once the
+                    // platform confirms (never on a server without the void route).
+                    const platformOp = acceptedForItems <= 0 && reservedInOp[i]?.operationId;
+                    if (!platformOp)
+                        chargingManager.applyAcceptedCount(name, reserved, acceptedForItems);
                     results[i] = {
                         ...results[i],
                         chargedCount: acceptedForItems,
@@ -824,6 +870,20 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
             }
             chargeResult = results.reduce(mergeChargeResults);
             if (acceptedForItems <= 0) {
+                // Nothing will be stored (another event of this push was capped to
+                // zero): undo every platform charge this operation made, so no
+                // result is billed for items that are never saved.
+                for (let i = 0; i < reservedInOp.length; i++) {
+                    const { name, operationId, uncertain, reserved } = reservedInOp[i];
+                    if (reserved <= 0 || !operationId)
+                        continue;
+                    // Original count: the local reservation is released only when
+                    // the platform confirms the void (or that it never applied).
+                    if (uncertain)
+                        await chargingManager._cancelUncertainCharge?.(name, operationId, reserved);
+                    else
+                        await chargingManager._voidCharge(name, operationId, reserved);
+                }
                 return {
                     eventChargeLimitReached: true,
                     chargedCount: chargeResult.chargedCount,
@@ -848,11 +908,15 @@ async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultD
                 if (!pushCertainlyFailed(pushErr))
                     throw pushErr;
                 for (let i = 0; i < reservedInOp.length; i++) {
-                    const { name, operationId } = reservedInOp[i];
+                    const { name, operationId, uncertain } = reservedInOp[i];
                     const reserved = results[i].chargedCount;
                     if (reserved <= 0)
                         continue;
-                    if (operationId) {
+                    if (operationId && uncertain) {
+                        // Never re-post a charge for items that were not stored.
+                        await chargingManager._cancelUncertainCharge?.(name, operationId, reserved);
+                    }
+                    else if (operationId) {
                         await chargingManager._voidCharge(name, operationId, reserved);
                     }
                     else if (isSyntheticEvent(name)) {
@@ -895,3 +959,9 @@ function pushCertainlyFailed(err) {
     }
     return false;
 }
+/** API error type (e.g. 'charge-not-found') of a client error, if the server sent one. */
+function apiErrorTypeOf(err) {
+    const e = err;
+    const t = e?.type ?? e?.data?.error?.type ?? e?.response?.data?.error?.type;
+    return typeof t === 'string' ? t : undefined;
+}

package/dist/storage/api-storage-client.d.ts

@@ -12,6 +12,19 @@ export declare class ApiStorageClientError extends Error {
     readonly statusCode: number;
     constructor(message: string, statusCode: number);
 }
+/**
+ * Whether a non-2xx answer is retried. Idempotent methods: 500/502/503/504.
+ * Others (dataset push, KV PUT) only on a 503 that certainly was not
+ * processed: the rate limiter refused it ("rate-limit-unavailable", before the
+ * handler), or HAProxy had no backend (its own page, nothing forwarded). Any
+ * other 503 may mean "outcome unknown" (e.g. a direct dataset write whose
+ * commit was not confirmed), and re-sending could store the items twice.
+ */
+export declare function isRetryableResponse(status: number, idempotent: boolean, errorType?: string, bodyText?: string): boolean;
+/** Exponential backoff with jitter; honours a Retry-After (seconds or HTTP date) of up to 30 s. */
+export declare function serverRetryDelayMs(serverAttempt: number, retryAfter: string | null, random?: () => number, now?: number): number;
+/** fetch() failed before the request reached the server. */
+export declare function isConnectFailure(err: unknown): boolean;
 /**
  * Options for API requests (passed through from RecordOptions)
  */
@@ -20,19 +33,29 @@ interface ApiRequestOptions {
     timeoutSecs?: number;
     /** If true, don't retry on timeout errors */
     doNotRetryTimeouts?: boolean;
+    /** Override the method-based idempotency (e.g. get-or-create by name is safe to repeat). */
+    idempotent?: boolean;
 }
 /**
  * Base class for API-based storage clients.
  * Communicates with internal storage API instead of direct DB access.
  * No sensitive credentials are stored - only the user's API key.
  */
-declare abstract class BaseApiClient {
+export declare abstract class BaseApiClient {
     protected config: ApiStorageConfig;
     protected headers: Record<string, string>;
     private static rateLimiter;
     private static maxRetries;
+    static maxServerRetries: number;
     private static defaultTimeoutMs;
     constructor(config: ApiStorageConfig);
+    /**
+     * Sends the request and returns the successful (2xx) response, with the
+     * retry policy: 429 (rate limiter), timeouts, and the API being briefly
+     * unavailable (503, 5xx for idempotent methods, connection failures).
+     * Non-2xx answers that are not retried throw ApiStorageClientError.
+     */
+    private send;
     protected request<T>(method: string, path: string, body?: any, contentType?: string, options?: ApiRequestOptions): Promise<T>;
     protected requestWithHeaders<T>(method: string, path: string, body?: any, contentType?: string): Promise<{
         data: T;

package/dist/storage/api-storage-client.js

@@ -1,6 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ApiStorageClientFactory = exports.ApiRequestQueueClient = exports.ApiKeyValueStoreClient = exports.ApiDatasetClient = exports.ApiStorageClientError = void 0;
+exports.ApiStorageClientFactory = exports.ApiRequestQueueClient = exports.ApiKeyValueStoreClient = exports.ApiDatasetClient = exports.BaseApiClient = exports.ApiStorageClientError = void 0;
+exports.isRetryableResponse = isRetryableResponse;
+exports.serverRetryDelayMs = serverRetryDelayMs;
+exports.isConnectFailure = isConnectFailure;
 const node_crypto_1 = require("node:crypto");
 const kvs_iteration_js_1 = require("./kvs-iteration.js");
 function buildRqQueryParams(baseClientKey, options) {
@@ -155,6 +158,53 @@ exports.ApiStorageClientError = ApiStorageClientError;
 function isApiStorageClientError(err) {
     return err instanceof ApiStorageClientError;
 }
+// PUT is left out: a KV record PUT is counted (and billed) as a write each
+// time it is applied, so a lost response must not be re-sent blindly.
+const IDEMPOTENT_METHODS = new Set(['GET', 'HEAD', 'DELETE', 'OPTIONS']);
+// HAProxy's own 503 page: no backend was available, so nothing was forwarded.
+const NO_BACKEND_MARKER = 'No server is available to handle this request';
+const CONNECT_FAILURE_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'EHOSTUNREACH', 'ENETUNREACH', 'UND_ERR_CONNECT_TIMEOUT']);
+/**
+ * Whether a non-2xx answer is retried. Idempotent methods: 500/502/503/504.
+ * Others (dataset push, KV PUT) only on a 503 that certainly was not
+ * processed: the rate limiter refused it ("rate-limit-unavailable", before the
+ * handler), or HAProxy had no backend (its own page, nothing forwarded). Any
+ * other 503 may mean "outcome unknown" (e.g. a direct dataset write whose
+ * commit was not confirmed), and re-sending could store the items twice.
+ */
+function isRetryableResponse(status, idempotent, errorType, bodyText) {
+    if (idempotent)
+        return status === 500 || status === 502 || status === 503 || status === 504;
+    if (status !== 503)
+        return false;
+    return errorType === 'rate-limit-unavailable' || (bodyText ?? '').includes(NO_BACKEND_MARKER);
+}
+/** Exponential backoff with jitter; honours a Retry-After (seconds or HTTP date) of up to 30 s. */
+function serverRetryDelayMs(serverAttempt, retryAfter, random = Math.random, now = Date.now()) {
+    if (retryAfter !== null && retryAfter.trim() !== '') {
+        let secs = Number(retryAfter);
+        if (!Number.isFinite(secs)) {
+            const at = Date.parse(retryAfter);
+            secs = Number.isFinite(at) ? Math.max(0, (at - now) / 1000) : NaN;
+        }
+        if (Number.isFinite(secs) && secs >= 0)
+            return Math.min(secs, 30) * 1000 + random() * 100;
+    }
+    return Math.min(8000, 500 * 2 ** serverAttempt) + random() * 250;
+}
+function fetchErrorCode(err) {
+    const e = err;
+    return e?.cause?.code ?? e?.cause?.cause?.code ?? e?.code;
+}
+/** fetch() failed before the request reached the server. */
+function isConnectFailure(err) {
+    const code = fetchErrorCode(err);
+    return code !== undefined && CONNECT_FAILURE_CODES.has(code);
+}
+function describeFetchError(err) {
+    return fetchErrorCode(err) ?? err?.message ?? String(err);
+}
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 /** Map GET storage errors: rethrow 403, return undefined for 404 (triggers getOrCreate for names only). */
 function handleStorageGetError(err) {
     if (isApiStorageClientError(err)) {
@@ -175,17 +225,11 @@ function storageCollectionHeaders(config) {
 /** POST /v2/{datasets|key-value-stores|request-queues} to create or get-by-name. */
 async function postStorageCollection(config, path, name) {
     const query = name ? `?name=${encodeURIComponent(name)}` : '';
-    const url = `${config.apiUrl}/v2${path}${query}`;
-    const response = await fetch(url, {
-        method: 'POST',
-        headers: storageCollectionHeaders(config),
-    });
-    if (!response.ok) {
-        const error = await response.json().catch(() => ({ error: { message: response.statusText } }));
-        throw new Error(error.error?.message || `Failed to create storage: ${response.status}`);
-    }
-    const body = await response.json();
-    return unwrapData(body);
+    // Not marked repeatable even by name: a 500 after the create (e.g. scoped
+    // token registration failed) must surface, since a re-send would take the
+    // existing-name path and hide it. POST rules apply (limiter/HAProxy 503,
+    // refused connection).
+    return new CollectionApiClient(config).call('POST', `${path}${query}`);
 }
 /**
  * Helper to create AsyncIterable + Promise result (matches Crawlee pattern)
@@ -226,6 +270,10 @@ class BaseApiClient {
     static rateLimiter = new AdaptiveRateLimiter();
     // Maximum retries for rate-limited requests
     static maxRetries = 3;
+    // Retries while the API is briefly unavailable (503/5xx, connection
+    // refused): backoff 0.5 s doubling to 8 s, about 45 s in total, which
+    // covers a main redeploy.
+    static maxServerRetries = 8;
     // Default timeout for requests (30 seconds)
     static defaultTimeoutMs = 30000;
     constructor(config) {
@@ -236,7 +284,13 @@ class BaseApiClient {
             'Content-Type': 'application/json',
         };
     }
-    async request(method, path, body, contentType, options) {
+    /**
+     * Sends the request and returns the successful (2xx) response, with the
+     * retry policy: 429 (rate limiter), timeouts, and the API being briefly
+     * unavailable (503, 5xx for idempotent methods, connection failures).
+     * Non-2xx answers that are not retried throw ApiStorageClientError.
+     */
+    async send(method, path, body, contentType, options, decode) {
         const url = `${this.config.apiUrl}/v2${path}`;
         const headers = { ...this.headers };
         if (contentType) {
@@ -246,8 +300,10 @@ class BaseApiClient {
             ? options.timeoutSecs * 1000
             : BaseApiClient.defaultTimeoutMs;
         const shouldRetryTimeouts = options?.doNotRetryTimeouts !== true;
-        let lastError = null;
-        for (let attempt = 0; attempt <= BaseApiClient.maxRetries; attempt++) {
+        const idempotent = options?.idempotent ?? IDEMPOTENT_METHODS.has(method.toUpperCase());
+        let attempt = 0; // timeouts and 429s
+        let serverAttempt = 0; // 503/5xx and connection failures (API briefly unavailable)
+        for (;;) {
             // Wait for rate limiter slot before making request
             await BaseApiClient.rateLimiter.waitForSlot();
             // Create AbortController for timeout
@@ -265,89 +321,110 @@ class BaseApiClient {
                 // Handle rate limiting (429)
                 if (response.status === 429) {
                     if (attempt < BaseApiClient.maxRetries) {
+                        attempt++;
                         await BaseApiClient.rateLimiter.handleRateLimited(response);
                         continue; // Retry
                     }
-                    // Max retries exceeded
                     const error = await response.json().catch(() => ({ error: { message: 'Rate limit exceeded' } }));
-                    throw new Error(error.error?.message || 'Rate limit exceeded');
+                    throw new ApiStorageClientError(error.error?.message || 'Rate limit exceeded', 429);
                 }
-                if (!response.ok) {
-                    const error = await response.json().catch(() => ({ error: { message: response.statusText } }));
-                    throw new ApiStorageClientError(error.error?.message || `API request failed: ${response.status}`, response.status);
+                if (response.ok) {
+                    try {
+                        // Decoded inside the retry loop: a body cut off after
+                        // the headers is a transport failure like any other.
+                        return await decode(response);
+                    }
+                    catch (bodyErr) {
+                        // The request was applied (2xx); a write whose answer
+                        // timed out while being read must not be sent again.
+                        if (bodyErr.name === 'AbortError' && !idempotent) {
+                            const timeoutError = new Error(`Request timed out after ${timeoutMs}ms (response body; the request was applied)`);
+                            timeoutError.isTimeout = true;
+                            throw timeoutError;
+                        }
+                        if (idempotent && bodyErr.name !== 'AbortError' && serverAttempt < BaseApiClient.maxServerRetries) {
+                            const waitMs = serverRetryDelayMs(serverAttempt, null);
+                            serverAttempt++;
+                            console.warn(`[Scrapely] ${method} ${path} response body failed (${describeFetchError(bodyErr)}); retry ${serverAttempt}/${BaseApiClient.maxServerRetries} in ${Math.round(waitMs)}ms`);
+                            await sleep(waitMs);
+                            continue;
+                        }
+                        throw bodyErr;
+                    }
                 }
-                // Handle empty responses (204 No Content)
-                if (response.status === 204) {
-                    return undefined;
+                // API briefly unavailable (main redeploy, limiter store blip).
+                const text = await response.text().catch(() => '');
+                let errorBody;
+                try {
+                    errorBody = JSON.parse(text);
                 }
-                // Handle text/plain responses
-                const responseContentType = response.headers.get('content-type') || '';
-                if (responseContentType.includes('text/plain')) {
-                    return response.text();
+                catch {
+                    errorBody = undefined;
                 }
-                // Handle binary responses
-                if (responseContentType.includes('application/octet-stream')) {
-                    const buffer = Buffer.from(await response.arrayBuffer());
-                    return buffer;
+                if (isRetryableResponse(response.status, idempotent, errorBody?.error?.type, text)
+                    && serverAttempt < BaseApiClient.maxServerRetries) {
+                    const waitMs = serverRetryDelayMs(serverAttempt, response.headers.get('retry-after'));
+                    serverAttempt++;
+                    console.warn(`[Scrapely] ${method} ${path} answered ${response.status}; retry ${serverAttempt}/${BaseApiClient.maxServerRetries} in ${Math.round(waitMs)}ms`);
+                    await sleep(waitMs);
+                    continue;
                 }
-                const json = await response.json();
-                return unwrapData(json);
+                throw new ApiStorageClientError(errorBody?.error?.message || response.statusText || `API request failed: ${response.status}`, response.status);
             }
             catch (err) {
                 // Handle timeout/abort errors
                 if (err.name === 'AbortError') {
                     const timeoutError = new Error(`Request timed out after ${timeoutMs}ms`);
                     timeoutError.isTimeout = true;
-                    lastError = timeoutError;
                     // Don't retry if doNotRetryTimeouts is true
                     if (!shouldRetryTimeouts || attempt >= BaseApiClient.maxRetries) {
                         throw timeoutError;
                     }
+                    attempt++;
                     // Wait a bit before retrying
-                    await new Promise(resolve => setTimeout(resolve, 100 * (attempt + 1)));
+                    await sleep(100 * attempt);
                     continue;
                 }
-                lastError = err;
-                // Don't retry non-timeout errors unless it's a network issue
-                const isNetworkError = !err.response && !err.status;
-                if (!isNetworkError || attempt >= BaseApiClient.maxRetries) {
+                // HTTP errors (already decided above) are final.
+                if (isApiStorageClientError(err))
                     throw err;
+                // fetch() failed without a response. A refused/unresolvable
+                // connection never reached the API, so it is safe for every
+                // method; anything else (reset mid-request) only for idempotent ones.
+                if ((idempotent || isConnectFailure(err)) && serverAttempt < BaseApiClient.maxServerRetries) {
+                    const waitMs = serverRetryDelayMs(serverAttempt, null);
+                    serverAttempt++;
+                    console.warn(`[Scrapely] ${method} ${path} failed (${describeFetchError(err)}); retry ${serverAttempt}/${BaseApiClient.maxServerRetries} in ${Math.round(waitMs)}ms`);
+                    await sleep(waitMs);
+                    continue;
                 }
-                // Wait before retrying network errors
-                await new Promise(resolve => setTimeout(resolve, 100 * (attempt + 1)));
+                throw err;
             }
             finally {
                 clearTimeout(timeoutId);
             }
         }
-        throw lastError || new Error('Request failed after retries');
     }
-    async requestWithHeaders(method, path, body, contentType) {
-        const url = `${this.config.apiUrl}/v2${path}`;
-        const headers = { ...this.headers };
-        if (contentType) {
-            headers['Content-Type'] = contentType;
-        }
-        for (let attempt = 0; attempt <= BaseApiClient.maxRetries; attempt++) {
-            await BaseApiClient.rateLimiter.waitForSlot();
-            const response = await fetch(url, {
-                method,
-                headers,
-                body: body !== undefined ? (typeof body === 'string' ? body : JSON.stringify(body)) : undefined,
-            });
-            BaseApiClient.rateLimiter.updateFromHeaders(response.headers);
-            if (response.status === 429) {
-                if (attempt < BaseApiClient.maxRetries) {
-                    await BaseApiClient.rateLimiter.handleRateLimited(response);
-                    continue;
-                }
-                const error = await response.json().catch(() => ({ error: { message: 'Rate limit exceeded' } }));
-                throw new Error(error.error?.message || 'Rate limit exceeded');
+    async request(method, path, body, contentType, options) {
+        return this.send(method, path, body, contentType, options, async (response) => {
+            // Handle empty responses (204 No Content)
+            if (response.status === 204) {
+                return undefined;
             }
-            if (!response.ok) {
-                const error = await response.json().catch(() => ({ error: { message: response.statusText } }));
-                throw new Error(error.error?.message || `API request failed: ${response.status}`);
+            // Handle text/plain responses
+            const responseContentType = response.headers.get('content-type') || '';
+            if (responseContentType.includes('text/plain')) {
+                return (await response.text());
             }
+            // Handle binary responses
+            if (responseContentType.includes('application/octet-stream')) {
+                return Buffer.from(await response.arrayBuffer());
+            }
+            return unwrapData(await response.json());
+        });
+    }
+    async requestWithHeaders(method, path, body, contentType) {
+        return this.send(method, path, body, contentType, undefined, async (response) => {
             if (response.status === 204) {
                 return { data: undefined, contentType: '' };
             }
@@ -356,13 +433,18 @@ class BaseApiClient {
                 return { data: await response.text(), contentType: 'text/plain' };
             }
             if (responseContentType.includes('application/octet-stream')) {
-                const buffer = Buffer.from(await response.arrayBuffer());
-                return { data: buffer, contentType: 'application/octet-stream' };
+                return { data: Buffer.from(await response.arrayBuffer()), contentType: 'application/octet-stream' };
             }
             const json = await response.json();
             return { data: json, contentType: responseContentType.split(';')[0].trim() || 'application/json' };
-        }
-        throw new Error('Request failed after retries');
+        });
+    }
+}
+exports.BaseApiClient = BaseApiClient;
+/** Collection-level calls (create/get by name, list) with the same retry policy. */
+class CollectionApiClient extends BaseApiClient {
+    call(method, path, options) {
+        return this.request(method, path, undefined, undefined, options);
     }
 }
 /**
@@ -531,8 +613,12 @@ class ApiKeyValueStoreClient extends BaseApiClient {
             }
             return { key, value, contentType: contentType || 'application/json' };
         }
-        catch {
-            return undefined;
+        catch (err) {
+            // Missing record only; anything else (API unavailable after the
+            // retries) must not look like "no record" (lost state, restarts).
+            if (isApiStorageClientError(err) && err.statusCode === 404)
+                return undefined;
+            throw err;
         }
     }
     async setRecord(record, options) {
@@ -850,17 +936,7 @@ class ApiRequestQueueCollectionClient {
         };
     }
     async list() {
-        const url = `${this.config.apiUrl}/v2/request-queues`;
-        const response = await fetch(url, {
-            headers: storageCollectionHeaders(this.config),
-        });
-        if (!response.ok) {
-            const error = await response.json().catch(() => ({ error: { message: response.statusText } }));
-            throw new Error(error.error?.message
-                || `Failed to list request queues: ${response.status}`);
-        }
-        const body = await response.json();
-        const data = unwrapData(body);
+        const data = await new CollectionApiClient(this.config).call('GET', '/request-queues');
         return {
             total: data.total,
             count: data.count,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.9",
+    "version": "1.2.11",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -19,6 +19,7 @@
     "scripts": {
         "test": "npx tsx tests/test-mvp.ts",
         "test:charging": "npx tsx --test tests/charging-pending.test.ts",
+        "test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",
         "test:stdio-drain": "npx tsx --test tests/stdio-drain.test.ts",
         "test:charging-client": "npx tsx --test tests/charging-client.test.ts",
         "test:proxy": "npx tsx --test tests/proxy-configuration.test.ts",