scrapely 1.2.17 → 1.2.19

package/dist/actor.js

@@ -1062,9 +1062,16 @@ class Actor {
             label: 'PUT status message',
             retryWindowMs: options?.retryWindowMs,
         }, async (response) => (await response.json().catch(() => ({})))).catch((err) => {
-            throw new Error(`Failed to update status message: ${describeRunApiError(err)}`);
+            const error = new Error(`Failed to update status message: ${describeRunApiError(err)}`);
+            // A progress update is best effort: if the API still fails after the
+            // retry window (e.g. a long 503), log it and carry on instead of
+            // crashing the run. Terminal updates keep throwing.
+            if (isStatusMessageTerminal)
+                throw error;
+            console.warn(`[Scrapely] ${error.message} (continuing)`);
+            return undefined;
         });
-        return (body.data ?? {});
+        return (body?.data ?? {});
     }
     /**
      * Creates a proxy configuration and returns a promise resolving to an instance

package/dist/charging.d.ts

@@ -234,6 +234,8 @@ export declare function pushDataAndCharge<T>({ chargingManager, items, eventName
 /**
  * @internal True when a dataset push certainly stored nothing: the storage API
  * answered with an error status, or the request never reached it. Timeouts and
- * resets after the request was sent are not proof (the write may have landed).
+ * resets after the request was sent are not proof (the write may have landed),
+ * and neither is any error of a push one of whose earlier attempts may have
+ * been applied (`outcomeUnknown`, set by the storage client's retries).
  */
 export declare function pushCertainlyFailed(err: unknown): boolean;

package/dist/charging.js

@@ -25,7 +25,7 @@ const PENDING_MAX_ENTRIES = 1024;
 // so the retries here can honour deadlines: 429, 408, 5xx, timeouts and
 // network errors are re-sent with the same idempotency key, with exponential
 // backoff and jitter, never sooner than the server's Retry-After, within the API
-// retry window (2 minutes, SCRAPELY_API_RETRY_WINDOW_SECS) or the caller's
+// retry window (5 minutes, SCRAPELY_API_RETRY_WINDOW_SECS) or the caller's
 // deadline. Then the charge stays pending / the void is re-queued and retried
 // before the next push and at exit, the same as the Python SDK.
 const POST_ATTEMPTS = 64;
@@ -1182,9 +1182,17 @@ const NOT_SENT_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'EHOST
 /**
  * @internal True when a dataset push certainly stored nothing: the storage API
  * answered with an error status, or the request never reached it. Timeouts and
- * resets after the request was sent are not proof (the write may have landed).
+ * resets after the request was sent are not proof (the write may have landed),
+ * and neither is any error of a push one of whose earlier attempts may have
+ * been applied (`outcomeUnknown`, set by the storage client's retries).
  */
 function pushCertainlyFailed(err) {
+    for (let e = err, depth = 0; e && depth < 4; e = e.cause, depth++) {
+        if (e.outcomeUnknown === true)
+            return false;
+    }
+    if ((0, api_storage_client_js_1.isConnectFailure)(err))
+        return true;
     for (let e = err, depth = 0; e && depth < 4; e = e.cause, depth++) {
         const status = httpStatusOf(e);
         if (status !== undefined)

package/dist/storage/api-storage-client.d.ts

@@ -24,11 +24,15 @@ export declare class ApiStorageClientError extends Error {
  */
 export declare function isRetryableResponse(status: number, idempotent: boolean, errorType?: string, bodyText?: string): boolean;
 /** Default time budget for retrying one API call (429, 503, network errors). */
-export declare const DEFAULT_API_RETRY_WINDOW_MS = 120000;
+export declare const DEFAULT_API_RETRY_WINDOW_MS = 300000;
+/** Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes. */
+export declare function apiDedupesKeyedWrites(url?: string): boolean;
+/** Test hook: forget what was learned about the API, or force an answer for every origin. */
+export declare function _resetKeyedWriteSupport(value?: boolean): void;
 /**
  * How long one API call keeps retrying before it gives up:
  * SCRAPELY_API_RETRY_WINDOW_SECS when set, else {@link BaseApiClient.retryWindowMs}
- * (2 minutes by default).
+ * (5 minutes by default).
  */
 export declare function apiRetryWindowMs(): number;
 /**
@@ -51,6 +55,13 @@ export interface RetryingFetchOptions {
     retryWindowMs?: number;
     /** Short label for log lines, e.g. "POST /datasets/x/items". */
     label: string;
+    /**
+     * Idempotency-Key for a write (one per logical write, the same on every
+     * attempt). Once the API is known to de-duplicate keyed writes, the write
+     * is retried like an idempotent call: a re-send of an applied write returns
+     * the original answer instead of applying it again.
+     */
+    idempotencyKey?: string;
 }
 export interface RetryingFetchInit {
     method: string;
@@ -59,15 +70,17 @@ export interface RetryingFetchInit {
 }
 /**
  * fetch() with the API retry policy. Retries, with exponential backoff and
- * jitter that honours Retry-After, until the retry window (2 minutes by
+ * jitter that honours Retry-After, until the retry window (5 minutes by
  * default) runs out:
  * - 429 (run API rate limit, dataset backpressure): always; nothing was applied.
  * - 503 from the limiter or HAProxy: always; nothing was forwarded.
- * - 500/502/503/504, timeouts and dropped connections: idempotent calls only.
+ * - 500/502/503/504, timeouts and dropped connections: idempotent calls, and
+ *   writes with an Idempotency-Key once the API is known to de-duplicate them.
  * - Refused/unresolvable connections: always; the request never left.
- * A write (dataset push, KV PUT, metamorph) whose request may have reached the
- * API is never re-sent: there is no idempotency key for it, so a re-send could
- * store the same rows twice. Such failures surface to the caller instead.
+ * Any other write (KV PUT, metamorph, a keyed push to an API that did not say
+ * it de-duplicates) whose request may have reached the API is never re-sent:
+ * a re-send could apply it twice. Such failures surface to the caller instead,
+ * marked `outcomeUnknown` when an attempt may have been applied.
  * Non-2xx answers that are not retried throw ApiStorageClientError.
  */
 export declare function fetchWithRetries<R>(url: string, init: RetryingFetchInit, options: RetryingFetchOptions, decode: (response: Response) => Promise<R>): Promise<R>;
@@ -81,6 +94,8 @@ interface ApiRequestOptions {
     doNotRetryTimeouts?: boolean;
     /** Override the method-based idempotency (e.g. get-or-create by name is safe to repeat). */
     idempotent?: boolean;
+    /** Idempotency-Key for a write; see {@link RetryingFetchOptions.idempotencyKey}. */
+    idempotencyKey?: string;
 }
 /**
  * Base class for API-based storage clients.

package/dist/storage/api-storage-client.js

@@ -2,6 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ApiStorageClientFactory = exports.ApiRequestQueueClient = exports.ApiKeyValueStoreClient = exports.ApiDatasetClient = exports.BaseApiClient = exports.DEFAULT_API_RETRY_WINDOW_MS = exports.ApiStorageClientError = void 0;
 exports.isRetryableResponse = isRetryableResponse;
+exports.apiDedupesKeyedWrites = apiDedupesKeyedWrites;
+exports._resetKeyedWriteSupport = _resetKeyedWriteSupport;
 exports.apiRetryWindowMs = apiRetryWindowMs;
 exports.serverRetryDelayMs = serverRetryDelayMs;
 exports.isConnectFailure = isConnectFailure;
@@ -161,12 +163,72 @@ function isRetryableResponse(status, idempotent, errorType, bodyText) {
     return errorType === 'rate-limit-unavailable' || (bodyText ?? '').includes(NO_BACKEND_MARKER);
 }
 /** Default time budget for retrying one API call (429, 503, network errors). */
-exports.DEFAULT_API_RETRY_WINDOW_MS = 120_000;
+exports.DEFAULT_API_RETRY_WINDOW_MS = 300_000;
+/**
+ * Response header the API sets on every dataset push that sent an
+ * Idempotency-Key, once it de-duplicates such pushes. An older API ignores the
+ * key, and re-sending a push there could store the rows twice, so ambiguous
+ * failures of keyed writes are retried only while the latest keyed answer from
+ * that API origin carried this header. A 2xx without it (an older instance,
+ * e.g. mid-deploy or after a rollback) switches the retries off again.
+ */
+const IDEMPOTENCY_SUPPORT_HEADER = 'x-scrapely-idempotency';
+const keyedWritesDeduped = new Map();
+/** Test override for every origin (undefined: learn from answers). */
+let keyedWritesDedupedOverride;
+function originOf(url) {
+    try {
+        return new URL(url).origin;
+    }
+    catch {
+        return url;
+    }
+}
+/** Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes. */
+function apiDedupesKeyedWrites(url) {
+    if (keyedWritesDedupedOverride !== undefined)
+        return keyedWritesDedupedOverride;
+    if (url !== undefined)
+        return keyedWritesDeduped.get(originOf(url)) === true;
+    return [...keyedWritesDeduped.values()].pop() === true;
+}
+/** Test hook: forget what was learned about the API, or force an answer for every origin. */
+function _resetKeyedWriteSupport(value) {
+    keyedWritesDeduped.clear();
+    keyedWritesDedupedOverride = value === true ? true : undefined;
+}
+function noteKeyedAnswer(url, response) {
+    const origin = originOf(url);
+    if (response.headers.has(IDEMPOTENCY_SUPPORT_HEADER)) {
+        keyedWritesDeduped.delete(origin);
+        keyedWritesDeduped.set(origin, true);
+    }
+    else if (response.ok) {
+        // Stored by a handler that ignored the key: this API (instance) does not de-duplicate.
+        keyedWritesDeduped.delete(origin);
+        keyedWritesDeduped.set(origin, false);
+        if (keyedWritesDedupedOverride === true)
+            keyedWritesDedupedOverride = undefined;
+    }
+}
+/** API error types of a keyed write: still being stored (retry), or stored-or-not unknown (final). */
+const KEY_IN_PROGRESS = 'idempotency-key-in-progress';
+const OUTCOME_UNKNOWN = 'push-outcome-unknown';
+/**
+ * Marks an error whose request may have been applied by an earlier attempt
+ * (timeout, dropped connection or 5xx before the final answer), so callers
+ * don't treat it as "nothing was stored".
+ */
+function markOutcomeUnknown(err) {
+    if (err && typeof err === 'object')
+        err.outcomeUnknown = true;
+    return err;
+}
 const MAX_BACKOFF_MS = 30_000;
 /**
  * How long one API call keeps retrying before it gives up:
  * SCRAPELY_API_RETRY_WINDOW_SECS when set, else {@link BaseApiClient.retryWindowMs}
- * (2 minutes by default).
+ * (5 minutes by default).
  */
 function apiRetryWindowMs() {
     const env = Number(process.env.SCRAPELY_API_RETRY_WINDOW_SECS);
@@ -206,7 +268,21 @@ function fetchErrorCode(err) {
 /** fetch() failed before the request reached the server. */
 function isConnectFailure(err) {
     const code = fetchErrorCode(err);
-    return code !== undefined && CONNECT_FAILURE_CODES.has(code);
+    if (code !== undefined && CONNECT_FAILURE_CODES.has(code))
+        return true;
+    // A TCP connect that timed out (`connect ETIMEDOUT`, or Node's happy-eyeballs
+    // AggregateError of connect attempts): nothing was sent. An ETIMEDOUT on an
+    // established socket (syscall "read") is not, so the syscall decides.
+    for (let e = err, depth = 0; e && depth < 4; e = e.cause, depth++) {
+        if (e.code === 'ETIMEDOUT' || e.code === 'ECONNREFUSED' || e.code === 'EHOSTUNREACH' || e.code === 'ENETUNREACH') {
+            if (e.syscall === 'connect')
+                return true;
+            const attempts = Array.isArray(e.errors) ? e.errors : [];
+            if (attempts.length > 0 && attempts.every((a) => a?.syscall === 'connect'))
+                return true;
+        }
+    }
+    return false;
 }
 function describeFetchError(err) {
     return fetchErrorCode(err) ?? err?.message ?? String(err);
@@ -217,22 +293,32 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const sharedRateLimiter = new AdaptiveRateLimiter();
 /**
  * fetch() with the API retry policy. Retries, with exponential backoff and
- * jitter that honours Retry-After, until the retry window (2 minutes by
+ * jitter that honours Retry-After, until the retry window (5 minutes by
  * default) runs out:
  * - 429 (run API rate limit, dataset backpressure): always; nothing was applied.
  * - 503 from the limiter or HAProxy: always; nothing was forwarded.
- * - 500/502/503/504, timeouts and dropped connections: idempotent calls only.
+ * - 500/502/503/504, timeouts and dropped connections: idempotent calls, and
+ *   writes with an Idempotency-Key once the API is known to de-duplicate them.
  * - Refused/unresolvable connections: always; the request never left.
- * A write (dataset push, KV PUT, metamorph) whose request may have reached the
- * API is never re-sent: there is no idempotency key for it, so a re-send could
- * store the same rows twice. Such failures surface to the caller instead.
+ * Any other write (KV PUT, metamorph, a keyed push to an API that did not say
+ * it de-duplicates) whose request may have reached the API is never re-sent:
+ * a re-send could apply it twice. Such failures surface to the caller instead,
+ * marked `outcomeUnknown` when an attempt may have been applied.
  * Non-2xx answers that are not retried throw ApiStorageClientError.
  */
 async function fetchWithRetries(url, init, options, decode) {
-    const { idempotent, timeoutMs, label } = options;
-    const retryTimeouts = options.retryTimeouts !== false && idempotent;
+    const { timeoutMs, label, idempotencyKey } = options;
+    const headers = idempotencyKey ? { ...init.headers, 'Idempotency-Key': idempotencyKey } : init.headers;
+    // Re-sending is safe for idempotent calls, and for keyed writes once the
+    // API showed it de-duplicates them (learned from any keyed answer).
+    const safeToResend = () => options.idempotent || (idempotencyKey !== undefined && apiDedupesKeyedWrites(url));
+    const retryTimeouts = () => options.retryTimeouts !== false && safeToResend();
     const deadline = Date.now() + Math.max(0, options.retryWindowMs ?? apiRetryWindowMs());
     let attempt = 0;
+    // An attempt of a non-idempotent call may have been applied (timeout,
+    // dropped connection, 5xx): the final error must not claim "nothing applied".
+    let maybeApplied = false;
+    const fail = (err) => (maybeApplied && !options.idempotent ? markOutcomeUnknown(err) : err);
     /** Sleeps before the next attempt; false once the retry window is used up. */
     const backoff = async (why, retryAfter) => {
         // Retrying sooner than asked would only be refused again: give up when
@@ -250,8 +336,10 @@ async function fetchWithRetries(url, init, options, decode) {
         const controller = new AbortController();
         const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
         try {
-            const response = await fetch(url, { method: init.method, headers: init.headers, body: init.body, signal: controller.signal });
+            const response = await fetch(url, { method: init.method, headers, body: init.body, signal: controller.signal });
             sharedRateLimiter.updateFromHeaders(response.headers);
+            if (idempotencyKey)
+                noteKeyedAnswer(url, response);
             if (response.ok) {
                 try {
                     // Decoded inside the retry loop: a body cut off after
@@ -261,19 +349,20 @@ async function fetchWithRetries(url, init, options, decode) {
                 catch (bodyErr) {
                     // The request was applied (2xx); a write whose answer
                     // timed out while being read must not be sent again.
+                    maybeApplied = true;
                     if (bodyErr.name === 'AbortError') {
-                        if (!idempotent || !retryTimeouts) {
-                            const timeoutError = new Error(`Request timed out after ${timeoutMs}ms (response body${idempotent ? '' : '; the request was applied'})`);
+                        if (!retryTimeouts()) {
+                            const timeoutError = new Error(`Request timed out after ${timeoutMs}ms (response body${options.idempotent ? '' : '; the request was applied'})`);
                             timeoutError.isTimeout = true;
-                            throw timeoutError;
+                            throw fail(timeoutError);
                         }
                         if (await backoff(`response body timed out after ${timeoutMs}ms`, null))
                             continue;
-                        throw bodyErr;
+                        throw fail(bodyErr);
                     }
-                    if (idempotent && await backoff(`response body failed (${describeFetchError(bodyErr)})`, null))
+                    if (safeToResend() && await backoff(`response body failed (${describeFetchError(bodyErr)})`, null))
                         continue;
-                    throw bodyErr;
+                    throw fail(bodyErr);
                 }
             }
             const text = await response.text().catch(() => '');
@@ -284,11 +373,27 @@ async function fetchWithRetries(url, init, options, decode) {
             catch {
                 errorBody = undefined;
             }
-            if (isRetryableResponse(response.status, idempotent, errorBody?.error?.type, text)
+            const errorType = errorBody?.error?.type;
+            const apiError = new ApiStorageClientError(errorBody?.error?.message || response.statusText || `API request failed: ${response.status}`, response.status);
+            if (idempotencyKey && errorType === OUTCOME_UNKNOWN) {
+                // Stored or not, nobody knows: the API refuses repeats of this key.
+                maybeApplied = true;
+                throw fail(apiError);
+            }
+            // A 5xx other than the limiter's / HAProxy's 503 may come after the write was applied.
+            if (!options.idempotent && response.status >= 500 && !isRetryableResponse(response.status, false, errorType, text)) {
+                maybeApplied = true;
+            }
+            const resend = safeToResend();
+            // 409 for a keyed write: an earlier attempt is still being stored.
+            const inProgress = idempotencyKey !== undefined && response.status === 409 && errorType === KEY_IN_PROGRESS;
+            if ((inProgress || isRetryableResponse(response.status, resend, errorType, text))
                 && await backoff(`answered ${response.status}`, response.headers.get('retry-after'))) {
                 continue;
             }
-            throw new ApiStorageClientError(errorBody?.error?.message || response.statusText || `API request failed: ${response.status}`, response.status);
+            if (inProgress)
+                maybeApplied = true;
+            throw fail(apiError);
         }
         catch (err) {
             // HTTP errors (already decided above) are final.
@@ -296,19 +401,25 @@ async function fetchWithRetries(url, init, options, decode) {
                 throw err;
             if (err?.isTimeout)
                 throw err;
+            if (err?.outcomeUnknown)
+                throw err;
             if (err.name === 'AbortError') {
+                maybeApplied = true;
                 const timeoutError = new Error(`Request timed out after ${timeoutMs}ms`);
                 timeoutError.isTimeout = true;
-                if (retryTimeouts && await backoff(`timed out after ${timeoutMs}ms`, null))
+                if (retryTimeouts() && await backoff(`timed out after ${timeoutMs}ms`, null))
                     continue;
-                throw timeoutError;
+                throw fail(timeoutError);
             }
             // fetch() failed without a response. A refused/unresolvable
             // connection never reached the API, so it is safe for every
-            // method; anything else (reset mid-request) only for idempotent ones.
-            if ((idempotent || isConnectFailure(err)) && await backoff(`failed (${describeFetchError(err)})`, null))
+            // method; anything else (reset mid-request) only when re-sending is safe.
+            const connectFailure = isConnectFailure(err);
+            if (!connectFailure)
+                maybeApplied = true;
+            if ((safeToResend() || connectFailure) && await backoff(`failed (${describeFetchError(err)})`, null))
                 continue;
-            throw err;
+            throw fail(err);
         }
         finally {
             clearTimeout(timeoutId);
@@ -377,7 +488,7 @@ class BaseApiClient {
     headers;
     // How long one call keeps retrying 429/503/network errors before it gives
     // up (SCRAPELY_API_RETRY_WINDOW_SECS overrides it). Long enough that a
-    // throttled run slows down instead of failing, and covers a main redeploy.
+    // throttled run slows down instead of failing, and covers an API redeploy.
     static retryWindowMs = exports.DEFAULT_API_RETRY_WINDOW_MS;
     // Default timeout for requests (30 seconds)
     static defaultTimeoutMs = 30000;
@@ -413,6 +524,7 @@ class BaseApiClient {
             timeoutMs,
             retryTimeouts: options?.doNotRetryTimeouts !== true,
             label: `${method} ${path}`,
+            idempotencyKey: options?.idempotencyKey,
         }, decode);
     }
     async request(method, path, body, contentType, options) {
@@ -570,7 +682,9 @@ class ApiDatasetClient extends BaseApiClient {
         // must not store unbilled rows in its default dataset.
         if (this.id === (0, charging_js_1.defaultDatasetIdFromEnv)() && (0, charging_js_1.refuseDefaultDatasetPush)(parsed.length))
             return;
-        await this.request('POST', `/datasets/${this.id}/items`, parsed);
+        // One key per push, the same on every attempt: the API stores a
+        // re-sent push once, so a push whose connection dropped can be retried.
+        await this.request('POST', `/datasets/${this.id}/items`, parsed, undefined, { idempotencyKey: (0, node_crypto_1.randomUUID)() });
     }
 }
 exports.ApiDatasetClient = ApiDatasetClient;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.17",
+    "version": "1.2.19",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -24,6 +24,7 @@
         "test:charging": "npx tsx --test tests/charging-pending.test.ts",
         "test:charge-limit-guard": "npx tsx --test tests/charge-limit-guard.test.ts",
         "test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",
+        "test:push-idempotency": "npx tsx --test tests/push-idempotency.test.ts",
         "test:stdio-drain": "npx tsx --test tests/stdio-drain.test.ts",
         "test:charging-client": "npx tsx --test tests/charging-client.test.ts",
         "test:proxy": "npx tsx --test tests/proxy-configuration.test.ts",