scrapely 1.2.36 → 1.2.38

package/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 All notable changes to this project will be documented in this file.
 
+## [Unreleased]
+
+## [1.2.38] - 2026-10-27
+
+- A key-value store record write (`Actor.setValue`, `KeyValueStore.setValue`) is no longer lost to one failed answer. Each write now sends an `Idempotency-Key`, and when its answer is lost (a `502`/`504` page from the edge or load balancer, a dropped connection, a timeout) it is sent again with the same key; the API applies and counts it once. Before, the write was never re-sent, so one such page on the final `OUTPUT` write failed the whole run. An API that answers a keyed write without `X-Scrapely-Idempotency` (an older server) gets no re-sends, as before.
+
+## [1.2.37] - 2026-10-26
+
+- Opening a named dataset, key-value store or request queue (`Actor.openDataset('name')` and the like) retries when the API answers `503 storage-name-busy`. The API sends it when another request kept creating or renaming a storage of that name for over 15 s; nothing was changed, so the open is simply repeated. It used to fail the open.
+- Once `Actor.fail()` has run, the run stays failed: a later `Actor.exit()`, or the end of `Actor.main()`, keeps the failure's exit code and status message. `fail()` normally ends the process at once, so this only changed runs that called `fail({ exit: false })` and then `exit()` or returned from `Actor.main()`; they used to exit with 0. A second `fail()` keeps the first failure. `fail({ exitCode: undefined })` exits with 1.
+
 ## [1.2.36] - 2026-10-22
 
 - Debug captures (`Actor.debugCapture`, `Actor.debugCaptureSync`) keep payloads over 256 KiB whole. They are stored gzip-compressed and base64-encoded as `{ key, payload, encoding: 'gzip-base64', length, truncated }` instead of being cut at 262,144 characters; up to 8 Mi characters are kept. A payload that does not compress enough keeps its first 256 KiB, now marked `truncated: true` with its full `length`. `Uint8Array` / `Buffer` payloads are stored as UTF-8 text. Captures are only active on the platform's own diagnostic runs (`SCRAPELY_DEBUG_CAPTURE=1`).

package/dist/actor.d.ts

@@ -332,6 +332,12 @@ export declare class Actor {
      * succeeds.
      */
     private _lifecycleStarted;
+    /**
+     * Set by {@link Actor.fail} for the rest of the run: a later {@link Actor.exit}
+     * keeps the failure's exit code and status message, so a failed run never
+     * ends as SUCCEEDED (e.g. `fail({ exit: false })` followed by `exit()`).
+     */
+    private static _failure;
     /** Created on first use: scrapely-client is only loaded when something needs the API client. */
     private _apifyClient;
     private readonly chargingManager;
@@ -388,6 +394,7 @@ export declare class Actor {
      * @param options - Exit options
      */
     static exit(messageOrOptions?: string | ExitOptions, options?: ExitOptions): Promise<void>;
+    private static _exitWith;
     static fail(messageOrOptions?: string | ExitOptions, options?: ExitOptions): Promise<void>;
     static getInput<T = ActorInput>(): Promise<T | null>;
     static getInputOrThrow<T = ActorInput>(): Promise<T>;

package/dist/actor.js

@@ -112,6 +112,12 @@ class Actor {
      * succeeds.
      */
     _lifecycleStarted = false;
+    /**
+     * Set by {@link Actor.fail} for the rest of the run: a later {@link Actor.exit}
+     * keeps the failure's exit code and status message, so a failed run never
+     * ends as SUCCEEDED (e.g. `fail({ exit: false })` followed by `exit()`).
+     */
+    static _failure;
     /** Created on first use: scrapely-client is only loaded when something needs the API client. */
     _apifyClient;
     chargingManager;
@@ -189,6 +195,7 @@ class Actor {
         if (this.initialized)
             return;
         cachedInput = undefined; // a fresh init reads this run's input again
+        Actor._failure = undefined; // a new lifecycle starts unfailed
         // NOTE: `initialized` is set at the END of init — a failed platform init
         // must stay retryable instead of leaving a half-initialized Actor.
         Actor.installFatalChargeHandlers();
@@ -375,7 +382,8 @@ class Actor {
             const result = await userFunc();
             await Actor._shutdownActor(options?.timeoutSecs ?? 30);
             await new Promise((resolve) => process.stdout.write('', () => resolve()));
-            process.exit(options?.exitCode ?? 0);
+            // A run that called fail({ exit: false }) stays failed.
+            process.exit(Actor._failure?.exitCode ?? options?.exitCode ?? 0);
             return result;
         }
         catch (err) {
@@ -403,10 +411,22 @@ class Actor {
         const opts = typeof messageOrOptions === 'string'
             ? { ...options, statusMessage: messageOrOptions }
             : { ...messageOrOptions, ...options };
+        Actor._ensureActorInit('exit');
+        const failure = Actor._failure;
+        if (failure) {
+            // Actor.fail() already ran: the run stays failed. Keep its exit code and
+            // status message; this exit() only finishes the shutdown.
+            if (opts.statusMessage)
+                console.log(`[Scrapely] ${opts.statusMessage}`);
+            console.log(`[Scrapely] Actor.exit() after Actor.fail(): the run stays failed (exit code ${failure.exitCode}).`);
+            return Actor._exitWith({ ...opts, exitCode: failure.exitCode, statusMessage: undefined });
+        }
+        return Actor._exitWith(opts);
+    }
+    static async _exitWith(opts) {
         const code = opts.exitCode ?? 0;
         const timeoutSecs = opts.timeoutSecs ?? 30;
         const shouldExit = opts.exit ?? true;
-        Actor._ensureActorInit('exit');
         // The exit/fail message becomes the run's terminal status
         // message. Stored before teardown, while the run and process are alive.
         if (opts.statusMessage)
@@ -423,7 +443,13 @@ class Actor {
             : { exitCode: 1, ...messageOrOptions, ...options };
         console.error(`[Scrapely] Actor failed: ${opts.statusMessage ?? 'Unknown error'}`);
         await new Promise((resolve) => process.stderr.write('', () => resolve()));
-        await Actor.exit(opts);
+        Actor._ensureActorInit('fail');
+        if (Actor._failure) {
+            // Failed already: the first failure's code and message stay.
+            return Actor._exitWith({ ...opts, exitCode: Actor._failure.exitCode, statusMessage: undefined });
+        }
+        Actor._failure = { exitCode: opts.exitCode ?? 1, statusMessage: opts.statusMessage };
+        await Actor._exitWith({ ...opts, exitCode: opts.exitCode ?? 1 });
     }
     static async getInput() {
         Actor._ensureActorInit('getInput');

package/dist/storage/api-storage-client.d.ts

@@ -15,17 +15,25 @@ export declare class ApiStorageClientError extends Error {
 /**
  * Whether a non-2xx answer is retried. 429 always: the run API limiter and
  * dataset backpressure refuse it before anything is written. Idempotent
- * methods: also 500/502/503/504. Others (dataset push, KV PUT) only on a 503
- * that certainly was not processed: the rate limiter refused it
- * ("rate-limit-unavailable", before the handler), or HAProxy had no backend
- * (its own page, nothing forwarded). Any other 503 may mean "outcome unknown"
- * (e.g. a direct dataset write whose commit was not confirmed), and re-sending
- * could store the items twice.
+ * methods: also 500/502/503/504. Others (dataset push, KV PUT, named storage
+ * create) only on a 503 that certainly was not processed: the rate limiter
+ * refused it ("rate-limit-unavailable", before the handler), another request
+ * held the storage name ("storage-name-busy", nothing changed), or HAProxy had
+ * no backend (its own page, nothing forwarded). Any other 503 may mean
+ * "outcome unknown" (e.g. a direct dataset write whose commit was not
+ * confirmed), and re-sending could store the items twice.
  */
 export declare function isRetryableResponse(status: number, idempotent: boolean, errorType?: string, bodyText?: string): boolean;
 /** Default time budget for retrying one API call (429, 503, network errors). */
 export declare const DEFAULT_API_RETRY_WINDOW_MS = 300000;
-/** Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes. */
+/** A key-value store record URL (`.../key-value-stores/<id>/records/<key>`). */
+export declare function isKvRecordUrl(url: string): boolean;
+/**
+ * Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes.
+ * Record PUTs count as de-duplicated until a keyed record PUT was stored without the support
+ * header: re-applying the same body to the same record leaves the same record, so the worst
+ * case on an API that ignores the key is one extra counted write, while not re-sending fails the run.
+ */
 export declare function apiDedupesKeyedWrites(url?: string): boolean;
 /** Test hook: forget what was learned about the API, or force an answer for every origin. */
 export declare function _resetKeyedWriteSupport(value?: boolean): void;

package/dist/storage/api-storage-client.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ApiStorageClientFactory = exports.ApiRequestQueueClient = exports.ApiKeyValueStoreClient = exports.ApiDatasetClient = exports.BaseApiClient = exports.DEFAULT_API_RETRY_WINDOW_MS = exports.ApiStorageClientError = void 0;
 exports.isRetryableResponse = isRetryableResponse;
+exports.isKvRecordUrl = isKvRecordUrl;
 exports.apiDedupesKeyedWrites = apiDedupesKeyedWrites;
 exports._resetKeyedWriteSupport = _resetKeyedWriteSupport;
 exports.apiRetryWindowMs = apiRetryWindowMs;
@@ -139,7 +140,9 @@ function isApiStorageClientError(err) {
     return err instanceof ApiStorageClientError;
 }
 // PUT is left out: a KV record PUT is counted (and billed) as a write each
-// time it is applied, so a lost response must not be re-sent blindly.
+// time it is applied, so a lost response must not be re-sent blindly. Record
+// PUTs carry an Idempotency-Key instead (see isKvRecordUrl), which the API
+// de-duplicates, so they are re-sent after a lost answer (e.g. an edge 502).
 const IDEMPOTENT_METHODS = new Set(['GET', 'HEAD', 'DELETE', 'OPTIONS']);
 // HAProxy's own 503 page: no backend was available, so nothing was forwarded.
 const NO_BACKEND_MARKER = 'No server is available to handle this request';
@@ -147,12 +150,13 @@ const CONNECT_FAILURE_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN',
 /**
  * Whether a non-2xx answer is retried. 429 always: the run API limiter and
  * dataset backpressure refuse it before anything is written. Idempotent
- * methods: also 500/502/503/504. Others (dataset push, KV PUT) only on a 503
- * that certainly was not processed: the rate limiter refused it
- * ("rate-limit-unavailable", before the handler), or HAProxy had no backend
- * (its own page, nothing forwarded). Any other 503 may mean "outcome unknown"
- * (e.g. a direct dataset write whose commit was not confirmed), and re-sending
- * could store the items twice.
+ * methods: also 500/502/503/504. Others (dataset push, KV PUT, named storage
+ * create) only on a 503 that certainly was not processed: the rate limiter
+ * refused it ("rate-limit-unavailable", before the handler), another request
+ * held the storage name ("storage-name-busy", nothing changed), or HAProxy had
+ * no backend (its own page, nothing forwarded). Any other 503 may mean
+ * "outcome unknown" (e.g. a direct dataset write whose commit was not
+ * confirmed), and re-sending could store the items twice.
  */
 function isRetryableResponse(status, idempotent, errorType, bodyText) {
     if (status === 429)
@@ -161,7 +165,9 @@ function isRetryableResponse(status, idempotent, errorType, bodyText) {
         return status === 500 || status === 502 || status === 503 || status === 504;
     if (status !== 503)
         return false;
-    return errorType === 'rate-limit-unavailable' || (bodyText ?? '').includes(NO_BACKEND_MARKER);
+    return errorType === 'rate-limit-unavailable'
+        || errorType === 'storage-name-busy'
+        || (bodyText ?? '').includes(NO_BACKEND_MARKER);
 }
 /** Default time budget for retrying one API call (429, 503, network errors). */
 exports.DEFAULT_API_RETRY_WINDOW_MS = 300_000;
@@ -177,20 +183,39 @@ const IDEMPOTENCY_SUPPORT_HEADER = 'x-scrapely-idempotency';
 const keyedWritesDeduped = new Map();
 /** Test override for every origin (undefined: learn from answers). */
 let keyedWritesDedupedOverride;
-function originOf(url) {
+/** A key-value store record URL (`.../key-value-stores/<id>/records/<key>`). */
+function isKvRecordUrl(url) {
+    let path = url;
     try {
-        return new URL(url).origin;
+        path = new URL(url).pathname;
     }
-    catch {
-        return url;
+    catch { /* a path already */ }
+    return path.includes('/key-value-stores/') && path.includes('/records/');
+}
+/** What is learned per API: its origin, separately for record PUTs and other keyed writes. */
+function originOf(url) {
+    let origin = url;
+    try {
+        origin = new URL(url).origin;
     }
+    catch { /* keep the input */ }
+    return isKvRecordUrl(url) ? `${origin}#kv-record` : origin;
 }
-/** Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes. */
+/**
+ * Whether the API at `url` (any URL of it; default: the last one seen) de-duplicates keyed writes.
+ * Record PUTs count as de-duplicated until a keyed record PUT was stored without the support
+ * header: re-applying the same body to the same record leaves the same record, so the worst
+ * case on an API that ignores the key is one extra counted write, while not re-sending fails the run.
+ */
 function apiDedupesKeyedWrites(url) {
     if (keyedWritesDedupedOverride !== undefined)
         return keyedWritesDedupedOverride;
-    if (url !== undefined)
-        return keyedWritesDeduped.get(originOf(url)) === true;
+    if (url !== undefined) {
+        const learned = keyedWritesDeduped.get(originOf(url));
+        if (learned === undefined && isKvRecordUrl(url))
+            return true;
+        return learned === true;
+    }
     return [...keyedWritesDeduped.values()].pop() === true;
 }
 /** Test hook: forget what was learned about the API, or force an answer for every origin. */
@@ -450,7 +475,8 @@ async function postStorageCollection(config, path, name) {
     // Not marked repeatable even by name: a 500 after the create (e.g. scoped
     // token registration failed) must surface, since a re-send would take the
     // existing-name path and hide it. POST rules apply (limiter/HAProxy 503,
-    // refused connection).
+    // "storage-name-busy" 503 while a parallel create holds the name, refused
+    // connection).
     return new CollectionApiClient(config).call('POST', `${path}${query}`);
 }
 /**
@@ -788,10 +814,13 @@ class ApiKeyValueStoreClient extends BaseApiClient {
             ct = 'application/json';
         }
         // Pass timeout/retry options from KeyValueStoreRecordOptions to the API request
-        const requestOptions = options ? {
-            timeoutSecs: options.timeoutSecs,
-            doNotRetryTimeouts: options.doNotRetryTimeouts,
-        } : undefined;
+        // One Idempotency-Key for every attempt of this write: a re-send after a lost answer
+        // (edge 502/504, dropped connection) is applied and counted once.
+        const requestOptions = {
+            timeoutSecs: options?.timeoutSecs,
+            doNotRetryTimeouts: options?.doNotRetryTimeouts,
+            idempotencyKey: (0, node_crypto_1.randomUUID)(),
+        };
         await this.request('PUT', `/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`, body, contentType || ct, requestOptions);
     }
     async deleteRecord(key) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.36",
+    "version": "1.2.38",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -69,12 +69,14 @@
         "test:run-control": "npx tsx --test tests/run-control.test.ts",
         "test:rq-list": "npx tsx --test tests/rq-list-requests.test.ts",
         "test:terminal-status": "npx tsx --test tests/terminal-status-message.test.ts",
+        "test:fail-then-exit": "npx tsx --test tests/fail-then-exit.test.ts",
         "test:cdp-retry": "npx tsx --test tests/cdp-retry.test.ts",
         "test:proxy-input-aliases": "npx tsx --test tests/proxy-input-aliases.test.ts",
         "test:charge-retries": "npx tsx --test tests/charge-retries.test.ts",
         "test:solver": "npx tsx --test tests/solver.test.ts",
         "test:lazy-preload": "npx tsx --test tests/lazy-preload.test.ts",
-        "test:v2-utils": "npx tsx --test tests/v2-utils.test.ts"
+        "test:v2-utils": "npx tsx --test tests/v2-utils.test.ts",
+        "test:default-api-url": "npx tsx --test tests/default-api-url.test.ts"
     },
     "author": "TeraShift GmbH (Scrape.ly)",
     "license": "ISC AND Apache-2.0",
@@ -91,7 +93,7 @@
         "lodash": "^4.18.1",
         "mysql2": "^3.23.1",
         "playwright": "^1.58.2",
-        "scrapely-client": "^2.25.2",
+        "scrapely-client": "^2.25.9",
         "tar-fs": "2.1.5",
         "ws": "^8.18.0"
     },