scrapely 1.2.35 → 1.2.37

package/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 All notable changes to this project will be documented in this file.
 
+## [Unreleased]
+
+## [1.2.37] - 2026-10-26
+
+- Opening a named dataset, key-value store or request queue (`Actor.openDataset('name')` and the like) retries when the API answers `503 storage-name-busy`. The API sends it when another request kept creating or renaming a storage of that name for over 15 s; nothing was changed, so the open is simply repeated. It used to fail the open.
+- Once `Actor.fail()` has run, the run stays failed: a later `Actor.exit()`, or the end of `Actor.main()`, keeps the failure's exit code and status message. `fail()` normally ends the process at once, so this only changed runs that called `fail({ exit: false })` and then `exit()` or returned from `Actor.main()`; they used to exit with 0. A second `fail()` keeps the first failure. `fail({ exitCode: undefined })` exits with 1.
+
+## [1.2.36] - 2026-10-22
+
+- Debug captures (`Actor.debugCapture`, `Actor.debugCaptureSync`) keep payloads over 256 KiB whole. They are stored gzip-compressed and base64-encoded as `{ key, payload, encoding: 'gzip-base64', length, truncated }` instead of being cut at 262,144 characters; up to 8 Mi characters are kept. A payload that does not compress enough keeps its first 256 KiB, now marked `truncated: true` with its full `length`. `Uint8Array` / `Buffer` payloads are stored as UTF-8 text. Captures are only active on the platform's own diagnostic runs (`SCRAPELY_DEBUG_CAPTURE=1`).
+
 ## [1.2.35] - 2026-10-20
 
 - The storage client percent-encodes request ids and key-value store record keys as one URL path segment and refuses `''`, `.` and `..`. A request id such as `..` read from a queue used to send the call to another resource.

package/dist/actor.d.ts

@@ -332,6 +332,12 @@ export declare class Actor {
      * succeeds.
      */
     private _lifecycleStarted;
+    /**
+     * Set by {@link Actor.fail} for the rest of the run: a later {@link Actor.exit}
+     * keeps the failure's exit code and status message, so a failed run never
+     * ends as SUCCEEDED (e.g. `fail({ exit: false })` followed by `exit()`).
+     */
+    private static _failure;
     /** Created on first use: scrapely-client is only loaded when something needs the API client. */
     private _apifyClient;
     private readonly chargingManager;
@@ -388,6 +394,7 @@ export declare class Actor {
      * @param options - Exit options
      */
     static exit(messageOrOptions?: string | ExitOptions, options?: ExitOptions): Promise<void>;
+    private static _exitWith;
     static fail(messageOrOptions?: string | ExitOptions, options?: ExitOptions): Promise<void>;
     static getInput<T = ActorInput>(): Promise<T | null>;
     static getInputOrThrow<T = ActorInput>(): Promise<T>;
@@ -459,7 +466,15 @@ export declare class Actor {
      */
     static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
     private static _debugCaptureCount;
+    /**
+     * A payload up to DEBUG_CAPTURE_MAX_BYTES characters is stored as it is; a larger one gzip-compressed
+     * (base64, at most DEBUG_CAPTURE_MAX_BYTES characters again), keeping up to DEBUG_CAPTURE_MAX_CHARS
+     * characters / DEBUG_CAPTURE_MAX_ENCODED bytes of it. A plain cut at 256 KiB lost the part a check
+     * needed when a large API document starts with site navigation.
+     */
     static readonly DEBUG_CAPTURE_MAX_BYTES: number;
+    static readonly DEBUG_CAPTURE_MAX_CHARS: number;
+    static readonly DEBUG_CAPTURE_MAX_ENCODED: number;
     static readonly DEBUG_CAPTURE_MAX_RECORDS = 500;
     /** The key-value store record name for a capture key (item URL or id). */
     static debugCaptureRecordKey(key: string): Promise<string>;
@@ -471,6 +486,8 @@ export declare class Actor {
     /** debugCapture for synchronous code: fire and forget. Does nothing unless capture is enabled for the run. */
     static debugCaptureSync(key: string, payload: unknown): void;
     static debugCapture(key: string, payload: unknown): Promise<void>;
+    /** The record for one capture: plain when small, else gzip-base64 (plain head as the last resort). */
+    static debugCaptureRecord(key: string, text: string): Promise<Record<string, unknown>>;
     /**
      * Helper to resolve storage identifier to a string ID or name.
      * Supports: string, { id: '...' }, { name: '...' }, { alias: '...' }

package/dist/actor.js

@@ -112,6 +112,12 @@ class Actor {
      * succeeds.
      */
     _lifecycleStarted = false;
+    /**
+     * Set by {@link Actor.fail} for the rest of the run: a later {@link Actor.exit}
+     * keeps the failure's exit code and status message, so a failed run never
+     * ends as SUCCEEDED (e.g. `fail({ exit: false })` followed by `exit()`).
+     */
+    static _failure;
     /** Created on first use: scrapely-client is only loaded when something needs the API client. */
     _apifyClient;
     chargingManager;
@@ -189,6 +195,7 @@ class Actor {
         if (this.initialized)
             return;
         cachedInput = undefined; // a fresh init reads this run's input again
+        Actor._failure = undefined; // a new lifecycle starts unfailed
         // NOTE: `initialized` is set at the END of init — a failed platform init
         // must stay retryable instead of leaving a half-initialized Actor.
         Actor.installFatalChargeHandlers();
@@ -375,7 +382,8 @@ class Actor {
             const result = await userFunc();
             await Actor._shutdownActor(options?.timeoutSecs ?? 30);
             await new Promise((resolve) => process.stdout.write('', () => resolve()));
-            process.exit(options?.exitCode ?? 0);
+            // A run that called fail({ exit: false }) stays failed.
+            process.exit(Actor._failure?.exitCode ?? options?.exitCode ?? 0);
             return result;
         }
         catch (err) {
@@ -403,10 +411,22 @@ class Actor {
         const opts = typeof messageOrOptions === 'string'
             ? { ...options, statusMessage: messageOrOptions }
             : { ...messageOrOptions, ...options };
+        Actor._ensureActorInit('exit');
+        const failure = Actor._failure;
+        if (failure) {
+            // Actor.fail() already ran: the run stays failed. Keep its exit code and
+            // status message; this exit() only finishes the shutdown.
+            if (opts.statusMessage)
+                console.log(`[Scrapely] ${opts.statusMessage}`);
+            console.log(`[Scrapely] Actor.exit() after Actor.fail(): the run stays failed (exit code ${failure.exitCode}).`);
+            return Actor._exitWith({ ...opts, exitCode: failure.exitCode, statusMessage: undefined });
+        }
+        return Actor._exitWith(opts);
+    }
+    static async _exitWith(opts) {
         const code = opts.exitCode ?? 0;
         const timeoutSecs = opts.timeoutSecs ?? 30;
         const shouldExit = opts.exit ?? true;
-        Actor._ensureActorInit('exit');
         // The exit/fail message becomes the run's terminal status
         // message. Stored before teardown, while the run and process are alive.
         if (opts.statusMessage)
@@ -423,7 +443,13 @@ class Actor {
             : { exitCode: 1, ...messageOrOptions, ...options };
         console.error(`[Scrapely] Actor failed: ${opts.statusMessage ?? 'Unknown error'}`);
         await new Promise((resolve) => process.stderr.write('', () => resolve()));
-        await Actor.exit(opts);
+        Actor._ensureActorInit('fail');
+        if (Actor._failure) {
+            // Failed already: the first failure's code and message stay.
+            return Actor._exitWith({ ...opts, exitCode: Actor._failure.exitCode, statusMessage: undefined });
+        }
+        Actor._failure = { exitCode: opts.exitCode ?? 1, statusMessage: opts.statusMessage };
+        await Actor._exitWith({ ...opts, exitCode: opts.exitCode ?? 1 });
     }
     static async getInput() {
         Actor._ensureActorInit('getInput');
@@ -660,7 +686,15 @@ class Actor {
         await store.setValue(key, value, options);
     }
     static _debugCaptureCount = 0;
+    /**
+     * A payload up to DEBUG_CAPTURE_MAX_BYTES characters is stored as it is; a larger one gzip-compressed
+     * (base64, at most DEBUG_CAPTURE_MAX_BYTES characters again), keeping up to DEBUG_CAPTURE_MAX_CHARS
+     * characters / DEBUG_CAPTURE_MAX_ENCODED bytes of it. A plain cut at 256 KiB lost the part a check
+     * needed when a large API document starts with site navigation.
+     */
     static DEBUG_CAPTURE_MAX_BYTES = 256 * 1024;
+    static DEBUG_CAPTURE_MAX_CHARS = 8 * 1024 * 1024;
+    static DEBUG_CAPTURE_MAX_ENCODED = 8 * 1024 * 1024;
     static DEBUG_CAPTURE_MAX_RECORDS = 500;
     /** The key-value store record name for a capture key (item URL or id). */
     static async debugCaptureRecordKey(key) {
@@ -684,14 +718,32 @@ class Actor {
         if (Actor._debugCaptureCount >= Actor.DEBUG_CAPTURE_MAX_RECORDS)
             return;
         try {
-            const text = (typeof payload === 'string' ? payload : JSON.stringify(payload) ?? '').slice(0, Actor.DEBUG_CAPTURE_MAX_BYTES);
+            const text = typeof payload === 'string' ? payload
+                : payload instanceof Uint8Array ? Buffer.from(payload.buffer, payload.byteOffset, payload.byteLength).toString('utf8')
+                    : JSON.stringify(payload) ?? '';
+            const record = await Actor.debugCaptureRecord(String(key), text);
             Actor._debugCaptureCount++;
-            await Actor.setValue(await Actor.debugCaptureRecordKey(key), { key: String(key), payload: text });
+            await Actor.setValue(await Actor.debugCaptureRecordKey(key), record);
         }
         catch {
             // never break the run
         }
     }
+    /** The record for one capture: plain when small, else gzip-base64 (plain head as the last resort). */
+    static async debugCaptureRecord(key, text) {
+        const limit = Actor.DEBUG_CAPTURE_MAX_BYTES;
+        if (text.length <= limit)
+            return { key, payload: text };
+        const { gzipSync } = await import('node:zlib');
+        const raw = Buffer.from(text.slice(0, Actor.DEBUG_CAPTURE_MAX_CHARS), 'utf8');
+        const kept = raw.subarray(0, Actor.DEBUG_CAPTURE_MAX_ENCODED);
+        const packed = gzipSync(kept, { level: 6 }).toString('base64');
+        if (packed.length <= limit) {
+            const truncated = text.length > Actor.DEBUG_CAPTURE_MAX_CHARS || kept.length < raw.length;
+            return { key, payload: packed, encoding: 'gzip-base64', length: text.length, truncated };
+        }
+        return { key, payload: text.slice(0, limit), length: text.length, truncated: true };
+    }
     /**
      * Helper to resolve storage identifier to a string ID or name.
      * Supports: string, { id: '...' }, { name: '...' }, { alias: '...' }

package/dist/storage/api-storage-client.d.ts

@@ -15,12 +15,13 @@ export declare class ApiStorageClientError extends Error {
 /**
  * Whether a non-2xx answer is retried. 429 always: the run API limiter and
  * dataset backpressure refuse it before anything is written. Idempotent
- * methods: also 500/502/503/504. Others (dataset push, KV PUT) only on a 503
- * that certainly was not processed: the rate limiter refused it
- * ("rate-limit-unavailable", before the handler), or HAProxy had no backend
- * (its own page, nothing forwarded). Any other 503 may mean "outcome unknown"
- * (e.g. a direct dataset write whose commit was not confirmed), and re-sending
- * could store the items twice.
+ * methods: also 500/502/503/504. Others (dataset push, KV PUT, named storage
+ * create) only on a 503 that certainly was not processed: the rate limiter
+ * refused it ("rate-limit-unavailable", before the handler), another request
+ * held the storage name ("storage-name-busy", nothing changed), or HAProxy had
+ * no backend (its own page, nothing forwarded). Any other 503 may mean
+ * "outcome unknown" (e.g. a direct dataset write whose commit was not
+ * confirmed), and re-sending could store the items twice.
  */
 export declare function isRetryableResponse(status: number, idempotent: boolean, errorType?: string, bodyText?: string): boolean;
 /** Default time budget for retrying one API call (429, 503, network errors). */

package/dist/storage/api-storage-client.js

@@ -147,12 +147,13 @@ const CONNECT_FAILURE_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN',
 /**
  * Whether a non-2xx answer is retried. 429 always: the run API limiter and
  * dataset backpressure refuse it before anything is written. Idempotent
- * methods: also 500/502/503/504. Others (dataset push, KV PUT) only on a 503
- * that certainly was not processed: the rate limiter refused it
- * ("rate-limit-unavailable", before the handler), or HAProxy had no backend
- * (its own page, nothing forwarded). Any other 503 may mean "outcome unknown"
- * (e.g. a direct dataset write whose commit was not confirmed), and re-sending
- * could store the items twice.
+ * methods: also 500/502/503/504. Others (dataset push, KV PUT, named storage
+ * create) only on a 503 that certainly was not processed: the rate limiter
+ * refused it ("rate-limit-unavailable", before the handler), another request
+ * held the storage name ("storage-name-busy", nothing changed), or HAProxy had
+ * no backend (its own page, nothing forwarded). Any other 503 may mean
+ * "outcome unknown" (e.g. a direct dataset write whose commit was not
+ * confirmed), and re-sending could store the items twice.
  */
 function isRetryableResponse(status, idempotent, errorType, bodyText) {
     if (status === 429)
@@ -161,7 +162,9 @@ function isRetryableResponse(status, idempotent, errorType, bodyText) {
         return status === 500 || status === 502 || status === 503 || status === 504;
     if (status !== 503)
         return false;
-    return errorType === 'rate-limit-unavailable' || (bodyText ?? '').includes(NO_BACKEND_MARKER);
+    return errorType === 'rate-limit-unavailable'
+        || errorType === 'storage-name-busy'
+        || (bodyText ?? '').includes(NO_BACKEND_MARKER);
 }
 /** Default time budget for retrying one API call (429, 503, network errors). */
 exports.DEFAULT_API_RETRY_WINDOW_MS = 300_000;
@@ -450,7 +453,8 @@ async function postStorageCollection(config, path, name) {
     // Not marked repeatable even by name: a 500 after the create (e.g. scoped
     // token registration failed) must surface, since a re-send would take the
     // existing-name path and hide it. POST rules apply (limiter/HAProxy 503,
-    // refused connection).
+    // "storage-name-busy" 503 while a parallel create holds the name, refused
+    // connection).
     return new CollectionApiClient(config).call('POST', `${path}${query}`);
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.35",
+    "version": "1.2.37",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -48,6 +48,7 @@
         "test:charge-platform-budget": "npx tsx --test tests/charge-platform-budget.test.ts",
         "test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",
         "test:storage-path-segments": "npx tsx --test tests/storage-path-segments.test.ts",
+        "test:storage-record-keys": "npx tsx --test tests/storage-record-keys.test.ts",
         "test:push-idempotency": "npx tsx --test tests/push-idempotency.test.ts",
         "test:stdio-drain": "npx tsx --test tests/stdio-drain.test.ts",
         "test:charging-client": "npx tsx --test tests/charging-client.test.ts",
@@ -68,6 +69,7 @@
         "test:run-control": "npx tsx --test tests/run-control.test.ts",
         "test:rq-list": "npx tsx --test tests/rq-list-requests.test.ts",
         "test:terminal-status": "npx tsx --test tests/terminal-status-message.test.ts",
+        "test:fail-then-exit": "npx tsx --test tests/fail-then-exit.test.ts",
         "test:cdp-retry": "npx tsx --test tests/cdp-retry.test.ts",
         "test:proxy-input-aliases": "npx tsx --test tests/proxy-input-aliases.test.ts",
         "test:charge-retries": "npx tsx --test tests/charge-retries.test.ts",