scrapely 1.2.11 → 1.2.12

package/dist/actor.d.ts

@@ -517,6 +517,11 @@ export declare class Actor {
     /** One-shot bounded reconciliation on fatal process errors. */
     private static installFatalChargeHandlers;
     private static _shutdownActor;
+    /**
+     * Store the exit/fail status message on the run as terminal. Best effort and
+     * bounded: a failing or slow API is logged and never blocks the exit.
+     */
+    private static _persistTerminalStatusMessage;
     /**
      * Sets the status message for the current Actor run.
      *

package/dist/actor.js

@@ -382,9 +382,11 @@ class Actor {
         const code = opts.exitCode ?? 0;
         const timeoutSecs = opts.timeoutSecs ?? 30;
         const shouldExit = opts.exit ?? true;
-        if (opts.statusMessage)
-            console.log(`[Scrapely] ${opts.statusMessage}`);
         Actor._ensureActorInit('exit');
+        // Apify parity: the exit/fail message becomes the run's terminal status
+        // message. Stored before teardown, while the run and process are alive.
+        if (opts.statusMessage)
+            await Actor._persistTerminalStatusMessage(opts.statusMessage, code);
         await Actor._shutdownActor(timeoutSecs);
         if (shouldExit) {
             await new Promise((resolve) => process.stdout.write('', () => resolve()));
@@ -949,6 +951,31 @@ class Actor {
         // drain, usually within milliseconds, and is still bounded.
         await waitForStdioDrain(Math.min(1000, flushDeadlineMs));
     }
+    /**
+     * Store the exit/fail status message on the run as terminal. Best effort and
+     * bounded: a failing or slow API is logged and never blocks the exit.
+     */
+    static async _persistTerminalStatusMessage(message, exitCode, timeoutMs = 10_000) {
+        let timer;
+        try {
+            await Promise.race([
+                Actor.setStatusMessage(message, {
+                    isStatusMessageTerminal: true,
+                    level: exitCode > 0 ? 'ERROR' : 'INFO',
+                }),
+                new Promise((_, reject) => {
+                    timer = setTimeout(() => reject(new Error(`timed out after ${timeoutMs}ms`)), timeoutMs);
+                }),
+            ]);
+        }
+        catch (err) {
+            console.warn(`[Scrapely] Failed to store the final status message on the run: ${err.message}`);
+        }
+        finally {
+            if (timer)
+                clearTimeout(timer);
+        }
+    }
     /**
      * Sets the status message for the current Actor run.
      *
@@ -1000,6 +1027,7 @@ class Actor {
                 statusMessage: message,
                 isStatusMessageTerminal,
             }),
+            signal: AbortSignal.timeout(10_000),
         });
         if (!response.ok) {
             const error = await response.text();

package/dist/cdp-retry.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Retry remote-browser (CDP) connects that fail for transient reasons.
+ *
+ * The remote browser answers 429 when it is at capacity and 503 when it cannot
+ * start a browser or validate the session right now, both with a Retry-After
+ * delay (repeated in the response body, because Playwright only puts the
+ * status line and body into its error message). Connection errors and
+ * timeouts are transient too. Those are retried with exponential backoff,
+ * waiting at least the Retry-After delay when the error carries one. Any other
+ * status (400/401/402/403/404) fails at once.
+ *
+ * Tuning (environment):
+ * - SCRAPELY_CDP_CONNECT_ATTEMPTS: total attempts, default 5 (1 disables retries).
+ * - SCRAPELY_CDP_RETRY_BASE_SECS: first backoff delay, default 1; doubles per attempt.
+ * - SCRAPELY_CDP_RETRY_MAX_SECS: cap for one backoff delay, default 30.
+ * - SCRAPELY_CDP_RETRY_AFTER_MAX_SECS: longest Retry-After the SDK waits out,
+ *   default 120. A longer Retry-After ends the retries (the SDK never retries
+ *   earlier than the server asked).
+ */
+/** @internal Exported for tests. */
+export declare function classifyConnectError(err: unknown): {
+    retryable: boolean;
+    status?: number;
+    retryAfterSecs?: number;
+};
+/** @internal Delay in ms before retry number `attempt` (1-based), honoring Retry-After. */
+export declare function backoffDelayMs(attempt: number, retryAfterSecs?: number): number;
+/** @internal Overridable for tests. */
+export declare const retryTiming: {
+    sleep: (ms: number) => Promise<void>;
+};
+/** Run a CDP connect, retrying transient failures with backoff. */
+export declare function connectWithRetry<T>(connect: () => Promise<T>): Promise<T>;
+/** True when `url` points at the same host as the platform's CDP_URL. */
+export declare function isPlatformCdpUrl(url: unknown, platformCdpUrl: string | undefined): boolean;
+export declare function isRetryWrapped(fn: unknown): boolean;
+/**
+ * Make `target[method]` (Playwright `connectOverCDP` or Puppeteer `connect`)
+ * retry transient failures when it connects to the platform CDP host. Covers
+ * actor code that connects to CDP_URL directly as well as the SDK's own
+ * redirected launches. Other hosts are left alone. Idempotent.
+ */
+export declare function wrapConnectWithRetry(target: any, method: 'connectOverCDP' | 'connect', getPlatformCdpUrl: () => string | undefined): void;

package/dist/cdp-retry.js

@@ -0,0 +1,171 @@
+"use strict";
+/**
+ * Retry remote-browser (CDP) connects that fail for transient reasons.
+ *
+ * The remote browser answers 429 when it is at capacity and 503 when it cannot
+ * start a browser or validate the session right now, both with a Retry-After
+ * delay (repeated in the response body, because Playwright only puts the
+ * status line and body into its error message). Connection errors and
+ * timeouts are transient too. Those are retried with exponential backoff,
+ * waiting at least the Retry-After delay when the error carries one. Any other
+ * status (400/401/402/403/404) fails at once.
+ *
+ * Tuning (environment):
+ * - SCRAPELY_CDP_CONNECT_ATTEMPTS: total attempts, default 5 (1 disables retries).
+ * - SCRAPELY_CDP_RETRY_BASE_SECS: first backoff delay, default 1; doubles per attempt.
+ * - SCRAPELY_CDP_RETRY_MAX_SECS: cap for one backoff delay, default 30.
+ * - SCRAPELY_CDP_RETRY_AFTER_MAX_SECS: longest Retry-After the SDK waits out,
+ *   default 120. A longer Retry-After ends the retries (the SDK never retries
+ *   earlier than the server asked).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.retryTiming = void 0;
+exports.classifyConnectError = classifyConnectError;
+exports.backoffDelayMs = backoffDelayMs;
+exports.connectWithRetry = connectWithRetry;
+exports.isPlatformCdpUrl = isPlatformCdpUrl;
+exports.isRetryWrapped = isRetryWrapped;
+exports.wrapConnectWithRetry = wrapConnectWithRetry;
+const RETRY_WRAPPED = Symbol.for('scrapely.cdpRetryWrapped');
+const REASONS = 'Too Many Requests|Internal Server Error|Bad Gateway|Service Unavailable|Gateway Timeout|' +
+    'Request Timeout|Bad Request|Unauthorized|Payment Required|Forbidden|Not Found';
+const STATUS_RE = new RegExp(`\\b([1-5]\\d\\d) (?:${REASONS})\\b`);
+const WS_STATUS_RE = /Unexpected server response:?\s*(\d{3})/i;
+const RETRY_AFTER_RE = /Retry-After:?\s*(\d+(?:\.\d+)?)/i;
+const TRANSIENT_MARKERS = [
+    'econnrefused',
+    'econnreset',
+    'etimedout',
+    'epipe',
+    'eai_again',
+    'ehostunreach',
+    'enetunreach',
+    'socket hang up',
+    'timeout',
+    'timed out',
+    'connection closed',
+    'connection refused',
+    'connection reset',
+    'websocket error',
+    'net::err_',
+    'target closed',
+];
+function envNumber(name, fallback) {
+    const value = Number(process.env[name]);
+    return Number.isFinite(value) && value > 0 ? value : fallback;
+}
+function maxAttempts() {
+    return Math.max(1, Math.floor(envNumber('SCRAPELY_CDP_CONNECT_ATTEMPTS', 5)));
+}
+function errorText(err) {
+    if (err instanceof Error) {
+        const code = err.code;
+        return `${err.name}: ${err.message}${code ? ` ${code}` : ''}`;
+    }
+    return String(err);
+}
+/** @internal Exported for tests. */
+function classifyConnectError(err) {
+    const text = errorText(err);
+    const statusMatch = WS_STATUS_RE.exec(text) ?? STATUS_RE.exec(text);
+    const retryAfterMatch = RETRY_AFTER_RE.exec(text);
+    const retryAfterSecs = retryAfterMatch ? Number(retryAfterMatch[1]) : undefined;
+    if (statusMatch) {
+        const status = Number(statusMatch[1]);
+        return { retryable: status === 408 || status === 429 || status >= 500, status, retryAfterSecs };
+    }
+    const lowered = text.toLowerCase();
+    return { retryable: TRANSIENT_MARKERS.some((m) => lowered.includes(m)), retryAfterSecs };
+}
+/** @internal Delay in ms before retry number `attempt` (1-based), honoring Retry-After. */
+function backoffDelayMs(attempt, retryAfterSecs) {
+    const base = envNumber('SCRAPELY_CDP_RETRY_BASE_SECS', 1);
+    const cap = envNumber('SCRAPELY_CDP_RETRY_MAX_SECS', 30);
+    let delay = Math.min(cap, base * 2 ** (attempt - 1)) * (0.75 + Math.random() * 0.5);
+    if (retryAfterSecs !== undefined)
+        delay = Math.max(delay, retryAfterSecs);
+    return Math.round(delay * 1000);
+}
+function describe(err, status) {
+    if (status !== undefined)
+        return `HTTP ${status}`;
+    // First line only, URL masked: connect errors can carry the session URL with ids.
+    const first = (err instanceof Error ? err.message : String(err)).trim().split('\n')[0] || 'error';
+    return first.replace(/(wss?|https?):\/\/\S+/g, '<cdp-url>').slice(0, 200);
+}
+/** @internal Overridable for tests. */
+exports.retryTiming = {
+    sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+};
+/** Run a CDP connect, retrying transient failures with backoff. */
+async function connectWithRetry(connect) {
+    const attempts = maxAttempts();
+    for (let attempt = 1;; attempt++) {
+        try {
+            return await connect();
+        }
+        catch (err) {
+            const { retryable, status, retryAfterSecs } = classifyConnectError(err);
+            if (!retryable || attempt >= attempts)
+                throw err;
+            if (retryAfterSecs !== undefined && retryAfterSecs > envNumber('SCRAPELY_CDP_RETRY_AFTER_MAX_SECS', 120)) {
+                // Never retry earlier than the server asked; give up instead of waiting that long.
+                throw err;
+            }
+            const delayMs = backoffDelayMs(attempt, retryAfterSecs);
+            console.warn(`[scrapely] Remote browser connect failed (${describe(err, status)}); ` +
+                `retrying in ${(delayMs / 1000).toFixed(1)}s (attempt ${attempt + 1}/${attempts})`);
+            await exports.retryTiming.sleep(delayMs);
+        }
+    }
+}
+/** True when `url` points at the same host as the platform's CDP_URL. */
+function isPlatformCdpUrl(url, platformCdpUrl) {
+    if (typeof url !== 'string' || !platformCdpUrl)
+        return false;
+    try {
+        const target = new URL(url);
+        const platform = new URL(platformCdpUrl);
+        return !!target.host && target.host.toLowerCase() === platform.host.toLowerCase();
+    }
+    catch {
+        return false;
+    }
+}
+function connectTarget(arg) {
+    if (typeof arg === 'string')
+        return arg;
+    if (arg && typeof arg === 'object') {
+        const o = arg;
+        return o.endpointURL ?? o.wsEndpoint ?? o.browserWSEndpoint ?? o.browserURL;
+    }
+    return undefined;
+}
+function isRetryWrapped(fn) {
+    return typeof fn === 'function' && fn[RETRY_WRAPPED] === true;
+}
+/**
+ * Make `target[method]` (Playwright `connectOverCDP` or Puppeteer `connect`)
+ * retry transient failures when it connects to the platform CDP host. Covers
+ * actor code that connects to CDP_URL directly as well as the SDK's own
+ * redirected launches. Other hosts are left alone. Idempotent.
+ */
+function wrapConnectWithRetry(target, method, getPlatformCdpUrl) {
+    const original = target?.[method];
+    if (typeof original !== 'function' || isRetryWrapped(original))
+        return;
+    const wrapped = function (...args) {
+        const self = this ?? target;
+        if (!isPlatformCdpUrl(connectTarget(args[0]), getPlatformCdpUrl())) {
+            return original.apply(self, args);
+        }
+        return connectWithRetry(() => original.apply(self, args));
+    };
+    wrapped[RETRY_WRAPPED] = true;
+    try {
+        target[method] = wrapped;
+    }
+    catch {
+        // frozen object: leave it unwrapped
+    }
+}

package/dist/patch-browser-pool.js

@@ -18,6 +18,7 @@ exports.patchRawBrowserType = patchRawBrowserType;
  */
 const browser_configuration_js_1 = require("./browser-configuration.js");
 const browser_locations_js_1 = require("./browser-locations.js");
+const cdp_retry_js_1 = require("./cdp-retry.js");
 let _cdpUrl;
 /**
  * Lazily get CDP URL from environment or cached value.
@@ -321,6 +322,7 @@ function patchPlaywrightPluginClass(PlaywrightPlugin) {
         }
         const cdpUrl = applyProxyInfoToCdpUrl(requireCdpUrl(), proxyInfoFromLaunchContext(launchContext), launchContext?.browserConfiguration);
         const chromium = getPlaywrightChromium(this.library);
+        (0, cdp_retry_js_1.wrapConnectWithRetry)(chromium, 'connectOverCDP', getCdpUrl);
         return chromium.connectOverCDP(cdpUrl);
     };
     proto._scrapelyCdpPatched = true;
@@ -344,6 +346,7 @@ function patchPuppeteerPluginClass(PuppeteerPlugin) {
         if (!launcher || typeof launcher.connect !== 'function') {
             throw new Error('Puppeteer library is missing connect(). Install puppeteer or puppeteer-core.');
         }
+        (0, cdp_retry_js_1.wrapConnectWithRetry)(launcher, 'connect', getCdpUrl);
         return launcher.connect({ browserWSEndpoint: cdpUrl });
     };
     proto._scrapelyCdpPatched = true;
@@ -669,6 +672,10 @@ function patchRawBrowserType(chromium, browserType, name) {
 function patchRawPlaywrightModule(pw) {
     if (!pw?.chromium || typeof pw.chromium.connectOverCDP !== 'function')
         return;
+    // Retry transient connect failures (429/5xx/network) for the platform CDP
+    // host: covers direct chromium.connectOverCDP(CDP_URL) calls in actor code
+    // and the redirected launches, which call this same property.
+    (0, cdp_retry_js_1.wrapConnectWithRetry)(pw.chromium, 'connectOverCDP', getCdpUrl);
     for (const name of ['chromium', 'firefox', 'webkit']) {
         patchRawBrowserType(pw.chromium, pw[name], name);
     }
@@ -683,6 +690,7 @@ function patchRawPuppeteerModule(mod, moduleName) {
     if (typeof launcher?.launch !== 'function' || launcher._scrapelyRawCdpPatched) {
         return;
     }
+    (0, cdp_retry_js_1.wrapConnectWithRetry)(launcher, 'connect', getCdpUrl);
     const origLaunch = launcher.launch.bind(launcher);
     launcher.launch = async function rawPuppeteerLaunchPatched(launchOptions = {}) {
         const { options, browserConfiguration } = takeBrowserConfiguration(launchOptions);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scrapely",
-    "version": "1.2.11",
+    "version": "1.2.12",
     "description": "Scrapely SDK for building JavaScript and TypeScript Actors",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -36,7 +36,9 @@
         "test:init-rollback": "npx tsx --test tests/init-rollback.test.ts",
         "test:apify-parity": "npx tsx --test tests/apify-parity.test.ts",
         "test:run-control": "npx tsx --test tests/run-control.test.ts",
-        "test:rq-list": "npx tsx --test tests/rq-list-requests.test.ts"
+        "test:rq-list": "npx tsx --test tests/rq-list-requests.test.ts",
+        "test:terminal-status": "npx tsx --test tests/terminal-status-message.test.ts",
+        "test:cdp-retry": "npx tsx --test tests/cdp-retry.test.ts"
     },
     "author": "SandeepDev1",
     "license": "ISC",