scrapely-python 0.1.5__py3-none-any.whl → 0.1.6__py3-none-any.whl

scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.5'
+__version__ = '0.1.6'
 __all__ = [
     # Core
     'Actor',

scrapely/actor.py

@@ -18,7 +18,7 @@ import warnings
 from collections.abc import MutableMapping
 from datetime import datetime, timedelta
 from decimal import Decimal
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union, cast
 
 from crawlee.configuration import Configuration as CrawleeConfiguration
 from crawlee.storage_clients._base import StorageClient
@@ -287,6 +287,7 @@ class Actor(metaclass=_ActorMeta):
             cls._exit_code = self._instance_exit_code
         if self._instance_status_message:
             cls._status_message = self._instance_status_message
+            await cls._persist_terminal_status_message(self._instance_status_message)
         await cls._shutdown(timeout_secs=int(self._cleanup_timeout.total_seconds()))
 
     @classmethod
@@ -589,6 +590,7 @@ class Actor(metaclass=_ActorMeta):
         cls._exit_code = opts.exit_code
         cls._status_message = opts.status_message
 
+        await cls._persist_terminal_status_message(opts.status_message)
         await cls._shutdown(timeout_secs=opts.timeout_secs)
 
         if opts.exit:
@@ -1021,13 +1023,32 @@ class Actor(metaclass=_ActorMeta):
         else:
             logger.info(logged)
 
+        result = await cls._put_run_status_message(message, is_terminal=is_terminal)
+        if result is None:
+            return ActorRun(id='', actor_id='', status='')
+        run_id, data = result
+
+        return ActorRun(
+            id=data.get('id', run_id),
+            actor_id=data.get('actorId', ''),
+            status=data.get('status', ''),
+            started_at=data.get('startedAt'),
+            finished_at=data.get('finishedAt'),
+            status_message=data.get('statusMessage'),
+        )
+
+    @classmethod
+    async def _put_run_status_message(
+        cls, message: str, *, is_terminal: bool
+    ) -> Optional[Tuple[str, Dict[str, Any]]]:
+        """PUT the status message to the current run. Returns ``None`` when not on the platform."""
         config = cls.get_config()
         api_url = config.get('api_base_url')
         token = config.get('token')
         run_id = config.get('actor_run_id')
 
         if not api_url or not token or not run_id:
-            return ActorRun(id='', actor_id='', status='')
+            return None
 
         client = cls.new_client(token=token, base_url=api_url)
         response = await client.put(
@@ -1041,15 +1062,27 @@ class Actor(metaclass=_ActorMeta):
         if response.content:
             body = response.json()
         data = body.get('data', body) if isinstance(body, dict) else {}
+        return run_id, data if isinstance(data, dict) else {}
 
-        return ActorRun(
-            id=data.get('id', run_id),
-            actor_id=data.get('actorId', ''),
-            status=data.get('status', ''),
-            started_at=data.get('startedAt'),
-            finished_at=data.get('finishedAt'),
-            status_message=data.get('statusMessage'),
-        )
+    @classmethod
+    async def _persist_terminal_status_message(cls, message: Optional[str], timeout_secs: float = 10.0) -> None:
+        """Store the exit/fail status message on the run as terminal (Apify parity).
+
+        Runs before teardown so it lands while the run still exists and the
+        process is alive, on success and failure alike. Best effort: a failed
+        update is logged and never blocks the exit.
+        """
+        if not message or not cls._initialized or cls._config is None:
+            return
+        try:
+            await asyncio.wait_for(
+                cls._put_run_status_message(message, is_terminal=True),
+                timeout=max(1.0, timeout_secs),
+            )
+        except asyncio.CancelledError:
+            raise
+        except BaseException as e:
+            logger.warning('Failed to store the final status message on the run: %s', e)
 
     @_ClassProperty
     def log(cls) -> logging.Logger:

scrapely/cdp_retry.py

@@ -0,0 +1,226 @@
+"""Retry remote-browser (CDP) connects that fail for transient reasons.
+
+The remote browser answers ``429`` when it is at capacity and ``503`` when it
+cannot start a browser or validate the session right now, both with a
+``Retry-After`` delay (also repeated in the response body, because Playwright
+only puts the status line and body into its error message). Connection errors
+and timeouts are transient too. Those are retried with exponential backoff,
+waiting at least the ``Retry-After`` delay when the error carries one. Any
+other status (``400``/``401``/``402``/``403``/``404``) fails at once.
+
+Tuning (environment):
+
+- ``SCRAPELY_CDP_CONNECT_ATTEMPTS``: total attempts, default ``5`` (``1`` disables retries).
+- ``SCRAPELY_CDP_RETRY_BASE_SECS``: first backoff delay, default ``1``; doubles per attempt.
+- ``SCRAPELY_CDP_RETRY_MAX_SECS``: cap for one backoff delay, default ``30``.
+- ``SCRAPELY_CDP_RETRY_AFTER_MAX_SECS``: longest ``Retry-After`` the SDK waits
+  out, default ``120``. A longer ``Retry-After`` ends the retries (the SDK never
+  retries earlier than the server asked).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import logging
+import os
+import random
+import re
+import time
+from typing import Any, Awaitable, Callable, Optional, Tuple, TypeVar
+from urllib.parse import urlsplit
+
+logger = logging.getLogger('scrapely.browser')
+
+T = TypeVar('T')
+
+_RETRY_WRAPPED_ATTR = '_scrapely_cdp_retry_wrapped'
+
+_REASONS = (
+    'Too Many Requests|Internal Server Error|Bad Gateway|Service Unavailable|Gateway Timeout|'
+    'Request Timeout|Bad Request|Unauthorized|Payment Required|Forbidden|Not Found'
+)
+_STATUS_RE = re.compile(rf'\b([1-5]\d\d) (?:{_REASONS})\b')
+_WS_STATUS_RE = re.compile(r'Unexpected server response:?\s*(\d{3})', re.IGNORECASE)
+_RETRY_AFTER_RE = re.compile(r'Retry-After:?\s*(\d+(?:\.\d+)?)', re.IGNORECASE)
+_TRANSIENT_MARKERS = (
+    'econnrefused',
+    'econnreset',
+    'etimedout',
+    'epipe',
+    'eai_again',
+    'ehostunreach',
+    'enetunreach',
+    'socket hang up',
+    'timeout',
+    'timed out',
+    'connection closed',
+    'connection refused',
+    'connection reset',
+    'websocket error',
+    'net::err_',
+    'target closed',
+)
+
+
+def _env_float(name: str, default: float) -> float:
+    try:
+        value = float(os.environ.get(name, ''))
+    except ValueError:
+        return default
+    return value if value > 0 else default
+
+
+def _max_attempts() -> int:
+    return max(1, int(_env_float('SCRAPELY_CDP_CONNECT_ATTEMPTS', 5)))
+
+
+def parse_status(message: str) -> Optional[int]:
+    """HTTP status from a Playwright/Puppeteer connect error message, if any."""
+    match = _WS_STATUS_RE.search(message) or _STATUS_RE.search(message)
+    return int(match.group(1)) if match else None
+
+
+def parse_retry_after(message: str) -> Optional[float]:
+    match = _RETRY_AFTER_RE.search(message)
+    if not match:
+        return None
+    return float(match.group(1))
+
+
+def classify_connect_error(err: BaseException) -> Tuple[bool, Optional[int], Optional[float]]:
+    """Return ``(retryable, status, retry_after_secs)`` for a connect error."""
+    message = f'{type(err).__name__}: {err}'
+    status = parse_status(message)
+    retry_after = parse_retry_after(message)
+    if status is not None:
+        return (status == 408 or status == 429 or status >= 500), status, retry_after
+    if isinstance(err, (asyncio.TimeoutError, TimeoutError, ConnectionError, OSError)):
+        return True, None, retry_after
+    lowered = message.lower()
+    return any(marker in lowered for marker in _TRANSIENT_MARKERS), None, retry_after
+
+
+def backoff_delay(attempt: int, retry_after: Optional[float]) -> float:
+    """Delay before retry number ``attempt`` (1-based), honoring Retry-After."""
+    base = _env_float('SCRAPELY_CDP_RETRY_BASE_SECS', 1.0)
+    cap = _env_float('SCRAPELY_CDP_RETRY_MAX_SECS', 30.0)
+    delay = min(cap, base * (2 ** (attempt - 1)))
+    delay = delay * (0.75 + random.random() * 0.5)
+    if retry_after is not None:
+        delay = max(delay, retry_after)
+    return delay
+
+
+def _retry_after_too_long(retry_after: Optional[float]) -> bool:
+    return retry_after is not None and retry_after > _env_float('SCRAPELY_CDP_RETRY_AFTER_MAX_SECS', 120.0)
+
+
+def _describe(err: BaseException, status: Optional[int]) -> str:
+    if status is not None:
+        return f'HTTP {status}'
+    # First line only: connect errors can carry the session URL with ids.
+    text = str(err).strip().splitlines()[0] if str(err).strip() else type(err).__name__
+    return re.sub(r'(wss?|https?)://\S+', '<cdp-url>', text)[:200]
+
+
+def _log_retry(err: BaseException, status: Optional[int], attempt: int, attempts: int, delay: float) -> None:
+    logger.warning(
+        'Remote browser connect failed (%s); retrying in %.1fs (attempt %d/%d)',
+        _describe(err, status),
+        delay,
+        attempt + 1,
+        attempts,
+    )
+
+
+async def connect_with_retry_async(connect: Callable[[], Awaitable[T]]) -> T:
+    """Run an async CDP connect, retrying transient failures."""
+    attempts = _max_attempts()
+    attempt = 1
+    while True:
+        try:
+            return await connect()
+        except asyncio.CancelledError:
+            raise
+        except Exception as err:
+            retryable, status, retry_after = classify_connect_error(err)
+            # Never retry earlier than the server asked; give up instead of waiting that long.
+            if not retryable or attempt >= attempts or _retry_after_too_long(retry_after):
+                raise
+            delay = backoff_delay(attempt, retry_after)
+            _log_retry(err, status, attempt, attempts, delay)
+            await asyncio.sleep(delay)
+            attempt += 1
+
+
+def connect_with_retry_sync(connect: Callable[[], T]) -> T:
+    """Run a blocking CDP connect, retrying transient failures."""
+    attempts = _max_attempts()
+    attempt = 1
+    while True:
+        try:
+            return connect()
+        except Exception as err:
+            retryable, status, retry_after = classify_connect_error(err)
+            # Never retry earlier than the server asked; give up instead of waiting that long.
+            if not retryable or attempt >= attempts or _retry_after_too_long(retry_after):
+                raise
+            delay = backoff_delay(attempt, retry_after)
+            _log_retry(err, status, attempt, attempts, delay)
+            time.sleep(delay)
+            attempt += 1
+
+
+def is_platform_cdp_url(url: Any, platform_cdp_url: Optional[str]) -> bool:
+    """True when ``url`` points at the same host as the platform's CDP_URL."""
+    if not isinstance(url, str) or not platform_cdp_url:
+        return False
+    try:
+        target, platform = urlsplit(url), urlsplit(platform_cdp_url)
+    except ValueError:
+        return False
+    return bool(target.netloc) and target.netloc.lower() == platform.netloc.lower()
+
+
+def wrap_connect_over_cdp(chromium: Any, is_async: bool, get_platform_cdp_url: Callable[[], Optional[str]]) -> None:
+    """Make ``chromium.connect_over_cdp`` retry transient failures for the platform CDP host.
+
+    Covers actor code that connects to ``CDP_URL`` directly as well as the
+    SDK's own redirected launches. Connects to other hosts are left alone.
+    Idempotent.
+    """
+    original = getattr(chromium, 'connect_over_cdp', None)
+    if original is None or getattr(original, _RETRY_WRAPPED_ATTR, False):
+        return
+
+    def _target(args: Tuple[Any, ...], kwargs: dict) -> Any:
+        if args:
+            return args[0]
+        return kwargs.get('endpoint_url') or kwargs.get('endpointURL')
+
+    if is_async:
+
+        @functools.wraps(original)
+        async def connect_over_cdp(*args: Any, **kwargs: Any) -> Any:
+            if not is_platform_cdp_url(_target(args, kwargs), get_platform_cdp_url()):
+                return await original(*args, **kwargs)
+            return await connect_with_retry_async(lambda: original(*args, **kwargs))
+
+    else:
+
+        @functools.wraps(original)
+        def connect_over_cdp(*args: Any, **kwargs: Any) -> Any:  # type: ignore[misc]
+            if not is_platform_cdp_url(_target(args, kwargs), get_platform_cdp_url()):
+                return original(*args, **kwargs)
+            return connect_with_retry_sync(lambda: original(*args, **kwargs))
+
+    setattr(connect_over_cdp, _RETRY_WRAPPED_ATTR, True)
+    try:
+        chromium.connect_over_cdp = connect_over_cdp
+    except (AttributeError, TypeError):
+        pass
+
+
+def is_retry_wrapped(fn: Any) -> bool:
+    return bool(getattr(fn, _RETRY_WRAPPED_ATTR, False))

scrapely/patch_browser_pool.py

@@ -70,6 +70,7 @@ from typing import Any, List, Optional, Tuple
 from urllib.parse import SplitResult, parse_qsl, unquote, urlencode, urlsplit, urlunsplit
 
 from ._browser_locations import subdivision_to_state_name
+from .cdp_retry import connect_with_retry_async, wrap_connect_over_cdp
 
 logger = logging.getLogger(__name__)
 
@@ -427,7 +428,9 @@ def _apply_patch() -> None:
         cdp_url = apply_proxy_info_to_cdp_url(cdp_url, proxy_info, browser_configuration)
 
         # Remote browser infrastructure is Chromium CDP (same as Node connectOverCDP).
-        browser = await self._playwright.chromium.connect_over_cdp(cdp_url)
+        chromium = self._playwright.chromium
+        wrap_connect_over_cdp(chromium, True, get_cdp_url)
+        browser = await chromium.connect_over_cdp(cdp_url)
 
         controller = PlaywrightBrowserController(
             browser,
@@ -518,7 +521,9 @@ def _apply_camoufox_patch() -> None:
         # Same as the PlaywrightBrowserPlugin patch above: the first page's
         # proxy info / browser configuration pins the connection's location.
         cdp_url = apply_proxy_info_to_cdp_url(cdp_url, _launch_proxy_info.get(), browser_configuration)
-        return await playwright.chromium.connect_over_cdp(cdp_url)
+        chromium = playwright.chromium
+        wrap_connect_over_cdp(chromium, True, get_cdp_url)
+        return await chromium.connect_over_cdp(cdp_url)
 
     camoufox.AsyncNewBrowser = _patched_async_new_browser
     camoufox._scrapely_cdp_patched = True
@@ -726,6 +731,10 @@ def _patch_playwright_object(playwright: Any, is_async: bool) -> Any:
     chromium = getattr(playwright, 'chromium', None)
     if chromium is None or getattr(chromium, 'connect_over_cdp', None) is None:
         return playwright
+    # Retry transient connect failures (429/5xx/network) for the platform CDP
+    # host: covers direct chromium.connect_over_cdp(CDP_URL) calls in actor
+    # code and the redirected launches below, which call this same attribute.
+    wrap_connect_over_cdp(chromium, is_async, get_cdp_url)
     for name in ('chromium', 'firefox', 'webkit'):
         browser_type = getattr(playwright, name, None)
         if browser_type is None:
@@ -845,7 +854,9 @@ def _apply_pyppeteer_patch() -> None:
                 'ignored: %s (the remote browser manages headless mode, binary and flags).',
                 ', '.join(sorted(merged.keys())),
             )
-        return await pyppeteer_launcher.connect(browserWSEndpoint=cdp_url)
+        return await connect_with_retry_async(
+            lambda: pyppeteer_launcher.connect(browserWSEndpoint=cdp_url)
+        )
 
     pyppeteer_launcher.launch = _patched_launch
     if hasattr(pyppeteer_mod, 'launch'):

{scrapely_python-0.1.5.dist-info → scrapely_python-0.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.5
+Version: 0.1.6
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT

{scrapely_python-0.1.5.dist-info → scrapely_python-0.1.6.dist-info}/RECORD

@@ -1,13 +1,14 @@
-scrapely/__init__.py,sha256=m8qsFMznobEHdjjlLu5KCP8Y1yKA3EM2pSoJJn-En_4,5745
+scrapely/__init__.py,sha256=IcAS39wGSGYhV2mkHPkaTPFdZyoTjdVln3wBZaCAwfo,5745
 scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
-scrapely/actor.py,sha256=QczHOj2GVQnMbQZ_76vjjJNHxPrinWB_WCdZ6_41l00,75126
+scrapely/actor.py,sha256=j61rLc2JXxmIkRi1M_Gym_G4PVcK19NfOp0dVpEu_Bw,76660
 scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
+scrapely/cdp_retry.py,sha256=hdTJEAIKhApM7IOk1DrQEl8NoflSJSmqd4RbNJ2IsgI,8540
 scrapely/charging.py,sha256=MFz_Z4qJ-N3qLffleQb2BxjomygM3UMvHSBw1HabG80,56857
 scrapely/charging_client.py,sha256=tDtlPwLSVFvoRa2NBgNjyfHkAUM77SSIyYT2ACgpDm8,3710
 scrapely/configuration.py,sha256=YUnbft9JeWlVTgWalr-CgJCiyJ_e_rgoQItVPvjeWRs,15406
 scrapely/errors.py,sha256=DCAlVvE3jToZuPEG2HVDL7i8D8hLboomQ5JpSOcsXoQ,4107
 scrapely/log.py,sha256=iVKoQdMLQXEWz1STzbSXrNUxsS_FNwiIdqoEg7CzeqU,1440
-scrapely/patch_browser_pool.py,sha256=sRAjuzKoOEgn0X8695c_9qioRALzkTWXoDlorVnK9jg,37426
+scrapely/patch_browser_pool.py,sha256=qZstdQ1rFw8kGjzB9fm3GXco8Jt40eklTw1Czf_vh-8,38018
 scrapely/pricing_types.py,sha256=J6wCddZd-U1MYeOtaaEdeHSsp1HBgiQLfvSInBOvhIo,5305
 scrapely/proxy_configuration.py,sha256=rSCXJFuUEIK31chk05FVLRQKeVA34h_pBRbr8IKNXo0,36195
 scrapely/py.typed,sha256=XtWlywqUjc6x_M2gAXlDtYXbAsYkCjKZEjFyJvaHvm4,105
@@ -37,7 +38,7 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9
 scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
 scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
 scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
-scrapely_python-0.1.5.dist-info/METADATA,sha256=9Jl9D1ijxCxthjVQY0YB3ryI6jipYzXONbjyOjA8YEs,3104
-scrapely_python-0.1.5.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-scrapely_python-0.1.5.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
-scrapely_python-0.1.5.dist-info/RECORD,,
+scrapely_python-0.1.6.dist-info/METADATA,sha256=M7O94gZCIK2wmgvm4JCGYK6Z2pcaolNCjpRlkF_jsQI,3104
+scrapely_python-0.1.6.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+scrapely_python-0.1.6.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
+scrapely_python-0.1.6.dist-info/RECORD,,