scrapely-python 0.1.5__tar.gz → 0.1.6__tar.gz

{scrapely_python-0.1.5 → scrapely_python-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.5
+Version: 0.1.6
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT

{scrapely_python-0.1.5 → scrapely_python-0.1.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.5"
+version = "0.1.6"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT"

{scrapely_python-0.1.5 → scrapely_python-0.1.6}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.5'
+__version__ = '0.1.6'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.5 → scrapely_python-0.1.6}/src/scrapely/actor.py

@@ -18,7 +18,7 @@ import warnings
 from collections.abc import MutableMapping
 from datetime import datetime, timedelta
 from decimal import Decimal
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union, cast
 
 from crawlee.configuration import Configuration as CrawleeConfiguration
 from crawlee.storage_clients._base import StorageClient
@@ -287,6 +287,7 @@ class Actor(metaclass=_ActorMeta):
             cls._exit_code = self._instance_exit_code
         if self._instance_status_message:
             cls._status_message = self._instance_status_message
+            await cls._persist_terminal_status_message(self._instance_status_message)
         await cls._shutdown(timeout_secs=int(self._cleanup_timeout.total_seconds()))
 
     @classmethod
@@ -589,6 +590,7 @@ class Actor(metaclass=_ActorMeta):
         cls._exit_code = opts.exit_code
         cls._status_message = opts.status_message
 
+        await cls._persist_terminal_status_message(opts.status_message)
         await cls._shutdown(timeout_secs=opts.timeout_secs)
 
         if opts.exit:
@@ -1021,13 +1023,32 @@ class Actor(metaclass=_ActorMeta):
         else:
             logger.info(logged)
 
+        result = await cls._put_run_status_message(message, is_terminal=is_terminal)
+        if result is None:
+            return ActorRun(id='', actor_id='', status='')
+        run_id, data = result
+
+        return ActorRun(
+            id=data.get('id', run_id),
+            actor_id=data.get('actorId', ''),
+            status=data.get('status', ''),
+            started_at=data.get('startedAt'),
+            finished_at=data.get('finishedAt'),
+            status_message=data.get('statusMessage'),
+        )
+
+    @classmethod
+    async def _put_run_status_message(
+        cls, message: str, *, is_terminal: bool
+    ) -> Optional[Tuple[str, Dict[str, Any]]]:
+        """PUT the status message to the current run. Returns ``None`` when not on the platform."""
         config = cls.get_config()
         api_url = config.get('api_base_url')
         token = config.get('token')
         run_id = config.get('actor_run_id')
 
         if not api_url or not token or not run_id:
-            return ActorRun(id='', actor_id='', status='')
+            return None
 
         client = cls.new_client(token=token, base_url=api_url)
         response = await client.put(
@@ -1041,15 +1062,27 @@ class Actor(metaclass=_ActorMeta):
         if response.content:
             body = response.json()
         data = body.get('data', body) if isinstance(body, dict) else {}
+        return run_id, data if isinstance(data, dict) else {}
 
-        return ActorRun(
-            id=data.get('id', run_id),
-            actor_id=data.get('actorId', ''),
-            status=data.get('status', ''),
-            started_at=data.get('startedAt'),
-            finished_at=data.get('finishedAt'),
-            status_message=data.get('statusMessage'),
-        )
+    @classmethod
+    async def _persist_terminal_status_message(cls, message: Optional[str], timeout_secs: float = 10.0) -> None:
+        """Store the exit/fail status message on the run as terminal (Apify parity).
+
+        Runs before teardown so it lands while the run still exists and the
+        process is alive, on success and failure alike. Best effort: a failed
+        update is logged and never blocks the exit.
+        """
+        if not message or not cls._initialized or cls._config is None:
+            return
+        try:
+            await asyncio.wait_for(
+                cls._put_run_status_message(message, is_terminal=True),
+                timeout=max(1.0, timeout_secs),
+            )
+        except asyncio.CancelledError:
+            raise
+        except BaseException as e:
+            logger.warning('Failed to store the final status message on the run: %s', e)
 
     @_ClassProperty
     def log(cls) -> logging.Logger:

scrapely_python-0.1.6/src/scrapely/cdp_retry.py

@@ -0,0 +1,226 @@
+"""Retry remote-browser (CDP) connects that fail for transient reasons.
+
+The remote browser answers ``429`` when it is at capacity and ``503`` when it
+cannot start a browser or validate the session right now, both with a
+``Retry-After`` delay (also repeated in the response body, because Playwright
+only puts the status line and body into its error message). Connection errors
+and timeouts are transient too. Those are retried with exponential backoff,
+waiting at least the ``Retry-After`` delay when the error carries one. Any
+other status (``400``/``401``/``402``/``403``/``404``) fails at once.
+
+Tuning (environment):
+
+- ``SCRAPELY_CDP_CONNECT_ATTEMPTS``: total attempts, default ``5`` (``1`` disables retries).
+- ``SCRAPELY_CDP_RETRY_BASE_SECS``: first backoff delay, default ``1``; doubles per attempt.
+- ``SCRAPELY_CDP_RETRY_MAX_SECS``: cap for one backoff delay, default ``30``.
+- ``SCRAPELY_CDP_RETRY_AFTER_MAX_SECS``: longest ``Retry-After`` the SDK waits
+  out, default ``120``. A longer ``Retry-After`` ends the retries (the SDK never
+  retries earlier than the server asked).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import logging
+import os
+import random
+import re
+import time
+from typing import Any, Awaitable, Callable, Optional, Tuple, TypeVar
+from urllib.parse import urlsplit
+
+logger = logging.getLogger('scrapely.browser')
+
+T = TypeVar('T')
+
+_RETRY_WRAPPED_ATTR = '_scrapely_cdp_retry_wrapped'
+
+_REASONS = (
+    'Too Many Requests|Internal Server Error|Bad Gateway|Service Unavailable|Gateway Timeout|'
+    'Request Timeout|Bad Request|Unauthorized|Payment Required|Forbidden|Not Found'
+)
+_STATUS_RE = re.compile(rf'\b([1-5]\d\d) (?:{_REASONS})\b')
+_WS_STATUS_RE = re.compile(r'Unexpected server response:?\s*(\d{3})', re.IGNORECASE)
+_RETRY_AFTER_RE = re.compile(r'Retry-After:?\s*(\d+(?:\.\d+)?)', re.IGNORECASE)
+_TRANSIENT_MARKERS = (
+    'econnrefused',
+    'econnreset',
+    'etimedout',
+    'epipe',
+    'eai_again',
+    'ehostunreach',
+    'enetunreach',
+    'socket hang up',
+    'timeout',
+    'timed out',
+    'connection closed',
+    'connection refused',
+    'connection reset',
+    'websocket error',
+    'net::err_',
+    'target closed',
+)
+
+
+def _env_float(name: str, default: float) -> float:
+    try:
+        value = float(os.environ.get(name, ''))
+    except ValueError:
+        return default
+    return value if value > 0 else default
+
+
+def _max_attempts() -> int:
+    return max(1, int(_env_float('SCRAPELY_CDP_CONNECT_ATTEMPTS', 5)))
+
+
+def parse_status(message: str) -> Optional[int]:
+    """HTTP status from a Playwright/Puppeteer connect error message, if any."""
+    match = _WS_STATUS_RE.search(message) or _STATUS_RE.search(message)
+    return int(match.group(1)) if match else None
+
+
+def parse_retry_after(message: str) -> Optional[float]:
+    match = _RETRY_AFTER_RE.search(message)
+    if not match:
+        return None
+    return float(match.group(1))
+
+
+def classify_connect_error(err: BaseException) -> Tuple[bool, Optional[int], Optional[float]]:
+    """Return ``(retryable, status, retry_after_secs)`` for a connect error."""
+    message = f'{type(err).__name__}: {err}'
+    status = parse_status(message)
+    retry_after = parse_retry_after(message)
+    if status is not None:
+        return (status == 408 or status == 429 or status >= 500), status, retry_after
+    if isinstance(err, (asyncio.TimeoutError, TimeoutError, ConnectionError, OSError)):
+        return True, None, retry_after
+    lowered = message.lower()
+    return any(marker in lowered for marker in _TRANSIENT_MARKERS), None, retry_after
+
+
+def backoff_delay(attempt: int, retry_after: Optional[float]) -> float:
+    """Delay before retry number ``attempt`` (1-based), honoring Retry-After."""
+    base = _env_float('SCRAPELY_CDP_RETRY_BASE_SECS', 1.0)
+    cap = _env_float('SCRAPELY_CDP_RETRY_MAX_SECS', 30.0)
+    delay = min(cap, base * (2 ** (attempt - 1)))
+    delay = delay * (0.75 + random.random() * 0.5)
+    if retry_after is not None:
+        delay = max(delay, retry_after)
+    return delay
+
+
+def _retry_after_too_long(retry_after: Optional[float]) -> bool:
+    return retry_after is not None and retry_after > _env_float('SCRAPELY_CDP_RETRY_AFTER_MAX_SECS', 120.0)
+
+
+def _describe(err: BaseException, status: Optional[int]) -> str:
+    if status is not None:
+        return f'HTTP {status}'
+    # First line only: connect errors can carry the session URL with ids.
+    text = str(err).strip().splitlines()[0] if str(err).strip() else type(err).__name__
+    return re.sub(r'(wss?|https?)://\S+', '<cdp-url>', text)[:200]
+
+
+def _log_retry(err: BaseException, status: Optional[int], attempt: int, attempts: int, delay: float) -> None:
+    logger.warning(
+        'Remote browser connect failed (%s); retrying in %.1fs (attempt %d/%d)',
+        _describe(err, status),
+        delay,
+        attempt + 1,
+        attempts,
+    )
+
+
+async def connect_with_retry_async(connect: Callable[[], Awaitable[T]]) -> T:
+    """Run an async CDP connect, retrying transient failures."""
+    attempts = _max_attempts()
+    attempt = 1
+    while True:
+        try:
+            return await connect()
+        except asyncio.CancelledError:
+            raise
+        except Exception as err:
+            retryable, status, retry_after = classify_connect_error(err)
+            # Never retry earlier than the server asked; give up instead of waiting that long.
+            if not retryable or attempt >= attempts or _retry_after_too_long(retry_after):
+                raise
+            delay = backoff_delay(attempt, retry_after)
+            _log_retry(err, status, attempt, attempts, delay)
+            await asyncio.sleep(delay)
+            attempt += 1
+
+
+def connect_with_retry_sync(connect: Callable[[], T]) -> T:
+    """Run a blocking CDP connect, retrying transient failures."""
+    attempts = _max_attempts()
+    attempt = 1
+    while True:
+        try:
+            return connect()
+        except Exception as err:
+            retryable, status, retry_after = classify_connect_error(err)
+            # Never retry earlier than the server asked; give up instead of waiting that long.
+            if not retryable or attempt >= attempts or _retry_after_too_long(retry_after):
+                raise
+            delay = backoff_delay(attempt, retry_after)
+            _log_retry(err, status, attempt, attempts, delay)
+            time.sleep(delay)
+            attempt += 1
+
+
+def is_platform_cdp_url(url: Any, platform_cdp_url: Optional[str]) -> bool:
+    """True when ``url`` points at the same host as the platform's CDP_URL."""
+    if not isinstance(url, str) or not platform_cdp_url:
+        return False
+    try:
+        target, platform = urlsplit(url), urlsplit(platform_cdp_url)
+    except ValueError:
+        return False
+    return bool(target.netloc) and target.netloc.lower() == platform.netloc.lower()
+
+
+def wrap_connect_over_cdp(chromium: Any, is_async: bool, get_platform_cdp_url: Callable[[], Optional[str]]) -> None:
+    """Make ``chromium.connect_over_cdp`` retry transient failures for the platform CDP host.
+
+    Covers actor code that connects to ``CDP_URL`` directly as well as the
+    SDK's own redirected launches. Connects to other hosts are left alone.
+    Idempotent.
+    """
+    original = getattr(chromium, 'connect_over_cdp', None)
+    if original is None or getattr(original, _RETRY_WRAPPED_ATTR, False):
+        return
+
+    def _target(args: Tuple[Any, ...], kwargs: dict) -> Any:
+        if args:
+            return args[0]
+        return kwargs.get('endpoint_url') or kwargs.get('endpointURL')
+
+    if is_async:
+
+        @functools.wraps(original)
+        async def connect_over_cdp(*args: Any, **kwargs: Any) -> Any:
+            if not is_platform_cdp_url(_target(args, kwargs), get_platform_cdp_url()):
+                return await original(*args, **kwargs)
+            return await connect_with_retry_async(lambda: original(*args, **kwargs))
+
+    else:
+
+        @functools.wraps(original)
+        def connect_over_cdp(*args: Any, **kwargs: Any) -> Any:  # type: ignore[misc]
+            if not is_platform_cdp_url(_target(args, kwargs), get_platform_cdp_url()):
+                return original(*args, **kwargs)
+            return connect_with_retry_sync(lambda: original(*args, **kwargs))
+
+    setattr(connect_over_cdp, _RETRY_WRAPPED_ATTR, True)
+    try:
+        chromium.connect_over_cdp = connect_over_cdp
+    except (AttributeError, TypeError):
+        pass
+
+
+def is_retry_wrapped(fn: Any) -> bool:
+    return bool(getattr(fn, _RETRY_WRAPPED_ATTR, False))

{scrapely_python-0.1.5 → scrapely_python-0.1.6}/src/scrapely/patch_browser_pool.py

@@ -70,6 +70,7 @@ from typing import Any, List, Optional, Tuple
 from urllib.parse import SplitResult, parse_qsl, unquote, urlencode, urlsplit, urlunsplit
 
 from ._browser_locations import subdivision_to_state_name
+from .cdp_retry import connect_with_retry_async, wrap_connect_over_cdp
 
 logger = logging.getLogger(__name__)
 
@@ -427,7 +428,9 @@ def _apply_patch() -> None:
         cdp_url = apply_proxy_info_to_cdp_url(cdp_url, proxy_info, browser_configuration)
 
         # Remote browser infrastructure is Chromium CDP (same as Node connectOverCDP).
-        browser = await self._playwright.chromium.connect_over_cdp(cdp_url)
+        chromium = self._playwright.chromium
+        wrap_connect_over_cdp(chromium, True, get_cdp_url)
+        browser = await chromium.connect_over_cdp(cdp_url)
 
         controller = PlaywrightBrowserController(
             browser,
@@ -518,7 +521,9 @@ def _apply_camoufox_patch() -> None:
         # Same as the PlaywrightBrowserPlugin patch above: the first page's
         # proxy info / browser configuration pins the connection's location.
         cdp_url = apply_proxy_info_to_cdp_url(cdp_url, _launch_proxy_info.get(), browser_configuration)
-        return await playwright.chromium.connect_over_cdp(cdp_url)
+        chromium = playwright.chromium
+        wrap_connect_over_cdp(chromium, True, get_cdp_url)
+        return await chromium.connect_over_cdp(cdp_url)
 
     camoufox.AsyncNewBrowser = _patched_async_new_browser
     camoufox._scrapely_cdp_patched = True
@@ -726,6 +731,10 @@ def _patch_playwright_object(playwright: Any, is_async: bool) -> Any:
     chromium = getattr(playwright, 'chromium', None)
     if chromium is None or getattr(chromium, 'connect_over_cdp', None) is None:
         return playwright
+    # Retry transient connect failures (429/5xx/network) for the platform CDP
+    # host: covers direct chromium.connect_over_cdp(CDP_URL) calls in actor
+    # code and the redirected launches below, which call this same attribute.
+    wrap_connect_over_cdp(chromium, is_async, get_cdp_url)
     for name in ('chromium', 'firefox', 'webkit'):
         browser_type = getattr(playwright, name, None)
         if browser_type is None:
@@ -845,7 +854,9 @@ def _apply_pyppeteer_patch() -> None:
                 'ignored: %s (the remote browser manages headless mode, binary and flags).',
                 ', '.join(sorted(merged.keys())),
             )
-        return await pyppeteer_launcher.connect(browserWSEndpoint=cdp_url)
+        return await connect_with_retry_async(
+            lambda: pyppeteer_launcher.connect(browserWSEndpoint=cdp_url)
+        )
 
     pyppeteer_launcher.launch = _patched_launch
     if hasattr(pyppeteer_mod, 'launch'):

{scrapely_python-0.1.5 → scrapely_python-0.1.6}/src/scrapely_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.5
+Version: 0.1.6
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT

{scrapely_python-0.1.5 → scrapely_python-0.1.6}/src/scrapely_python.egg-info/SOURCES.txt

@@ -4,6 +4,7 @@ src/scrapely/__init__.py
 src/scrapely/_browser_locations.py
 src/scrapely/actor.py
 src/scrapely/browser_configuration.py
+src/scrapely/cdp_retry.py
 src/scrapely/charging.py
 src/scrapely/charging_client.py
 src/scrapely/configuration.py
@@ -47,6 +48,7 @@ src/scrapely_python.egg-info/top_level.txt
 tests/test_api_client_retries.py
 tests/test_apify_parity.py
 tests/test_browser_configuration.py
+tests/test_cdp_retry.py
 tests/test_charging_parity.py
 tests/test_charging_pending.py
 tests/test_parity.py
@@ -54,4 +56,5 @@ tests/test_patch_browser_pool.py
 tests/test_proxy_configuration.py
 tests/test_run_control.py
 tests/test_secrets.py
-tests/test_storage.py
+tests/test_storage.py
+tests/test_terminal_status_message.py

scrapely_python-0.1.6/tests/test_cdp_retry.py

@@ -0,0 +1,169 @@
+"""Remote-browser connects retry 429/5xx/connection failures and honor Retry-After."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, List
+from unittest.mock import patch
+
+import pytest
+
+from scrapely import cdp_retry
+from scrapely.cdp_retry import (
+    classify_connect_error,
+    connect_with_retry_async,
+    connect_with_retry_sync,
+    wrap_connect_over_cdp,
+)
+
+PLATFORM = 'wss://browser.example.test?scrapely_run_id=r&scrapely_user_id=u'
+
+
+class PlaywrightError(Exception):
+    pass
+
+
+def _ws_error(status: str, body: str = '') -> PlaywrightError:
+    msg = f'BrowserType.connect_over_cdp: WebSocket error: wss://browser.example.test/ {status}'
+    if body:
+        msg += f'\n{body}'
+    return PlaywrightError(msg)
+
+
+@pytest.fixture(autouse=True)
+def _fast(monkeypatch: pytest.MonkeyPatch) -> List[float]:
+    sleeps: List[float] = []
+
+    async def fake_sleep(delay: float) -> None:
+        sleeps.append(delay)
+
+    monkeypatch.setattr(cdp_retry.asyncio, 'sleep', fake_sleep)
+    monkeypatch.setattr(cdp_retry.time, 'sleep', lambda d: sleeps.append(d))
+    monkeypatch.delenv('SCRAPELY_CDP_CONNECT_ATTEMPTS', raising=False)
+    return sleeps
+
+
+@pytest.mark.parametrize(
+    'err,retryable,status',
+    [
+        (_ws_error('429 Too Many Requests'), True, 429),
+        (_ws_error('500 Internal Server Error'), True, 500),
+        (_ws_error('503 Service Unavailable'), True, 503),
+        (PlaywrightError('Unexpected server response: 502'), True, 502),
+        (_ws_error('401 Unauthorized', 'Scrapely: run_not_active'), False, 401),
+        (_ws_error('402 Payment Required', 'Scrapely: no_credits'), False, 402),
+        (PlaywrightError('connect ECONNREFUSED 1.2.3.4:443'), True, None),
+        (PlaywrightError('BrowserType.connect_over_cdp: Timeout 25000ms exceeded.'), True, None),
+        (ConnectionResetError('reset'), True, None),
+        (ValueError('bad option'), False, None),
+    ],
+)
+def test_classify(err: BaseException, retryable: bool, status: Any) -> None:
+    got_retryable, got_status, _ = classify_connect_error(err)
+    assert (got_retryable, got_status) == (retryable, status)
+
+
+def test_retry_after_from_body() -> None:
+    err = _ws_error('429 Too Many Requests', 'Too many requests\nRetry-After: 7')
+    assert classify_connect_error(err) == (True, 429, 7.0)
+
+
+async def test_async_retries_then_succeeds_and_honors_retry_after(_fast: List[float]) -> None:
+    calls = 0
+
+    async def connect() -> str:
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            raise _ws_error('429 Too Many Requests', 'Retry-After: 9')
+        if calls == 2:
+            raise _ws_error('500 Internal Server Error')
+        return 'browser'
+
+    assert await connect_with_retry_async(connect) == 'browser'
+    assert calls == 3
+    assert _fast[0] >= 9
+    assert len(_fast) == 2
+
+
+async def test_async_does_not_retry_auth_failure(_fast: List[float]) -> None:
+    calls = 0
+
+    async def connect() -> str:
+        nonlocal calls
+        calls += 1
+        raise _ws_error('401 Unauthorized')
+
+    with pytest.raises(PlaywrightError):
+        await connect_with_retry_async(connect)
+    assert calls == 1
+    assert _fast == []
+
+
+async def test_async_gives_up_after_max_attempts(monkeypatch: pytest.MonkeyPatch, _fast: List[float]) -> None:
+    monkeypatch.setenv('SCRAPELY_CDP_CONNECT_ATTEMPTS', '3')
+    calls = 0
+
+    async def connect() -> str:
+        nonlocal calls
+        calls += 1
+        raise _ws_error('503 Service Unavailable')
+
+    with pytest.raises(PlaywrightError):
+        await connect_with_retry_async(connect)
+    assert calls == 3
+    assert len(_fast) == 2
+
+
+def test_sync_retries() -> None:
+    calls = 0
+
+    def connect() -> str:
+        nonlocal calls
+        calls += 1
+        if calls < 3:
+            raise ConnectionRefusedError('refused')
+        return 'browser'
+
+    assert connect_with_retry_sync(connect) == 'browser'
+    assert calls == 3
+
+
+class _Chromium:
+    def __init__(self, failures: int) -> None:
+        self.failures = failures
+        self.urls: List[str] = []
+
+    async def connect_over_cdp(self, endpoint_url: str, **kwargs: Any) -> str:
+        self.urls.append(endpoint_url)
+        if len(self.urls) <= self.failures:
+            raise _ws_error('429 Too Many Requests')
+        return 'browser'
+
+
+async def test_wrapped_connect_over_cdp_retries_platform_host_only() -> None:
+    chromium = _Chromium(failures=2)
+    wrap_connect_over_cdp(chromium, True, lambda: PLATFORM)
+    wrap_connect_over_cdp(chromium, True, lambda: PLATFORM)  # idempotent
+    assert await chromium.connect_over_cdp(PLATFORM + '&proxy_country=US') == 'browser'
+    assert len(chromium.urls) == 3
+
+    other = _Chromium(failures=1)
+    wrap_connect_over_cdp(other, True, lambda: PLATFORM)
+    with pytest.raises(PlaywrightError):
+        await other.connect_over_cdp('ws://localhost:9222/devtools/browser/x')
+    assert len(other.urls) == 1
+
+
+async def test_long_retry_after_ends_retries_instead_of_retrying_early(_fast: List[float]) -> None:
+    calls = 0
+
+    async def connect() -> str:
+        nonlocal calls
+        calls += 1
+        raise _ws_error('429 Too Many Requests', 'Retry-After: 600')
+
+    with pytest.raises(PlaywrightError):
+        await connect_with_retry_async(connect)
+    assert calls == 1
+    assert _fast == []

scrapely_python-0.1.6/tests/test_terminal_status_message.py

@@ -0,0 +1,130 @@
+"""Actor.exit()/fail() store their status message on the run as terminal, before teardown."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Dict, List, Optional, Tuple
+
+import pytest
+
+from scrapely import Actor
+from scrapely.configuration import Configuration
+
+
+class _FakeResponse:
+    def __init__(self, data: Dict[str, Any]) -> None:
+        self._data = data
+        self.content = b'x'
+
+    def json(self) -> Dict[str, Any]:
+        return {'data': self._data}
+
+
+class _FakeClient:
+    def __init__(self, events: List[Tuple[str, Any]], fail: bool = False, hang: bool = False) -> None:
+        self.events = events
+        self.fail = fail
+        self.hang = hang
+
+    async def put(self, path: str, json_data: Optional[Dict[str, Any]] = None) -> _FakeResponse:
+        if self.hang:
+            await asyncio.sleep(60)
+        if self.fail:
+            raise RuntimeError('API down')
+        self.events.append(('put', (path, json_data)))
+        return _FakeResponse({'id': 'run123', **(json_data or {})})
+
+
+@pytest.fixture
+def lifecycle(monkeypatch: pytest.MonkeyPatch):
+    events: List[Tuple[str, Any]] = []
+    exit_codes: List[int] = []
+    state = {
+        '_exit_code': Actor._exit_code,
+        '_status_message': Actor._status_message,
+        '_initialized': Actor._initialized,
+        '_config': Actor._config,
+    }
+    config = Configuration({'api_base_url': 'http://api.test', 'token': 'tok', 'actor_run_id': 'run123'})
+    Actor._config = config
+    Actor._initialized = True
+
+    async def fake_shutdown(timeout_secs: int = 30) -> None:
+        events.append(('shutdown', timeout_secs))
+        Actor._initialized = False
+
+    client = _FakeClient(events)
+    monkeypatch.setattr(Actor, '_shutdown', fake_shutdown)
+    monkeypatch.setattr(Actor, 'new_client', classmethod(lambda cls, token=None, base_url=None: client))
+    monkeypatch.setattr('sys.exit', lambda code=0: exit_codes.append(code))
+    try:
+        yield events, exit_codes, client
+    finally:
+        for key, value in state.items():
+            setattr(Actor, key, value)
+
+
+async def test_fail_stores_terminal_status_message_before_shutdown(lifecycle) -> None:
+    events, exit_codes, _ = lifecycle
+    await Actor.fail(status_message='No results: bad input')
+    assert events[0] == (
+        'put',
+        ('/v2/actor-runs/run123', {'statusMessage': 'No results: bad input', 'isStatusMessageTerminal': True}),
+    )
+    assert events[1][0] == 'shutdown'
+    assert exit_codes == [1]
+
+
+async def test_exit_stores_terminal_status_message(lifecycle) -> None:
+    events, exit_codes, _ = lifecycle
+    await Actor.exit('Done: 10 items')
+    assert events[0] == (
+        'put',
+        ('/v2/actor-runs/run123', {'statusMessage': 'Done: 10 items', 'isStatusMessageTerminal': True}),
+    )
+    assert exit_codes == [0]
+
+
+async def test_exit_without_message_sends_nothing(lifecycle) -> None:
+    events, _, _ = lifecycle
+    await Actor.exit(exit_code=1)
+    assert [e[0] for e in events] == ['shutdown']
+
+
+async def test_failed_update_does_not_block_exit(lifecycle) -> None:
+    events, exit_codes, client = lifecycle
+    client.fail = True
+    await Actor.fail(status_message='boom')
+    assert [e[0] for e in events] == ['shutdown']
+    assert exit_codes == [1]
+
+
+async def test_hanging_update_is_bounded(lifecycle, monkeypatch: pytest.MonkeyPatch) -> None:
+    events, exit_codes, client = lifecycle
+    client.hang = True
+    original = Actor._persist_terminal_status_message.__func__
+
+    async def quick(cls, message, timeout_secs: float = 10.0):
+        await original(cls, message, timeout_secs=0.01)
+
+    monkeypatch.setattr(Actor, '_persist_terminal_status_message', classmethod(quick))
+    await asyncio.wait_for(Actor.fail(status_message='boom'), timeout=5)
+    assert [e[0] for e in events] == ['shutdown']
+    assert exit_codes == [1]
+
+
+async def test_context_manager_status_message_is_terminal(lifecycle, monkeypatch: pytest.MonkeyPatch) -> None:
+    events, _, _ = lifecycle
+
+    async def fake_init(options: Any = None) -> None:
+        Actor._initialized = True
+
+    monkeypatch.setattr(Actor, 'init', fake_init)
+    with pytest.raises(ValueError):
+        async with Actor(status_message='Crashed while parsing'):
+            raise ValueError('parse error')
+    assert events[0] == (
+        'put',
+        ('/v2/actor-runs/run123', {'statusMessage': 'Crashed while parsing', 'isStatusMessageTerminal': True}),
+    )
+    assert events[1][0] == 'shutdown'