scrapely-python 0.1.13__tar.gz → 0.1.15__tar.gz

{scrapely_python-0.1.13/src/scrapely_python.egg-info → scrapely_python-0.1.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.13
+Version: 0.1.15
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.13 → scrapely_python-0.1.15}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.13"
+version = "0.1.15"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.13 → scrapely_python-0.1.15}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.13'
+__version__ = '0.1.15'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.13 → scrapely_python-0.1.15}/src/scrapely/actor.py

@@ -213,6 +213,14 @@ class _ClassProperty:
         return self.func(owner)
 
 
+def _system_exit_code(exc: Any) -> int:
+    """Process exit code carried by a SystemExit (sys.exit semantics)."""
+    code = getattr(exc, 'code', None)
+    if code is None:
+        return 0
+    return code if isinstance(code, int) else 1
+
+
 class _ActorMeta(type):
     """Metaclass enabling async with Actor: by delegating to classmethods."""
 
@@ -226,7 +234,11 @@ class _ActorMeta(type):
         return cls
 
     async def __aexit__(cls, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
-        if exc_type is not None:
+        if exc_type is not None and issubclass(exc_type, SystemExit):
+            # Actor.exit()/fail()/sys.exit() inside `async with Actor:` is a deliberate
+            # exit, not a failure (Apify parity): keep its exit code, log nothing.
+            cls._exit_code = _system_exit_code(exc_val)
+        elif exc_type is not None:
             cls._exit_code = 1
             logger.error(f'Actor failed: {exc_val}')
         await cls._shutdown()
@@ -286,7 +298,10 @@ class Actor(metaclass=_ActorMeta):
 
     async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
         cls = type(self)
-        if exc_type is not None:
+        if exc_type is not None and issubclass(exc_type, SystemExit):
+            # Deliberate exit via Actor.exit()/fail()/sys.exit(), not a failure.
+            cls._exit_code = _system_exit_code(exc_val)
+        elif exc_type is not None:
             cls._exit_code = 1
             logger.error(f'Actor failed: {exc_val}')
         else:
@@ -1039,7 +1054,19 @@ class Actor(metaclass=_ActorMeta):
         else:
             logger.info(logged)
 
-        result = await cls._put_run_status_message(message, is_terminal=is_terminal)
+        if is_terminal:
+            result = await cls._put_run_status_message(message, is_terminal=True)
+        else:
+            # A progress update is best effort: if the API still fails after the
+            # retry window (e.g. a long 503), log it and carry on instead of
+            # crashing the run. Terminal updates keep raising.
+            try:
+                result = await cls._put_run_status_message(message, is_terminal=False)
+            except asyncio.CancelledError:
+                raise
+            except Exception as e:  # noqa: BLE001
+                logger.warning('Failed to update the status message (continuing): %s', e)
+                return ActorRun(id='', actor_id='', status='')
         if result is None:
             return ActorRun(id='', actor_id='', status='')
         run_id, data = result

{scrapely_python-0.1.13 → scrapely_python-0.1.15}/src/scrapely/charging.py

@@ -52,7 +52,7 @@ _PENDING_MAX_ENTRIES = 1024
 # Same-key POST attempts before leaving an intent pending / re-queueing a void.
 # The API client itself retries 429 and 5xx answers and refused or reset
 # connections of these POSTs with exponential backoff (honouring Retry-After)
-# for its retry window (2 minutes, SCRAPELY_API_RETRY_WINDOW_SECS): re-sending
+# for its retry window (5 minutes, SCRAPELY_API_RETRY_WINDOW_SECS): re-sending
 # is safe because every attempt carries the same Idempotency-Key. These outer
 # attempts cover what the client gives up on quickly (timeouts, other
 # transport errors), within _RETRY_BUDGET_SECS. Then the charge stays pending
@@ -62,7 +62,7 @@ _POST_ATTEMPTS = 8
 
 # Outer retries stop once this much time has passed since the first attempt
 # (the API retry window; JS SDK parity).
-_RETRY_BUDGET_SECS = 120.0
+_RETRY_BUDGET_SECS = 300.0
 
 
 def _is_transient_status(status: Optional[int]) -> bool:
@@ -1567,10 +1567,20 @@ def _push_certainly_failed(err: BaseException) -> bool:
     """True when a dataset push certainly stored nothing.
 
     The storage API answered with an error status, or the connection was never
-    made. Timeouts and resets after sending are not proof: the write may have landed.
+    made. Timeouts and resets after sending are not proof: the write may have landed,
+    and neither is any error of a push one of whose earlier attempts may have been
+    applied (``outcome_unknown``, set by the storage client's retries).
     """
     import httpx
 
+    c: Optional[BaseException] = err
+    for _ in range(4):
+        if c is None:
+            break
+        if getattr(c, 'outcome_unknown', False) is True:
+            return False
+        c = c.__cause__ or c.__context__
+
     e: Optional[BaseException] = err
     for _ in range(4):
         if e is None:

{scrapely_python-0.1.13 → scrapely_python-0.1.15}/src/scrapely/storage/api_client.py

@@ -76,15 +76,73 @@ def _error_type(response: httpx.Response) -> Optional[str]:
 
 
 # Default time budget for retrying one API call (429, 503, network errors).
-DEFAULT_API_RETRY_WINDOW_SECS = 120.0
+DEFAULT_API_RETRY_WINDOW_SECS = 300.0
 _MAX_BACKOFF_SECS = 30.0
 
+# Response header the API sets on every dataset push that sent an
+# Idempotency-Key, once it de-duplicates such pushes. An older API ignores the
+# key, and re-sending a push there could store the rows twice, so ambiguous
+# failures of keyed writes are retried only while the latest keyed answer from
+# that API origin carried this header. A 2xx without it (an older instance,
+# e.g. mid-deploy or after a rollback) switches the retries off again.
+IDEMPOTENCY_SUPPORT_HEADER = 'X-Scrapely-Idempotency'
+_KEY_IN_PROGRESS = 'idempotency-key-in-progress'
+_OUTCOME_UNKNOWN = 'push-outcome-unknown'
+_keyed_writes_deduped: Dict[str, bool] = {}
+_keyed_writes_override: Optional[bool] = None
+
+
+def _origin_of(url: str) -> str:
+    try:
+        u = httpx.URL(url)
+        return f'{u.scheme}://{u.netloc.decode()}'
+    except Exception:
+        return url
+
+
+def api_dedupes_keyed_writes(url: Optional[str] = None) -> bool:
+    """Whether the API at ``url`` (default: the last one seen) de-duplicates keyed writes."""
+    if _keyed_writes_override is not None:
+        return _keyed_writes_override
+    if url is not None:
+        return _keyed_writes_deduped.get(_origin_of(url)) is True
+    return bool(_keyed_writes_deduped) and list(_keyed_writes_deduped.values())[-1] is True
+
+
+def _reset_keyed_write_support(value: Optional[bool] = None) -> None:
+    """Test hook: forget what was learned about the API, or force True for every origin."""
+    global _keyed_writes_override
+    _keyed_writes_deduped.clear()
+    _keyed_writes_override = True if value is True else None
+
+
+def _note_keyed_answer(url: str, response: httpx.Response) -> None:
+    global _keyed_writes_override
+    origin = _origin_of(url)
+    if IDEMPOTENCY_SUPPORT_HEADER in response.headers:
+        _keyed_writes_deduped.pop(origin, None)
+        _keyed_writes_deduped[origin] = True
+    elif response.is_success:
+        # Stored by a handler that ignored the key: this API (instance) does not de-duplicate.
+        _keyed_writes_deduped.pop(origin, None)
+        _keyed_writes_deduped[origin] = False
+        _keyed_writes_override = None
+
+
+def _mark_outcome_unknown(exc: BaseException) -> BaseException:
+    """Mark an error whose request an earlier attempt may have applied."""
+    try:
+        setattr(exc, 'outcome_unknown', True)
+    except Exception:
+        pass
+    return exc
+
 
 def api_retry_window_secs() -> float:
     """How long one API call keeps retrying before it gives up.
 
     ``SCRAPELY_API_RETRY_WINDOW_SECS`` when set, else ``ApiClient.retry_window_secs``
-    (2 minutes by default).
+    (5 minutes by default).
     """
     raw = os.environ.get('SCRAPELY_API_RETRY_WINDOW_SECS')
     if raw:
@@ -198,7 +256,7 @@ class ApiClient:
     _rate_limiter = AdaptiveRateLimiter()
     # How long one call keeps retrying 429/503/network errors before it gives
     # up (SCRAPELY_API_RETRY_WINDOW_SECS overrides it). Long enough that a
-    # throttled run slows down instead of failing, and covers a main redeploy.
+    # throttled run slows down instead of failing, and covers an API redeploy.
     retry_window_secs = DEFAULT_API_RETRY_WINDOW_SECS
 
     def __init__(
@@ -284,22 +342,26 @@ class ApiClient:
         do_not_retry_timeouts: bool = False,
         idempotent: Optional[bool] = None,
         retry_deadline: Optional[float] = None,
+        idempotency_key: Optional[str] = None,
     ) -> httpx.Response:
         """Make an HTTP request with the API retry policy.
 
         Retries, with exponential backoff and jitter that honours Retry-After,
-        until the retry window (2 minutes by default) runs out:
+        until the retry window (5 minutes by default) runs out:
 
         - 429 (run API rate limit, dataset backpressure): always; nothing was applied.
         - 503 from the limiter or HAProxy: always; nothing was forwarded.
         - 500/502/503/504, read/write timeouts and dropped connections:
-          idempotent calls only.
+          idempotent calls, and writes with an ``idempotency_key`` once the API
+          is known to de-duplicate them (a re-send returns the original answer).
         - Refused connections and connect/pool timeouts: always; the request
           never left.
 
-        A write (dataset push, KV PUT) whose request may have reached the API is
-        never re-sent: there is no idempotency key for it, so a re-send could
-        store the same rows twice. Such failures surface to the caller instead.
+        Any other write (KV PUT, a keyed push to an API that did not say it
+        de-duplicates) whose request may have reached the API is never re-sent:
+        a re-send could apply it twice. Such failures surface to the caller
+        instead, with ``outcome_unknown = True`` when an attempt may have been
+        applied.
 
         ``retry_deadline`` (a ``time.monotonic()`` value) ends the retries
         earlier, so a caller with a time budget (e.g. the exit flush) gets an
@@ -319,6 +381,21 @@ class ApiClient:
 
         if idempotent is None:
             idempotent = method.upper() in IDEMPOTENT_METHODS
+        if idempotency_key is not None:
+            request_headers['Idempotency-Key'] = idempotency_key
+
+        def safe_to_resend() -> bool:
+            # Idempotent calls, and keyed writes once the API showed it
+            # de-duplicates them (learned from any keyed answer).
+            return bool(idempotent) or (idempotency_key is not None and api_dedupes_keyed_writes(url))
+
+        # An attempt of a non-idempotent call may have been applied (timeout,
+        # dropped connection, 5xx): the final error must not claim "nothing applied".
+        maybe_applied = False
+
+        def fail(exc: BaseException) -> BaseException:
+            return _mark_outcome_unknown(exc) if maybe_applied and not idempotent else exc
+
         deadline = time.monotonic() + max(0.0, api_retry_window_secs())
         if retry_deadline is not None:
             deadline = min(deadline, retry_deadline)
@@ -355,36 +432,53 @@ class ApiClient:
             except (httpx.ConnectTimeout, httpx.PoolTimeout, httpx.ConnectError) as exc:
                 # Never reached the API: safe for every method.
                 if isinstance(exc, (httpx.ConnectTimeout, httpx.PoolTimeout)) and do_not_retry_timeouts:
-                    raise
+                    raise fail(exc)
                 if await backoff(f'failed ({type(exc).__name__})', None):
                     continue
-                raise
+                raise fail(exc)
             except httpx.TimeoutException as exc:
                 # Read/write timeout: the request may have been applied.
-                if idempotent and not do_not_retry_timeouts and await backoff(f'timed out ({type(exc).__name__})', None):
+                maybe_applied = True
+                if safe_to_resend() and not do_not_retry_timeouts and await backoff(f'timed out ({type(exc).__name__})', None):
                     continue
-                raise
+                raise fail(exc)
             except httpx.TransportError as exc:
-                # Reset mid-request: only idempotent calls are re-sent.
-                if idempotent and await backoff(f'failed ({type(exc).__name__})', None):
+                # Reset mid-request: re-sent only when that is safe.
+                maybe_applied = True
+                if safe_to_resend() and await backoff(f'failed ({type(exc).__name__})', None):
                     continue
-                raise
+                raise fail(exc)
 
             self._rate_limiter.update_from_headers(response.headers)
             if response.status_code == 429:
                 self._rate_limiter.record_rate_limited()
-
-            if not response.is_success and is_retryable_response(
-                response.status_code,
-                idempotent,
-                _error_type(response) if response.status_code == 503 else None,
-                response.text if response.status_code == 503 else None,
-            ):
-                if await backoff(f'answered {response.status_code}', response.headers.get('Retry-After')):
-                    continue
+            if idempotency_key is not None:
+                _note_keyed_answer(url, response)
+
+            if not response.is_success:
+                status = response.status_code
+                error_type = _error_type(response) if status == 409 or status >= 500 else None
+                body_text = response.text if status == 503 else None
+                # A 5xx other than the limiter's / HAProxy's 503 may come after the write was applied.
+                if not idempotent and status >= 500 and not is_retryable_response(status, False, error_type, body_text):
+                    maybe_applied = True
+                if idempotency_key is not None and error_type == _OUTCOME_UNKNOWN:
+                    # Stored or not, nobody knows: the API refuses repeats of this key. Final.
+                    maybe_applied = True
+                else:
+                    # 409 for a keyed write: an earlier attempt is still being stored.
+                    in_progress = idempotency_key is not None and status == 409 and error_type == _KEY_IN_PROGRESS
+                    if in_progress or is_retryable_response(status, safe_to_resend(), error_type, body_text):
+                        if await backoff(f'answered {status}', response.headers.get('Retry-After')):
+                            continue
+                        if in_progress:
+                            maybe_applied = True
 
             if raise_for_status:
-                await self._raise_for_status(response)
+                try:
+                    await self._raise_for_status(response)
+                except ApiClientError as exc:
+                    raise fail(exc)
             return response
 
     async def get(
@@ -416,6 +510,7 @@ class ApiClient:
         raise_for_status: bool = True,
         idempotent: Optional[bool] = None,
         retry_deadline: Optional[float] = None,
+        idempotency_key: Optional[str] = None,
     ) -> httpx.Response:
         return await self.request(
             'POST',
@@ -427,6 +522,7 @@ class ApiClient:
             raise_for_status=raise_for_status,
             idempotent=idempotent,
             retry_deadline=retry_deadline,
+            idempotency_key=idempotency_key,
         )
 
     async def put(
@@ -479,8 +575,11 @@ class ApiClient:
         headers: Optional[Dict[str, str]] = None,
         *,
         idempotent: Optional[bool] = None,
+        idempotency_key: Optional[str] = None,
     ) -> Any:
-        response = await self.post(path, json_data=data, params=params, headers=headers, idempotent=idempotent)
+        response = await self.post(
+            path, json_data=data, params=params, headers=headers, idempotent=idempotent, idempotency_key=idempotency_key
+        )
         if not response.content:
             return {}
         try:

{scrapely_python-0.1.13 → scrapely_python-0.1.15}/src/scrapely/storage/client.py

@@ -182,7 +182,11 @@ class ApiStorageClient:
         dataset_id: str,
         items: Sequence[Mapping[str, Any]],
     ) -> None:
-        await self.api_client.post_json(f'/v2/datasets/{dataset_id}/items', list(items))
+        # One key per push, the same on every attempt: the API stores a re-sent
+        # push once, so a push whose connection dropped can be retried.
+        await self.api_client.post_json(
+            f'/v2/datasets/{dataset_id}/items', list(items), idempotency_key=str(uuid.uuid4())
+        )
 
     async def dataset_list_items(
         self,

{scrapely_python-0.1.13 → scrapely_python-0.1.15/src/scrapely_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.13
+Version: 0.1.15
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.13 → scrapely_python-0.1.15}/src/scrapely_python.egg-info/SOURCES.txt

@@ -60,6 +60,7 @@ tests/test_input_proxy_aliases.py
 tests/test_parity.py
 tests/test_patch_browser_pool.py
 tests/test_proxy_configuration.py
+tests/test_push_idempotency.py
 tests/test_run_control.py
 tests/test_secrets.py
 tests/test_storage.py

scrapely_python-0.1.15/tests/test_push_idempotency.py

@@ -0,0 +1,242 @@
+"""Dataset pushes carry an Idempotency-Key, so a push whose connection drops
+after sending (socket closed, timeout, 5xx) is re-sent with the same key and
+the API stores it once. The SDK re-sends such a push only after the API showed
+it de-duplicates (X-Scrapely-Idempotency), so an older API never gets a blind
+re-send. A charged push charges once, and its charge is not voided when an
+attempt may have stored the rows.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, Iterator, List, Optional
+
+import httpx
+import pytest
+
+from scrapely.charging import _push_certainly_failed, push_data_and_charge
+from scrapely.storage import api_client as mod
+from scrapely.storage.api_client import (
+    DEFAULT_API_RETRY_WINDOW_SECS,
+    ApiClient,
+    ApiClientError,
+    _reset_keyed_write_support,
+    api_dedupes_keyed_writes,
+    api_retry_window_secs,
+)
+from scrapely.storage.client import ApiStorageClient
+
+from test_charging_pending import FakeClient, _make_manager
+
+
+@pytest.fixture(autouse=True)
+def no_sleep(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    clock = [1000.0]
+
+    async def fake_sleep(secs: float) -> None:
+        clock[0] += secs
+
+    monkeypatch.setattr(mod.asyncio, 'sleep', fake_sleep)
+    monkeypatch.setattr(mod.time, 'monotonic', lambda: clock[0])
+    _reset_keyed_write_support()
+    yield
+    _reset_keyed_write_support()
+
+
+class FakeApi:
+    """Stores pushes once per Idempotency-Key, like the server; scripted failures first."""
+
+    def __init__(self, script: List[str]) -> None:
+        self.script = list(script)
+        self.seen: List[Dict[str, Any]] = []
+        self.stored: List[Any] = []
+        self.keys: set = set()
+
+    def _store(self, key: Optional[str], body: bytes) -> bool:
+        if key and key in self.keys:
+            return False
+        if key:
+            self.keys.add(key)
+        self.stored.append(json.loads(body))
+        return True
+
+    def handler(self, request: httpx.Request) -> httpx.Response:
+        key = request.headers.get('Idempotency-Key')
+        body = request.read()
+        self.seen.append({'key': key, 'body': body})
+        step = self.script.pop(0) if len(self.script) > 1 else self.script[0]
+        if step == 'drop':  # stored, then the connection closed before the answer
+            self._store(key, body)
+            raise httpx.RemoteProtocolError('Server disconnected without sending a response.', request=request)
+        if step == 'timeout':  # stored, answer never arrived
+            self._store(key, body)
+            raise httpx.ReadTimeout('timed out', request=request)
+        if step == 'plain201':  # an API without idempotency
+            self._store(key, body)
+            return httpx.Response(201, json={})
+        if step == '502':
+            return httpx.Response(502, json={'error': {'type': 'bad-gateway'}})
+        if step == '409':
+            return httpx.Response(409, headers={'Retry-After': '0'}, json={'error': {'type': 'idempotency-key-in-progress'}})
+        if step == 'unknown':
+            return httpx.Response(503, headers={'X-Scrapely-Idempotency': 'refused'}, json={'error': {'type': 'push-outcome-unknown', 'message': 'unknown'}})
+        if step == '400':
+            return httpx.Response(400, json={'error': {'type': 'invalid-input', 'message': 'bad'}})
+        assert step == 'dedupe'
+        fresh = self._store(key, body)
+        headers = {'X-Scrapely-Idempotency': 'stored' if fresh else 'replayed'} if key else {}
+        return httpx.Response(201, headers=headers, json={})
+
+    def storage(self) -> ApiStorageClient:
+        return ApiStorageClient('http://api.test', 'st_x', 'run1', transport=httpx.MockTransport(self.handler))
+
+
+def test_default_retry_window_is_five_minutes(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv('SCRAPELY_API_RETRY_WINDOW_SECS', raising=False)
+    assert DEFAULT_API_RETRY_WINDOW_SECS == 300.0
+    assert ApiClient.retry_window_secs == 300.0
+    assert api_retry_window_secs() == 300.0
+
+
+async def test_each_push_has_its_own_key_and_the_header_teaches_support() -> None:
+    api = FakeApi(['dedupe'])
+    s = api.storage()
+    await s.dataset_push_items('ds', [{'a': 1}])
+    await s.dataset_push_items('ds', [{'a': 2}])
+    keys = [x['key'] for x in api.seen]
+    assert all(keys) and keys[0] != keys[1]
+    assert len(keys[0]) == 36
+    assert api_dedupes_keyed_writes() is True
+
+
+async def test_api_without_idempotency_does_not_get_a_resend() -> None:
+    api = FakeApi(['plain201', 'drop', 'plain201'])
+    s = api.storage()
+    await s.dataset_push_items('ds', [{'warmup': True}])
+    assert api_dedupes_keyed_writes() is False
+    with pytest.raises(httpx.RemoteProtocolError) as e:
+        await s.dataset_push_items('ds', [{'a': 1}])
+    assert len(api.seen) == 2, 'not re-sent'
+    assert getattr(e.value, 'outcome_unknown', False) is True
+    assert _push_certainly_failed(e.value) is False
+
+
+async def test_support_is_per_origin_and_revoked_by_a_plain_2xx() -> None:
+    api_a = FakeApi(['dedupe'])
+    await api_a.storage().dataset_push_items('ds', [{'a': 1}])
+    assert api_dedupes_keyed_writes('http://api.test/v2/x') is True
+    api_b = FakeApi(['drop'])
+    b = ApiStorageClient('http://other.test', 'st_x', 'run1', transport=httpx.MockTransport(api_b.handler))
+    assert api_dedupes_keyed_writes('http://other.test/v2/x') is False
+    with pytest.raises(httpx.RemoteProtocolError):
+        await b.dataset_push_items('ds', [{'b': 1}])
+    assert len(api_b.seen) == 1, 'another origin is not trusted'
+
+    # Mixed versions behind one origin: a 2xx without the header turns retries off.
+    _reset_keyed_write_support()
+    api = FakeApi(['dedupe', 'plain201', 'drop', 'dedupe'])
+    s = api.storage()
+    await s.dataset_push_items('ds', [{'a': 1}])
+    assert api_dedupes_keyed_writes('http://api.test') is True
+    await s.dataset_push_items('ds', [{'a': 2}])
+    assert api_dedupes_keyed_writes('http://api.test') is False
+    with pytest.raises(httpx.RemoteProtocolError):
+        await s.dataset_push_items('ds', [{'a': 3}])
+    assert len(api.seen) == 3, 'the dropped push was not re-sent'
+
+
+@pytest.mark.parametrize('failure', ['drop', 'timeout'])
+async def test_dropped_push_is_resent_with_the_same_key_and_stored_once(failure: str) -> None:
+    _reset_keyed_write_support(True)
+    api = FakeApi([failure, 'dedupe'])
+    await api.storage().dataset_push_items('ds', [{'a': 1}, {'a': 2}])
+    assert len(api.seen) == 2
+    assert api.seen[0]['key'] == api.seen[1]['key']
+    assert api.seen[0]['body'] == api.seen[1]['body']
+    assert api.stored == [[{'a': 1}, {'a': 2}]]
+
+
+async def test_5xx_and_409_are_retried_outcome_unknown_is_final() -> None:
+    _reset_keyed_write_support(True)
+    api = FakeApi(['502', '409', 'dedupe'])
+    await api.storage().dataset_push_items('ds', [{'a': 1}])
+    assert len(api.seen) == 3
+    assert len({x['key'] for x in api.seen}) == 1
+    assert len(api.stored) == 1
+
+    api = FakeApi(['unknown'])
+    with pytest.raises(ApiClientError) as e:
+        await api.storage().dataset_push_items('ds', [{'a': 1}])
+    assert e.value.status_code == 503
+    assert len(api.seen) == 1
+    assert _push_certainly_failed(e.value) is False
+
+
+async def test_final_4xx_after_a_maybe_applied_attempt_is_not_certain() -> None:
+    _reset_keyed_write_support(True)
+    api = FakeApi(['drop', '400'])
+    with pytest.raises(ApiClientError) as e:
+        await api.storage().dataset_push_items('ds', [{'a': 1}])
+    assert len(api.seen) == 2
+    assert _push_certainly_failed(e.value) is False
+
+    api = FakeApi(['400'])
+    with pytest.raises(ApiClientError) as e2:
+        await api.storage().dataset_push_items('ds', [{'a': 1}])
+    assert _push_certainly_failed(e2.value) is True
+
+
+async def test_kv_writes_send_no_key_and_are_not_resent() -> None:
+    _reset_keyed_write_support(True)
+    api = FakeApi(['drop', 'plain201'])
+    client = api.storage().api_client
+    with pytest.raises(httpx.RemoteProtocolError):
+        await client.put('/v2/key-value-stores/kv/records/k', json_data={'a': 1})
+    assert len(api.seen) == 1
+    assert api.seen[0]['key'] is None
+
+
+async def test_charged_push_resent_after_drop_charges_once_and_stores_once() -> None:
+    _reset_keyed_write_support(True)
+    client = FakeClient()
+    mgr = _make_manager(client)
+    api = FakeApi(['drop', 'dedupe'])
+    storage = api.storage()
+
+    async def push(items: Any) -> None:
+        await storage.dataset_push_items('ds', items)
+
+    result = await push_data_and_charge(mgr, [{'a': 1}, {'a': 2}], 'page-open', False, push)
+    assert result.charged_count == 2
+    assert len(api.seen) == 2
+    assert len(api.stored) == 1
+    assert len([p for p in client.posts if p['path'].endswith('/charge')]) == 1
+    assert not [p for p in client.posts if p['path'].endswith('/charge/void')]
+    assert mgr.get_charged_event_count('page-open') == 2
+
+
+async def test_charged_push_that_may_have_stored_keeps_its_charge() -> None:
+    _reset_keyed_write_support(True)
+    client = FakeClient()
+    mgr = _make_manager(client)
+    api = FakeApi(['drop', 'unknown'])
+    storage = api.storage()
+
+    async def push(items: Any) -> None:
+        await storage.dataset_push_items('ds', items)
+
+    with pytest.raises(ApiClientError):
+        await push_data_and_charge(mgr, [{'a': 1}], 'page-open', False, push)
+    assert len([p for p in client.posts if p['path'].endswith('/charge')]) == 1
+    assert not [p for p in client.posts if p['path'].endswith('/charge/void')], 'rows may be stored: charge stands'
+
+    # Certainly not stored (a 4xx on the only attempt): voided as before.
+    api2 = FakeApi(['400'])
+    storage2 = api2.storage()
+
+    async def push2(items: Any) -> None:
+        await storage2.dataset_push_items('ds', items)
+
+    with pytest.raises(ApiClientError):
+        await push_data_and_charge(mgr, [{'a': 2}], 'page-open', False, push2)
+    assert len([p for p in client.posts if p['path'].endswith('/charge/void')]) == 1

{scrapely_python-0.1.13 → scrapely_python-0.1.15}/tests/test_terminal_status_message.py

@@ -128,3 +128,22 @@ async def test_context_manager_status_message_is_terminal(lifecycle, monkeypatch
         ('/v2/actor-runs/run123', {'statusMessage': 'Crashed while parsing', 'isStatusMessageTerminal': True}),
     )
     assert events[1][0] == 'shutdown'
+
+
+async def test_progress_status_message_is_best_effort(lifecycle, caplog: pytest.LogCaptureFixture) -> None:
+    """A non-terminal update that still fails after the retry window (e.g. a long 503)
+    logs a warning and returns instead of raising and crashing the run."""
+    events, exit_codes, client = lifecycle
+    client.fail = True
+    with caplog.at_level('WARNING'):
+        run = await Actor.set_status_message('Scraped 10 pages')
+    assert run.id == ''
+    assert 'Failed to update the status message' in caplog.text
+    assert exit_codes == []
+
+
+async def test_terminal_status_message_still_raises(lifecycle) -> None:
+    _, _, client = lifecycle
+    client.fail = True
+    with pytest.raises(RuntimeError, match='API down'):
+        await Actor.set_status_message('Done', is_terminal=True)