scrapely-python 0.1.3__tar.gz → 0.1.5__tar.gz

{scrapely_python-0.1.3 → scrapely_python-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.3
+Version: 0.1.5
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT

{scrapely_python-0.1.3 → scrapely_python-0.1.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.3"
+version = "0.1.5"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT"

{scrapely_python-0.1.3 → scrapely_python-0.1.5}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.3'
+__version__ = '0.1.5'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.3 → scrapely_python-0.1.5}/src/scrapely/charging.py

@@ -199,7 +199,7 @@ class ChargingManager:
         # (push_data_and_charge voids it when the push then fails).
         self._last_accepted_operation_id: Optional[str] = None
         # Voids not yet confirmed by the platform; retried before the next push and at exit.
-        self._pending_voids: List[tuple[str, str, int]] = []
+        self._pending_voids: List[tuple[str, str, int, bool]] = []
 
     def close(self) -> None:
         """Mark the manager closed; subsequent charge/push operations are rejected."""
@@ -439,7 +439,18 @@ class ChargingManager:
         self._last_accepted_operation_id = None
         return op_id
 
-    async def _void_charge(self, event_name: str, operation_id: str, reserved: int) -> bool:
+    async def _cancel_uncertain_charge(self, event_name: str, operation_id: str, reserved: int) -> bool:
+        """Cancel an unconfirmed charge for items that were then not stored.
+
+        Drops the pending intent (so the exit flush never posts it) and voids it in
+        case the original request did apply; a 404 means it never applied.
+        """
+        self._drop_pending(operation_id)
+        return await self._void_charge(event_name, operation_id, reserved, not_found_means_voided=True)
+
+    async def _void_charge(
+        self, event_name: str, operation_id: str, reserved: int, not_found_means_voided: bool = False
+    ) -> bool:
         """Reverse an accepted charge whose items were never stored, then unwind locally.
 
         POST /v2/actor-runs/{run}/charge/void with the charge's Idempotency-Key.
@@ -458,13 +469,19 @@ class ChargingManager:
                 return True
             except Exception as err:  # noqa: BLE001
                 status = _http_status_of(err)
+                # Only the void route's own answer proves it never applied; a server
+                # without the route (older deployment) also answers 404.
+                if status == 404 and not_found_means_voided and getattr(err, 'type', None) == 'charge-not-found':
+                    # The unconfirmed charge never reached the ledger: nothing billed.
+                    self._unwind_reservation(event_name, reserved)
+                    return True
                 # 404: nothing to void; 409: already settled; 4xx auth: permanent.
                 if status is not None and 400 <= status < 500:
                     return False
                 if attempt + 1 < self._POST_ATTEMPTS:
                     await asyncio.sleep(0.25 * (attempt + 1))
         # Outcome unknown: keep the reservation and retry before the next push / at exit.
-        self._pending_voids.append((event_name, operation_id, reserved))
+        self._pending_voids.append((event_name, operation_id, reserved, not_found_means_voided))
         logger.error('Void of charge %s (%r) not confirmed yet; will retry', operation_id, event_name)
         return False
 
@@ -474,8 +491,8 @@ class ChargingManager:
         Returns False while any earlier void is still unconfirmed.
         """
         pending, self._pending_voids = self._pending_voids, []
-        for event_name, operation_id, reserved in pending:
-            await self._void_charge(event_name, operation_id, reserved)
+        for event_name, operation_id, reserved, not_found_means_voided in pending:
+            await self._void_charge(event_name, operation_id, reserved, not_found_means_voided)
         return not self._pending_voids
 
     def _quarantine_stale_pending(self) -> None:
@@ -1158,19 +1175,39 @@ async def push_data_and_charge(
             # Synthetic storage event first (local-only); then explicit billable event.
             reserved_in_op: list[tuple[str, int]] = []
             operation_ids: list[Optional[str]] = []
+            uncertain_ops: set[str] = set()
+
+            async def charge_one(name: str) -> None:
+                charging_manager._take_last_accepted_operation_id()
+                try:
+                    r = await charging_manager._charge_exclusive(name, len(limited_items))
+                    op_id = charging_manager._take_last_accepted_operation_id()
+                except ChargeUncertainError as unconfirmed:
+                    # The platform did not confirm the charge (5xx / timeouts after
+                    # retries). Never drop the items for that: the intent stays
+                    # pending with its idempotency key and is re-posted by the exit
+                    # flush, so the items are stored now and billed once.
+                    logger.warning(
+                        'Charge for %r not confirmed yet (%s); storing the items and retrying the charge later',
+                        name,
+                        unconfirmed.operation_id,
+                    )
+                    r = ChargeResult(
+                        chargeable_within_limit=charging_manager.compute_chargeable(),
+                        charged_count=unconfirmed.count,
+                        event_charge_limit_reached=False,
+                    )
+                    op_id = unconfirmed.operation_id
+                    uncertain_ops.add(op_id)
+                results.append(r)
+                reserved_in_op.append((name, r.charged_count))
+                operation_ids.append(op_id)
+
             try:
                 if default_event:
-                    charging_manager._take_last_accepted_operation_id()
-                    r = await charging_manager._charge_exclusive(default_event, len(limited_items))
-                    results.append(r)
-                    reserved_in_op.append((default_event, r.charged_count))
-                    operation_ids.append(charging_manager._take_last_accepted_operation_id())
+                    await charge_one(default_event)
                 if event_name and event_name != default_event:
-                    charging_manager._take_last_accepted_operation_id()
-                    r = await charging_manager._charge_exclusive(event_name, len(limited_items))
-                    results.append(r)
-                    reserved_in_op.append((event_name, r.charged_count))
-                    operation_ids.append(charging_manager._take_last_accepted_operation_id())
+                    await charge_one(event_name)
             except Exception:
                 for name, reserved in reserved_in_op:
                     if reserved > 0:
@@ -1192,11 +1229,31 @@ async def push_data_and_charge(
                 )
                 # Unwind any event that reserved more than we will persist
                 # (synthetic default-dataset is local-only and otherwise sticks).
+                platform_op_names = {
+                    name for (name, _), op_id in zip(reserved_in_op, operation_ids) if op_id
+                }
                 for name, result in charged_events:
                     if result.charged_count > accepted_for_items:
+                        # Nothing stored and a platform charge was made: the void
+                        # below releases its local count once the platform confirms.
+                        if accepted_for_items <= 0 and name in platform_op_names:
+                            continue
                         charging_manager._unwind_reservation(
                             name, result.charged_count - accepted_for_items
                         )
+                if accepted_for_items <= 0:
+                    # Nothing will be stored (another event of this push was capped
+                    # to zero): undo every platform charge this operation made, so
+                    # no result is billed for items that are never saved. Local
+                    # counts were unwound just above.
+                    for (name, reserved), op_id in zip(reserved_in_op, operation_ids):
+                        if reserved <= 0 or not op_id:
+                            continue
+                        # Original count: released only when the platform confirms.
+                        if op_id in uncertain_ops:
+                            await charging_manager._cancel_uncertain_charge(name, op_id, reserved)
+                        else:
+                            await charging_manager._void_charge(name, op_id, reserved)
                 if accepted_for_items > 0:
                     try:
                         await push_fn(limited_items[:accepted_for_items])
@@ -1218,7 +1275,10 @@ async def push_data_and_charge(
                             kept = min(accepted_for_items, next(r.charged_count for n, r in charged_events if n == name))
                             if kept <= 0:
                                 continue
-                            if op_id:
+                            if op_id and op_id in uncertain_ops:
+                                # Never re-post a charge for items that were not stored.
+                                await charging_manager._cancel_uncertain_charge(name, op_id, kept)
+                            elif op_id:
                                 await charging_manager._void_charge(name, op_id, kept)
                             elif is_default_dataset_item_event(name):
                                 charging_manager._unwind_reservation(name, kept)

{scrapely_python-0.1.3 → scrapely_python-0.1.5}/src/scrapely/storage/api_client.py

@@ -24,6 +24,32 @@ class ApiClientError(Exception):
         self.body = body
 
 
+# PUT is left out: a KV record PUT is counted (and billed) as a write each time
+# it is applied, so a lost response must not be re-sent blindly.
+IDEMPOTENT_METHODS = frozenset({'GET', 'HEAD', 'DELETE', 'OPTIONS'})
+# HAProxy's own 503 page: no backend was available, so nothing was forwarded.
+NO_BACKEND_MARKER = 'No server is available to handle this request'
+
+
+def is_retryable_response(
+    status: int, idempotent: bool, error_type: Optional[str] = None, body_text: Optional[str] = None
+) -> bool:
+    """Whether a non-2xx answer is retried.
+
+    Idempotent methods: 500/502/503/504. Others (dataset push, KV PUT) only on
+    a 503 that certainly was not processed: the rate limiter refused it
+    ("rate-limit-unavailable", before the handler), or HAProxy had no backend
+    (its own page, nothing forwarded). Any other 503 may mean "outcome unknown"
+    (a direct dataset write whose commit was not confirmed), and re-sending
+    could store the items twice.
+    """
+    if idempotent:
+        return status in (500, 502, 503, 504)
+    if status != 503:
+        return False
+    return error_type == 'rate-limit-unavailable' or NO_BACKEND_MARKER in (body_text or '')
+
+
 def unwrap_data(value: Any) -> Any:
     """Unwrap Apify-style v2 response envelopes ({ data: ... })."""
     if isinstance(value, list):
@@ -33,6 +59,35 @@ def unwrap_data(value: Any) -> Any:
     return value
 
 
+def _error_type(response: httpx.Response) -> Optional[str]:
+    try:
+        body = response.json()
+    except Exception:
+        return None
+    if isinstance(body, dict) and isinstance(body.get('error'), dict):
+        t = body['error'].get('type')
+        return t if isinstance(t, str) else None
+    return None
+
+
+def server_retry_delay(server_attempt: int, retry_after: Optional[str], now: Optional[float] = None) -> float:
+    """Exponential backoff with jitter, in seconds; honours a Retry-After (seconds or HTTP date) of up to 30 s."""
+    if retry_after is not None and retry_after.strip():
+        secs: Optional[float] = None
+        try:
+            secs = float(retry_after)
+        except ValueError:
+            try:
+                from email.utils import parsedate_to_datetime
+
+                secs = max(0.0, parsedate_to_datetime(retry_after).timestamp() - (time.time() if now is None else now))
+            except Exception:
+                secs = None
+        if secs is not None and secs >= 0:
+            return min(secs, 30.0) + random.random() * 0.1
+    return min(8.0, 0.5 * 2 ** server_attempt) + random.random() * 0.25
+
+
 class AdaptiveRateLimiter:
     """Adaptive throttling based on X-RateLimit-* headers (TS AdaptiveRateLimiter)."""
 
@@ -116,6 +171,10 @@ class ApiClient:
 
     _rate_limiter = AdaptiveRateLimiter()
     _max_retries = 3
+    # Retries while the API is briefly unavailable (503/5xx, connection
+    # refused): backoff 0.5 s doubling to 8 s, about 45 s in total, which
+    # covers a main redeploy.
+    _max_server_retries = 8
 
     def __init__(
         self,
@@ -198,6 +257,7 @@ class ApiClient:
         headers: Optional[Dict[str, str]] = None,
         raise_for_status: bool = True,
         do_not_retry_timeouts: bool = False,
+        idempotent: Optional[bool] = None,
     ) -> httpx.Response:
         """Make an HTTP request with adaptive rate limiting and 429 retries."""
         client = await self._ensure_client()
@@ -208,8 +268,11 @@ class ApiClient:
         if json_data is not None and 'Content-Type' not in request_headers:
             request_headers['Content-Type'] = 'application/json'
 
-        last_error: Optional[Exception] = None
-        for attempt in range(self._max_retries + 1):
+        if idempotent is None:
+            idempotent = method.upper() in IDEMPOTENT_METHODS
+        attempt = 0  # timeouts and 429s
+        server_attempt = 0  # 503/5xx and connection failures (API briefly unavailable)
+        while True:
             await self._rate_limiter.wait_for_slot()
             try:
                 response = await client.request(
@@ -220,27 +283,59 @@ class ApiClient:
                     content=content,
                     headers=request_headers or None,
                 )
-            except httpx.TimeoutException as exc:
-                last_error = exc
+            except httpx.TimeoutException:
                 if do_not_retry_timeouts or attempt >= self._max_retries:
                     raise
-                await asyncio.sleep(0.25 * (attempt + 1))
+                attempt += 1
+                await asyncio.sleep(0.25 * attempt)
                 continue
+            except httpx.TransportError as exc:
+                # No response. A refused/unresolvable connection never reached
+                # the API, so it is safe for every method; anything else (reset
+                # mid-request) only for idempotent ones.
+                if (idempotent or isinstance(exc, httpx.ConnectError)) and server_attempt < self._max_server_retries:
+                    wait = server_retry_delay(server_attempt, None)
+                    server_attempt += 1
+                    logger.warning(
+                        '%s %s failed (%s); retry %d/%d in %.1fs',
+                        method, path, type(exc).__name__, server_attempt, self._max_server_retries, wait,
+                    )
+                    await asyncio.sleep(wait)
+                    continue
+                raise
 
             self._rate_limiter.update_from_headers(response.headers)
 
             if response.status_code == 429 and attempt < self._max_retries:
+                attempt += 1
                 await self._rate_limiter.handle_rate_limited(response)
                 continue
 
+            # API briefly unavailable (main redeploy, limiter store blip); see
+            # is_retryable_response for which answers are safe to re-send.
+            if (
+                not response.is_success
+                and server_attempt < self._max_server_retries
+                and is_retryable_response(
+                    response.status_code,
+                    idempotent,
+                    _error_type(response) if response.status_code == 503 else None,
+                    response.text if response.status_code == 503 else None,
+                )
+            ):
+                wait = server_retry_delay(server_attempt, response.headers.get('Retry-After'))
+                server_attempt += 1
+                logger.warning(
+                    '%s %s answered %d; retry %d/%d in %.1fs',
+                    method, path, response.status_code, server_attempt, self._max_server_retries, wait,
+                )
+                await asyncio.sleep(wait)
+                continue
+
             if raise_for_status:
                 await self._raise_for_status(response)
             return response
 
-        if last_error is not None:
-            raise last_error
-        raise ApiClientError('Request failed after retries', 429)
-
     async def get(
         self,
         path: str,
@@ -268,6 +363,7 @@ class ApiClient:
         params: Optional[Dict[str, Any]] = None,
         *,
         raise_for_status: bool = True,
+        idempotent: Optional[bool] = None,
     ) -> httpx.Response:
         return await self.request(
             'POST',
@@ -277,6 +373,7 @@ class ApiClient:
             content=content,
             headers=headers,
             raise_for_status=raise_for_status,
+            idempotent=idempotent,
         )
 
     async def put(
@@ -325,8 +422,10 @@ class ApiClient:
         data: Any = None,
         params: Optional[Dict[str, Any]] = None,
         headers: Optional[Dict[str, str]] = None,
+        *,
+        idempotent: Optional[bool] = None,
     ) -> Any:
-        response = await self.post(path, json_data=data, params=params, headers=headers)
+        response = await self.post(path, json_data=data, params=params, headers=headers, idempotent=idempotent)
         if not response.content:
             return {}
         try:

{scrapely_python-0.1.3 → scrapely_python-0.1.5}/src/scrapely_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.3
+Version: 0.1.5
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT

{scrapely_python-0.1.3 → scrapely_python-0.1.5}/src/scrapely_python.egg-info/SOURCES.txt

@@ -44,6 +44,7 @@ src/scrapely_python.egg-info/SOURCES.txt
 src/scrapely_python.egg-info/dependency_links.txt
 src/scrapely_python.egg-info/requires.txt
 src/scrapely_python.egg-info/top_level.txt
+tests/test_api_client_retries.py
 tests/test_apify_parity.py
 tests/test_browser_configuration.py
 tests/test_charging_parity.py

scrapely_python-0.1.5/tests/test_api_client_retries.py

@@ -0,0 +1,170 @@
+"""Storage calls ride out a briefly unavailable API.
+
+Staging 2026-10-15: while main redeployed, storage calls got 503 "Rate limiter
+is temporarily unavailable" (and refused connections) and runs failed, because
+the client retried only 429s. 503 and refused connections now retry with
+backoff for every method; 500/502/504 and mid-request transport errors only for
+idempotent methods.
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+import httpx
+import pytest
+
+from scrapely.storage import api_client as mod
+from scrapely.storage.api_client import ApiClient, ApiClientError, is_retryable_response, server_retry_delay
+
+
+@pytest.fixture(autouse=True)
+def no_sleep(monkeypatch: pytest.MonkeyPatch) -> List[float]:
+    slept: List[float] = []
+
+    async def fake_sleep(secs: float) -> None:
+        slept.append(secs)
+
+    monkeypatch.setattr(mod.asyncio, 'sleep', fake_sleep)
+    return slept
+
+
+def client_for(responses: list) -> tuple[ApiClient, list]:
+    calls: list = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls.append(request.method)
+        r = responses.pop(0) if len(responses) > 1 else responses[0]
+        if isinstance(r, Exception):
+            raise r
+        return r
+
+    return ApiClient('http://api.test', 'st_x', 'run1', transport=httpx.MockTransport(handler)), calls
+
+
+def unavailable() -> httpx.Response:
+    return httpx.Response(503, json={'error': {'type': 'rate-limit-unavailable', 'message': 'Rate limiter is temporarily unavailable'}})
+
+
+def outcome_unknown() -> httpx.Response:
+    return httpx.Response(503, json={'error': {'type': 'service-unavailable', 'message': 'Dataset storage is temporarily unavailable, try again'}})
+
+
+async def test_post_not_resent_after_outcome_unknown_503() -> None:
+    c, calls = client_for([outcome_unknown(), httpx.Response(201, json={})])
+    with pytest.raises(ApiClientError) as e:
+        await c.request('POST', '/v2/datasets/d/items', json_data=[{}])
+    assert e.value.status_code == 503
+    assert calls == ['POST'], 'the items may already be stored'
+
+    c, calls = client_for([outcome_unknown(), httpx.Response(200, json={})])
+    assert (await c.request('GET', '/v2/datasets/d')).status_code == 200
+    assert len(calls) == 2
+
+
+async def test_post_resent_after_proxy_503() -> None:
+    proxy = httpx.Response(503, text='<html>No server is available to handle this request.</html>', headers={'content-type': 'text/html'})
+    other = httpx.Response(503, text='<html>Service Unavailable</html>', headers={'content-type': 'text/html'})
+    c, calls = client_for([other, httpx.Response(201, json={})])
+    with pytest.raises(ApiClientError):
+        await c.request('POST', '/v2/datasets/d/items', json_data=[{}])
+    assert len(calls) == 1, 'a non-JSON 503 without the no-backend marker may have been forwarded'
+    c, calls = client_for([proxy, httpx.Response(201, json={})])
+    assert (await c.request('POST', '/v2/datasets/d/items', json_data=[{}])).status_code == 201
+    assert len(calls) == 2
+
+
+async def test_record_read_with_raise_for_status_false_still_retries() -> None:
+    c, calls = client_for([unavailable(), httpx.Response(404, json={'error': {'message': 'Record not found'}})])
+    r = await c.request('GET', '/v2/key-value-stores/k/records/STATE', raise_for_status=False)
+    assert r.status_code == 404
+    assert len(calls) == 2
+
+
+async def test_post_retries_503_until_success(no_sleep: List[float]) -> None:
+    c, calls = client_for([unavailable(), unavailable(), httpx.Response(201, json={'data': {}})])
+    r = await c.request('POST', '/v2/datasets/d/items', json_data=[{'a': 1}])
+    assert r.status_code == 201
+    assert calls == ['POST'] * 3
+    backoff = [x for x in no_sleep if x >= 0.4]  # the rate limiter's own pacing sleeps are shorter
+    assert len(backoff) == 2 and 0.5 <= backoff[0] < 0.8 and 1.0 <= backoff[1] < 1.3  # exponential
+
+
+async def test_retry_after_is_honoured(no_sleep: List[float]) -> None:
+    c, _ = client_for([httpx.Response(503, headers={'Retry-After': '3'}), httpx.Response(200, json={})])
+    await c.request('GET', '/v2/key-value-stores/k/records/x')
+    assert any(3.0 <= x < 3.2 for x in no_sleep)
+
+
+async def test_gives_up_after_budget() -> None:
+    c, calls = client_for([unavailable()])
+    with pytest.raises(ApiClientError) as e:
+        await c.request('GET', '/v2/datasets/d')
+    assert e.value.status_code == 503
+    assert len(calls) == ApiClient._max_server_retries + 1
+
+
+async def test_502_retried_for_get_not_for_post() -> None:
+    c, calls = client_for([httpx.Response(502), httpx.Response(200, json={})])
+    assert (await c.request('GET', '/v2/datasets/d')).status_code == 200
+    assert len(calls) == 2
+
+    c, calls = client_for([httpx.Response(502), httpx.Response(201, json={})])
+    with pytest.raises(ApiClientError) as e:
+        await c.request('POST', '/v2/datasets/d/items', json_data=[{}])
+    assert e.value.status_code == 502
+    assert calls == ['POST'], 'a POST that may have been processed must not be re-sent'
+
+
+async def test_connection_refused_retried_for_every_method() -> None:
+    c, calls = client_for([httpx.ConnectError('refused'), httpx.Response(201, json={})])
+    assert (await c.request('POST', '/v2/datasets/d/items', json_data=[{}])).status_code == 201
+    assert len(calls) == 2
+
+
+async def test_mid_request_reset_only_retried_when_idempotent() -> None:
+    c, calls = client_for([httpx.ReadError('reset'), httpx.Response(200, json={})])
+    assert (await c.request('GET', '/v2/key-value-stores/k/records/x')).status_code == 200
+    assert len(calls) == 2
+
+    # PUT is a billed KV write: not re-sent after a reset or a 502 (may have been applied) ...
+    c, calls = client_for([httpx.ReadError('reset'), httpx.Response(200, json={})])
+    with pytest.raises(httpx.ReadError):
+        await c.request('PUT', '/v2/key-value-stores/k/records/x', content=b'x')
+    assert len(calls) == 1
+    c, calls = client_for([httpx.Response(502), httpx.Response(200, json={})])
+    with pytest.raises(ApiClientError):
+        await c.request('PUT', '/v2/key-value-stores/k/records/x', content=b'x')
+    assert len(calls) == 1
+    # ... but is after the limiter refused it.
+    c, calls = client_for([unavailable(), httpx.Response(200, json={})])
+    assert (await c.request('PUT', '/v2/key-value-stores/k/records/x', content=b'x')).status_code == 200
+    assert len(calls) == 2
+
+    c, calls = client_for([httpx.ReadError('reset'), httpx.Response(201, json={})])
+    with pytest.raises(httpx.ReadError):
+        await c.request('POST', '/v2/datasets/d/items', json_data=[{}])
+    assert len(calls) == 1
+
+
+async def test_client_errors_are_not_retried() -> None:
+    c, calls = client_for([httpx.Response(404, json={'error': {'message': 'nope'}})])
+    with pytest.raises(ApiClientError):
+        await c.request('GET', '/v2/datasets/missing')
+    assert len(calls) == 1
+
+
+def test_helpers() -> None:
+    assert is_retryable_response(503, True, 'service-unavailable')
+    assert not is_retryable_response(503, False, 'service-unavailable')
+    assert is_retryable_response(503, False, 'rate-limit-unavailable')
+    assert is_retryable_response(503, False, None, '<h1>503</h1> No server is available to handle this request.')
+    assert not is_retryable_response(503, False, None, '<html>Service Unavailable</html>')
+    assert is_retryable_response(504, True) and not is_retryable_response(504, False)
+    assert not is_retryable_response(400, True) and not is_retryable_response(501, True)
+    from email.utils import parsedate_to_datetime
+    now = parsedate_to_datetime('Thu, 15 Oct 2026 00:00:00 GMT').timestamp()
+    assert 5.0 <= server_retry_delay(0, 'Thu, 15 Oct 2026 00:00:05 GMT', now=now) < 5.2
+    assert 4.0 <= server_retry_delay(3, 'garbage') < 4.3
+    assert server_retry_delay(10, None) < 8.3
+    assert 30.0 <= server_retry_delay(0, '120') < 30.2

{scrapely_python-0.1.3 → scrapely_python-0.1.5}/tests/test_charging_pending.py

@@ -659,3 +659,86 @@ async def test_unpriced_explicit_event_with_priced_default_is_gated() -> None:
 
     with pytest.raises(RuntimeError, match='could not be voided yet'):
         await push_data_and_charge(mgr, [{'b': 1}], 'not-priced', True, ok_push)
+
+
+@pytest.mark.asyncio
+async def test_charge_5xx_never_drops_items_and_flush_bills_once() -> None:
+    client = FakeClient(fail_times=1)  # first charge POST fails (unconfirmed)
+    mgr = _make_manager(client)
+    pushed: List[Any] = []
+
+    async def ok_push(items: Any) -> None:
+        pushed.extend(items)
+
+    result = await push_data_and_charge(mgr, [{'a': 1}, {'a': 2}], 'page-open', False, ok_push)
+    assert pushed == [{'a': 1}, {'a': 2}], 'items must be stored despite the failed charge'
+    assert result.charged_count == 2
+    charge_keys = [p['headers']['Idempotency-Key'] for p in client.posts if p['path'].endswith('/charge')]
+    await mgr.flush_pending()
+    charge_keys_after = [p['headers']['Idempotency-Key'] for p in client.posts if p['path'].endswith('/charge')]
+    assert len(charge_keys) == 1 and charge_keys_after == charge_keys * 2, 'same key re-posted once by the flush'
+    assert mgr.get_charged_event_count('page-open') == 2
+
+
+class Void404Client(FakeClient):
+    async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, **kwargs: Any) -> Dict[str, Any]:
+        if path.endswith('/charge/void'):
+            self.posts.append({'path': path, 'json': json_data, 'headers': headers or {}})
+
+            class NotFound(Exception):
+                status_code = 404
+                type = 'charge-not-found'
+
+            raise NotFound('charge not found')
+        return await super().post(path, json_data=json_data, headers=headers, **kwargs)
+
+
+@pytest.mark.asyncio
+async def test_unconfirmed_charge_for_unsaved_items_is_cancelled() -> None:
+    client = Void404Client(fail_times=1)
+    mgr = _make_manager(client)
+
+    class PushHttpError(Exception):
+        status_code = 502
+
+    async def failing_push(_items: Any) -> None:
+        raise PushHttpError('bad gateway')
+
+    with pytest.raises(PushHttpError):
+        await push_data_and_charge(mgr, [{'a': 1}], 'page-open', False, failing_push)
+    key = next(p['headers']['Idempotency-Key'] for p in client.posts if p['path'].endswith('/charge'))
+    assert any(p['path'].endswith('/charge/void') and p['headers']['Idempotency-Key'] == key for p in client.posts)
+    await mgr.flush_pending()
+    assert len([p for p in client.posts if p['path'].endswith('/charge')]) == 1, 'never re-posted'
+    assert mgr.get_charged_event_count('page-open') == 0
+
+
+
+@pytest.mark.asyncio
+async def test_void_route_missing_404_is_not_never_applied() -> None:
+    class GenericNotFoundClient(FakeClient):
+        async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, **kwargs: Any) -> Dict[str, Any]:
+            if path.endswith('/charge/void'):
+                self.posts.append({'path': path, 'json': json_data, 'headers': headers or {}})
+
+                class RouteNotFound(Exception):
+                    status_code = 404
+                    type = 'not-found'
+
+                raise RouteNotFound('route not found')
+            return await super().post(path, json_data=json_data, headers=headers, **kwargs)
+
+    client = GenericNotFoundClient(fail_times=1)
+    mgr = _make_manager(client)
+
+    class PushHttpError(Exception):
+        status_code = 502
+
+    async def failing_push(_items: Any) -> None:
+        raise PushHttpError('bad gateway')
+
+    with pytest.raises(PushHttpError):
+        await push_data_and_charge(mgr, [{'a': 1}], 'page-open', False, failing_push)
+    await mgr.flush_pending()
+    assert len([p for p in client.posts if p['path'].endswith('/charge')]) == 1, 'never re-posted for unsaved items'
+    assert mgr.get_charged_event_count('page-open') == 1, 'local count kept: it may have applied'

{scrapely_python-0.1.3 → scrapely_python-0.1.5}/tests/test_storage.py

@@ -256,7 +256,11 @@ async def test_rq_reclaim_after_unclaimed_lock_expiry_persists_retry_count(trans
 
 
 @pytest.mark.asyncio
-async def test_rq_reclaim_still_raises_other_errors(transport, mock_api):
+async def test_rq_reclaim_still_raises_other_errors(transport, mock_api, monkeypatch):
+    # A persistent 500 is raised once the server-error retries are spent (none here, to keep the test fast).
+    from scrapely.storage.api_client import ApiClient
+
+    monkeypatch.setattr(ApiClient, '_max_server_retries', 0)
     client = ScrapelyStorageClient(
         api_url='https://api.test.scrape.ly',
         token='t',