scrapely-python 0.1.11__tar.gz → 0.1.12__tar.gz

{scrapely_python-0.1.11/src/scrapely_python.egg-info → scrapely_python-0.1.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.11
+Version: 0.1.12
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.11 → scrapely_python-0.1.12}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.11"
+version = "0.1.12"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.11 → scrapely_python-0.1.12}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.11'
+__version__ = '0.1.12'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.11 → scrapely_python-0.1.12}/src/scrapely/actor.py

@@ -491,7 +491,7 @@ class Actor(metaclass=_ActorMeta):
                     if flush is not None:
                         deadline_ms = min(max(1, timeout_secs) * 1000, 30_000)
                         # The flush's own deadline ends its retries (charge POSTs back off
-                        # for up to ~45 s) a little before wait_for would cancel it.
+                        # for up to ~2 min) a little before wait_for would cancel it.
                         await asyncio.wait_for(
                             flush(max_intents=64, deadline_ms=max(500, deadline_ms - 2_000)),
                             timeout=max(1.0, deadline_ms / 1000.0),
@@ -1067,12 +1067,15 @@ class Actor(metaclass=_ActorMeta):
             return None
 
         client = cls.new_client(token=token, base_url=api_url)
+        # Setting the message is idempotent: a 429 (run API rate limit), 5xx or
+        # dropped connection is retried with backoff instead of failing the run.
         response = await client.put(
             f'/v2/actor-runs/{run_id}',
             json_data={
                 'statusMessage': message,
                 'isStatusMessageTerminal': is_terminal,
             },
+            idempotent=True,
         )
         body: Any = {}
         if response.content:

{scrapely_python-0.1.11 → scrapely_python-0.1.12}/src/scrapely/charging.py

@@ -49,16 +49,26 @@ CHARGE_IDEMPOTENCY_SAFE_AGE_SECS = 30 * 24 * 60 * 60
 # Soft cap on open unresolved intents. Refuse NEW billable charges when full.
 _PENDING_MAX_ENTRIES = 1024
 
-# Same-key POST attempts before leaving an intent pending / raising. The API
-# client itself retries 5xx answers and refused or reset connections of these
-# POSTs with its long backoff (0.5 s doubling to 8 s, about 45 s): re-sending
+# Same-key POST attempts before leaving an intent pending / re-queueing a void.
+# The API client itself retries 429 and 5xx answers and refused or reset
+# connections of these POSTs with exponential backoff (honouring Retry-After)
+# for its retry window (2 minutes, SCRAPELY_API_RETRY_WINDOW_SECS): re-sending
 # is safe because every attempt carries the same Idempotency-Key. These outer
 # attempts cover what the client gives up on quickly (timeouts, other
-# transport errors), with the same backoff, within _RETRY_BUDGET_SECS.
+# transport errors), within _RETRY_BUDGET_SECS. Then the charge stays pending
+# (the void is re-queued) and is retried before the next push and at exit,
+# the same as the JS SDK.
 _POST_ATTEMPTS = 8
 
-# Outer retries stop once this much time has passed since the first attempt.
-_RETRY_BUDGET_SECS = 60.0
+# Outer retries stop once this much time has passed since the first attempt
+# (the API retry window; JS SDK parity).
+_RETRY_BUDGET_SECS = 120.0
+
+
+def _is_transient_status(status: Optional[int]) -> bool:
+    """408/429 and 5xx: the platform could not take the request now; try again later."""
+    return status is not None and (status in (408, 429) or status >= 500)
+
 
 # Local-only dataset collecting one entry per logical charge operation
 # (JS ChargingManager LOCAL_CHARGING_LOG_DATASET_NAME parity).
@@ -563,7 +573,12 @@ class ChargingManager:
         return await self._void_charge(event_name, operation_id, reserved, not_found_means_voided=True)
 
     async def _void_charge(
-        self, event_name: str, operation_id: str, reserved: int, not_found_means_voided: bool = False
+        self,
+        event_name: str,
+        operation_id: str,
+        reserved: int,
+        not_found_means_voided: bool = False,
+        deadline: Optional[float] = None,
     ) -> bool:
         """Reverse an accepted charge whose items were never stored, then unwind locally.
 
@@ -573,6 +588,8 @@ class ChargingManager:
         if self._client is None or not self._is_at_home or not self._actor_run_id:
             return False
         started = time.monotonic()
+        budget_end = started + self._RETRY_BUDGET_SECS
+        deadline = budget_end if deadline is None else min(deadline, budget_end)
         for attempt in range(self._POST_ATTEMPTS):
             try:
                 await self._client.post(
@@ -580,6 +597,7 @@ class ChargingManager:
                     json_data={},
                     headers={'Idempotency-Key': operation_id},
                     idempotent=True,
+                    retry_deadline=deadline,
                 )
                 self._unwind_reservation(event_name, reserved)
                 return True
@@ -592,24 +610,37 @@ class ChargingManager:
                     self._unwind_reservation(event_name, reserved)
                     return True
                 # 404: nothing to void; 409: already settled; 4xx auth: permanent.
-                if status is not None and 400 <= status < 500:
+                # 408/429 (rate limited) and 5xx are not answers: re-queue below.
+                if status is not None and 400 <= status < 500 and not _is_transient_status(status):
                     return False
-                if not await self._backoff_before_retry(attempt, started, err):
+                if not await self._backoff_before_retry(attempt, started, err, deadline):
                     break
         # Outcome unknown: keep the reservation and retry before the next push / at exit.
         self._pending_voids.append((event_name, operation_id, reserved, not_found_means_voided))
         logger.error('Void of charge %s (%r) not confirmed yet; will retry', operation_id, event_name)
         return False
 
-    async def _retry_pending_voids(self) -> bool:
+    async def _retry_pending_voids(self, deadline: Optional[float] = None) -> bool:
         """Retry unconfirmed voids (caller holds the operation lock).
 
-        Returns False while any earlier void is still unconfirmed.
+        With ``deadline`` (``time.monotonic()``, the exit flush) keeps retrying
+        with backoff while time is left. Returns False while any earlier void
+        is still unconfirmed.
         """
-        pending, self._pending_voids = self._pending_voids, []
-        for event_name, operation_id, reserved, not_found_means_voided in pending:
-            await self._void_charge(event_name, operation_id, reserved, not_found_means_voided)
-        return not self._pending_voids
+        from .storage.api_client import server_retry_delay
+
+        round_ = 0
+        while True:
+            pending, self._pending_voids = self._pending_voids, []
+            for event_name, operation_id, reserved, not_found_means_voided in pending:
+                await self._void_charge(event_name, operation_id, reserved, not_found_means_voided, deadline)
+            if not self._pending_voids:
+                return True
+            left = 0.0 if deadline is None else deadline - time.monotonic()
+            if left <= 0:
+                return False
+            await asyncio.sleep(min(server_retry_delay(round_, None), left))
+            round_ += 1
 
     def _quarantine_stale_pending(self) -> None:
         """Remove intents older than the safe replay window (fail closed, no replay).
@@ -656,7 +687,7 @@ class ChargingManager:
             },
             headers={'Idempotency-Key': idempotency_key},
             idempotent=True,
-            **({'retry_deadline': deadline} if deadline is not None else {}),
+            retry_deadline=deadline,
         )
         if isinstance(response, dict):
             return response
@@ -674,17 +705,17 @@ class ChargingManager:
     ) -> bool:
         """Sleep before the next same-key attempt; False when no attempt is left.
 
-        A 5xx answer reaching this point was already retried by the API client
-        with its long backoff, so it ends the attempts. Timeouts and other
-        errors without an answer back off the same way (0.5 s doubling to 8 s)
-        while attempts and the retry budget last.
+        A 429/5xx answer reaching this point was already retried by the API
+        client for the whole budget, so it ends the attempts. Timeouts, 408 and
+        other errors back off the same way while attempts and the retry budget
+        last.
         """
         from .storage.api_client import server_retry_delay
 
         if attempt + 1 >= self._POST_ATTEMPTS:
             return False
         status = _http_status_of(err)
-        if status is not None and status >= 500:
+        if status is not None and (status == 429 or status >= 500):
             return False
         delay = server_retry_delay(attempt, None)
         if time.monotonic() - started + delay > self._RETRY_BUDGET_SECS:
@@ -714,6 +745,9 @@ class ChargingManager:
         """
         last_error: Optional[BaseException] = None
         started = time.monotonic()
+        # The client's own retries count against the same budget.
+        budget_end = started + self._RETRY_BUDGET_SECS
+        deadline = budget_end if deadline is None else min(deadline, budget_end)
         for attempt in range(self._POST_ATTEMPTS):
             try:
                 body = await self._post_charge_once(
@@ -762,11 +796,10 @@ class ChargingManager:
         """Caller must hold _operation_lock."""
         if not self._is_at_home or self._client is None or not self._actor_run_id:
             return 0
-        await self._retry_pending_voids()
-
-        import time
-
         deadline = None if deadline_ms is None else (time.monotonic() + max(0.0, deadline_ms) / 1000.0)
+        # Voids first: an unconfirmed one means the user may be billed for items
+        # that were never stored. At exit (with a deadline) keep retrying them.
+        await self._retry_pending_voids(deadline)
         confirmed_total = 0
         async with self._flush_lock:
             while True:

{scrapely_python-0.1.11 → scrapely_python-0.1.12}/src/scrapely/storage/api_client.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import os
 import random
 import time
 from typing import Any, Dict, Optional, Union
@@ -36,13 +37,16 @@ def is_retryable_response(
 ) -> bool:
     """Whether a non-2xx answer is retried.
 
-    Idempotent methods: 500/502/503/504. Others (dataset push, KV PUT) only on
-    a 503 that certainly was not processed: the rate limiter refused it
-    ("rate-limit-unavailable", before the handler), or HAProxy had no backend
-    (its own page, nothing forwarded). Any other 503 may mean "outcome unknown"
-    (a direct dataset write whose commit was not confirmed), and re-sending
-    could store the items twice.
+    429 always: the run API limiter and dataset backpressure refuse it before
+    anything is written. Idempotent methods: also 500/502/503/504. Others
+    (dataset push, KV PUT) only on a 503 that certainly was not processed: the
+    rate limiter refused it ("rate-limit-unavailable", before the handler), or
+    HAProxy had no backend (its own page, nothing forwarded). Any other 503 may
+    mean "outcome unknown" (a direct dataset write whose commit was not
+    confirmed), and re-sending could store the items twice.
     """
+    if status == 429:
+        return True
     if idempotent:
         return status in (500, 502, 503, 504)
     if status != 503:
@@ -70,22 +74,60 @@ def _error_type(response: httpx.Response) -> Optional[str]:
     return None
 
 
-def server_retry_delay(server_attempt: int, retry_after: Optional[str], now: Optional[float] = None) -> float:
-    """Exponential backoff with jitter, in seconds; honours a Retry-After (seconds or HTTP date) of up to 30 s."""
-    if retry_after is not None and retry_after.strip():
-        secs: Optional[float] = None
+# Default time budget for retrying one API call (429, 503, network errors).
+DEFAULT_API_RETRY_WINDOW_SECS = 120.0
+_MAX_BACKOFF_SECS = 30.0
+
+
+def api_retry_window_secs() -> float:
+    """How long one API call keeps retrying before it gives up.
+
+    ``SCRAPELY_API_RETRY_WINDOW_SECS`` when set, else ``ApiClient.retry_window_secs``
+    (2 minutes by default).
+    """
+    raw = os.environ.get('SCRAPELY_API_RETRY_WINDOW_SECS')
+    if raw:
         try:
-            secs = float(retry_after)
+            value = float(raw)
+            if value >= 0:
+                return value
         except ValueError:
-            try:
-                from email.utils import parsedate_to_datetime
+            pass
+    return ApiClient.retry_window_secs
+
+
+def _parse_retry_after(retry_after: Optional[str], now: Optional[float]) -> Optional[float]:
+    if retry_after is None or not retry_after.strip():
+        return None
+    secs: Optional[float] = None
+    try:
+        secs = float(retry_after)
+    except ValueError:
+        try:
+            from email.utils import parsedate_to_datetime
 
-                secs = max(0.0, parsedate_to_datetime(retry_after).timestamp() - (time.time() if now is None else now))
-            except Exception:
-                secs = None
-        if secs is not None and secs >= 0:
-            return min(secs, 30.0) + random.random() * 0.1
-    return min(8.0, 0.5 * 2 ** server_attempt) + random.random() * 0.25
+            secs = max(0.0, parsedate_to_datetime(retry_after).timestamp() - (time.time() if now is None else now))
+        except Exception:
+            secs = None
+    if secs is None or secs < 0 or secs != secs:
+        return None
+    # Not capped: a Retry-After past the retry window ends the retries instead.
+    return secs
+
+
+def server_retry_delay(attempt: int, retry_after: Optional[str], now: Optional[float] = None) -> float:
+    """Delay in seconds before retry number ``attempt + 1``.
+
+    Exponential backoff (0.5 s doubling, capped at 30 s) with equal jitter, and
+    never earlier than the server's Retry-After (seconds or HTTP date) plus a little jitter so throttled clients don't all come back in
+    the same instant.
+    """
+    exp: float = min(_MAX_BACKOFF_SECS, 0.5 * float(2 ** min(attempt, 16)))
+    backoff = exp / 2 + random.random() * (exp / 2)
+    ra = _parse_retry_after(retry_after, now)
+    if ra is None:
+        return backoff
+    return max(ra + random.random() * min(1.0, ra * 0.2 + 0.1), backoff)
 
 
 class AdaptiveRateLimiter:
@@ -139,25 +181,8 @@ class AdaptiveRateLimiter:
 
         self.last_request_time = time.time() * 1000
 
-    async def handle_rate_limited(self, response: httpx.Response) -> None:
+    def record_rate_limited(self) -> None:
         self._rate_limit_errors[429] = self._rate_limit_errors.get(429, 0) + 1
-        retry_after = response.headers.get('Retry-After')
-        wait_ms = 1000.0
-        if retry_after:
-            try:
-                wait_ms = float(retry_after) * 1000
-            except ValueError:
-                try:
-                    from email.utils import parsedate_to_datetime
-
-                    retry_date = parsedate_to_datetime(retry_after).timestamp() * 1000
-                    wait_ms = max(0.0, retry_date - time.time() * 1000)
-                except Exception:
-                    wait_ms = 1000.0
-
-        wait_ms = wait_ms + random.random() * 100
-        logger.warning('[RateLimiter] Rate limited, waiting %.0fms before retry', wait_ms)
-        await asyncio.sleep(wait_ms / 1000)
 
     def get_rate_limit_errors(self) -> Dict[int, int]:
         return dict(self._rate_limit_errors)
@@ -170,11 +195,10 @@ class ApiClient:
     """
 
     _rate_limiter = AdaptiveRateLimiter()
-    _max_retries = 3
-    # Retries while the API is briefly unavailable (503/5xx, connection
-    # refused): backoff 0.5 s doubling to 8 s, about 45 s in total, which
-    # covers a main redeploy.
-    _max_server_retries = 8
+    # How long one call keeps retrying 429/503/network errors before it gives
+    # up (SCRAPELY_API_RETRY_WINDOW_SECS overrides it). Long enough that a
+    # throttled run slows down instead of failing, and covers a main redeploy.
+    retry_window_secs = DEFAULT_API_RETRY_WINDOW_SECS
 
     def __init__(
         self,
@@ -260,12 +284,25 @@ class ApiClient:
         idempotent: Optional[bool] = None,
         retry_deadline: Optional[float] = None,
     ) -> httpx.Response:
-        """Make an HTTP request with adaptive rate limiting and 429 retries.
+        """Make an HTTP request with the API retry policy.
+
+        Retries, with exponential backoff and jitter that honours Retry-After,
+        until the retry window (2 minutes by default) runs out:
+
+        - 429 (run API rate limit, dataset backpressure): always; nothing was applied.
+        - 503 from the limiter or HAProxy: always; nothing was forwarded.
+        - 500/502/503/504, read/write timeouts and dropped connections:
+          idempotent calls only.
+        - Refused connections and connect/pool timeouts: always; the request
+          never left.
+
+        A write (dataset push, KV PUT) whose request may have reached the API is
+        never re-sent: there is no idempotency key for it, so a re-send could
+        store the same rows twice. Such failures surface to the caller instead.
 
-        ``retry_deadline`` (a ``time.monotonic()`` value) stops the 5xx /
-        connection retries once the next backoff would end after it, so a
-        caller with a time budget (e.g. the exit flush) gets an answer instead
-        of being cancelled mid-retry.
+        ``retry_deadline`` (a ``time.monotonic()`` value) ends the retries
+        earlier, so a caller with a time budget (e.g. the exit flush) gets an
+        answer instead of being cancelled mid-retry.
         """
         client = await self._ensure_client()
         url = self._url(path)
@@ -277,10 +314,30 @@ class ApiClient:
 
         if idempotent is None:
             idempotent = method.upper() in IDEMPOTENT_METHODS
-        attempt = 0  # timeouts and 429s
-        server_attempt = 0  # 503/5xx and connection failures (API briefly unavailable)
+        deadline = time.monotonic() + max(0.0, api_retry_window_secs())
+        if retry_deadline is not None:
+            deadline = min(deadline, retry_deadline)
+        attempt = 0
+
+        async def backoff(why: str, retry_after: Optional[str]) -> bool:
+            """Sleep before the next attempt; False once the retry window is used up."""
+            nonlocal attempt
+            # Retrying sooner than asked would only be refused again: give up
+            # when the next try would land past the window / deadline.
+            wait = server_retry_delay(attempt, retry_after)
+            if time.monotonic() + wait > deadline:
+                return False
+            attempt += 1
+            logger.warning('%s %s %s; retry %d in %.1fs', method, path, why, attempt, wait)
+            await asyncio.sleep(wait)
+            return True
+
         while True:
             await self._rate_limiter.wait_for_slot()
+            request_kwargs: Dict[str, Any] = {}
+            if retry_deadline is not None:
+                # A caller with a time budget: no attempt may outlive it.
+                request_kwargs['timeout'] = max(0.5, min(self.timeout, retry_deadline - time.monotonic()))
             try:
                 response = await client.request(
                     method,
@@ -289,63 +346,38 @@ class ApiClient:
                     json=json_data,
                     content=content,
                     headers=request_headers or None,
+                    **request_kwargs,
                 )
-            except httpx.TimeoutException:
-                if do_not_retry_timeouts or attempt >= self._max_retries:
+            except (httpx.ConnectTimeout, httpx.PoolTimeout, httpx.ConnectError) as exc:
+                # Never reached the API: safe for every method.
+                if isinstance(exc, (httpx.ConnectTimeout, httpx.PoolTimeout)) and do_not_retry_timeouts:
                     raise
-                attempt += 1
-                await asyncio.sleep(0.25 * attempt)
-                continue
+                if await backoff(f'failed ({type(exc).__name__})', None):
+                    continue
+                raise
+            except httpx.TimeoutException as exc:
+                # Read/write timeout: the request may have been applied.
+                if idempotent and not do_not_retry_timeouts and await backoff(f'timed out ({type(exc).__name__})', None):
+                    continue
+                raise
             except httpx.TransportError as exc:
-                # No response. A refused/unresolvable connection never reached
-                # the API, so it is safe for every method; anything else (reset
-                # mid-request) only for idempotent ones.
-                wait = server_retry_delay(server_attempt, None)
-                if (
-                    (idempotent or isinstance(exc, httpx.ConnectError))
-                    and server_attempt < self._max_server_retries
-                    and (retry_deadline is None or time.monotonic() + wait <= retry_deadline)
-                ):
-                    server_attempt += 1
-                    logger.warning(
-                        '%s %s failed (%s); retry %d/%d in %.1fs',
-                        method, path, type(exc).__name__, server_attempt, self._max_server_retries, wait,
-                    )
-                    await asyncio.sleep(wait)
+                # Reset mid-request: only idempotent calls are re-sent.
+                if idempotent and await backoff(f'failed ({type(exc).__name__})', None):
                     continue
                 raise
 
             self._rate_limiter.update_from_headers(response.headers)
-
-            if response.status_code == 429 and attempt < self._max_retries:
-                attempt += 1
-                await self._rate_limiter.handle_rate_limited(response)
-                continue
-
-            # API briefly unavailable (main redeploy, limiter store blip); see
-            # is_retryable_response for which answers are safe to re-send.
-            if (
-                not response.is_success
-                and server_attempt < self._max_server_retries
-                and (
-                    retry_deadline is None
-                    or time.monotonic() + server_retry_delay(server_attempt, response.headers.get('Retry-After')) <= retry_deadline
-                )
-                and is_retryable_response(
-                    response.status_code,
-                    idempotent,
-                    _error_type(response) if response.status_code == 503 else None,
-                    response.text if response.status_code == 503 else None,
-                )
+            if response.status_code == 429:
+                self._rate_limiter.record_rate_limited()
+
+            if not response.is_success and is_retryable_response(
+                response.status_code,
+                idempotent,
+                _error_type(response) if response.status_code == 503 else None,
+                response.text if response.status_code == 503 else None,
             ):
-                wait = server_retry_delay(server_attempt, response.headers.get('Retry-After'))
-                server_attempt += 1
-                logger.warning(
-                    '%s %s answered %d; retry %d/%d in %.1fs',
-                    method, path, response.status_code, server_attempt, self._max_server_retries, wait,
-                )
-                await asyncio.sleep(wait)
-                continue
+                if await backoff(f'answered {response.status_code}', response.headers.get('Retry-After')):
+                    continue
 
             if raise_for_status:
                 await self._raise_for_status(response)
@@ -402,6 +434,7 @@ class ApiClient:
         params: Optional[Dict[str, Any]] = None,
         *,
         raise_for_status: bool = True,
+        idempotent: Optional[bool] = None,
     ) -> httpx.Response:
         return await self.request(
             'PUT',
@@ -411,6 +444,7 @@ class ApiClient:
             content=content,
             headers=headers,
             raise_for_status=raise_for_status,
+            idempotent=idempotent,
         )
 
     async def delete(

{scrapely_python-0.1.11 → scrapely_python-0.1.12/src/scrapely_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.11
+Version: 0.1.12
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.11 → scrapely_python-0.1.12}/tests/test_api_client_retries.py

@@ -20,12 +20,16 @@ from scrapely.storage.api_client import ApiClient, ApiClientError, is_retryable_
 
 @pytest.fixture(autouse=True)
 def no_sleep(monkeypatch: pytest.MonkeyPatch) -> List[float]:
+    """Sleeps return at once but advance a fake monotonic clock, so retry windows run out."""
     slept: List[float] = []
+    clock = [1000.0]
 
     async def fake_sleep(secs: float) -> None:
         slept.append(secs)
+        clock[0] += secs
 
     monkeypatch.setattr(mod.asyncio, 'sleep', fake_sleep)
+    monkeypatch.setattr(mod.time, 'monotonic', lambda: clock[0])
     return slept
 
 
@@ -86,22 +90,69 @@ async def test_post_retries_503_until_success(no_sleep: List[float]) -> None:
     r = await c.request('POST', '/v2/datasets/d/items', json_data=[{'a': 1}])
     assert r.status_code == 201
     assert calls == ['POST'] * 3
-    backoff = [x for x in no_sleep if x >= 0.4]  # the rate limiter's own pacing sleeps are shorter
-    assert len(backoff) == 2 and 0.5 <= backoff[0] < 0.8 and 1.0 <= backoff[1] < 1.3  # exponential
+    backoff = [x for x in no_sleep if x >= 0.2]  # the rate limiter's own pacing sleeps are shorter
+    assert len(backoff) == 2 and 0.25 <= backoff[0] <= 0.5 and 0.5 <= backoff[1] <= 1.0  # exponential
 
 
 async def test_retry_after_is_honoured(no_sleep: List[float]) -> None:
     c, _ = client_for([httpx.Response(503, headers={'Retry-After': '3'}), httpx.Response(200, json={})])
     await c.request('GET', '/v2/key-value-stores/k/records/x')
-    assert any(3.0 <= x < 3.2 for x in no_sleep)
+    assert any(3.0 <= x < 3.8 for x in no_sleep)
 
 
-async def test_gives_up_after_budget() -> None:
+async def test_gives_up_after_the_retry_window(no_sleep: List[float]) -> None:
     c, calls = client_for([unavailable()])
     with pytest.raises(ApiClientError) as e:
         await c.request('GET', '/v2/datasets/d')
     assert e.value.status_code == 503
-    assert len(calls) == ApiClient._max_server_retries + 1
+    backoff = sum(x for x in no_sleep if x >= 0.2)  # the rate limiter's own pacing sleeps are shorter
+    assert 75 <= backoff <= ApiClient.retry_window_secs, 'most of 2 minutes, never past it'
+    assert len(calls) >= 8
+
+
+async def test_retry_window_is_configurable(no_sleep: List[float], monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv('SCRAPELY_API_RETRY_WINDOW_SECS', '10')
+    c, calls = client_for([httpx.Response(429, headers={'Retry-After': '1'})])
+    with pytest.raises(ApiClientError) as e:
+        await c.request('POST', '/v2/datasets/d/items', json_data=[{}])
+    assert e.value.status_code == 429
+    assert sum(x for x in no_sleep if x >= 0.2) <= 10.01 and len(calls) >= 3
+
+
+async def test_429_sequence_backs_off_honouring_retry_after(no_sleep: List[float]) -> None:
+    """Run API rate limit (PR #664): 429 + Retry-After 1-10 s. Nothing was stored, so a push is re-sent."""
+    limited = [httpx.Response(429, headers={'Retry-After': str(s)}, json={'error': {'type': 'rate-limit-exceeded'}}) for s in (3, 1, 1, 1, 1, 1)]
+    c, calls = client_for(limited + [httpx.Response(201, json={})])
+    assert (await c.request('POST', '/v2/datasets/d/items', json_data=[{'a': 1}])).status_code == 201
+    assert calls == ['POST'] * 7
+    backoff = [x for x in no_sleep if x >= 0.2]
+    assert len(backoff) == 6
+    assert 3.0 <= backoff[0] < 3.8, 'never earlier than Retry-After'
+    assert all(b >= 1.0 for b in backoff)
+    assert backoff[5] >= 8.0, f'grows past a short Retry-After: {backoff}'
+    assert c.get_rate_limit_errors().get(429, 0) >= 6
+
+
+async def test_429_past_the_old_three_retries_still_succeeds(no_sleep: List[float]) -> None:
+    c, calls = client_for([httpx.Response(429, headers={'Retry-After': '1'})] * 8 + [httpx.Response(200, json={})])
+    assert (await c.request('GET', '/v2/datasets/d')).status_code == 200
+    assert len(calls) == 9
+
+
+async def test_post_read_timeout_is_not_resent(no_sleep: List[float]) -> None:
+    """No idempotency key for dataset pushes: a timed-out push may be stored, so it is not re-sent."""
+    c, calls = client_for([httpx.ReadTimeout('slow'), httpx.Response(201, json={})])
+    with pytest.raises(httpx.ReadTimeout):
+        await c.request('POST', '/v2/datasets/d/items', json_data=[{}])
+    assert len(calls) == 1
+    # Connect timeout: the request never left, so it is re-sent.
+    c, calls = client_for([httpx.ConnectTimeout('slow'), httpx.Response(201, json={})])
+    assert (await c.request('POST', '/v2/datasets/d/items', json_data=[{}])).status_code == 201
+    assert len(calls) == 2
+    # GET read timeouts are retried.
+    c, calls = client_for([httpx.ReadTimeout('slow'), httpx.Response(200, json={})])
+    assert (await c.request('GET', '/v2/datasets/d')).status_code == 200
+    assert len(calls) == 2
 
 
 async def test_502_retried_for_get_not_for_post() -> None:
@@ -164,7 +215,27 @@ def test_helpers() -> None:
     assert not is_retryable_response(400, True) and not is_retryable_response(501, True)
     from email.utils import parsedate_to_datetime
     now = parsedate_to_datetime('Thu, 15 Oct 2026 00:00:00 GMT').timestamp()
-    assert 5.0 <= server_retry_delay(0, 'Thu, 15 Oct 2026 00:00:05 GMT', now=now) < 5.2
-    assert 4.0 <= server_retry_delay(3, 'garbage') < 4.3
-    assert server_retry_delay(10, None) < 8.3
-    assert 30.0 <= server_retry_delay(0, '120') < 30.2
+    assert is_retryable_response(429, False) and is_retryable_response(429, True)
+    assert 5.0 <= server_retry_delay(0, 'Thu, 15 Oct 2026 00:00:05 GMT', now=now) < 6.1
+    assert 2.0 <= server_retry_delay(3, 'garbage') <= 4.0
+    assert 0.25 <= server_retry_delay(0, None) <= 0.5
+    assert 15.0 <= server_retry_delay(10, None) <= 30.0, 'capped at 30 s'
+    assert 120.0 <= server_retry_delay(0, '120') < 121.1, 'Retry-After is not capped'
+    assert server_retry_delay(6, '1') >= 15.0, 'backoff grows past a short Retry-After'
+    assert server_retry_delay(0, '0') <= 0.5
+
+
+async def test_status_message_put_is_retried_as_idempotent(no_sleep: List[float]) -> None:
+    """Actor.set_status_message PUTs with idempotent=True: a 429 or 502 is retried, not raised."""
+    c, calls = client_for([httpx.Response(429, headers={'Retry-After': '1'}), httpx.Response(502), httpx.Response(200, json={'data': {}})])
+    r = await c.put('/v2/actor-runs/run1', json_data={'statusMessage': 'x'}, idempotent=True)
+    assert r.status_code == 200
+    assert calls == ['PUT'] * 3
+
+
+async def test_retry_after_past_the_window_gives_up_at_once(no_sleep: List[float]) -> None:
+    c, calls = client_for([httpx.Response(429, headers={'Retry-After': '300'}), httpx.Response(200, json={})])
+    with pytest.raises(ApiClientError) as e:
+        await c.request('GET', '/v2/datasets/d')
+    assert e.value.status_code == 429
+    assert len(calls) == 1, 'no early retry that the server would refuse again'

{scrapely_python-0.1.11 → scrapely_python-0.1.12}/tests/test_charge_retries.py

@@ -27,13 +27,17 @@ from scrapely.storage.api_client import ApiClient
 
 @pytest.fixture(autouse=True)
 def no_sleep(monkeypatch: pytest.MonkeyPatch) -> List[float]:
+    """Sleeps return at once but advance a fake monotonic clock, so retry windows run out."""
     slept: List[float] = []
+    clock = [1000.0]
 
     async def fake_sleep(secs: float) -> None:
         slept.append(secs)
+        clock[0] += secs
 
     monkeypatch.setattr(client_mod.asyncio, 'sleep', fake_sleep)
     monkeypatch.setattr(charging_mod.asyncio, 'sleep', fake_sleep)
+    monkeypatch.setattr(client_mod.time, 'monotonic', lambda: clock[0])
     return slept
 
 
@@ -116,26 +120,76 @@ async def test_flush_deadline_bounds_the_client_retries(no_sleep: List[float]) -
 
     mgr, calls = manager_for([httpx.Response(503, json={'error': {'type': 'service-unavailable'}})])
     mgr._pending_charge_keys['k1'] = {'event_name': 'page-open', 'count': 1, 'created_at': _time.monotonic()}
-    clock = [1000.0]
-
-    def fake_monotonic() -> float:
-        return clock[0]
-
-    async def advancing_sleep(secs: float) -> None:
-        no_sleep.append(secs)
-        clock[0] += secs
-
-    import pytest as _pytest
-    mp = _pytest.MonkeyPatch()
-    try:
-        mp.setattr(client_mod.time, 'monotonic', fake_monotonic)
-        mp.setattr(charging_mod.time, 'monotonic', fake_monotonic)
-        mp.setattr(client_mod.asyncio, 'sleep', advancing_sleep)
-        mp.setattr(charging_mod.asyncio, 'sleep', advancing_sleep)
-        confirmed = await mgr.flush_pending(deadline_ms=3000)
-    finally:
-        mp.undo()
+    confirmed = await mgr.flush_pending(deadline_ms=3000)
     assert confirmed == 0
     assert 'k1' in mgr._pending_charge_keys, 'still pending for a later flush'
-    assert sum(no_sleep) <= 3.0, f'no retry sleeps past the deadline: {no_sleep}'
+    assert sum(x for x in no_sleep if x >= 0.2) <= 3.0, f'no retry sleeps past the deadline: {no_sleep}'
     assert len(calls) >= 2, 'it did retry within the budget'
+
+
+async def test_charge_429_backs_off_honouring_retry_after(no_sleep: List[float]) -> None:
+    limited = httpx.Response(429, headers={'Retry-After': '4'}, json={'error': {'type': 'rate-limit-exceeded'}})
+    mgr, calls = manager_for([limited, limited, limited, ok()])
+    assert (await mgr.charge('page-open', 1)).charged_count == 1
+    assert len(calls) == 4 and len({key for _, key in calls}) == 1
+    backoff = [x for x in no_sleep if x >= 0.4]
+    assert len(backoff) == 3 and all(4.0 <= x < 5.0 for x in backoff[:2]), backoff
+
+
+async def test_charge_429_for_longer_than_the_window_stays_pending(no_sleep: List[float]) -> None:
+    limited = httpx.Response(429, headers={'Retry-After': '10'}, json={'error': {'type': 'rate-limit-exceeded'}})
+    mgr, calls = manager_for([limited])
+    assert await mgr._post_charge_with_retries(event_name='page-open', count=1, idempotency_key='k1') is None
+    assert sum(no_sleep) <= ChargingManager._RETRY_BUDGET_SECS + 1
+    assert len(calls) >= 8, 'kept retrying through the window'
+
+
+async def test_void_that_429s_is_requeued_then_succeeds(no_sleep: List[float]) -> None:
+    """A 429 on a void is not an answer: the void is re-queued and retried, never dropped."""
+    limited = httpx.Response(429, headers={'Retry-After': '10'}, json={'error': {'type': 'rate-limit-exceeded'}})
+    mgr, calls = manager_for([limited])
+    assert await mgr._void_charge('page-open', 'op-1', 1) is False
+    assert mgr._pending_voids == [('page-open', 'op-1', 1, False)], 're-queued, not treated as final'
+    # The limiter lets it through on the next try (e.g. before the next push).
+    ok_void = httpx.Response(200, json={'data': {'status': 'voided'}})
+    mgr._client._transport = None  # noqa: SLF001
+    mgr2_calls: list = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        mgr2_calls.append(request.url.path)
+        return ok_void
+
+    mgr._client = ApiClient('http://api.test', 'st_x', 'run-1', transport=httpx.MockTransport(handler))
+    assert await mgr._retry_pending_voids() is True
+    assert mgr2_calls == ['/v2/actor-runs/run-1/charge/void']
+    assert not mgr._pending_voids
+
+
+async def test_exit_flush_keeps_retrying_a_rate_limited_void(no_sleep: List[float]) -> None:
+    limited = httpx.Response(429, headers={'Retry-After': '2'}, json={'error': {'type': 'rate-limit-exceeded'}})
+    responses = [limited] * 6 + [httpx.Response(200, json={'data': {'status': 'voided'}})]
+    mgr, calls = manager_for(responses)
+    mgr._RETRY_BUDGET_SECS = 5.0  # one void call gives up quickly; the flush loop retries it
+    mgr._pending_voids.append(('page-open', 'op-1', 1, False))
+    await mgr.flush_pending(deadline_ms=60_000)
+    assert not mgr._pending_voids, 'voided before the run exits'
+    assert len(calls) == 7
+
+
+async def test_charge_408_is_retried_with_the_same_key(no_sleep: List[float]) -> None:
+    """The API client does not retry 408; the charging manager does (same key)."""
+    mgr, calls = manager_for([httpx.Response(408, json={'error': {'type': 'timeout'}}), ok()])
+    assert (await mgr.charge('page-open', 1)).charged_count == 1
+    assert len(calls) == 2 and len({key for _, key in calls}) == 1
+
+
+async def test_flush_retry_never_starts_past_the_deadline(no_sleep: List[float]) -> None:
+    """A Retry-After longer than the time left ends the retries instead of firing at the deadline."""
+    import time as _time
+
+    limited = httpx.Response(429, headers={'Retry-After': '10'}, json={'error': {'type': 'rate-limit-exceeded'}})
+    mgr, calls = manager_for([limited])
+    mgr._pending_charge_keys['k1'] = {'event_name': 'page-open', 'count': 1, 'created_at': _time.monotonic()}
+    await mgr.flush_pending(deadline_ms=5000)
+    assert len(calls) == 1, 'no second request: it would come after the deadline'
+    assert 'k1' in mgr._pending_charge_keys

{scrapely_python-0.1.11 → scrapely_python-0.1.12}/tests/test_storage.py

@@ -277,7 +277,7 @@ async def test_rq_reclaim_still_raises_other_errors(transport, mock_api, monkeyp
     # A persistent 500 is raised once the server-error retries are spent (none here, to keep the test fast).
     from scrapely.storage.api_client import ApiClient
 
-    monkeypatch.setattr(ApiClient, '_max_server_retries', 0)
+    monkeypatch.setattr(ApiClient, 'retry_window_secs', 0.0)
     client = ScrapelyStorageClient(
         api_url='https://api.test.scrape.ly',
         token='t',

{scrapely_python-0.1.11 → scrapely_python-0.1.12}/tests/test_terminal_status_message.py

@@ -26,7 +26,7 @@ class _FakeClient:
         self.fail = fail
         self.hang = hang
 
-    async def put(self, path: str, json_data: Optional[Dict[str, Any]] = None) -> _FakeResponse:
+    async def put(self, path: str, json_data: Optional[Dict[str, Any]] = None, **_kwargs: Any) -> _FakeResponse:
         if self.hang:
             await asyncio.sleep(60)
         if self.fail: