scrapely-python 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.4'
+__version__ = '0.1.5'
 __all__ = [
     # Core
     'Actor',

scrapely/storage/api_client.py

@@ -24,6 +24,32 @@ class ApiClientError(Exception):
         self.body = body
 
 
+# PUT is left out: a KV record PUT is counted (and billed) as a write each time
+# it is applied, so a lost response must not be re-sent blindly.
+IDEMPOTENT_METHODS = frozenset({'GET', 'HEAD', 'DELETE', 'OPTIONS'})
+# HAProxy's own 503 page: no backend was available, so nothing was forwarded.
+NO_BACKEND_MARKER = 'No server is available to handle this request'
+
+
+def is_retryable_response(
+    status: int, idempotent: bool, error_type: Optional[str] = None, body_text: Optional[str] = None
+) -> bool:
+    """Whether a non-2xx answer is retried.
+
+    Idempotent methods: 500/502/503/504. Others (dataset push, KV PUT) only on
+    a 503 that certainly was not processed: the rate limiter refused it
+    ("rate-limit-unavailable", before the handler), or HAProxy had no backend
+    (its own page, nothing forwarded). Any other 503 may mean "outcome unknown"
+    (a direct dataset write whose commit was not confirmed), and re-sending
+    could store the items twice.
+    """
+    if idempotent:
+        return status in (500, 502, 503, 504)
+    if status != 503:
+        return False
+    return error_type == 'rate-limit-unavailable' or NO_BACKEND_MARKER in (body_text or '')
+
+
 def unwrap_data(value: Any) -> Any:
     """Unwrap Apify-style v2 response envelopes ({ data: ... })."""
     if isinstance(value, list):
@@ -33,6 +59,35 @@ def unwrap_data(value: Any) -> Any:
     return value
 
 
+def _error_type(response: httpx.Response) -> Optional[str]:
+    try:
+        body = response.json()
+    except Exception:
+        return None
+    if isinstance(body, dict) and isinstance(body.get('error'), dict):
+        t = body['error'].get('type')
+        return t if isinstance(t, str) else None
+    return None
+
+
+def server_retry_delay(server_attempt: int, retry_after: Optional[str], now: Optional[float] = None) -> float:
+    """Exponential backoff with jitter, in seconds; honours a Retry-After (seconds or HTTP date) of up to 30 s."""
+    if retry_after is not None and retry_after.strip():
+        secs: Optional[float] = None
+        try:
+            secs = float(retry_after)
+        except ValueError:
+            try:
+                from email.utils import parsedate_to_datetime
+
+                secs = max(0.0, parsedate_to_datetime(retry_after).timestamp() - (time.time() if now is None else now))
+            except Exception:
+                secs = None
+        if secs is not None and secs >= 0:
+            return min(secs, 30.0) + random.random() * 0.1
+    return min(8.0, 0.5 * 2 ** server_attempt) + random.random() * 0.25
+
+
 class AdaptiveRateLimiter:
     """Adaptive throttling based on X-RateLimit-* headers (TS AdaptiveRateLimiter)."""
 
@@ -116,6 +171,10 @@ class ApiClient:
 
     _rate_limiter = AdaptiveRateLimiter()
     _max_retries = 3
+    # Retries while the API is briefly unavailable (503/5xx, connection
+    # refused): backoff 0.5 s doubling to 8 s, about 45 s in total, which
+    # covers a main redeploy.
+    _max_server_retries = 8
 
     def __init__(
         self,
@@ -198,6 +257,7 @@ class ApiClient:
         headers: Optional[Dict[str, str]] = None,
         raise_for_status: bool = True,
         do_not_retry_timeouts: bool = False,
+        idempotent: Optional[bool] = None,
     ) -> httpx.Response:
         """Make an HTTP request with adaptive rate limiting and 429 retries."""
         client = await self._ensure_client()
@@ -208,8 +268,11 @@ class ApiClient:
         if json_data is not None and 'Content-Type' not in request_headers:
             request_headers['Content-Type'] = 'application/json'
 
-        last_error: Optional[Exception] = None
-        for attempt in range(self._max_retries + 1):
+        if idempotent is None:
+            idempotent = method.upper() in IDEMPOTENT_METHODS
+        attempt = 0  # timeouts and 429s
+        server_attempt = 0  # 503/5xx and connection failures (API briefly unavailable)
+        while True:
             await self._rate_limiter.wait_for_slot()
             try:
                 response = await client.request(
@@ -220,27 +283,59 @@ class ApiClient:
                     content=content,
                     headers=request_headers or None,
                 )
-            except httpx.TimeoutException as exc:
-                last_error = exc
+            except httpx.TimeoutException:
                 if do_not_retry_timeouts or attempt >= self._max_retries:
                     raise
-                await asyncio.sleep(0.25 * (attempt + 1))
+                attempt += 1
+                await asyncio.sleep(0.25 * attempt)
                 continue
+            except httpx.TransportError as exc:
+                # No response. A refused/unresolvable connection never reached
+                # the API, so it is safe for every method; anything else (reset
+                # mid-request) only for idempotent ones.
+                if (idempotent or isinstance(exc, httpx.ConnectError)) and server_attempt < self._max_server_retries:
+                    wait = server_retry_delay(server_attempt, None)
+                    server_attempt += 1
+                    logger.warning(
+                        '%s %s failed (%s); retry %d/%d in %.1fs',
+                        method, path, type(exc).__name__, server_attempt, self._max_server_retries, wait,
+                    )
+                    await asyncio.sleep(wait)
+                    continue
+                raise
 
             self._rate_limiter.update_from_headers(response.headers)
 
             if response.status_code == 429 and attempt < self._max_retries:
+                attempt += 1
                 await self._rate_limiter.handle_rate_limited(response)
                 continue
 
+            # API briefly unavailable (main redeploy, limiter store blip); see
+            # is_retryable_response for which answers are safe to re-send.
+            if (
+                not response.is_success
+                and server_attempt < self._max_server_retries
+                and is_retryable_response(
+                    response.status_code,
+                    idempotent,
+                    _error_type(response) if response.status_code == 503 else None,
+                    response.text if response.status_code == 503 else None,
+                )
+            ):
+                wait = server_retry_delay(server_attempt, response.headers.get('Retry-After'))
+                server_attempt += 1
+                logger.warning(
+                    '%s %s answered %d; retry %d/%d in %.1fs',
+                    method, path, response.status_code, server_attempt, self._max_server_retries, wait,
+                )
+                await asyncio.sleep(wait)
+                continue
+
             if raise_for_status:
                 await self._raise_for_status(response)
             return response
 
-        if last_error is not None:
-            raise last_error
-        raise ApiClientError('Request failed after retries', 429)
-
     async def get(
         self,
         path: str,
@@ -268,6 +363,7 @@ class ApiClient:
         params: Optional[Dict[str, Any]] = None,
         *,
         raise_for_status: bool = True,
+        idempotent: Optional[bool] = None,
     ) -> httpx.Response:
         return await self.request(
             'POST',
@@ -277,6 +373,7 @@ class ApiClient:
             content=content,
             headers=headers,
             raise_for_status=raise_for_status,
+            idempotent=idempotent,
         )
 
     async def put(
@@ -325,8 +422,10 @@ class ApiClient:
         data: Any = None,
         params: Optional[Dict[str, Any]] = None,
         headers: Optional[Dict[str, str]] = None,
+        *,
+        idempotent: Optional[bool] = None,
     ) -> Any:
-        response = await self.post(path, json_data=data, params=params, headers=headers)
+        response = await self.post(path, json_data=data, params=params, headers=headers, idempotent=idempotent)
         if not response.content:
             return {}
         try:

{scrapely_python-0.1.4.dist-info → scrapely_python-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.4
+Version: 0.1.5
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT

{scrapely_python-0.1.4.dist-info → scrapely_python-0.1.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-scrapely/__init__.py,sha256=nVwtGU9e9kfzoiX6MdfbK4U-p88J7eMDUkqNkpKY1Ug,5745
+scrapely/__init__.py,sha256=m8qsFMznobEHdjjlLu5KCP8Y1yKA3EM2pSoJJn-En_4,5745
 scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
 scrapely/actor.py,sha256=QczHOj2GVQnMbQZ_76vjjJNHxPrinWB_WCdZ6_41l00,75126
 scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
@@ -22,7 +22,7 @@ scrapely/request_loaders/__init__.py,sha256=aNgr4pRt3MziUbU6UQ_bjTT3moMNvvyUsPeC
 scrapely/request_loaders/_scrapely_request_list.py,sha256=cq7xx7Ai4qOhcqGV2TLqPR3o53FPCPKpVxWEMw5qKaM,7367
 scrapely/storage/__init__.py,sha256=diAhxEvOXjc3YVhZvzkO3MttJUXQWJ5ogxstiApsh7U,1405
 scrapely/storage/_metadata.py,sha256=WWawBnVN0b1iK2K-SlxZXUc6czb5wXWf8iC_0tzkgAQ,4764
-scrapely/storage/api_client.py,sha256=wrRxQ8h4jKdWY50HI0gXS-XHaFpzWPbVyLQ1nOYnsnA,11305
+scrapely/storage/api_client.py,sha256=q48maCfgEnyWGD_gIhehiBvA30AlbqKkNjouNu6BIa4,15870
 scrapely/storage/client.py,sha256=M3o_a_9azHNn6eL_UyEeVz-VQJvnT_th1HJPuCA4PkE,21746
 scrapely/storage/patch_storage.py,sha256=o3Fv0BByeZ1T9HhkzhY6bjolOFsIQcnqBoHq1EdG8Fw,8758
 scrapely/storage/scrapely_dataset_client.py,sha256=gRvlCDw9c0yO5GE5LQOa1xAyXc9pnb-E37Eq7fHqHCM,10338
@@ -37,7 +37,7 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9
 scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
 scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
 scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
-scrapely_python-0.1.4.dist-info/METADATA,sha256=ACM2A5RD1zJ8JV3cEY0jZigW3dpvsjQyiwJBFvKlLv4,3104
-scrapely_python-0.1.4.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-scrapely_python-0.1.4.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
-scrapely_python-0.1.4.dist-info/RECORD,,
+scrapely_python-0.1.5.dist-info/METADATA,sha256=9Jl9D1ijxCxthjVQY0YB3ryI6jipYzXONbjyOjA8YEs,3104
+scrapely_python-0.1.5.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+scrapely_python-0.1.5.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
+scrapely_python-0.1.5.dist-info/RECORD,,