scrapely-python 0.1.36__py3-none-any.whl → 0.1.40__py3-none-any.whl

scrapely/__init__.py

@@ -227,7 +227,7 @@ if TYPE_CHECKING:  # Static analysers and IDEs see the lazy names as plain impor
         set_api_storage,
     )
 
-__version__ = '0.1.36'
+__version__ = '0.1.40'
 __all__ = [
     # Core
     'Actor',

scrapely/charging.py

@@ -308,7 +308,8 @@ class ChargingManager:
 
     def _remaining_budget_usd(self) -> Decimal:
         """maxTotalChargeUsd minus this SDK's charges and the platform's other run spend."""
-        assert self._max_total_charge_usd is not None
+        if self._max_total_charge_usd is None:
+            raise RuntimeError('maxTotalChargeUsd is not set')
         return self._max_total_charge_usd - self._total_charged_usd - self._platform_other_usd
 
     def _sync_platform_budget(self) -> None:

scrapely/events/event_manager.py

@@ -122,13 +122,15 @@ class ScrapelyEventManager(CrawleeEventManager):
         # Run tasks but don't await them
         ws_url = self._configuration.get('actor_events_ws_url')
         if ws_url:
-            # Append authentication parameters to the WebSocket URL
-            # This matches the Node SDK behavior in events-client.ts
+            # Authenticate with the token in a header and the runId in the URL (URLs end up in access
+            # logs). This matches the Node SDK behavior in events-client.ts
             token = self._configuration.get('token')
             run_id = self._configuration.get('actor_run_id')
+            headers: dict[str, str] = {}
             if token and run_id:
                 separator = '&' if '?' in ws_url else '?'
-                ws_url = f"{ws_url}{separator}{urlencode({'token': token, 'runId': run_id})}"
+                ws_url = f"{ws_url}{separator}{urlencode({'runId': run_id})}"
+                headers['Authorization'] = f'Bearer {token}'
                 logger.debug(f'Connecting to platform events websocket with runId={run_id}')
             else:
                 logger.warning(
@@ -139,7 +141,7 @@ class ScrapelyEventManager(CrawleeEventManager):
             # Connect in the background without blocking init, matching the Node SDK: a slow or failed
             # connection is retried and must not crash the Actor.
             self._process_platform_messages_task = asyncio.create_task(
-                self._process_platform_messages(ws_url)
+                self._process_platform_messages(ws_url, headers)
             )
         else:
             logger.debug(
@@ -170,7 +172,7 @@ class ScrapelyEventManager(CrawleeEventManager):
 
         await super().__aexit__(exc_type, exc_value, exc_traceback)
 
-    async def _process_platform_messages(self, ws_url: str) -> None:
+    async def _process_platform_messages(self, ws_url: str, headers: Optional[dict[str, str]] = None) -> None:
         """Process messages from the platform WebSocket with reconnect logic."""
         try:
             import websockets.asyncio.client
@@ -189,7 +191,7 @@ class ScrapelyEventManager(CrawleeEventManager):
         try:
             # Used as an async iterator, `connect` reconnects with exponential backoff on failed connection attempts,
             # including the first one (timeouts, network errors and 5xx responses are retried).
-            async for websocket in websockets.asyncio.client.connect(ws_url):
+            async for websocket in websockets.asyncio.client.connect(ws_url, additional_headers=headers or None):
                 self._platform_events_websocket = websocket
                 if self._connected_to_platform_websocket and not self._connected_to_platform_websocket.done():
                     self._connected_to_platform_websocket.set_result(True)

scrapely/http/__init__.py

@@ -0,0 +1,57 @@
+"""scrapely.http: an HTTP client that looks like a real browser and handles blocks.
+
+Every request goes out with a current browser fingerprint whose TLS, HTTP/2,
+User-Agent, client hints and header order all come from one backend profile.
+Requests use identities (profile + proxy session + cookie jar) that are
+retired together when a site blocks them; the request is retried on a fresh
+identity at once, without waiting: the block was tied to the old identity.
+``polite=True`` turns on Retry-After, per-host backoff and a slow mode for
+actors that must be gentle with a site.
+
+Quick start::
+
+    from scrapely import Actor
+    from scrapely.http import AsyncClient
+
+    proxy = await Actor.create_proxy_configuration(groups=["RESIDENTIAL"], country_code="DE")
+    async with AsyncClient(proxy_configuration=proxy) as http:
+        r = await http.get("https://example.com/")
+        data = (await http.get("https://example.com/api/items", kind="json")).json()
+
+Migrating from curl_cffi is one line: ``from scrapely.http import AsyncSession``.
+"""
+
+from ._client import AsyncClient, AsyncHttpSession, Client, HttpSession
+from ._compat import AsyncSession, Session
+from ._detect import detect_block
+from ._options import FingerprintOptions
+from ._policy import BackoffPolicy
+from ._profiles import BrowserProfile, BrowserSpec, NoMatchingProfile, load_table
+from ._response import Headers, HTTPError, Response
+from ._stats import HttpStats
+from ._transport import TransportError
+
+# curl_cffi / requests-style name for transport errors.
+RequestsError = TransportError
+
+__all__ = [
+    'AsyncClient',
+    'AsyncHttpSession',
+    'AsyncSession',
+    'BackoffPolicy',
+    'BrowserProfile',
+    'BrowserSpec',
+    'Client',
+    'FingerprintOptions',
+    'HTTPError',
+    'Headers',
+    'HttpSession',
+    'HttpStats',
+    'NoMatchingProfile',
+    'RequestsError',
+    'Response',
+    'Session',
+    'TransportError',
+    'detect_block',
+    'load_table',
+]

scrapely/http/_capture.py

@@ -0,0 +1,97 @@
+"""Read the headers a backend profile really sends, over loopback HTTP/1.1.
+
+Used by the profile-table tests and to regenerate the ``headers`` of each
+profile in ``profiles.json`` after a backend upgrade::
+
+    python -m scrapely.http._capture > /tmp/native.json
+
+No network access: a one-shot server on 127.0.0.1 records the request.
+"""
+
+from __future__ import annotations
+
+import json
+import socket
+import sys
+import threading
+from typing import Dict, List, Tuple
+
+
+class LoopbackServer:
+    """Accepts connections on 127.0.0.1 and records each request's header lines in order."""
+
+    def __init__(self, response: bytes = b'HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok') -> None:
+        self.response = response
+        self.requests: List[str] = []
+        self._sock = socket.socket()
+        self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self._sock.bind(('127.0.0.1', 0))
+        self._sock.listen(64)
+        self.port = self._sock.getsockname()[1]
+        threading.Thread(target=self._loop, daemon=True).start()
+
+    @property
+    def url(self) -> str:
+        return f'http://127.0.0.1:{self.port}'
+
+    def _loop(self) -> None:
+        while True:
+            try:
+                conn, _ = self._sock.accept()
+            except OSError:
+                return
+            threading.Thread(target=self._one, args=(conn,), daemon=True).start()
+
+    def _one(self, conn: socket.socket) -> None:
+        data = b''
+        try:
+            while b'\r\n\r\n' not in data:
+                chunk = conn.recv(65536)
+                if not chunk:
+                    break
+                data += chunk
+            self.requests.append(data.decode('latin-1'))
+            conn.sendall(self.response)
+        finally:
+            conn.close()
+
+    def close(self) -> None:
+        self._sock.close()
+
+
+def parse_headers(raw: str) -> List[Tuple[str, str]]:
+    head = raw.split('\r\n\r\n', 1)[0]
+    out = []
+    for line in head.split('\r\n')[1:]:
+        if ':' in line:
+            k, v = line.split(':', 1)
+            out.append((k.strip().lower(), v.strip()))
+    return out
+
+
+def native_headers(wreq_profile: str, wreq_platform: str) -> List[Tuple[str, str]]:
+    """Header (name, value) pairs wreq sends for a profile, in its native order (host excluded)."""
+    import wreq
+    from wreq.blocking import Client
+
+    srv = LoopbackServer()
+    try:
+        emu = wreq.Emulation(profile=getattr(wreq.Profile, wreq_profile), platform=getattr(wreq.Platform, wreq_platform))
+        Client(emulation=emu).get(srv.url + '/').text()
+        return [(k, v) for k, v in parse_headers(srv.requests[-1]) if k != 'host']
+    finally:
+        srv.close()
+
+
+def main() -> None:
+    from ._profiles import load_table
+
+    out: Dict[str, Dict[str, str]] = {}
+    for p in load_table().profiles:
+        out[p.id] = dict(native_headers(p.wreq_profile, p.wreq_platform))
+    json.dump(out, sys.stdout, indent=2)
+    sys.stdout.write('\n')
+
+
+if __name__ == '__main__':
+    main()