scrapely-python 0.1.38__py3-none-any.whl → 0.1.40__py3-none-any.whl

scrapely/__init__.py

@@ -227,7 +227,7 @@ if TYPE_CHECKING:  # Static analysers and IDEs see the lazy names as plain impor
         set_api_storage,
     )
 
-__version__ = '0.1.38'
+__version__ = '0.1.40'
 __all__ = [
     # Core
     'Actor',

scrapely/charging.py

@@ -308,7 +308,8 @@ class ChargingManager:
 
     def _remaining_budget_usd(self) -> Decimal:
         """maxTotalChargeUsd minus this SDK's charges and the platform's other run spend."""
-        assert self._max_total_charge_usd is not None
+        if self._max_total_charge_usd is None:
+            raise RuntimeError('maxTotalChargeUsd is not set')
         return self._max_total_charge_usd - self._total_charged_usd - self._platform_other_usd
 
     def _sync_platform_budget(self) -> None:

scrapely/events/event_manager.py

@@ -122,13 +122,15 @@ class ScrapelyEventManager(CrawleeEventManager):
         # Run tasks but don't await them
         ws_url = self._configuration.get('actor_events_ws_url')
         if ws_url:
-            # Append authentication parameters to the WebSocket URL
-            # This matches the Node SDK behavior in events-client.ts
+            # Authenticate with the token in a header and the runId in the URL (URLs end up in access
+            # logs). This matches the Node SDK behavior in events-client.ts
             token = self._configuration.get('token')
             run_id = self._configuration.get('actor_run_id')
+            headers: dict[str, str] = {}
             if token and run_id:
                 separator = '&' if '?' in ws_url else '?'
-                ws_url = f"{ws_url}{separator}{urlencode({'token': token, 'runId': run_id})}"
+                ws_url = f"{ws_url}{separator}{urlencode({'runId': run_id})}"
+                headers['Authorization'] = f'Bearer {token}'
                 logger.debug(f'Connecting to platform events websocket with runId={run_id}')
             else:
                 logger.warning(
@@ -139,7 +141,7 @@ class ScrapelyEventManager(CrawleeEventManager):
             # Connect in the background without blocking init, matching the Node SDK: a slow or failed
             # connection is retried and must not crash the Actor.
             self._process_platform_messages_task = asyncio.create_task(
-                self._process_platform_messages(ws_url)
+                self._process_platform_messages(ws_url, headers)
             )
         else:
             logger.debug(
@@ -170,7 +172,7 @@ class ScrapelyEventManager(CrawleeEventManager):
 
         await super().__aexit__(exc_type, exc_value, exc_traceback)
 
-    async def _process_platform_messages(self, ws_url: str) -> None:
+    async def _process_platform_messages(self, ws_url: str, headers: Optional[dict[str, str]] = None) -> None:
         """Process messages from the platform WebSocket with reconnect logic."""
         try:
             import websockets.asyncio.client
@@ -189,7 +191,7 @@ class ScrapelyEventManager(CrawleeEventManager):
         try:
             # Used as an async iterator, `connect` reconnects with exponential backoff on failed connection attempts,
             # including the first one (timeouts, network errors and 5xx responses are retried).
-            async for websocket in websockets.asyncio.client.connect(ws_url):
+            async for websocket in websockets.asyncio.client.connect(ws_url, additional_headers=headers or None):
                 self._platform_events_websocket = websocket
                 if self._connected_to_platform_websocket and not self._connected_to_platform_websocket.done():
                     self._connected_to_platform_websocket.set_result(True)

scrapely/http/_client.py

@@ -11,7 +11,7 @@ from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple
 from urllib.parse import urlencode, urlsplit, urlunsplit
 
 from ._detect import detect_block, parse_retry_after
-from ._headers import accept_language_for, encode_body, prepare
+from ._headers import accept_language_for, country_for_locale, encode_body, pick_accept_language, pick_country, prepare
 from ._options import FingerprintOptions
 from ._policy import BackoffPolicy, HostState, Identity, RetryPolicy, country_from_proxy, decide, new_session_id
 from ._profiles import BrowserProfile, impersonate_to_options, load_table, pick, select_candidates
@@ -58,6 +58,29 @@ def _with_params(url: str, params: Any) -> str:
     return urlunsplit((parts.scheme, parts.netloc, parts.path, joined, parts.fragment))
 
 
+def _normalize_url(url: str) -> str:
+    """``https://host?q`` → ``https://host/?q``: an empty path is sent as ``/``, as browsers do.
+
+    wreq 0.13 sends ``:path: ?q`` over HTTP/2 for such URLs, and servers
+    (Cloudflare, nginx) answer 400 Bad Request.
+    """
+    parts = urlsplit(url)
+    if parts.scheme.lower() in ('http', 'https') and parts.netloc and not parts.path:
+        return urlunsplit((parts.scheme, parts.netloc, '/', parts.query, parts.fragment))
+    return url
+
+
+def _can_pick_country(proxy_configuration: Any) -> bool:
+    """True for a platform proxy that lets each identity choose its exit country."""
+    check = getattr(proxy_configuration, '_can_pick_country', None)
+    if not callable(check):
+        return False
+    try:
+        return bool(check())
+    except Exception:  # noqa: BLE001 - a foreign proxy configuration: keep its own behaviour
+        return False
+
+
 def _rotations(max_rotations: Optional[int], max_retries: Optional[int], default: Optional[int]) -> Optional[int]:
     """``max_rotations`` wins; ``max_retries`` is its older name."""
     if max_rotations is not None:
@@ -198,7 +221,12 @@ class _Base:
         # Only a proxy configuration gives each identity its own exit IP.
         self.rotates_ip = proxy_configuration is not None
         country = country_from_proxy(proxy_configuration if proxy_configuration is not None else self.fixed_proxy)
+        self.proxy_country = country
+        # Default Accept-Language of this client (identities draw their own, see _identity_geo).
         self.accept_language = accept_language_for(country, self.options.locales)
+        # A platform proxy without a country: every identity picks its exit
+        # country and sends the matching Accept-Language.
+        self.picks_country = country is None and proxy_configuration is not None and _can_pick_country(proxy_configuration)
         if backoff is None and polite:
             backoff = BackoffPolicy()
         self.policy = RetryPolicy(max_rotations=_rotations(max_rotations, max_retries, 6), backoff=backoff)
@@ -244,14 +272,44 @@ class _Base:
     def _pick_profile(self, avoid: Optional[BrowserProfile]) -> BrowserProfile:
         return pick(self.candidates, self.rng, avoid)
 
-    def _new_identity(self, client: Any, profile: BrowserProfile, session_id: str, proxy_url: Optional[str]) -> Identity:
+    def _identity_geo(self) -> Tuple[Optional[str], str]:
+        """(exit country to request from the proxy or None, Accept-Language) for a new identity.
+
+        - ``locales``: one of them per identity, as a browser has one language
+          setup; with a country-pickable proxy the exit is that locale's country.
+        - proxy with a country: a language setup common in that country.
+        - platform proxy without a country: a country weighted like real
+          traffic, used for both the proxy session and the language.
+        - otherwise (no proxy, own proxy URLs, datacenter): the client default.
+        """
+        locales = self.options.locales
+        if locales:
+            loc = locales[0] if len(locales) == 1 else self.rng.choice(locales)
+            return (country_for_locale(loc) if self.picks_country else None), accept_language_for(None, [loc])
+        if self.proxy_country:
+            return None, pick_accept_language(self.proxy_country, self.rng)
+        if self.picks_country:
+            cc = pick_country(self.rng)
+            return cc, pick_accept_language(cc, self.rng)
+        return None, self.accept_language
+
+    def _new_identity(
+        self,
+        client: Any,
+        profile: BrowserProfile,
+        session_id: str,
+        proxy_url: Optional[str],
+        accept_language: Optional[str] = None,
+        country: Optional[str] = None,
+    ) -> Identity:
         ident = Identity(
             session_id=session_id,
             profile=profile,
             proxy_url=proxy_url,
             client=client,
-            accept_language=self.accept_language,
+            accept_language=accept_language or self.accept_language,
             created_at=self._clock(),
+            country=country or self.proxy_country,
         )
         for name, value, domain, path in self._caller_cookies._cookies:
             self._apply_cookie(ident, name, value, domain, path)
@@ -428,18 +486,22 @@ class AsyncClient(_Base):
             self._cond = asyncio.Condition()
         return self._cond
 
-    async def _proxy_url(self, session_id: str) -> Optional[str]:
+    async def _proxy_url(self, session_id: str, country: Optional[str] = None) -> Optional[str]:
         if self.proxy_configuration is not None:
-            url = await self.proxy_configuration.new_url(session_id=session_id)
+            if country:
+                url = await self.proxy_configuration.new_url(session_id=session_id, country_code=country)
+            else:
+                url = await self.proxy_configuration.new_url(session_id=session_id)
             return str(url) if url else None
         return self.fixed_proxy
 
     async def _open(self, avoid: Optional[BrowserProfile]) -> Identity:
         profile = self._pick_profile(avoid)
         sid = new_session_id(self.rng)
-        proxy_url = await self._proxy_url(sid)
+        country, al = self._identity_geo()
+        proxy_url = await self._proxy_url(sid, country)
         client = self.transport.new_client(profile, proxy_url, self.timeout, self.verify, self.http1_only)
-        return self._new_identity(client, profile, sid, proxy_url)
+        return self._new_identity(client, profile, sid, proxy_url, al, country)
 
     async def _retire(self, ident: Identity, force: bool = False) -> None:
         """Retire an identity; its client closes once no request uses it any more."""
@@ -536,7 +598,7 @@ class AsyncClient(_Base):
     ) -> Response:
         """Send a request; a block or error rotates to a fresh identity at once. Returns the last response."""
         method = method.upper()
-        url = _with_params(url, params)
+        url = _normalize_url(_with_params(url, params))
         body, ctype = encode_body(data, json, content)
         host, hs = self._host(url)
         started = self._clock()
@@ -684,6 +746,16 @@ class AsyncHttpSession:
     def session_id(self) -> Optional[str]:
         return self._pin.identity.session_id if self._pin.identity else None
 
+    @property
+    def accept_language(self) -> Optional[str]:
+        """Accept-Language of the current identity."""
+        return self._pin.identity.accept_language if self._pin.identity else None
+
+    @property
+    def country(self) -> Optional[str]:
+        """Exit country the current identity asked the proxy for (None when the proxy chooses)."""
+        return self._pin.identity.country if self._pin.identity else None
+
     async def request(self, method: str, url: str, **kwargs: Any) -> Response:
         return await self._client.request(method, url, _pin=self._pin, **kwargs)
 
@@ -755,18 +827,18 @@ class Client(_Base):
     def _make_transport(self, backend: str) -> Any:
         return WreqSyncTransport() if backend == 'wreq' else CurlSyncTransport(self.table.curl_cffi_targets)
 
-    def _proxy_url(self, session_id: str) -> Optional[str]:
+    def _proxy_url(self, session_id: str, country: Optional[str] = None) -> Optional[str]:
         pc = self.proxy_configuration
         if pc is None:
             return self.fixed_proxy
         sync = getattr(pc, '_new_url_sync', None)
         if sync is not None and not getattr(pc, '_new_url_function', None):
-            url = sync(session_id)
+            url = sync(session_id, country_code=country) if country else sync(session_id)
         else:
             try:
                 asyncio.get_running_loop()
             except RuntimeError:
-                url = asyncio.run(pc.new_url(session_id=session_id))
+                url = asyncio.run(pc.new_url(session_id=session_id, country_code=country) if country else pc.new_url(session_id=session_id))
             else:
                 raise RuntimeError('scrapely.http.Client: use AsyncClient inside async code (this proxy configuration needs await)')
         return str(url) if url else None
@@ -775,10 +847,11 @@ class Client(_Base):
         profile = self._pick_profile(avoid)
         with self._lock:
             sid = new_session_id(self.rng)
-        proxy_url = self._proxy_url(sid)
+            country, al = self._identity_geo()
+        proxy_url = self._proxy_url(sid, country)
         client = self.transport.new_client(profile, proxy_url, self.timeout, self.verify, self.http1_only)
         with self._lock:
-            return self._new_identity(client, profile, sid, proxy_url)
+            return self._new_identity(client, profile, sid, proxy_url, al, country)
 
     def _retire(self, ident: Identity, force: bool = False) -> None:
         """Retire an identity; its client closes once no request uses it any more."""
@@ -885,7 +958,7 @@ class Client(_Base):
     ) -> Response:
         """Send a request; a block or error rotates to a fresh identity at once. Returns the last response."""
         method = method.upper()
-        url = _with_params(url, params)
+        url = _normalize_url(_with_params(url, params))
         body, ctype = encode_body(data, json, content)
         with self._lock:
             host, hs = self._host(url)
@@ -1030,6 +1103,16 @@ class HttpSession:
     def session_id(self) -> Optional[str]:
         return self._pin.identity.session_id if self._pin.identity else None
 
+    @property
+    def accept_language(self) -> Optional[str]:
+        """Accept-Language of the current identity."""
+        return self._pin.identity.accept_language if self._pin.identity else None
+
+    @property
+    def country(self) -> Optional[str]:
+        """Exit country the current identity asked the proxy for (None when the proxy chooses)."""
+        return self._pin.identity.country if self._pin.identity else None
+
     def request(self, method: str, url: str, **kwargs: Any) -> Response:
         return self._client.request(method, url, _pin=self._pin, **kwargs)
 

scrapely/http/_headers.py

@@ -107,6 +107,74 @@ def accept_language_for(country_code: Optional[str] = None, locales: Sequence[st
     return f'{lang}-{cc},{lang};q=0.9,en;q=0.8'
 
 
+# ─── per-identity geo: exit country + matching Accept-Language ──────────────
+
+# Exit countries for identities whose proxy has no country set, weighted like
+# desktop web traffic (rough shares by country; only countries the residential
+# pool serves well). The proxy session and Accept-Language use the same draw,
+# so the language always matches the exit IP.
+TRAFFIC_COUNTRIES: Tuple[Tuple[str, float], ...] = (
+    ('US', 32.0), ('GB', 7.0), ('DE', 7.0), ('FR', 5.0), ('CA', 4.0), ('BR', 4.0), ('IN', 3.0),
+    ('IT', 3.5), ('ES', 3.5), ('NL', 2.5), ('PL', 2.5), ('MX', 2.5), ('AU', 2.5), ('JP', 2.0),
+    ('TR', 1.5), ('SE', 1.0), ('BE', 1.0), ('AT', 1.0), ('CH', 1.0), ('PT', 1.0), ('AR', 1.0),
+    ('CO', 1.0), ('RO', 1.0), ('CZ', 1.0), ('ID', 1.0),
+)
+
+# Main country for a bare language (``locales=["de"]``): where its speakers mostly are.
+_LANG_COUNTRY: Dict[str, str] = {
+    'en': 'US', 'de': 'DE', 'fr': 'FR', 'es': 'ES', 'it': 'IT', 'pt': 'BR', 'nl': 'NL', 'pl': 'PL',
+    'cs': 'CZ', 'sk': 'SK', 'hu': 'HU', 'ro': 'RO', 'bg': 'BG', 'el': 'GR', 'tr': 'TR', 'ru': 'RU',
+    'uk': 'UA', 'sv': 'SE', 'nb': 'NO', 'no': 'NO', 'da': 'DK', 'fi': 'FI', 'ja': 'JP', 'ko': 'KR',
+    'zh': 'TW', 'vi': 'VN', 'th': 'TH', 'id': 'ID', 'ms': 'MY', 'he': 'IL', 'ar': 'SA', 'hi': 'IN',
+}
+
+
+def pick_country(rng: Any) -> str:
+    """A country drawn from ``TRAFFIC_COUNTRIES``."""
+    names = [c for c, _ in TRAFFIC_COUNTRIES]
+    return rng.choices(names, weights=[w for _, w in TRAFFIC_COUNTRIES], k=1)[0]
+
+
+def country_for_locale(locale: str) -> Optional[str]:
+    """The country a locale points at: ``de-AT`` → AT, ``de`` → DE, ``en`` → US."""
+    parts = str(locale).strip().replace('_', '-').split('-')
+    for p in parts[1:]:
+        if len(p) == 2 and p.isalpha():
+            return p.upper()
+    return _LANG_COUNTRY.get(parts[0].lower())
+
+
+def accept_language_variants(country_code: Optional[str]) -> List[Tuple[str, float]]:
+    """Accept-Language values real Chrome users in a country send, with rough weights.
+
+    All variants put the country's language first (``de-DE,de;q=0.9`` with or
+    without English after it), so any of them matches an exit IP in that
+    country. Chrome builds the header from the user's language list: the
+    regional tag, its base language, then further languages 0.1 lower each.
+    """
+    cc = (country_code or '').strip().upper()
+    if cc == 'UK':
+        cc = 'GB'
+    if not cc or cc == 'US':
+        return [(DEFAULT_ACCEPT_LANGUAGE, 0.92), ('en-US,en;q=0.9,es;q=0.8', 0.08)]
+    if cc in _ENGLISH_COUNTRIES:
+        return [(f'en-{cc},en;q=0.9', 0.5), (f'en-{cc},en-US;q=0.9,en;q=0.8', 0.2), (DEFAULT_ACCEPT_LANGUAGE, 0.3)]
+    lang = _COUNTRY_LANG.get(cc)
+    if not lang:
+        return [(DEFAULT_ACCEPT_LANGUAGE, 1.0)]
+    return [
+        (f'{lang}-{cc},{lang};q=0.9,en-US;q=0.8,en;q=0.7', 0.45),
+        (f'{lang}-{cc},{lang};q=0.9', 0.30),
+        (f'{lang}-{cc},{lang};q=0.9,en;q=0.8', 0.25),
+    ]
+
+
+def pick_accept_language(country_code: Optional[str], rng: Any) -> str:
+    """One Accept-Language for an identity whose exit is in ``country_code``."""
+    variants = accept_language_variants(country_code)
+    return rng.choices([v for v, _ in variants], weights=[w for _, w in variants], k=1)[0]
+
+
 # ─── sec-fetch-site ─────────────────────────────────────────────────────────
 
 _TWO_LEVEL_SUFFIXES = frozenset(
@@ -257,6 +325,10 @@ def prepare(
     if kind == 'navigate':
         # Backend default headers; we only adjust values that depend on the request.
         prep.default_headers = True
+        for h in profile.header_overrides:
+            # Newer version on the backend's TLS profile: its own UA, client hints and Accept.
+            if h in native:
+                out[h] = native[h]
         if al != native.get('accept-language'):
             out['accept-language'] = al
         site = fetch_site(referer, url)

scrapely/http/_options.py

@@ -99,16 +99,16 @@ class FingerprintOptions:
     """Which browsers, operating systems, devices and locales a client may present.
 
     All fields are optional. The defaults are desktop Chrome on Windows, macOS
-    and Linux (weighted toward real traffic), with Accept-Language taken from
-    the proxy country.
+    and Linux (weighted toward real traffic), with Accept-Language matching
+    each identity's exit country.
 
     Attributes:
         browsers: ``"chrome"``, ``"edge"``, ``"firefox"``, ``"safari"``, or
             ``{"name": "chrome", "min_version": 150}``.
         operating_systems: ``"windows"``, ``"macos"``, ``"linux"``, ``"android"``, ``"ios"``.
         devices: ``"desktop"`` (default) and/or ``"mobile"``.
-        locales: e.g. ``["de-DE"]``; the first is the primary language. Wins over
-            the proxy country.
+        locales: e.g. ``["de-DE"]``. Each identity uses one of them, as a
+            browser has one language setup. Wins over the proxy country.
         strict: only profiles whose fingerprint was verified against the real
             browser, and fail instead of relaxing options that match nothing.
         http_version: ``"2"`` (default) or ``"1"`` to force HTTP/1.1.

scrapely/http/_policy.py

@@ -198,6 +198,7 @@ class Identity:
     retired: bool = False
     in_flight: int = 0  # requests using it right now (a session's identity is shared)
     closed: bool = False
+    country: Optional[str] = None  # exit country requested from the proxy, when known
 
     def expired(self, now: float, max_usage: int, max_age: float) -> bool:
         if self.retired:

scrapely/http/_profiles.py

@@ -39,6 +39,10 @@ class BrowserProfile:
     weight: float
     headers: Dict[str, str] = field(default_factory=dict, compare=False, hash=False)
     note: str = field(default='', compare=False, hash=False)
+    # Headers whose value differs from the backend profile's own default
+    # (a newer browser version on the same TLS/HTTP2 profile: User-Agent,
+    # client hints, Accept). They are sent explicitly on every request.
+    header_overrides: Tuple[str, ...] = field(default=(), compare=False, hash=False)
 
     @property
     def label(self) -> str:
@@ -83,6 +87,7 @@ def _parse_table(raw: Dict[str, Any]) -> ProfileTable:
                 weight=float(row.get('weight') or 0.0),
                 headers={k.lower(): v for k, v in (row.get('headers') or {}).items()},
                 note=row.get('note') or '',
+                header_overrides=tuple(h.lower() for h in (row.get('header_overrides') or ())),
             )
         )
     return ProfileTable(

scrapely/http/profiles.json

@@ -2,7 +2,7 @@
   "schema": 1,
   "backend": "wreq",
   "backend_version": "0.13.0",
-  "generated": "2026-10-27",
+  "generated": "2026-10-28",
   "reference_versions": {
     "chrome": 156,
     "edge": 156,
@@ -221,24 +221,24 @@
   },
   "profiles": [
     {
-      "id": "chrome154-win",
+      "id": "chrome156-win",
       "family": "chrome",
-      "version": 154,
-      "version_label": "154",
+      "version": 156,
+      "version_label": "156",
       "os": "windows",
       "device": "desktop",
       "wreq_profile": "Chrome154",
       "wreq_platform": "Windows",
       "verified": true,
-      "weight": 0.39,
-      "note": "TLS, HTTP/2 and header order equal to real Chrome 156 on tls.peet.ws (JA4 t13d1517h2_8daaf6152771_cb7bf5808d99, Akamai 1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p), 2026-10-27",
+      "weight": 0.3575,
+      "note": "Chrome 156 on the Chrome154 TLS/HTTP2 profile. Chrome for Testing 154, 155 and 156 send the same cipher suites, extension set, groups, signature algorithms and HTTP/2 settings (local captures 2026-10-28; those builds add extension 0x12e0, which real Chrome 156 stable did not send on tls.peet.ws, where it equals this profile, 2026-10-27). UA, client hints and Accept (image/jxl since 155) are 156's own.",
       "headers": {
-        "sec-ch-ua": "\"Chromium\";v=\"154\", \"Google Chrome\";v=\"154\", \"Not A(Brand\";v=\"99\"",
+        "sec-ch-ua": "\"Not:A-Brand\";v=\"8\", \"Chromium\";v=\"156\", \"Google Chrome\";v=\"156\"",
         "sec-ch-ua-mobile": "?0",
         "sec-ch-ua-platform": "\"Windows\"",
         "upgrade-insecure-requests": "1",
-        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36",
-        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
         "sec-fetch-site": "none",
         "sec-fetch-mode": "navigate",
         "sec-fetch-user": "?1",
@@ -246,27 +246,98 @@
         "accept-encoding": "gzip, deflate, br, zstd",
         "accept-language": "en-US,en;q=0.9",
         "priority": "u=0, i"
-      }
+      },
+      "header_overrides": [
+        "user-agent",
+        "sec-ch-ua",
+        "accept"
+      ]
     },
     {
-      "id": "chrome153-win",
+      "id": "chrome156-mac",
       "family": "chrome",
-      "version": 153,
-      "version_label": "153",
+      "version": 156,
+      "version_label": "156",
+      "os": "macos",
+      "device": "desktop",
+      "wreq_profile": "Chrome154",
+      "wreq_platform": "MacOS",
+      "verified": true,
+      "weight": 0.1485,
+      "note": "Chrome 156 on the Chrome154 TLS/HTTP2 profile. Chrome for Testing 154, 155 and 156 send the same cipher suites, extension set, groups, signature algorithms and HTTP/2 settings (local captures 2026-10-28; those builds add extension 0x12e0, which real Chrome 156 stable did not send on tls.peet.ws, where it equals this profile, 2026-10-27). UA, client hints and Accept (image/jxl since 155) are 156's own.",
+      "headers": {
+        "sec-ch-ua": "\"Not:A-Brand\";v=\"8\", \"Chromium\";v=\"156\", \"Google Chrome\";v=\"156\"",
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": "\"macOS\"",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "sec-fetch-site": "none",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-user": "?1",
+        "sec-fetch-dest": "document",
+        "accept-encoding": "gzip, deflate, br, zstd",
+        "accept-language": "en-US,en;q=0.9",
+        "priority": "u=0, i"
+      },
+      "header_overrides": [
+        "user-agent",
+        "sec-ch-ua",
+        "accept"
+      ]
+    },
+    {
+      "id": "chrome156-linux",
+      "family": "chrome",
+      "version": 156,
+      "version_label": "156",
+      "os": "linux",
+      "device": "desktop",
+      "wreq_profile": "Chrome154",
+      "wreq_platform": "Linux",
+      "verified": true,
+      "weight": 0.044,
+      "note": "Chrome 156 on the Chrome154 TLS/HTTP2 profile. Chrome for Testing 154, 155 and 156 send the same cipher suites, extension set, groups, signature algorithms and HTTP/2 settings (local captures 2026-10-28; those builds add extension 0x12e0, which real Chrome 156 stable did not send on tls.peet.ws, where it equals this profile, 2026-10-27). UA, client hints and Accept (image/jxl since 155) are 156's own.",
+      "headers": {
+        "sec-ch-ua": "\"Not:A-Brand\";v=\"8\", \"Chromium\";v=\"156\", \"Google Chrome\";v=\"156\"",
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": "\"Linux\"",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "sec-fetch-site": "none",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-user": "?1",
+        "sec-fetch-dest": "document",
+        "accept-encoding": "gzip, deflate, br, zstd",
+        "accept-language": "en-US,en;q=0.9",
+        "priority": "u=0, i"
+      },
+      "header_overrides": [
+        "user-agent",
+        "sec-ch-ua",
+        "accept"
+      ]
+    },
+    {
+      "id": "chrome155-win",
+      "family": "chrome",
+      "version": 155,
+      "version_label": "155",
       "os": "windows",
       "device": "desktop",
-      "wreq_profile": "Chrome153",
+      "wreq_profile": "Chrome154",
       "wreq_platform": "Windows",
       "verified": true,
-      "weight": 0.26,
-      "note": "TLS, HTTP/2 and header order equal to real Chrome 156 on tls.peet.ws (JA4 t13d1517h2_8daaf6152771_cb7bf5808d99, Akamai 1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p), 2026-10-27",
+      "weight": 0.195,
+      "note": "Chrome 155 on the Chrome154 TLS/HTTP2 profile. Chrome for Testing 154, 155 and 156 send the same cipher suites, extension set, groups, signature algorithms and HTTP/2 settings (local captures 2026-10-28; those builds add extension 0x12e0, which real Chrome 156 stable did not send on tls.peet.ws, where it equals this profile, 2026-10-27). UA, client hints and Accept (image/jxl since 155) are 155's own.",
       "headers": {
-        "sec-ch-ua": "\"Google Chrome\";v=\"153\", \"Not_A Brand\";v=\"8\", \"Chromium\";v=\"153\"",
+        "sec-ch-ua": "\"Google Chrome\";v=\"155\", \"Chromium\";v=\"155\", \"Not(A:Brand\";v=\"24\"",
         "sec-ch-ua-mobile": "?0",
         "sec-ch-ua-platform": "\"Windows\"",
         "upgrade-insecure-requests": "1",
-        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36",
-        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
         "sec-fetch-site": "none",
         "sec-fetch-mode": "navigate",
         "sec-fetch-user": "?1",
@@ -274,26 +345,97 @@
         "accept-encoding": "gzip, deflate, br, zstd",
         "accept-language": "en-US,en;q=0.9",
         "priority": "u=0, i"
-      }
+      },
+      "header_overrides": [
+        "user-agent",
+        "sec-ch-ua",
+        "accept"
+      ]
     },
     {
-      "id": "chrome154-mac",
+      "id": "chrome155-mac",
       "family": "chrome",
-      "version": 154,
-      "version_label": "154",
+      "version": 155,
+      "version_label": "155",
       "os": "macos",
       "device": "desktop",
       "wreq_profile": "Chrome154",
       "wreq_platform": "MacOS",
       "verified": true,
-      "weight": 0.162,
+      "weight": 0.081,
+      "note": "Chrome 155 on the Chrome154 TLS/HTTP2 profile. Chrome for Testing 154, 155 and 156 send the same cipher suites, extension set, groups, signature algorithms and HTTP/2 settings (local captures 2026-10-28; those builds add extension 0x12e0, which real Chrome 156 stable did not send on tls.peet.ws, where it equals this profile, 2026-10-27). UA, client hints and Accept (image/jxl since 155) are 155's own.",
+      "headers": {
+        "sec-ch-ua": "\"Google Chrome\";v=\"155\", \"Chromium\";v=\"155\", \"Not(A:Brand\";v=\"24\"",
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": "\"macOS\"",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "sec-fetch-site": "none",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-user": "?1",
+        "sec-fetch-dest": "document",
+        "accept-encoding": "gzip, deflate, br, zstd",
+        "accept-language": "en-US,en;q=0.9",
+        "priority": "u=0, i"
+      },
+      "header_overrides": [
+        "user-agent",
+        "sec-ch-ua",
+        "accept"
+      ]
+    },
+    {
+      "id": "chrome155-linux",
+      "family": "chrome",
+      "version": 155,
+      "version_label": "155",
+      "os": "linux",
+      "device": "desktop",
+      "wreq_profile": "Chrome154",
+      "wreq_platform": "Linux",
+      "verified": true,
+      "weight": 0.024,
+      "note": "Chrome 155 on the Chrome154 TLS/HTTP2 profile. Chrome for Testing 154, 155 and 156 send the same cipher suites, extension set, groups, signature algorithms and HTTP/2 settings (local captures 2026-10-28; those builds add extension 0x12e0, which real Chrome 156 stable did not send on tls.peet.ws, where it equals this profile, 2026-10-27). UA, client hints and Accept (image/jxl since 155) are 155's own.",
+      "headers": {
+        "sec-ch-ua": "\"Google Chrome\";v=\"155\", \"Chromium\";v=\"155\", \"Not(A:Brand\";v=\"24\"",
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": "\"Linux\"",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "sec-fetch-site": "none",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-user": "?1",
+        "sec-fetch-dest": "document",
+        "accept-encoding": "gzip, deflate, br, zstd",
+        "accept-language": "en-US,en;q=0.9",
+        "priority": "u=0, i"
+      },
+      "header_overrides": [
+        "user-agent",
+        "sec-ch-ua",
+        "accept"
+      ]
+    },
+    {
+      "id": "chrome154-win",
+      "family": "chrome",
+      "version": 154,
+      "version_label": "154",
+      "os": "windows",
+      "device": "desktop",
+      "wreq_profile": "Chrome154",
+      "wreq_platform": "Windows",
+      "verified": true,
+      "weight": 0.065,
       "note": "TLS, HTTP/2 and header order equal to real Chrome 156 on tls.peet.ws (JA4 t13d1517h2_8daaf6152771_cb7bf5808d99, Akamai 1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p), 2026-10-27",
       "headers": {
         "sec-ch-ua": "\"Chromium\";v=\"154\", \"Google Chrome\";v=\"154\", \"Not A(Brand\";v=\"99\"",
         "sec-ch-ua-mobile": "?0",
-        "sec-ch-ua-platform": "\"macOS\"",
+        "sec-ch-ua-platform": "\"Windows\"",
         "upgrade-insecure-requests": "1",
-        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36",
         "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
         "sec-fetch-site": "none",
         "sec-fetch-mode": "navigate",
@@ -305,23 +447,23 @@
       }
     },
     {
-      "id": "chrome153-mac",
+      "id": "chrome154-mac",
       "family": "chrome",
-      "version": 153,
-      "version_label": "153",
+      "version": 154,
+      "version_label": "154",
       "os": "macos",
       "device": "desktop",
-      "wreq_profile": "Chrome153",
+      "wreq_profile": "Chrome154",
       "wreq_platform": "MacOS",
       "verified": true,
-      "weight": 0.108,
+      "weight": 0.027,
       "note": "TLS, HTTP/2 and header order equal to real Chrome 156 on tls.peet.ws (JA4 t13d1517h2_8daaf6152771_cb7bf5808d99, Akamai 1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p), 2026-10-27",
       "headers": {
-        "sec-ch-ua": "\"Google Chrome\";v=\"153\", \"Not_A Brand\";v=\"8\", \"Chromium\";v=\"153\"",
+        "sec-ch-ua": "\"Chromium\";v=\"154\", \"Google Chrome\";v=\"154\", \"Not A(Brand\";v=\"99\"",
         "sec-ch-ua-mobile": "?0",
         "sec-ch-ua-platform": "\"macOS\"",
         "upgrade-insecure-requests": "1",
-        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36",
+        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36",
         "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
         "sec-fetch-site": "none",
         "sec-fetch-mode": "navigate",
@@ -342,7 +484,7 @@
       "wreq_profile": "Chrome154",
       "wreq_platform": "Linux",
       "verified": true,
-      "weight": 0.048,
+      "weight": 0.008,
       "note": "TLS, HTTP/2 and header order equal to real Chrome 156 on tls.peet.ws (JA4 t13d1517h2_8daaf6152771_cb7bf5808d99, Akamai 1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p), 2026-10-27",
       "headers": {
         "sec-ch-ua": "\"Chromium\";v=\"154\", \"Google Chrome\";v=\"154\", \"Not A(Brand\";v=\"99\"",
@@ -360,6 +502,62 @@
         "priority": "u=0, i"
       }
     },
+    {
+      "id": "chrome153-win",
+      "family": "chrome",
+      "version": 153,
+      "version_label": "153",
+      "os": "windows",
+      "device": "desktop",
+      "wreq_profile": "Chrome153",
+      "wreq_platform": "Windows",
+      "verified": true,
+      "weight": 0.0325,
+      "note": "TLS, HTTP/2 and header order equal to real Chrome 156 on tls.peet.ws (JA4 t13d1517h2_8daaf6152771_cb7bf5808d99, Akamai 1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p), 2026-10-27",
+      "headers": {
+        "sec-ch-ua": "\"Google Chrome\";v=\"153\", \"Not_A Brand\";v=\"8\", \"Chromium\";v=\"153\"",
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": "\"Windows\"",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "sec-fetch-site": "none",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-user": "?1",
+        "sec-fetch-dest": "document",
+        "accept-encoding": "gzip, deflate, br, zstd",
+        "accept-language": "en-US,en;q=0.9",
+        "priority": "u=0, i"
+      }
+    },
+    {
+      "id": "chrome153-mac",
+      "family": "chrome",
+      "version": 153,
+      "version_label": "153",
+      "os": "macos",
+      "device": "desktop",
+      "wreq_profile": "Chrome153",
+      "wreq_platform": "MacOS",
+      "verified": true,
+      "weight": 0.0135,
+      "note": "TLS, HTTP/2 and header order equal to real Chrome 156 on tls.peet.ws (JA4 t13d1517h2_8daaf6152771_cb7bf5808d99, Akamai 1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p), 2026-10-27",
+      "headers": {
+        "sec-ch-ua": "\"Google Chrome\";v=\"153\", \"Not_A Brand\";v=\"8\", \"Chromium\";v=\"153\"",
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": "\"macOS\"",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "sec-fetch-site": "none",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-user": "?1",
+        "sec-fetch-dest": "document",
+        "accept-encoding": "gzip, deflate, br, zstd",
+        "accept-language": "en-US,en;q=0.9",
+        "priority": "u=0, i"
+      }
+    },
     {
       "id": "chrome153-linux",
       "family": "chrome",
@@ -370,7 +568,7 @@
       "wreq_profile": "Chrome153",
       "wreq_platform": "Linux",
       "verified": true,
-      "weight": 0.032,
+      "weight": 0.004,
       "note": "TLS, HTTP/2 and header order equal to real Chrome 156 on tls.peet.ws (JA4 t13d1517h2_8daaf6152771_cb7bf5808d99, Akamai 1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p), 2026-10-27",
       "headers": {
         "sec-ch-ua": "\"Google Chrome\";v=\"153\", \"Not_A Brand\";v=\"8\", \"Chromium\";v=\"153\"",
@@ -388,6 +586,72 @@
         "priority": "u=0, i"
       }
     },
+    {
+      "id": "chrome156-android",
+      "family": "chrome",
+      "version": 156,
+      "version_label": "156",
+      "os": "android",
+      "device": "mobile",
+      "wreq_profile": "Chrome154",
+      "wreq_platform": "Android",
+      "verified": false,
+      "weight": 0.275,
+      "note": "Chrome 156 Android on the Chrome154 TLS profile (equal to desktop); no real Android Chrome capture to compare against",
+      "headers": {
+        "sec-ch-ua": "\"Not:A-Brand\";v=\"8\", \"Chromium\";v=\"156\", \"Google Chrome\";v=\"156\"",
+        "sec-ch-ua-mobile": "?1",
+        "sec-ch-ua-platform": "\"Android\"",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Mobile Safari/537.36",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "sec-fetch-site": "none",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-user": "?1",
+        "sec-fetch-dest": "document",
+        "accept-encoding": "gzip, deflate, br, zstd",
+        "accept-language": "en-US,en;q=0.9",
+        "priority": "u=0, i"
+      },
+      "header_overrides": [
+        "user-agent",
+        "sec-ch-ua",
+        "accept"
+      ]
+    },
+    {
+      "id": "chrome155-android",
+      "family": "chrome",
+      "version": 155,
+      "version_label": "155",
+      "os": "android",
+      "device": "mobile",
+      "wreq_profile": "Chrome154",
+      "wreq_platform": "Android",
+      "verified": false,
+      "weight": 0.15,
+      "note": "Chrome 155 Android on the Chrome154 TLS profile (equal to desktop); no real Android Chrome capture to compare against",
+      "headers": {
+        "sec-ch-ua": "\"Google Chrome\";v=\"155\", \"Chromium\";v=\"155\", \"Not(A:Brand\";v=\"24\"",
+        "sec-ch-ua-mobile": "?1",
+        "sec-ch-ua-platform": "\"Android\"",
+        "upgrade-insecure-requests": "1",
+        "user-agent": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Mobile Safari/537.36",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        "sec-fetch-site": "none",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-user": "?1",
+        "sec-fetch-dest": "document",
+        "accept-encoding": "gzip, deflate, br, zstd",
+        "accept-language": "en-US,en;q=0.9",
+        "priority": "u=0, i"
+      },
+      "header_overrides": [
+        "user-agent",
+        "sec-ch-ua",
+        "accept"
+      ]
+    },
     {
       "id": "chrome154-android",
       "family": "chrome",
@@ -398,7 +662,7 @@
       "wreq_profile": "Chrome154",
       "wreq_platform": "Android",
       "verified": false,
-      "weight": 0.5,
+      "weight": 0.075,
       "note": "TLS equals desktop Chrome 154; no real Android Chrome capture to compare against",
       "headers": {
         "sec-ch-ua": "\"Chromium\";v=\"154\", \"Google Chrome\";v=\"154\", \"Not A(Brand\";v=\"99\"",
@@ -472,6 +736,93 @@
         "priority": "u=0, i"
       }
     },
+    {
+      "id": "firefox153-win",
+      "family": "firefox",
+      "version": 153,
+      "version_label": "153",
+      "os": "windows",
+      "device": "desktop",
+      "wreq_profile": "Firefox152",
+      "wreq_platform": "Windows",
+      "verified": true,
+      "weight": 0.032,
+      "note": "Firefox 153 ESR on the Firefox152 profile. TLS ClientHello (ciphers incl. c00a, extensions, groups incl. ffdhe2048/3072, signature algorithms, key shares) and HTTP/2 (1:65536;2:0;4:131072;5:16384|12517377|0|m,p,a,s, header order) equal to real Firefox 153.6.0esr (and 152.0), Linux builds from archive.mozilla.org captured 2026-10-28. Firefox 157+ differ (no c00a, no ffdhe groups).",
+      "headers": {
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "accept-language": "en-US,en;q=0.9",
+        "accept-encoding": "gzip, deflate, br, zstd",
+        "upgrade-insecure-requests": "1",
+        "sec-fetch-dest": "document",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-site": "none",
+        "sec-fetch-user": "?1",
+        "priority": "u=0, i",
+        "te": "trailers"
+      },
+      "header_overrides": [
+        "user-agent"
+      ]
+    },
+    {
+      "id": "firefox153-mac",
+      "family": "firefox",
+      "version": 153,
+      "version_label": "153",
+      "os": "macos",
+      "device": "desktop",
+      "wreq_profile": "Firefox152",
+      "wreq_platform": "MacOS",
+      "verified": true,
+      "weight": 0.008,
+      "note": "Firefox 153 ESR on the Firefox152 profile. TLS ClientHello (ciphers incl. c00a, extensions, groups incl. ffdhe2048/3072, signature algorithms, key shares) and HTTP/2 (1:65536;2:0;4:131072;5:16384|12517377|0|m,p,a,s, header order) equal to real Firefox 153.6.0esr (and 152.0), Linux builds from archive.mozilla.org captured 2026-10-28. Firefox 157+ differ (no c00a, no ffdhe groups).",
+      "headers": {
+        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:153.0) Gecko/20100101 Firefox/153.0",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "accept-language": "en-US,en;q=0.9",
+        "accept-encoding": "gzip, deflate, br, zstd",
+        "upgrade-insecure-requests": "1",
+        "sec-fetch-dest": "document",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-site": "none",
+        "sec-fetch-user": "?1",
+        "priority": "u=0, i",
+        "te": "trailers"
+      },
+      "header_overrides": [
+        "user-agent"
+      ]
+    },
+    {
+      "id": "firefox153-linux",
+      "family": "firefox",
+      "version": 153,
+      "version_label": "153",
+      "os": "linux",
+      "device": "desktop",
+      "wreq_profile": "Firefox152",
+      "wreq_platform": "Linux",
+      "verified": true,
+      "weight": 0.008,
+      "note": "Firefox 153 ESR on the Firefox152 profile. TLS ClientHello (ciphers incl. c00a, extensions, groups incl. ffdhe2048/3072, signature algorithms, key shares) and HTTP/2 (1:65536;2:0;4:131072;5:16384|12517377|0|m,p,a,s, header order) equal to real Firefox 153.6.0esr (and 152.0), Linux builds from archive.mozilla.org captured 2026-10-28. Firefox 157+ differ (no c00a, no ffdhe groups).",
+      "headers": {
+        "user-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0",
+        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "accept-language": "en-US,en;q=0.9",
+        "accept-encoding": "gzip, deflate, br, zstd",
+        "upgrade-insecure-requests": "1",
+        "sec-fetch-dest": "document",
+        "sec-fetch-mode": "navigate",
+        "sec-fetch-site": "none",
+        "sec-fetch-user": "?1",
+        "priority": "u=0, i",
+        "te": "trailers"
+      },
+      "header_overrides": [
+        "user-agent"
+      ]
+    },
     {
       "id": "firefox152-win",
       "family": "firefox",
@@ -481,9 +832,9 @@
       "device": "desktop",
       "wreq_profile": "Firefox152",
       "wreq_platform": "Windows",
-      "verified": false,
-      "weight": 0.04,
-      "note": "Akamai H2 and header order equal real Firefox 159; key-share groups and ciphers differ",
+      "verified": true,
+      "weight": 0.008,
+      "note": "TLS ClientHello (ciphers incl. c00a, extensions, groups incl. ffdhe2048/3072, signature algorithms, key shares) and HTTP/2 (1:65536;2:0;4:131072;5:16384|12517377|0|m,p,a,s, header order) equal to real Firefox 152.0, Linux builds from archive.mozilla.org captured 2026-10-28. Firefox 157+ differ (no c00a, no ffdhe groups).",
       "headers": {
         "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:152.0) Gecko/20100101 Firefox/152.0",
         "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
@@ -507,9 +858,9 @@
       "device": "desktop",
       "wreq_profile": "Firefox152",
       "wreq_platform": "MacOS",
-      "verified": false,
-      "weight": 0.01,
-      "note": "Akamai H2 and header order equal real Firefox 159; key-share groups and ciphers differ",
+      "verified": true,
+      "weight": 0.002,
+      "note": "TLS ClientHello (ciphers incl. c00a, extensions, groups incl. ffdhe2048/3072, signature algorithms, key shares) and HTTP/2 (1:65536;2:0;4:131072;5:16384|12517377|0|m,p,a,s, header order) equal to real Firefox 152.0, Linux builds from archive.mozilla.org captured 2026-10-28. Firefox 157+ differ (no c00a, no ffdhe groups).",
       "headers": {
         "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:152.0) Gecko/20100101 Firefox/152.0",
         "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
@@ -533,9 +884,9 @@
       "device": "desktop",
       "wreq_profile": "Firefox152",
       "wreq_platform": "Linux",
-      "verified": false,
-      "weight": 0.01,
-      "note": "Akamai H2 and header order equal real Firefox 159; key-share groups and ciphers differ",
+      "verified": true,
+      "weight": 0.002,
+      "note": "TLS ClientHello (ciphers incl. c00a, extensions, groups incl. ffdhe2048/3072, signature algorithms, key shares) and HTTP/2 (1:65536;2:0;4:131072;5:16384|12517377|0|m,p,a,s, header order) equal to real Firefox 152.0, Linux builds from archive.mozilla.org captured 2026-10-28. Firefox 157+ differ (no c00a, no ffdhe groups).",
       "headers": {
         "user-agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:152.0) Gecko/20100101 Firefox/152.0",
         "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
@@ -597,4 +948,4 @@
       }
     }
   ]
-}
+}

scrapely/proxy_configuration.py

@@ -631,11 +631,26 @@ class ProxyConfiguration:
             result = await result
         return cast(str, result)
 
+    def _can_pick_country(self) -> bool:
+        """True when each URL may get its own exit country (``new_url(country_code=...)``).
+
+        Only for the plain platform proxy with no country set, no custom or
+        tiered URLs and no datacenter group: the datacenter pool exits in the
+        US only, and a datacenter request pinned to another country is routed
+        and billed as residential.
+        """
+        if self._country_code or self._subdivision_code or self._proxy_urls or self._tiered_proxy_urls or self._new_url_function:
+            return False
+        if not self._needs_platform:
+            return False
+        return all(str(g).upper() == 'RESIDENTIAL' for g in self._groups)
+
     async def new_url(
         self,
         session_id: Optional[str] = None,
         request: Optional[Any] = None,  # Request object from crawlee (unused, kept for API compatibility)
         proxy_tier: Optional[int] = None,
+        country_code: Optional[str] = None,
     ) -> str:
         """Get a new proxy URL.
 
@@ -649,19 +664,33 @@ class ProxyConfiguration:
                 configured, the session ID is passed to it.
             request: Request object from crawlee (unused, for API compatibility).
             proxy_tier: Optional tier to select when tiered proxies are used.
+            country_code: Exit country for this URL only (two-letter ISO
+                code). Only for the plain platform proxy without a country of
+                its own (residential); ignored otherwise.
 
         Returns:
             Proxy URL string
         """
         if self._new_url_function:
             return await self._call_new_url_function(session_id)
-        return self._new_url_sync(session_id, proxy_tier)
+        return self._new_url_sync(session_id, proxy_tier, country_code=country_code)
 
-    def _new_url_sync(self, session_id: Optional[str] = None, proxy_tier: Optional[int] = None) -> str:
+    def _new_url_sync(self, session_id: Optional[str] = None, proxy_tier: Optional[int] = None, country_code: Optional[str] = None) -> str:
         """Resolve a proxy URL from the configured proxy_urls / platform gateway."""
         if session_id is not None:
             _validate_session_id(str(session_id))
 
+        if country_code and self._can_pick_country():
+            country_code = country_code.upper()
+            _validate_country_options(country_code, None)
+            key: Tuple[str, int] = (f'{session_id}@{country_code}', -1)
+            if session_id is not None and key in self._session_urls:
+                return self._session_urls[key]
+            url = self._build_platform_url(session_id=session_id, country_code=country_code)
+            if session_id is not None:
+                self._session_urls[key] = url
+            return url
+
         # Session stickiness is scoped per tier (parity with Crawlee's tiered
         # handling): the same session ID on a different tier resolves to that
         # tier's own URL instead of leaking the pinned URL across tiers. The

scrapely/storage/api_client.py

@@ -27,7 +27,9 @@ class ApiClientError(Exception):
 
 
 # PUT is left out: a KV record PUT is counted (and billed) as a write each time
-# it is applied, so a lost response must not be re-sent blindly.
+# it is applied, so a lost response must not be re-sent blindly. Record PUTs
+# carry an Idempotency-Key instead (see is_kv_record_url), which the API
+# de-duplicates, so they are re-sent after a lost answer (e.g. an edge 502).
 IDEMPOTENT_METHODS = frozenset({'GET', 'HEAD', 'DELETE', 'OPTIONS'})
 # HAProxy's own 503 page: no backend was available, so nothing was forwarded.
 NO_BACKEND_MARKER = 'No server is available to handle this request'
@@ -94,20 +96,40 @@ _keyed_writes_deduped: Dict[str, bool] = {}
 _keyed_writes_override: Optional[bool] = None
 
 
+def is_kv_record_url(url: str) -> bool:
+    """A key-value store record URL (``.../key-value-stores/<id>/records/<key>``)."""
+    try:
+        path = httpx.URL(url).path
+    except Exception:
+        path = url
+    return '/key-value-stores/' in path and '/records/' in path
+
+
 def _origin_of(url: str) -> str:
+    """What is learned per API: its origin, separately for record PUTs and other keyed writes."""
     try:
         u = httpx.URL(url)
-        return f'{u.scheme}://{u.netloc.decode()}'
+        origin = f'{u.scheme}://{u.netloc.decode()}'
     except Exception:
-        return url
+        origin = url
+    return origin + '#kv-record' if is_kv_record_url(url) else origin
 
 
 def api_dedupes_keyed_writes(url: Optional[str] = None) -> bool:
-    """Whether the API at ``url`` (default: the last one seen) de-duplicates keyed writes."""
+    """Whether the API at ``url`` (default: the last one seen) de-duplicates keyed writes.
+
+    Record PUTs count as de-duplicated until a keyed record PUT was stored
+    without the support header: re-applying the same body to the same record
+    leaves the same record, so the worst case on an API that ignores the key
+    is one extra counted write, while not re-sending fails the run.
+    """
     if _keyed_writes_override is not None:
         return _keyed_writes_override
     if url is not None:
-        return _keyed_writes_deduped.get(_origin_of(url)) is True
+        learned = _keyed_writes_deduped.get(_origin_of(url))
+        if learned is None and is_kv_record_url(url):
+            return True
+        return learned is True
     return bool(_keyed_writes_deduped) and list(_keyed_writes_deduped.values())[-1] is True
 
 
@@ -537,6 +559,7 @@ class ApiClient:
         *,
         raise_for_status: bool = True,
         idempotent: Optional[bool] = None,
+        idempotency_key: Optional[str] = None,
     ) -> httpx.Response:
         return await self.request(
             'PUT',
@@ -547,6 +570,7 @@ class ApiClient:
             headers=headers,
             raise_for_status=raise_for_status,
             idempotent=idempotent,
+            idempotency_key=idempotency_key,
         )
 
     async def delete(

scrapely/storage/client.py

@@ -334,23 +334,29 @@ class ApiStorageClient:
 
         path = kvs_record_path(store_id, key)
         headers = {'Content-Type': content_type}
+        # One key for every attempt of this write: a re-send after a lost answer
+        # (edge 502/504, dropped connection) is applied and counted once.
+        idempotency_key = uuid.uuid4().hex
         if isinstance(value, bytes):
             await self.api_client.put(
                 path,
                 content=value,
                 headers=headers,
+                idempotency_key=idempotency_key,
             )
         elif isinstance(value, str) and content_type != 'application/json':
             await self.api_client.put(
                 path,
                 content=value.encode('utf-8'),
                 headers=headers,
+                idempotency_key=idempotency_key,
             )
         else:
             await self.api_client.put(
                 path,
                 json_data=value,
                 headers=headers,
+                idempotency_key=idempotency_key,
             )
 
     async def kvs_delete_record(self, store_id: str, key: str) -> None:

{scrapely_python-0.1.38.dist-info → scrapely_python-0.1.40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.38
+Version: 0.1.40
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <hello@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.38.dist-info → scrapely_python-0.1.40.dist-info}/RECORD

@@ -1,16 +1,16 @@
-scrapely/__init__.py,sha256=FSwal-iDUoXIor_yZBy_p_Kq_QXGti9j9dnDZhOdNng,9668
+scrapely/__init__.py,sha256=cTs0rZD-AnrB6WLOzrhgGkh9XqmYj8Ihvqk80JFPjsM,9668
 scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
 scrapely/actor.py,sha256=z-kL24Y6m98v6lKLOxGYCnFmhVhaHJ6c0uOdYZW_4ak,93514
 scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
 scrapely/cdp_retry.py,sha256=hdTJEAIKhApM7IOk1DrQEl8NoflSJSmqd4RbNJ2IsgI,8540
-scrapely/charging.py,sha256=osi2Ys99IyIJwFj0eRtV_RshokPs4mA2CE7oftNbN58,95559
+scrapely/charging.py,sha256=xWYBFSr20UN5ySW0Vp1g0Dl6SppXN1y9qvMPQ1B0F1M,95615
 scrapely/charging_client.py,sha256=-4q-Ww7gu9EZXpM9QIn1Vd4vwAXmgL_WEKqaQIx3ViA,3961
 scrapely/configuration.py,sha256=ONnzw4mhtrY2jVvoVUQ48dMgmMWnx3Gv7fXrT4pY9z8,16919
 scrapely/errors.py,sha256=V8JBCGSyqrbwMD9GDv6ahWhZ92n2cb8AWxkxth6mcI8,4062
 scrapely/log.py,sha256=H3SoBvmNNRp8ChD9qv_J-hG0Nc5qDy3RldLQcTTgWa8,1635
 scrapely/patch_browser_pool.py,sha256=eZck-JSOvCuXdoGQaX6xUsKVGzolDlB4uKiBSBrFyr8,42190
 scrapely/pricing_types.py,sha256=yy2nXmCxL0BvUXRtWnznwhoInYeVdWObMRT5Fo0fc4I,5083
-scrapely/proxy_configuration.py,sha256=LCKXCV5huSba2rwT_QnOv36tQJYgqRoHsfvxDjjnMEw,37447
+scrapely/proxy_configuration.py,sha256=YAg2xm8eK4B14vCRWsSJxgtMIxdDJXbM66pZDx-HShk,39018
 scrapely/py.typed,sha256=XtWlywqUjc6x_M2gAXlDtYXbAsYkCjKZEjFyJvaHvm4,105
 scrapely/solver.py,sha256=RVoLNJTxniTI5SzDRpUdxe0ajzy4qqjuEmToTL1GRzA,35863
 scrapely/types.py,sha256=2yGedPfb5786UNqnlnRG5Cw4Pbyc4EPBiYeoU2kix3I,8812
@@ -19,20 +19,20 @@ scrapely/crypto/__init__.py,sha256=MbMPs7Eq_fGSVjOZxNts_JckDCUtxCVqdkHU0VCfN4M,4
 scrapely/crypto/_secrets.py,sha256=-ETpq1yCMaW-mheTGedFaQGGhu_pGT9_VMdJ3--V1rk,7614
 scrapely/events/__init__.py,sha256=XYk_qVzcw3jme9jErm9eMAEaMlj6MGpQ5UJfv7syzgc,1216
 scrapely/events/_types.py,sha256=XslMIEAQOe7othqevnxO86c7i7AysbvDLHH0lep64QU,5993
-scrapely/events/event_manager.py,sha256=9x3VqNy1gBi5tNQRv_1KptCRJQ9QnPwCDLT7JdlvjIM,12774
+scrapely/events/event_manager.py,sha256=svqrMb3KtSLvWBBoDrL6ozXaBE-IlHrdrh-rFPDnveM,12988
 scrapely/http/__init__.py,sha256=KGbINWi2WY6tN4CTM6c4-RIOQGL6iV3giNCYoGYJfWU,1949
 scrapely/http/_capture.py,sha256=wtS3bl_hNjICgxg3Iu_aL4uniQ_6xzXk_XsxGzpoBLE,3024
-scrapely/http/_client.py,sha256=o65Mb4OywW7gTtBnvE7g9aTojaXwhwhHkS3CmmUewuU,44585
+scrapely/http/_client.py,sha256=_JoW2u3NCpmBdfZbGyJAqrCTIkCBcxvnLPO1Cw0yeKc,48651
 scrapely/http/_compat.py,sha256=si8V3SAb-FDBcdUKuQHMCthRjNR6_oFlg8rpIKyEv2g,4082
 scrapely/http/_detect.py,sha256=NXMonGYv0jFwG7ONY6MPdhJadTRL2LbyPss-HVbE5J4,5433
-scrapely/http/_headers.py,sha256=CeYTaXYVDCoC6n56vqvVDfmAIS8fhHQWWRcmpXWr1cA,13137
-scrapely/http/_options.py,sha256=FAxjamCMJildHCQTxze7-3-QXPBE_BpBUyv_EPyzsmo,6418
-scrapely/http/_policy.py,sha256=jWC287sFtdpL8ZgyOMNUxbhf9lURRHcJuyCsbVsIVOE,8732
-scrapely/http/_profiles.py,sha256=b52ZXYvwSIVnGww6kqT6GCmq4HBFkGZJcHw7YM42EJ0,9542
+scrapely/http/_headers.py,sha256=jJMYYb6SpcMmIQ5ETTLVZi2xSA40wW_pt8UxzP2EMMQ,16633
+scrapely/http/_options.py,sha256=rEn_xYD93On-Fe0taWzDpf9c6_nTyKKUc-qMC-uGHGI,6461
+scrapely/http/_policy.py,sha256=Je_XwW6aeTOr9B6ghaAPdg9eq5Nic7YK6CHUDUibBX0,8819
+scrapely/http/_profiles.py,sha256=FUtXHHNQqbgRbMdodg_-IT6gMFsHcat49-L0JK5SvNM,9942
 scrapely/http/_response.py,sha256=kKSLJClZcLq5wWhXrtcdswGJxap1DObsidVfzUzi0Yk,5076
 scrapely/http/_stats.py,sha256=wi64avHSUnGnN-Irfw5bD3Fv2UvpCPi0fw4hJJR7fgQ,6190
 scrapely/http/_transport.py,sha256=EzdL7NK0epIHKeX71Z0GLVHwkepd16DDoBkxCnfFrC4,13238
-scrapely/http/profiles.json,sha256=8PP2SbbYI-bjPfS5rHX-TZQNo1dc6DQ8RCyQ8xl9UKQ,20954
+scrapely/http/profiles.json,sha256=D4gM25kQ-dUnAfSL_fG7AEyVGF4BCT4khF-J6_0R180,38009
 scrapely/request_loaders/__init__.py,sha256=aNgr4pRt3MziUbU6UQ_bjTT3moMNvvyUsPeCJyMQcwU,196
 scrapely/request_loaders/_scrapely_request_list.py,sha256=HGOWmQptWqBr1DGY2ZSZdQrSXtw9dmgSfxERF-3GPBE,7616
 scrapely/scrapy/__init__.py,sha256=3W2DwT6KlqiEKhlUCIn73dqb0V21EasVqfGmlN0PK34,1289
@@ -53,8 +53,8 @@ scrapely/scrapy/pipelines/__init__.py,sha256=gGUSWQtVC-7SvhvXbkQs0JlxpfezRMsBRVr
 scrapely/scrapy/pipelines/actor_dataset_push.py,sha256=6dL5Ulesqkjq3wOL2wnnGY0sC9gCoSUR9ouvBi5s5UA,924
 scrapely/storage/__init__.py,sha256=yN2gqhmPf3EVIekJpvJvKmEAYaE5zUA9oyUr8a2HQRo,3840
 scrapely/storage/_metadata.py,sha256=WWawBnVN0b1iK2K-SlxZXUc6czb5wXWf8iC_0tzkgAQ,4764
-scrapely/storage/api_client.py,sha256=61G2MIYlTnWzEuA4RRmBapBF0eGLNmdhrMzefciQuuo,22997
-scrapely/storage/client.py,sha256=VKieti-7RbEPGS9JlQS_vCUM8yD7GcFdjs8oSbKG7K0,23394
+scrapely/storage/api_client.py,sha256=w74FYCdquGgxAZB6abFziaHEDBXUSaDdnO1t4Z9JIiU,24107
+scrapely/storage/client.py,sha256=bMQLDmdXp8kRJ2c9JjVd1kOqoqxXxKqCvfCVPyr17Xk,23739
 scrapely/storage/patch_storage.py,sha256=6vOW5yjnZ8eiqb4XeF9dwa7rBQoa6XayyftVCrZcJ1w,8743
 scrapely/storage/scrapely_dataset_client.py,sha256=u5dnXxe1BAsq7rh-haR5W1ChE7UZ_k7B9ShR8YtwFkg,10695
 scrapely/storage/scrapely_kvs_client.py,sha256=ItQhe2rcSl20NkSmegfqHiRyKncH4Qg9BjQVpF6jHfk,7403
@@ -68,10 +68,10 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=1C22qY-RWlARzMyW
 scrapely/storage/_file_system/_storage_client.py,sha256=QHa3IULEb6AmgRFwpBqHD2OUTmtA_gl3lId-j6QdSFo,2027
 scrapely/storage_clients/__init__.py,sha256=q2pC-Rwe-BfHl018OyN_P0EDSx4-hNdxbk3pOCLLd4U,618
 scrapely/storages/__init__.py,sha256=kDNtd9VodjdHd4u7GzruJo0aB2HeiBN2_-HrkeqjmFg,139
-scrapely_python-0.1.38.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
-scrapely_python-0.1.38.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
-scrapely_python-0.1.38.dist-info/licenses/NOTICE,sha256=1ltrzKk2ez7x2BPNpXeyvOlTty5AfABYDBlwNilpORk,788
-scrapely_python-0.1.38.dist-info/METADATA,sha256=s2UUbg1otzuMt4_-Bfw8bKA70ofUgwaQts8Ry_4b6t8,4914
-scrapely_python-0.1.38.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
-scrapely_python-0.1.38.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
-scrapely_python-0.1.38.dist-info/RECORD,,
+scrapely_python-0.1.40.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
+scrapely_python-0.1.40.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
+scrapely_python-0.1.40.dist-info/licenses/NOTICE,sha256=1ltrzKk2ez7x2BPNpXeyvOlTty5AfABYDBlwNilpORk,788
+scrapely_python-0.1.40.dist-info/METADATA,sha256=Dqdbn37BPsue4GTs3scUAmHGJUjmHGRDOX9ByjWuah8,4914
+scrapely_python-0.1.40.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+scrapely_python-0.1.40.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
+scrapely_python-0.1.40.dist-info/RECORD,,