scrapely-python 0.1.14__py3-none-any.whl → 0.1.16__py3-none-any.whl

scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.14'
+__version__ = '0.1.16'
 __all__ = [
     # Core
     'Actor',

scrapely/actor.py

@@ -1054,7 +1054,19 @@ class Actor(metaclass=_ActorMeta):
         else:
             logger.info(logged)
 
-        result = await cls._put_run_status_message(message, is_terminal=is_terminal)
+        if is_terminal:
+            result = await cls._put_run_status_message(message, is_terminal=True)
+        else:
+            # A progress update is best effort: if the API still fails after the
+            # retry window (e.g. a long 503), log it and carry on instead of
+            # crashing the run. Terminal updates keep raising.
+            try:
+                result = await cls._put_run_status_message(message, is_terminal=False)
+            except asyncio.CancelledError:
+                raise
+            except Exception as e:  # noqa: BLE001
+                logger.warning('Failed to update the status message (continuing): %s', e)
+                return ActorRun(id='', actor_id='', status='')
         if result is None:
             return ActorRun(id='', actor_id='', status='')
         run_id, data = result
@@ -1068,6 +1080,34 @@ class Actor(metaclass=_ActorMeta):
             status_message=data.get('statusMessage'),
         )
 
+    @classmethod
+    async def solver_status(cls, vendor: str) -> Dict[str, Any]:
+        """Anti-bot solver status for a vendor (free, cached for 15 s, never raises).
+
+        ``usable`` False means the vendor cannot be solved right now: take your
+        fallback (for example the hosted browser), or fail the items that need
+        a solve at once and call ``set_solver_degraded_status()``. See
+        ``scrapely.solver``.
+        """
+        from .solver import solver_status_async
+
+        return await solver_status_async(vendor)
+
+    @classmethod
+    async def set_solver_degraded_status(cls, skipped: Optional[int] = None, total: Optional[int] = None) -> None:
+        """Set the standard status message for a run whose site protection cannot be solved right now.
+
+        "Site protection temporarily unavailable, results may be incomplete",
+        with "(N of M items skipped)" when both counts are given. Never raises:
+        the run goes on.
+        """
+        from .solver import degraded_status_message
+
+        try:
+            await cls.set_status_message(degraded_status_message(skipped, total))
+        except Exception as exc:  # noqa: BLE001 - a status message must never stop the run
+            logger.warning(f'Could not set the solver status message: {exc}')
+
     @classmethod
     async def _put_run_status_message(
         cls, message: str, *, is_terminal: bool

scrapely/charging.py

@@ -52,7 +52,7 @@ _PENDING_MAX_ENTRIES = 1024
 # Same-key POST attempts before leaving an intent pending / re-queueing a void.
 # The API client itself retries 429 and 5xx answers and refused or reset
 # connections of these POSTs with exponential backoff (honouring Retry-After)
-# for its retry window (2 minutes, SCRAPELY_API_RETRY_WINDOW_SECS): re-sending
+# for its retry window (5 minutes, SCRAPELY_API_RETRY_WINDOW_SECS): re-sending
 # is safe because every attempt carries the same Idempotency-Key. These outer
 # attempts cover what the client gives up on quickly (timeouts, other
 # transport errors), within _RETRY_BUDGET_SECS. Then the charge stays pending
@@ -62,7 +62,7 @@ _POST_ATTEMPTS = 8
 
 # Outer retries stop once this much time has passed since the first attempt
 # (the API retry window; JS SDK parity).
-_RETRY_BUDGET_SECS = 120.0
+_RETRY_BUDGET_SECS = 300.0
 
 
 def _is_transient_status(status: Optional[int]) -> bool:
@@ -1567,10 +1567,20 @@ def _push_certainly_failed(err: BaseException) -> bool:
     """True when a dataset push certainly stored nothing.
 
     The storage API answered with an error status, or the connection was never
-    made. Timeouts and resets after sending are not proof: the write may have landed.
+    made. Timeouts and resets after sending are not proof: the write may have landed,
+    and neither is any error of a push one of whose earlier attempts may have been
+    applied (``outcome_unknown``, set by the storage client's retries).
     """
     import httpx
 
+    c: Optional[BaseException] = err
+    for _ in range(4):
+        if c is None:
+            break
+        if getattr(c, 'outcome_unknown', False) is True:
+            return False
+        c = c.__cause__ or c.__context__
+
     e: Optional[BaseException] = err
     for _ in range(4):
         if e is None:

scrapely/solver.py

@@ -0,0 +1,875 @@
+"""Anti-bot solver helpers for Scrapely Actors.
+
+The platform runs the challenge solver (xhr.dev today) behind
+``/v2/solver``. Actors never hold a solver key: the platform injects
+``SCRAPELY_SOLVER_URL`` and ``SCRAPELY_SOLVER_API_KEY`` (the run's own token)
+into runs of Actors that may use it.
+
+Three layers, use what you need:
+
+* ``solver_status(vendor)`` / ``await Actor.solver_status(vendor)``: a free,
+  cached status check. When a vendor is ``down`` take your fallback (for
+  example the hosted browser) or fail the items that need a solve at once.
+* ``Solver``: solves one challenge on your own HTTP session. The server tells
+  the SDK which requests to send (challenge document, scripts, submission);
+  they go out through *your* session so the clearance is bound to your exit IP.
+  Cookies land in the session's jar; the clearance is verified and reported.
+* ``SolvePolicy``: retries on fresh proxy sessions with backoff and a breaker
+  (the policy the solver Actors already use), plus ``VendorUnavailable``,
+  which is raised at once, without retries, when the platform says the
+  vendor is down.
+
+Typical use::
+
+    from scrapely.solver import Solver, SolvePolicy, SolveFailed, VendorUnavailable
+
+    solver = Solver(vendor="datadome", accept_language="es-ES,es;q=0.9", timezone="Europe/Madrid")
+    policy = SolvePolicy()
+
+    def attempt(i):
+        session = new_session(i)                      # fresh exit per try
+        r = session.get(url)
+        if solver.is_blocked(r):
+            solver.solve(session, url, r, session_id=f"s{i}")
+            r = session.get(url)
+        return r
+
+    try:
+        page = policy.run(attempt, label=url)
+    except VendorUnavailable:
+        ...  # vendor down: fail this item at once (or use your fallback)
+    except SolveFailed:
+        ...  # every try failed: record a per-item error, keep going
+
+When a vendor is down and the Actor has no fallback, the run must still
+start and finish: call ``await Actor.set_solver_degraded_status()`` once, fail
+the items that need a solve, keep everything else.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import json
+import os
+import random
+import re
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Any, Awaitable, Callable, Optional
+
+import httpx
+
+__all__ = [
+    "Clearance",
+    "DEGRADED_STATUS_MESSAGE",
+    "IpBanned",
+    "SolveAttemptError",
+    "SolveFailed",
+    "SolvePolicy",
+    "Solver",
+    "SolverAuthError",
+    "SolverRateLimited",
+    "SolverRequestInvalid",
+    "SolverUnavailable",
+    "VendorUnavailable",
+    "check_solver_response",
+    "degraded_status_message",
+    "detect_vendor",
+    "is_blocked",
+    "solver_env",
+    "solver_status",
+    "solver_status_async",
+]
+
+DEGRADED_STATUS_MESSAGE = "Site protection temporarily unavailable, results may be incomplete"
+
+# Codes the platform answers with when a vendor cannot be solved right now.
+UNAVAILABLE_CODES = ("vendor_degraded", "vendor_down", "provider_unavailable", "vendor_unsupported", "solver_not_enabled")
+
+_COMPAT_SUFFIX = "/compat/xhrdev"
+_STATUS_TTL_SECS = 15.0
+_MAX_BODY = 4 * 1024 * 1024
+
+
+# ─── environment ─────────────────────────────────────────────────────────────
+
+
+def solver_env() -> tuple[str, str] | None:
+    """Solver URL and key (the run token), injected by the platform. None when not set.
+
+    Never hardcode the host or the key, and never log the key.
+    """
+    url = (os.environ.get("SCRAPELY_SOLVER_URL") or "").strip().rstrip("/")
+    key = (os.environ.get("SCRAPELY_SOLVER_API_KEY") or "").strip()
+    if not url or not key:
+        return None
+    return url, key
+
+
+def _gateway() -> tuple[str, str] | None:
+    """Base of the platform solver API (…/v2/solver) and the run token, or None.
+
+    Only available when the platform serves the solver (the URL ends with the
+    compat route); a raw solver URL means the old direct mode.
+    """
+    env = solver_env()
+    if env is None:
+        return None
+    url, key = env
+    if not url.endswith(_COMPAT_SUFFIX):
+        return None
+    return url[: -len(_COMPAT_SUFFIX)], key
+
+
+# ─── errors ──────────────────────────────────────────────────────────────────
+
+
+class SolveAttemptError(RuntimeError):
+    """One try failed. Worth another try on a fresh exit."""
+
+
+class IpBanned(SolveAttemptError):
+    """The address is refused outright. Only a new exit helps."""
+
+
+class SolverRateLimited(SolveAttemptError):
+    """The solver answered 429. Not retried for this item."""
+
+    def __init__(self, retry_after: float | None = None) -> None:
+        self.retry_after = retry_after
+        suffix = f", retry after {retry_after:g}s" if retry_after is not None else ""
+        super().__init__(f"solver rate limit hit (HTTP 429{suffix})")
+
+
+class SolverRequestInvalid(SolveAttemptError):
+    """The solver rejected the request itself (HTTP 400). Not retried."""
+
+
+class SolverAuthError(SolveAttemptError):
+    """The solver refused the key (HTTP 401/403). Opens the breaker."""
+
+
+class SolveFailed(RuntimeError):
+    """Every try for one item failed."""
+
+    def __init__(self, message: str, attempts: int, last_error: BaseException | None = None) -> None:
+        super().__init__(message)
+        self.attempts = attempts
+        self.last_error = last_error
+
+
+class SolverUnavailable(SolveFailed):
+    """The breaker is open: the solver failed for everything recently."""
+
+
+class VendorUnavailable(SolverUnavailable):
+    """The platform says this vendor cannot be solved right now (fail fast, no retries).
+
+    ``code`` is one of ``vendor_down``, ``vendor_degraded``,
+    ``provider_unavailable``, ``vendor_unsupported`` or ``solver_not_enabled``.
+    """
+
+    def __init__(self, code: str, vendor: str | None = None, retry_after: float | None = None, message: str | None = None) -> None:
+        self.code = code
+        self.vendor = vendor
+        self.retry_after = retry_after
+        super().__init__(message or f"anti-bot solver unavailable ({code}{f', {vendor}' if vendor else ''})", 0)
+
+
+def _json(body: str) -> dict:
+    try:
+        parsed = json.loads(body or "")
+        return parsed if isinstance(parsed, dict) else {}
+    except (TypeError, ValueError):
+        return {}
+
+
+def check_solver_response(status: int, body: str = "", retry_after: str | None = None) -> None:
+    """Raise the right error for a non-2xx solver answer; return on 2xx."""
+    if 200 <= status < 300:
+        return
+    detail = (body or "")[:200].replace("\n", " ")
+    seconds: float | None = None
+    try:
+        seconds = float(retry_after) if retry_after is not None else None
+    except (TypeError, ValueError):
+        seconds = None
+    if status == 503:
+        parsed = _json(body)
+        code = str(parsed.get("code") or "")
+        if code in UNAVAILABLE_CODES:
+            raise VendorUnavailable(code, parsed.get("vendor"), seconds if seconds is not None else parsed.get("retryAfterSecs"))
+    if status == 429:
+        raise SolverRateLimited(seconds)
+    if status in (401, 403):
+        parsed = _json(body)
+        if parsed.get("code") == "solver_not_enabled":
+            raise VendorUnavailable("solver_not_enabled", None, None, "the anti-bot solver is not enabled for this Actor")
+        raise SolverAuthError(f"solver refused the key (HTTP {status})")
+    if status == 400:
+        raise SolverRequestInvalid(f"solver rejected the request (HTTP 400): {detail}")
+    raise SolveAttemptError(f"solver HTTP {status}: {detail}")
+
+
+_NOT_RETRIED = (SolverRateLimited, SolverRequestInvalid, SolverAuthError)
+
+
+# ─── retry policy ────────────────────────────────────────────────────────────
+
+
+class SolvePolicy:
+    """Up to `attempts` tries per item, each on a fresh proxy session, with backoff and a breaker.
+
+    * HTTP 5xx, timeouts, transport errors, empty solutions and a page still
+      blocked after the clearance are failed tries;
+    * HTTP 429 ends the tries for that item;
+    * HTTP 401/403 opens the breaker at once;
+    * after `trip_after` consecutive failed tries the breaker opens and
+      further solves fail fast with SolverUnavailable; after `cooldown`
+      seconds one probe is let through;
+    * VendorUnavailable (the platform says the vendor is down) is raised at
+      once, without retries and without touching the breaker.
+    """
+
+    def __init__(
+        self,
+        attempts: int = 4,
+        base_delay: float = 1.5,
+        max_delay: float = 15.0,
+        trip_after: int = 12,
+        cooldown: float = 180.0,
+        sleep: Callable[[float], None] = time.sleep,
+        async_sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+        clock: Callable[[], float] = time.monotonic,
+        rng: Callable[[], float] = random.random,
+    ) -> None:
+        self.attempts = max(1, min(5, attempts))
+        self.base_delay = base_delay
+        self.max_delay = max_delay
+        self.trip_after = max(1, trip_after)
+        self.cooldown = cooldown
+        self._sleep = sleep
+        self._async_sleep = async_sleep
+        self._clock = clock
+        self._rng = rng
+        self._lock = threading.Lock()
+        self.consecutive_failures = 0
+        self.opened_at: float | None = None
+        self.calls = 0
+        self.solved = 0
+        self.failed_items = 0
+        self.unavailable_items = 0
+        self.last_error = ""
+
+    @property
+    def down(self) -> bool:
+        with self._lock:
+            return self.opened_at is not None and self._clock() - self.opened_at < self.cooldown
+
+    @property
+    def ever_tripped(self) -> bool:
+        return self.opened_at is not None
+
+    def _admit(self) -> None:
+        with self._lock:
+            if self.opened_at is None:
+                return
+            if self._clock() - self.opened_at < self.cooldown:
+                raise SolverUnavailable(
+                    f"challenge solver unavailable after {self.trip_after} consecutive failed solves"
+                    + (f" (last error: {self.last_error})" if self.last_error else ""),
+                    0,
+                )
+            self.opened_at = self._clock()
+            self.consecutive_failures = self.trip_after - 1
+
+    def _success(self) -> None:
+        with self._lock:
+            self.calls += 1
+            self.solved += 1
+            self.consecutive_failures = 0
+            self.opened_at = None
+
+    def _failure(self, exc: BaseException) -> None:
+        with self._lock:
+            self.calls += 1
+            self.consecutive_failures += 1
+            self.last_error = _short(exc)
+            if isinstance(exc, SolverAuthError) or self.consecutive_failures >= self.trip_after:
+                self.opened_at = self._clock()
+
+    def _unavailable(self) -> None:
+        with self._lock:
+            self.unavailable_items += 1
+
+    def delay(self, index: int) -> float:
+        raw = min(self.max_delay, self.base_delay * (2 ** index))
+        return raw * (0.5 + self._rng())
+
+    def run(self, attempt: Callable[[int], Any], label: str = "page") -> Any:
+        """Call attempt(i) until it returns. Sync version (for worker threads)."""
+        last: BaseException | None = None
+        for index in range(self.attempts):
+            self._admit()
+            try:
+                result = attempt(index)
+            except VendorUnavailable:
+                self._unavailable()
+                raise
+            except Exception as exc:  # noqa: BLE001 - every failure is one failed try
+                last = exc
+                self._failure(exc)
+                if isinstance(exc, _NOT_RETRIED):
+                    break
+                if index + 1 < self.attempts and not self.down:
+                    self._sleep(self.delay(index))
+                continue
+            self._success()
+            return result
+        return self._give_up(label, last)
+
+    async def run_async(self, attempt: Callable[[int], Any], label: str = "page") -> Any:
+        """Async version. attempt(i) may be a coroutine function or a plain one."""
+        last: BaseException | None = None
+        for index in range(self.attempts):
+            self._admit()
+            try:
+                result = attempt(index)
+                if inspect.isawaitable(result):
+                    result = await result
+            except asyncio.CancelledError:
+                raise
+            except VendorUnavailable:
+                self._unavailable()
+                raise
+            except Exception as exc:  # noqa: BLE001
+                last = exc
+                self._failure(exc)
+                if isinstance(exc, _NOT_RETRIED):
+                    break
+                if index + 1 < self.attempts and not self.down:
+                    await self._async_sleep(self.delay(index))
+                continue
+            self._success()
+            return result
+        return self._give_up(label, last)
+
+    def _give_up(self, label: str, last: BaseException | None) -> Any:
+        with self._lock:
+            self.failed_items += 1
+        if isinstance(last, SolverUnavailable):
+            raise last
+        raise SolveFailed(f"anti-bot challenge on {label} was not solved: {_short(last)}", self.attempts, last)
+
+    def summary(self) -> dict[str, int]:
+        return {
+            "solverCalls": self.calls,
+            "solved": self.solved,
+            "failedItems": self.failed_items,
+            "unavailableItems": self.unavailable_items,
+        }
+
+
+def _short(exc: BaseException | None) -> str:
+    if exc is None:
+        return "unknown error"
+    text = str(exc) or type(exc).__name__
+    return text[:200]
+
+
+# ─── detection ───────────────────────────────────────────────────────────────
+
+_DD_RE = re.compile(r"var\s+dd\s*=\s*\{")
+_SBSD_RE = re.compile(r'<script[^>]+src="([^"]+\?v=[0-9a-f-]{36}[^"]*)"', re.I)
+_PX_RE = re.compile(r"_pxappid|px-captcha|perimeterx|/_px\d?/", re.I)
+_KASADA_RE = re.compile(r"kpsdk", re.I)
+
+
+def detect_vendor(status: int, body: str) -> str | None:
+    """The protection behind a *challenge* response: datadome, akamai-sbsd, perimeterx, kasada or None.
+
+    Ordinary pages that merely load a vendor's script are not challenges:
+    DataDome, PerimeterX and Kasada need an error status (4xx), and an SBSD
+    bundle counts only on a short interstitial page.
+    """
+    body = body or ""
+    sample = body[:20000]
+    head = sample[:4000].lower()
+    challenged = status >= 400
+    if challenged and ("captcha-delivery.com" in head or _DD_RE.search(sample) or "datadome" in head):
+        return "datadome"
+    if len(body) <= 20000 and _SBSD_RE.search(sample):
+        return "akamai-sbsd"
+    if challenged and _PX_RE.search(sample):
+        return "perimeterx"
+    if challenged and _KASADA_RE.search(sample):
+        return "kasada"
+    return None
+
+
+def _status_body(response: Any) -> tuple[int, str]:
+    if isinstance(response, tuple):
+        return int(response[0]), str(response[1] or "")
+    if isinstance(response, dict):
+        return int(response.get("status") or 0), str(response.get("body") or "")
+    status = getattr(response, "status_code", None)
+    if status is None:
+        status = getattr(response, "status", 0)
+    text = getattr(response, "text", "")
+    if callable(text):  # aiohttp-style
+        text = ""
+    return int(status or 0), str(text or "")
+
+
+def is_blocked(response: Any, vendor: str | None = None) -> bool:
+    """True when the response is an anti-bot challenge (optionally of one vendor)."""
+    status, body = _status_body(response)
+    found = detect_vendor(status, body)
+    return found is not None if vendor in (None, "auto") else found == vendor
+
+
+def degraded_status_message(skipped: int | None = None, total: int | None = None) -> str:
+    """The standard status message for runs whose site protection cannot be solved right now."""
+    if skipped is not None and total is not None:
+        return f"{DEGRADED_STATUS_MESSAGE} ({skipped} of {total} items skipped)"
+    return DEGRADED_STATUS_MESSAGE
+
+
+# ─── status ──────────────────────────────────────────────────────────────────
+
+_status_cache: dict[str, tuple[float, dict]] = {}
+_status_lock = threading.Lock()
+
+
+def _unknown_status(code: str | None = None) -> dict:
+    # Without the platform gateway nothing is known: let the Actor proceed as before.
+    return {"status": "unknown", "usable": code is None, "code": code, "successRate30m": None, "since": None, "recheckAfterSecs": 300}
+
+
+def _status_from_answer(vendor: str, status: int, body: str) -> dict:
+    if status == 200:
+        data = _json(body)
+        v = (data.get("vendors") or {}).get(vendor)
+        if isinstance(v, dict):
+            return v
+    return _unknown_status()
+
+
+def _cached(vendor: str) -> dict | None:
+    with _status_lock:
+        hit = _status_cache.get(vendor)
+    if hit and time.monotonic() - hit[0] < _STATUS_TTL_SECS:
+        return hit[1]
+    return None
+
+
+def _store(vendor: str, value: dict) -> dict:
+    with _status_lock:
+        _status_cache[vendor] = (time.monotonic(), value)
+    return value
+
+
+def solver_status(vendor: str, timeout: float = 10.0) -> dict:
+    """Vendor status from the platform (free, cached for 15 s). Never raises.
+
+    Returns a dict with ``status`` (healthy, degraded, down, unsupported or
+    unknown), ``usable`` (False means: take your fallback or fail the items
+    that need a solve), ``code``, ``successRate30m``, ``since`` and
+    ``recheckAfterSecs``.
+    """
+    vendor = (vendor or "").strip().lower()
+    hit = _cached(vendor)
+    if hit is not None:
+        return hit
+    gw = _gateway()
+    if gw is None:
+        return _unknown_status(None if solver_env() else "solver_not_enabled")
+    base, key = gw
+    try:
+        r = httpx.get(f"{base}/status", params={"vendors": vendor}, headers={"authorization": f"Bearer {key}"}, timeout=timeout)
+        return _store(vendor, _status_from_answer(vendor, r.status_code, r.text))
+    except Exception:  # noqa: BLE001 - a failed check must never stop the run
+        return _unknown_status()
+
+
+async def solver_status_async(vendor: str, timeout: float = 10.0) -> dict:
+    """Async ``solver_status``."""
+    vendor = (vendor or "").strip().lower()
+    hit = _cached(vendor)
+    if hit is not None:
+        return hit
+    gw = _gateway()
+    if gw is None:
+        return _unknown_status(None if solver_env() else "solver_not_enabled")
+    base, key = gw
+    try:
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            r = await client.get(f"{base}/status", params={"vendors": vendor}, headers={"authorization": f"Bearer {key}"})
+        return _store(vendor, _status_from_answer(vendor, r.status_code, r.text))
+    except Exception:  # noqa: BLE001
+        return _unknown_status()
+
+
+# ─── solving ─────────────────────────────────────────────────────────────────
+
+
+@dataclass
+class Clearance:
+    """A solved challenge: cookies already set on the session, and the verification response."""
+
+    vendor: str
+    solve_id: str | None
+    cookies: list[dict] = field(default_factory=list)
+    ttl_hint_secs: int = 0
+    cached: bool = False
+    verified: bool | None = None
+    response: Any = None
+
+
+def _is_httpx(session: Any) -> bool:
+    return type(session).__module__.split(".")[0] == "httpx"
+
+
+def _cookie_applies(cookie: Any, host: str, path: str, secure: bool, now: float) -> bool:
+    """RFC 6265 matching: the cookies the session itself would send to this URL."""
+    domain = (getattr(cookie, "domain", "") or "").lower()
+    if not domain:
+        return False
+    bare = domain.lstrip(".")
+    host_only = not domain.startswith(".") and not getattr(cookie, "domain_specified", False)
+    if host_only:
+        if host != bare:
+            return False
+    elif host != bare and not host.endswith("." + bare):
+        return False
+    cpath = getattr(cookie, "path", "/") or "/"
+    if not (path == cpath or path.startswith(cpath.rstrip("/") + "/") or cpath == "/"):
+        return False
+    if getattr(cookie, "secure", False) and not secure:
+        return False
+    expires = getattr(cookie, "expires", None)
+    return not (expires is not None and expires <= now)
+
+
+def _cookie_header(session: Any, url: str) -> str | None:
+    """The Cookie header the session would send to `url` (host, path, secure and expiry rules)."""
+    u = httpx.URL(url)
+    host = (u.host or "").lower()
+    path = u.path or "/"
+    secure = u.scheme == "https"
+    jar = getattr(getattr(session, "cookies", None), "jar", None)
+    pairs: list[str] = []
+    now = time.time()
+    try:
+        for c in jar or []:
+            if _cookie_applies(c, host, path, secure, now):
+                pairs.append(f"{c.name}={c.value}")
+    except Exception:  # noqa: BLE001
+        return None
+    return "; ".join(pairs) or None
+
+
+def _set_cookies(session: Any, cookies: list[dict]) -> None:
+    jar = getattr(session, "cookies", None)
+    if jar is None:
+        return
+    for c in cookies:
+        try:
+            jar.set(c["name"], c["value"], domain=c.get("domain") or "", path=c.get("path") or "/")
+        except Exception:  # noqa: BLE001 - a jar without domain support
+            jar.set(c["name"], c["value"])
+
+
+def _step_allowed(step_url: str, target_url: str) -> bool:
+    """A step may only go to the target page's own host or to captcha-delivery.com, over https."""
+    try:
+        step = httpx.URL(step_url)
+        target = httpx.URL(target_url)
+    except Exception:  # noqa: BLE001
+        return False
+    if step.scheme != "https":
+        return False
+    host = (step.host or "").lower()
+    if host == (target.host or "").lower():
+        return True
+    return host == "captcha-delivery.com" or host.endswith(".captcha-delivery.com")
+
+
+def _checked(req: dict, target_url: str) -> dict:
+    if not isinstance(req, dict) or not _step_allowed(str(req.get("url") or ""), target_url):
+        raise SolveAttemptError("the solver asked for a request outside the target site; refused")
+    if req.get("method") not in ("GET", "POST"):
+        raise SolveAttemptError("the solver asked for an unsupported request method; refused")
+    return req
+
+
+def _send_sync(session: Any, req: dict, timeout: float) -> tuple[int, str]:
+    body = req.get("body")
+    kwargs: dict[str, Any] = {"headers": req.get("headers") or {}, "timeout": timeout}
+    if body is not None:
+        kwargs["content" if _is_httpx(session) else "data"] = body.encode() if isinstance(body, str) else body
+    r = session.request(req["method"], req["url"], **kwargs)
+    return _status_body(r)
+
+
+async def _send_async(session: Any, req: dict, timeout: float) -> tuple[int, str]:
+    body = req.get("body")
+    kwargs: dict[str, Any] = {"headers": req.get("headers") or {}, "timeout": timeout}
+    if body is not None:
+        kwargs["content" if _is_httpx(session) else "data"] = body.encode() if isinstance(body, str) else body
+    r = await session.request(req["method"], req["url"], **kwargs)
+    return _status_body(r)
+
+
+class Solver:
+    """Solves anti-bot challenges on the Actor's own HTTP session through the platform.
+
+    Args:
+        vendor: ``"datadome"``, ``"akamai-sbsd"`` or ``"auto"`` (detected from the blocked response).
+        identity: the browser profile your TLS impersonation claims (``"chrome-146-macos"``
+            matches curl_cffi ``impersonate="chrome146"``).
+        accept_language / timezone / languages: locale of that browser (match the site and proxy country).
+        fetch: optional ``fetch(session, request) -> (status, body)`` to send the step requests
+            yourself (sync), or ``afetch`` for async.
+        request_timeout: seconds per step request on your session.
+    """
+
+    def __init__(
+        self,
+        vendor: str = "auto",
+        identity: str = "chrome-146-macos",
+        accept_language: str | None = None,
+        timezone: str | None = None,
+        languages: str | None = None,
+        timeout_ms: int = 60_000,
+        request_timeout: float = 30.0,
+        fetch: Callable[[Any, dict], tuple[int, str]] | None = None,
+        afetch: Callable[[Any, dict], Awaitable[tuple[int, str]]] | None = None,
+    ) -> None:
+        self.vendor = vendor
+        self.identity = {"profile": identity}
+        if accept_language:
+            self.identity["acceptLanguage"] = accept_language
+        if timezone:
+            self.identity["timezone"] = timezone
+        if languages:
+            self.identity["languages"] = languages
+        self.timeout_ms = timeout_ms
+        self.request_timeout = request_timeout
+        self._fetch = fetch
+        self._afetch = afetch
+        self.solves = 0
+        self.cleared = 0
+
+    @staticmethod
+    def configured() -> bool:
+        """True when the platform solver API is available to this run."""
+        return _gateway() is not None
+
+    def is_blocked(self, response: Any) -> bool:
+        return is_blocked(response, None if self.vendor == "auto" else self.vendor)
+
+    # -- wire ------------------------------------------------------------------
+
+    def _start_payload(self, url: str, blocked: Any, session: Any, session_id: str | None) -> dict:
+        status, body = _status_body(blocked)
+        payload: dict[str, Any] = {
+            "vendor": self.vendor,
+            "url": url,
+            "blocked": {"status": status, "body": body[:_MAX_BODY]},
+            "identity": self.identity,
+            "timeoutMs": self.timeout_ms,
+            "session": {"cookieHeader": _cookie_header(session, url)},
+        }
+        if session_id:
+            payload["session"]["id"] = str(session_id)
+        return payload
+
+    @staticmethod
+    def _handle(status: int, text: str, headers: Any, vendor: str) -> dict:
+        data = _json(text)
+        if status == 503 and str(data.get("code") or "") in UNAVAILABLE_CODES:
+            raise VendorUnavailable(str(data["code"]), data.get("vendor") or vendor, data.get("retryAfterSecs"))
+        if status == 429:
+            ra = headers.get("retry-after") if headers is not None else None
+            raise SolverRateLimited(float(ra) if ra else data.get("retryAfterSecs"))
+        if status in (401, 403):
+            if data.get("code") == "solver_not_enabled":
+                raise VendorUnavailable("solver_not_enabled", vendor, None, "the anti-bot solver is not enabled for this Actor")
+            raise SolverAuthError(f"solver refused the run token (HTTP {status}, {data.get('code') or ''})")
+        if status == 404 and data.get("code") == "not_found":
+            raise SolveAttemptError("the solve expired; start again")
+        if status >= 400:
+            raise SolveAttemptError(f"solver HTTP {status}: {text[:160]}")
+        return data
+
+    @staticmethod
+    def _raise_failed(data: dict) -> None:
+        reason = str(data.get("reason") or "rejected")
+        message = str(data.get("message") or reason)
+        if reason == "ip_banned":
+            raise IpBanned(message)
+        if reason in ("no_challenge", "bad_request") and not data.get("retryable", True):
+            raise SolverRequestInvalid(message)
+        raise SolveAttemptError(message)
+
+    def _finish(self, session: Any, data: dict) -> Clearance:
+        cookies = data.get("cookies") or []
+        _set_cookies(session, cookies)
+        self.cleared += 1
+        return Clearance(
+            vendor=self.vendor,
+            solve_id=data.get("solveId"),
+            cookies=cookies,
+            ttl_hint_secs=int(data.get("ttlHintSecs") or 0),
+            cached=bool(data.get("cached")),
+        )
+
+    # -- sync ------------------------------------------------------------------
+
+    def solve(self, session: Any, url: str, blocked: Any, *, session_id: str | None = None, verify: Any = True) -> Clearance:
+        """Solve the challenge in `blocked` (a response of `session` for `url`).
+
+        verify: True re-fetches `url` and checks it is no longer blocked (the
+        result is reported to the platform); a callable ``verify(session) ->
+        bool`` does your own check; False skips it (call ``report()`` later).
+        Raises SolveAttemptError / IpBanned (try again on a fresh exit),
+        VendorUnavailable (fail fast), SolverRateLimited, SolverAuthError.
+        """
+        gw = _gateway()
+        if gw is None:
+            raise VendorUnavailable("solver_not_enabled", self.vendor, None, "the platform solver is not available to this run")
+        base, key = gw
+        headers = {"authorization": f"Bearer {key}"}
+        self.solves += 1
+        with httpx.Client(timeout=self.timeout_ms / 1000 + 30) as api:
+            r = api.post(f"{base}/solve", json=self._start_payload(url, blocked, session, session_id), headers=headers)
+            data = self._handle(r.status_code, r.text, r.headers, self.vendor)
+            for _ in range(8):
+                action = data.get("action")
+                if action == "done":
+                    clearance = self._finish(session, data)
+                    break
+                if action == "failed":
+                    self._raise_failed(data)
+                if action not in ("fetch", "submit"):
+                    raise SolveAttemptError(f"unexpected solver answer: {action}")
+                results = []
+                for req in data.get("requests") or []:
+                    req = _checked(req, url)
+                    try:
+                        status, body = (self._fetch or (lambda s, q: _send_sync(s, q, self.request_timeout)))(session, req)
+                    except Exception as exc:  # noqa: BLE001
+                        raise SolveAttemptError(f"challenge request failed: {type(exc).__name__}") from exc
+                    results.append({"id": req["id"], "status": status, "body": body[:_MAX_BODY]})
+                r = api.post(
+                    f"{base}/solve",
+                    json={"solveId": data.get("solveId"), "results": results, "session": {"id": session_id, "cookieHeader": _cookie_header(session, url)}},
+                    headers=headers,
+                )
+                data = self._handle(r.status_code, r.text, r.headers, self.vendor)
+            else:
+                raise SolveAttemptError("the solve did not finish")
+            if verify is False:
+                return clearance
+            ok, response = self._verify_sync(session, url, verify)
+            clearance.verified = ok
+            clearance.response = response
+            self._report_sync(api, base, headers, clearance.solve_id, ok)
+        if not ok:
+            raise SolveAttemptError("still blocked after the clearance was applied")
+        return clearance
+
+    def _verify_sync(self, session: Any, url: str, verify: Any) -> tuple[bool, Any]:
+        if callable(verify):
+            return bool(verify(session)), None
+        try:
+            r = session.get(url, timeout=self.request_timeout)
+        except Exception:  # noqa: BLE001
+            return False, None
+        return not is_blocked(r), r
+
+    @staticmethod
+    def _report_sync(api: httpx.Client, base: str, headers: dict, solve_id: str | None, ok: bool) -> None:
+        if not solve_id:
+            return
+        try:
+            api.post(f"{base}/solves/{solve_id}/outcome", json={"verified": ok}, headers=headers)
+        except Exception:  # noqa: BLE001 - reporting is best effort
+            pass
+
+    def report(self, clearance: Clearance, verified: bool) -> None:
+        """Report your own verification of a clearance (after ``solve(..., verify=False)``)."""
+        gw = _gateway()
+        if gw is None or not clearance.solve_id:
+            return
+        base, key = gw
+        clearance.verified = verified
+        with httpx.Client(timeout=15) as api:
+            self._report_sync(api, base, {"authorization": f"Bearer {key}"}, clearance.solve_id, verified)
+
+    # -- async -----------------------------------------------------------------
+
+    async def solve_async(self, session: Any, url: str, blocked: Any, *, session_id: str | None = None, verify: Any = True) -> Clearance:
+        """Async ``solve`` (curl_cffi AsyncSession, httpx.AsyncClient, or `afetch`)."""
+        gw = _gateway()
+        if gw is None:
+            raise VendorUnavailable("solver_not_enabled", self.vendor, None, "the platform solver is not available to this run")
+        base, key = gw
+        headers = {"authorization": f"Bearer {key}"}
+        self.solves += 1
+        async with httpx.AsyncClient(timeout=self.timeout_ms / 1000 + 30) as api:
+            r = await api.post(f"{base}/solve", json=self._start_payload(url, blocked, session, session_id), headers=headers)
+            data = self._handle(r.status_code, r.text, r.headers, self.vendor)
+            for _ in range(8):
+                action = data.get("action")
+                if action == "done":
+                    clearance = self._finish(session, data)
+                    break
+                if action == "failed":
+                    self._raise_failed(data)
+                if action not in ("fetch", "submit"):
+                    raise SolveAttemptError(f"unexpected solver answer: {action}")
+                results = []
+                for req in data.get("requests") or []:
+                    req = _checked(req, url)
+                    try:
+                        if self._afetch is not None:
+                            status, body = await self._afetch(session, req)
+                        else:
+                            status, body = await _send_async(session, req, self.request_timeout)
+                    except Exception as exc:  # noqa: BLE001
+                        raise SolveAttemptError(f"challenge request failed: {type(exc).__name__}") from exc
+                    results.append({"id": req["id"], "status": status, "body": body[:_MAX_BODY]})
+                r = await api.post(
+                    f"{base}/solve",
+                    json={"solveId": data.get("solveId"), "results": results, "session": {"id": session_id, "cookieHeader": _cookie_header(session, url)}},
+                    headers=headers,
+                )
+                data = self._handle(r.status_code, r.text, r.headers, self.vendor)
+            else:
+                raise SolveAttemptError("the solve did not finish")
+            if verify is False:
+                return clearance
+            if callable(verify):
+                result = verify(session)
+                ok = bool(await result) if inspect.isawaitable(result) else bool(result)
+                response = None
+            else:
+                try:
+                    response = await session.get(url, timeout=self.request_timeout)
+                    ok = not is_blocked(response)
+                except Exception:  # noqa: BLE001
+                    ok, response = False, None
+            clearance.verified = ok
+            clearance.response = response
+            if clearance.solve_id:
+                try:
+                    await api.post(f"{base}/solves/{clearance.solve_id}/outcome", json={"verified": ok}, headers=headers)
+                except Exception:  # noqa: BLE001
+                    pass
+        if not ok:
+            raise SolveAttemptError("still blocked after the clearance was applied")
+        return clearance

scrapely/storage/api_client.py

@@ -76,15 +76,73 @@ def _error_type(response: httpx.Response) -> Optional[str]:
 
 
 # Default time budget for retrying one API call (429, 503, network errors).
-DEFAULT_API_RETRY_WINDOW_SECS = 120.0
+DEFAULT_API_RETRY_WINDOW_SECS = 300.0
 _MAX_BACKOFF_SECS = 30.0
 
+# Response header the API sets on every dataset push that sent an
+# Idempotency-Key, once it de-duplicates such pushes. An older API ignores the
+# key, and re-sending a push there could store the rows twice, so ambiguous
+# failures of keyed writes are retried only while the latest keyed answer from
+# that API origin carried this header. A 2xx without it (an older instance,
+# e.g. mid-deploy or after a rollback) switches the retries off again.
+IDEMPOTENCY_SUPPORT_HEADER = 'X-Scrapely-Idempotency'
+_KEY_IN_PROGRESS = 'idempotency-key-in-progress'
+_OUTCOME_UNKNOWN = 'push-outcome-unknown'
+_keyed_writes_deduped: Dict[str, bool] = {}
+_keyed_writes_override: Optional[bool] = None
+
+
+def _origin_of(url: str) -> str:
+    try:
+        u = httpx.URL(url)
+        return f'{u.scheme}://{u.netloc.decode()}'
+    except Exception:
+        return url
+
+
+def api_dedupes_keyed_writes(url: Optional[str] = None) -> bool:
+    """Whether the API at ``url`` (default: the last one seen) de-duplicates keyed writes."""
+    if _keyed_writes_override is not None:
+        return _keyed_writes_override
+    if url is not None:
+        return _keyed_writes_deduped.get(_origin_of(url)) is True
+    return bool(_keyed_writes_deduped) and list(_keyed_writes_deduped.values())[-1] is True
+
+
+def _reset_keyed_write_support(value: Optional[bool] = None) -> None:
+    """Test hook: forget what was learned about the API, or force True for every origin."""
+    global _keyed_writes_override
+    _keyed_writes_deduped.clear()
+    _keyed_writes_override = True if value is True else None
+
+
+def _note_keyed_answer(url: str, response: httpx.Response) -> None:
+    global _keyed_writes_override
+    origin = _origin_of(url)
+    if IDEMPOTENCY_SUPPORT_HEADER in response.headers:
+        _keyed_writes_deduped.pop(origin, None)
+        _keyed_writes_deduped[origin] = True
+    elif response.is_success:
+        # Stored by a handler that ignored the key: this API (instance) does not de-duplicate.
+        _keyed_writes_deduped.pop(origin, None)
+        _keyed_writes_deduped[origin] = False
+        _keyed_writes_override = None
+
+
+def _mark_outcome_unknown(exc: BaseException) -> BaseException:
+    """Mark an error whose request an earlier attempt may have applied."""
+    try:
+        setattr(exc, 'outcome_unknown', True)
+    except Exception:
+        pass
+    return exc
+
 
 def api_retry_window_secs() -> float:
     """How long one API call keeps retrying before it gives up.
 
     ``SCRAPELY_API_RETRY_WINDOW_SECS`` when set, else ``ApiClient.retry_window_secs``
-    (2 minutes by default).
+    (5 minutes by default).
     """
     raw = os.environ.get('SCRAPELY_API_RETRY_WINDOW_SECS')
     if raw:
@@ -198,7 +256,7 @@ class ApiClient:
     _rate_limiter = AdaptiveRateLimiter()
     # How long one call keeps retrying 429/503/network errors before it gives
     # up (SCRAPELY_API_RETRY_WINDOW_SECS overrides it). Long enough that a
-    # throttled run slows down instead of failing, and covers a main redeploy.
+    # throttled run slows down instead of failing, and covers an API redeploy.
     retry_window_secs = DEFAULT_API_RETRY_WINDOW_SECS
 
     def __init__(
@@ -284,22 +342,26 @@ class ApiClient:
         do_not_retry_timeouts: bool = False,
         idempotent: Optional[bool] = None,
         retry_deadline: Optional[float] = None,
+        idempotency_key: Optional[str] = None,
     ) -> httpx.Response:
         """Make an HTTP request with the API retry policy.
 
         Retries, with exponential backoff and jitter that honours Retry-After,
-        until the retry window (2 minutes by default) runs out:
+        until the retry window (5 minutes by default) runs out:
 
         - 429 (run API rate limit, dataset backpressure): always; nothing was applied.
         - 503 from the limiter or HAProxy: always; nothing was forwarded.
         - 500/502/503/504, read/write timeouts and dropped connections:
-          idempotent calls only.
+          idempotent calls, and writes with an ``idempotency_key`` once the API
+          is known to de-duplicate them (a re-send returns the original answer).
         - Refused connections and connect/pool timeouts: always; the request
           never left.
 
-        A write (dataset push, KV PUT) whose request may have reached the API is
-        never re-sent: there is no idempotency key for it, so a re-send could
-        store the same rows twice. Such failures surface to the caller instead.
+        Any other write (KV PUT, a keyed push to an API that did not say it
+        de-duplicates) whose request may have reached the API is never re-sent:
+        a re-send could apply it twice. Such failures surface to the caller
+        instead, with ``outcome_unknown = True`` when an attempt may have been
+        applied.
 
         ``retry_deadline`` (a ``time.monotonic()`` value) ends the retries
         earlier, so a caller with a time budget (e.g. the exit flush) gets an
@@ -319,6 +381,21 @@ class ApiClient:
 
         if idempotent is None:
             idempotent = method.upper() in IDEMPOTENT_METHODS
+        if idempotency_key is not None:
+            request_headers['Idempotency-Key'] = idempotency_key
+
+        def safe_to_resend() -> bool:
+            # Idempotent calls, and keyed writes once the API showed it
+            # de-duplicates them (learned from any keyed answer).
+            return bool(idempotent) or (idempotency_key is not None and api_dedupes_keyed_writes(url))
+
+        # An attempt of a non-idempotent call may have been applied (timeout,
+        # dropped connection, 5xx): the final error must not claim "nothing applied".
+        maybe_applied = False
+
+        def fail(exc: BaseException) -> BaseException:
+            return _mark_outcome_unknown(exc) if maybe_applied and not idempotent else exc
+
         deadline = time.monotonic() + max(0.0, api_retry_window_secs())
         if retry_deadline is not None:
             deadline = min(deadline, retry_deadline)
@@ -355,36 +432,53 @@ class ApiClient:
             except (httpx.ConnectTimeout, httpx.PoolTimeout, httpx.ConnectError) as exc:
                 # Never reached the API: safe for every method.
                 if isinstance(exc, (httpx.ConnectTimeout, httpx.PoolTimeout)) and do_not_retry_timeouts:
-                    raise
+                    raise fail(exc)
                 if await backoff(f'failed ({type(exc).__name__})', None):
                     continue
-                raise
+                raise fail(exc)
             except httpx.TimeoutException as exc:
                 # Read/write timeout: the request may have been applied.
-                if idempotent and not do_not_retry_timeouts and await backoff(f'timed out ({type(exc).__name__})', None):
+                maybe_applied = True
+                if safe_to_resend() and not do_not_retry_timeouts and await backoff(f'timed out ({type(exc).__name__})', None):
                     continue
-                raise
+                raise fail(exc)
             except httpx.TransportError as exc:
-                # Reset mid-request: only idempotent calls are re-sent.
-                if idempotent and await backoff(f'failed ({type(exc).__name__})', None):
+                # Reset mid-request: re-sent only when that is safe.
+                maybe_applied = True
+                if safe_to_resend() and await backoff(f'failed ({type(exc).__name__})', None):
                     continue
-                raise
+                raise fail(exc)
 
             self._rate_limiter.update_from_headers(response.headers)
             if response.status_code == 429:
                 self._rate_limiter.record_rate_limited()
-
-            if not response.is_success and is_retryable_response(
-                response.status_code,
-                idempotent,
-                _error_type(response) if response.status_code == 503 else None,
-                response.text if response.status_code == 503 else None,
-            ):
-                if await backoff(f'answered {response.status_code}', response.headers.get('Retry-After')):
-                    continue
+            if idempotency_key is not None:
+                _note_keyed_answer(url, response)
+
+            if not response.is_success:
+                status = response.status_code
+                error_type = _error_type(response) if status == 409 or status >= 500 else None
+                body_text = response.text if status == 503 else None
+                # A 5xx other than the limiter's / HAProxy's 503 may come after the write was applied.
+                if not idempotent and status >= 500 and not is_retryable_response(status, False, error_type, body_text):
+                    maybe_applied = True
+                if idempotency_key is not None and error_type == _OUTCOME_UNKNOWN:
+                    # Stored or not, nobody knows: the API refuses repeats of this key. Final.
+                    maybe_applied = True
+                else:
+                    # 409 for a keyed write: an earlier attempt is still being stored.
+                    in_progress = idempotency_key is not None and status == 409 and error_type == _KEY_IN_PROGRESS
+                    if in_progress or is_retryable_response(status, safe_to_resend(), error_type, body_text):
+                        if await backoff(f'answered {status}', response.headers.get('Retry-After')):
+                            continue
+                        if in_progress:
+                            maybe_applied = True
 
             if raise_for_status:
-                await self._raise_for_status(response)
+                try:
+                    await self._raise_for_status(response)
+                except ApiClientError as exc:
+                    raise fail(exc)
             return response
 
     async def get(
@@ -416,6 +510,7 @@ class ApiClient:
         raise_for_status: bool = True,
         idempotent: Optional[bool] = None,
         retry_deadline: Optional[float] = None,
+        idempotency_key: Optional[str] = None,
     ) -> httpx.Response:
         return await self.request(
             'POST',
@@ -427,6 +522,7 @@ class ApiClient:
             raise_for_status=raise_for_status,
             idempotent=idempotent,
             retry_deadline=retry_deadline,
+            idempotency_key=idempotency_key,
         )
 
     async def put(
@@ -479,8 +575,11 @@ class ApiClient:
         headers: Optional[Dict[str, str]] = None,
         *,
         idempotent: Optional[bool] = None,
+        idempotency_key: Optional[str] = None,
     ) -> Any:
-        response = await self.post(path, json_data=data, params=params, headers=headers, idempotent=idempotent)
+        response = await self.post(
+            path, json_data=data, params=params, headers=headers, idempotent=idempotent, idempotency_key=idempotency_key
+        )
         if not response.content:
             return {}
         try:

scrapely/storage/client.py

@@ -182,7 +182,11 @@ class ApiStorageClient:
         dataset_id: str,
         items: Sequence[Mapping[str, Any]],
     ) -> None:
-        await self.api_client.post_json(f'/v2/datasets/{dataset_id}/items', list(items))
+        # One key per push, the same on every attempt: the API stores a re-sent
+        # push once, so a push whose connection dropped can be retried.
+        await self.api_client.post_json(
+            f'/v2/datasets/{dataset_id}/items', list(items), idempotency_key=str(uuid.uuid4())
+        )
 
     async def dataset_list_items(
         self,

{scrapely_python-0.1.14.dist-info → scrapely_python-0.1.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.14
+Version: 0.1.16
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.14.dist-info → scrapely_python-0.1.16.dist-info}/RECORD

@@ -1,9 +1,9 @@
-scrapely/__init__.py,sha256=ufWE21XlAwFdGRoSHi0mdTUUnJ4OMaa7ShbGFO-V6mM,5746
+scrapely/__init__.py,sha256=liCCoe7VolXKiiQlJmaBGJt7DlkLFi904rNpFMT-kX8,5746
 scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
-scrapely/actor.py,sha256=dGim2A5OqA0raiRNJk7E2O19QnEik4xgypFv0rXx3PA,78892
+scrapely/actor.py,sha256=1MXGyHql5sRfh15HMRY8vAp_TNr_yhavDALlHBcjLO0,80810
 scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
 scrapely/cdp_retry.py,sha256=hdTJEAIKhApM7IOk1DrQEl8NoflSJSmqd4RbNJ2IsgI,8540
-scrapely/charging.py,sha256=_GHEKEgQ7lRYTVPacBiBuvlaqGTNcsF65K8xhiJ0dNM,68526
+scrapely/charging.py,sha256=HQHDkJ9buhlgArmmAC0I4lON8v-B6jgLD594-MpB-Rs,68905
 scrapely/charging_client.py,sha256=-4q-Ww7gu9EZXpM9QIn1Vd4vwAXmgL_WEKqaQIx3ViA,3961
 scrapely/configuration.py,sha256=Of-lKm7hoPGUQO8d08kV7fAy2co9QTfb8Of_8ds5PSA,17028
 scrapely/errors.py,sha256=DCAlVvE3jToZuPEG2HVDL7i8D8hLboomQ5JpSOcsXoQ,4107
@@ -12,6 +12,7 @@ scrapely/patch_browser_pool.py,sha256=qZstdQ1rFw8kGjzB9fm3GXco8Jt40eklTw1Czf_vh-
 scrapely/pricing_types.py,sha256=J6wCddZd-U1MYeOtaaEdeHSsp1HBgiQLfvSInBOvhIo,5305
 scrapely/proxy_configuration.py,sha256=4passyiUBfBsuzaBDyf5003rY0ayLYwwSnev0FaxCs0,38125
 scrapely/py.typed,sha256=XtWlywqUjc6x_M2gAXlDtYXbAsYkCjKZEjFyJvaHvm4,105
+scrapely/solver.py,sha256=ahuORJwZu9aJzCKWF14qTjQSnwT6dEbhMophQQ03JHw,35854
 scrapely/types.py,sha256=ceVuHvhYbPai2CL3vFmXLTMoGaMR27IefJFgWhzd7ws,8900
 scrapely/webhook.py,sha256=4WOM42ypOSOs4lKtP5ofxAh6lX-7c8p5U9lOfuFzx-g,2830
 scrapely/crypto/__init__.py,sha256=MbMPs7Eq_fGSVjOZxNts_JckDCUtxCVqdkHU0VCfN4M,481
@@ -23,8 +24,8 @@ scrapely/request_loaders/__init__.py,sha256=aNgr4pRt3MziUbU6UQ_bjTT3moMNvvyUsPeC
 scrapely/request_loaders/_scrapely_request_list.py,sha256=vguj-EvBjvrGY5_vf2fGXnJBfnhlPeRjdkjPFaxUA7g,7628
 scrapely/storage/__init__.py,sha256=diAhxEvOXjc3YVhZvzkO3MttJUXQWJ5ogxstiApsh7U,1405
 scrapely/storage/_metadata.py,sha256=WWawBnVN0b1iK2K-SlxZXUc6czb5wXWf8iC_0tzkgAQ,4764
-scrapely/storage/api_client.py,sha256=SRtDOR58fggYwWjcz-S5RshDCM4kO2opxqn_k9Wdyxg,18183
-scrapely/storage/client.py,sha256=qQtFf1G2wv2RU5uHil521j3EAI7JDLX9VngPePe3VMU,22353
+scrapely/storage/api_client.py,sha256=uhIU5LnOmW0x6MOMt1H7iWlvs-b55Sn4zVCgAnynTug,22900
+scrapely/storage/client.py,sha256=dfTD3424kb0AAFdFU5F6HMcS2XaKcJ1ACiViYniKDXg,22562
 scrapely/storage/patch_storage.py,sha256=o3Fv0BByeZ1T9HhkzhY6bjolOFsIQcnqBoHq1EdG8Fw,8758
 scrapely/storage/scrapely_dataset_client.py,sha256=u5dnXxe1BAsq7rh-haR5W1ChE7UZ_k7B9ShR8YtwFkg,10695
 scrapely/storage/scrapely_kvs_client.py,sha256=ItQhe2rcSl20NkSmegfqHiRyKncH4Qg9BjQVpF6jHfk,7403
@@ -38,10 +39,10 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9
 scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
 scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
 scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
-scrapely_python-0.1.14.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
-scrapely_python-0.1.14.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
-scrapely_python-0.1.14.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
-scrapely_python-0.1.14.dist-info/METADATA,sha256=81hJV4H066ZpErdWq7wvAKXDta_3xjcnMWdAVr_zu2g,3487
-scrapely_python-0.1.14.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-scrapely_python-0.1.14.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
-scrapely_python-0.1.14.dist-info/RECORD,,
+scrapely_python-0.1.16.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
+scrapely_python-0.1.16.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
+scrapely_python-0.1.16.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
+scrapely_python-0.1.16.dist-info/METADATA,sha256=wxPrakVU3_4JfSWaOgNs6hYKp3vkCbssBHR5OH9Jcoo,3487
+scrapely_python-0.1.16.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+scrapely_python-0.1.16.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
+scrapely_python-0.1.16.dist-info/RECORD,,