scrapely-python 0.1.15__py3-none-any.whl → 0.1.16__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
scrapely/__init__.py CHANGED
@@ -132,7 +132,7 @@ from .errors import (
132
132
  raise_for_status,
133
133
  )
134
134
 
135
- __version__ = '0.1.15'
135
+ __version__ = '0.1.16'
136
136
  __all__ = [
137
137
  # Core
138
138
  'Actor',
scrapely/actor.py CHANGED
@@ -1080,6 +1080,34 @@ class Actor(metaclass=_ActorMeta):
1080
1080
  status_message=data.get('statusMessage'),
1081
1081
  )
1082
1082
 
1083
+ @classmethod
1084
+ async def solver_status(cls, vendor: str) -> Dict[str, Any]:
1085
+ """Anti-bot solver status for a vendor (free, cached for 15 s, never raises).
1086
+
1087
+ ``usable`` False means the vendor cannot be solved right now: take your
1088
+ fallback (for example the hosted browser), or fail the items that need
1089
+ a solve at once and call ``set_solver_degraded_status()``. See
1090
+ ``scrapely.solver``.
1091
+ """
1092
+ from .solver import solver_status_async
1093
+
1094
+ return await solver_status_async(vendor)
1095
+
1096
+ @classmethod
1097
+ async def set_solver_degraded_status(cls, skipped: Optional[int] = None, total: Optional[int] = None) -> None:
1098
+ """Set the standard status message for a run whose site protection cannot be solved right now.
1099
+
1100
+ "Site protection temporarily unavailable, results may be incomplete",
1101
+ with "(N of M items skipped)" when both counts are given. Never raises:
1102
+ the run goes on.
1103
+ """
1104
+ from .solver import degraded_status_message
1105
+
1106
+ try:
1107
+ await cls.set_status_message(degraded_status_message(skipped, total))
1108
+ except Exception as exc: # noqa: BLE001 - a status message must never stop the run
1109
+ logger.warning(f'Could not set the solver status message: {exc}')
1110
+
1083
1111
  @classmethod
1084
1112
  async def _put_run_status_message(
1085
1113
  cls, message: str, *, is_terminal: bool
scrapely/solver.py ADDED
@@ -0,0 +1,875 @@
1
+ """Anti-bot solver helpers for Scrapely Actors.
2
+
3
+ The platform runs the challenge solver (xhr.dev today) behind
4
+ ``/v2/solver``. Actors never hold a solver key: the platform injects
5
+ ``SCRAPELY_SOLVER_URL`` and ``SCRAPELY_SOLVER_API_KEY`` (the run's own token)
6
+ into runs of Actors that may use it.
7
+
8
+ Three layers, use what you need:
9
+
10
+ * ``solver_status(vendor)`` / ``await Actor.solver_status(vendor)``: a free,
11
+ cached status check. When a vendor is ``down`` take your fallback (for
12
+ example the hosted browser) or fail the items that need a solve at once.
13
+ * ``Solver``: solves one challenge on your own HTTP session. The server tells
14
+ the SDK which requests to send (challenge document, scripts, submission);
15
+ they go out through *your* session so the clearance is bound to your exit IP.
16
+ Cookies land in the session's jar; the clearance is verified and reported.
17
+ * ``SolvePolicy``: retries on fresh proxy sessions with backoff and a breaker
18
+ (the policy the solver Actors already use), plus ``VendorUnavailable``,
19
+ which is raised at once, without retries, when the platform says the
20
+ vendor is down.
21
+
22
+ Typical use::
23
+
24
+ from scrapely.solver import Solver, SolvePolicy, SolveFailed, VendorUnavailable
25
+
26
+ solver = Solver(vendor="datadome", accept_language="es-ES,es;q=0.9", timezone="Europe/Madrid")
27
+ policy = SolvePolicy()
28
+
29
+ def attempt(i):
30
+ session = new_session(i) # fresh exit per try
31
+ r = session.get(url)
32
+ if solver.is_blocked(r):
33
+ solver.solve(session, url, r, session_id=f"s{i}")
34
+ r = session.get(url)
35
+ return r
36
+
37
+ try:
38
+ page = policy.run(attempt, label=url)
39
+ except VendorUnavailable:
40
+ ... # vendor down: fail this item at once (or use your fallback)
41
+ except SolveFailed:
42
+ ... # every try failed: record a per-item error, keep going
43
+
44
+ When a vendor is down and the Actor has no fallback, the run must still
45
+ start and finish: call ``await Actor.set_solver_degraded_status()`` once, fail
46
+ the items that need a solve, keep everything else.
47
+ """
48
+
49
+ from __future__ import annotations
50
+
51
+ import asyncio
52
+ import inspect
53
+ import json
54
+ import os
55
+ import random
56
+ import re
57
+ import threading
58
+ import time
59
+ from dataclasses import dataclass, field
60
+ from typing import Any, Awaitable, Callable, Optional
61
+
62
+ import httpx
63
+
64
+ __all__ = [
65
+ "Clearance",
66
+ "DEGRADED_STATUS_MESSAGE",
67
+ "IpBanned",
68
+ "SolveAttemptError",
69
+ "SolveFailed",
70
+ "SolvePolicy",
71
+ "Solver",
72
+ "SolverAuthError",
73
+ "SolverRateLimited",
74
+ "SolverRequestInvalid",
75
+ "SolverUnavailable",
76
+ "VendorUnavailable",
77
+ "check_solver_response",
78
+ "degraded_status_message",
79
+ "detect_vendor",
80
+ "is_blocked",
81
+ "solver_env",
82
+ "solver_status",
83
+ "solver_status_async",
84
+ ]
85
+
86
+ DEGRADED_STATUS_MESSAGE = "Site protection temporarily unavailable, results may be incomplete"
87
+
88
+ # Codes the platform answers with when a vendor cannot be solved right now.
89
+ UNAVAILABLE_CODES = ("vendor_degraded", "vendor_down", "provider_unavailable", "vendor_unsupported", "solver_not_enabled")
90
+
91
+ _COMPAT_SUFFIX = "/compat/xhrdev"
92
+ _STATUS_TTL_SECS = 15.0
93
+ _MAX_BODY = 4 * 1024 * 1024
94
+
95
+
96
+ # ─── environment ─────────────────────────────────────────────────────────────
97
+
98
+
99
+ def solver_env() -> tuple[str, str] | None:
100
+ """Solver URL and key (the run token), injected by the platform. None when not set.
101
+
102
+ Never hardcode the host or the key, and never log the key.
103
+ """
104
+ url = (os.environ.get("SCRAPELY_SOLVER_URL") or "").strip().rstrip("/")
105
+ key = (os.environ.get("SCRAPELY_SOLVER_API_KEY") or "").strip()
106
+ if not url or not key:
107
+ return None
108
+ return url, key
109
+
110
+
111
+ def _gateway() -> tuple[str, str] | None:
112
+ """Base of the platform solver API (…/v2/solver) and the run token, or None.
113
+
114
+ Only available when the platform serves the solver (the URL ends with the
115
+ compat route); a raw solver URL means the old direct mode.
116
+ """
117
+ env = solver_env()
118
+ if env is None:
119
+ return None
120
+ url, key = env
121
+ if not url.endswith(_COMPAT_SUFFIX):
122
+ return None
123
+ return url[: -len(_COMPAT_SUFFIX)], key
124
+
125
+
126
+ # ─── errors ──────────────────────────────────────────────────────────────────
127
+
128
+
129
+ class SolveAttemptError(RuntimeError):
130
+ """One try failed. Worth another try on a fresh exit."""
131
+
132
+
133
+ class IpBanned(SolveAttemptError):
134
+ """The address is refused outright. Only a new exit helps."""
135
+
136
+
137
+ class SolverRateLimited(SolveAttemptError):
138
+ """The solver answered 429. Not retried for this item."""
139
+
140
+ def __init__(self, retry_after: float | None = None) -> None:
141
+ self.retry_after = retry_after
142
+ suffix = f", retry after {retry_after:g}s" if retry_after is not None else ""
143
+ super().__init__(f"solver rate limit hit (HTTP 429{suffix})")
144
+
145
+
146
+ class SolverRequestInvalid(SolveAttemptError):
147
+ """The solver rejected the request itself (HTTP 400). Not retried."""
148
+
149
+
150
+ class SolverAuthError(SolveAttemptError):
151
+ """The solver refused the key (HTTP 401/403). Opens the breaker."""
152
+
153
+
154
+ class SolveFailed(RuntimeError):
155
+ """Every try for one item failed."""
156
+
157
+ def __init__(self, message: str, attempts: int, last_error: BaseException | None = None) -> None:
158
+ super().__init__(message)
159
+ self.attempts = attempts
160
+ self.last_error = last_error
161
+
162
+
163
+ class SolverUnavailable(SolveFailed):
164
+ """The breaker is open: the solver failed for everything recently."""
165
+
166
+
167
+ class VendorUnavailable(SolverUnavailable):
168
+ """The platform says this vendor cannot be solved right now (fail fast, no retries).
169
+
170
+ ``code`` is one of ``vendor_down``, ``vendor_degraded``,
171
+ ``provider_unavailable``, ``vendor_unsupported`` or ``solver_not_enabled``.
172
+ """
173
+
174
+ def __init__(self, code: str, vendor: str | None = None, retry_after: float | None = None, message: str | None = None) -> None:
175
+ self.code = code
176
+ self.vendor = vendor
177
+ self.retry_after = retry_after
178
+ super().__init__(message or f"anti-bot solver unavailable ({code}{f', {vendor}' if vendor else ''})", 0)
179
+
180
+
181
+ def _json(body: str) -> dict:
182
+ try:
183
+ parsed = json.loads(body or "")
184
+ return parsed if isinstance(parsed, dict) else {}
185
+ except (TypeError, ValueError):
186
+ return {}
187
+
188
+
189
+ def check_solver_response(status: int, body: str = "", retry_after: str | None = None) -> None:
190
+ """Raise the right error for a non-2xx solver answer; return on 2xx."""
191
+ if 200 <= status < 300:
192
+ return
193
+ detail = (body or "")[:200].replace("\n", " ")
194
+ seconds: float | None = None
195
+ try:
196
+ seconds = float(retry_after) if retry_after is not None else None
197
+ except (TypeError, ValueError):
198
+ seconds = None
199
+ if status == 503:
200
+ parsed = _json(body)
201
+ code = str(parsed.get("code") or "")
202
+ if code in UNAVAILABLE_CODES:
203
+ raise VendorUnavailable(code, parsed.get("vendor"), seconds if seconds is not None else parsed.get("retryAfterSecs"))
204
+ if status == 429:
205
+ raise SolverRateLimited(seconds)
206
+ if status in (401, 403):
207
+ parsed = _json(body)
208
+ if parsed.get("code") == "solver_not_enabled":
209
+ raise VendorUnavailable("solver_not_enabled", None, None, "the anti-bot solver is not enabled for this Actor")
210
+ raise SolverAuthError(f"solver refused the key (HTTP {status})")
211
+ if status == 400:
212
+ raise SolverRequestInvalid(f"solver rejected the request (HTTP 400): {detail}")
213
+ raise SolveAttemptError(f"solver HTTP {status}: {detail}")
214
+
215
+
216
+ _NOT_RETRIED = (SolverRateLimited, SolverRequestInvalid, SolverAuthError)
217
+
218
+
219
+ # ─── retry policy ────────────────────────────────────────────────────────────
220
+
221
+
222
+ class SolvePolicy:
223
+ """Up to `attempts` tries per item, each on a fresh proxy session, with backoff and a breaker.
224
+
225
+ * HTTP 5xx, timeouts, transport errors, empty solutions and a page still
226
+ blocked after the clearance are failed tries;
227
+ * HTTP 429 ends the tries for that item;
228
+ * HTTP 401/403 opens the breaker at once;
229
+ * after `trip_after` consecutive failed tries the breaker opens and
230
+ further solves fail fast with SolverUnavailable; after `cooldown`
231
+ seconds one probe is let through;
232
+ * VendorUnavailable (the platform says the vendor is down) is raised at
233
+ once, without retries and without touching the breaker.
234
+ """
235
+
236
+ def __init__(
237
+ self,
238
+ attempts: int = 4,
239
+ base_delay: float = 1.5,
240
+ max_delay: float = 15.0,
241
+ trip_after: int = 12,
242
+ cooldown: float = 180.0,
243
+ sleep: Callable[[float], None] = time.sleep,
244
+ async_sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
245
+ clock: Callable[[], float] = time.monotonic,
246
+ rng: Callable[[], float] = random.random,
247
+ ) -> None:
248
+ self.attempts = max(1, min(5, attempts))
249
+ self.base_delay = base_delay
250
+ self.max_delay = max_delay
251
+ self.trip_after = max(1, trip_after)
252
+ self.cooldown = cooldown
253
+ self._sleep = sleep
254
+ self._async_sleep = async_sleep
255
+ self._clock = clock
256
+ self._rng = rng
257
+ self._lock = threading.Lock()
258
+ self.consecutive_failures = 0
259
+ self.opened_at: float | None = None
260
+ self.calls = 0
261
+ self.solved = 0
262
+ self.failed_items = 0
263
+ self.unavailable_items = 0
264
+ self.last_error = ""
265
+
266
+ @property
267
+ def down(self) -> bool:
268
+ with self._lock:
269
+ return self.opened_at is not None and self._clock() - self.opened_at < self.cooldown
270
+
271
+ @property
272
+ def ever_tripped(self) -> bool:
273
+ return self.opened_at is not None
274
+
275
+ def _admit(self) -> None:
276
+ with self._lock:
277
+ if self.opened_at is None:
278
+ return
279
+ if self._clock() - self.opened_at < self.cooldown:
280
+ raise SolverUnavailable(
281
+ f"challenge solver unavailable after {self.trip_after} consecutive failed solves"
282
+ + (f" (last error: {self.last_error})" if self.last_error else ""),
283
+ 0,
284
+ )
285
+ self.opened_at = self._clock()
286
+ self.consecutive_failures = self.trip_after - 1
287
+
288
+ def _success(self) -> None:
289
+ with self._lock:
290
+ self.calls += 1
291
+ self.solved += 1
292
+ self.consecutive_failures = 0
293
+ self.opened_at = None
294
+
295
+ def _failure(self, exc: BaseException) -> None:
296
+ with self._lock:
297
+ self.calls += 1
298
+ self.consecutive_failures += 1
299
+ self.last_error = _short(exc)
300
+ if isinstance(exc, SolverAuthError) or self.consecutive_failures >= self.trip_after:
301
+ self.opened_at = self._clock()
302
+
303
+ def _unavailable(self) -> None:
304
+ with self._lock:
305
+ self.unavailable_items += 1
306
+
307
+ def delay(self, index: int) -> float:
308
+ raw = min(self.max_delay, self.base_delay * (2 ** index))
309
+ return raw * (0.5 + self._rng())
310
+
311
+ def run(self, attempt: Callable[[int], Any], label: str = "page") -> Any:
312
+ """Call attempt(i) until it returns. Sync version (for worker threads)."""
313
+ last: BaseException | None = None
314
+ for index in range(self.attempts):
315
+ self._admit()
316
+ try:
317
+ result = attempt(index)
318
+ except VendorUnavailable:
319
+ self._unavailable()
320
+ raise
321
+ except Exception as exc: # noqa: BLE001 - every failure is one failed try
322
+ last = exc
323
+ self._failure(exc)
324
+ if isinstance(exc, _NOT_RETRIED):
325
+ break
326
+ if index + 1 < self.attempts and not self.down:
327
+ self._sleep(self.delay(index))
328
+ continue
329
+ self._success()
330
+ return result
331
+ return self._give_up(label, last)
332
+
333
+ async def run_async(self, attempt: Callable[[int], Any], label: str = "page") -> Any:
334
+ """Async version. attempt(i) may be a coroutine function or a plain one."""
335
+ last: BaseException | None = None
336
+ for index in range(self.attempts):
337
+ self._admit()
338
+ try:
339
+ result = attempt(index)
340
+ if inspect.isawaitable(result):
341
+ result = await result
342
+ except asyncio.CancelledError:
343
+ raise
344
+ except VendorUnavailable:
345
+ self._unavailable()
346
+ raise
347
+ except Exception as exc: # noqa: BLE001
348
+ last = exc
349
+ self._failure(exc)
350
+ if isinstance(exc, _NOT_RETRIED):
351
+ break
352
+ if index + 1 < self.attempts and not self.down:
353
+ await self._async_sleep(self.delay(index))
354
+ continue
355
+ self._success()
356
+ return result
357
+ return self._give_up(label, last)
358
+
359
+ def _give_up(self, label: str, last: BaseException | None) -> Any:
360
+ with self._lock:
361
+ self.failed_items += 1
362
+ if isinstance(last, SolverUnavailable):
363
+ raise last
364
+ raise SolveFailed(f"anti-bot challenge on {label} was not solved: {_short(last)}", self.attempts, last)
365
+
366
+ def summary(self) -> dict[str, int]:
367
+ return {
368
+ "solverCalls": self.calls,
369
+ "solved": self.solved,
370
+ "failedItems": self.failed_items,
371
+ "unavailableItems": self.unavailable_items,
372
+ }
373
+
374
+
375
+ def _short(exc: BaseException | None) -> str:
376
+ if exc is None:
377
+ return "unknown error"
378
+ text = str(exc) or type(exc).__name__
379
+ return text[:200]
380
+
381
+
382
+ # ─── detection ───────────────────────────────────────────────────────────────
383
+
384
+ _DD_RE = re.compile(r"var\s+dd\s*=\s*\{")
385
+ _SBSD_RE = re.compile(r'<script[^>]+src="([^"]+\?v=[0-9a-f-]{36}[^"]*)"', re.I)
386
+ _PX_RE = re.compile(r"_pxappid|px-captcha|perimeterx|/_px\d?/", re.I)
387
+ _KASADA_RE = re.compile(r"kpsdk", re.I)
388
+
389
+
390
+ def detect_vendor(status: int, body: str) -> str | None:
391
+ """The protection behind a *challenge* response: datadome, akamai-sbsd, perimeterx, kasada or None.
392
+
393
+ Ordinary pages that merely load a vendor's script are not challenges:
394
+ DataDome, PerimeterX and Kasada need an error status (4xx), and an SBSD
395
+ bundle counts only on a short interstitial page.
396
+ """
397
+ body = body or ""
398
+ sample = body[:20000]
399
+ head = sample[:4000].lower()
400
+ challenged = status >= 400
401
+ if challenged and ("captcha-delivery.com" in head or _DD_RE.search(sample) or "datadome" in head):
402
+ return "datadome"
403
+ if len(body) <= 20000 and _SBSD_RE.search(sample):
404
+ return "akamai-sbsd"
405
+ if challenged and _PX_RE.search(sample):
406
+ return "perimeterx"
407
+ if challenged and _KASADA_RE.search(sample):
408
+ return "kasada"
409
+ return None
410
+
411
+
412
+ def _status_body(response: Any) -> tuple[int, str]:
413
+ if isinstance(response, tuple):
414
+ return int(response[0]), str(response[1] or "")
415
+ if isinstance(response, dict):
416
+ return int(response.get("status") or 0), str(response.get("body") or "")
417
+ status = getattr(response, "status_code", None)
418
+ if status is None:
419
+ status = getattr(response, "status", 0)
420
+ text = getattr(response, "text", "")
421
+ if callable(text): # aiohttp-style
422
+ text = ""
423
+ return int(status or 0), str(text or "")
424
+
425
+
426
+ def is_blocked(response: Any, vendor: str | None = None) -> bool:
427
+ """True when the response is an anti-bot challenge (optionally of one vendor)."""
428
+ status, body = _status_body(response)
429
+ found = detect_vendor(status, body)
430
+ return found is not None if vendor in (None, "auto") else found == vendor
431
+
432
+
433
+ def degraded_status_message(skipped: int | None = None, total: int | None = None) -> str:
434
+ """The standard status message for runs whose site protection cannot be solved right now."""
435
+ if skipped is not None and total is not None:
436
+ return f"{DEGRADED_STATUS_MESSAGE} ({skipped} of {total} items skipped)"
437
+ return DEGRADED_STATUS_MESSAGE
438
+
439
+
440
+ # ─── status ──────────────────────────────────────────────────────────────────
441
+
442
+ _status_cache: dict[str, tuple[float, dict]] = {}
443
+ _status_lock = threading.Lock()
444
+
445
+
446
+ def _unknown_status(code: str | None = None) -> dict:
447
+ # Without the platform gateway nothing is known: let the Actor proceed as before.
448
+ return {"status": "unknown", "usable": code is None, "code": code, "successRate30m": None, "since": None, "recheckAfterSecs": 300}
449
+
450
+
451
+ def _status_from_answer(vendor: str, status: int, body: str) -> dict:
452
+ if status == 200:
453
+ data = _json(body)
454
+ v = (data.get("vendors") or {}).get(vendor)
455
+ if isinstance(v, dict):
456
+ return v
457
+ return _unknown_status()
458
+
459
+
460
+ def _cached(vendor: str) -> dict | None:
461
+ with _status_lock:
462
+ hit = _status_cache.get(vendor)
463
+ if hit and time.monotonic() - hit[0] < _STATUS_TTL_SECS:
464
+ return hit[1]
465
+ return None
466
+
467
+
468
+ def _store(vendor: str, value: dict) -> dict:
469
+ with _status_lock:
470
+ _status_cache[vendor] = (time.monotonic(), value)
471
+ return value
472
+
473
+
474
+ def solver_status(vendor: str, timeout: float = 10.0) -> dict:
475
+ """Vendor status from the platform (free, cached for 15 s). Never raises.
476
+
477
+ Returns a dict with ``status`` (healthy, degraded, down, unsupported or
478
+ unknown), ``usable`` (False means: take your fallback or fail the items
479
+ that need a solve), ``code``, ``successRate30m``, ``since`` and
480
+ ``recheckAfterSecs``.
481
+ """
482
+ vendor = (vendor or "").strip().lower()
483
+ hit = _cached(vendor)
484
+ if hit is not None:
485
+ return hit
486
+ gw = _gateway()
487
+ if gw is None:
488
+ return _unknown_status(None if solver_env() else "solver_not_enabled")
489
+ base, key = gw
490
+ try:
491
+ r = httpx.get(f"{base}/status", params={"vendors": vendor}, headers={"authorization": f"Bearer {key}"}, timeout=timeout)
492
+ return _store(vendor, _status_from_answer(vendor, r.status_code, r.text))
493
+ except Exception: # noqa: BLE001 - a failed check must never stop the run
494
+ return _unknown_status()
495
+
496
+
497
+ async def solver_status_async(vendor: str, timeout: float = 10.0) -> dict:
498
+ """Async ``solver_status``."""
499
+ vendor = (vendor or "").strip().lower()
500
+ hit = _cached(vendor)
501
+ if hit is not None:
502
+ return hit
503
+ gw = _gateway()
504
+ if gw is None:
505
+ return _unknown_status(None if solver_env() else "solver_not_enabled")
506
+ base, key = gw
507
+ try:
508
+ async with httpx.AsyncClient(timeout=timeout) as client:
509
+ r = await client.get(f"{base}/status", params={"vendors": vendor}, headers={"authorization": f"Bearer {key}"})
510
+ return _store(vendor, _status_from_answer(vendor, r.status_code, r.text))
511
+ except Exception: # noqa: BLE001
512
+ return _unknown_status()
513
+
514
+
515
+ # ─── solving ─────────────────────────────────────────────────────────────────
516
+
517
+
518
+ @dataclass
519
+ class Clearance:
520
+ """A solved challenge: cookies already set on the session, and the verification response."""
521
+
522
+ vendor: str
523
+ solve_id: str | None
524
+ cookies: list[dict] = field(default_factory=list)
525
+ ttl_hint_secs: int = 0
526
+ cached: bool = False
527
+ verified: bool | None = None
528
+ response: Any = None
529
+
530
+
531
+ def _is_httpx(session: Any) -> bool:
532
+ return type(session).__module__.split(".")[0] == "httpx"
533
+
534
+
535
+ def _cookie_applies(cookie: Any, host: str, path: str, secure: bool, now: float) -> bool:
536
+ """RFC 6265 matching: the cookies the session itself would send to this URL."""
537
+ domain = (getattr(cookie, "domain", "") or "").lower()
538
+ if not domain:
539
+ return False
540
+ bare = domain.lstrip(".")
541
+ host_only = not domain.startswith(".") and not getattr(cookie, "domain_specified", False)
542
+ if host_only:
543
+ if host != bare:
544
+ return False
545
+ elif host != bare and not host.endswith("." + bare):
546
+ return False
547
+ cpath = getattr(cookie, "path", "/") or "/"
548
+ if not (path == cpath or path.startswith(cpath.rstrip("/") + "/") or cpath == "/"):
549
+ return False
550
+ if getattr(cookie, "secure", False) and not secure:
551
+ return False
552
+ expires = getattr(cookie, "expires", None)
553
+ return not (expires is not None and expires <= now)
554
+
555
+
556
+ def _cookie_header(session: Any, url: str) -> str | None:
557
+ """The Cookie header the session would send to `url` (host, path, secure and expiry rules)."""
558
+ u = httpx.URL(url)
559
+ host = (u.host or "").lower()
560
+ path = u.path or "/"
561
+ secure = u.scheme == "https"
562
+ jar = getattr(getattr(session, "cookies", None), "jar", None)
563
+ pairs: list[str] = []
564
+ now = time.time()
565
+ try:
566
+ for c in jar or []:
567
+ if _cookie_applies(c, host, path, secure, now):
568
+ pairs.append(f"{c.name}={c.value}")
569
+ except Exception: # noqa: BLE001
570
+ return None
571
+ return "; ".join(pairs) or None
572
+
573
+
574
+ def _set_cookies(session: Any, cookies: list[dict]) -> None:
575
+ jar = getattr(session, "cookies", None)
576
+ if jar is None:
577
+ return
578
+ for c in cookies:
579
+ try:
580
+ jar.set(c["name"], c["value"], domain=c.get("domain") or "", path=c.get("path") or "/")
581
+ except Exception: # noqa: BLE001 - a jar without domain support
582
+ jar.set(c["name"], c["value"])
583
+
584
+
585
+ def _step_allowed(step_url: str, target_url: str) -> bool:
586
+ """A step may only go to the target page's own host or to captcha-delivery.com, over https."""
587
+ try:
588
+ step = httpx.URL(step_url)
589
+ target = httpx.URL(target_url)
590
+ except Exception: # noqa: BLE001
591
+ return False
592
+ if step.scheme != "https":
593
+ return False
594
+ host = (step.host or "").lower()
595
+ if host == (target.host or "").lower():
596
+ return True
597
+ return host == "captcha-delivery.com" or host.endswith(".captcha-delivery.com")
598
+
599
+
600
+ def _checked(req: dict, target_url: str) -> dict:
601
+ if not isinstance(req, dict) or not _step_allowed(str(req.get("url") or ""), target_url):
602
+ raise SolveAttemptError("the solver asked for a request outside the target site; refused")
603
+ if req.get("method") not in ("GET", "POST"):
604
+ raise SolveAttemptError("the solver asked for an unsupported request method; refused")
605
+ return req
606
+
607
+
608
+ def _send_sync(session: Any, req: dict, timeout: float) -> tuple[int, str]:
609
+ body = req.get("body")
610
+ kwargs: dict[str, Any] = {"headers": req.get("headers") or {}, "timeout": timeout}
611
+ if body is not None:
612
+ kwargs["content" if _is_httpx(session) else "data"] = body.encode() if isinstance(body, str) else body
613
+ r = session.request(req["method"], req["url"], **kwargs)
614
+ return _status_body(r)
615
+
616
+
617
+ async def _send_async(session: Any, req: dict, timeout: float) -> tuple[int, str]:
618
+ body = req.get("body")
619
+ kwargs: dict[str, Any] = {"headers": req.get("headers") or {}, "timeout": timeout}
620
+ if body is not None:
621
+ kwargs["content" if _is_httpx(session) else "data"] = body.encode() if isinstance(body, str) else body
622
+ r = await session.request(req["method"], req["url"], **kwargs)
623
+ return _status_body(r)
624
+
625
+
626
+ class Solver:
627
+ """Solves anti-bot challenges on the Actor's own HTTP session through the platform.
628
+
629
+ Args:
630
+ vendor: ``"datadome"``, ``"akamai-sbsd"`` or ``"auto"`` (detected from the blocked response).
631
+ identity: the browser profile your TLS impersonation claims (``"chrome-146-macos"``
632
+ matches curl_cffi ``impersonate="chrome146"``).
633
+ accept_language / timezone / languages: locale of that browser (match the site and proxy country).
634
+ fetch: optional ``fetch(session, request) -> (status, body)`` to send the step requests
635
+ yourself (sync), or ``afetch`` for async.
636
+ request_timeout: seconds per step request on your session.
637
+ """
638
+
639
+ def __init__(
640
+ self,
641
+ vendor: str = "auto",
642
+ identity: str = "chrome-146-macos",
643
+ accept_language: str | None = None,
644
+ timezone: str | None = None,
645
+ languages: str | None = None,
646
+ timeout_ms: int = 60_000,
647
+ request_timeout: float = 30.0,
648
+ fetch: Callable[[Any, dict], tuple[int, str]] | None = None,
649
+ afetch: Callable[[Any, dict], Awaitable[tuple[int, str]]] | None = None,
650
+ ) -> None:
651
+ self.vendor = vendor
652
+ self.identity = {"profile": identity}
653
+ if accept_language:
654
+ self.identity["acceptLanguage"] = accept_language
655
+ if timezone:
656
+ self.identity["timezone"] = timezone
657
+ if languages:
658
+ self.identity["languages"] = languages
659
+ self.timeout_ms = timeout_ms
660
+ self.request_timeout = request_timeout
661
+ self._fetch = fetch
662
+ self._afetch = afetch
663
+ self.solves = 0
664
+ self.cleared = 0
665
+
666
+ @staticmethod
667
+ def configured() -> bool:
668
+ """True when the platform solver API is available to this run."""
669
+ return _gateway() is not None
670
+
671
+ def is_blocked(self, response: Any) -> bool:
672
+ return is_blocked(response, None if self.vendor == "auto" else self.vendor)
673
+
674
+ # -- wire ------------------------------------------------------------------
675
+
676
+ def _start_payload(self, url: str, blocked: Any, session: Any, session_id: str | None) -> dict:
677
+ status, body = _status_body(blocked)
678
+ payload: dict[str, Any] = {
679
+ "vendor": self.vendor,
680
+ "url": url,
681
+ "blocked": {"status": status, "body": body[:_MAX_BODY]},
682
+ "identity": self.identity,
683
+ "timeoutMs": self.timeout_ms,
684
+ "session": {"cookieHeader": _cookie_header(session, url)},
685
+ }
686
+ if session_id:
687
+ payload["session"]["id"] = str(session_id)
688
+ return payload
689
+
690
+ @staticmethod
691
+ def _handle(status: int, text: str, headers: Any, vendor: str) -> dict:
692
+ data = _json(text)
693
+ if status == 503 and str(data.get("code") or "") in UNAVAILABLE_CODES:
694
+ raise VendorUnavailable(str(data["code"]), data.get("vendor") or vendor, data.get("retryAfterSecs"))
695
+ if status == 429:
696
+ ra = headers.get("retry-after") if headers is not None else None
697
+ raise SolverRateLimited(float(ra) if ra else data.get("retryAfterSecs"))
698
+ if status in (401, 403):
699
+ if data.get("code") == "solver_not_enabled":
700
+ raise VendorUnavailable("solver_not_enabled", vendor, None, "the anti-bot solver is not enabled for this Actor")
701
+ raise SolverAuthError(f"solver refused the run token (HTTP {status}, {data.get('code') or ''})")
702
+ if status == 404 and data.get("code") == "not_found":
703
+ raise SolveAttemptError("the solve expired; start again")
704
+ if status >= 400:
705
+ raise SolveAttemptError(f"solver HTTP {status}: {text[:160]}")
706
+ return data
707
+
708
+ @staticmethod
709
+ def _raise_failed(data: dict) -> None:
710
+ reason = str(data.get("reason") or "rejected")
711
+ message = str(data.get("message") or reason)
712
+ if reason == "ip_banned":
713
+ raise IpBanned(message)
714
+ if reason in ("no_challenge", "bad_request") and not data.get("retryable", True):
715
+ raise SolverRequestInvalid(message)
716
+ raise SolveAttemptError(message)
717
+
718
+ def _finish(self, session: Any, data: dict) -> Clearance:
719
+ cookies = data.get("cookies") or []
720
+ _set_cookies(session, cookies)
721
+ self.cleared += 1
722
+ return Clearance(
723
+ vendor=self.vendor,
724
+ solve_id=data.get("solveId"),
725
+ cookies=cookies,
726
+ ttl_hint_secs=int(data.get("ttlHintSecs") or 0),
727
+ cached=bool(data.get("cached")),
728
+ )
729
+
730
+ # -- sync ------------------------------------------------------------------
731
+
732
+ def solve(self, session: Any, url: str, blocked: Any, *, session_id: str | None = None, verify: Any = True) -> Clearance:
733
+ """Solve the challenge in `blocked` (a response of `session` for `url`).
734
+
735
+ verify: True re-fetches `url` and checks it is no longer blocked (the
736
+ result is reported to the platform); a callable ``verify(session) ->
737
+ bool`` does your own check; False skips it (call ``report()`` later).
738
+ Raises SolveAttemptError / IpBanned (try again on a fresh exit),
739
+ VendorUnavailable (fail fast), SolverRateLimited, SolverAuthError.
740
+ """
741
+ gw = _gateway()
742
+ if gw is None:
743
+ raise VendorUnavailable("solver_not_enabled", self.vendor, None, "the platform solver is not available to this run")
744
+ base, key = gw
745
+ headers = {"authorization": f"Bearer {key}"}
746
+ self.solves += 1
747
+ with httpx.Client(timeout=self.timeout_ms / 1000 + 30) as api:
748
+ r = api.post(f"{base}/solve", json=self._start_payload(url, blocked, session, session_id), headers=headers)
749
+ data = self._handle(r.status_code, r.text, r.headers, self.vendor)
750
+ for _ in range(8):
751
+ action = data.get("action")
752
+ if action == "done":
753
+ clearance = self._finish(session, data)
754
+ break
755
+ if action == "failed":
756
+ self._raise_failed(data)
757
+ if action not in ("fetch", "submit"):
758
+ raise SolveAttemptError(f"unexpected solver answer: {action}")
759
+ results = []
760
+ for req in data.get("requests") or []:
761
+ req = _checked(req, url)
762
+ try:
763
+ status, body = (self._fetch or (lambda s, q: _send_sync(s, q, self.request_timeout)))(session, req)
764
+ except Exception as exc: # noqa: BLE001
765
+ raise SolveAttemptError(f"challenge request failed: {type(exc).__name__}") from exc
766
+ results.append({"id": req["id"], "status": status, "body": body[:_MAX_BODY]})
767
+ r = api.post(
768
+ f"{base}/solve",
769
+ json={"solveId": data.get("solveId"), "results": results, "session": {"id": session_id, "cookieHeader": _cookie_header(session, url)}},
770
+ headers=headers,
771
+ )
772
+ data = self._handle(r.status_code, r.text, r.headers, self.vendor)
773
+ else:
774
+ raise SolveAttemptError("the solve did not finish")
775
+ if verify is False:
776
+ return clearance
777
+ ok, response = self._verify_sync(session, url, verify)
778
+ clearance.verified = ok
779
+ clearance.response = response
780
+ self._report_sync(api, base, headers, clearance.solve_id, ok)
781
+ if not ok:
782
+ raise SolveAttemptError("still blocked after the clearance was applied")
783
+ return clearance
784
+
785
+ def _verify_sync(self, session: Any, url: str, verify: Any) -> tuple[bool, Any]:
786
+ if callable(verify):
787
+ return bool(verify(session)), None
788
+ try:
789
+ r = session.get(url, timeout=self.request_timeout)
790
+ except Exception: # noqa: BLE001
791
+ return False, None
792
+ return not is_blocked(r), r
793
+
794
+ @staticmethod
795
+ def _report_sync(api: httpx.Client, base: str, headers: dict, solve_id: str | None, ok: bool) -> None:
796
+ if not solve_id:
797
+ return
798
+ try:
799
+ api.post(f"{base}/solves/{solve_id}/outcome", json={"verified": ok}, headers=headers)
800
+ except Exception: # noqa: BLE001 - reporting is best effort
801
+ pass
802
+
803
+ def report(self, clearance: Clearance, verified: bool) -> None:
804
+ """Report your own verification of a clearance (after ``solve(..., verify=False)``)."""
805
+ gw = _gateway()
806
+ if gw is None or not clearance.solve_id:
807
+ return
808
+ base, key = gw
809
+ clearance.verified = verified
810
+ with httpx.Client(timeout=15) as api:
811
+ self._report_sync(api, base, {"authorization": f"Bearer {key}"}, clearance.solve_id, verified)
812
+
813
+ # -- async -----------------------------------------------------------------
814
+
815
+ async def solve_async(self, session: Any, url: str, blocked: Any, *, session_id: str | None = None, verify: Any = True) -> Clearance:
816
+ """Async ``solve`` (curl_cffi AsyncSession, httpx.AsyncClient, or `afetch`)."""
817
+ gw = _gateway()
818
+ if gw is None:
819
+ raise VendorUnavailable("solver_not_enabled", self.vendor, None, "the platform solver is not available to this run")
820
+ base, key = gw
821
+ headers = {"authorization": f"Bearer {key}"}
822
+ self.solves += 1
823
+ async with httpx.AsyncClient(timeout=self.timeout_ms / 1000 + 30) as api:
824
+ r = await api.post(f"{base}/solve", json=self._start_payload(url, blocked, session, session_id), headers=headers)
825
+ data = self._handle(r.status_code, r.text, r.headers, self.vendor)
826
+ for _ in range(8):
827
+ action = data.get("action")
828
+ if action == "done":
829
+ clearance = self._finish(session, data)
830
+ break
831
+ if action == "failed":
832
+ self._raise_failed(data)
833
+ if action not in ("fetch", "submit"):
834
+ raise SolveAttemptError(f"unexpected solver answer: {action}")
835
+ results = []
836
+ for req in data.get("requests") or []:
837
+ req = _checked(req, url)
838
+ try:
839
+ if self._afetch is not None:
840
+ status, body = await self._afetch(session, req)
841
+ else:
842
+ status, body = await _send_async(session, req, self.request_timeout)
843
+ except Exception as exc: # noqa: BLE001
844
+ raise SolveAttemptError(f"challenge request failed: {type(exc).__name__}") from exc
845
+ results.append({"id": req["id"], "status": status, "body": body[:_MAX_BODY]})
846
+ r = await api.post(
847
+ f"{base}/solve",
848
+ json={"solveId": data.get("solveId"), "results": results, "session": {"id": session_id, "cookieHeader": _cookie_header(session, url)}},
849
+ headers=headers,
850
+ )
851
+ data = self._handle(r.status_code, r.text, r.headers, self.vendor)
852
+ else:
853
+ raise SolveAttemptError("the solve did not finish")
854
+ if verify is False:
855
+ return clearance
856
+ if callable(verify):
857
+ result = verify(session)
858
+ ok = bool(await result) if inspect.isawaitable(result) else bool(result)
859
+ response = None
860
+ else:
861
+ try:
862
+ response = await session.get(url, timeout=self.request_timeout)
863
+ ok = not is_blocked(response)
864
+ except Exception: # noqa: BLE001
865
+ ok, response = False, None
866
+ clearance.verified = ok
867
+ clearance.response = response
868
+ if clearance.solve_id:
869
+ try:
870
+ await api.post(f"{base}/solves/{clearance.solve_id}/outcome", json={"verified": ok}, headers=headers)
871
+ except Exception: # noqa: BLE001
872
+ pass
873
+ if not ok:
874
+ raise SolveAttemptError("still blocked after the clearance was applied")
875
+ return clearance
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapely-python
3
- Version: 0.1.15
3
+ Version: 0.1.16
4
4
  Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
5
5
  Author-email: Scrapely <support@scrape.ly>
6
6
  License-Expression: MIT AND Apache-2.0
@@ -1,6 +1,6 @@
1
- scrapely/__init__.py,sha256=aFN2f1YJxIkmmsJ9TqUiOAFzUxZQhzNllwytSQiVMZA,5746
1
+ scrapely/__init__.py,sha256=liCCoe7VolXKiiQlJmaBGJt7DlkLFi904rNpFMT-kX8,5746
2
2
  scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
3
- scrapely/actor.py,sha256=h9CW-aFgeoyc1_W_KnR3jN3F2koQWEf4omlvqWTBwZI,79521
3
+ scrapely/actor.py,sha256=1MXGyHql5sRfh15HMRY8vAp_TNr_yhavDALlHBcjLO0,80810
4
4
  scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
5
5
  scrapely/cdp_retry.py,sha256=hdTJEAIKhApM7IOk1DrQEl8NoflSJSmqd4RbNJ2IsgI,8540
6
6
  scrapely/charging.py,sha256=HQHDkJ9buhlgArmmAC0I4lON8v-B6jgLD594-MpB-Rs,68905
@@ -12,6 +12,7 @@ scrapely/patch_browser_pool.py,sha256=qZstdQ1rFw8kGjzB9fm3GXco8Jt40eklTw1Czf_vh-
12
12
  scrapely/pricing_types.py,sha256=J6wCddZd-U1MYeOtaaEdeHSsp1HBgiQLfvSInBOvhIo,5305
13
13
  scrapely/proxy_configuration.py,sha256=4passyiUBfBsuzaBDyf5003rY0ayLYwwSnev0FaxCs0,38125
14
14
  scrapely/py.typed,sha256=XtWlywqUjc6x_M2gAXlDtYXbAsYkCjKZEjFyJvaHvm4,105
15
+ scrapely/solver.py,sha256=ahuORJwZu9aJzCKWF14qTjQSnwT6dEbhMophQQ03JHw,35854
15
16
  scrapely/types.py,sha256=ceVuHvhYbPai2CL3vFmXLTMoGaMR27IefJFgWhzd7ws,8900
16
17
  scrapely/webhook.py,sha256=4WOM42ypOSOs4lKtP5ofxAh6lX-7c8p5U9lOfuFzx-g,2830
17
18
  scrapely/crypto/__init__.py,sha256=MbMPs7Eq_fGSVjOZxNts_JckDCUtxCVqdkHU0VCfN4M,481
@@ -38,10 +39,10 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9
38
39
  scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
39
40
  scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
40
41
  scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
41
- scrapely_python-0.1.15.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
42
- scrapely_python-0.1.15.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
43
- scrapely_python-0.1.15.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
44
- scrapely_python-0.1.15.dist-info/METADATA,sha256=lwSsoEdYgoZZoO7AWadWStjR0Aa6KOLDMtp2rZP4wnw,3487
45
- scrapely_python-0.1.15.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
46
- scrapely_python-0.1.15.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
47
- scrapely_python-0.1.15.dist-info/RECORD,,
42
+ scrapely_python-0.1.16.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
43
+ scrapely_python-0.1.16.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
44
+ scrapely_python-0.1.16.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
45
+ scrapely_python-0.1.16.dist-info/METADATA,sha256=wxPrakVU3_4JfSWaOgNs6hYKp3vkCbssBHR5OH9Jcoo,3487
46
+ scrapely_python-0.1.16.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
47
+ scrapely_python-0.1.16.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
48
+ scrapely_python-0.1.16.dist-info/RECORD,,