scrapely-python 0.1.15__py3-none-any.whl → 0.1.16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapely/__init__.py +1 -1
- scrapely/actor.py +28 -0
- scrapely/solver.py +875 -0
- {scrapely_python-0.1.15.dist-info → scrapely_python-0.1.16.dist-info}/METADATA +1 -1
- {scrapely_python-0.1.15.dist-info → scrapely_python-0.1.16.dist-info}/RECORD +10 -9
- {scrapely_python-0.1.15.dist-info → scrapely_python-0.1.16.dist-info}/WHEEL +0 -0
- {scrapely_python-0.1.15.dist-info → scrapely_python-0.1.16.dist-info}/licenses/LICENSE +0 -0
- {scrapely_python-0.1.15.dist-info → scrapely_python-0.1.16.dist-info}/licenses/LICENSE-APACHE +0 -0
- {scrapely_python-0.1.15.dist-info → scrapely_python-0.1.16.dist-info}/licenses/NOTICE +0 -0
- {scrapely_python-0.1.15.dist-info → scrapely_python-0.1.16.dist-info}/top_level.txt +0 -0
scrapely/__init__.py
CHANGED
scrapely/actor.py
CHANGED
|
@@ -1080,6 +1080,34 @@ class Actor(metaclass=_ActorMeta):
|
|
|
1080
1080
|
status_message=data.get('statusMessage'),
|
|
1081
1081
|
)
|
|
1082
1082
|
|
|
1083
|
+
@classmethod
|
|
1084
|
+
async def solver_status(cls, vendor: str) -> Dict[str, Any]:
|
|
1085
|
+
"""Anti-bot solver status for a vendor (free, cached for 15 s, never raises).
|
|
1086
|
+
|
|
1087
|
+
``usable`` False means the vendor cannot be solved right now: take your
|
|
1088
|
+
fallback (for example the hosted browser), or fail the items that need
|
|
1089
|
+
a solve at once and call ``set_solver_degraded_status()``. See
|
|
1090
|
+
``scrapely.solver``.
|
|
1091
|
+
"""
|
|
1092
|
+
from .solver import solver_status_async
|
|
1093
|
+
|
|
1094
|
+
return await solver_status_async(vendor)
|
|
1095
|
+
|
|
1096
|
+
@classmethod
|
|
1097
|
+
async def set_solver_degraded_status(cls, skipped: Optional[int] = None, total: Optional[int] = None) -> None:
|
|
1098
|
+
"""Set the standard status message for a run whose site protection cannot be solved right now.
|
|
1099
|
+
|
|
1100
|
+
"Site protection temporarily unavailable, results may be incomplete",
|
|
1101
|
+
with "(N of M items skipped)" when both counts are given. Never raises:
|
|
1102
|
+
the run goes on.
|
|
1103
|
+
"""
|
|
1104
|
+
from .solver import degraded_status_message
|
|
1105
|
+
|
|
1106
|
+
try:
|
|
1107
|
+
await cls.set_status_message(degraded_status_message(skipped, total))
|
|
1108
|
+
except Exception as exc: # noqa: BLE001 - a status message must never stop the run
|
|
1109
|
+
logger.warning(f'Could not set the solver status message: {exc}')
|
|
1110
|
+
|
|
1083
1111
|
@classmethod
|
|
1084
1112
|
async def _put_run_status_message(
|
|
1085
1113
|
cls, message: str, *, is_terminal: bool
|
scrapely/solver.py
ADDED
|
@@ -0,0 +1,875 @@
|
|
|
1
|
+
"""Anti-bot solver helpers for Scrapely Actors.
|
|
2
|
+
|
|
3
|
+
The platform runs the challenge solver (xhr.dev today) behind
|
|
4
|
+
``/v2/solver``. Actors never hold a solver key: the platform injects
|
|
5
|
+
``SCRAPELY_SOLVER_URL`` and ``SCRAPELY_SOLVER_API_KEY`` (the run's own token)
|
|
6
|
+
into runs of Actors that may use it.
|
|
7
|
+
|
|
8
|
+
Three layers, use what you need:
|
|
9
|
+
|
|
10
|
+
* ``solver_status(vendor)`` / ``await Actor.solver_status(vendor)``: a free,
|
|
11
|
+
cached status check. When a vendor is ``down`` take your fallback (for
|
|
12
|
+
example the hosted browser) or fail the items that need a solve at once.
|
|
13
|
+
* ``Solver``: solves one challenge on your own HTTP session. The server tells
|
|
14
|
+
the SDK which requests to send (challenge document, scripts, submission);
|
|
15
|
+
they go out through *your* session so the clearance is bound to your exit IP.
|
|
16
|
+
Cookies land in the session's jar; the clearance is verified and reported.
|
|
17
|
+
* ``SolvePolicy``: retries on fresh proxy sessions with backoff and a breaker
|
|
18
|
+
(the policy the solver Actors already use), plus ``VendorUnavailable``,
|
|
19
|
+
which is raised at once, without retries, when the platform says the
|
|
20
|
+
vendor is down.
|
|
21
|
+
|
|
22
|
+
Typical use::
|
|
23
|
+
|
|
24
|
+
from scrapely.solver import Solver, SolvePolicy, SolveFailed, VendorUnavailable
|
|
25
|
+
|
|
26
|
+
solver = Solver(vendor="datadome", accept_language="es-ES,es;q=0.9", timezone="Europe/Madrid")
|
|
27
|
+
policy = SolvePolicy()
|
|
28
|
+
|
|
29
|
+
def attempt(i):
|
|
30
|
+
session = new_session(i) # fresh exit per try
|
|
31
|
+
r = session.get(url)
|
|
32
|
+
if solver.is_blocked(r):
|
|
33
|
+
solver.solve(session, url, r, session_id=f"s{i}")
|
|
34
|
+
r = session.get(url)
|
|
35
|
+
return r
|
|
36
|
+
|
|
37
|
+
try:
|
|
38
|
+
page = policy.run(attempt, label=url)
|
|
39
|
+
except VendorUnavailable:
|
|
40
|
+
... # vendor down: fail this item at once (or use your fallback)
|
|
41
|
+
except SolveFailed:
|
|
42
|
+
... # every try failed: record a per-item error, keep going
|
|
43
|
+
|
|
44
|
+
When a vendor is down and the Actor has no fallback, the run must still
|
|
45
|
+
start and finish: call ``await Actor.set_solver_degraded_status()`` once, fail
|
|
46
|
+
the items that need a solve, keep everything else.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
from __future__ import annotations
|
|
50
|
+
|
|
51
|
+
import asyncio
|
|
52
|
+
import inspect
|
|
53
|
+
import json
|
|
54
|
+
import os
|
|
55
|
+
import random
|
|
56
|
+
import re
|
|
57
|
+
import threading
|
|
58
|
+
import time
|
|
59
|
+
from dataclasses import dataclass, field
|
|
60
|
+
from typing import Any, Awaitable, Callable, Optional
|
|
61
|
+
|
|
62
|
+
import httpx
|
|
63
|
+
|
|
64
|
+
__all__ = [
|
|
65
|
+
"Clearance",
|
|
66
|
+
"DEGRADED_STATUS_MESSAGE",
|
|
67
|
+
"IpBanned",
|
|
68
|
+
"SolveAttemptError",
|
|
69
|
+
"SolveFailed",
|
|
70
|
+
"SolvePolicy",
|
|
71
|
+
"Solver",
|
|
72
|
+
"SolverAuthError",
|
|
73
|
+
"SolverRateLimited",
|
|
74
|
+
"SolverRequestInvalid",
|
|
75
|
+
"SolverUnavailable",
|
|
76
|
+
"VendorUnavailable",
|
|
77
|
+
"check_solver_response",
|
|
78
|
+
"degraded_status_message",
|
|
79
|
+
"detect_vendor",
|
|
80
|
+
"is_blocked",
|
|
81
|
+
"solver_env",
|
|
82
|
+
"solver_status",
|
|
83
|
+
"solver_status_async",
|
|
84
|
+
]
|
|
85
|
+
|
|
86
|
+
DEGRADED_STATUS_MESSAGE = "Site protection temporarily unavailable, results may be incomplete"
|
|
87
|
+
|
|
88
|
+
# Codes the platform answers with when a vendor cannot be solved right now.
|
|
89
|
+
UNAVAILABLE_CODES = ("vendor_degraded", "vendor_down", "provider_unavailable", "vendor_unsupported", "solver_not_enabled")
|
|
90
|
+
|
|
91
|
+
_COMPAT_SUFFIX = "/compat/xhrdev"
|
|
92
|
+
_STATUS_TTL_SECS = 15.0
|
|
93
|
+
_MAX_BODY = 4 * 1024 * 1024
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# ─── environment ─────────────────────────────────────────────────────────────
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def solver_env() -> tuple[str, str] | None:
|
|
100
|
+
"""Solver URL and key (the run token), injected by the platform. None when not set.
|
|
101
|
+
|
|
102
|
+
Never hardcode the host or the key, and never log the key.
|
|
103
|
+
"""
|
|
104
|
+
url = (os.environ.get("SCRAPELY_SOLVER_URL") or "").strip().rstrip("/")
|
|
105
|
+
key = (os.environ.get("SCRAPELY_SOLVER_API_KEY") or "").strip()
|
|
106
|
+
if not url or not key:
|
|
107
|
+
return None
|
|
108
|
+
return url, key
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _gateway() -> tuple[str, str] | None:
|
|
112
|
+
"""Base of the platform solver API (…/v2/solver) and the run token, or None.
|
|
113
|
+
|
|
114
|
+
Only available when the platform serves the solver (the URL ends with the
|
|
115
|
+
compat route); a raw solver URL means the old direct mode.
|
|
116
|
+
"""
|
|
117
|
+
env = solver_env()
|
|
118
|
+
if env is None:
|
|
119
|
+
return None
|
|
120
|
+
url, key = env
|
|
121
|
+
if not url.endswith(_COMPAT_SUFFIX):
|
|
122
|
+
return None
|
|
123
|
+
return url[: -len(_COMPAT_SUFFIX)], key
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# ─── errors ──────────────────────────────────────────────────────────────────
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class SolveAttemptError(RuntimeError):
|
|
130
|
+
"""One try failed. Worth another try on a fresh exit."""
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class IpBanned(SolveAttemptError):
|
|
134
|
+
"""The address is refused outright. Only a new exit helps."""
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class SolverRateLimited(SolveAttemptError):
|
|
138
|
+
"""The solver answered 429. Not retried for this item."""
|
|
139
|
+
|
|
140
|
+
def __init__(self, retry_after: float | None = None) -> None:
|
|
141
|
+
self.retry_after = retry_after
|
|
142
|
+
suffix = f", retry after {retry_after:g}s" if retry_after is not None else ""
|
|
143
|
+
super().__init__(f"solver rate limit hit (HTTP 429{suffix})")
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class SolverRequestInvalid(SolveAttemptError):
|
|
147
|
+
"""The solver rejected the request itself (HTTP 400). Not retried."""
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class SolverAuthError(SolveAttemptError):
|
|
151
|
+
"""The solver refused the key (HTTP 401/403). Opens the breaker."""
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class SolveFailed(RuntimeError):
|
|
155
|
+
"""Every try for one item failed."""
|
|
156
|
+
|
|
157
|
+
def __init__(self, message: str, attempts: int, last_error: BaseException | None = None) -> None:
|
|
158
|
+
super().__init__(message)
|
|
159
|
+
self.attempts = attempts
|
|
160
|
+
self.last_error = last_error
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class SolverUnavailable(SolveFailed):
|
|
164
|
+
"""The breaker is open: the solver failed for everything recently."""
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
class VendorUnavailable(SolverUnavailable):
|
|
168
|
+
"""The platform says this vendor cannot be solved right now (fail fast, no retries).
|
|
169
|
+
|
|
170
|
+
``code`` is one of ``vendor_down``, ``vendor_degraded``,
|
|
171
|
+
``provider_unavailable``, ``vendor_unsupported`` or ``solver_not_enabled``.
|
|
172
|
+
"""
|
|
173
|
+
|
|
174
|
+
def __init__(self, code: str, vendor: str | None = None, retry_after: float | None = None, message: str | None = None) -> None:
|
|
175
|
+
self.code = code
|
|
176
|
+
self.vendor = vendor
|
|
177
|
+
self.retry_after = retry_after
|
|
178
|
+
super().__init__(message or f"anti-bot solver unavailable ({code}{f', {vendor}' if vendor else ''})", 0)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _json(body: str) -> dict:
|
|
182
|
+
try:
|
|
183
|
+
parsed = json.loads(body or "")
|
|
184
|
+
return parsed if isinstance(parsed, dict) else {}
|
|
185
|
+
except (TypeError, ValueError):
|
|
186
|
+
return {}
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def check_solver_response(status: int, body: str = "", retry_after: str | None = None) -> None:
|
|
190
|
+
"""Raise the right error for a non-2xx solver answer; return on 2xx."""
|
|
191
|
+
if 200 <= status < 300:
|
|
192
|
+
return
|
|
193
|
+
detail = (body or "")[:200].replace("\n", " ")
|
|
194
|
+
seconds: float | None = None
|
|
195
|
+
try:
|
|
196
|
+
seconds = float(retry_after) if retry_after is not None else None
|
|
197
|
+
except (TypeError, ValueError):
|
|
198
|
+
seconds = None
|
|
199
|
+
if status == 503:
|
|
200
|
+
parsed = _json(body)
|
|
201
|
+
code = str(parsed.get("code") or "")
|
|
202
|
+
if code in UNAVAILABLE_CODES:
|
|
203
|
+
raise VendorUnavailable(code, parsed.get("vendor"), seconds if seconds is not None else parsed.get("retryAfterSecs"))
|
|
204
|
+
if status == 429:
|
|
205
|
+
raise SolverRateLimited(seconds)
|
|
206
|
+
if status in (401, 403):
|
|
207
|
+
parsed = _json(body)
|
|
208
|
+
if parsed.get("code") == "solver_not_enabled":
|
|
209
|
+
raise VendorUnavailable("solver_not_enabled", None, None, "the anti-bot solver is not enabled for this Actor")
|
|
210
|
+
raise SolverAuthError(f"solver refused the key (HTTP {status})")
|
|
211
|
+
if status == 400:
|
|
212
|
+
raise SolverRequestInvalid(f"solver rejected the request (HTTP 400): {detail}")
|
|
213
|
+
raise SolveAttemptError(f"solver HTTP {status}: {detail}")
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
_NOT_RETRIED = (SolverRateLimited, SolverRequestInvalid, SolverAuthError)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
# ─── retry policy ────────────────────────────────────────────────────────────
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
class SolvePolicy:
|
|
223
|
+
"""Up to `attempts` tries per item, each on a fresh proxy session, with backoff and a breaker.
|
|
224
|
+
|
|
225
|
+
* HTTP 5xx, timeouts, transport errors, empty solutions and a page still
|
|
226
|
+
blocked after the clearance are failed tries;
|
|
227
|
+
* HTTP 429 ends the tries for that item;
|
|
228
|
+
* HTTP 401/403 opens the breaker at once;
|
|
229
|
+
* after `trip_after` consecutive failed tries the breaker opens and
|
|
230
|
+
further solves fail fast with SolverUnavailable; after `cooldown`
|
|
231
|
+
seconds one probe is let through;
|
|
232
|
+
* VendorUnavailable (the platform says the vendor is down) is raised at
|
|
233
|
+
once, without retries and without touching the breaker.
|
|
234
|
+
"""
|
|
235
|
+
|
|
236
|
+
def __init__(
|
|
237
|
+
self,
|
|
238
|
+
attempts: int = 4,
|
|
239
|
+
base_delay: float = 1.5,
|
|
240
|
+
max_delay: float = 15.0,
|
|
241
|
+
trip_after: int = 12,
|
|
242
|
+
cooldown: float = 180.0,
|
|
243
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
244
|
+
async_sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
|
245
|
+
clock: Callable[[], float] = time.monotonic,
|
|
246
|
+
rng: Callable[[], float] = random.random,
|
|
247
|
+
) -> None:
|
|
248
|
+
self.attempts = max(1, min(5, attempts))
|
|
249
|
+
self.base_delay = base_delay
|
|
250
|
+
self.max_delay = max_delay
|
|
251
|
+
self.trip_after = max(1, trip_after)
|
|
252
|
+
self.cooldown = cooldown
|
|
253
|
+
self._sleep = sleep
|
|
254
|
+
self._async_sleep = async_sleep
|
|
255
|
+
self._clock = clock
|
|
256
|
+
self._rng = rng
|
|
257
|
+
self._lock = threading.Lock()
|
|
258
|
+
self.consecutive_failures = 0
|
|
259
|
+
self.opened_at: float | None = None
|
|
260
|
+
self.calls = 0
|
|
261
|
+
self.solved = 0
|
|
262
|
+
self.failed_items = 0
|
|
263
|
+
self.unavailable_items = 0
|
|
264
|
+
self.last_error = ""
|
|
265
|
+
|
|
266
|
+
@property
|
|
267
|
+
def down(self) -> bool:
|
|
268
|
+
with self._lock:
|
|
269
|
+
return self.opened_at is not None and self._clock() - self.opened_at < self.cooldown
|
|
270
|
+
|
|
271
|
+
@property
|
|
272
|
+
def ever_tripped(self) -> bool:
|
|
273
|
+
return self.opened_at is not None
|
|
274
|
+
|
|
275
|
+
def _admit(self) -> None:
|
|
276
|
+
with self._lock:
|
|
277
|
+
if self.opened_at is None:
|
|
278
|
+
return
|
|
279
|
+
if self._clock() - self.opened_at < self.cooldown:
|
|
280
|
+
raise SolverUnavailable(
|
|
281
|
+
f"challenge solver unavailable after {self.trip_after} consecutive failed solves"
|
|
282
|
+
+ (f" (last error: {self.last_error})" if self.last_error else ""),
|
|
283
|
+
0,
|
|
284
|
+
)
|
|
285
|
+
self.opened_at = self._clock()
|
|
286
|
+
self.consecutive_failures = self.trip_after - 1
|
|
287
|
+
|
|
288
|
+
def _success(self) -> None:
|
|
289
|
+
with self._lock:
|
|
290
|
+
self.calls += 1
|
|
291
|
+
self.solved += 1
|
|
292
|
+
self.consecutive_failures = 0
|
|
293
|
+
self.opened_at = None
|
|
294
|
+
|
|
295
|
+
def _failure(self, exc: BaseException) -> None:
|
|
296
|
+
with self._lock:
|
|
297
|
+
self.calls += 1
|
|
298
|
+
self.consecutive_failures += 1
|
|
299
|
+
self.last_error = _short(exc)
|
|
300
|
+
if isinstance(exc, SolverAuthError) or self.consecutive_failures >= self.trip_after:
|
|
301
|
+
self.opened_at = self._clock()
|
|
302
|
+
|
|
303
|
+
def _unavailable(self) -> None:
|
|
304
|
+
with self._lock:
|
|
305
|
+
self.unavailable_items += 1
|
|
306
|
+
|
|
307
|
+
def delay(self, index: int) -> float:
|
|
308
|
+
raw = min(self.max_delay, self.base_delay * (2 ** index))
|
|
309
|
+
return raw * (0.5 + self._rng())
|
|
310
|
+
|
|
311
|
+
def run(self, attempt: Callable[[int], Any], label: str = "page") -> Any:
|
|
312
|
+
"""Call attempt(i) until it returns. Sync version (for worker threads)."""
|
|
313
|
+
last: BaseException | None = None
|
|
314
|
+
for index in range(self.attempts):
|
|
315
|
+
self._admit()
|
|
316
|
+
try:
|
|
317
|
+
result = attempt(index)
|
|
318
|
+
except VendorUnavailable:
|
|
319
|
+
self._unavailable()
|
|
320
|
+
raise
|
|
321
|
+
except Exception as exc: # noqa: BLE001 - every failure is one failed try
|
|
322
|
+
last = exc
|
|
323
|
+
self._failure(exc)
|
|
324
|
+
if isinstance(exc, _NOT_RETRIED):
|
|
325
|
+
break
|
|
326
|
+
if index + 1 < self.attempts and not self.down:
|
|
327
|
+
self._sleep(self.delay(index))
|
|
328
|
+
continue
|
|
329
|
+
self._success()
|
|
330
|
+
return result
|
|
331
|
+
return self._give_up(label, last)
|
|
332
|
+
|
|
333
|
+
async def run_async(self, attempt: Callable[[int], Any], label: str = "page") -> Any:
|
|
334
|
+
"""Async version. attempt(i) may be a coroutine function or a plain one."""
|
|
335
|
+
last: BaseException | None = None
|
|
336
|
+
for index in range(self.attempts):
|
|
337
|
+
self._admit()
|
|
338
|
+
try:
|
|
339
|
+
result = attempt(index)
|
|
340
|
+
if inspect.isawaitable(result):
|
|
341
|
+
result = await result
|
|
342
|
+
except asyncio.CancelledError:
|
|
343
|
+
raise
|
|
344
|
+
except VendorUnavailable:
|
|
345
|
+
self._unavailable()
|
|
346
|
+
raise
|
|
347
|
+
except Exception as exc: # noqa: BLE001
|
|
348
|
+
last = exc
|
|
349
|
+
self._failure(exc)
|
|
350
|
+
if isinstance(exc, _NOT_RETRIED):
|
|
351
|
+
break
|
|
352
|
+
if index + 1 < self.attempts and not self.down:
|
|
353
|
+
await self._async_sleep(self.delay(index))
|
|
354
|
+
continue
|
|
355
|
+
self._success()
|
|
356
|
+
return result
|
|
357
|
+
return self._give_up(label, last)
|
|
358
|
+
|
|
359
|
+
def _give_up(self, label: str, last: BaseException | None) -> Any:
|
|
360
|
+
with self._lock:
|
|
361
|
+
self.failed_items += 1
|
|
362
|
+
if isinstance(last, SolverUnavailable):
|
|
363
|
+
raise last
|
|
364
|
+
raise SolveFailed(f"anti-bot challenge on {label} was not solved: {_short(last)}", self.attempts, last)
|
|
365
|
+
|
|
366
|
+
def summary(self) -> dict[str, int]:
|
|
367
|
+
return {
|
|
368
|
+
"solverCalls": self.calls,
|
|
369
|
+
"solved": self.solved,
|
|
370
|
+
"failedItems": self.failed_items,
|
|
371
|
+
"unavailableItems": self.unavailable_items,
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _short(exc: BaseException | None) -> str:
|
|
376
|
+
if exc is None:
|
|
377
|
+
return "unknown error"
|
|
378
|
+
text = str(exc) or type(exc).__name__
|
|
379
|
+
return text[:200]
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
# ─── detection ───────────────────────────────────────────────────────────────
|
|
383
|
+
|
|
384
|
+
_DD_RE = re.compile(r"var\s+dd\s*=\s*\{")
|
|
385
|
+
_SBSD_RE = re.compile(r'<script[^>]+src="([^"]+\?v=[0-9a-f-]{36}[^"]*)"', re.I)
|
|
386
|
+
_PX_RE = re.compile(r"_pxappid|px-captcha|perimeterx|/_px\d?/", re.I)
|
|
387
|
+
_KASADA_RE = re.compile(r"kpsdk", re.I)
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def detect_vendor(status: int, body: str) -> str | None:
|
|
391
|
+
"""The protection behind a *challenge* response: datadome, akamai-sbsd, perimeterx, kasada or None.
|
|
392
|
+
|
|
393
|
+
Ordinary pages that merely load a vendor's script are not challenges:
|
|
394
|
+
DataDome, PerimeterX and Kasada need an error status (4xx), and an SBSD
|
|
395
|
+
bundle counts only on a short interstitial page.
|
|
396
|
+
"""
|
|
397
|
+
body = body or ""
|
|
398
|
+
sample = body[:20000]
|
|
399
|
+
head = sample[:4000].lower()
|
|
400
|
+
challenged = status >= 400
|
|
401
|
+
if challenged and ("captcha-delivery.com" in head or _DD_RE.search(sample) or "datadome" in head):
|
|
402
|
+
return "datadome"
|
|
403
|
+
if len(body) <= 20000 and _SBSD_RE.search(sample):
|
|
404
|
+
return "akamai-sbsd"
|
|
405
|
+
if challenged and _PX_RE.search(sample):
|
|
406
|
+
return "perimeterx"
|
|
407
|
+
if challenged and _KASADA_RE.search(sample):
|
|
408
|
+
return "kasada"
|
|
409
|
+
return None
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _status_body(response: Any) -> tuple[int, str]:
|
|
413
|
+
if isinstance(response, tuple):
|
|
414
|
+
return int(response[0]), str(response[1] or "")
|
|
415
|
+
if isinstance(response, dict):
|
|
416
|
+
return int(response.get("status") or 0), str(response.get("body") or "")
|
|
417
|
+
status = getattr(response, "status_code", None)
|
|
418
|
+
if status is None:
|
|
419
|
+
status = getattr(response, "status", 0)
|
|
420
|
+
text = getattr(response, "text", "")
|
|
421
|
+
if callable(text): # aiohttp-style
|
|
422
|
+
text = ""
|
|
423
|
+
return int(status or 0), str(text or "")
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def is_blocked(response: Any, vendor: str | None = None) -> bool:
|
|
427
|
+
"""True when the response is an anti-bot challenge (optionally of one vendor)."""
|
|
428
|
+
status, body = _status_body(response)
|
|
429
|
+
found = detect_vendor(status, body)
|
|
430
|
+
return found is not None if vendor in (None, "auto") else found == vendor
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def degraded_status_message(skipped: int | None = None, total: int | None = None) -> str:
|
|
434
|
+
"""The standard status message for runs whose site protection cannot be solved right now."""
|
|
435
|
+
if skipped is not None and total is not None:
|
|
436
|
+
return f"{DEGRADED_STATUS_MESSAGE} ({skipped} of {total} items skipped)"
|
|
437
|
+
return DEGRADED_STATUS_MESSAGE
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
# ─── status ──────────────────────────────────────────────────────────────────
|
|
441
|
+
|
|
442
|
+
_status_cache: dict[str, tuple[float, dict]] = {}
|
|
443
|
+
_status_lock = threading.Lock()
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _unknown_status(code: str | None = None) -> dict:
|
|
447
|
+
# Without the platform gateway nothing is known: let the Actor proceed as before.
|
|
448
|
+
return {"status": "unknown", "usable": code is None, "code": code, "successRate30m": None, "since": None, "recheckAfterSecs": 300}
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _status_from_answer(vendor: str, status: int, body: str) -> dict:
|
|
452
|
+
if status == 200:
|
|
453
|
+
data = _json(body)
|
|
454
|
+
v = (data.get("vendors") or {}).get(vendor)
|
|
455
|
+
if isinstance(v, dict):
|
|
456
|
+
return v
|
|
457
|
+
return _unknown_status()
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def _cached(vendor: str) -> dict | None:
|
|
461
|
+
with _status_lock:
|
|
462
|
+
hit = _status_cache.get(vendor)
|
|
463
|
+
if hit and time.monotonic() - hit[0] < _STATUS_TTL_SECS:
|
|
464
|
+
return hit[1]
|
|
465
|
+
return None
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _store(vendor: str, value: dict) -> dict:
|
|
469
|
+
with _status_lock:
|
|
470
|
+
_status_cache[vendor] = (time.monotonic(), value)
|
|
471
|
+
return value
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def solver_status(vendor: str, timeout: float = 10.0) -> dict:
|
|
475
|
+
"""Vendor status from the platform (free, cached for 15 s). Never raises.
|
|
476
|
+
|
|
477
|
+
Returns a dict with ``status`` (healthy, degraded, down, unsupported or
|
|
478
|
+
unknown), ``usable`` (False means: take your fallback or fail the items
|
|
479
|
+
that need a solve), ``code``, ``successRate30m``, ``since`` and
|
|
480
|
+
``recheckAfterSecs``.
|
|
481
|
+
"""
|
|
482
|
+
vendor = (vendor or "").strip().lower()
|
|
483
|
+
hit = _cached(vendor)
|
|
484
|
+
if hit is not None:
|
|
485
|
+
return hit
|
|
486
|
+
gw = _gateway()
|
|
487
|
+
if gw is None:
|
|
488
|
+
return _unknown_status(None if solver_env() else "solver_not_enabled")
|
|
489
|
+
base, key = gw
|
|
490
|
+
try:
|
|
491
|
+
r = httpx.get(f"{base}/status", params={"vendors": vendor}, headers={"authorization": f"Bearer {key}"}, timeout=timeout)
|
|
492
|
+
return _store(vendor, _status_from_answer(vendor, r.status_code, r.text))
|
|
493
|
+
except Exception: # noqa: BLE001 - a failed check must never stop the run
|
|
494
|
+
return _unknown_status()
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
async def solver_status_async(vendor: str, timeout: float = 10.0) -> dict:
|
|
498
|
+
"""Async ``solver_status``."""
|
|
499
|
+
vendor = (vendor or "").strip().lower()
|
|
500
|
+
hit = _cached(vendor)
|
|
501
|
+
if hit is not None:
|
|
502
|
+
return hit
|
|
503
|
+
gw = _gateway()
|
|
504
|
+
if gw is None:
|
|
505
|
+
return _unknown_status(None if solver_env() else "solver_not_enabled")
|
|
506
|
+
base, key = gw
|
|
507
|
+
try:
|
|
508
|
+
async with httpx.AsyncClient(timeout=timeout) as client:
|
|
509
|
+
r = await client.get(f"{base}/status", params={"vendors": vendor}, headers={"authorization": f"Bearer {key}"})
|
|
510
|
+
return _store(vendor, _status_from_answer(vendor, r.status_code, r.text))
|
|
511
|
+
except Exception: # noqa: BLE001
|
|
512
|
+
return _unknown_status()
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
# ─── solving ─────────────────────────────────────────────────────────────────
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
@dataclass
|
|
519
|
+
class Clearance:
|
|
520
|
+
"""A solved challenge: cookies already set on the session, and the verification response."""
|
|
521
|
+
|
|
522
|
+
vendor: str
|
|
523
|
+
solve_id: str | None
|
|
524
|
+
cookies: list[dict] = field(default_factory=list)
|
|
525
|
+
ttl_hint_secs: int = 0
|
|
526
|
+
cached: bool = False
|
|
527
|
+
verified: bool | None = None
|
|
528
|
+
response: Any = None
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _is_httpx(session: Any) -> bool:
|
|
532
|
+
return type(session).__module__.split(".")[0] == "httpx"
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _cookie_applies(cookie: Any, host: str, path: str, secure: bool, now: float) -> bool:
|
|
536
|
+
"""RFC 6265 matching: the cookies the session itself would send to this URL."""
|
|
537
|
+
domain = (getattr(cookie, "domain", "") or "").lower()
|
|
538
|
+
if not domain:
|
|
539
|
+
return False
|
|
540
|
+
bare = domain.lstrip(".")
|
|
541
|
+
host_only = not domain.startswith(".") and not getattr(cookie, "domain_specified", False)
|
|
542
|
+
if host_only:
|
|
543
|
+
if host != bare:
|
|
544
|
+
return False
|
|
545
|
+
elif host != bare and not host.endswith("." + bare):
|
|
546
|
+
return False
|
|
547
|
+
cpath = getattr(cookie, "path", "/") or "/"
|
|
548
|
+
if not (path == cpath or path.startswith(cpath.rstrip("/") + "/") or cpath == "/"):
|
|
549
|
+
return False
|
|
550
|
+
if getattr(cookie, "secure", False) and not secure:
|
|
551
|
+
return False
|
|
552
|
+
expires = getattr(cookie, "expires", None)
|
|
553
|
+
return not (expires is not None and expires <= now)
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _cookie_header(session: Any, url: str) -> str | None:
|
|
557
|
+
"""The Cookie header the session would send to `url` (host, path, secure and expiry rules)."""
|
|
558
|
+
u = httpx.URL(url)
|
|
559
|
+
host = (u.host or "").lower()
|
|
560
|
+
path = u.path or "/"
|
|
561
|
+
secure = u.scheme == "https"
|
|
562
|
+
jar = getattr(getattr(session, "cookies", None), "jar", None)
|
|
563
|
+
pairs: list[str] = []
|
|
564
|
+
now = time.time()
|
|
565
|
+
try:
|
|
566
|
+
for c in jar or []:
|
|
567
|
+
if _cookie_applies(c, host, path, secure, now):
|
|
568
|
+
pairs.append(f"{c.name}={c.value}")
|
|
569
|
+
except Exception: # noqa: BLE001
|
|
570
|
+
return None
|
|
571
|
+
return "; ".join(pairs) or None
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
def _set_cookies(session: Any, cookies: list[dict]) -> None:
|
|
575
|
+
jar = getattr(session, "cookies", None)
|
|
576
|
+
if jar is None:
|
|
577
|
+
return
|
|
578
|
+
for c in cookies:
|
|
579
|
+
try:
|
|
580
|
+
jar.set(c["name"], c["value"], domain=c.get("domain") or "", path=c.get("path") or "/")
|
|
581
|
+
except Exception: # noqa: BLE001 - a jar without domain support
|
|
582
|
+
jar.set(c["name"], c["value"])
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def _step_allowed(step_url: str, target_url: str) -> bool:
|
|
586
|
+
"""A step may only go to the target page's own host or to captcha-delivery.com, over https."""
|
|
587
|
+
try:
|
|
588
|
+
step = httpx.URL(step_url)
|
|
589
|
+
target = httpx.URL(target_url)
|
|
590
|
+
except Exception: # noqa: BLE001
|
|
591
|
+
return False
|
|
592
|
+
if step.scheme != "https":
|
|
593
|
+
return False
|
|
594
|
+
host = (step.host or "").lower()
|
|
595
|
+
if host == (target.host or "").lower():
|
|
596
|
+
return True
|
|
597
|
+
return host == "captcha-delivery.com" or host.endswith(".captcha-delivery.com")
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _checked(req: dict, target_url: str) -> dict:
|
|
601
|
+
if not isinstance(req, dict) or not _step_allowed(str(req.get("url") or ""), target_url):
|
|
602
|
+
raise SolveAttemptError("the solver asked for a request outside the target site; refused")
|
|
603
|
+
if req.get("method") not in ("GET", "POST"):
|
|
604
|
+
raise SolveAttemptError("the solver asked for an unsupported request method; refused")
|
|
605
|
+
return req
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def _send_sync(session: Any, req: dict, timeout: float) -> tuple[int, str]:
|
|
609
|
+
body = req.get("body")
|
|
610
|
+
kwargs: dict[str, Any] = {"headers": req.get("headers") or {}, "timeout": timeout}
|
|
611
|
+
if body is not None:
|
|
612
|
+
kwargs["content" if _is_httpx(session) else "data"] = body.encode() if isinstance(body, str) else body
|
|
613
|
+
r = session.request(req["method"], req["url"], **kwargs)
|
|
614
|
+
return _status_body(r)
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
async def _send_async(session: Any, req: dict, timeout: float) -> tuple[int, str]:
|
|
618
|
+
body = req.get("body")
|
|
619
|
+
kwargs: dict[str, Any] = {"headers": req.get("headers") or {}, "timeout": timeout}
|
|
620
|
+
if body is not None:
|
|
621
|
+
kwargs["content" if _is_httpx(session) else "data"] = body.encode() if isinstance(body, str) else body
|
|
622
|
+
r = await session.request(req["method"], req["url"], **kwargs)
|
|
623
|
+
return _status_body(r)
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
class Solver:
|
|
627
|
+
"""Solves anti-bot challenges on the Actor's own HTTP session through the platform.
|
|
628
|
+
|
|
629
|
+
Args:
|
|
630
|
+
vendor: ``"datadome"``, ``"akamai-sbsd"`` or ``"auto"`` (detected from the blocked response).
|
|
631
|
+
identity: the browser profile your TLS impersonation claims (``"chrome-146-macos"``
|
|
632
|
+
matches curl_cffi ``impersonate="chrome146"``).
|
|
633
|
+
accept_language / timezone / languages: locale of that browser (match the site and proxy country).
|
|
634
|
+
fetch: optional ``fetch(session, request) -> (status, body)`` to send the step requests
|
|
635
|
+
yourself (sync), or ``afetch`` for async.
|
|
636
|
+
request_timeout: seconds per step request on your session.
|
|
637
|
+
"""
|
|
638
|
+
|
|
639
|
+
def __init__(
|
|
640
|
+
self,
|
|
641
|
+
vendor: str = "auto",
|
|
642
|
+
identity: str = "chrome-146-macos",
|
|
643
|
+
accept_language: str | None = None,
|
|
644
|
+
timezone: str | None = None,
|
|
645
|
+
languages: str | None = None,
|
|
646
|
+
timeout_ms: int = 60_000,
|
|
647
|
+
request_timeout: float = 30.0,
|
|
648
|
+
fetch: Callable[[Any, dict], tuple[int, str]] | None = None,
|
|
649
|
+
afetch: Callable[[Any, dict], Awaitable[tuple[int, str]]] | None = None,
|
|
650
|
+
) -> None:
|
|
651
|
+
self.vendor = vendor
|
|
652
|
+
self.identity = {"profile": identity}
|
|
653
|
+
if accept_language:
|
|
654
|
+
self.identity["acceptLanguage"] = accept_language
|
|
655
|
+
if timezone:
|
|
656
|
+
self.identity["timezone"] = timezone
|
|
657
|
+
if languages:
|
|
658
|
+
self.identity["languages"] = languages
|
|
659
|
+
self.timeout_ms = timeout_ms
|
|
660
|
+
self.request_timeout = request_timeout
|
|
661
|
+
self._fetch = fetch
|
|
662
|
+
self._afetch = afetch
|
|
663
|
+
self.solves = 0
|
|
664
|
+
self.cleared = 0
|
|
665
|
+
|
|
666
|
+
@staticmethod
|
|
667
|
+
def configured() -> bool:
|
|
668
|
+
"""True when the platform solver API is available to this run."""
|
|
669
|
+
return _gateway() is not None
|
|
670
|
+
|
|
671
|
+
def is_blocked(self, response: Any) -> bool:
|
|
672
|
+
return is_blocked(response, None if self.vendor == "auto" else self.vendor)
|
|
673
|
+
|
|
674
|
+
# -- wire ------------------------------------------------------------------
|
|
675
|
+
|
|
676
|
+
def _start_payload(self, url: str, blocked: Any, session: Any, session_id: str | None) -> dict:
|
|
677
|
+
status, body = _status_body(blocked)
|
|
678
|
+
payload: dict[str, Any] = {
|
|
679
|
+
"vendor": self.vendor,
|
|
680
|
+
"url": url,
|
|
681
|
+
"blocked": {"status": status, "body": body[:_MAX_BODY]},
|
|
682
|
+
"identity": self.identity,
|
|
683
|
+
"timeoutMs": self.timeout_ms,
|
|
684
|
+
"session": {"cookieHeader": _cookie_header(session, url)},
|
|
685
|
+
}
|
|
686
|
+
if session_id:
|
|
687
|
+
payload["session"]["id"] = str(session_id)
|
|
688
|
+
return payload
|
|
689
|
+
|
|
690
|
+
@staticmethod
|
|
691
|
+
def _handle(status: int, text: str, headers: Any, vendor: str) -> dict:
|
|
692
|
+
data = _json(text)
|
|
693
|
+
if status == 503 and str(data.get("code") or "") in UNAVAILABLE_CODES:
|
|
694
|
+
raise VendorUnavailable(str(data["code"]), data.get("vendor") or vendor, data.get("retryAfterSecs"))
|
|
695
|
+
if status == 429:
|
|
696
|
+
ra = headers.get("retry-after") if headers is not None else None
|
|
697
|
+
raise SolverRateLimited(float(ra) if ra else data.get("retryAfterSecs"))
|
|
698
|
+
if status in (401, 403):
|
|
699
|
+
if data.get("code") == "solver_not_enabled":
|
|
700
|
+
raise VendorUnavailable("solver_not_enabled", vendor, None, "the anti-bot solver is not enabled for this Actor")
|
|
701
|
+
raise SolverAuthError(f"solver refused the run token (HTTP {status}, {data.get('code') or ''})")
|
|
702
|
+
if status == 404 and data.get("code") == "not_found":
|
|
703
|
+
raise SolveAttemptError("the solve expired; start again")
|
|
704
|
+
if status >= 400:
|
|
705
|
+
raise SolveAttemptError(f"solver HTTP {status}: {text[:160]}")
|
|
706
|
+
return data
|
|
707
|
+
|
|
708
|
+
@staticmethod
|
|
709
|
+
def _raise_failed(data: dict) -> None:
|
|
710
|
+
reason = str(data.get("reason") or "rejected")
|
|
711
|
+
message = str(data.get("message") or reason)
|
|
712
|
+
if reason == "ip_banned":
|
|
713
|
+
raise IpBanned(message)
|
|
714
|
+
if reason in ("no_challenge", "bad_request") and not data.get("retryable", True):
|
|
715
|
+
raise SolverRequestInvalid(message)
|
|
716
|
+
raise SolveAttemptError(message)
|
|
717
|
+
|
|
718
|
+
def _finish(self, session: Any, data: dict) -> Clearance:
|
|
719
|
+
cookies = data.get("cookies") or []
|
|
720
|
+
_set_cookies(session, cookies)
|
|
721
|
+
self.cleared += 1
|
|
722
|
+
return Clearance(
|
|
723
|
+
vendor=self.vendor,
|
|
724
|
+
solve_id=data.get("solveId"),
|
|
725
|
+
cookies=cookies,
|
|
726
|
+
ttl_hint_secs=int(data.get("ttlHintSecs") or 0),
|
|
727
|
+
cached=bool(data.get("cached")),
|
|
728
|
+
)
|
|
729
|
+
|
|
730
|
+
# -- sync ------------------------------------------------------------------
|
|
731
|
+
|
|
732
|
+
def solve(self, session: Any, url: str, blocked: Any, *, session_id: str | None = None, verify: Any = True) -> Clearance:
|
|
733
|
+
"""Solve the challenge in `blocked` (a response of `session` for `url`).
|
|
734
|
+
|
|
735
|
+
verify: True re-fetches `url` and checks it is no longer blocked (the
|
|
736
|
+
result is reported to the platform); a callable ``verify(session) ->
|
|
737
|
+
bool`` does your own check; False skips it (call ``report()`` later).
|
|
738
|
+
Raises SolveAttemptError / IpBanned (try again on a fresh exit),
|
|
739
|
+
VendorUnavailable (fail fast), SolverRateLimited, SolverAuthError.
|
|
740
|
+
"""
|
|
741
|
+
gw = _gateway()
|
|
742
|
+
if gw is None:
|
|
743
|
+
raise VendorUnavailable("solver_not_enabled", self.vendor, None, "the platform solver is not available to this run")
|
|
744
|
+
base, key = gw
|
|
745
|
+
headers = {"authorization": f"Bearer {key}"}
|
|
746
|
+
self.solves += 1
|
|
747
|
+
with httpx.Client(timeout=self.timeout_ms / 1000 + 30) as api:
|
|
748
|
+
r = api.post(f"{base}/solve", json=self._start_payload(url, blocked, session, session_id), headers=headers)
|
|
749
|
+
data = self._handle(r.status_code, r.text, r.headers, self.vendor)
|
|
750
|
+
for _ in range(8):
|
|
751
|
+
action = data.get("action")
|
|
752
|
+
if action == "done":
|
|
753
|
+
clearance = self._finish(session, data)
|
|
754
|
+
break
|
|
755
|
+
if action == "failed":
|
|
756
|
+
self._raise_failed(data)
|
|
757
|
+
if action not in ("fetch", "submit"):
|
|
758
|
+
raise SolveAttemptError(f"unexpected solver answer: {action}")
|
|
759
|
+
results = []
|
|
760
|
+
for req in data.get("requests") or []:
|
|
761
|
+
req = _checked(req, url)
|
|
762
|
+
try:
|
|
763
|
+
status, body = (self._fetch or (lambda s, q: _send_sync(s, q, self.request_timeout)))(session, req)
|
|
764
|
+
except Exception as exc: # noqa: BLE001
|
|
765
|
+
raise SolveAttemptError(f"challenge request failed: {type(exc).__name__}") from exc
|
|
766
|
+
results.append({"id": req["id"], "status": status, "body": body[:_MAX_BODY]})
|
|
767
|
+
r = api.post(
|
|
768
|
+
f"{base}/solve",
|
|
769
|
+
json={"solveId": data.get("solveId"), "results": results, "session": {"id": session_id, "cookieHeader": _cookie_header(session, url)}},
|
|
770
|
+
headers=headers,
|
|
771
|
+
)
|
|
772
|
+
data = self._handle(r.status_code, r.text, r.headers, self.vendor)
|
|
773
|
+
else:
|
|
774
|
+
raise SolveAttemptError("the solve did not finish")
|
|
775
|
+
if verify is False:
|
|
776
|
+
return clearance
|
|
777
|
+
ok, response = self._verify_sync(session, url, verify)
|
|
778
|
+
clearance.verified = ok
|
|
779
|
+
clearance.response = response
|
|
780
|
+
self._report_sync(api, base, headers, clearance.solve_id, ok)
|
|
781
|
+
if not ok:
|
|
782
|
+
raise SolveAttemptError("still blocked after the clearance was applied")
|
|
783
|
+
return clearance
|
|
784
|
+
|
|
785
|
+
def _verify_sync(self, session: Any, url: str, verify: Any) -> tuple[bool, Any]:
|
|
786
|
+
if callable(verify):
|
|
787
|
+
return bool(verify(session)), None
|
|
788
|
+
try:
|
|
789
|
+
r = session.get(url, timeout=self.request_timeout)
|
|
790
|
+
except Exception: # noqa: BLE001
|
|
791
|
+
return False, None
|
|
792
|
+
return not is_blocked(r), r
|
|
793
|
+
|
|
794
|
+
@staticmethod
|
|
795
|
+
def _report_sync(api: httpx.Client, base: str, headers: dict, solve_id: str | None, ok: bool) -> None:
|
|
796
|
+
if not solve_id:
|
|
797
|
+
return
|
|
798
|
+
try:
|
|
799
|
+
api.post(f"{base}/solves/{solve_id}/outcome", json={"verified": ok}, headers=headers)
|
|
800
|
+
except Exception: # noqa: BLE001 - reporting is best effort
|
|
801
|
+
pass
|
|
802
|
+
|
|
803
|
+
def report(self, clearance: Clearance, verified: bool) -> None:
|
|
804
|
+
"""Report your own verification of a clearance (after ``solve(..., verify=False)``)."""
|
|
805
|
+
gw = _gateway()
|
|
806
|
+
if gw is None or not clearance.solve_id:
|
|
807
|
+
return
|
|
808
|
+
base, key = gw
|
|
809
|
+
clearance.verified = verified
|
|
810
|
+
with httpx.Client(timeout=15) as api:
|
|
811
|
+
self._report_sync(api, base, {"authorization": f"Bearer {key}"}, clearance.solve_id, verified)
|
|
812
|
+
|
|
813
|
+
# -- async -----------------------------------------------------------------
|
|
814
|
+
|
|
815
|
+
async def solve_async(self, session: Any, url: str, blocked: Any, *, session_id: str | None = None, verify: Any = True) -> Clearance:
|
|
816
|
+
"""Async ``solve`` (curl_cffi AsyncSession, httpx.AsyncClient, or `afetch`)."""
|
|
817
|
+
gw = _gateway()
|
|
818
|
+
if gw is None:
|
|
819
|
+
raise VendorUnavailable("solver_not_enabled", self.vendor, None, "the platform solver is not available to this run")
|
|
820
|
+
base, key = gw
|
|
821
|
+
headers = {"authorization": f"Bearer {key}"}
|
|
822
|
+
self.solves += 1
|
|
823
|
+
async with httpx.AsyncClient(timeout=self.timeout_ms / 1000 + 30) as api:
|
|
824
|
+
r = await api.post(f"{base}/solve", json=self._start_payload(url, blocked, session, session_id), headers=headers)
|
|
825
|
+
data = self._handle(r.status_code, r.text, r.headers, self.vendor)
|
|
826
|
+
for _ in range(8):
|
|
827
|
+
action = data.get("action")
|
|
828
|
+
if action == "done":
|
|
829
|
+
clearance = self._finish(session, data)
|
|
830
|
+
break
|
|
831
|
+
if action == "failed":
|
|
832
|
+
self._raise_failed(data)
|
|
833
|
+
if action not in ("fetch", "submit"):
|
|
834
|
+
raise SolveAttemptError(f"unexpected solver answer: {action}")
|
|
835
|
+
results = []
|
|
836
|
+
for req in data.get("requests") or []:
|
|
837
|
+
req = _checked(req, url)
|
|
838
|
+
try:
|
|
839
|
+
if self._afetch is not None:
|
|
840
|
+
status, body = await self._afetch(session, req)
|
|
841
|
+
else:
|
|
842
|
+
status, body = await _send_async(session, req, self.request_timeout)
|
|
843
|
+
except Exception as exc: # noqa: BLE001
|
|
844
|
+
raise SolveAttemptError(f"challenge request failed: {type(exc).__name__}") from exc
|
|
845
|
+
results.append({"id": req["id"], "status": status, "body": body[:_MAX_BODY]})
|
|
846
|
+
r = await api.post(
|
|
847
|
+
f"{base}/solve",
|
|
848
|
+
json={"solveId": data.get("solveId"), "results": results, "session": {"id": session_id, "cookieHeader": _cookie_header(session, url)}},
|
|
849
|
+
headers=headers,
|
|
850
|
+
)
|
|
851
|
+
data = self._handle(r.status_code, r.text, r.headers, self.vendor)
|
|
852
|
+
else:
|
|
853
|
+
raise SolveAttemptError("the solve did not finish")
|
|
854
|
+
if verify is False:
|
|
855
|
+
return clearance
|
|
856
|
+
if callable(verify):
|
|
857
|
+
result = verify(session)
|
|
858
|
+
ok = bool(await result) if inspect.isawaitable(result) else bool(result)
|
|
859
|
+
response = None
|
|
860
|
+
else:
|
|
861
|
+
try:
|
|
862
|
+
response = await session.get(url, timeout=self.request_timeout)
|
|
863
|
+
ok = not is_blocked(response)
|
|
864
|
+
except Exception: # noqa: BLE001
|
|
865
|
+
ok, response = False, None
|
|
866
|
+
clearance.verified = ok
|
|
867
|
+
clearance.response = response
|
|
868
|
+
if clearance.solve_id:
|
|
869
|
+
try:
|
|
870
|
+
await api.post(f"{base}/solves/{clearance.solve_id}/outcome", json={"verified": ok}, headers=headers)
|
|
871
|
+
except Exception: # noqa: BLE001
|
|
872
|
+
pass
|
|
873
|
+
if not ok:
|
|
874
|
+
raise SolveAttemptError("still blocked after the clearance was applied")
|
|
875
|
+
return clearance
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
scrapely/__init__.py,sha256=
|
|
1
|
+
scrapely/__init__.py,sha256=liCCoe7VolXKiiQlJmaBGJt7DlkLFi904rNpFMT-kX8,5746
|
|
2
2
|
scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
|
|
3
|
-
scrapely/actor.py,sha256=
|
|
3
|
+
scrapely/actor.py,sha256=1MXGyHql5sRfh15HMRY8vAp_TNr_yhavDALlHBcjLO0,80810
|
|
4
4
|
scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
|
|
5
5
|
scrapely/cdp_retry.py,sha256=hdTJEAIKhApM7IOk1DrQEl8NoflSJSmqd4RbNJ2IsgI,8540
|
|
6
6
|
scrapely/charging.py,sha256=HQHDkJ9buhlgArmmAC0I4lON8v-B6jgLD594-MpB-Rs,68905
|
|
@@ -12,6 +12,7 @@ scrapely/patch_browser_pool.py,sha256=qZstdQ1rFw8kGjzB9fm3GXco8Jt40eklTw1Czf_vh-
|
|
|
12
12
|
scrapely/pricing_types.py,sha256=J6wCddZd-U1MYeOtaaEdeHSsp1HBgiQLfvSInBOvhIo,5305
|
|
13
13
|
scrapely/proxy_configuration.py,sha256=4passyiUBfBsuzaBDyf5003rY0ayLYwwSnev0FaxCs0,38125
|
|
14
14
|
scrapely/py.typed,sha256=XtWlywqUjc6x_M2gAXlDtYXbAsYkCjKZEjFyJvaHvm4,105
|
|
15
|
+
scrapely/solver.py,sha256=ahuORJwZu9aJzCKWF14qTjQSnwT6dEbhMophQQ03JHw,35854
|
|
15
16
|
scrapely/types.py,sha256=ceVuHvhYbPai2CL3vFmXLTMoGaMR27IefJFgWhzd7ws,8900
|
|
16
17
|
scrapely/webhook.py,sha256=4WOM42ypOSOs4lKtP5ofxAh6lX-7c8p5U9lOfuFzx-g,2830
|
|
17
18
|
scrapely/crypto/__init__.py,sha256=MbMPs7Eq_fGSVjOZxNts_JckDCUtxCVqdkHU0VCfN4M,481
|
|
@@ -38,10 +39,10 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9
|
|
|
38
39
|
scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
|
|
39
40
|
scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
|
|
40
41
|
scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
|
|
41
|
-
scrapely_python-0.1.
|
|
42
|
-
scrapely_python-0.1.
|
|
43
|
-
scrapely_python-0.1.
|
|
44
|
-
scrapely_python-0.1.
|
|
45
|
-
scrapely_python-0.1.
|
|
46
|
-
scrapely_python-0.1.
|
|
47
|
-
scrapely_python-0.1.
|
|
42
|
+
scrapely_python-0.1.16.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
|
|
43
|
+
scrapely_python-0.1.16.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
|
|
44
|
+
scrapely_python-0.1.16.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
|
|
45
|
+
scrapely_python-0.1.16.dist-info/METADATA,sha256=wxPrakVU3_4JfSWaOgNs6hYKp3vkCbssBHR5OH9Jcoo,3487
|
|
46
|
+
scrapely_python-0.1.16.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
47
|
+
scrapely_python-0.1.16.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
|
|
48
|
+
scrapely_python-0.1.16.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
{scrapely_python-0.1.15.dist-info → scrapely_python-0.1.16.dist-info}/licenses/LICENSE-APACHE
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|