scraperack-turnstile 0.1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scraperack_turnstile-0.1.0a1/PKG-INFO +31 -0
- scraperack_turnstile-0.1.0a1/README.md +19 -0
- scraperack_turnstile-0.1.0a1/pyproject.toml +22 -0
- scraperack_turnstile-0.1.0a1/scraperack_turnstile/__init__.py +6 -0
- scraperack_turnstile-0.1.0a1/scraperack_turnstile/client.py +289 -0
- scraperack_turnstile-0.1.0a1/scraperack_turnstile/py.typed +0 -0
- scraperack_turnstile-0.1.0a1/scraperack_turnstile.egg-info/PKG-INFO +31 -0
- scraperack_turnstile-0.1.0a1/scraperack_turnstile.egg-info/SOURCES.txt +11 -0
- scraperack_turnstile-0.1.0a1/scraperack_turnstile.egg-info/dependency_links.txt +1 -0
- scraperack_turnstile-0.1.0a1/scraperack_turnstile.egg-info/requires.txt +4 -0
- scraperack_turnstile-0.1.0a1/scraperack_turnstile.egg-info/top_level.txt +1 -0
- scraperack_turnstile-0.1.0a1/setup.cfg +4 -0
- scraperack_turnstile-0.1.0a1/tests/test_client.py +117 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scraperack-turnstile
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: Pools of live Cloudflare bundles (proxy, User-Agent, cf_clearance or Turnstile token) from a self-hosted Scraperack cluster's turnstile-solver plugin
|
|
5
|
+
License-Expression: LicenseRef-Proprietary
|
|
6
|
+
Classifier: Development Status :: 3 - Alpha
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: httpx>=0.27
|
|
10
|
+
Provides-Extra: curl
|
|
11
|
+
Requires-Dist: curl_cffi>=0.7; extra == "curl"
|
|
12
|
+
|
|
13
|
+
# scraperack-turnstile
|
|
14
|
+
|
|
15
|
+
Pools of live Cloudflare **bundles** from a Scraperack cluster's turnstile-solver plugin. A
|
|
16
|
+
bundle is a proxy, a User-Agent with client hints, and a `cf_clearance` cookie or a Turnstile
|
|
17
|
+
token, all solved together in one browser. Pull one and make requests with it.
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
from scraperack_turnstile import Turnstile
|
|
21
|
+
|
|
22
|
+
with Turnstile("http://controller:7070", key="ts_...") as ts:
|
|
23
|
+
ts.put_pool("shop", "https://shop.example.com/", size=5)
|
|
24
|
+
bundle = ts.pull("shop", wait=120)
|
|
25
|
+
with bundle.curl_cffi_session() as http: # pip install scraperack-turnstile[curl]
|
|
26
|
+
page = http.get("https://shop.example.com/products")
|
|
27
|
+
if page.status_code == 403:
|
|
28
|
+
ts.report(bundle, "challenged")
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
See [docs/turnstile-solver-sdk.md](../../docs/turnstile-solver-sdk.md).
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# scraperack-turnstile
|
|
2
|
+
|
|
3
|
+
Pools of live Cloudflare **bundles** from a Scraperack cluster's turnstile-solver plugin. A
|
|
4
|
+
bundle is a proxy, a User-Agent with client hints, and a `cf_clearance` cookie or a Turnstile
|
|
5
|
+
token, all solved together in one browser. Pull one and make requests with it.
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
from scraperack_turnstile import Turnstile
|
|
9
|
+
|
|
10
|
+
with Turnstile("http://controller:7070", key="ts_...") as ts:
|
|
11
|
+
ts.put_pool("shop", "https://shop.example.com/", size=5)
|
|
12
|
+
bundle = ts.pull("shop", wait=120)
|
|
13
|
+
with bundle.curl_cffi_session() as http: # pip install scraperack-turnstile[curl]
|
|
14
|
+
page = http.get("https://shop.example.com/products")
|
|
15
|
+
if page.status_code == 403:
|
|
16
|
+
ts.report(bundle, "challenged")
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
See [docs/turnstile-solver-sdk.md](../../docs/turnstile-solver-sdk.md).
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "scraperack-turnstile"
|
|
7
|
+
version = "0.1.0a1"
|
|
8
|
+
description = "Pools of live Cloudflare bundles (proxy, User-Agent, cf_clearance or Turnstile token) from a self-hosted Scraperack cluster's turnstile-solver plugin"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
dependencies = ["httpx>=0.27"]
|
|
12
|
+
license = "LicenseRef-Proprietary"
|
|
13
|
+
classifiers = ["Development Status :: 3 - Alpha"]
|
|
14
|
+
|
|
15
|
+
[project.optional-dependencies]
|
|
16
|
+
curl = ["curl_cffi>=0.7"]
|
|
17
|
+
|
|
18
|
+
[tool.setuptools]
|
|
19
|
+
packages = ["scraperack_turnstile"]
|
|
20
|
+
|
|
21
|
+
[tool.setuptools.package-data]
|
|
22
|
+
scraperack_turnstile = ["py.typed"]
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""Pools of live Cloudflare bundles on Scraperack, through the turnstile-solver plugin."""
|
|
2
|
+
from .client import (AsyncTurnstile, Bundle, BundleNotFound, PoolEmpty, PoolNotFound, Turnstile,
|
|
3
|
+
TurnstileError)
|
|
4
|
+
|
|
5
|
+
__all__ = ["AsyncTurnstile", "Bundle", "BundleNotFound", "PoolEmpty", "PoolNotFound", "Turnstile",
|
|
6
|
+
"TurnstileError"]
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
"""Clients for the Scraperack turnstile-solver plugin: pools of live Cloudflare bundles.
|
|
2
|
+
|
|
3
|
+
A **bundle** is everything needed to make requests the way the browser that solved the
|
|
4
|
+
challenge did: the proxy (a clearance only works from the IP that earned it), the User-Agent
|
|
5
|
+
and client hints, and the ``cf_clearance`` cookie or a Turnstile widget token. Pulling a
|
|
6
|
+
bundle takes it out of its pool; the pool solves a replacement on its own.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import time
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from datetime import datetime, timezone
|
|
14
|
+
from typing import Any, Literal
|
|
15
|
+
from urllib.parse import quote
|
|
16
|
+
|
|
17
|
+
import httpx
|
|
18
|
+
|
|
19
|
+
DEFAULT_URL = "http://127.0.0.1:7070"
|
|
20
|
+
Kind = Literal["clearance", "turnstile"]
|
|
21
|
+
Reason = Literal["challenged", "blocked", "proxy_dead"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# --- Errors ----------------------------------------------------------------------------------
|
|
25
|
+
|
|
26
|
+
class TurnstileError(RuntimeError):
|
|
27
|
+
"""The plugin answered with an error. ``code`` is its machine-readable kind."""
|
|
28
|
+
|
|
29
|
+
def __init__(self, status: int, code: str, message: str, data: dict[str, Any] | None = None) -> None:
|
|
30
|
+
super().__init__(f"{status} {code}: {message}")
|
|
31
|
+
self.status, self.code, self.message, self.data = status, code, message, data or {}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class PoolEmpty(TurnstileError):
|
|
35
|
+
"""No bundle is ready yet. ``retry_after``: seconds until one probably is."""
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def retry_after(self) -> int:
|
|
39
|
+
value = self.data.get("retry_after")
|
|
40
|
+
return 5 if value is None else max(0, int(value))
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class PoolNotFound(TurnstileError):
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class BundleNotFound(TurnstileError):
|
|
48
|
+
pass
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
ERRORS = {"pool_empty": PoolEmpty, "pool_not_found": PoolNotFound, "bundle_not_found": BundleNotFound}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _check(response: httpx.Response) -> Any:
|
|
55
|
+
if response.status_code >= 400:
|
|
56
|
+
try:
|
|
57
|
+
data = response.json()
|
|
58
|
+
except ValueError:
|
|
59
|
+
data = {"error": response.text or response.reason_phrase}
|
|
60
|
+
code = data.get("code") or "error"
|
|
61
|
+
raise ERRORS.get(code, TurnstileError)(response.status_code, code, data.get("error", ""), data)
|
|
62
|
+
return response.json() if response.content else None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _name(value: str) -> str:
|
|
66
|
+
return quote(str(value), safe="")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# --- Bundles ---------------------------------------------------------------------------------
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class Bundle:
|
|
73
|
+
"""A pulled bundle. Send every request with its ``proxy``, ``headers`` and ``cookies``."""
|
|
74
|
+
|
|
75
|
+
id: str
|
|
76
|
+
pool: str
|
|
77
|
+
kind: str
|
|
78
|
+
url: str
|
|
79
|
+
proxy: str | None
|
|
80
|
+
ip: str | None
|
|
81
|
+
country: str | None
|
|
82
|
+
timezone: str | None
|
|
83
|
+
user_agent: str
|
|
84
|
+
headers: dict[str, str]
|
|
85
|
+
cookie_list: list[dict[str, Any]]
|
|
86
|
+
cookie_header: str
|
|
87
|
+
token: str | None
|
|
88
|
+
challenged: bool
|
|
89
|
+
created_at_ms: int
|
|
90
|
+
expires_at_ms: int
|
|
91
|
+
raw: dict[str, Any] = field(repr=False, default_factory=dict)
|
|
92
|
+
|
|
93
|
+
@classmethod
|
|
94
|
+
def from_api(cls, data: dict[str, Any]) -> "Bundle":
|
|
95
|
+
return cls(id=data["id"], pool=data["pool"], kind=data["kind"], url=data["url"], proxy=data.get("proxy"),
|
|
96
|
+
ip=data.get("ip"), country=data.get("country"), timezone=data.get("timezone"),
|
|
97
|
+
user_agent=data["user_agent"], headers=dict(data.get("headers") or {}),
|
|
98
|
+
cookie_list=list(data.get("cookies") or []), cookie_header=data.get("cookie_header") or "",
|
|
99
|
+
token=data.get("token"), challenged=bool(data.get("challenged")),
|
|
100
|
+
created_at_ms=data["created_at_ms"], expires_at_ms=data["expires_at_ms"], raw=data)
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def cookies(self) -> dict[str, str]:
|
|
104
|
+
"""``{name: value}``, ready for any HTTP client."""
|
|
105
|
+
return {c["name"]: c["value"] for c in self.cookie_list}
|
|
106
|
+
|
|
107
|
+
@property
|
|
108
|
+
def proxies(self) -> dict[str, str] | None:
|
|
109
|
+
"""The proxy in the ``{"http": ..., "https": ...}`` form requests and curl_cffi take."""
|
|
110
|
+
return {"http": self.proxy, "https": self.proxy} if self.proxy else None
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def expires_at(self) -> datetime:
|
|
114
|
+
return datetime.fromtimestamp(self.expires_at_ms / 1000, timezone.utc)
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def seconds_left(self) -> float:
|
|
118
|
+
return max(0.0, self.expires_at_ms / 1000 - time.time())
|
|
119
|
+
|
|
120
|
+
@property
|
|
121
|
+
def expired(self) -> bool:
|
|
122
|
+
return self.seconds_left <= 0
|
|
123
|
+
|
|
124
|
+
def curl_cffi_session(self, impersonate: str = "chrome", **kwargs: Any) -> Any:
|
|
125
|
+
"""A ``curl_cffi`` session set up as this bundle: proxy, headers, cookies, and Chrome's
|
|
126
|
+
TLS fingerprint (Cloudflare checks it; plain ``requests`` looks like Python).
|
|
127
|
+
Needs ``pip install scraperack-turnstile[curl]``."""
|
|
128
|
+
from curl_cffi import requests as curl_requests
|
|
129
|
+
session = curl_requests.Session(impersonate=impersonate, **kwargs)
|
|
130
|
+
session.headers.update(self.headers)
|
|
131
|
+
for cookie in self.cookie_list:
|
|
132
|
+
session.cookies.set(cookie["name"], cookie["value"], domain=cookie.get("domain", ""),
|
|
133
|
+
path=cookie.get("path", "/"))
|
|
134
|
+
if self.proxy:
|
|
135
|
+
session.proxies = self.proxies
|
|
136
|
+
return session
|
|
137
|
+
|
|
138
|
+
def httpx_client(self, **kwargs: Any) -> httpx.Client:
|
|
139
|
+
"""An ``httpx.Client`` set up as this bundle. Its TLS fingerprint is not Chrome's, so
|
|
140
|
+
prefer :meth:`curl_cffi_session` where Cloudflare checks it. SOCKS proxies need
|
|
141
|
+
``httpx[socks]``."""
|
|
142
|
+
return httpx.Client(proxy=self.proxy, headers=self.headers, cookies=self.cookies, **kwargs)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# --- Pools -------------------------------------------------------------------------------------
|
|
146
|
+
|
|
147
|
+
def pool_body(url: str, kind: Kind, size: int, parallel: int, proxy: str, country: str | list[str] | None,
|
|
148
|
+
max_age: int) -> dict[str, Any]:
|
|
149
|
+
if isinstance(country, (list, tuple)):
|
|
150
|
+
country = ",".join(country)
|
|
151
|
+
return {"url": url, "kind": kind, "size": size, "parallel": parallel, "proxy": proxy,
|
|
152
|
+
"country": country or "", "max_age": max_age}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _headers(key: str | None) -> dict[str, str]:
|
|
156
|
+
return {"Authorization": f"Bearer {key}"} if key else {}
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class Turnstile:
|
|
160
|
+
"""Synchronous client.
|
|
161
|
+
|
|
162
|
+
>>> with Turnstile("http://controller:7070", key="ts_...") as ts:
|
|
163
|
+
... ts.put_pool("shop", "https://shop.example.com/", size=5)
|
|
164
|
+
... bundle = ts.pull("shop", wait=120)
|
|
165
|
+
"""
|
|
166
|
+
|
|
167
|
+
def __init__(self, url: str = DEFAULT_URL, key: str | None = None, *, plugin: str = "turnstile-solver",
|
|
168
|
+
client: httpx.Client | None = None, timeout: float = 30) -> None:
|
|
169
|
+
self.url = url.rstrip("/")
|
|
170
|
+
self.api = f"/plugins/{_name(plugin)}/api"
|
|
171
|
+
self._owns_client = client is None
|
|
172
|
+
self.client = client or httpx.Client(base_url=self.url, timeout=timeout)
|
|
173
|
+
self.client.headers.update(_headers(key))
|
|
174
|
+
|
|
175
|
+
def __enter__(self) -> "Turnstile":
|
|
176
|
+
return self
|
|
177
|
+
|
|
178
|
+
def __exit__(self, *_: object) -> None:
|
|
179
|
+
self.close()
|
|
180
|
+
|
|
181
|
+
def close(self) -> None:
|
|
182
|
+
if self._owns_client:
|
|
183
|
+
self.client.close()
|
|
184
|
+
|
|
185
|
+
def _call(self, method: str, path: str, **kwargs: Any) -> Any:
|
|
186
|
+
return _check(self.client.request(method, self.api + path, **kwargs))
|
|
187
|
+
|
|
188
|
+
def pools(self) -> list[dict[str, Any]]:
|
|
189
|
+
"""Every pool with its counts: ``ready``, ``solving``, ``status``, ``stats``..."""
|
|
190
|
+
return self._call("GET", "/pools")
|
|
191
|
+
|
|
192
|
+
def pool(self, name: str) -> dict[str, Any]:
|
|
193
|
+
"""One pool, with its ready bundles listed (without their secrets)."""
|
|
194
|
+
return self._call("GET", f"/pools/{_name(name)}")
|
|
195
|
+
|
|
196
|
+
def put_pool(self, name: str, url: str, *, size: int = 1, kind: Kind = "clearance", parallel: int = 1,
|
|
197
|
+
proxy: str = "proxies", country: str | list[str] | None = None,
|
|
198
|
+
max_age: int = 1800) -> dict[str, Any]:
|
|
199
|
+
"""Create or change a pool that keeps ``size`` bundles ready for ``url``.
|
|
200
|
+
|
|
201
|
+
``proxy`` is ``"proxies"`` (anonymous proxies from the proxies plugin) or
|
|
202
|
+
``"direct"``. Changing ``url``, ``kind``, ``proxy`` or ``country`` drops the pool's
|
|
203
|
+
bundles; changing only the numbers keeps them.
|
|
204
|
+
"""
|
|
205
|
+
return self._call("PUT", f"/pools/{_name(name)}",
|
|
206
|
+
json=pool_body(url, kind, size, parallel, proxy, country, max_age))
|
|
207
|
+
|
|
208
|
+
def delete_pool(self, name: str) -> dict[str, Any]:
|
|
209
|
+
return self._call("DELETE", f"/pools/{_name(name)}")
|
|
210
|
+
|
|
211
|
+
def pull(self, pool: str, *, wait: float = 0) -> Bundle:
|
|
212
|
+
"""Take a ready bundle out of ``pool``.
|
|
213
|
+
|
|
214
|
+
With ``wait``, keep trying for up to that many seconds while the pool is empty
|
|
215
|
+
(following the plugin's ``retry_after``); then raise :class:`PoolEmpty`.
|
|
216
|
+
"""
|
|
217
|
+
deadline = time.monotonic() + wait
|
|
218
|
+
while True:
|
|
219
|
+
try:
|
|
220
|
+
return Bundle.from_api(self._call("POST", f"/pools/{_name(pool)}/pull"))
|
|
221
|
+
except PoolEmpty as empty:
|
|
222
|
+
left = deadline - time.monotonic()
|
|
223
|
+
if left <= 0:
|
|
224
|
+
raise
|
|
225
|
+
time.sleep(min(empty.retry_after, left, 10))
|
|
226
|
+
|
|
227
|
+
def report(self, bundle: Bundle | str, reason: Reason = "challenged") -> dict[str, Any]:
|
|
228
|
+
"""Tell the plugin a bundle stopped working.
|
|
229
|
+
|
|
230
|
+
``challenged`` / ``blocked``: the site turned that IP away (the pool avoids it for a
|
|
231
|
+
while). ``proxy_dead``: the proxy doesn't answer (every bundle on it goes).
|
|
232
|
+
"""
|
|
233
|
+
bundle_id = bundle.id if isinstance(bundle, Bundle) else bundle
|
|
234
|
+
return self._call("POST", "/reports", json={"id": bundle_id, "reason": reason})
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class AsyncTurnstile:
|
|
238
|
+
"""Asynchronous client; the same methods as :class:`Turnstile`, awaited."""
|
|
239
|
+
|
|
240
|
+
def __init__(self, url: str = DEFAULT_URL, key: str | None = None, *, plugin: str = "turnstile-solver",
|
|
241
|
+
client: httpx.AsyncClient | None = None, timeout: float = 30) -> None:
|
|
242
|
+
self.url = url.rstrip("/")
|
|
243
|
+
self.api = f"/plugins/{_name(plugin)}/api"
|
|
244
|
+
self._owns_client = client is None
|
|
245
|
+
self.client = client or httpx.AsyncClient(base_url=self.url, timeout=timeout)
|
|
246
|
+
self.client.headers.update(_headers(key))
|
|
247
|
+
|
|
248
|
+
async def __aenter__(self) -> "AsyncTurnstile":
|
|
249
|
+
return self
|
|
250
|
+
|
|
251
|
+
async def __aexit__(self, *_: object) -> None:
|
|
252
|
+
await self.aclose()
|
|
253
|
+
|
|
254
|
+
async def aclose(self) -> None:
|
|
255
|
+
if self._owns_client:
|
|
256
|
+
await self.client.aclose()
|
|
257
|
+
|
|
258
|
+
async def _call(self, method: str, path: str, **kwargs: Any) -> Any:
|
|
259
|
+
return _check(await self.client.request(method, self.api + path, **kwargs))
|
|
260
|
+
|
|
261
|
+
async def pools(self) -> list[dict[str, Any]]:
|
|
262
|
+
return await self._call("GET", "/pools")
|
|
263
|
+
|
|
264
|
+
async def pool(self, name: str) -> dict[str, Any]:
|
|
265
|
+
return await self._call("GET", f"/pools/{_name(name)}")
|
|
266
|
+
|
|
267
|
+
async def put_pool(self, name: str, url: str, *, size: int = 1, kind: Kind = "clearance", parallel: int = 1,
|
|
268
|
+
proxy: str = "proxies", country: str | list[str] | None = None,
|
|
269
|
+
max_age: int = 1800) -> dict[str, Any]:
|
|
270
|
+
return await self._call("PUT", f"/pools/{_name(name)}",
|
|
271
|
+
json=pool_body(url, kind, size, parallel, proxy, country, max_age))
|
|
272
|
+
|
|
273
|
+
async def delete_pool(self, name: str) -> dict[str, Any]:
|
|
274
|
+
return await self._call("DELETE", f"/pools/{_name(name)}")
|
|
275
|
+
|
|
276
|
+
async def pull(self, pool: str, *, wait: float = 0) -> Bundle:
|
|
277
|
+
deadline = time.monotonic() + wait
|
|
278
|
+
while True:
|
|
279
|
+
try:
|
|
280
|
+
return Bundle.from_api(await self._call("POST", f"/pools/{_name(pool)}/pull"))
|
|
281
|
+
except PoolEmpty as empty:
|
|
282
|
+
left = deadline - time.monotonic()
|
|
283
|
+
if left <= 0:
|
|
284
|
+
raise
|
|
285
|
+
await asyncio.sleep(min(empty.retry_after, left, 10))
|
|
286
|
+
|
|
287
|
+
async def report(self, bundle: Bundle | str, reason: Reason = "challenged") -> dict[str, Any]:
|
|
288
|
+
bundle_id = bundle.id if isinstance(bundle, Bundle) else bundle
|
|
289
|
+
return await self._call("POST", "/reports", json={"id": bundle_id, "reason": reason})
|
|
File without changes
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scraperack-turnstile
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: Pools of live Cloudflare bundles (proxy, User-Agent, cf_clearance or Turnstile token) from a self-hosted Scraperack cluster's turnstile-solver plugin
|
|
5
|
+
License-Expression: LicenseRef-Proprietary
|
|
6
|
+
Classifier: Development Status :: 3 - Alpha
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: httpx>=0.27
|
|
10
|
+
Provides-Extra: curl
|
|
11
|
+
Requires-Dist: curl_cffi>=0.7; extra == "curl"
|
|
12
|
+
|
|
13
|
+
# scraperack-turnstile
|
|
14
|
+
|
|
15
|
+
Pools of live Cloudflare **bundles** from a Scraperack cluster's turnstile-solver plugin. A
|
|
16
|
+
bundle is a proxy, a User-Agent with client hints, and a `cf_clearance` cookie or a Turnstile
|
|
17
|
+
token, all solved together in one browser. Pull one and make requests with it.
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
from scraperack_turnstile import Turnstile
|
|
21
|
+
|
|
22
|
+
with Turnstile("http://controller:7070", key="ts_...") as ts:
|
|
23
|
+
ts.put_pool("shop", "https://shop.example.com/", size=5)
|
|
24
|
+
bundle = ts.pull("shop", wait=120)
|
|
25
|
+
with bundle.curl_cffi_session() as http: # pip install scraperack-turnstile[curl]
|
|
26
|
+
page = http.get("https://shop.example.com/products")
|
|
27
|
+
if page.status_code == 403:
|
|
28
|
+
ts.report(bundle, "challenged")
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
See [docs/turnstile-solver-sdk.md](../../docs/turnstile-solver-sdk.md).
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
scraperack_turnstile/__init__.py
|
|
4
|
+
scraperack_turnstile/client.py
|
|
5
|
+
scraperack_turnstile/py.typed
|
|
6
|
+
scraperack_turnstile.egg-info/PKG-INFO
|
|
7
|
+
scraperack_turnstile.egg-info/SOURCES.txt
|
|
8
|
+
scraperack_turnstile.egg-info/dependency_links.txt
|
|
9
|
+
scraperack_turnstile.egg-info/requires.txt
|
|
10
|
+
scraperack_turnstile.egg-info/top_level.txt
|
|
11
|
+
tests/test_client.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
scraperack_turnstile
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
import sys
|
|
4
|
+
import time
|
|
5
|
+
import unittest
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
|
|
10
|
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
11
|
+
from scraperack_turnstile import (AsyncTurnstile, Bundle, BundleNotFound, PoolEmpty, PoolNotFound, Turnstile,
|
|
12
|
+
TurnstileError)
|
|
13
|
+
|
|
14
|
+
API = "/plugins/turnstile-solver/api"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def bundle(**extra):
|
|
18
|
+
now = int(time.time() * 1000)
|
|
19
|
+
return {"id": "bd_1", "pool": "shop", "kind": "clearance", "url": "https://shop.example.com/",
|
|
20
|
+
"proxy": "http://1.2.3.4:8080", "ip": "1.2.3.4", "country": "DE", "timezone": "Europe/Berlin",
|
|
21
|
+
"user_agent": "UA", "headers": {"User-Agent": "UA", "sec-ch-ua": '"Chromium";v="158"'},
|
|
22
|
+
"cookies": [{"name": "cf_clearance", "value": "abc", "domain": ".shop.example.com", "path": "/"}],
|
|
23
|
+
"cookie_header": "cf_clearance=abc", "token": None, "challenged": True,
|
|
24
|
+
"created_at_ms": now, "expires_at_ms": now + 1_800_000, **extra}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def client(handler, key="ts_key"):
|
|
28
|
+
return Turnstile("http://controller:7070", key, client=httpx.Client(
|
|
29
|
+
base_url="http://controller:7070", transport=httpx.MockTransport(handler)))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class ClientTests(unittest.TestCase):
|
|
33
|
+
def test_put_pool_sends_the_contract_and_the_key(self):
|
|
34
|
+
seen = []
|
|
35
|
+
|
|
36
|
+
def handler(request):
|
|
37
|
+
seen.append(request)
|
|
38
|
+
return httpx.Response(201, json={"name": "shop", "ready": 0})
|
|
39
|
+
|
|
40
|
+
client(handler).put_pool("shop", "https://shop.example.com/", size=5, parallel=2, country=["DE", "NL"])
|
|
41
|
+
request = seen[0]
|
|
42
|
+
self.assertEqual((request.method, request.url.path), ("PUT", f"{API}/pools/shop"))
|
|
43
|
+
self.assertEqual(request.headers["Authorization"], "Bearer ts_key")
|
|
44
|
+
self.assertEqual(json.loads(request.content), {
|
|
45
|
+
"url": "https://shop.example.com/", "kind": "clearance", "size": 5, "parallel": 2,
|
|
46
|
+
"proxy": "proxies", "country": "DE,NL", "max_age": 1800})
|
|
47
|
+
|
|
48
|
+
def test_pull_returns_a_bundle(self):
|
|
49
|
+
got = client(lambda r: httpx.Response(200, json=bundle())).pull("shop")
|
|
50
|
+
self.assertIsInstance(got, Bundle)
|
|
51
|
+
self.assertEqual(got.cookies, {"cf_clearance": "abc"})
|
|
52
|
+
self.assertEqual(got.proxies, {"http": "http://1.2.3.4:8080", "https": "http://1.2.3.4:8080"})
|
|
53
|
+
self.assertFalse(got.expired)
|
|
54
|
+
self.assertGreater(got.seconds_left, 1700)
|
|
55
|
+
self.assertEqual(got.headers["User-Agent"], "UA")
|
|
56
|
+
direct = Bundle.from_api(bundle(proxy=None))
|
|
57
|
+
self.assertIsNone(direct.proxies)
|
|
58
|
+
|
|
59
|
+
def test_pull_waits_while_the_pool_fills(self):
|
|
60
|
+
answers = iter([httpx.Response(503, json={"error": "empty", "code": "pool_empty", "retry_after": 0}),
|
|
61
|
+
httpx.Response(200, json=bundle())])
|
|
62
|
+
self.assertEqual(client(lambda r: next(answers)).pull("shop", wait=5).id, "bd_1")
|
|
63
|
+
|
|
64
|
+
def test_errors_carry_their_code(self):
|
|
65
|
+
empty = {"error": "pool shop has no ready bundle", "code": "pool_empty", "retry_after": 30}
|
|
66
|
+
with self.assertRaises(PoolEmpty) as raised:
|
|
67
|
+
client(lambda r: httpx.Response(503, json=empty)).pull("shop")
|
|
68
|
+
self.assertEqual((raised.exception.retry_after, raised.exception.code), (30, "pool_empty"))
|
|
69
|
+
with self.assertRaises(PoolNotFound):
|
|
70
|
+
client(lambda r: httpx.Response(404, json={"error": "x", "code": "pool_not_found"})).pull("nope")
|
|
71
|
+
with self.assertRaises(BundleNotFound):
|
|
72
|
+
client(lambda r: httpx.Response(404, json={"error": "x", "code": "bundle_not_found"})).report("bd_x")
|
|
73
|
+
with self.assertRaises(TurnstileError) as raised:
|
|
74
|
+
client(lambda r: httpx.Response(401, json={"error": "unknown API key", "code": "unauthorized"})).pools()
|
|
75
|
+
self.assertEqual(raised.exception.status, 401)
|
|
76
|
+
|
|
77
|
+
def test_report_takes_a_bundle_or_an_id(self):
|
|
78
|
+
bodies = []
|
|
79
|
+
|
|
80
|
+
def handler(request):
|
|
81
|
+
bodies.append(json.loads(request.content))
|
|
82
|
+
return httpx.Response(200, json={"bundles_dropped": 0})
|
|
83
|
+
|
|
84
|
+
ts = client(handler)
|
|
85
|
+
ts.report(Bundle.from_api(bundle()), "proxy_dead")
|
|
86
|
+
ts.report("bd_2")
|
|
87
|
+
self.assertEqual(bodies, [{"id": "bd_1", "reason": "proxy_dead"}, {"id": "bd_2", "reason": "challenged"}])
|
|
88
|
+
|
|
89
|
+
def test_httpx_client_is_set_up_as_the_bundle(self):
|
|
90
|
+
with Bundle.from_api(bundle(proxy=None)).httpx_client() as http:
|
|
91
|
+
self.assertEqual(http.headers["User-Agent"], "UA")
|
|
92
|
+
self.assertEqual(http.cookies["cf_clearance"], "abc")
|
|
93
|
+
|
|
94
|
+
def test_curl_cffi_session_is_set_up_as_the_bundle(self):
|
|
95
|
+
try:
|
|
96
|
+
import curl_cffi # noqa: F401
|
|
97
|
+
except ImportError:
|
|
98
|
+
self.skipTest("curl_cffi is not installed")
|
|
99
|
+
session = Bundle.from_api(bundle()).curl_cffi_session()
|
|
100
|
+
self.assertEqual(session.headers["User-Agent"], "UA")
|
|
101
|
+
self.assertEqual(session.cookies.get("cf_clearance"), "abc")
|
|
102
|
+
self.assertEqual(session.proxies["https"], "http://1.2.3.4:8080")
|
|
103
|
+
session.close()
|
|
104
|
+
|
|
105
|
+
def test_async_client(self):
|
|
106
|
+
async def main():
|
|
107
|
+
answers = iter([httpx.Response(503, json={"error": "e", "code": "pool_empty", "retry_after": 0}),
|
|
108
|
+
httpx.Response(200, json=bundle())])
|
|
109
|
+
transport = httpx.MockTransport(lambda r: next(answers))
|
|
110
|
+
async with AsyncTurnstile("http://c:7070", client=httpx.AsyncClient(
|
|
111
|
+
base_url="http://c:7070", transport=transport)) as ts:
|
|
112
|
+
return await ts.pull("shop", wait=5)
|
|
113
|
+
self.assertEqual(asyncio.run(main()).id, "bd_1")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
if __name__ == "__main__":
|
|
117
|
+
unittest.main()
|