http-tls 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {http_tls-0.2.1 → http_tls-0.3.0}/Cargo.lock +38 -7
- {http_tls-0.2.1 → http_tls-0.3.0}/Cargo.toml +5 -2
- http_tls-0.2.1/README.md → http_tls-0.3.0/PKG-INFO +91 -1
- http_tls-0.2.1/PKG-INFO → http_tls-0.3.0/README.md +72 -16
- {http_tls-0.2.1 → http_tls-0.3.0}/pyproject.toml +9 -1
- {http_tls-0.2.1 → http_tls-0.3.0}/python/http_tls/__init__.py +3 -3
- {http_tls-0.2.1 → http_tls-0.3.0}/python/http_tls/__init__.pyi +50 -24
- http_tls-0.3.0/python/http_tls/challenge.py +82 -0
- http_tls-0.3.0/python/http_tls/requests.py +766 -0
- http_tls-0.3.0/src/lib.rs +844 -0
- http_tls-0.2.1/python/http_tls/requests.py +0 -546
- http_tls-0.2.1/src/lib.rs +0 -591
- {http_tls-0.2.1 → http_tls-0.3.0}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {http_tls-0.2.1 → http_tls-0.3.0}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {http_tls-0.2.1 → http_tls-0.3.0}/.github/workflows/release.yml +0 -0
- {http_tls-0.2.1 → http_tls-0.3.0}/.gitignore +0 -0
- {http_tls-0.2.1 → http_tls-0.3.0}/LICENSE +0 -0
- {http_tls-0.2.1 → http_tls-0.3.0}/python/http_tls/exceptions.py +0 -0
- {http_tls-0.2.1 → http_tls-0.3.0}/python/http_tls/py.typed +0 -0
- {http_tls-0.2.1 → http_tls-0.3.0}/python/http_tls/status_codes.py +0 -0
- {http_tls-0.2.1 → http_tls-0.3.0}/python/http_tls/structures.py +0 -0
|
@@ -84,9 +84,9 @@ checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
|
|
|
84
84
|
|
|
85
85
|
[[package]]
|
|
86
86
|
name = "brotli"
|
|
87
|
-
version = "9.1.
|
|
87
|
+
version = "9.1.3"
|
|
88
88
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
89
|
-
checksum = "
|
|
89
|
+
checksum = "9071f285ed78f04f37d675abaaef5de7689e2f30a216c8dc6795465d81910a13"
|
|
90
90
|
dependencies = [
|
|
91
91
|
"alloc-no-stdlib",
|
|
92
92
|
"alloc-stdlib",
|
|
@@ -95,9 +95,9 @@ dependencies = [
|
|
|
95
95
|
|
|
96
96
|
[[package]]
|
|
97
97
|
name = "brotli-decompressor"
|
|
98
|
-
version = "6.3.
|
|
98
|
+
version = "6.3.2"
|
|
99
99
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
100
|
-
checksum = "
|
|
100
|
+
checksum = "cc031cd87c6529082b0b65c604e86af0ed7744841b17ccfe60b1e31fa606179e"
|
|
101
101
|
dependencies = [
|
|
102
102
|
"alloc-no-stdlib",
|
|
103
103
|
"alloc-stdlib",
|
|
@@ -179,6 +179,8 @@ dependencies = [
|
|
|
179
179
|
"compression-core",
|
|
180
180
|
"flate2",
|
|
181
181
|
"memchr",
|
|
182
|
+
"zstd",
|
|
183
|
+
"zstd-safe",
|
|
182
184
|
]
|
|
183
185
|
|
|
184
186
|
[[package]]
|
|
@@ -478,8 +480,9 @@ dependencies = [
|
|
|
478
480
|
|
|
479
481
|
[[package]]
|
|
480
482
|
name = "http_tls"
|
|
481
|
-
version = "0.
|
|
483
|
+
version = "0.3.0"
|
|
482
484
|
dependencies = [
|
|
485
|
+
"bytes",
|
|
483
486
|
"encoding_rs",
|
|
484
487
|
"pyo3",
|
|
485
488
|
"reqwest",
|
|
@@ -1051,9 +1054,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
|
|
1051
1054
|
|
|
1052
1055
|
[[package]]
|
|
1053
1056
|
name = "rand"
|
|
1054
|
-
version = "0.10.
|
|
1057
|
+
version = "0.10.4"
|
|
1055
1058
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1056
|
-
checksum = "
|
|
1059
|
+
checksum = "8f6fa277c67e6ea590466f04c3fe82cb509f0864fe470643e5570de9514a9b11"
|
|
1057
1060
|
dependencies = [
|
|
1058
1061
|
"chacha20",
|
|
1059
1062
|
"getrandom 0.4.3",
|
|
@@ -1967,3 +1970,31 @@ name = "zmij"
|
|
|
1967
1970
|
version = "1.0.23"
|
|
1968
1971
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1969
1972
|
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
|
1973
|
+
|
|
1974
|
+
[[package]]
|
|
1975
|
+
name = "zstd"
|
|
1976
|
+
version = "0.14.4"
|
|
1977
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1978
|
+
checksum = "ad5020ab8f27bfc0024ad0a07aed6bf022a16208f479686fbed4fdd0c85eb0b3"
|
|
1979
|
+
dependencies = [
|
|
1980
|
+
"zstd-safe",
|
|
1981
|
+
]
|
|
1982
|
+
|
|
1983
|
+
[[package]]
|
|
1984
|
+
name = "zstd-safe"
|
|
1985
|
+
version = "8.3.0"
|
|
1986
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1987
|
+
checksum = "5a9ea9cc2576c2fcc104e9cec04f46e3ede564f4fbc206831e4ef7e859a4140d"
|
|
1988
|
+
dependencies = [
|
|
1989
|
+
"zstd-sys",
|
|
1990
|
+
]
|
|
1991
|
+
|
|
1992
|
+
[[package]]
|
|
1993
|
+
name = "zstd-sys"
|
|
1994
|
+
version = "2.1.3+zstd.1.5.7"
|
|
1995
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1996
|
+
checksum = "f729a06d0257428fd9d997ac8db7e1407bd0262ab016b085b36bc52957c0052d"
|
|
1997
|
+
dependencies = [
|
|
1998
|
+
"cc",
|
|
1999
|
+
"pkg-config",
|
|
2000
|
+
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "http_tls"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
edition = "2021"
|
|
5
5
|
description = "Rust native core for http_tls Python library"
|
|
6
6
|
license = "MIT"
|
|
@@ -19,10 +19,12 @@ reqwest = { version = "0.13", default-features = false, features = [
|
|
|
19
19
|
"gzip",
|
|
20
20
|
"brotli",
|
|
21
21
|
"deflate",
|
|
22
|
+
"zstd",
|
|
22
23
|
"socks",
|
|
23
24
|
"json",
|
|
24
25
|
"http2"
|
|
25
26
|
] }
|
|
27
|
+
bytes = "1"
|
|
26
28
|
serde = { version = "1.0", features = ["derive"] }
|
|
27
29
|
serde_json = "1.0"
|
|
28
30
|
encoding_rs = "0.8"
|
|
@@ -32,5 +34,6 @@ url = "2.5"
|
|
|
32
34
|
opt-level = 3
|
|
33
35
|
lto = "fat"
|
|
34
36
|
codegen-units = 1
|
|
35
|
-
panic
|
|
37
|
+
# unwind (default) so an unexpected panic surfaces as a Python exception
|
|
38
|
+
# instead of aborting the interpreter.
|
|
36
39
|
strip = true
|
|
@@ -1,3 +1,21 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: http-tls
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Classifier: Programming Language :: Rust
|
|
5
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
6
|
+
Classifier: Programming Language :: Python :: 3
|
|
7
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
8
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
9
|
+
Requires-Dist: curl-cffi>=0.17 ; extra == 'impersonate'
|
|
10
|
+
Provides-Extra: impersonate
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Summary: High-performance Rust-powered TLS & Browser Impersonating HTTP client for Python
|
|
13
|
+
License: MIT
|
|
14
|
+
Requires-Python: >=3.8
|
|
15
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
16
|
+
Project-URL: Homepage, https://github.com/real-coder-xD/http_tls
|
|
17
|
+
Project-URL: Issues, https://github.com/real-coder-xD/http_tls/issues
|
|
18
|
+
|
|
1
19
|
<div align="center">
|
|
2
20
|
|
|
3
21
|
# ⚡ http_tls
|
|
@@ -157,6 +175,62 @@ with http_tls.Session(impersonate="chrome_152", engine="rust") as s:
|
|
|
157
175
|
|
|
158
176
|
---
|
|
159
177
|
|
|
178
|
+
### Mode 5: Streaming Large Responses (`stream=True`)
|
|
179
|
+
|
|
180
|
+
Stream large payloads, downloads, and EventStreams directly from the socket without loading entire bodies into RAM:
|
|
181
|
+
|
|
182
|
+
```python
|
|
183
|
+
import http_tls as requests
|
|
184
|
+
|
|
185
|
+
# Stream download chunk-by-chunk
|
|
186
|
+
with requests.get("https://httpbin.org/stream/20", stream=True) as resp:
|
|
187
|
+
for line in resp.iter_lines():
|
|
188
|
+
if line:
|
|
189
|
+
print("Received line:", line.decode("utf-8"))
|
|
190
|
+
|
|
191
|
+
# Custom chunk size
|
|
192
|
+
with requests.get("https://example.com/large-file.zip", stream=True) as resp:
|
|
193
|
+
with open("local_file.zip", "wb") as f:
|
|
194
|
+
for chunk in resp.iter_content(chunk_size=65536):
|
|
195
|
+
f.write(chunk)
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
200
|
+
### Mode 6: Built-in Anti-Bot & JS Challenge Detection
|
|
201
|
+
|
|
202
|
+
Inspect whether an anti-bot system is challenging your request (`resp.challenge` returns the provider name or `None`):
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
import http_tls as requests
|
|
206
|
+
|
|
207
|
+
resp = requests.get("https://protected-site.com", impersonate="chrome_152")
|
|
208
|
+
|
|
209
|
+
if resp.challenge:
|
|
210
|
+
print(f"Intercepted by: {resp.challenge}")
|
|
211
|
+
# Detected providers: "cloudflare", "datadome", "ddos-guard", "imperva", "perimeterx", "sucuri"
|
|
212
|
+
else:
|
|
213
|
+
print("Normal response:", resp.status_code)
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
---
|
|
217
|
+
|
|
218
|
+
### Mode 7: Custom CA Bundles & Client Certificates (mTLS)
|
|
219
|
+
|
|
220
|
+
```python
|
|
221
|
+
import http_tls as requests
|
|
222
|
+
|
|
223
|
+
# Custom Root CA bundle (PEM)
|
|
224
|
+
session = requests.Session(verify="./custom_ca_bundle.pem")
|
|
225
|
+
|
|
226
|
+
# Client mTLS authentication
|
|
227
|
+
session = requests.Session(cert=("client.crt", "client.key"))
|
|
228
|
+
# Or combined PEM:
|
|
229
|
+
session = requests.Session(cert="client_combined.pem")
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
---
|
|
233
|
+
|
|
160
234
|
## 🎯 Supported Browser Presets (Updated to Nov 2026)
|
|
161
235
|
|
|
162
236
|
- **Chrome (Default `chrome_152`):** `chrome_152`, `chrome_152_macos`, `chrome_151`, `chrome_150`, `chrome_146`, `chrome_145`, `chrome_142`, `chrome_136`, `chrome_133`, `chrome_131`, `chrome_130`, `chrome_128`, `chrome_124`
|
|
@@ -196,7 +270,22 @@ with http_tls.Session(impersonate="chrome_152", engine="rust") as s:
|
|
|
196
270
|
from http_tls import requests, BrowserType
|
|
197
271
|
resp = requests.get("https://target.com", impersonate=BrowserType.chrome152)
|
|
198
272
|
```
|
|
199
|
-
3. **
|
|
273
|
+
3. **Tải file lớn tiết kiệm RAM (Streaming):**
|
|
274
|
+
```python
|
|
275
|
+
import http_tls as requests
|
|
276
|
+
with requests.get("https://httpbin.org/stream/10", stream=True) as resp:
|
|
277
|
+
for line in resp.iter_lines():
|
|
278
|
+
if line:
|
|
279
|
+
print(line.decode("utf-8"))
|
|
280
|
+
```
|
|
281
|
+
4. **Tự động nhận diện rào chắn Anti-Bot:**
|
|
282
|
+
```python
|
|
283
|
+
import http_tls as requests
|
|
284
|
+
resp = requests.get("https://target.com", impersonate="chrome_152")
|
|
285
|
+
if resp.challenge:
|
|
286
|
+
print("Bị chặn bởi:", resp.challenge) # 'cloudflare', 'datadome', 'ddos-guard'...
|
|
287
|
+
```
|
|
288
|
+
5. **Cào dữ liệu Async đa luồng:**
|
|
200
289
|
```python
|
|
201
290
|
import asyncio, http_tls
|
|
202
291
|
async def main():
|
|
@@ -213,3 +302,4 @@ with http_tls.Session(impersonate="chrome_152", engine="rust") as s:
|
|
|
213
302
|
## 📄 License
|
|
214
303
|
|
|
215
304
|
Released under the [MIT License](LICENSE).
|
|
305
|
+
|
|
@@ -1,17 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: http-tls
|
|
3
|
-
Version: 0.2.1
|
|
4
|
-
Classifier: Programming Language :: Rust
|
|
5
|
-
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
6
|
-
Classifier: Programming Language :: Python :: 3
|
|
7
|
-
Classifier: Topic :: Internet :: WWW/HTTP
|
|
8
|
-
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
9
|
-
License-File: LICENSE
|
|
10
|
-
Summary: High-performance Rust-powered TLS & Browser Impersonating HTTP client for Python
|
|
11
|
-
License: MIT
|
|
12
|
-
Requires-Python: >=3.8
|
|
13
|
-
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
14
|
-
|
|
15
1
|
<div align="center">
|
|
16
2
|
|
|
17
3
|
# ⚡ http_tls
|
|
@@ -171,6 +157,62 @@ with http_tls.Session(impersonate="chrome_152", engine="rust") as s:
|
|
|
171
157
|
|
|
172
158
|
---
|
|
173
159
|
|
|
160
|
+
### Mode 5: Streaming Large Responses (`stream=True`)
|
|
161
|
+
|
|
162
|
+
Stream large payloads, downloads, and EventStreams directly from the socket without loading entire bodies into RAM:
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
import http_tls as requests
|
|
166
|
+
|
|
167
|
+
# Stream download chunk-by-chunk
|
|
168
|
+
with requests.get("https://httpbin.org/stream/20", stream=True) as resp:
|
|
169
|
+
for line in resp.iter_lines():
|
|
170
|
+
if line:
|
|
171
|
+
print("Received line:", line.decode("utf-8"))
|
|
172
|
+
|
|
173
|
+
# Custom chunk size
|
|
174
|
+
with requests.get("https://example.com/large-file.zip", stream=True) as resp:
|
|
175
|
+
with open("local_file.zip", "wb") as f:
|
|
176
|
+
for chunk in resp.iter_content(chunk_size=65536):
|
|
177
|
+
f.write(chunk)
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
---
|
|
181
|
+
|
|
182
|
+
### Mode 6: Built-in Anti-Bot & JS Challenge Detection
|
|
183
|
+
|
|
184
|
+
Inspect whether an anti-bot system is challenging your request (`resp.challenge` returns the provider name or `None`):
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
import http_tls as requests
|
|
188
|
+
|
|
189
|
+
resp = requests.get("https://protected-site.com", impersonate="chrome_152")
|
|
190
|
+
|
|
191
|
+
if resp.challenge:
|
|
192
|
+
print(f"Intercepted by: {resp.challenge}")
|
|
193
|
+
# Detected providers: "cloudflare", "datadome", "ddos-guard", "imperva", "perimeterx", "sucuri"
|
|
194
|
+
else:
|
|
195
|
+
print("Normal response:", resp.status_code)
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
200
|
+
### Mode 7: Custom CA Bundles & Client Certificates (mTLS)
|
|
201
|
+
|
|
202
|
+
```python
|
|
203
|
+
import http_tls as requests
|
|
204
|
+
|
|
205
|
+
# Custom Root CA bundle (PEM)
|
|
206
|
+
session = requests.Session(verify="./custom_ca_bundle.pem")
|
|
207
|
+
|
|
208
|
+
# Client mTLS authentication
|
|
209
|
+
session = requests.Session(cert=("client.crt", "client.key"))
|
|
210
|
+
# Or combined PEM:
|
|
211
|
+
session = requests.Session(cert="client_combined.pem")
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
---
|
|
215
|
+
|
|
174
216
|
## 🎯 Supported Browser Presets (Updated to Nov 2026)
|
|
175
217
|
|
|
176
218
|
- **Chrome (Default `chrome_152`):** `chrome_152`, `chrome_152_macos`, `chrome_151`, `chrome_150`, `chrome_146`, `chrome_145`, `chrome_142`, `chrome_136`, `chrome_133`, `chrome_131`, `chrome_130`, `chrome_128`, `chrome_124`
|
|
@@ -210,7 +252,22 @@ with http_tls.Session(impersonate="chrome_152", engine="rust") as s:
|
|
|
210
252
|
from http_tls import requests, BrowserType
|
|
211
253
|
resp = requests.get("https://target.com", impersonate=BrowserType.chrome152)
|
|
212
254
|
```
|
|
213
|
-
3. **
|
|
255
|
+
3. **Tải file lớn tiết kiệm RAM (Streaming):**
|
|
256
|
+
```python
|
|
257
|
+
import http_tls as requests
|
|
258
|
+
with requests.get("https://httpbin.org/stream/10", stream=True) as resp:
|
|
259
|
+
for line in resp.iter_lines():
|
|
260
|
+
if line:
|
|
261
|
+
print(line.decode("utf-8"))
|
|
262
|
+
```
|
|
263
|
+
4. **Tự động nhận diện rào chắn Anti-Bot:**
|
|
264
|
+
```python
|
|
265
|
+
import http_tls as requests
|
|
266
|
+
resp = requests.get("https://target.com", impersonate="chrome_152")
|
|
267
|
+
if resp.challenge:
|
|
268
|
+
print("Bị chặn bởi:", resp.challenge) # 'cloudflare', 'datadome', 'ddos-guard'...
|
|
269
|
+
```
|
|
270
|
+
5. **Cào dữ liệu Async đa luồng:**
|
|
214
271
|
```python
|
|
215
272
|
import asyncio, http_tls
|
|
216
273
|
async def main():
|
|
@@ -227,4 +284,3 @@ with http_tls.Session(impersonate="chrome_152", engine="rust") as s:
|
|
|
227
284
|
## 📄 License
|
|
228
285
|
|
|
229
286
|
Released under the [MIT License](LICENSE).
|
|
230
|
-
|
|
@@ -4,7 +4,7 @@ build-backend = "maturin"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "http-tls"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "High-performance Rust-powered TLS & Browser Impersonating HTTP client for Python"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.8"
|
|
@@ -18,6 +18,14 @@ classifiers = [
|
|
|
18
18
|
]
|
|
19
19
|
dependencies = []
|
|
20
20
|
|
|
21
|
+
[project.optional-dependencies]
|
|
22
|
+
# Byte-exact browser TLS/HTTP2 fingerprints via libcurl-impersonate.
|
|
23
|
+
impersonate = ["curl_cffi>=0.17"]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Homepage = "https://github.com/real-coder-xD/http_tls"
|
|
27
|
+
Issues = "https://github.com/real-coder-xD/http_tls/issues"
|
|
28
|
+
|
|
21
29
|
[tool.maturin]
|
|
22
30
|
python-source = "python"
|
|
23
31
|
module-name = "http_tls._core"
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""http_tls: High-performance Rust & TLS Fingerprint Impersonating HTTP client for Python.
|
|
1
|
+
"""http_tls: High-performance Rust & TLS Fingerprint Impersonating HTTP client for Python.
|
|
2
2
|
|
|
3
3
|
100% Drop-in replacement for both `requests` and `curl_cffi.requests`:
|
|
4
4
|
import http_tls as requests
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import sys
|
|
11
|
-
from http_tls import exceptions, requests, status_codes, structures
|
|
11
|
+
from http_tls import challenge, exceptions, requests, status_codes, structures
|
|
12
12
|
from http_tls.exceptions import (
|
|
13
13
|
ConnectTimeout,
|
|
14
14
|
ConnectionError,
|
|
@@ -41,7 +41,7 @@ from http_tls.structures import CaseInsensitiveDict
|
|
|
41
41
|
try:
|
|
42
42
|
from http_tls._core import __version__
|
|
43
43
|
except ImportError:
|
|
44
|
-
__version__ = "0.
|
|
44
|
+
__version__ = "0.3.0"
|
|
45
45
|
|
|
46
46
|
# Register module alias in sys.modules so `import http_tls.requests as requests` works
|
|
47
47
|
sys.modules["http_tls.requests"] = requests
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
from typing import Any, Dict, List, Optional, Union
|
|
1
|
+
from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
|
|
2
|
+
from http_tls.structures import CaseInsensitiveDict
|
|
2
3
|
|
|
3
4
|
__version__: str
|
|
4
5
|
|
|
@@ -6,41 +7,72 @@ def supported_presets() -> List[str]: ...
|
|
|
6
7
|
|
|
7
8
|
class Response:
|
|
8
9
|
status_code: int
|
|
9
|
-
headers:
|
|
10
|
+
headers: CaseInsensitiveDict
|
|
10
11
|
url: str
|
|
11
12
|
content: bytes
|
|
12
13
|
text: str
|
|
13
|
-
encoding: str
|
|
14
|
+
encoding: Optional[str]
|
|
14
15
|
elapsed: float
|
|
15
16
|
cookies: Dict[str, str]
|
|
17
|
+
history: List["Response"]
|
|
16
18
|
|
|
17
|
-
|
|
19
|
+
@property
|
|
18
20
|
def ok(self) -> bool: ...
|
|
21
|
+
@property
|
|
22
|
+
def reason(self) -> str: ...
|
|
23
|
+
@property
|
|
24
|
+
def challenge(self) -> Optional[str]: ...
|
|
25
|
+
|
|
26
|
+
def json(self, **kwargs: Any) -> Any: ...
|
|
27
|
+
def raise_for_status(self) -> None: ...
|
|
28
|
+
def iter_content(self, chunk_size: Optional[int] = 1, decode_unicode: bool = False) -> Iterator[Any]: ...
|
|
29
|
+
def iter_lines(self, chunk_size: int = 512, decode_unicode: bool = False, delimiter: Optional[Any] = None) -> Iterator[Any]: ...
|
|
30
|
+
def close(self) -> None: ...
|
|
31
|
+
def __enter__(self) -> "Response": ...
|
|
32
|
+
def __exit__(self, *exc: Any) -> None: ...
|
|
19
33
|
|
|
20
34
|
class Session:
|
|
35
|
+
impersonate: str
|
|
36
|
+
headers: CaseInsensitiveDict
|
|
37
|
+
cookies: Dict[str, str]
|
|
38
|
+
proxies: Dict[str, str]
|
|
39
|
+
mode: str
|
|
40
|
+
|
|
21
41
|
def __init__(
|
|
22
42
|
self,
|
|
23
43
|
impersonate: str = "chrome_152",
|
|
24
44
|
timeout: float = 30.0,
|
|
25
|
-
|
|
26
|
-
max_redirects: int = 10,
|
|
45
|
+
verify: Union[bool, str] = True,
|
|
27
46
|
verify_ssl: bool = True,
|
|
47
|
+
proxies: Optional[Dict[str, str]] = None,
|
|
28
48
|
proxy: Optional[str] = None,
|
|
49
|
+
allow_redirects: bool = True,
|
|
50
|
+
follow_redirects: bool = True,
|
|
51
|
+
max_redirects: int = 10,
|
|
29
52
|
headers: Optional[Dict[str, str]] = None,
|
|
53
|
+
cookies: Optional[Dict[str, str]] = None,
|
|
54
|
+
auth: Optional[Tuple[str, str]] = None,
|
|
55
|
+
cert: Union[str, Tuple[str, str], None] = None,
|
|
56
|
+
engine: str = "auto",
|
|
57
|
+
**kwargs: Any,
|
|
30
58
|
) -> None: ...
|
|
31
|
-
@property
|
|
32
|
-
def impersonate(self) -> str: ...
|
|
33
59
|
def request(
|
|
34
60
|
self,
|
|
35
61
|
method: str,
|
|
36
62
|
url: str,
|
|
37
|
-
params: Optional[Dict[str,
|
|
63
|
+
params: Optional[Dict[str, Any]] = None,
|
|
38
64
|
headers: Optional[Dict[str, str]] = None,
|
|
39
65
|
cookies: Optional[Dict[str, str]] = None,
|
|
40
|
-
data: Optional[Union[bytes, str]] = None,
|
|
66
|
+
data: Optional[Union[bytes, str, Dict[str, Any]]] = None,
|
|
41
67
|
json: Optional[Any] = None,
|
|
42
68
|
timeout: Optional[float] = None,
|
|
69
|
+
verify: Optional[Union[bool, str]] = None,
|
|
70
|
+
proxies: Optional[Dict[str, str]] = None,
|
|
43
71
|
proxy: Optional[str] = None,
|
|
72
|
+
allow_redirects: Optional[bool] = None,
|
|
73
|
+
auth: Optional[Tuple[str, str]] = None,
|
|
74
|
+
stream: bool = False,
|
|
75
|
+
**kwargs: Any,
|
|
44
76
|
) -> Response: ...
|
|
45
77
|
def get(self, url: str, **kwargs: Any) -> Response: ...
|
|
46
78
|
def post(self, url: str, **kwargs: Any) -> Response: ...
|
|
@@ -49,22 +81,15 @@ class Session:
|
|
|
49
81
|
def patch(self, url: str, **kwargs: Any) -> Response: ...
|
|
50
82
|
def head(self, url: str, **kwargs: Any) -> Response: ...
|
|
51
83
|
def options(self, url: str, **kwargs: Any) -> Response: ...
|
|
52
|
-
def
|
|
84
|
+
def close(self) -> None: ...
|
|
85
|
+
def __enter__(self) -> "Session": ...
|
|
53
86
|
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: ...
|
|
54
87
|
|
|
55
88
|
class AsyncSession:
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
follow_redirects: bool = True,
|
|
61
|
-
max_redirects: int = 10,
|
|
62
|
-
verify_ssl: bool = True,
|
|
63
|
-
proxy: Optional[str] = None,
|
|
64
|
-
headers: Optional[Dict[str, str]] = None,
|
|
65
|
-
) -> None: ...
|
|
66
|
-
@property
|
|
67
|
-
def impersonate(self) -> str: ...
|
|
89
|
+
headers: CaseInsensitiveDict
|
|
90
|
+
cookies: Dict[str, str]
|
|
91
|
+
|
|
92
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None: ...
|
|
68
93
|
async def request(self, method: str, url: str, **kwargs: Any) -> Response: ...
|
|
69
94
|
async def get(self, url: str, **kwargs: Any) -> Response: ...
|
|
70
95
|
async def post(self, url: str, **kwargs: Any) -> Response: ...
|
|
@@ -73,7 +98,8 @@ class AsyncSession:
|
|
|
73
98
|
async def patch(self, url: str, **kwargs: Any) -> Response: ...
|
|
74
99
|
async def head(self, url: str, **kwargs: Any) -> Response: ...
|
|
75
100
|
async def options(self, url: str, **kwargs: Any) -> Response: ...
|
|
76
|
-
async def
|
|
101
|
+
async def close(self) -> None: ...
|
|
102
|
+
async def __aenter__(self) -> "AsyncSession": ...
|
|
77
103
|
async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: ...
|
|
78
104
|
|
|
79
105
|
def request(method: str, url: str, impersonate: str = "chrome_152", **kwargs: Any) -> Response: ...
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Anti-bot interstitial detection.
|
|
2
|
+
|
|
3
|
+
Classifies a response as a JavaScript / browser-verification interstitial
|
|
4
|
+
served by a known edge provider. Pure inspection: nothing is executed.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Any, Dict, Optional, Tuple
|
|
10
|
+
|
|
11
|
+
__all__ = ["detect", "CLEARANCE_COOKIES"]
|
|
12
|
+
|
|
13
|
+
# Cookie(s) each provider sets once the browser check has passed.
|
|
14
|
+
CLEARANCE_COOKIES: Dict[str, Tuple[str, ...]] = {
|
|
15
|
+
"cloudflare": ("cf_clearance",),
|
|
16
|
+
"ddos-guard": ("__ddg2_", "__ddgid_", "__ddg1_"),
|
|
17
|
+
"datadome": ("datadome",),
|
|
18
|
+
"perimeterx": ("_px3", "_pxvid"),
|
|
19
|
+
"imperva": ("reese84", "incap_ses_"),
|
|
20
|
+
"sucuri": ("sucuri_cloudproxy_uuid_",),
|
|
21
|
+
"vercel": ("_vcrcs",),
|
|
22
|
+
"akamai": ("_abck",),
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
_CHALLENGE_STATUSES = (403, 429, 503)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _body(resp: Any, limit: int = 32768) -> str:
|
|
29
|
+
# Never force-read a streamed body just to classify it.
|
|
30
|
+
if getattr(resp, "_content", b"") is None:
|
|
31
|
+
return ""
|
|
32
|
+
try:
|
|
33
|
+
return (resp.text or "")[:limit]
|
|
34
|
+
except Exception:
|
|
35
|
+
return ""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def detect(resp: Any) -> Optional[str]:
|
|
39
|
+
"""Return the provider name if `resp` is a browser-check interstitial, else None."""
|
|
40
|
+
headers = resp.headers
|
|
41
|
+
status = int(resp.status_code)
|
|
42
|
+
server = str(headers.get("server", "")).lower()
|
|
43
|
+
|
|
44
|
+
# Explicit signals first: these headers are only sent on challenge pages.
|
|
45
|
+
if str(headers.get("cf-mitigated", "")).lower() == "challenge":
|
|
46
|
+
return "cloudflare"
|
|
47
|
+
if str(headers.get("x-vercel-mitigated", "")).lower() == "challenge":
|
|
48
|
+
return "vercel"
|
|
49
|
+
|
|
50
|
+
if status not in _CHALLENGE_STATUSES and status != 200:
|
|
51
|
+
return None
|
|
52
|
+
|
|
53
|
+
body = _body(resp)
|
|
54
|
+
low = body.lower()
|
|
55
|
+
|
|
56
|
+
if "cloudflare" in server and status in _CHALLENGE_STATUSES and (
|
|
57
|
+
"challenge-platform" in low or "cf_chl_opt" in low or "<title>just a moment" in low
|
|
58
|
+
):
|
|
59
|
+
return "cloudflare"
|
|
60
|
+
|
|
61
|
+
if ("ddos-guard" in server or "ddg" in server) and status in _CHALLENGE_STATUSES:
|
|
62
|
+
return "ddos-guard"
|
|
63
|
+
if "check.ddos-guard.net" in low or "ddos-guard/js-challenge" in low:
|
|
64
|
+
return "ddos-guard"
|
|
65
|
+
|
|
66
|
+
if "x-datadome" in headers or "x-dd-b" in headers:
|
|
67
|
+
if status in _CHALLENGE_STATUSES or "captcha-delivery.com" in low:
|
|
68
|
+
return "datadome"
|
|
69
|
+
|
|
70
|
+
if status in _CHALLENGE_STATUSES and ("px-captcha" in low or "_pxappid" in low):
|
|
71
|
+
return "perimeterx"
|
|
72
|
+
|
|
73
|
+
if "_incapsula_resource" in low or ("x-iinfo" in headers and status in _CHALLENGE_STATUSES):
|
|
74
|
+
return "imperva"
|
|
75
|
+
|
|
76
|
+
if "sucuri" in server and "sucuri_cloudproxy_js" in low:
|
|
77
|
+
return "sucuri"
|
|
78
|
+
|
|
79
|
+
if "akamaighost" in server and status == 403 and "reference #" in low:
|
|
80
|
+
return "akamai"
|
|
81
|
+
|
|
82
|
+
return None
|