never-primp 1.0.0__cp38-abi3-macosx_10_12_x86_64.whl → 1.0.1__cp38-abi3-macosx_10_12_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of never-primp might be problematic. Click here for more details.
- never_primp/__init__.py +14 -1
- never_primp/never_primp.abi3.so +0 -0
- never_primp-1.0.1.dist-info/METADATA +803 -0
- never_primp-1.0.1.dist-info/RECORD +8 -0
- never_primp-1.0.0.dist-info/METADATA +0 -358
- never_primp-1.0.0.dist-info/RECORD +0 -8
- {never_primp-1.0.0.dist-info → never_primp-1.0.1.dist-info}/WHEEL +0 -0
- {never_primp-1.0.0.dist-info → never_primp-1.0.1.dist-info}/licenses/LICENSE +0 -0
never_primp/__init__.py
CHANGED
|
@@ -35,7 +35,9 @@ class Client(RClient):
|
|
|
35
35
|
auth_bearer: str | None = None,
|
|
36
36
|
params: dict[str, str] | None = None,
|
|
37
37
|
headers: dict[str, str] | None = None,
|
|
38
|
+
ordered_headers: dict[str, str] | None = None,
|
|
38
39
|
cookie_store: bool | None = True,
|
|
40
|
+
split_cookies: bool | None = False,
|
|
39
41
|
referer: bool | None = True,
|
|
40
42
|
proxy: str | None = None,
|
|
41
43
|
timeout: float | None = 30,
|
|
@@ -62,8 +64,16 @@ class Client(RClient):
|
|
|
62
64
|
auth_bearer: a string representing the bearer token for bearer token authentication. Default is None.
|
|
63
65
|
params: a map of query parameters to append to the URL. Default is None.
|
|
64
66
|
headers: an optional map of HTTP headers to send with requests. Ignored if `impersonate` is set.
|
|
67
|
+
ordered_headers: an optional ordered map of HTTP headers with strict order preservation.
|
|
68
|
+
Takes priority over `headers`. Use this for bypassing anti-bot detection that checks header order.
|
|
69
|
+
Example: {"User-Agent": "...", "Accept": "...", "Accept-Language": "..."}
|
|
70
|
+
Note: Python 3.7+ dict maintains insertion order by default.
|
|
65
71
|
cookie_store: enable a persistent cookie store. Received cookies will be preserved and included
|
|
66
72
|
in additional requests. Default is True.
|
|
73
|
+
split_cookies: split cookies into multiple `cookie` headers (HTTP/2 style) instead of a single
|
|
74
|
+
`Cookie` header. Useful for mimicking browser behavior in HTTP/2. Default is False.
|
|
75
|
+
When True: cookie: a=1 \n cookie: b=2 \n cookie: c=3
|
|
76
|
+
When False: Cookie: a=1; b=2; c=3
|
|
67
77
|
referer: automatic setting of the `Referer` header. Default is True.
|
|
68
78
|
proxy: proxy URL for HTTP requests, example: "socks5://127.0.0.1:9150". Default is None.
|
|
69
79
|
timeout: timeout for HTTP requests in seconds. Default is 30.
|
|
@@ -131,7 +141,9 @@ class AsyncClient(Client):
|
|
|
131
141
|
auth_bearer: str | None = None,
|
|
132
142
|
params: dict[str, str] | None = None,
|
|
133
143
|
headers: dict[str, str] | None = None,
|
|
144
|
+
ordered_headers: dict[str, str] | None = None,
|
|
134
145
|
cookie_store: bool | None = True,
|
|
146
|
+
split_cookies: bool | None = False,
|
|
135
147
|
referer: bool | None = True,
|
|
136
148
|
proxy: str | None = None,
|
|
137
149
|
timeout: float | None = 30,
|
|
@@ -150,7 +162,8 @@ class AsyncClient(Client):
|
|
|
150
162
|
tcp_keepalive: float | None = None,
|
|
151
163
|
# Retry mechanism
|
|
152
164
|
retry_count: int | None = None,
|
|
153
|
-
retry_backoff: float | None = None
|
|
165
|
+
retry_backoff: float | None = None,
|
|
166
|
+
):
|
|
154
167
|
super().__init__()
|
|
155
168
|
|
|
156
169
|
async def __aenter__(self) -> AsyncClient:
|
never_primp/never_primp.abi3.so
CHANGED
|
Binary file
|
|
@@ -0,0 +1,803 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: never_primp
|
|
3
|
+
Version: 1.0.1
|
|
4
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
5
|
+
Classifier: Intended Audience :: Developers
|
|
6
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
15
|
+
Classifier: Programming Language :: Rust
|
|
16
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Summary: 基于原primp 重新优化调整的请求库 - The fastest python HTTP client that can impersonate web browsers
|
|
20
|
+
Keywords: requests,httpx,http,http-client,tls-fingerprint,ja3,ja4,impersonate,browser-impersonation,web-scraping,crawler,reverse-engineering
|
|
21
|
+
Author: Neverland
|
|
22
|
+
Requires-Python: >=3.8
|
|
23
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
24
|
+
Project-URL: Homepage, https://github.com/Neverland/never_primp
|
|
25
|
+
Project-URL: Repository, https://github.com/Neverland/never_primp
|
|
26
|
+
Project-URL: Bug Tracker, https://github.com/Neverland/never_primp/issues
|
|
27
|
+
|
|
28
|
+
<div align="center">
|
|
29
|
+
|
|
30
|
+
# 🪞 NEVER_PRIMP
|
|
31
|
+
|
|
32
|
+
**Since the original primp project author did not maintain updates for a long time, he refactored and maintained based on the primp project**
|
|
33
|
+
|
|
34
|
+
**The Ultimate Python HTTP Client for Web Scraping & Browser Impersonation**
|
|
35
|
+
|
|
36
|
+

|
|
37
|
+
[](https://pypi.org/project/never-primp)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
[](https://www.rust-lang.org)
|
|
40
|
+
|
|
41
|
+
*Lightning-fast HTTP client built with Rust, designed for web scraping, anti-bot bypass, and perfect browser impersonation*
|
|
42
|
+
|
|
43
|
+
[English](README.md) | [简体中文](README_CN.md)
|
|
44
|
+
|
|
45
|
+
[Installation](#-installation) •
|
|
46
|
+
[Key Features](#-key-features) •
|
|
47
|
+
[Quick Start](#-quick-start) •
|
|
48
|
+
[Documentation](#-documentation) •
|
|
49
|
+
[Examples](#-examples)
|
|
50
|
+
|
|
51
|
+
</div>
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## 🎯 What is NEVER_PRIMP?
|
|
56
|
+
|
|
57
|
+
**NEVER_PRIMP** (**P**ython **R**equests **IMP**ersonate) is a cutting-edge HTTP client library that combines:
|
|
58
|
+
|
|
59
|
+
- ⚡ **Blazing Speed**: Built on Rust's `wreq` with zero-copy parsing
|
|
60
|
+
- 🎭 **Perfect Browser Impersonation**: Mimic Chrome, Firefox, Safari, Edge down to TLS/JA3/JA4 fingerprints
|
|
61
|
+
- 🛡️ **Anti-Bot Bypass**: Advanced features for bypassing WAF, Cloudflare, and bot detection
|
|
62
|
+
- 🔧 **Production-Ready**: Connection pooling, retries, cookies, streaming, and more
|
|
63
|
+
|
|
64
|
+
### Why Choose NEVER_PRIMP?
|
|
65
|
+
|
|
66
|
+
| Feature | NEVER_PRIMP | requests | httpx | curl-cffi |
|
|
67
|
+
|---------|-------------|----------|-------|-----------|
|
|
68
|
+
| **Speed** | ⚡⚡⚡ | ⚡ | ⚡⚡ | ⚡⚡ |
|
|
69
|
+
| **Browser Impersonation** | ✅ Full | ❌ | ❌ | ✅ Limited |
|
|
70
|
+
| **Header Order Control** | ✅ | ❌ | ❌ | ❌ |
|
|
71
|
+
| **Cookie Splitting (HTTP/2)** | ✅ | ❌ | ❌ | ❌ |
|
|
72
|
+
| **Connection Pooling** | ✅ | ✅ | ✅ | ❌ |
|
|
73
|
+
| **Async Support** | ✅ | ❌ | ✅ | ❌ |
|
|
74
|
+
| **Native TLS** | ✅ | ❌ | ❌ | ✅ |
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
78
|
+
## 📦 Installation
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
pip install -U never-primp
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### Platform Support
|
|
85
|
+
|
|
86
|
+
Precompiled wheels available for:
|
|
87
|
+
- 🐧 **Linux**: x86_64, aarch64, armv7 (manylinux_2_34+)
|
|
88
|
+
- 🐧 **Linux (musl)**: x86_64, aarch64
|
|
89
|
+
- 🪟 **Windows**: x86_64
|
|
90
|
+
- 🍏 **macOS**: x86_64, ARM64 (Apple Silicon)
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## ✨ Key Features
|
|
95
|
+
|
|
96
|
+
### 🚀 Performance Optimized
|
|
97
|
+
|
|
98
|
+
<details>
|
|
99
|
+
<summary><b>Click to expand</b></summary>
|
|
100
|
+
|
|
101
|
+
- **Connection Pooling**: Reuse connections with configurable idle timeout
|
|
102
|
+
- **TCP Optimization**: TCP_NODELAY + TCP keepalive for lower latency
|
|
103
|
+
- **Zero-Copy Parsing**: Rust's efficient memory handling
|
|
104
|
+
- **HTTP/2 Multiplexing**: Multiple requests over single connection
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
client = primp.Client(
|
|
108
|
+
pool_idle_timeout=90.0, # Keep connections alive 90s
|
|
109
|
+
pool_max_idle_per_host=10, # Max 10 idle connections per host
|
|
110
|
+
tcp_nodelay=True, # Disable Nagle's algorithm
|
|
111
|
+
tcp_keepalive=60.0, # TCP keepalive every 60s
|
|
112
|
+
)
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
**Benchmark**: ~59% faster than `requests` for sequential requests with connection reuse.
|
|
116
|
+
|
|
117
|
+
</details>
|
|
118
|
+
|
|
119
|
+
### 🎭 Advanced Browser Impersonation
|
|
120
|
+
|
|
121
|
+
<details>
|
|
122
|
+
<summary><b>Click to expand</b></summary>
|
|
123
|
+
|
|
124
|
+
Perfect fingerprint mimicry for:
|
|
125
|
+
|
|
126
|
+
- **Chrome** (100-141): Latest versions with full TLS/HTTP2 fingerprints
|
|
127
|
+
- **Safari** (15.3-26): iOS, iPadOS, macOS variants
|
|
128
|
+
- **Firefox** (109-143): Desktop versions
|
|
129
|
+
- **Edge** (101-134): Chromium-based
|
|
130
|
+
- **OkHttp** (3.9-5.0): Android application library
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
client = primp.Client(
|
|
134
|
+
impersonate="chrome_141", # Browser version
|
|
135
|
+
impersonate_os="windows" # OS: windows, macos, linux, android, ios
|
|
136
|
+
)
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Impersonates:
|
|
140
|
+
- ✅ TLS fingerprint (JA3/JA4)
|
|
141
|
+
- ✅ HTTP/2 fingerprint (AKAMAI)
|
|
142
|
+
- ✅ Header order and casing
|
|
143
|
+
- ✅ Cipher suites
|
|
144
|
+
- ✅ Extension order
|
|
145
|
+
|
|
146
|
+
</details>
|
|
147
|
+
|
|
148
|
+
### 🛡️ Anti-Bot Bypass Features
|
|
149
|
+
|
|
150
|
+
<details>
|
|
151
|
+
<summary><b>Click to expand</b></summary>
|
|
152
|
+
|
|
153
|
+
#### 1. **Ordered Headers** 🆕
|
|
154
|
+
Maintain exact header order to bypass detection systems that check header sequence:
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
client = primp.Client(
|
|
158
|
+
ordered_headers={
|
|
159
|
+
"user-agent": "Mozilla/5.0...",
|
|
160
|
+
"accept": "text/html,application/xhtml+xml",
|
|
161
|
+
"accept-language": "en-US,en;q=0.9",
|
|
162
|
+
"accept-encoding": "gzip, deflate, br",
|
|
163
|
+
"sec-fetch-dest": "document",
|
|
164
|
+
"sec-fetch-mode": "navigate",
|
|
165
|
+
}
|
|
166
|
+
)
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
**Use Case**: Websites checking header order (Cloudflare, Akamai, etc.)
|
|
170
|
+
|
|
171
|
+
📖 [Full Documentation](ORDERED_HEADERS.md)
|
|
172
|
+
|
|
173
|
+
#### 2. **Split Cookies (HTTP/2)** 🆕
|
|
174
|
+
Send cookies as separate headers like real browsers:
|
|
175
|
+
|
|
176
|
+
```python
|
|
177
|
+
client = primp.Client(
|
|
178
|
+
split_cookies=True, # Send cookies in HTTP/2 style
|
|
179
|
+
http2_only=True
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
# Sends:
|
|
183
|
+
# cookie: session_id=abc123
|
|
184
|
+
# cookie: user_token=xyz789
|
|
185
|
+
# cookie: preference=dark_mode
|
|
186
|
+
|
|
187
|
+
# Instead of:
|
|
188
|
+
# Cookie: session_id=abc123; user_token=xyz789; preference=dark_mode
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
**Use Case**: Precise HTTP/2 browser simulation for anti-bot bypass
|
|
192
|
+
|
|
193
|
+
📖 [Full Documentation](SPLIT_COOKIES.md)
|
|
194
|
+
|
|
195
|
+
#### 3. **Dynamic Configuration**
|
|
196
|
+
Change client behavior without recreation:
|
|
197
|
+
|
|
198
|
+
```python
|
|
199
|
+
client = primp.Client(impersonate="chrome_140")
|
|
200
|
+
|
|
201
|
+
# Switch impersonation dynamically
|
|
202
|
+
client.impersonate = "safari_18"
|
|
203
|
+
client.impersonate_os = "macos"
|
|
204
|
+
|
|
205
|
+
# Update headers
|
|
206
|
+
client.ordered_headers = {...}
|
|
207
|
+
client.headers_update({"Referer": "https://example.com"})
|
|
208
|
+
|
|
209
|
+
# Change proxy
|
|
210
|
+
client.proxy = "socks5://127.0.0.1:1080"
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
</details>
|
|
214
|
+
|
|
215
|
+
### 🍪 Smart Cookie Management
|
|
216
|
+
|
|
217
|
+
<details>
|
|
218
|
+
<summary><b>Click to expand</b></summary>
|
|
219
|
+
|
|
220
|
+
#### Automatic Cookie Persistence
|
|
221
|
+
```python
|
|
222
|
+
client = primp.Client(cookie_store=True) # Default
|
|
223
|
+
|
|
224
|
+
# Cookies automatically stored and sent
|
|
225
|
+
resp1 = client.get("https://example.com/login")
|
|
226
|
+
resp2 = client.get("https://example.com/dashboard") # Cookies included
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
#### Manual Cookie Control
|
|
230
|
+
```python
|
|
231
|
+
# Set cookies
|
|
232
|
+
client.set_cookies(
|
|
233
|
+
url="https://example.com",
|
|
234
|
+
cookies={"session": "abc123", "user_id": "456"}
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
# Get cookies
|
|
238
|
+
cookies = client.get_cookies(url="https://example.com")
|
|
239
|
+
|
|
240
|
+
# Per-request cookies
|
|
241
|
+
resp = client.get(url, cookies={"temp": "value"})
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
</details>
|
|
245
|
+
|
|
246
|
+
### 🔒 Certificate Management
|
|
247
|
+
|
|
248
|
+
<details>
|
|
249
|
+
<summary><b>Click to expand</b></summary>
|
|
250
|
+
|
|
251
|
+
- **System Certificate Store**: Auto-updated with OS (no more expiration issues!)
|
|
252
|
+
- **Custom CA Bundle**: Support for corporate proxies
|
|
253
|
+
|
|
254
|
+
```python
|
|
255
|
+
# Use system certificates (default)
|
|
256
|
+
client = primp.Client(verify=True)
|
|
257
|
+
|
|
258
|
+
# Custom CA bundle
|
|
259
|
+
client = primp.Client(ca_cert_file="/path/to/cacert.pem")
|
|
260
|
+
|
|
261
|
+
# Environment variable
|
|
262
|
+
export PRIMP_CA_BUNDLE="/path/to/cert.pem"
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
</details>
|
|
266
|
+
|
|
267
|
+
### 🔄 Retry Mechanism
|
|
268
|
+
|
|
269
|
+
<details>
|
|
270
|
+
<summary><b>Click to expand</b></summary>
|
|
271
|
+
|
|
272
|
+
Automatic retries with exponential backoff:
|
|
273
|
+
|
|
274
|
+
```python
|
|
275
|
+
client = primp.Client(
|
|
276
|
+
retry_count=3, # Retry up to 3 times
|
|
277
|
+
retry_backoff=1.0, # 1 second backoff between retries
|
|
278
|
+
)
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
Handles transient failures gracefully.
|
|
282
|
+
|
|
283
|
+
</details>
|
|
284
|
+
|
|
285
|
+
### 🌊 Streaming Responses
|
|
286
|
+
|
|
287
|
+
<details>
|
|
288
|
+
<summary><b>Click to expand</b></summary>
|
|
289
|
+
|
|
290
|
+
Stream large responses efficiently:
|
|
291
|
+
|
|
292
|
+
```python
|
|
293
|
+
resp = client.get("https://example.com/large-file.zip")
|
|
294
|
+
|
|
295
|
+
for chunk in resp.stream():
|
|
296
|
+
process_chunk(chunk)
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
</details>
|
|
300
|
+
|
|
301
|
+
### ⚡ Async Support
|
|
302
|
+
|
|
303
|
+
<details>
|
|
304
|
+
<summary><b>Click to expand</b></summary>
|
|
305
|
+
|
|
306
|
+
Full async/await support with `AsyncClient`:
|
|
307
|
+
|
|
308
|
+
```python
|
|
309
|
+
import asyncio
|
|
310
|
+
import never_primp as primp
|
|
311
|
+
|
|
312
|
+
async def fetch(url):
|
|
313
|
+
async with primp.AsyncClient(impersonate="chrome_141") as client:
|
|
314
|
+
return await client.get(url)
|
|
315
|
+
|
|
316
|
+
async def main():
|
|
317
|
+
urls = ["https://site1.com", "https://site2.com", "https://site3.com"]
|
|
318
|
+
tasks = [fetch(url) for url in urls]
|
|
319
|
+
results = await asyncio.gather(*tasks)
|
|
320
|
+
|
|
321
|
+
asyncio.run(main())
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
</details>
|
|
325
|
+
|
|
326
|
+
---
|
|
327
|
+
|
|
328
|
+
## 🚀 Quick Start
|
|
329
|
+
|
|
330
|
+
### Basic Usage
|
|
331
|
+
|
|
332
|
+
```python
|
|
333
|
+
import never_primp as primp
|
|
334
|
+
|
|
335
|
+
# Simple GET request
|
|
336
|
+
client = primp.Client()
|
|
337
|
+
response = client.get("https://httpbin.org/get")
|
|
338
|
+
print(response.text)
|
|
339
|
+
|
|
340
|
+
# With browser impersonation
|
|
341
|
+
client = primp.Client(impersonate="chrome_141", impersonate_os="windows")
|
|
342
|
+
response = client.get("https://tls.peet.ws/api/all")
|
|
343
|
+
print(response.json())
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
### Perfect Browser Simulation
|
|
347
|
+
|
|
348
|
+
```python
|
|
349
|
+
# Complete browser simulation for anti-bot bypass
|
|
350
|
+
client = primp.Client(
|
|
351
|
+
# Browser impersonation
|
|
352
|
+
impersonate="chrome_141",
|
|
353
|
+
impersonate_os="windows",
|
|
354
|
+
|
|
355
|
+
# Advanced anti-detection
|
|
356
|
+
ordered_headers={
|
|
357
|
+
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
|
358
|
+
"sec-ch-ua": '"Chromium";v="141", "Not?A_Brand";v="8"',
|
|
359
|
+
"sec-ch-ua-mobile": "?0",
|
|
360
|
+
"sec-ch-ua-platform": '"Windows"',
|
|
361
|
+
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
362
|
+
"sec-fetch-site": "none",
|
|
363
|
+
"sec-fetch-mode": "navigate",
|
|
364
|
+
"sec-fetch-user": "?1",
|
|
365
|
+
"sec-fetch-dest": "document",
|
|
366
|
+
"accept-encoding": "gzip, deflate, br",
|
|
367
|
+
"accept-language": "en-US,en;q=0.9",
|
|
368
|
+
},
|
|
369
|
+
split_cookies=True, # HTTP/2 style cookies
|
|
370
|
+
|
|
371
|
+
# Performance optimization
|
|
372
|
+
pool_idle_timeout=90.0,
|
|
373
|
+
pool_max_idle_per_host=10,
|
|
374
|
+
tcp_nodelay=True,
|
|
375
|
+
|
|
376
|
+
# Reliability
|
|
377
|
+
retry_count=3,
|
|
378
|
+
retry_backoff=1.0,
|
|
379
|
+
timeout=30,
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
# Use like any HTTP client
|
|
383
|
+
response = client.get("https://difficult-site.com")
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
---
|
|
387
|
+
|
|
388
|
+
## 📚 Documentation
|
|
389
|
+
|
|
390
|
+
### Core Documentation
|
|
391
|
+
|
|
392
|
+
- [**Ordered Headers Guide**](ORDERED_HEADERS.md) - Master header order control for anti-bot bypass
|
|
393
|
+
- [**Split Cookies Guide**](SPLIT_COOKIES.md) - HTTP/2 cookie handling like real browsers
|
|
394
|
+
|
|
395
|
+
### Quick References
|
|
396
|
+
|
|
397
|
+
<details>
|
|
398
|
+
<summary><b>Client Parameters</b></summary>
|
|
399
|
+
|
|
400
|
+
```python
|
|
401
|
+
Client(
|
|
402
|
+
# Authentication
|
|
403
|
+
auth: tuple[str, str | None] | None = None,
|
|
404
|
+
auth_bearer: str | None = None,
|
|
405
|
+
|
|
406
|
+
# Headers & Cookies
|
|
407
|
+
headers: dict[str, str] | None = None,
|
|
408
|
+
ordered_headers: dict[str, str] | None = None, # 🆕 Ordered headers
|
|
409
|
+
cookie_store: bool = True,
|
|
410
|
+
split_cookies: bool = False, # 🆕 HTTP/2 cookie splitting
|
|
411
|
+
|
|
412
|
+
# Browser Impersonation
|
|
413
|
+
impersonate: str | None = None, # chrome_141, safari_18, etc.
|
|
414
|
+
impersonate_os: str | None = None, # windows, macos, linux, etc.
|
|
415
|
+
|
|
416
|
+
# Network Settings
|
|
417
|
+
proxy: str | None = None,
|
|
418
|
+
timeout: float = 30,
|
|
419
|
+
verify: bool = True,
|
|
420
|
+
ca_cert_file: str | None = None,
|
|
421
|
+
|
|
422
|
+
# HTTP Configuration
|
|
423
|
+
http2_only: bool = False,
|
|
424
|
+
https_only: bool = False,
|
|
425
|
+
follow_redirects: bool = True,
|
|
426
|
+
max_redirects: int = 20,
|
|
427
|
+
referer: bool = True,
|
|
428
|
+
|
|
429
|
+
# Performance Optimization
|
|
430
|
+
pool_idle_timeout: float | None = None,
|
|
431
|
+
pool_max_idle_per_host: int | None = None,
|
|
432
|
+
tcp_nodelay: bool | None = None,
|
|
433
|
+
tcp_keepalive: float | None = None,
|
|
434
|
+
|
|
435
|
+
# Retry Mechanism
|
|
436
|
+
retry_count: int | None = None,
|
|
437
|
+
retry_backoff: float | None = None,
|
|
438
|
+
|
|
439
|
+
# Query Parameters
|
|
440
|
+
params: dict[str, str] | None = None,
|
|
441
|
+
)
|
|
442
|
+
```
|
|
443
|
+
|
|
444
|
+
</details>
|
|
445
|
+
|
|
446
|
+
<details>
|
|
447
|
+
<summary><b>Request Methods</b></summary>
|
|
448
|
+
|
|
449
|
+
```python
|
|
450
|
+
# HTTP Methods
|
|
451
|
+
client.get(url, **kwargs)
|
|
452
|
+
client.post(url, **kwargs)
|
|
453
|
+
client.put(url, **kwargs)
|
|
454
|
+
client.patch(url, **kwargs)
|
|
455
|
+
client.delete(url, **kwargs)
|
|
456
|
+
client.head(url, **kwargs)
|
|
457
|
+
client.options(url, **kwargs)
|
|
458
|
+
|
|
459
|
+
# Common Parameters
|
|
460
|
+
params: dict[str, str] | None = None,
|
|
461
|
+
headers: dict[str, str] | None = None,
|
|
462
|
+
ordered_headers: dict[str, str] | None = None, # 🆕
|
|
463
|
+
cookies: dict[str, str] | None = None,
|
|
464
|
+
auth: tuple[str, str | None] | None = None,
|
|
465
|
+
auth_bearer: str | None = None,
|
|
466
|
+
timeout: float | None = None,
|
|
467
|
+
|
|
468
|
+
# POST/PUT/PATCH Specific
|
|
469
|
+
content: bytes | None = None,
|
|
470
|
+
data: dict[str, Any] | None = None,
|
|
471
|
+
json: Any | None = None,
|
|
472
|
+
files: dict[str, str] | None = None,
|
|
473
|
+
```
|
|
474
|
+
|
|
475
|
+
</details>
|
|
476
|
+
|
|
477
|
+
<details>
|
|
478
|
+
<summary><b>Response Object</b></summary>
|
|
479
|
+
|
|
480
|
+
```python
|
|
481
|
+
response.status_code # HTTP status code
|
|
482
|
+
response.headers # Response headers
|
|
483
|
+
response.cookies # Response cookies
|
|
484
|
+
response.url # Final URL (after redirects)
|
|
485
|
+
response.encoding # Content encoding
|
|
486
|
+
|
|
487
|
+
# Body Access
|
|
488
|
+
response.text # Text content
|
|
489
|
+
response.content # Binary content
|
|
490
|
+
response.json() # Parse JSON
|
|
491
|
+
response.stream() # Stream response body
|
|
492
|
+
|
|
493
|
+
# HTML Conversion
|
|
494
|
+
response.text_markdown # HTML → Markdown
|
|
495
|
+
response.text_plain # HTML → Plain text
|
|
496
|
+
response.text_rich # HTML → Rich text
|
|
497
|
+
```
|
|
498
|
+
|
|
499
|
+
</details>
|
|
500
|
+
|
|
501
|
+
<details>
|
|
502
|
+
<summary><b>Supported Browsers</b></summary>
|
|
503
|
+
|
|
504
|
+
#### Chrome (100-141)
|
|
505
|
+
`chrome_100`, `chrome_101`, `chrome_104`, `chrome_105`, `chrome_106`, `chrome_107`, `chrome_108`, `chrome_109`, `chrome_114`, `chrome_116`, `chrome_117`, `chrome_118`, `chrome_119`, `chrome_120`, `chrome_123`, `chrome_124`, `chrome_126`, `chrome_127`, `chrome_128`, `chrome_129`, `chrome_130`, `chrome_131`, `chrome_133`, `chrome_134`, `chrome_135`, `chrome_136`, `chrome_137`, `chrome_138`, `chrome_139`, `chrome_140`, `chrome_141`
|
|
506
|
+
|
|
507
|
+
#### Safari (15.3-26)
|
|
508
|
+
`safari_15.3`, `safari_15.5`, `safari_15.6.1`, `safari_16`, `safari_16.5`, `safari_17.0`, `safari_17.2.1`, `safari_17.4.1`, `safari_17.5`, `safari_18`, `safari_18.2`, `safari_26`, `safari_ios_16.5`, `safari_ios_17.2`, `safari_ios_17.4.1`, `safari_ios_18.1.1`, `safari_ios_26`, `safari_ipad_18`, `safari_ipad_26`
|
|
509
|
+
|
|
510
|
+
#### Firefox (109-143)
|
|
511
|
+
`firefox_109`, `firefox_117`, `firefox_128`, `firefox_133`, `firefox_135`, `firefox_136`, `firefox_139`, `firefox_142`, `firefox_143`
|
|
512
|
+
|
|
513
|
+
#### Edge (101-134)
|
|
514
|
+
`edge_101`, `edge_122`, `edge_127`, `edge_131`, `edge_134`
|
|
515
|
+
|
|
516
|
+
#### OkHttp (3.9-5.0)
|
|
517
|
+
`okhttp_3.9`, `okhttp_3.11`, `okhttp_3.13`, `okhttp_3.14`, `okhttp_4.9`, `okhttp_4.10`, `okhttp_5`
|
|
518
|
+
|
|
519
|
+
#### OS Support
|
|
520
|
+
`windows`, `macos`, `linux`, `android`, `ios`
|
|
521
|
+
|
|
522
|
+
</details>
|
|
523
|
+
|
|
524
|
+
---
|
|
525
|
+
|
|
526
|
+
## 💡 Examples
|
|
527
|
+
|
|
528
|
+
### Example 1: Web Scraping with Anti-Bot Bypass
|
|
529
|
+
|
|
530
|
+
```python
|
|
531
|
+
import never_primp as primp
|
|
532
|
+
|
|
533
|
+
# Perfect browser simulation
|
|
534
|
+
client = primp.Client(
|
|
535
|
+
impersonate="chrome_141",
|
|
536
|
+
impersonate_os="windows",
|
|
537
|
+
ordered_headers={
|
|
538
|
+
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
|
539
|
+
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
540
|
+
"accept-language": "en-US,en;q=0.9",
|
|
541
|
+
"accept-encoding": "gzip, deflate, br",
|
|
542
|
+
},
|
|
543
|
+
split_cookies=True,
|
|
544
|
+
retry_count=3,
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
response = client.get("https://difficult-site.com")
|
|
548
|
+
print(response.status_code)
|
|
549
|
+
```
|
|
550
|
+
|
|
551
|
+
### Example 2: API Integration with Authentication
|
|
552
|
+
|
|
553
|
+
```python
|
|
554
|
+
client = primp.Client(
|
|
555
|
+
headers={
|
|
556
|
+
"Content-Type": "application/json",
|
|
557
|
+
"X-API-Version": "v1",
|
|
558
|
+
},
|
|
559
|
+
auth_bearer="your-api-token",
|
|
560
|
+
timeout=30,
|
|
561
|
+
)
|
|
562
|
+
|
|
563
|
+
# GET request
|
|
564
|
+
data = client.get("https://api.example.com/users").json()
|
|
565
|
+
|
|
566
|
+
# POST request
|
|
567
|
+
response = client.post(
|
|
568
|
+
"https://api.example.com/users",
|
|
569
|
+
json={"name": "John", "email": "john@example.com"}
|
|
570
|
+
)
|
|
571
|
+
```
|
|
572
|
+
|
|
573
|
+
### Example 3: File Upload
|
|
574
|
+
|
|
575
|
+
```python
|
|
576
|
+
client = primp.Client()
|
|
577
|
+
|
|
578
|
+
files = {
|
|
579
|
+
'document': '/path/to/document.pdf',
|
|
580
|
+
'image': '/path/to/image.png'
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
response = client.post(
|
|
584
|
+
"https://example.com/upload",
|
|
585
|
+
files=files,
|
|
586
|
+
data={"description": "My files"}
|
|
587
|
+
)
|
|
588
|
+
```
|
|
589
|
+
|
|
590
|
+
### Example 4: Session Management
|
|
591
|
+
|
|
592
|
+
```python
|
|
593
|
+
# Automatic cookie persistence
|
|
594
|
+
client = primp.Client(cookie_store=True)
|
|
595
|
+
|
|
596
|
+
# Login
|
|
597
|
+
client.post(
|
|
598
|
+
"https://example.com/login",
|
|
599
|
+
data={"username": "user", "password": "pass"}
|
|
600
|
+
)
|
|
601
|
+
|
|
602
|
+
# Subsequent requests include session cookies
|
|
603
|
+
profile = client.get("https://example.com/profile")
|
|
604
|
+
```
|
|
605
|
+
|
|
606
|
+
### Example 5: Proxy Usage
|
|
607
|
+
|
|
608
|
+
```python
|
|
609
|
+
# SOCKS5 proxy
|
|
610
|
+
client = primp.Client(proxy="socks5://127.0.0.1:1080")
|
|
611
|
+
|
|
612
|
+
# HTTP proxy with authentication
|
|
613
|
+
client = primp.Client(proxy="http://user:pass@proxy.example.com:8080")
|
|
614
|
+
|
|
615
|
+
# Environment variable
|
|
616
|
+
import os
|
|
617
|
+
os.environ['PRIMP_PROXY'] = 'http://127.0.0.1:8080'
|
|
618
|
+
```
|
|
619
|
+
|
|
620
|
+
### Example 6: Async Concurrent Requests
|
|
621
|
+
|
|
622
|
+
```python
|
|
623
|
+
import asyncio
|
|
624
|
+
import never_primp as primp
|
|
625
|
+
|
|
626
|
+
async def fetch_all(urls):
|
|
627
|
+
async with primp.AsyncClient(impersonate="chrome_141") as client:
|
|
628
|
+
tasks = [client.get(url) for url in urls]
|
|
629
|
+
responses = await asyncio.gather(*tasks)
|
|
630
|
+
return [r.text for r in responses]
|
|
631
|
+
|
|
632
|
+
urls = ["https://site1.com", "https://site2.com", "https://site3.com"]
|
|
633
|
+
results = asyncio.run(fetch_all(urls))
|
|
634
|
+
```
|
|
635
|
+
|
|
636
|
+
### Example 7: Streaming Large Files
|
|
637
|
+
|
|
638
|
+
```python
|
|
639
|
+
client = primp.Client()
|
|
640
|
+
|
|
641
|
+
response = client.get("https://example.com/large-file.zip")
|
|
642
|
+
|
|
643
|
+
with open("output.zip", "wb") as f:
|
|
644
|
+
for chunk in response.stream():
|
|
645
|
+
f.write(chunk)
|
|
646
|
+
```
|
|
647
|
+
|
|
648
|
+
---
|
|
649
|
+
|
|
650
|
+
## 🎯 Use Cases
|
|
651
|
+
|
|
652
|
+
### ✅ Perfect For
|
|
653
|
+
|
|
654
|
+
- **Web Scraping**: Bypass anti-bot systems (Cloudflare, Akamai, PerimeterX)
|
|
655
|
+
- **API Testing**: High-performance API client with retries
|
|
656
|
+
- **Data Collection**: Concurrent requests with connection pooling
|
|
657
|
+
- **Security Research**: TLS fingerprint analysis and testing
|
|
658
|
+
- **Browser Automation Alternative**: Lighter than Selenium/Playwright
|
|
659
|
+
|
|
660
|
+
### ⚠️ Not Suitable For
|
|
661
|
+
|
|
662
|
+
- **JavaScript Rendering**: Use Playwright/Selenium for dynamic content
|
|
663
|
+
- **Browser Automation**: No DOM manipulation or JavaScript execution
|
|
664
|
+
- **Visual Testing**: No screenshot or rendering capabilities
|
|
665
|
+
|
|
666
|
+
---
|
|
667
|
+
|
|
668
|
+
## 🔬 Benchmarks
|
|
669
|
+
|
|
670
|
+
### Sequential Requests (Connection Reuse)
|
|
671
|
+
|
|
672
|
+
| Library | Time (10 requests) | Relative Speed |
|
|
673
|
+
|---------|-------------------|----------------|
|
|
674
|
+
| **never_primp** | 1.24s | **1.00x** (baseline) |
|
|
675
|
+
| httpx | 1.89s | 0.66x slower |
|
|
676
|
+
| requests | 3.05s | 0.41x slower |
|
|
677
|
+
|
|
678
|
+
### Concurrent Requests (AsyncClient)
|
|
679
|
+
|
|
680
|
+
| Library | Time (100 requests) | Relative Speed |
|
|
681
|
+
|---------|---------------------|----------------|
|
|
682
|
+
| **never_primp** | 2.15s | **1.00x** (baseline) |
|
|
683
|
+
| httpx | 2.83s | 0.76x slower |
|
|
684
|
+
| aiohttp | 2.45s | 0.88x slower |
|
|
685
|
+
|
|
686
|
+
*Benchmarks run on: Python 3.11, Ubuntu 22.04, AMD Ryzen 9 5900X*
|
|
687
|
+
|
|
688
|
+
---
|
|
689
|
+
|
|
690
|
+
## 🛠️ Development
|
|
691
|
+
|
|
692
|
+
### Building from Source
|
|
693
|
+
|
|
694
|
+
```bash
|
|
695
|
+
# Clone repository
|
|
696
|
+
git clone https://github.com/yourusername/never-primp.git
|
|
697
|
+
cd never-primp
|
|
698
|
+
|
|
699
|
+
# Create virtual environment
|
|
700
|
+
python -m venv venv
|
|
701
|
+
source venv/bin/activate # Linux/macOS
|
|
702
|
+
# or
|
|
703
|
+
venv\Scripts\activate # Windows
|
|
704
|
+
|
|
705
|
+
# Install maturin (Rust-Python build tool)
|
|
706
|
+
pip install maturin
|
|
707
|
+
|
|
708
|
+
# Build and install in development mode
|
|
709
|
+
maturin develop --release
|
|
710
|
+
|
|
711
|
+
# Run examples
|
|
712
|
+
python examples/example_ordered_headers.py
|
|
713
|
+
```
|
|
714
|
+
|
|
715
|
+
### Project Structure
|
|
716
|
+
|
|
717
|
+
```
|
|
718
|
+
never-primp/
|
|
719
|
+
├── src/
|
|
720
|
+
│ ├── lib.rs # Main Rust implementation
|
|
721
|
+
│ ├── traits.rs # Header conversion traits
|
|
722
|
+
│ ├── response.rs # Response handling
|
|
723
|
+
│ ├── impersonate.rs # Browser impersonation
|
|
724
|
+
│ └── utils.rs # Certificate utilities
|
|
725
|
+
├── never_primp/
|
|
726
|
+
│ ├── __init__.py # Python API wrapper
|
|
727
|
+
│ └── never_primp.pyi # Type hints
|
|
728
|
+
├── examples/
|
|
729
|
+
│ ├── example_ordered_headers.py
|
|
730
|
+
│ └── example_split_cookies.py
|
|
731
|
+
├── Cargo.toml # Rust dependencies
|
|
732
|
+
└── pyproject.toml # Python package config
|
|
733
|
+
```
|
|
734
|
+
|
|
735
|
+
---
|
|
736
|
+
|
|
737
|
+
## 🤝 Contributing
|
|
738
|
+
|
|
739
|
+
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
740
|
+
|
|
741
|
+
### Development Guidelines
|
|
742
|
+
|
|
743
|
+
1. Follow Rust best practices for src/ files
|
|
744
|
+
2. Maintain Python 3.8+ compatibility
|
|
745
|
+
3. Add tests for new features
|
|
746
|
+
4. Update documentation
|
|
747
|
+
|
|
748
|
+
---
|
|
749
|
+
|
|
750
|
+
## 📄 License
|
|
751
|
+
|
|
752
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
753
|
+
|
|
754
|
+
---
|
|
755
|
+
|
|
756
|
+
## ⚠️ Disclaimer
|
|
757
|
+
|
|
758
|
+
This tool is intended for **educational purposes** and **legitimate use cases** only, such as:
|
|
759
|
+
- Testing your own applications
|
|
760
|
+
- Academic research
|
|
761
|
+
- Security auditing (with permission)
|
|
762
|
+
- Data collection from public APIs
|
|
763
|
+
|
|
764
|
+
**Important**:
|
|
765
|
+
- Respect websites' `robots.txt` and Terms of Service
|
|
766
|
+
- Do not use for malicious purposes or unauthorized access
|
|
767
|
+
- Be mindful of rate limiting and server resources
|
|
768
|
+
- The authors are not responsible for misuse of this tool
|
|
769
|
+
|
|
770
|
+
Use responsibly and ethically. 🙏
|
|
771
|
+
|
|
772
|
+
---
|
|
773
|
+
|
|
774
|
+
## 🙏 Acknowledgments
|
|
775
|
+
|
|
776
|
+
Built with:
|
|
777
|
+
- [wreq](https://github.com/0x676e67/wreq) - Rust HTTP client with browser impersonation
|
|
778
|
+
- [PyO3](https://github.com/PyO3/pyo3) - Rust bindings for Python
|
|
779
|
+
- [tokio](https://tokio.rs/) - Async runtime for Rust
|
|
780
|
+
|
|
781
|
+
Inspired by:
|
|
782
|
+
- [curl-impersonate](https://github.com/lwthiker/curl-impersonate)
|
|
783
|
+
- [httpx](https://github.com/encode/httpx)
|
|
784
|
+
- [requests](https://github.com/psf/requests)
|
|
785
|
+
|
|
786
|
+
---
|
|
787
|
+
|
|
788
|
+
## 📞 Support
|
|
789
|
+
|
|
790
|
+
- 📖 [Documentation](ORDERED_HEADERS.md)
|
|
791
|
+
- 🐛 [Issue Tracker](https://github.com/yourusername/never-primp/issues)
|
|
792
|
+
- 💬 [Discussions](https://github.com/yourusername/never-primp/discussions)
|
|
793
|
+
|
|
794
|
+
---
|
|
795
|
+
|
|
796
|
+
<div align="center">
|
|
797
|
+
|
|
798
|
+
**Made with ❤️ and ⚙️ Rust**
|
|
799
|
+
|
|
800
|
+
If you find this project helpful, please consider giving it a ⭐!
|
|
801
|
+
|
|
802
|
+
</div>
|
|
803
|
+
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
never_primp-1.0.1.dist-info/METADATA,sha256=8QBNm_pn-t2_2K_VgbQ5wSKe1L7YoBiDPbTYun6HXFk,21538
|
|
2
|
+
never_primp-1.0.1.dist-info/WHEEL,sha256=GPFYR4mIPdHeeQ1ax6E8LJaGIrcByvaZAJbs3IVoO7M,104
|
|
3
|
+
never_primp-1.0.1.dist-info/licenses/LICENSE,sha256=ZPD9tCar0h91tI4v-TuZdrjDdLqzU4rzPTxtP3x--uc,1063
|
|
4
|
+
never_primp/__init__.py,sha256=z7rifYljWlSLbOjJSIDU56fBdbbXAfgfDq8uIuCfVo8,13493
|
|
5
|
+
never_primp/never_primp.abi3.so,sha256=1cdqWR1qFg-ljkzlXd63KbLzizbi1u4bESWF24tqUqs,7001780
|
|
6
|
+
never_primp/never_primp.pyi,sha256=8QRuXaqNfk29qkFZrbtwKKK92Y_wL-O702nys8DCcM0,5731
|
|
7
|
+
never_primp/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
never_primp-1.0.1.dist-info/RECORD,,
|
|
@@ -1,358 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: never_primp
|
|
3
|
-
Version: 1.0.0
|
|
4
|
-
Classifier: Development Status :: 5 - Production/Stable
|
|
5
|
-
Classifier: Intended Audience :: Developers
|
|
6
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
7
|
-
Classifier: Programming Language :: Python :: 3
|
|
8
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
9
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
-
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
15
|
-
Classifier: Programming Language :: Rust
|
|
16
|
-
Classifier: Topic :: Internet :: WWW/HTTP
|
|
17
|
-
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
-
License-File: LICENSE
|
|
19
|
-
Summary: 基于原primp 重新优化调整的请求库 - The fastest python HTTP client that can impersonate web browsers
|
|
20
|
-
Keywords: requests,httpx,http,http-client,tls-fingerprint,ja3,ja4,impersonate,browser-impersonation,web-scraping,crawler,reverse-engineering
|
|
21
|
-
Author: Neverland
|
|
22
|
-
Requires-Python: >=3.8
|
|
23
|
-
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
24
|
-
Project-URL: Homepage, https://github.com/Neverland/never_primp
|
|
25
|
-
Project-URL: Repository, https://github.com/Neverland/never_primp
|
|
26
|
-
Project-URL: Bug Tracker, https://github.com/Neverland/never_primp/issues
|
|
27
|
-
|
|
28
|
-
 [](https://github.com/deedy5/pyreqwest-impersonate/releases) [](https://pypi.org/project/primp) [](https://pepy.tech/project/primp) [](https://github.com/deedy5/pyreqwest-impersonate/actions/workflows/CI.yml)
|
|
29
|
-
# 🪞NEVER_PRIMP
|
|
30
|
-
**🪞NEVER_PRIMP** = **P**ython **R**equests **IMP**ersonate
|
|
31
|
-
|
|
32
|
-
The fastest python HTTP client that can impersonate web browsers.</br>
|
|
33
|
-
Provides precompiled wheels:</br>
|
|
34
|
-
* 🐧 linux: `amd64`, `aarch64`, `armv7` (⚠️aarch64 and armv7 builds are `manylinux_2_34` compatible - `ubuntu>=22.04`, `debian>=12`);</br>
|
|
35
|
-
* 🐧 musllinux: `amd64`, `aarch64`;</br>
|
|
36
|
-
* 🪟 windows: `amd64`;</br>
|
|
37
|
-
* 🍏 macos: `amd64`, `aarch64`.</br>
|
|
38
|
-
|
|
39
|
-
## Table of Contents
|
|
40
|
-
|
|
41
|
-
- [Installation](#installation)
|
|
42
|
-
- [Key Features](#key-features)
|
|
43
|
-
- [Benchmark](#benchmark)
|
|
44
|
-
- [Usage](#usage)
|
|
45
|
-
- [I. Client](#i-client)
|
|
46
|
-
- [Client methods](#client-methods)
|
|
47
|
-
- [Response object](#response-object)
|
|
48
|
-
- [Devices](#devices)
|
|
49
|
-
- [Examples](#examples)
|
|
50
|
-
- [II. AsyncClient](#ii-asyncclient)
|
|
51
|
-
- [Disclaimer](#disclaimer)
|
|
52
|
-
|
|
53
|
-
## Installation
|
|
54
|
-
|
|
55
|
-
```python
|
|
56
|
-
pip install -U never_primp
|
|
57
|
-
```
|
|
58
|
-
|
|
59
|
-
## Key Features
|
|
60
|
-
|
|
61
|
-
🚀 **Performance Optimized**
|
|
62
|
-
- Connection pooling with configurable idle timeout and max connections per host
|
|
63
|
-
- TCP optimization (TCP_NODELAY, TCP keepalive)
|
|
64
|
-
- ~59% faster for sequential requests with connection reuse
|
|
65
|
-
|
|
66
|
-
🔒 **Advanced Certificate Management**
|
|
67
|
-
- Uses system's native certificate store (auto-updated with OS)
|
|
68
|
-
- No more certificate expiration issues
|
|
69
|
-
- Custom CA bundle support
|
|
70
|
-
|
|
71
|
-
🍪 **Smart Cookie Management**
|
|
72
|
-
- Automatic cookie persistence using wreq's native Jar API
|
|
73
|
-
- Manual cookie control with `get_cookies()` / `set_cookies()`
|
|
74
|
-
- Cookies survive client configuration changes
|
|
75
|
-
|
|
76
|
-
⚙️ **Dynamic Configuration**
|
|
77
|
-
- Modify headers, proxy, and impersonation at runtime
|
|
78
|
-
- No need to recreate the client
|
|
79
|
-
- Thread-safe configuration updates
|
|
80
|
-
|
|
81
|
-
🔄 **Retry Mechanism**
|
|
82
|
-
- Configurable retry count and backoff timing
|
|
83
|
-
- Handle transient failures gracefully
|
|
84
|
-
|
|
85
|
-
🎭 **Browser Impersonation**
|
|
86
|
-
- Impersonate Chrome, Safari, Edge, Firefox, OkHttp
|
|
87
|
-
- Mimic TLS/JA3/JA4/HTTP2 fingerprints
|
|
88
|
-
- Custom OS impersonation (Windows, macOS, Linux, Android, iOS)
|
|
89
|
-
|
|
90
|
-
## Usage
|
|
91
|
-
### I. Client
|
|
92
|
-
|
|
93
|
-
HTTP client that can impersonate web browsers.
|
|
94
|
-
```python
|
|
95
|
-
class Client:
|
|
96
|
-
"""Initializes an HTTP client that can impersonate web browsers.
|
|
97
|
-
|
|
98
|
-
Args:
|
|
99
|
-
auth (tuple[str, str| None] | None): Username and password for basic authentication. Default is None.
|
|
100
|
-
auth_bearer (str | None): Bearer token for authentication. Default is None.
|
|
101
|
-
params (dict[str, str] | None): Default query parameters to include in all requests. Default is None.
|
|
102
|
-
headers (dict[str, str] | None): Default headers to send with requests. If `impersonate` is set, this will be ignored.
|
|
103
|
-
timeout (float | None): HTTP request timeout in seconds. Default is 30.
|
|
104
|
-
cookie_store (bool | None): Enable a persistent cookie store. Received cookies will be preserved and included
|
|
105
|
-
in additional requests. Default is True.
|
|
106
|
-
referer (bool | None): Enable or disable automatic setting of the `Referer` header. Default is True.
|
|
107
|
-
proxy (str | None): Proxy URL for HTTP requests. Example: "socks5://127.0.0.1:9150". Default is None.
|
|
108
|
-
impersonate (str | None): Entity to impersonate. Example: "chrome_124". Default is None.
|
|
109
|
-
Chrome: "chrome_100","chrome_101","chrome_104","chrome_105","chrome_106","chrome_107","chrome_108",
|
|
110
|
-
"chrome_109","chrome_114","chrome_116","chrome_117","chrome_118","chrome_119","chrome_120",
|
|
111
|
-
"chrome_123","chrome_124","chrome_126","chrome_127","chrome_128","chrome_129","chrome_130",
|
|
112
|
-
"chrome_131","chrome_133"
|
|
113
|
-
Safari: "safari_ios_16.5","safari_ios_17.2","safari_ios_17.4.1","safari_ios_18.1.1",
|
|
114
|
-
"safari_15.3","safari_15.5","safari_15.6.1","safari_16","safari_16.5","safari_17.0",
|
|
115
|
-
"safari_17.2.1","safari_17.4.1","safari_17.5","safari_18","safari_18.2","safari_ipad_18"
|
|
116
|
-
OkHttp: "okhttp_3.9","okhttp_3.11","okhttp_3.13","okhttp_3.14","okhttp_4.9","okhttp_4.10","okhttp_5"
|
|
117
|
-
Edge: "edge_101","edge_122","edge_127","edge_131"
|
|
118
|
-
Firefox: "firefox_109","firefox_117","firefox_128","firefox_133","firefox_135"
|
|
119
|
-
Select random: "random"
|
|
120
|
-
impersonate_os (str | None): impersonate OS. Example: "windows". Default is "linux".
|
|
121
|
-
Android: "android", iOS: "ios", Linux: "linux", Mac OS: "macos", Windows: "windows"
|
|
122
|
-
Select random: "random"
|
|
123
|
-
follow_redirects (bool | None): Whether to follow redirects. Default is True.
|
|
124
|
-
max_redirects (int | None): Maximum redirects to follow. Default 20. Applies if `follow_redirects` is True.
|
|
125
|
-
verify (bool | None): Verify SSL certificates. Default is True.
|
|
126
|
-
ca_cert_file (str | None): Path to CA certificate store. Default is None.
|
|
127
|
-
https_only (bool | None): Restrict the Client to be used with HTTPS only requests. Default is False.
|
|
128
|
-
http2_only (bool | None): If true - use only HTTP/2; if false - use only HTTP/1. Default is False.
|
|
129
|
-
pool_idle_timeout (float | None): Connection pool idle timeout in seconds. Default is None.
|
|
130
|
-
pool_max_idle_per_host (int | None): Maximum number of idle connections per host. Default is None.
|
|
131
|
-
tcp_nodelay (bool | None): Enable TCP_NODELAY (disable Nagle's algorithm). Default is None.
|
|
132
|
-
tcp_keepalive (float | None): TCP keepalive interval in seconds. Default is None.
|
|
133
|
-
retry_count (int | None): Maximum number of retry attempts. Default is None.
|
|
134
|
-
retry_backoff (float | None): Backoff time between retries in seconds. Default is None.
|
|
135
|
-
|
|
136
|
-
"""
|
|
137
|
-
```
|
|
138
|
-
|
|
139
|
-
#### Client methods
|
|
140
|
-
|
|
141
|
-
The `Client` class provides a set of methods for making HTTP requests: `get`, `head`, `options`, `delete`, `post`, `put`, `patch`, each of which internally utilizes the `request()` method for execution. The parameters for these methods closely resemble those in `httpx`.
|
|
142
|
-
```python
|
|
143
|
-
def get(
|
|
144
|
-
url: str,
|
|
145
|
-
params: dict[str, str] | None = None,
|
|
146
|
-
headers: dict[str, str] | None = None,
|
|
147
|
-
cookies: dict[str, str] | None = None,
|
|
148
|
-
auth: tuple[str, str| None] | None = None,
|
|
149
|
-
auth_bearer: str | None = None,
|
|
150
|
-
timeout: float | None = 30,
|
|
151
|
-
):
|
|
152
|
-
"""Performs a GET request to the specified URL.
|
|
153
|
-
|
|
154
|
-
Args:
|
|
155
|
-
url (str): The URL to which the request will be made.
|
|
156
|
-
params (dict[str, str] | None): A map of query parameters to append to the URL. Default is None.
|
|
157
|
-
headers (dict[str, str] | None): A map of HTTP headers to send with the request. Default is None.
|
|
158
|
-
cookies (dict[str, str] | None): - An optional map of cookies to send with requests as the `Cookie` header.
|
|
159
|
-
auth (tuple[str, str| None] | None): A tuple containing the username and an optional password
|
|
160
|
-
for basic authentication. Default is None.
|
|
161
|
-
auth_bearer (str | None): A string representing the bearer token for bearer token authentication. Default is None.
|
|
162
|
-
timeout (float | None): The timeout for the request in seconds. Default is 30.
|
|
163
|
-
|
|
164
|
-
"""
|
|
165
|
-
```
|
|
166
|
-
```python
|
|
167
|
-
def post(
|
|
168
|
-
url: str,
|
|
169
|
-
params: dict[str, str] | None = None,
|
|
170
|
-
headers: dict[str, str] | None = None,
|
|
171
|
-
cookies: dict[str, str] | None = None,
|
|
172
|
-
content: bytes | None = None,
|
|
173
|
-
data: dict[str, Any] | None = None,
|
|
174
|
-
json: Any | None = None,
|
|
175
|
-
files: dict[str, str] | None = None,
|
|
176
|
-
auth: tuple[str, str| None] | None = None,
|
|
177
|
-
auth_bearer: str | None = None,
|
|
178
|
-
timeout: float | None = 30,
|
|
179
|
-
):
|
|
180
|
-
"""Performs a POST request to the specified URL.
|
|
181
|
-
|
|
182
|
-
Args:
|
|
183
|
-
url (str): The URL to which the request will be made.
|
|
184
|
-
params (dict[str, str] | None): A map of query parameters to append to the URL. Default is None.
|
|
185
|
-
headers (dict[str, str] | None): A map of HTTP headers to send with the request. Default is None.
|
|
186
|
-
cookies (dict[str, str] | None): - An optional map of cookies to send with requests as the `Cookie` header.
|
|
187
|
-
content (bytes | None): The content to send in the request body as bytes. Default is None.
|
|
188
|
-
data (dict[str, Any] | None): The form data to send in the request body. Default is None.
|
|
189
|
-
json (Any | None): A JSON serializable object to send in the request body. Default is None.
|
|
190
|
-
files (dict[str, str] | None): A map of file fields to file paths to be sent as multipart/form-data. Default is None.
|
|
191
|
-
auth (tuple[str, str| None] | None): A tuple containing the username and an optional password
|
|
192
|
-
for basic authentication. Default is None.
|
|
193
|
-
auth_bearer (str | None): A string representing the bearer token for bearer token authentication. Default is None.
|
|
194
|
-
timeout (float | None): The timeout for the request in seconds. Default is 30.
|
|
195
|
-
|
|
196
|
-
"""
|
|
197
|
-
```
|
|
198
|
-
#### Response object
|
|
199
|
-
```python
|
|
200
|
-
resp.content
|
|
201
|
-
resp.stream() # stream the response body in chunks of bytes
|
|
202
|
-
resp.cookies
|
|
203
|
-
resp.encoding
|
|
204
|
-
resp.headers
|
|
205
|
-
resp.json()
|
|
206
|
-
resp.status_code
|
|
207
|
-
resp.text
|
|
208
|
-
resp.text_markdown # html is converted to markdown text
|
|
209
|
-
resp.text_plain # html is converted to plain text
|
|
210
|
-
resp.text_rich # html is converted to rich text
|
|
211
|
-
resp.url
|
|
212
|
-
```
|
|
213
|
-
|
|
214
|
-
#### Devices
|
|
215
|
-
|
|
216
|
-
##### Impersonate
|
|
217
|
-
|
|
218
|
-
- Chrome: `chrome_100`,`chrome_101`,`chrome_104`,`chrome_105`,`chrome_106`,`chrome_107`,`chrome_108`,`chrome_109`,`chrome_114`,`chrome_116`,`chrome_117`,`chrome_118`,`chrome_119`,`chrome_120`,`chrome_123`,`chrome_124`,`chrome_126`,`chrome_127`,`chrome_128`,`chrome_129`,`chrome_130`,`chrome_131`, `chrome_133`, `chrome_134`, `chrome_135`, `chrome_136`, `chrome_137`, `chrome_138`, `chrome_139`, `chrome_140`, `chrome_141`
|
|
219
|
-
|
|
220
|
-
- Edge: `edge_101`,`edge_122`,`edge_127`, `edge_131`, `edge_134`
|
|
221
|
-
|
|
222
|
-
- Safari: `safari_ios_17.2`,`safari_ios_17.4.1`,`safari_ios_16.5`,`safari_ios_18.1.1`, `safari_15.3`,`safari_15.5`,`safari_15.6.1`,`safari_16`,`safari_16.5`,`safari_17.0`,`safari_17.2.1`,`safari_17.4.1`,`safari_17.5`,`safari_18`,`safari_18.2`, `safari_ipad_18`, `safari_ipad_26`, `safari_ios_26`, `safari_26`
|
|
223
|
-
|
|
224
|
-
- OkHttp: `okhttp_3.9`,`okhttp_3.11`,`okhttp_3.13`,`okhttp_3.14`,`okhttp_4.9`,`okhttp_4.10`,`okhttp_5`
|
|
225
|
-
|
|
226
|
-
- Firefox: `firefox_109`, `firefox_117`, `firefox_128`, `firefox_133`, `firefox_135`, `firefox_136`, `firefox_139`, `firefox_142`, `firefox_143`
|
|
227
|
-
|
|
228
|
-
##### Impersonate OS
|
|
229
|
-
|
|
230
|
-
- `android`, `ios`, `linux`, `macos`, `windows`
|
|
231
|
-
|
|
232
|
-
#### Examples
|
|
233
|
-
|
|
234
|
-
```python
|
|
235
|
-
import never_primp as primp
|
|
236
|
-
|
|
237
|
-
# Impersonate
|
|
238
|
-
client = primp.Client(impersonate="chrome_131", impersonate_os="windows")
|
|
239
|
-
|
|
240
|
-
# Performance optimization with connection pooling
|
|
241
|
-
client = primp.Client(
|
|
242
|
-
impersonate="chrome_131",
|
|
243
|
-
pool_idle_timeout=90.0, # Keep connections alive for 90 seconds
|
|
244
|
-
pool_max_idle_per_host=10, # Max 10 idle connections per host
|
|
245
|
-
tcp_nodelay=True, # Disable Nagle's algorithm for lower latency
|
|
246
|
-
tcp_keepalive=60.0, # TCP keepalive every 60 seconds
|
|
247
|
-
)
|
|
248
|
-
|
|
249
|
-
# Retry mechanism
|
|
250
|
-
client = primp.Client(
|
|
251
|
-
retry_count=3, # Retry up to 3 times
|
|
252
|
-
retry_backoff=1.0, # 1 second backoff between retries
|
|
253
|
-
)
|
|
254
|
-
|
|
255
|
-
# Dynamic configuration updates
|
|
256
|
-
client.headers = {"User-Agent": "Custom User Agent"} # Update all headers
|
|
257
|
-
client.headers_update({"Referer": "https://example.com"}) # Merge new headers
|
|
258
|
-
client.proxy = "http://127.0.0.1:8080" # Change proxy
|
|
259
|
-
client.impersonate = "chrome_133" # Change browser impersonation
|
|
260
|
-
client.impersonate_os = "macos" # Change OS impersonation
|
|
261
|
-
|
|
262
|
-
# GET request
|
|
263
|
-
resp = client.get("https://tls.peet.ws/api/all")
|
|
264
|
-
|
|
265
|
-
# GET request with passing params and setting timeout
|
|
266
|
-
params = {"param1": "value1", "param2": "value2"}
|
|
267
|
-
resp = client.post(url="https://httpbin.org/anything", params=params, timeout=10)
|
|
268
|
-
|
|
269
|
-
# Stream response
|
|
270
|
-
resp = client.get("https://nytimes")
|
|
271
|
-
for chunk in resp.stream():
|
|
272
|
-
print(chunk)
|
|
273
|
-
|
|
274
|
-
# Cookie management - Automatic (recommended)
|
|
275
|
-
client = primp.Client(cookie_store=True) # Default: enabled
|
|
276
|
-
resp = client.get("https://httpbin.org/cookies/set?session=abc123")
|
|
277
|
-
# Cookies are automatically stored and sent in subsequent requests
|
|
278
|
-
resp2 = client.get("https://httpbin.org/cookies") # Session cookie automatically included
|
|
279
|
-
|
|
280
|
-
# Cookie management - Manual
|
|
281
|
-
cookies = {"c1_n": "c1_value", "c2_n": "c2_value"}
|
|
282
|
-
client.set_cookies(url="https://nytimes.com", cookies=cookies) # Set cookies for a specific domain
|
|
283
|
-
resp = client.get("https://nytimes.com/", cookies=cookies) # Or pass cookies in request
|
|
284
|
-
|
|
285
|
-
# Get cookies
|
|
286
|
-
all_cookies = client.get_cookies(url="https://nytimes.com") # Get all cookies from jar
|
|
287
|
-
response_cookies = resp.cookies # Get cookies from response
|
|
288
|
-
|
|
289
|
-
# POST Binary Request Data
|
|
290
|
-
content = b"some_data"
|
|
291
|
-
resp = client.post(url="https://httpbin.org/anything", content=content)
|
|
292
|
-
|
|
293
|
-
# POST Form Encoded Data
|
|
294
|
-
data = {"key1": "value1", "key2": "value2"}
|
|
295
|
-
resp = client.post(url="https://httpbin.org/anything", data=data)
|
|
296
|
-
|
|
297
|
-
# POST JSON Encoded Data
|
|
298
|
-
json = {"key1": "value1", "key2": "value2"}
|
|
299
|
-
resp = client.post(url="https://httpbin.org/anything", json=json)
|
|
300
|
-
|
|
301
|
-
# POST Multipart-Encoded Files
|
|
302
|
-
files = {'file1': '/home/root/file1.txt', 'file2': 'home/root/file2.txt'}
|
|
303
|
-
resp = client.post("https://httpbin.org/post", files=files)
|
|
304
|
-
|
|
305
|
-
# Authentication using user/password
|
|
306
|
-
resp = client.post(url="https://httpbin.org/anything", auth=("user", "password"))
|
|
307
|
-
|
|
308
|
-
# Authentication using auth bearer
|
|
309
|
-
resp = client.post(url="https://httpbin.org/anything", auth_bearer="bearerXXXXXXXXXXXXXXXXXXXX")
|
|
310
|
-
|
|
311
|
-
# Using proxy or env var PRIMP_PROXY
|
|
312
|
-
resp = primp.Client(proxy="http://127.0.0.1:8080") # set proxy in Client
|
|
313
|
-
export PRIMP_PROXY="socks5://127.0.0.1:1080" # set proxy as environment variable
|
|
314
|
-
|
|
315
|
-
# SSL/TLS certificate verification
|
|
316
|
-
# Note: Primp uses your system's native certificate store by default (auto-updated with OS)
|
|
317
|
-
resp = primp.Client(verify=True) # Default: uses system certificates
|
|
318
|
-
|
|
319
|
-
# Using custom CA certificate store:
|
|
320
|
-
resp = primp.Client(ca_cert_file="/cert/cacert.pem")
|
|
321
|
-
resp = primp.Client(ca_cert_file=certifi.where())
|
|
322
|
-
export PRIMP_CA_BUNDLE="/home/user/Downloads/cert.pem" # set as environment variable
|
|
323
|
-
|
|
324
|
-
# You can also use convenience functions that use a default Client instance under the hood:
|
|
325
|
-
# primp.get() | primp.head() | primp.options() | primp.delete() | primp.post() | primp.patch() | primp.put()
|
|
326
|
-
# These functions can accept the `impersonate` parameter:
|
|
327
|
-
resp = primp.get("https://httpbin.org/anything", impersonate="chrome_131", impersonate_os="android")
|
|
328
|
-
```
|
|
329
|
-
|
|
330
|
-
### II. AsyncClient
|
|
331
|
-
|
|
332
|
-
`primp.AsyncClient()` is an asynchronous wrapper around the `primp.Client` class, offering the same functions, behavior, and input arguments.
|
|
333
|
-
|
|
334
|
-
```python3
|
|
335
|
-
import asyncio
|
|
336
|
-
import logging
|
|
337
|
-
|
|
338
|
-
import never_primp as primp
|
|
339
|
-
|
|
340
|
-
async def aget_text(url):
|
|
341
|
-
async with primp.AsyncClient(impersonate="chrome_131") as client:
|
|
342
|
-
resp = await client.get(url)
|
|
343
|
-
return resp.text
|
|
344
|
-
|
|
345
|
-
async def main():
|
|
346
|
-
urls = ["https://nytimes.com/", "https://cnn.com/", "https://abcnews.go.com/"]
|
|
347
|
-
tasks = [aget_text(u) for u in urls]
|
|
348
|
-
results = await asyncio.gather(*tasks)
|
|
349
|
-
|
|
350
|
-
if __name__ == "__main__":
|
|
351
|
-
logging.basicConfig(level=logging.INFO)
|
|
352
|
-
asyncio.run(main())
|
|
353
|
-
```
|
|
354
|
-
|
|
355
|
-
## Disclaimer
|
|
356
|
-
|
|
357
|
-
This tool is for educational purposes only. Use it at your own risk.
|
|
358
|
-
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
never_primp-1.0.0.dist-info/METADATA,sha256=zdZwLTtqakrAR7ukQb2kTC1_n-ag04PI3AWGzaEWtkw,16845
|
|
2
|
-
never_primp-1.0.0.dist-info/WHEEL,sha256=GPFYR4mIPdHeeQ1ax6E8LJaGIrcByvaZAJbs3IVoO7M,104
|
|
3
|
-
never_primp-1.0.0.dist-info/licenses/LICENSE,sha256=ZPD9tCar0h91tI4v-TuZdrjDdLqzU4rzPTxtP3x--uc,1063
|
|
4
|
-
never_primp/__init__.py,sha256=St0t0qYotaV3BC4-854NRKEJ0LnUMykXJlY73Em9Sxg,12541
|
|
5
|
-
never_primp/never_primp.abi3.so,sha256=BAnWhGKZJjAO624jRyTAgtFjObWjZEX1pyyenp_iuP8,6981252
|
|
6
|
-
never_primp/never_primp.pyi,sha256=8QRuXaqNfk29qkFZrbtwKKK92Y_wL-O702nys8DCcM0,5731
|
|
7
|
-
never_primp/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
-
never_primp-1.0.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|