scrapy-proxy-headers 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scrapy_proxy_headers-0.2.1/scrapy_proxy_headers.egg-info → scrapy_proxy_headers-0.3.0}/PKG-INFO +73 -17
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.3.0}/README.md +61 -13
- scrapy_proxy_headers-0.3.0/pyproject.toml +52 -0
- scrapy_proxy_headers-0.3.0/scrapy_proxy_headers/__init__.py +9 -0
- scrapy_proxy_headers-0.3.0/scrapy_proxy_headers/connect.py +122 -0
- scrapy_proxy_headers-0.3.0/scrapy_proxy_headers/download_handler.py +88 -0
- scrapy_proxy_headers-0.3.0/scrapy_proxy_headers/legacy.py +181 -0
- scrapy_proxy_headers-0.3.0/scrapy_proxy_headers/mode.py +9 -0
- scrapy_proxy_headers-0.3.0/scrapy_proxy_headers/native.py +77 -0
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.3.0/scrapy_proxy_headers.egg-info}/PKG-INFO +73 -17
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.3.0}/scrapy_proxy_headers.egg-info/SOURCES.txt +6 -2
- scrapy_proxy_headers-0.3.0/scrapy_proxy_headers.egg-info/requires.txt +4 -0
- scrapy_proxy_headers-0.3.0/tests/test_compat.py +164 -0
- scrapy_proxy_headers-0.2.1/pyproject.toml +0 -31
- scrapy_proxy_headers-0.2.1/scrapy_proxy_headers/__init__.py +0 -1
- scrapy_proxy_headers-0.2.1/scrapy_proxy_headers/agent.py +0 -177
- scrapy_proxy_headers-0.2.1/scrapy_proxy_headers/download_handler.py +0 -47
- scrapy_proxy_headers-0.2.1/scrapy_proxy_headers.egg-info/requires.txt +0 -1
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.3.0}/LICENSE +0 -0
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.3.0}/scrapy_proxy_headers.egg-info/dependency_links.txt +0 -0
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.3.0}/scrapy_proxy_headers.egg-info/top_level.txt +0 -0
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.3.0}/setup.cfg +0 -0
{scrapy_proxy_headers-0.2.1/scrapy_proxy_headers.egg-info → scrapy_proxy_headers-0.3.0}/PKG-INFO
RENAMED
|
@@ -1,23 +1,31 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scrapy-proxy-headers
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Add custom proxy headers to HTTPS requests in Scrapy
|
|
5
5
|
Author-email: ProxyMesh <support@proxymesh.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
|
|
7
|
-
Project-URL: Changelog, https://github.com/proxymesh/scrapy-proxy-headers/
|
|
7
|
+
Project-URL: Changelog, https://github.com/proxymesh/scrapy-proxy-headers/blob/main/CHANGELOG.md
|
|
8
8
|
Project-URL: Issues, https://github.com/proxymesh/scrapy-proxy-headers/issues
|
|
9
9
|
Project-URL: Documentation, https://scrapy-proxy-headers.readthedocs.io/en/latest/
|
|
10
10
|
Project-URL: ProxyMesh, https://proxymesh.com
|
|
11
|
+
Classifier: Framework :: Scrapy
|
|
11
12
|
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
12
18
|
Classifier: Operating System :: OS Independent
|
|
13
19
|
Classifier: License :: OSI Approved :: BSD License
|
|
14
20
|
Classifier: Intended Audience :: Developers
|
|
15
21
|
Classifier: Topic :: Internet :: WWW/HTTP
|
|
16
22
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
17
|
-
Requires-Python: >=3.
|
|
23
|
+
Requires-Python: >=3.10
|
|
18
24
|
Description-Content-Type: text/markdown
|
|
19
25
|
License-File: LICENSE
|
|
20
|
-
Requires-Dist: scrapy>=2.
|
|
26
|
+
Requires-Dist: scrapy>=2.17.0
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
21
29
|
Dynamic: license-file
|
|
22
30
|
|
|
23
31
|
# Scrapy Proxy Headers
|
|
@@ -29,7 +37,7 @@ Dynamic: license-file
|
|
|
29
37
|
|
|
30
38
|
## The Problem
|
|
31
39
|
|
|
32
|
-
When making HTTPS requests through a proxy,
|
|
40
|
+
When making HTTPS requests through a proxy, headers on `request.headers` go through the encrypted tunnel and the proxy does not see them. Through Scrapy 2.19, Scrapy also does not send custom headers on the CONNECT request. Scrapy 2.20.0 does. Either way, Scrapy does not keep the proxy's CONNECT response headers, such as `X-ProxyMesh-IP`.
|
|
33
41
|
|
|
34
42
|
```
|
|
35
43
|
┌──────────┐ CONNECT ┌───────┐ Encrypted ┌────────────┐
|
|
@@ -83,12 +91,12 @@ import scrapy
|
|
|
83
91
|
|
|
84
92
|
class MySpider(scrapy.Spider):
|
|
85
93
|
name = "example"
|
|
86
|
-
|
|
87
|
-
def
|
|
94
|
+
|
|
95
|
+
async def start(self):
|
|
88
96
|
yield scrapy.Request(
|
|
89
97
|
url="https://api.ipify.org?format=json",
|
|
90
98
|
meta={
|
|
91
|
-
"proxy": "http://
|
|
99
|
+
"proxy": "http://user:pass@proxy.example.com:31280",
|
|
92
100
|
"proxy_headers": {"X-ProxyMesh-Country": "US"}
|
|
93
101
|
}
|
|
94
102
|
)
|
|
@@ -99,9 +107,13 @@ class MySpider(scrapy.Spider):
|
|
|
99
107
|
self.logger.info(f"Proxy IP: {proxy_ip}")
|
|
100
108
|
```
|
|
101
109
|
|
|
110
|
+
Use `async def start()`. Scrapy 2.16 removed `start_requests()`, so a spider that only defines `start_requests()` sends no requests on every Scrapy version this package supports (2.17 and newer).
|
|
111
|
+
|
|
112
|
+
`X-ProxyMesh-Country` is only honored by some ProxyMesh proxies. See the [ProxyMesh docs](https://docs.proxymesh.com/).
|
|
113
|
+
|
|
102
114
|
### 3. Receive Proxy Response Headers
|
|
103
115
|
|
|
104
|
-
Headers from the proxy's CONNECT response are
|
|
116
|
+
Headers from the proxy's CONNECT response are merged into `response.headers`. Headers the origin already sent are left unchanged.
|
|
105
117
|
|
|
106
118
|
```python
|
|
107
119
|
def parse(self, response):
|
|
@@ -111,6 +123,40 @@ def parse(self, response):
|
|
|
111
123
|
print(f"Request made through IP: {proxy_ip.decode()}")
|
|
112
124
|
```
|
|
113
125
|
|
|
126
|
+
## Scrapy 2.20.0+ / native proxy_headers
|
|
127
|
+
|
|
128
|
+
Scrapy 2.20.0 (released 2026-11-06) is the first release with native `request.meta["proxy_headers"]` support ([#3329](https://github.com/scrapy/scrapy/issues/3329), [#7888](https://github.com/scrapy/scrapy/pull/7888)). It sends those headers to the proxy in `request.meta["proxy"]`, including on the HTTPS CONNECT request. Scrapy 2.20.0 requires Python 3.11 or newer.
|
|
129
|
+
|
|
130
|
+
scrapy-proxy-headers 0.3.0 detects that support from Scrapy's features, not from the version string. On Scrapy 2.20.0 and newer, including master, `PROXY_HEADERS_MODE` is `"native"`: Scrapy sends `proxy_headers`, and this package does not send them again. It only copies the proxy's CONNECT response headers, such as `X-ProxyMesh-IP`, onto `response.headers`. On Scrapy 2.17.0 through 2.19.x the mode is `"legacy"` and this package still writes `proxy_headers` on CONNECT itself.
|
|
131
|
+
|
|
132
|
+
Header-only users on Scrapy 2.20.0 and newer can drop the `DOWNLOAD_HANDLERS` override and keep the same meta value:
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
yield scrapy.Request(
|
|
136
|
+
url="https://api.ipify.org?format=json",
|
|
137
|
+
meta={
|
|
138
|
+
"proxy": "http://user:pass@proxy.example.com:31280",
|
|
139
|
+
"proxy_headers": {"X-ProxyMesh-Country": "US"},
|
|
140
|
+
},
|
|
141
|
+
)
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Scrapy reads the CONNECT status line and discards the rest of the proxy response headers. `X-ProxyMesh-IP` and other CONNECT response headers are still only available through this package. Keep the download handler when the spider reads them from `response.headers`:
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
DOWNLOAD_HANDLERS = {
|
|
148
|
+
"https": "scrapy_proxy_headers.HTTP11ProxyDownloadHandler",
|
|
149
|
+
}
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
`scrapy_proxy_headers.PROXY_HEADERS_MODE` is `"native"` or `"legacy"`. It is detected from the installed Scrapy. It is not a Scrapy setting. In native mode the captured CONNECT response is merged into the Scrapy response even when Scrapy reuses the tunnel.
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
import scrapy_proxy_headers
|
|
156
|
+
|
|
157
|
+
print(scrapy_proxy_headers.PROXY_HEADERS_MODE)
|
|
158
|
+
```
|
|
159
|
+
|
|
114
160
|
## Complete Example
|
|
115
161
|
|
|
116
162
|
```python
|
|
@@ -125,11 +171,11 @@ class ProxyHeadersSpider(scrapy.Spider):
|
|
|
125
171
|
}
|
|
126
172
|
}
|
|
127
173
|
|
|
128
|
-
def
|
|
174
|
+
async def start(self):
|
|
129
175
|
yield scrapy.Request(
|
|
130
176
|
url="https://api.ipify.org?format=json",
|
|
131
177
|
meta={
|
|
132
|
-
"proxy": "http://
|
|
178
|
+
"proxy": "http://user:pass@proxy.example.com:31280",
|
|
133
179
|
"proxy_headers": {"X-ProxyMesh-Country": "US"}
|
|
134
180
|
},
|
|
135
181
|
callback=self.parse_ip
|
|
@@ -152,11 +198,12 @@ class ProxyHeadersSpider(scrapy.Spider):
|
|
|
152
198
|
## How It Works
|
|
153
199
|
|
|
154
200
|
1. **HTTP11ProxyDownloadHandler** - Custom download handler that manages proxy header caching
|
|
155
|
-
2. **
|
|
156
|
-
3. **
|
|
157
|
-
|
|
201
|
+
2. **Legacy path** (`PROXY_HEADERS_MODE == "legacy"`) - `ScrapyProxyHeadersAgent` reads `proxy_headers` and `TunnelingHeadersAgent` sends them on CONNECT
|
|
202
|
+
3. **Native path** (`PROXY_HEADERS_MODE == "native"`) - Scrapy sends `proxy_headers`; the handler only records the CONNECT response
|
|
203
|
+
|
|
204
|
+
On the legacy path, `TunnelingHeadersTCP4ClientEndpoint` captures proxy response headers from the CONNECT response. On the native path, a subclass of Scrapy's tunnel protocol does that capture.
|
|
158
205
|
|
|
159
|
-
The handler
|
|
206
|
+
The handler caches proxy response headers by proxy URL and the normalized `proxy_headers` for that request. A reused tunnel still exposes the CONNECT response from the header set that opened it, and a different header set does not reuse another set's cached `X-ProxyMesh-IP`.
|
|
160
207
|
|
|
161
208
|
## Test Harness
|
|
162
209
|
|
|
@@ -200,8 +247,17 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
|
|
|
200
247
|
|
|
201
248
|
## Requirements
|
|
202
249
|
|
|
203
|
-
- Python 3.
|
|
204
|
-
- Scrapy 2.
|
|
250
|
+
- Python 3.10+
|
|
251
|
+
- Scrapy 2.17.0+ (no upper cap). Native `proxy_headers` starts at Scrapy 2.20.0.
|
|
252
|
+
|
|
253
|
+
Scrapy 2.17.0 and 2.18 support Python 3.10 through 3.14. Scrapy 2.19 and 2.20 also list 3.15. Scrapy 2.20.0 and Scrapy master require Python 3.11+.
|
|
254
|
+
|
|
255
|
+
Local compatibility tests do not need a proxy account:
|
|
256
|
+
|
|
257
|
+
```bash
|
|
258
|
+
pip install -e ".[dev]"
|
|
259
|
+
pytest
|
|
260
|
+
```
|
|
205
261
|
|
|
206
262
|
## License
|
|
207
263
|
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
## The Problem
|
|
9
9
|
|
|
10
|
-
When making HTTPS requests through a proxy,
|
|
10
|
+
When making HTTPS requests through a proxy, headers on `request.headers` go through the encrypted tunnel and the proxy does not see them. Through Scrapy 2.19, Scrapy also does not send custom headers on the CONNECT request. Scrapy 2.20.0 does. Either way, Scrapy does not keep the proxy's CONNECT response headers, such as `X-ProxyMesh-IP`.
|
|
11
11
|
|
|
12
12
|
```
|
|
13
13
|
┌──────────┐ CONNECT ┌───────┐ Encrypted ┌────────────┐
|
|
@@ -61,12 +61,12 @@ import scrapy
|
|
|
61
61
|
|
|
62
62
|
class MySpider(scrapy.Spider):
|
|
63
63
|
name = "example"
|
|
64
|
-
|
|
65
|
-
def
|
|
64
|
+
|
|
65
|
+
async def start(self):
|
|
66
66
|
yield scrapy.Request(
|
|
67
67
|
url="https://api.ipify.org?format=json",
|
|
68
68
|
meta={
|
|
69
|
-
"proxy": "http://
|
|
69
|
+
"proxy": "http://user:pass@proxy.example.com:31280",
|
|
70
70
|
"proxy_headers": {"X-ProxyMesh-Country": "US"}
|
|
71
71
|
}
|
|
72
72
|
)
|
|
@@ -77,9 +77,13 @@ class MySpider(scrapy.Spider):
|
|
|
77
77
|
self.logger.info(f"Proxy IP: {proxy_ip}")
|
|
78
78
|
```
|
|
79
79
|
|
|
80
|
+
Use `async def start()`. Scrapy 2.16 removed `start_requests()`, so a spider that only defines `start_requests()` sends no requests on every Scrapy version this package supports (2.17 and newer).
|
|
81
|
+
|
|
82
|
+
`X-ProxyMesh-Country` is only honored by some ProxyMesh proxies. See the [ProxyMesh docs](https://docs.proxymesh.com/).
|
|
83
|
+
|
|
80
84
|
### 3. Receive Proxy Response Headers
|
|
81
85
|
|
|
82
|
-
Headers from the proxy's CONNECT response are
|
|
86
|
+
Headers from the proxy's CONNECT response are merged into `response.headers`. Headers the origin already sent are left unchanged.
|
|
83
87
|
|
|
84
88
|
```python
|
|
85
89
|
def parse(self, response):
|
|
@@ -89,6 +93,40 @@ def parse(self, response):
|
|
|
89
93
|
print(f"Request made through IP: {proxy_ip.decode()}")
|
|
90
94
|
```
|
|
91
95
|
|
|
96
|
+
## Scrapy 2.20.0+ / native proxy_headers
|
|
97
|
+
|
|
98
|
+
Scrapy 2.20.0 (released 2026-11-06) is the first release with native `request.meta["proxy_headers"]` support ([#3329](https://github.com/scrapy/scrapy/issues/3329), [#7888](https://github.com/scrapy/scrapy/pull/7888)). It sends those headers to the proxy in `request.meta["proxy"]`, including on the HTTPS CONNECT request. Scrapy 2.20.0 requires Python 3.11 or newer.
|
|
99
|
+
|
|
100
|
+
scrapy-proxy-headers 0.3.0 detects that support from Scrapy's features, not from the version string. On Scrapy 2.20.0 and newer, including master, `PROXY_HEADERS_MODE` is `"native"`: Scrapy sends `proxy_headers`, and this package does not send them again. It only copies the proxy's CONNECT response headers, such as `X-ProxyMesh-IP`, onto `response.headers`. On Scrapy 2.17.0 through 2.19.x the mode is `"legacy"` and this package still writes `proxy_headers` on CONNECT itself.
|
|
101
|
+
|
|
102
|
+
Header-only users on Scrapy 2.20.0 and newer can drop the `DOWNLOAD_HANDLERS` override and keep the same meta value:
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
yield scrapy.Request(
|
|
106
|
+
url="https://api.ipify.org?format=json",
|
|
107
|
+
meta={
|
|
108
|
+
"proxy": "http://user:pass@proxy.example.com:31280",
|
|
109
|
+
"proxy_headers": {"X-ProxyMesh-Country": "US"},
|
|
110
|
+
},
|
|
111
|
+
)
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Scrapy reads the CONNECT status line and discards the rest of the proxy response headers. `X-ProxyMesh-IP` and other CONNECT response headers are still only available through this package. Keep the download handler when the spider reads them from `response.headers`:
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
DOWNLOAD_HANDLERS = {
|
|
118
|
+
"https": "scrapy_proxy_headers.HTTP11ProxyDownloadHandler",
|
|
119
|
+
}
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
`scrapy_proxy_headers.PROXY_HEADERS_MODE` is `"native"` or `"legacy"`. It is detected from the installed Scrapy. It is not a Scrapy setting. In native mode the captured CONNECT response is merged into the Scrapy response even when Scrapy reuses the tunnel.
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
import scrapy_proxy_headers
|
|
126
|
+
|
|
127
|
+
print(scrapy_proxy_headers.PROXY_HEADERS_MODE)
|
|
128
|
+
```
|
|
129
|
+
|
|
92
130
|
## Complete Example
|
|
93
131
|
|
|
94
132
|
```python
|
|
@@ -103,11 +141,11 @@ class ProxyHeadersSpider(scrapy.Spider):
|
|
|
103
141
|
}
|
|
104
142
|
}
|
|
105
143
|
|
|
106
|
-
def
|
|
144
|
+
async def start(self):
|
|
107
145
|
yield scrapy.Request(
|
|
108
146
|
url="https://api.ipify.org?format=json",
|
|
109
147
|
meta={
|
|
110
|
-
"proxy": "http://
|
|
148
|
+
"proxy": "http://user:pass@proxy.example.com:31280",
|
|
111
149
|
"proxy_headers": {"X-ProxyMesh-Country": "US"}
|
|
112
150
|
},
|
|
113
151
|
callback=self.parse_ip
|
|
@@ -130,11 +168,12 @@ class ProxyHeadersSpider(scrapy.Spider):
|
|
|
130
168
|
## How It Works
|
|
131
169
|
|
|
132
170
|
1. **HTTP11ProxyDownloadHandler** - Custom download handler that manages proxy header caching
|
|
133
|
-
2. **
|
|
134
|
-
3. **
|
|
135
|
-
|
|
171
|
+
2. **Legacy path** (`PROXY_HEADERS_MODE == "legacy"`) - `ScrapyProxyHeadersAgent` reads `proxy_headers` and `TunnelingHeadersAgent` sends them on CONNECT
|
|
172
|
+
3. **Native path** (`PROXY_HEADERS_MODE == "native"`) - Scrapy sends `proxy_headers`; the handler only records the CONNECT response
|
|
173
|
+
|
|
174
|
+
On the legacy path, `TunnelingHeadersTCP4ClientEndpoint` captures proxy response headers from the CONNECT response. On the native path, a subclass of Scrapy's tunnel protocol does that capture.
|
|
136
175
|
|
|
137
|
-
The handler
|
|
176
|
+
The handler caches proxy response headers by proxy URL and the normalized `proxy_headers` for that request. A reused tunnel still exposes the CONNECT response from the header set that opened it, and a different header set does not reuse another set's cached `X-ProxyMesh-IP`.
|
|
138
177
|
|
|
139
178
|
## Test Harness
|
|
140
179
|
|
|
@@ -178,8 +217,17 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
|
|
|
178
217
|
|
|
179
218
|
## Requirements
|
|
180
219
|
|
|
181
|
-
- Python 3.
|
|
182
|
-
- Scrapy 2.
|
|
220
|
+
- Python 3.10+
|
|
221
|
+
- Scrapy 2.17.0+ (no upper cap). Native `proxy_headers` starts at Scrapy 2.20.0.
|
|
222
|
+
|
|
223
|
+
Scrapy 2.17.0 and 2.18 support Python 3.10 through 3.14. Scrapy 2.19 and 2.20 also list 3.15. Scrapy 2.20.0 and Scrapy master require Python 3.11+.
|
|
224
|
+
|
|
225
|
+
Local compatibility tests do not need a proxy account:
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
pip install -e ".[dev]"
|
|
229
|
+
pytest
|
|
230
|
+
```
|
|
183
231
|
|
|
184
232
|
## License
|
|
185
233
|
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "scrapy-proxy-headers"
|
|
7
|
+
version = "0.3.0"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name="ProxyMesh", email="support@proxymesh.com" },
|
|
10
|
+
]
|
|
11
|
+
description = "Add custom proxy headers to HTTPS requests in Scrapy"
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
requires-python = ">=3.10"
|
|
14
|
+
dependencies = [
|
|
15
|
+
"scrapy>=2.17.0",
|
|
16
|
+
]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Framework :: Scrapy",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Programming Language :: Python :: 3.14",
|
|
25
|
+
"Operating System :: OS Independent",
|
|
26
|
+
"License :: OSI Approved :: BSD License",
|
|
27
|
+
"Intended Audience :: Developers",
|
|
28
|
+
"Topic :: Internet :: WWW/HTTP",
|
|
29
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.optional-dependencies]
|
|
33
|
+
dev = [
|
|
34
|
+
"pytest>=8",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[project.urls]
|
|
38
|
+
Homepage = "https://github.com/proxymesh/scrapy-proxy-headers"
|
|
39
|
+
Changelog = "https://github.com/proxymesh/scrapy-proxy-headers/blob/main/CHANGELOG.md"
|
|
40
|
+
Issues = "https://github.com/proxymesh/scrapy-proxy-headers/issues"
|
|
41
|
+
Documentation = "https://scrapy-proxy-headers.readthedocs.io/en/latest/"
|
|
42
|
+
ProxyMesh = "https://proxymesh.com"
|
|
43
|
+
|
|
44
|
+
[tool.setuptools.packages.find]
|
|
45
|
+
include = ["scrapy_proxy_headers*"]
|
|
46
|
+
|
|
47
|
+
[tool.pytest.ini_options]
|
|
48
|
+
testpaths = ["tests"]
|
|
49
|
+
|
|
50
|
+
[tool.ruff.lint]
|
|
51
|
+
# Pre-0.16 default set, plus no new inline imports and lazy log args.
|
|
52
|
+
select = ["E4", "E7", "E9", "F", "PLC0415", "G004"]
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""CONNECT header bytes and proxy-header cache keys.
|
|
2
|
+
|
|
3
|
+
These helpers do not touch Scrapy downloader internals.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from scrapy.http import Headers, Response
|
|
12
|
+
from scrapy.utils.python import to_bytes
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def tunnel_request_data_with_headers(
|
|
16
|
+
host: str, port: int, **proxy_headers: Any
|
|
17
|
+
) -> bytes:
|
|
18
|
+
r"""
|
|
19
|
+
Return binary content of a CONNECT request.
|
|
20
|
+
|
|
21
|
+
>>> from scrapy.utils.python import to_unicode as s
|
|
22
|
+
>>> s(tunnel_request_data_with_headers("example.com", 8080))
|
|
23
|
+
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
|
|
24
|
+
>>> s(tunnel_request_data_with_headers("example.com", 8080, **{"X-ProxyMesh-Country": "US"}))
|
|
25
|
+
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nX-ProxyMesh-Country: US\r\n\r\n'
|
|
26
|
+
>>> s(tunnel_request_data_with_headers(b"example.com", "8090"))
|
|
27
|
+
'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'
|
|
28
|
+
"""
|
|
29
|
+
host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port))
|
|
30
|
+
tunnel_req = b"CONNECT " + host_value + b" HTTP/1.1\r\n"
|
|
31
|
+
tunnel_req += b"Host: " + host_value + b"\r\n"
|
|
32
|
+
|
|
33
|
+
for key, val in proxy_headers.items():
|
|
34
|
+
key_b = to_bytes(key)
|
|
35
|
+
val_b = to_bytes(val)
|
|
36
|
+
if b"\r" in key_b or b"\n" in key_b or b"\r" in val_b or b"\n" in val_b:
|
|
37
|
+
raise ValueError("proxy header contains CR or LF")
|
|
38
|
+
tunnel_req += key_b + b": " + val_b + b"\r\n"
|
|
39
|
+
|
|
40
|
+
tunnel_req += b"\r\n"
|
|
41
|
+
return tunnel_req
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _as_text(value: Any) -> str:
|
|
45
|
+
if isinstance(value, bytes):
|
|
46
|
+
return value.decode("utf-8", "replace")
|
|
47
|
+
return str(value)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def normalize_proxy_url(proxy: str) -> str:
|
|
51
|
+
"""Return *proxy* with an ``http://`` scheme when the value omits one."""
|
|
52
|
+
proxy = proxy.strip()
|
|
53
|
+
if "://" not in proxy:
|
|
54
|
+
return "http://" + proxy
|
|
55
|
+
return proxy
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def normalize_proxy_headers(
|
|
59
|
+
proxy_headers: Mapping[Any, Any] | None,
|
|
60
|
+
) -> tuple[tuple[str, str], ...]:
|
|
61
|
+
"""Return ``proxy_headers`` as sorted ``(lowercase name, value)`` pairs.
|
|
62
|
+
|
|
63
|
+
Header names are compared case-insensitively. Values keep their spelling.
|
|
64
|
+
A list or tuple value becomes one pair per item. ``None`` and empty
|
|
65
|
+
mappings become an empty tuple.
|
|
66
|
+
"""
|
|
67
|
+
if not proxy_headers:
|
|
68
|
+
return ()
|
|
69
|
+
pairs: list[tuple[str, str]] = []
|
|
70
|
+
items = (
|
|
71
|
+
proxy_headers.items() if isinstance(proxy_headers, Mapping) else proxy_headers
|
|
72
|
+
)
|
|
73
|
+
for key, value in items:
|
|
74
|
+
name = _as_text(key).lower()
|
|
75
|
+
values = value if isinstance(value, (list, tuple)) else (value,)
|
|
76
|
+
for item in values:
|
|
77
|
+
pairs.append((name, _as_text(item)))
|
|
78
|
+
return tuple(sorted(pairs))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def proxy_cache_key(
|
|
82
|
+
proxy: str,
|
|
83
|
+
proxy_headers: Mapping[Any, Any] | None,
|
|
84
|
+
) -> tuple[str, tuple[tuple[str, str], ...]]:
|
|
85
|
+
"""Cache key for CONNECT response headers.
|
|
86
|
+
|
|
87
|
+
Requests that share a proxy URL but send different ``proxy_headers`` must
|
|
88
|
+
not reuse each other's CONNECT response (for example a different
|
|
89
|
+
``X-ProxyMesh-IP``).
|
|
90
|
+
"""
|
|
91
|
+
return (normalize_proxy_url(proxy), normalize_proxy_headers(proxy_headers))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def parse_connect_response_headers(header_block: bytes) -> Headers:
|
|
95
|
+
"""Parse a CONNECT response, skipping the status line.
|
|
96
|
+
|
|
97
|
+
*header_block* is the bytes before the blank line that ends the headers.
|
|
98
|
+
"""
|
|
99
|
+
pairs: list[tuple[bytes, bytes]] = []
|
|
100
|
+
lines = header_block.split(b"\r\n")
|
|
101
|
+
for line in lines[1:]:
|
|
102
|
+
if b":" not in line:
|
|
103
|
+
continue
|
|
104
|
+
key, value = line.split(b":", 1)
|
|
105
|
+
key = key.strip()
|
|
106
|
+
if not key:
|
|
107
|
+
continue
|
|
108
|
+
pairs.append((key, value.strip()))
|
|
109
|
+
return Headers(pairs)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def merge_proxy_response_headers(response: Response, proxy_headers: Headers) -> None:
|
|
113
|
+
"""Copy CONNECT headers the origin response does not already have.
|
|
114
|
+
|
|
115
|
+
Proxy headers do not replace headers the origin already sent. Merging the
|
|
116
|
+
same proxy headers again does not duplicate values.
|
|
117
|
+
"""
|
|
118
|
+
if not proxy_headers:
|
|
119
|
+
return
|
|
120
|
+
for name in proxy_headers:
|
|
121
|
+
if name not in response.headers:
|
|
122
|
+
response.headers[name] = proxy_headers.getlist(name)
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""HTTPS download handler that exposes proxy CONNECT response headers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
|
|
9
|
+
from scrapy.http import Request, Response
|
|
10
|
+
from scrapy.utils.defer import maybe_deferred_to_future
|
|
11
|
+
|
|
12
|
+
from .connect import merge_proxy_response_headers, proxy_cache_key
|
|
13
|
+
from .mode import PROXY_HEADERS_MODE
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
if PROXY_HEADERS_MODE == "native":
|
|
18
|
+
from .native import lookup_connect_response_headers
|
|
19
|
+
else:
|
|
20
|
+
from .legacy import ScrapyProxyHeadersAgent
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
|
|
24
|
+
"""Merge CONNECT response headers into the Scrapy response.
|
|
25
|
+
|
|
26
|
+
Configure it with::
|
|
27
|
+
|
|
28
|
+
DOWNLOAD_HANDLERS = {
|
|
29
|
+
"https": "scrapy_proxy_headers.HTTP11ProxyDownloadHandler",
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
``request.meta["proxy_headers"]`` is sent on CONNECT. On Scrapy 2.20+
|
|
33
|
+
Scrapy sends those headers and this handler only copies the proxy's
|
|
34
|
+
response, including ``X-ProxyMesh-IP``. On Scrapy 2.17–2.19 this handler
|
|
35
|
+
sends them. The cache key is the proxy URL plus the normalized headers,
|
|
36
|
+
so two header sets do not share one ``X-ProxyMesh-IP``.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
40
|
+
super().__init__(*args, **kwargs)
|
|
41
|
+
self._proxy_response_headers_cache: dict[tuple[Any, ...], Any] = {}
|
|
42
|
+
logger.info("scrapy-proxy-headers mode: %s", PROXY_HEADERS_MODE)
|
|
43
|
+
|
|
44
|
+
async def download_request(self, request: Request, spider: Any = None) -> Response:
|
|
45
|
+
"""Download *request* and merge captured CONNECT headers onto it."""
|
|
46
|
+
if PROXY_HEADERS_MODE == "native":
|
|
47
|
+
response = await super().download_request(request)
|
|
48
|
+
captured = lookup_connect_response_headers(request)
|
|
49
|
+
return self._apply_cached_headers(response, request, captured)
|
|
50
|
+
|
|
51
|
+
if spider is None:
|
|
52
|
+
spider = self._crawler.spider
|
|
53
|
+
agent = ScrapyProxyHeadersAgent(
|
|
54
|
+
contextFactory=self._contextFactory,
|
|
55
|
+
pool=self._pool,
|
|
56
|
+
maxsize=getattr(spider, "download_maxsize", self._default_maxsize),
|
|
57
|
+
warnsize=getattr(spider, "download_warnsize", self._default_warnsize),
|
|
58
|
+
fail_on_dataloss=self._fail_on_dataloss,
|
|
59
|
+
crawler=self._crawler,
|
|
60
|
+
bindAddress=self._bind_address,
|
|
61
|
+
tls_verbose_logging=self._tls_verbose_logging,
|
|
62
|
+
)
|
|
63
|
+
deferred = agent.download_request(request)
|
|
64
|
+
deferred.addCallback(
|
|
65
|
+
lambda response: self._apply_cached_headers(
|
|
66
|
+
response, request, agent.proxy_response_headers
|
|
67
|
+
)
|
|
68
|
+
)
|
|
69
|
+
return await maybe_deferred_to_future(deferred)
|
|
70
|
+
|
|
71
|
+
def _apply_cached_headers(
|
|
72
|
+
self,
|
|
73
|
+
response: Response,
|
|
74
|
+
request: Request,
|
|
75
|
+
captured: Any,
|
|
76
|
+
) -> Response:
|
|
77
|
+
proxy = request.meta.get("proxy")
|
|
78
|
+
if not proxy or not isinstance(response, Response):
|
|
79
|
+
return response
|
|
80
|
+
key = proxy_cache_key(proxy, request.meta.get("proxy_headers"))
|
|
81
|
+
if captured is not None:
|
|
82
|
+
self._proxy_response_headers_cache[key] = captured
|
|
83
|
+
cached = self._proxy_response_headers_cache.get(key)
|
|
84
|
+
if cached:
|
|
85
|
+
merge_proxy_response_headers(response, cached)
|
|
86
|
+
elif PROXY_HEADERS_MODE == "native":
|
|
87
|
+
logger.warning("No CONNECT response headers captured for proxy %s", proxy)
|
|
88
|
+
return response
|