scrapy-proxy-headers 0.2.2__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. {scrapy_proxy_headers-0.2.2/scrapy_proxy_headers.egg-info → scrapy_proxy_headers-0.3.0}/PKG-INFO +72 -16
  2. {scrapy_proxy_headers-0.2.2 → scrapy_proxy_headers-0.3.0}/README.md +60 -12
  3. scrapy_proxy_headers-0.3.0/pyproject.toml +52 -0
  4. scrapy_proxy_headers-0.3.0/scrapy_proxy_headers/__init__.py +9 -0
  5. scrapy_proxy_headers-0.3.0/scrapy_proxy_headers/connect.py +122 -0
  6. scrapy_proxy_headers-0.3.0/scrapy_proxy_headers/download_handler.py +88 -0
  7. scrapy_proxy_headers-0.3.0/scrapy_proxy_headers/legacy.py +181 -0
  8. scrapy_proxy_headers-0.3.0/scrapy_proxy_headers/mode.py +9 -0
  9. scrapy_proxy_headers-0.3.0/scrapy_proxy_headers/native.py +77 -0
  10. {scrapy_proxy_headers-0.2.2 → scrapy_proxy_headers-0.3.0/scrapy_proxy_headers.egg-info}/PKG-INFO +72 -16
  11. {scrapy_proxy_headers-0.2.2 → scrapy_proxy_headers-0.3.0}/scrapy_proxy_headers.egg-info/SOURCES.txt +6 -2
  12. scrapy_proxy_headers-0.3.0/scrapy_proxy_headers.egg-info/requires.txt +4 -0
  13. scrapy_proxy_headers-0.3.0/tests/test_compat.py +164 -0
  14. scrapy_proxy_headers-0.2.2/pyproject.toml +0 -31
  15. scrapy_proxy_headers-0.2.2/scrapy_proxy_headers/__init__.py +0 -1
  16. scrapy_proxy_headers-0.2.2/scrapy_proxy_headers/agent.py +0 -190
  17. scrapy_proxy_headers-0.2.2/scrapy_proxy_headers/download_handler.py +0 -47
  18. scrapy_proxy_headers-0.2.2/scrapy_proxy_headers.egg-info/requires.txt +0 -1
  19. {scrapy_proxy_headers-0.2.2 → scrapy_proxy_headers-0.3.0}/LICENSE +0 -0
  20. {scrapy_proxy_headers-0.2.2 → scrapy_proxy_headers-0.3.0}/scrapy_proxy_headers.egg-info/dependency_links.txt +0 -0
  21. {scrapy_proxy_headers-0.2.2 → scrapy_proxy_headers-0.3.0}/scrapy_proxy_headers.egg-info/top_level.txt +0 -0
  22. {scrapy_proxy_headers-0.2.2 → scrapy_proxy_headers-0.3.0}/setup.cfg +0 -0
@@ -1,23 +1,31 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapy-proxy-headers
3
- Version: 0.2.2
3
+ Version: 0.3.0
4
4
  Summary: Add custom proxy headers to HTTPS requests in Scrapy
5
5
  Author-email: ProxyMesh <support@proxymesh.com>
6
6
  Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
7
- Project-URL: Changelog, https://github.com/proxymesh/scrapy-proxy-headers/commits/main/
7
+ Project-URL: Changelog, https://github.com/proxymesh/scrapy-proxy-headers/blob/main/CHANGELOG.md
8
8
  Project-URL: Issues, https://github.com/proxymesh/scrapy-proxy-headers/issues
9
9
  Project-URL: Documentation, https://scrapy-proxy-headers.readthedocs.io/en/latest/
10
10
  Project-URL: ProxyMesh, https://proxymesh.com
11
+ Classifier: Framework :: Scrapy
11
12
  Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Programming Language :: Python :: 3.14
12
18
  Classifier: Operating System :: OS Independent
13
19
  Classifier: License :: OSI Approved :: BSD License
14
20
  Classifier: Intended Audience :: Developers
15
21
  Classifier: Topic :: Internet :: WWW/HTTP
16
22
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
17
- Requires-Python: >=3.8
23
+ Requires-Python: >=3.10
18
24
  Description-Content-Type: text/markdown
19
25
  License-File: LICENSE
20
- Requires-Dist: scrapy<2.20,>=2.17.0
26
+ Requires-Dist: scrapy>=2.17.0
27
+ Provides-Extra: dev
28
+ Requires-Dist: pytest>=8; extra == "dev"
21
29
  Dynamic: license-file
22
30
 
23
31
  # Scrapy Proxy Headers
@@ -29,7 +37,7 @@ Dynamic: license-file
29
37
 
30
38
  ## The Problem
31
39
 
32
- When making HTTPS requests through a proxy, Scrapy cannot send custom headers to the proxy itself. This is because HTTPS requests create an encrypted tunnel (via HTTP CONNECT) - any headers you add to `request.headers` are encrypted and only visible to the destination server, not the proxy.
40
+ When making HTTPS requests through a proxy, headers on `request.headers` go through the encrypted tunnel and the proxy does not see them. Through Scrapy 2.19, Scrapy also does not send custom headers on the CONNECT request. Scrapy 2.20.0 does. Either way, Scrapy does not keep the proxy's CONNECT response headers, such as `X-ProxyMesh-IP`.
33
41
 
34
42
  ```
35
43
  ┌──────────┐ CONNECT ┌───────┐ Encrypted ┌────────────┐
@@ -83,12 +91,12 @@ import scrapy
83
91
 
84
92
  class MySpider(scrapy.Spider):
85
93
  name = "example"
86
-
87
- def start_requests(self):
94
+
95
+ async def start(self):
88
96
  yield scrapy.Request(
89
97
  url="https://api.ipify.org?format=json",
90
98
  meta={
91
- "proxy": "http://your-proxy:port",
99
+ "proxy": "http://user:pass@proxy.example.com:31280",
92
100
  "proxy_headers": {"X-ProxyMesh-Country": "US"}
93
101
  }
94
102
  )
@@ -99,6 +107,10 @@ class MySpider(scrapy.Spider):
99
107
  self.logger.info(f"Proxy IP: {proxy_ip}")
100
108
  ```
101
109
 
110
+ Use `async def start()`. Scrapy 2.16 removed `start_requests()`, so a spider that only defines `start_requests()` sends no requests on every Scrapy version this package supports (2.17 and newer).
111
+
112
+ `X-ProxyMesh-Country` is only honored by some ProxyMesh proxies. See the [ProxyMesh docs](https://docs.proxymesh.com/).
113
+
102
114
  ### 3. Receive Proxy Response Headers
103
115
 
104
116
  Headers from the proxy's CONNECT response are merged into `response.headers`. Headers the origin already sent are left unchanged.
@@ -111,6 +123,40 @@ def parse(self, response):
111
123
  print(f"Request made through IP: {proxy_ip.decode()}")
112
124
  ```
113
125
 
126
+ ## Scrapy 2.20.0+ / native proxy_headers
127
+
128
+ Scrapy 2.20.0 (released 2026-11-06) is the first release with native `request.meta["proxy_headers"]` support ([#3329](https://github.com/scrapy/scrapy/issues/3329), [#7888](https://github.com/scrapy/scrapy/pull/7888)). It sends those headers to the proxy in `request.meta["proxy"]`, including on the HTTPS CONNECT request. Scrapy 2.20.0 requires Python 3.11 or newer.
129
+
130
+ scrapy-proxy-headers 0.3.0 detects that support from Scrapy's features, not from the version string. On Scrapy 2.20.0 and newer, including master, `PROXY_HEADERS_MODE` is `"native"`: Scrapy sends `proxy_headers`, and this package does not send them again. It only copies the proxy's CONNECT response headers, such as `X-ProxyMesh-IP`, onto `response.headers`. On Scrapy 2.17.0 through 2.19.x the mode is `"legacy"` and this package still writes `proxy_headers` on CONNECT itself.
131
+
132
+ Header-only users on Scrapy 2.20.0 and newer can drop the `DOWNLOAD_HANDLERS` override and keep the same meta value:
133
+
134
+ ```python
135
+ yield scrapy.Request(
136
+ url="https://api.ipify.org?format=json",
137
+ meta={
138
+ "proxy": "http://user:pass@proxy.example.com:31280",
139
+ "proxy_headers": {"X-ProxyMesh-Country": "US"},
140
+ },
141
+ )
142
+ ```
143
+
144
+ Scrapy reads the CONNECT status line and discards the rest of the proxy response headers. `X-ProxyMesh-IP` and other CONNECT response headers are still only available through this package. Keep the download handler when the spider reads them from `response.headers`:
145
+
146
+ ```python
147
+ DOWNLOAD_HANDLERS = {
148
+ "https": "scrapy_proxy_headers.HTTP11ProxyDownloadHandler",
149
+ }
150
+ ```
151
+
152
+ `scrapy_proxy_headers.PROXY_HEADERS_MODE` is `"native"` or `"legacy"`. It is detected from the installed Scrapy. It is not a Scrapy setting. In native mode the captured CONNECT response is merged into the Scrapy response even when Scrapy reuses the tunnel.
153
+
154
+ ```python
155
+ import scrapy_proxy_headers
156
+
157
+ print(scrapy_proxy_headers.PROXY_HEADERS_MODE)
158
+ ```
159
+
114
160
  ## Complete Example
115
161
 
116
162
  ```python
@@ -125,11 +171,11 @@ class ProxyHeadersSpider(scrapy.Spider):
125
171
  }
126
172
  }
127
173
 
128
- def start_requests(self):
174
+ async def start(self):
129
175
  yield scrapy.Request(
130
176
  url="https://api.ipify.org?format=json",
131
177
  meta={
132
- "proxy": "http://us.proxymesh.com:31280",
178
+ "proxy": "http://user:pass@proxy.example.com:31280",
133
179
  "proxy_headers": {"X-ProxyMesh-Country": "US"}
134
180
  },
135
181
  callback=self.parse_ip
@@ -152,11 +198,12 @@ class ProxyHeadersSpider(scrapy.Spider):
152
198
  ## How It Works
153
199
 
154
200
  1. **HTTP11ProxyDownloadHandler** - Custom download handler that manages proxy header caching
155
- 2. **ScrapyProxyHeadersAgent** - Agent that reads `proxy_headers` from request meta
156
- 3. **TunnelingHeadersAgent** - Sends custom headers in the CONNECT request
157
- 4. **TunnelingHeadersTCP4ClientEndpoint** - Captures proxy response headers from CONNECT response
201
+ 2. **Legacy path** (`PROXY_HEADERS_MODE == "legacy"`) - `ScrapyProxyHeadersAgent` reads `proxy_headers` and `TunnelingHeadersAgent` sends them on CONNECT
202
+ 3. **Native path** (`PROXY_HEADERS_MODE == "native"`) - Scrapy sends `proxy_headers`; the handler only records the CONNECT response
203
+
204
+ On the legacy path, `TunnelingHeadersTCP4ClientEndpoint` captures proxy response headers from the CONNECT response. On the native path, a subclass of Scrapy's tunnel protocol does that capture.
158
205
 
159
- The handler also caches proxy response headers by proxy URL. This ensures headers remain available even when Scrapy reuses existing tunnel connections for subsequent requests.
206
+ The handler caches proxy response headers by proxy URL and the normalized `proxy_headers` for that request. A reused tunnel still exposes the CONNECT response from the header set that opened it, and a different header set does not reuse another set's cached `X-ProxyMesh-IP`.
160
207
 
161
208
  ## Test Harness
162
209
 
@@ -200,8 +247,17 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
200
247
 
201
248
  ## Requirements
202
249
 
203
- - Python 3.8+
204
- - Scrapy >=2.17.0,<2.20
250
+ - Python 3.10+
251
+ - Scrapy 2.17.0+ (no upper cap). Native `proxy_headers` starts at Scrapy 2.20.0.
252
+
253
+ Scrapy 2.17.0 and 2.18 support Python 3.10 through 3.14. Scrapy 2.19 and 2.20 also list 3.15. Scrapy 2.20.0 and Scrapy master require Python 3.11+.
254
+
255
+ Local compatibility tests do not need a proxy account:
256
+
257
+ ```bash
258
+ pip install -e ".[dev]"
259
+ pytest
260
+ ```
205
261
 
206
262
  ## License
207
263
 
@@ -7,7 +7,7 @@
7
7
 
8
8
  ## The Problem
9
9
 
10
- When making HTTPS requests through a proxy, Scrapy cannot send custom headers to the proxy itself. This is because HTTPS requests create an encrypted tunnel (via HTTP CONNECT) - any headers you add to `request.headers` are encrypted and only visible to the destination server, not the proxy.
10
+ When making HTTPS requests through a proxy, headers on `request.headers` go through the encrypted tunnel and the proxy does not see them. Through Scrapy 2.19, Scrapy also does not send custom headers on the CONNECT request. Scrapy 2.20.0 does. Either way, Scrapy does not keep the proxy's CONNECT response headers, such as `X-ProxyMesh-IP`.
11
11
 
12
12
  ```
13
13
  ┌──────────┐ CONNECT ┌───────┐ Encrypted ┌────────────┐
@@ -61,12 +61,12 @@ import scrapy
61
61
 
62
62
  class MySpider(scrapy.Spider):
63
63
  name = "example"
64
-
65
- def start_requests(self):
64
+
65
+ async def start(self):
66
66
  yield scrapy.Request(
67
67
  url="https://api.ipify.org?format=json",
68
68
  meta={
69
- "proxy": "http://your-proxy:port",
69
+ "proxy": "http://user:pass@proxy.example.com:31280",
70
70
  "proxy_headers": {"X-ProxyMesh-Country": "US"}
71
71
  }
72
72
  )
@@ -77,6 +77,10 @@ class MySpider(scrapy.Spider):
77
77
  self.logger.info(f"Proxy IP: {proxy_ip}")
78
78
  ```
79
79
 
80
+ Use `async def start()`. Scrapy 2.16 removed `start_requests()`, so a spider that only defines `start_requests()` sends no requests on every Scrapy version this package supports (2.17 and newer).
81
+
82
+ `X-ProxyMesh-Country` is only honored by some ProxyMesh proxies. See the [ProxyMesh docs](https://docs.proxymesh.com/).
83
+
80
84
  ### 3. Receive Proxy Response Headers
81
85
 
82
86
  Headers from the proxy's CONNECT response are merged into `response.headers`. Headers the origin already sent are left unchanged.
@@ -89,6 +93,40 @@ def parse(self, response):
89
93
  print(f"Request made through IP: {proxy_ip.decode()}")
90
94
  ```
91
95
 
96
+ ## Scrapy 2.20.0+ / native proxy_headers
97
+
98
+ Scrapy 2.20.0 (released 2026-11-06) is the first release with native `request.meta["proxy_headers"]` support ([#3329](https://github.com/scrapy/scrapy/issues/3329), [#7888](https://github.com/scrapy/scrapy/pull/7888)). It sends those headers to the proxy in `request.meta["proxy"]`, including on the HTTPS CONNECT request. Scrapy 2.20.0 requires Python 3.11 or newer.
99
+
100
+ scrapy-proxy-headers 0.3.0 detects that support from Scrapy's features, not from the version string. On Scrapy 2.20.0 and newer, including master, `PROXY_HEADERS_MODE` is `"native"`: Scrapy sends `proxy_headers`, and this package does not send them again. It only copies the proxy's CONNECT response headers, such as `X-ProxyMesh-IP`, onto `response.headers`. On Scrapy 2.17.0 through 2.19.x the mode is `"legacy"` and this package still writes `proxy_headers` on CONNECT itself.
101
+
102
+ Header-only users on Scrapy 2.20.0 and newer can drop the `DOWNLOAD_HANDLERS` override and keep the same meta value:
103
+
104
+ ```python
105
+ yield scrapy.Request(
106
+ url="https://api.ipify.org?format=json",
107
+ meta={
108
+ "proxy": "http://user:pass@proxy.example.com:31280",
109
+ "proxy_headers": {"X-ProxyMesh-Country": "US"},
110
+ },
111
+ )
112
+ ```
113
+
114
+ Scrapy reads the CONNECT status line and discards the rest of the proxy response headers. `X-ProxyMesh-IP` and other CONNECT response headers are still only available through this package. Keep the download handler when the spider reads them from `response.headers`:
115
+
116
+ ```python
117
+ DOWNLOAD_HANDLERS = {
118
+ "https": "scrapy_proxy_headers.HTTP11ProxyDownloadHandler",
119
+ }
120
+ ```
121
+
122
+ `scrapy_proxy_headers.PROXY_HEADERS_MODE` is `"native"` or `"legacy"`. It is detected from the installed Scrapy. It is not a Scrapy setting. In native mode the captured CONNECT response is merged into the Scrapy response even when Scrapy reuses the tunnel.
123
+
124
+ ```python
125
+ import scrapy_proxy_headers
126
+
127
+ print(scrapy_proxy_headers.PROXY_HEADERS_MODE)
128
+ ```
129
+
92
130
  ## Complete Example
93
131
 
94
132
  ```python
@@ -103,11 +141,11 @@ class ProxyHeadersSpider(scrapy.Spider):
103
141
  }
104
142
  }
105
143
 
106
- def start_requests(self):
144
+ async def start(self):
107
145
  yield scrapy.Request(
108
146
  url="https://api.ipify.org?format=json",
109
147
  meta={
110
- "proxy": "http://us.proxymesh.com:31280",
148
+ "proxy": "http://user:pass@proxy.example.com:31280",
111
149
  "proxy_headers": {"X-ProxyMesh-Country": "US"}
112
150
  },
113
151
  callback=self.parse_ip
@@ -130,11 +168,12 @@ class ProxyHeadersSpider(scrapy.Spider):
130
168
  ## How It Works
131
169
 
132
170
  1. **HTTP11ProxyDownloadHandler** - Custom download handler that manages proxy header caching
133
- 2. **ScrapyProxyHeadersAgent** - Agent that reads `proxy_headers` from request meta
134
- 3. **TunnelingHeadersAgent** - Sends custom headers in the CONNECT request
135
- 4. **TunnelingHeadersTCP4ClientEndpoint** - Captures proxy response headers from CONNECT response
171
+ 2. **Legacy path** (`PROXY_HEADERS_MODE == "legacy"`) - `ScrapyProxyHeadersAgent` reads `proxy_headers` and `TunnelingHeadersAgent` sends them on CONNECT
172
+ 3. **Native path** (`PROXY_HEADERS_MODE == "native"`) - Scrapy sends `proxy_headers`; the handler only records the CONNECT response
173
+
174
+ On the legacy path, `TunnelingHeadersTCP4ClientEndpoint` captures proxy response headers from the CONNECT response. On the native path, a subclass of Scrapy's tunnel protocol does that capture.
136
175
 
137
- The handler also caches proxy response headers by proxy URL. This ensures headers remain available even when Scrapy reuses existing tunnel connections for subsequent requests.
176
+ The handler caches proxy response headers by proxy URL and the normalized `proxy_headers` for that request. A reused tunnel still exposes the CONNECT response from the header set that opened it, and a different header set does not reuse another set's cached `X-ProxyMesh-IP`.
138
177
 
139
178
  ## Test Harness
140
179
 
@@ -178,8 +217,17 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
178
217
 
179
218
  ## Requirements
180
219
 
181
- - Python 3.8+
182
- - Scrapy >=2.17.0,<2.20
220
+ - Python 3.10+
221
+ - Scrapy 2.17.0+ (no upper cap). Native `proxy_headers` starts at Scrapy 2.20.0.
222
+
223
+ Scrapy 2.17.0 and 2.18 support Python 3.10 through 3.14. Scrapy 2.19 and 2.20 also list 3.15. Scrapy 2.20.0 and Scrapy master require Python 3.11+.
224
+
225
+ Local compatibility tests do not need a proxy account:
226
+
227
+ ```bash
228
+ pip install -e ".[dev]"
229
+ pytest
230
+ ```
183
231
 
184
232
  ## License
185
233
 
@@ -0,0 +1,52 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "scrapy-proxy-headers"
7
+ version = "0.3.0"
8
+ authors = [
9
+ { name="ProxyMesh", email="support@proxymesh.com" },
10
+ ]
11
+ description = "Add custom proxy headers to HTTPS requests in Scrapy"
12
+ readme = "README.md"
13
+ requires-python = ">=3.10"
14
+ dependencies = [
15
+ "scrapy>=2.17.0",
16
+ ]
17
+ classifiers = [
18
+ "Framework :: Scrapy",
19
+ "Programming Language :: Python :: 3",
20
+ "Programming Language :: Python :: 3.10",
21
+ "Programming Language :: Python :: 3.11",
22
+ "Programming Language :: Python :: 3.12",
23
+ "Programming Language :: Python :: 3.13",
24
+ "Programming Language :: Python :: 3.14",
25
+ "Operating System :: OS Independent",
26
+ "License :: OSI Approved :: BSD License",
27
+ "Intended Audience :: Developers",
28
+ "Topic :: Internet :: WWW/HTTP",
29
+ "Topic :: Software Development :: Libraries :: Python Modules",
30
+ ]
31
+
32
+ [project.optional-dependencies]
33
+ dev = [
34
+ "pytest>=8",
35
+ ]
36
+
37
+ [project.urls]
38
+ Homepage = "https://github.com/proxymesh/scrapy-proxy-headers"
39
+ Changelog = "https://github.com/proxymesh/scrapy-proxy-headers/blob/main/CHANGELOG.md"
40
+ Issues = "https://github.com/proxymesh/scrapy-proxy-headers/issues"
41
+ Documentation = "https://scrapy-proxy-headers.readthedocs.io/en/latest/"
42
+ ProxyMesh = "https://proxymesh.com"
43
+
44
+ [tool.setuptools.packages.find]
45
+ include = ["scrapy_proxy_headers*"]
46
+
47
+ [tool.pytest.ini_options]
48
+ testpaths = ["tests"]
49
+
50
+ [tool.ruff.lint]
51
+ # Pre-0.16 default set, plus no new inline imports and lazy log args.
52
+ select = ["E4", "E7", "E9", "F", "PLC0415", "G004"]
@@ -0,0 +1,9 @@
1
+ """Send proxy headers on HTTPS CONNECT and read the proxy's response headers."""
2
+
3
+ from .download_handler import HTTP11ProxyDownloadHandler
4
+ from .mode import PROXY_HEADERS_MODE
5
+
6
+ __all__ = [
7
+ "PROXY_HEADERS_MODE",
8
+ "HTTP11ProxyDownloadHandler",
9
+ ]
@@ -0,0 +1,122 @@
1
+ """CONNECT header bytes and proxy-header cache keys.
2
+
3
+ These helpers do not touch Scrapy downloader internals.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from collections.abc import Mapping
9
+ from typing import Any
10
+
11
+ from scrapy.http import Headers, Response
12
+ from scrapy.utils.python import to_bytes
13
+
14
+
15
+ def tunnel_request_data_with_headers(
16
+ host: str, port: int, **proxy_headers: Any
17
+ ) -> bytes:
18
+ r"""
19
+ Return binary content of a CONNECT request.
20
+
21
+ >>> from scrapy.utils.python import to_unicode as s
22
+ >>> s(tunnel_request_data_with_headers("example.com", 8080))
23
+ 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
24
+ >>> s(tunnel_request_data_with_headers("example.com", 8080, **{"X-ProxyMesh-Country": "US"}))
25
+ 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nX-ProxyMesh-Country: US\r\n\r\n'
26
+ >>> s(tunnel_request_data_with_headers(b"example.com", "8090"))
27
+ 'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'
28
+ """
29
+ host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port))
30
+ tunnel_req = b"CONNECT " + host_value + b" HTTP/1.1\r\n"
31
+ tunnel_req += b"Host: " + host_value + b"\r\n"
32
+
33
+ for key, val in proxy_headers.items():
34
+ key_b = to_bytes(key)
35
+ val_b = to_bytes(val)
36
+ if b"\r" in key_b or b"\n" in key_b or b"\r" in val_b or b"\n" in val_b:
37
+ raise ValueError("proxy header contains CR or LF")
38
+ tunnel_req += key_b + b": " + val_b + b"\r\n"
39
+
40
+ tunnel_req += b"\r\n"
41
+ return tunnel_req
42
+
43
+
44
+ def _as_text(value: Any) -> str:
45
+ if isinstance(value, bytes):
46
+ return value.decode("utf-8", "replace")
47
+ return str(value)
48
+
49
+
50
+ def normalize_proxy_url(proxy: str) -> str:
51
+ """Return *proxy* with an ``http://`` scheme when the value omits one."""
52
+ proxy = proxy.strip()
53
+ if "://" not in proxy:
54
+ return "http://" + proxy
55
+ return proxy
56
+
57
+
58
+ def normalize_proxy_headers(
59
+ proxy_headers: Mapping[Any, Any] | None,
60
+ ) -> tuple[tuple[str, str], ...]:
61
+ """Return ``proxy_headers`` as sorted ``(lowercase name, value)`` pairs.
62
+
63
+ Header names are compared case-insensitively. Values keep their spelling.
64
+ A list or tuple value becomes one pair per item. ``None`` and empty
65
+ mappings become an empty tuple.
66
+ """
67
+ if not proxy_headers:
68
+ return ()
69
+ pairs: list[tuple[str, str]] = []
70
+ items = (
71
+ proxy_headers.items() if isinstance(proxy_headers, Mapping) else proxy_headers
72
+ )
73
+ for key, value in items:
74
+ name = _as_text(key).lower()
75
+ values = value if isinstance(value, (list, tuple)) else (value,)
76
+ for item in values:
77
+ pairs.append((name, _as_text(item)))
78
+ return tuple(sorted(pairs))
79
+
80
+
81
+ def proxy_cache_key(
82
+ proxy: str,
83
+ proxy_headers: Mapping[Any, Any] | None,
84
+ ) -> tuple[str, tuple[tuple[str, str], ...]]:
85
+ """Cache key for CONNECT response headers.
86
+
87
+ Requests that share a proxy URL but send different ``proxy_headers`` must
88
+ not reuse each other's CONNECT response (for example a different
89
+ ``X-ProxyMesh-IP``).
90
+ """
91
+ return (normalize_proxy_url(proxy), normalize_proxy_headers(proxy_headers))
92
+
93
+
94
+ def parse_connect_response_headers(header_block: bytes) -> Headers:
95
+ """Parse a CONNECT response, skipping the status line.
96
+
97
+ *header_block* is the bytes before the blank line that ends the headers.
98
+ """
99
+ pairs: list[tuple[bytes, bytes]] = []
100
+ lines = header_block.split(b"\r\n")
101
+ for line in lines[1:]:
102
+ if b":" not in line:
103
+ continue
104
+ key, value = line.split(b":", 1)
105
+ key = key.strip()
106
+ if not key:
107
+ continue
108
+ pairs.append((key, value.strip()))
109
+ return Headers(pairs)
110
+
111
+
112
+ def merge_proxy_response_headers(response: Response, proxy_headers: Headers) -> None:
113
+ """Copy CONNECT headers the origin response does not already have.
114
+
115
+ Proxy headers do not replace headers the origin already sent. Merging the
116
+ same proxy headers again does not duplicate values.
117
+ """
118
+ if not proxy_headers:
119
+ return
120
+ for name in proxy_headers:
121
+ if name not in response.headers:
122
+ response.headers[name] = proxy_headers.getlist(name)
@@ -0,0 +1,88 @@
1
+ """HTTPS download handler that exposes proxy CONNECT response headers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ from typing import Any
7
+
8
+ from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
9
+ from scrapy.http import Request, Response
10
+ from scrapy.utils.defer import maybe_deferred_to_future
11
+
12
+ from .connect import merge_proxy_response_headers, proxy_cache_key
13
+ from .mode import PROXY_HEADERS_MODE
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+ if PROXY_HEADERS_MODE == "native":
18
+ from .native import lookup_connect_response_headers
19
+ else:
20
+ from .legacy import ScrapyProxyHeadersAgent
21
+
22
+
23
+ class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
24
+ """Merge CONNECT response headers into the Scrapy response.
25
+
26
+ Configure it with::
27
+
28
+ DOWNLOAD_HANDLERS = {
29
+ "https": "scrapy_proxy_headers.HTTP11ProxyDownloadHandler",
30
+ }
31
+
32
+ ``request.meta["proxy_headers"]`` is sent on CONNECT. On Scrapy 2.20+
33
+ Scrapy sends those headers and this handler only copies the proxy's
34
+ response, including ``X-ProxyMesh-IP``. On Scrapy 2.17–2.19 this handler
35
+ sends them. The cache key is the proxy URL plus the normalized headers,
36
+ so two header sets do not share one ``X-ProxyMesh-IP``.
37
+ """
38
+
39
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
40
+ super().__init__(*args, **kwargs)
41
+ self._proxy_response_headers_cache: dict[tuple[Any, ...], Any] = {}
42
+ logger.info("scrapy-proxy-headers mode: %s", PROXY_HEADERS_MODE)
43
+
44
+ async def download_request(self, request: Request, spider: Any = None) -> Response:
45
+ """Download *request* and merge captured CONNECT headers onto it."""
46
+ if PROXY_HEADERS_MODE == "native":
47
+ response = await super().download_request(request)
48
+ captured = lookup_connect_response_headers(request)
49
+ return self._apply_cached_headers(response, request, captured)
50
+
51
+ if spider is None:
52
+ spider = self._crawler.spider
53
+ agent = ScrapyProxyHeadersAgent(
54
+ contextFactory=self._contextFactory,
55
+ pool=self._pool,
56
+ maxsize=getattr(spider, "download_maxsize", self._default_maxsize),
57
+ warnsize=getattr(spider, "download_warnsize", self._default_warnsize),
58
+ fail_on_dataloss=self._fail_on_dataloss,
59
+ crawler=self._crawler,
60
+ bindAddress=self._bind_address,
61
+ tls_verbose_logging=self._tls_verbose_logging,
62
+ )
63
+ deferred = agent.download_request(request)
64
+ deferred.addCallback(
65
+ lambda response: self._apply_cached_headers(
66
+ response, request, agent.proxy_response_headers
67
+ )
68
+ )
69
+ return await maybe_deferred_to_future(deferred)
70
+
71
+ def _apply_cached_headers(
72
+ self,
73
+ response: Response,
74
+ request: Request,
75
+ captured: Any,
76
+ ) -> Response:
77
+ proxy = request.meta.get("proxy")
78
+ if not proxy or not isinstance(response, Response):
79
+ return response
80
+ key = proxy_cache_key(proxy, request.meta.get("proxy_headers"))
81
+ if captured is not None:
82
+ self._proxy_response_headers_cache[key] = captured
83
+ cached = self._proxy_response_headers_cache.get(key)
84
+ if cached:
85
+ merge_proxy_response_headers(response, cached)
86
+ elif PROXY_HEADERS_MODE == "native":
87
+ logger.warning("No CONNECT response headers captured for proxy %s", proxy)
88
+ return response