scrapy-proxy-headers 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scrapy_proxy_headers-0.2.1/scrapy_proxy_headers.egg-info → scrapy_proxy_headers-0.2.2}/PKG-INFO +4 -4
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2}/README.md +2 -2
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2}/pyproject.toml +2 -2
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2}/scrapy_proxy_headers/agent.py +23 -10
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2}/scrapy_proxy_headers/download_handler.py +2 -2
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2/scrapy_proxy_headers.egg-info}/PKG-INFO +4 -4
- scrapy_proxy_headers-0.2.2/scrapy_proxy_headers.egg-info/requires.txt +1 -0
- scrapy_proxy_headers-0.2.1/scrapy_proxy_headers.egg-info/requires.txt +0 -1
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2}/LICENSE +0 -0
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2}/scrapy_proxy_headers/__init__.py +0 -0
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2}/scrapy_proxy_headers.egg-info/SOURCES.txt +0 -0
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2}/scrapy_proxy_headers.egg-info/dependency_links.txt +0 -0
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2}/scrapy_proxy_headers.egg-info/top_level.txt +0 -0
- {scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2}/setup.cfg +0 -0
{scrapy_proxy_headers-0.2.1/scrapy_proxy_headers.egg-info → scrapy_proxy_headers-0.2.2}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scrapy-proxy-headers
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Add custom proxy headers to HTTPS requests in Scrapy
|
|
5
5
|
Author-email: ProxyMesh <support@proxymesh.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
|
|
@@ -17,7 +17,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
|
17
17
|
Requires-Python: >=3.8
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: LICENSE
|
|
20
|
-
Requires-Dist: scrapy
|
|
20
|
+
Requires-Dist: scrapy<2.20,>=2.17.0
|
|
21
21
|
Dynamic: license-file
|
|
22
22
|
|
|
23
23
|
# Scrapy Proxy Headers
|
|
@@ -101,7 +101,7 @@ class MySpider(scrapy.Spider):
|
|
|
101
101
|
|
|
102
102
|
### 3. Receive Proxy Response Headers
|
|
103
103
|
|
|
104
|
-
Headers from the proxy's CONNECT response are
|
|
104
|
+
Headers from the proxy's CONNECT response are merged into `response.headers`. Headers the origin already sent are left unchanged.
|
|
105
105
|
|
|
106
106
|
```python
|
|
107
107
|
def parse(self, response):
|
|
@@ -201,7 +201,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
|
|
|
201
201
|
## Requirements
|
|
202
202
|
|
|
203
203
|
- Python 3.8+
|
|
204
|
-
- Scrapy 2.
|
|
204
|
+
- Scrapy >=2.17.0,<2.20
|
|
205
205
|
|
|
206
206
|
## License
|
|
207
207
|
|
|
@@ -79,7 +79,7 @@ class MySpider(scrapy.Spider):
|
|
|
79
79
|
|
|
80
80
|
### 3. Receive Proxy Response Headers
|
|
81
81
|
|
|
82
|
-
Headers from the proxy's CONNECT response are
|
|
82
|
+
Headers from the proxy's CONNECT response are merged into `response.headers`. Headers the origin already sent are left unchanged.
|
|
83
83
|
|
|
84
84
|
```python
|
|
85
85
|
def parse(self, response):
|
|
@@ -179,7 +179,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
|
|
|
179
179
|
## Requirements
|
|
180
180
|
|
|
181
181
|
- Python 3.8+
|
|
182
|
-
- Scrapy 2.
|
|
182
|
+
- Scrapy >=2.17.0,<2.20
|
|
183
183
|
|
|
184
184
|
## License
|
|
185
185
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scrapy-proxy-headers"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name="ProxyMesh", email="support@proxymesh.com" },
|
|
10
10
|
]
|
|
@@ -12,7 +12,7 @@ description = "Add custom proxy headers to HTTPS requests in Scrapy"
|
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
requires-python = ">=3.8"
|
|
14
14
|
dependencies = [
|
|
15
|
-
"scrapy>=2.
|
|
15
|
+
"scrapy>=2.17.0,<2.20",
|
|
16
16
|
]
|
|
17
17
|
classifiers = [
|
|
18
18
|
"Programming Language :: Python :: 3",
|
|
@@ -33,12 +33,23 @@ def tunnel_request_data_with_headers(host: str, port: int, **proxy_headers) -> b
|
|
|
33
33
|
tunnel_req += b"Host: " + host_value + b"\r\n"
|
|
34
34
|
|
|
35
35
|
for key, val in proxy_headers.items():
|
|
36
|
-
|
|
36
|
+
key_b = to_bytes(key)
|
|
37
|
+
val_b = to_bytes(val)
|
|
38
|
+
if b"\r" in key_b or b"\n" in key_b or b"\r" in val_b or b"\n" in val_b:
|
|
39
|
+
raise ValueError("proxy header contains CR or LF")
|
|
40
|
+
tunnel_req += key_b + b": " + val_b + b"\r\n"
|
|
37
41
|
|
|
38
42
|
tunnel_req += b"\r\n"
|
|
39
43
|
return tunnel_req
|
|
40
44
|
|
|
41
45
|
|
|
46
|
+
def _merge_proxy_headers(response: Response, proxy_headers: Headers) -> None:
|
|
47
|
+
"""Copy CONNECT headers that the origin response does not already have."""
|
|
48
|
+
for name in proxy_headers:
|
|
49
|
+
if name not in response.headers:
|
|
50
|
+
response.headers.setlist(name, proxy_headers.getlist(name))
|
|
51
|
+
|
|
52
|
+
|
|
42
53
|
class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
|
|
43
54
|
def __init__(
|
|
44
55
|
self,
|
|
@@ -71,17 +82,19 @@ class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
|
|
|
71
82
|
return protocol
|
|
72
83
|
|
|
73
84
|
def processProxyResponse(self, data: bytes):
|
|
74
|
-
#
|
|
75
|
-
#
|
|
85
|
+
# Parent appends this chunk to _connectBuffer. Parse that buffer, not
|
|
86
|
+
# this chunk alone, and stop at the first header terminator.
|
|
87
|
+
result = super(TunnelingHeadersTCP4ClientEndpoint, self).processProxyResponse(data)
|
|
88
|
+
if b"\r\n\r\n" not in self._connectBuffer or hasattr(self, "_proxy_response_headers"):
|
|
89
|
+
return result
|
|
90
|
+
header_block = bytes(self._connectBuffer).split(b"\r\n\r\n", 1)[0]
|
|
76
91
|
response_headers = {}
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
key, val = line.split(b':', 1)
|
|
92
|
+
for line in header_block.split(b"\r\n"):
|
|
93
|
+
if b":" in line:
|
|
94
|
+
key, val = line.split(b":", 1)
|
|
81
95
|
response_headers[key.strip()] = val.strip()
|
|
82
|
-
# save for endpoing & agent
|
|
83
96
|
self._proxy_response_headers = Headers(response_headers)
|
|
84
|
-
return
|
|
97
|
+
return result
|
|
85
98
|
|
|
86
99
|
|
|
87
100
|
class TunnelingHeadersAgent(TunnelingAgent):
|
|
@@ -172,6 +185,6 @@ class ScrapyProxyHeadersAgent(ScrapyAgent):
|
|
|
172
185
|
if self._agent and hasattr(self._agent, '_endpoint'):
|
|
173
186
|
proxy_response_headers = getattr(self._agent._endpoint, '_proxy_response_headers', None)
|
|
174
187
|
if proxy_response_headers:
|
|
175
|
-
r
|
|
188
|
+
_merge_proxy_headers(r, proxy_response_headers)
|
|
176
189
|
self.proxy_response_headers = proxy_response_headers
|
|
177
190
|
return r
|
{scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2}/scrapy_proxy_headers/download_handler.py
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
|
|
2
2
|
from scrapy.utils.defer import maybe_deferred_to_future
|
|
3
|
-
from scrapy_proxy_headers.agent import ScrapyProxyHeadersAgent
|
|
3
|
+
from scrapy_proxy_headers.agent import ScrapyProxyHeadersAgent, _merge_proxy_headers
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
|
|
@@ -39,7 +39,7 @@ class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
|
|
|
39
39
|
self._proxy_headers_by_proxy[proxy] = agent.proxy_response_headers
|
|
40
40
|
|
|
41
41
|
if proxy in self._proxy_headers_by_proxy:
|
|
42
|
-
response
|
|
42
|
+
_merge_proxy_headers(response, self._proxy_headers_by_proxy[proxy])
|
|
43
43
|
|
|
44
44
|
return response
|
|
45
45
|
|
{scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2/scrapy_proxy_headers.egg-info}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scrapy-proxy-headers
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Add custom proxy headers to HTTPS requests in Scrapy
|
|
5
5
|
Author-email: ProxyMesh <support@proxymesh.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
|
|
@@ -17,7 +17,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
|
17
17
|
Requires-Python: >=3.8
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: LICENSE
|
|
20
|
-
Requires-Dist: scrapy
|
|
20
|
+
Requires-Dist: scrapy<2.20,>=2.17.0
|
|
21
21
|
Dynamic: license-file
|
|
22
22
|
|
|
23
23
|
# Scrapy Proxy Headers
|
|
@@ -101,7 +101,7 @@ class MySpider(scrapy.Spider):
|
|
|
101
101
|
|
|
102
102
|
### 3. Receive Proxy Response Headers
|
|
103
103
|
|
|
104
|
-
Headers from the proxy's CONNECT response are
|
|
104
|
+
Headers from the proxy's CONNECT response are merged into `response.headers`. Headers the origin already sent are left unchanged.
|
|
105
105
|
|
|
106
106
|
```python
|
|
107
107
|
def parse(self, response):
|
|
@@ -201,7 +201,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
|
|
|
201
201
|
## Requirements
|
|
202
202
|
|
|
203
203
|
- Python 3.8+
|
|
204
|
-
- Scrapy 2.
|
|
204
|
+
- Scrapy >=2.17.0,<2.20
|
|
205
205
|
|
|
206
206
|
## License
|
|
207
207
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
scrapy<2.20,>=2.17.0
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
scrapy>=2.14.2
|
|
File without changes
|
|
File without changes
|
{scrapy_proxy_headers-0.2.1 → scrapy_proxy_headers-0.2.2}/scrapy_proxy_headers.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|