scrapy-proxy-headers 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapy-proxy-headers
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Add custom proxy headers to HTTPS requests in Scrapy
5
5
  Author-email: ProxyMesh <support@proxymesh.com>
6
6
  Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
@@ -17,7 +17,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
17
17
  Requires-Python: >=3.8
18
18
  Description-Content-Type: text/markdown
19
19
  License-File: LICENSE
20
- Requires-Dist: scrapy>=2.14.2
20
+ Requires-Dist: scrapy<2.20,>=2.17.0
21
21
  Dynamic: license-file
22
22
 
23
23
  # Scrapy Proxy Headers
@@ -101,7 +101,7 @@ class MySpider(scrapy.Spider):
101
101
 
102
102
  ### 3. Receive Proxy Response Headers
103
103
 
104
- Headers from the proxy's CONNECT response are automatically merged into `response.headers`:
104
+ Headers from the proxy's CONNECT response are merged into `response.headers`. Headers the origin already sent are left unchanged.
105
105
 
106
106
  ```python
107
107
  def parse(self, response):
@@ -201,7 +201,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
201
201
  ## Requirements
202
202
 
203
203
  - Python 3.8+
204
- - Scrapy 2.14.2+
204
+ - Scrapy >=2.17.0,<2.20
205
205
 
206
206
  ## License
207
207
 
@@ -79,7 +79,7 @@ class MySpider(scrapy.Spider):
79
79
 
80
80
  ### 3. Receive Proxy Response Headers
81
81
 
82
- Headers from the proxy's CONNECT response are automatically merged into `response.headers`:
82
+ Headers from the proxy's CONNECT response are merged into `response.headers`. Headers the origin already sent are left unchanged.
83
83
 
84
84
  ```python
85
85
  def parse(self, response):
@@ -179,7 +179,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
179
179
  ## Requirements
180
180
 
181
181
  - Python 3.8+
182
- - Scrapy 2.14.2+
182
+ - Scrapy >=2.17.0,<2.20
183
183
 
184
184
  ## License
185
185
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "scrapy-proxy-headers"
7
- version = "0.2.1"
7
+ version = "0.2.2"
8
8
  authors = [
9
9
  { name="ProxyMesh", email="support@proxymesh.com" },
10
10
  ]
@@ -12,7 +12,7 @@ description = "Add custom proxy headers to HTTPS requests in Scrapy"
12
12
  readme = "README.md"
13
13
  requires-python = ">=3.8"
14
14
  dependencies = [
15
- "scrapy>=2.14.2",
15
+ "scrapy>=2.17.0,<2.20",
16
16
  ]
17
17
  classifiers = [
18
18
  "Programming Language :: Python :: 3",
@@ -33,12 +33,23 @@ def tunnel_request_data_with_headers(host: str, port: int, **proxy_headers) -> b
33
33
  tunnel_req += b"Host: " + host_value + b"\r\n"
34
34
 
35
35
  for key, val in proxy_headers.items():
36
- tunnel_req += to_bytes(key) + b": " + to_bytes(val) + b"\r\n"
36
+ key_b = to_bytes(key)
37
+ val_b = to_bytes(val)
38
+ if b"\r" in key_b or b"\n" in key_b or b"\r" in val_b or b"\n" in val_b:
39
+ raise ValueError("proxy header contains CR or LF")
40
+ tunnel_req += key_b + b": " + val_b + b"\r\n"
37
41
 
38
42
  tunnel_req += b"\r\n"
39
43
  return tunnel_req
40
44
 
41
45
 
46
+ def _merge_proxy_headers(response: Response, proxy_headers: Headers) -> None:
47
+ """Copy CONNECT headers that the origin response does not already have."""
48
+ for name in proxy_headers:
49
+ if name not in response.headers:
50
+ response.headers.setlist(name, proxy_headers.getlist(name))
51
+
52
+
42
53
  class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
43
54
  def __init__(
44
55
  self,
@@ -71,17 +82,19 @@ class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
71
82
  return protocol
72
83
 
73
84
  def processProxyResponse(self, data: bytes):
74
- # data might have proxy headers, looks like
75
- # b'HTTP/1.1 200 Connection established\r\nProxy-Header: VALUE\r\n\r\n'
85
+ # Parent appends this chunk to _connectBuffer. Parse that buffer, not
86
+ # this chunk alone, and stop at the first header terminator.
87
+ result = super(TunnelingHeadersTCP4ClientEndpoint, self).processProxyResponse(data)
88
+ if b"\r\n\r\n" not in self._connectBuffer or hasattr(self, "_proxy_response_headers"):
89
+ return result
90
+ header_block = bytes(self._connectBuffer).split(b"\r\n\r\n", 1)[0]
76
91
  response_headers = {}
77
-
78
- for line in data.split(b'\r\n'):
79
- if b':' in line:
80
- key, val = line.split(b':', 1)
92
+ for line in header_block.split(b"\r\n"):
93
+ if b":" in line:
94
+ key, val = line.split(b":", 1)
81
95
  response_headers[key.strip()] = val.strip()
82
- # save for endpoing & agent
83
96
  self._proxy_response_headers = Headers(response_headers)
84
- return super(TunnelingHeadersTCP4ClientEndpoint, self).processProxyResponse(data)
97
+ return result
85
98
 
86
99
 
87
100
  class TunnelingHeadersAgent(TunnelingAgent):
@@ -172,6 +185,6 @@ class ScrapyProxyHeadersAgent(ScrapyAgent):
172
185
  if self._agent and hasattr(self._agent, '_endpoint'):
173
186
  proxy_response_headers = getattr(self._agent._endpoint, '_proxy_response_headers', None)
174
187
  if proxy_response_headers:
175
- r.headers.update(proxy_response_headers)
188
+ _merge_proxy_headers(r, proxy_response_headers)
176
189
  self.proxy_response_headers = proxy_response_headers
177
190
  return r
@@ -1,6 +1,6 @@
1
1
  from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
2
2
  from scrapy.utils.defer import maybe_deferred_to_future
3
- from scrapy_proxy_headers.agent import ScrapyProxyHeadersAgent
3
+ from scrapy_proxy_headers.agent import ScrapyProxyHeadersAgent, _merge_proxy_headers
4
4
 
5
5
 
6
6
  class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
@@ -39,7 +39,7 @@ class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
39
39
  self._proxy_headers_by_proxy[proxy] = agent.proxy_response_headers
40
40
 
41
41
  if proxy in self._proxy_headers_by_proxy:
42
- response.headers.update(self._proxy_headers_by_proxy[proxy])
42
+ _merge_proxy_headers(response, self._proxy_headers_by_proxy[proxy])
43
43
 
44
44
  return response
45
45
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapy-proxy-headers
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Add custom proxy headers to HTTPS requests in Scrapy
5
5
  Author-email: ProxyMesh <support@proxymesh.com>
6
6
  Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
@@ -17,7 +17,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
17
17
  Requires-Python: >=3.8
18
18
  Description-Content-Type: text/markdown
19
19
  License-File: LICENSE
20
- Requires-Dist: scrapy>=2.14.2
20
+ Requires-Dist: scrapy<2.20,>=2.17.0
21
21
  Dynamic: license-file
22
22
 
23
23
  # Scrapy Proxy Headers
@@ -101,7 +101,7 @@ class MySpider(scrapy.Spider):
101
101
 
102
102
  ### 3. Receive Proxy Response Headers
103
103
 
104
- Headers from the proxy's CONNECT response are automatically merged into `response.headers`:
104
+ Headers from the proxy's CONNECT response are merged into `response.headers`. Headers the origin already sent are left unchanged.
105
105
 
106
106
  ```python
107
107
  def parse(self, response):
@@ -201,7 +201,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
201
201
  ## Requirements
202
202
 
203
203
  - Python 3.8+
204
- - Scrapy 2.14.2+
204
+ - Scrapy >=2.17.0,<2.20
205
205
 
206
206
  ## License
207
207
 
@@ -0,0 +1 @@
1
+ scrapy<2.20,>=2.17.0