scrapy-proxy-headers 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapy-proxy-headers
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Summary: Add custom proxy headers to HTTPS requests in Scrapy
5
5
  Author-email: ProxyMesh <support@proxymesh.com>
6
6
  Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
@@ -17,7 +17,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
17
17
  Requires-Python: >=3.8
18
18
  Description-Content-Type: text/markdown
19
19
  License-File: LICENSE
20
- Requires-Dist: scrapy>=2.14.1
20
+ Requires-Dist: scrapy<2.20,>=2.17.0
21
21
  Dynamic: license-file
22
22
 
23
23
  # Scrapy Proxy Headers
@@ -101,7 +101,7 @@ class MySpider(scrapy.Spider):
101
101
 
102
102
  ### 3. Receive Proxy Response Headers
103
103
 
104
- Headers from the proxy's CONNECT response are automatically merged into `response.headers`:
104
+ Headers from the proxy's CONNECT response are merged into `response.headers`. Headers the origin already sent are left unchanged.
105
105
 
106
106
  ```python
107
107
  def parse(self, response):
@@ -201,7 +201,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
201
201
  ## Requirements
202
202
 
203
203
  - Python 3.8+
204
- - Scrapy 2.0+
204
+ - Scrapy >=2.17.0,<2.20
205
205
 
206
206
  ## License
207
207
 
@@ -79,7 +79,7 @@ class MySpider(scrapy.Spider):
79
79
 
80
80
  ### 3. Receive Proxy Response Headers
81
81
 
82
- Headers from the proxy's CONNECT response are automatically merged into `response.headers`:
82
+ Headers from the proxy's CONNECT response are merged into `response.headers`. Headers the origin already sent are left unchanged.
83
83
 
84
84
  ```python
85
85
  def parse(self, response):
@@ -179,7 +179,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
179
179
  ## Requirements
180
180
 
181
181
  - Python 3.8+
182
- - Scrapy 2.0+
182
+ - Scrapy >=2.17.0,<2.20
183
183
 
184
184
  ## License
185
185
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "scrapy-proxy-headers"
7
- version = "0.2.0"
7
+ version = "0.2.2"
8
8
  authors = [
9
9
  { name="ProxyMesh", email="support@proxymesh.com" },
10
10
  ]
@@ -12,7 +12,7 @@ description = "Add custom proxy headers to HTTPS requests in Scrapy"
12
12
  readme = "README.md"
13
13
  requires-python = ">=3.8"
14
14
  dependencies = [
15
- "scrapy>=2.14.1",
15
+ "scrapy>=2.17.0,<2.20",
16
16
  ]
17
17
  classifiers = [
18
18
  "Programming Language :: Python :: 3",
@@ -1,7 +1,20 @@
1
1
  from urllib.parse import urlparse
2
- from scrapy.core.downloader.handlers.http11 import TunnelingAgent, TunnelingTCP4ClientEndpoint, ScrapyAgent
3
- from scrapy.utils.python import to_bytes
2
+
3
+ from scrapy.core.downloader.handlers import http11 as _http11
4
4
  from scrapy.http import Headers, Response
5
+ from scrapy.utils.httpobj import urlparse_cached
6
+ from scrapy.utils.python import to_bytes
7
+ from scrapy.utils.url import add_http_if_no_scheme
8
+
9
+ # Scrapy 2.18 made these HTTP11 internals private (leading underscore).
10
+ TunnelingTCP4ClientEndpoint = getattr(
11
+ _http11, "TunnelingTCP4ClientEndpoint", None
12
+ ) or getattr(_http11, "_TunnelingTCP4ClientEndpoint")
13
+ TunnelingAgent = getattr(_http11, "TunnelingAgent", None) or getattr(
14
+ _http11, "_TunnelingAgent"
15
+ )
16
+ ScrapyAgent = getattr(_http11, "ScrapyAgent", None) or getattr(_http11, "_ScrapyAgent")
17
+
5
18
 
6
19
  def tunnel_request_data_with_headers(host: str, port: int, **proxy_headers) -> bytes:
7
20
  r"""
@@ -18,13 +31,25 @@ def tunnel_request_data_with_headers(host: str, port: int, **proxy_headers) -> b
18
31
  host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port))
19
32
  tunnel_req = b"CONNECT " + host_value + b" HTTP/1.1\r\n"
20
33
  tunnel_req += b"Host: " + host_value + b"\r\n"
21
-
34
+
22
35
  for key, val in proxy_headers.items():
23
- tunnel_req += to_bytes(key) + b": " + to_bytes(val) + b"\r\n"
24
-
36
+ key_b = to_bytes(key)
37
+ val_b = to_bytes(val)
38
+ if b"\r" in key_b or b"\n" in key_b or b"\r" in val_b or b"\n" in val_b:
39
+ raise ValueError("proxy header contains CR or LF")
40
+ tunnel_req += key_b + b": " + val_b + b"\r\n"
41
+
25
42
  tunnel_req += b"\r\n"
26
43
  return tunnel_req
27
44
 
45
+
46
+ def _merge_proxy_headers(response: Response, proxy_headers: Headers) -> None:
47
+ """Copy CONNECT headers that the origin response does not already have."""
48
+ for name in proxy_headers:
49
+ if name not in response.headers:
50
+ response.headers.setlist(name, proxy_headers.getlist(name))
51
+
52
+
28
53
  class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
29
54
  def __init__(
30
55
  self,
@@ -34,7 +59,7 @@ class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
34
59
  proxyConf,
35
60
  contextFactory,
36
61
  timeout: float = 30,
37
- bindAddress = None,
62
+ bindAddress=None,
38
63
  **proxy_headers
39
64
  ):
40
65
  super().__init__(reactor, host, port, proxyConf, contextFactory, timeout, bindAddress)
@@ -43,7 +68,7 @@ class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
43
68
  if self._proxyAuthHeader:
44
69
  self._proxy_headers['Proxy-Authorization'] = self._proxyAuthHeader
45
70
  self._proxy_headers.update(proxy_headers)
46
-
71
+
47
72
  def requestTunnel(self, protocol):
48
73
  """Asks the proxy to open a tunnel."""
49
74
  assert protocol.transport
@@ -55,19 +80,22 @@ class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
55
80
  protocol.dataReceived = self.processProxyResponse # type: ignore[method-assign]
56
81
  self._protocol = protocol
57
82
  return protocol
58
-
83
+
59
84
  def processProxyResponse(self, data: bytes):
60
- # data might have proxy headers, looks like
61
- # b'HTTP/1.1 200 Connection established\r\nProxy-Header: VALUE\r\n\r\n'
85
+ # Parent appends this chunk to _connectBuffer. Parse that buffer, not
86
+ # this chunk alone, and stop at the first header terminator.
87
+ result = super(TunnelingHeadersTCP4ClientEndpoint, self).processProxyResponse(data)
88
+ if b"\r\n\r\n" not in self._connectBuffer or hasattr(self, "_proxy_response_headers"):
89
+ return result
90
+ header_block = bytes(self._connectBuffer).split(b"\r\n\r\n", 1)[0]
62
91
  response_headers = {}
63
-
64
- for line in data.split(b'\r\n'):
65
- if b':' in line:
66
- key, val = line.split(b':', 1)
92
+ for line in header_block.split(b"\r\n"):
93
+ if b":" in line:
94
+ key, val = line.split(b":", 1)
67
95
  response_headers[key.strip()] = val.strip()
68
- # save for endpoing & agent
69
96
  self._proxy_response_headers = Headers(response_headers)
70
- return super(TunnelingHeadersTCP4ClientEndpoint, self).processProxyResponse(data)
97
+ return result
98
+
71
99
 
72
100
  class TunnelingHeadersAgent(TunnelingAgent):
73
101
  """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
@@ -79,7 +107,7 @@ class TunnelingHeadersAgent(TunnelingAgent):
79
107
  def __init__(self, *args, **kwargs):
80
108
  super().__init__(*args, **kwargs)
81
109
  self._proxy_headers = {}
82
-
110
+
83
111
  def set_proxy_headers(self, proxy_headers):
84
112
  self._proxy_headers = proxy_headers
85
113
 
@@ -97,6 +125,7 @@ class TunnelingHeadersAgent(TunnelingAgent):
97
125
  )
98
126
  return self._endpoint
99
127
 
128
+
100
129
  class ScrapyProxyHeadersAgent(ScrapyAgent):
101
130
  _TunnelingAgent = TunnelingHeadersAgent
102
131
 
@@ -104,17 +133,49 @@ class ScrapyProxyHeadersAgent(ScrapyAgent):
104
133
  super().__init__(*args, **kwargs)
105
134
  self._agent = None
106
135
  self.proxy_response_headers = None
107
-
108
- def _get_agent(self, request, timeout: float):
109
- self._agent = super()._get_agent(request, timeout)
110
136
 
137
+ def _get_agent(self, request, timeout: float):
138
+ # Scrapy 2.18+ instantiates _TunnelingAgent directly instead of
139
+ # self._TunnelingAgent, so always build our subclass for HTTPS proxies.
111
140
  proxy = request.meta.get("proxy")
112
- proxy_headers = request.meta.get('proxy_headers')
113
- if proxy and proxy_headers:
114
- scheme = urlparse(request.url).scheme
115
- if scheme == "https":
141
+ if proxy and urlparse_cached(request).scheme == "https":
142
+ from twisted.internet import reactor
143
+
144
+ proxy = add_http_if_no_scheme(proxy)
145
+ proxy_parsed = urlparse(proxy)
146
+ proxy_host = proxy_parsed.hostname
147
+ proxy_port = proxy_parsed.port
148
+ if not proxy_port:
149
+ proxy_port = 443 if proxy_parsed.scheme == "https" else 80
150
+ if proxy_parsed.scheme == "https":
151
+ raise NotImplementedError(
152
+ "HTTPS proxies for HTTPS destinations are not supported"
153
+ )
154
+ bindaddress = request.meta.get("bindaddress") or self._bindAddress
155
+ try:
156
+ from scrapy.utils._download_handlers import normalize_bind_address
157
+ bindaddress = normalize_bind_address(bindaddress)
158
+ except ImportError:
159
+ pass
160
+ proxyAuth = request.headers.get(b"Proxy-Authorization", None)
161
+ proxyConf = (proxy_host, proxy_port, proxyAuth)
162
+ self._agent = TunnelingHeadersAgent(
163
+ reactor=reactor,
164
+ proxyConf=proxyConf,
165
+ contextFactory=self._contextFactory,
166
+ connectTimeout=timeout,
167
+ bindAddress=bindaddress,
168
+ pool=self._pool,
169
+ )
170
+ proxy_headers = request.meta.get("proxy_headers")
171
+ if proxy_headers:
116
172
  self._agent.set_proxy_headers(proxy_headers)
117
-
173
+ return self._agent
174
+
175
+ self._agent = super()._get_agent(request, timeout)
176
+ proxy_headers = request.meta.get("proxy_headers")
177
+ if proxy and proxy_headers and hasattr(self._agent, "set_proxy_headers"):
178
+ self._agent.set_proxy_headers(proxy_headers)
118
179
  return self._agent
119
180
 
120
181
  def _cb_bodydone(self, result, *args):
@@ -124,6 +185,6 @@ class ScrapyProxyHeadersAgent(ScrapyAgent):
124
185
  if self._agent and hasattr(self._agent, '_endpoint'):
125
186
  proxy_response_headers = getattr(self._agent._endpoint, '_proxy_response_headers', None)
126
187
  if proxy_response_headers:
127
- r.headers.update(proxy_response_headers)
188
+ _merge_proxy_headers(r, proxy_response_headers)
128
189
  self.proxy_response_headers = proxy_response_headers
129
- return r
190
+ return r
@@ -0,0 +1,47 @@
1
+ from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
2
+ from scrapy.utils.defer import maybe_deferred_to_future
3
+ from scrapy_proxy_headers.agent import ScrapyProxyHeadersAgent, _merge_proxy_headers
4
+
5
+
6
+ class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
7
+ def __init__(self, *args, **kwargs):
8
+ super().__init__(*args, **kwargs)
9
+ self._proxy_headers_by_proxy = {}
10
+
11
+ async def download_request(self, request, spider=None):
12
+ """Download the request and merge CONNECT proxy headers into the response."""
13
+ # Support both old Scrapy (spider param) and new Scrapy (self._crawler.spider)
14
+ if spider is None:
15
+ spider = self._crawler.spider
16
+
17
+ agent_kwargs = {
18
+ "contextFactory": self._contextFactory,
19
+ "pool": self._pool,
20
+ "maxsize": getattr(spider, "download_maxsize", self._default_maxsize),
21
+ "warnsize": getattr(spider, "download_warnsize", self._default_warnsize),
22
+ "fail_on_dataloss": self._fail_on_dataloss,
23
+ "crawler": self._crawler,
24
+ }
25
+ bind_address = getattr(self, "_bind_address", None)
26
+ if bind_address is not None:
27
+ agent_kwargs["bindAddress"] = bind_address
28
+
29
+ agent = ScrapyProxyHeadersAgent(**agent_kwargs)
30
+ deferred = agent.download_request(request)
31
+ proxy = request.meta.get("proxy")
32
+
33
+ if proxy:
34
+ # Proxy tunnels can get re-used; when that happens, proxy headers
35
+ # are not available in subsequent responses. Save proxy headers by
36
+ # proxy URL from the first tunnel response to add to later responses.
37
+ def callback(response):
38
+ if agent.proxy_response_headers:
39
+ self._proxy_headers_by_proxy[proxy] = agent.proxy_response_headers
40
+
41
+ if proxy in self._proxy_headers_by_proxy:
42
+ _merge_proxy_headers(response, self._proxy_headers_by_proxy[proxy])
43
+
44
+ return response
45
+
46
+ deferred.addCallback(callback)
47
+ return await maybe_deferred_to_future(deferred)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapy-proxy-headers
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Summary: Add custom proxy headers to HTTPS requests in Scrapy
5
5
  Author-email: ProxyMesh <support@proxymesh.com>
6
6
  Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
@@ -17,7 +17,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
17
17
  Requires-Python: >=3.8
18
18
  Description-Content-Type: text/markdown
19
19
  License-File: LICENSE
20
- Requires-Dist: scrapy>=2.14.1
20
+ Requires-Dist: scrapy<2.20,>=2.17.0
21
21
  Dynamic: license-file
22
22
 
23
23
  # Scrapy Proxy Headers
@@ -101,7 +101,7 @@ class MySpider(scrapy.Spider):
101
101
 
102
102
  ### 3. Receive Proxy Response Headers
103
103
 
104
- Headers from the proxy's CONNECT response are automatically merged into `response.headers`:
104
+ Headers from the proxy's CONNECT response are merged into `response.headers`. Headers the origin already sent are left unchanged.
105
105
 
106
106
  ```python
107
107
  def parse(self, response):
@@ -201,7 +201,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
201
201
  ## Requirements
202
202
 
203
203
  - Python 3.8+
204
- - Scrapy 2.0+
204
+ - Scrapy >=2.17.0,<2.20
205
205
 
206
206
  ## License
207
207
 
@@ -0,0 +1 @@
1
+ scrapy<2.20,>=2.17.0
@@ -1,41 +0,0 @@
1
- from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
2
- from scrapy_proxy_headers.agent import ScrapyProxyHeadersAgent
3
-
4
-
5
- class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
6
- def __init__(self, *args, **kwargs):
7
- super().__init__(*args, **kwargs)
8
- self._proxy_headers_by_proxy = {}
9
-
10
- def download_request(self, request, spider=None):
11
- """Return a deferred for the HTTP download"""
12
- # Support both old Scrapy (spider param) and new Scrapy (self._crawler.spider)
13
- if spider is None:
14
- spider = self._crawler.spider
15
-
16
- agent = ScrapyProxyHeadersAgent(
17
- contextFactory=self._contextFactory,
18
- pool=self._pool,
19
- maxsize=getattr(spider, "download_maxsize", self._default_maxsize),
20
- warnsize=getattr(spider, "download_warnsize", self._default_warnsize),
21
- fail_on_dataloss=self._fail_on_dataloss,
22
- crawler=self._crawler,
23
- )
24
- response = agent.download_request(request)
25
- proxy = request.meta.get("proxy")
26
-
27
- if proxy:
28
- # Proxy tunnels can get re-used; when that happens, proxy headers
29
- # are not available in subsequent responses. Save proxy headers by
30
- # proxy URL from the first tunnel response to add to later responses.
31
- def callback(response):
32
- if agent.proxy_response_headers:
33
- self._proxy_headers_by_proxy[proxy] = agent.proxy_response_headers
34
-
35
- if proxy in self._proxy_headers_by_proxy:
36
- response.headers.update(self._proxy_headers_by_proxy[proxy])
37
-
38
- return response
39
-
40
- response.addCallback(callback)
41
- return response