scrapy-proxy-headers 0.1.6__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapy-proxy-headers
3
- Version: 0.1.6
3
+ Version: 0.2.1
4
4
  Summary: Add custom proxy headers to HTTPS requests in Scrapy
5
5
  Author-email: ProxyMesh <support@proxymesh.com>
6
6
  Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
@@ -17,6 +17,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
17
17
  Requires-Python: >=3.8
18
18
  Description-Content-Type: text/markdown
19
19
  License-File: LICENSE
20
+ Requires-Dist: scrapy>=2.14.2
20
21
  Dynamic: license-file
21
22
 
22
23
  # Scrapy Proxy Headers
@@ -200,7 +201,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
200
201
  ## Requirements
201
202
 
202
203
  - Python 3.8+
203
- - Scrapy 2.0+
204
+ - Scrapy 2.14.2+
204
205
 
205
206
  ## License
206
207
 
@@ -179,7 +179,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
179
179
  ## Requirements
180
180
 
181
181
  - Python 3.8+
182
- - Scrapy 2.0+
182
+ - Scrapy 2.14.2+
183
183
 
184
184
  ## License
185
185
 
@@ -4,13 +4,16 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "scrapy-proxy-headers"
7
- version = "0.1.6"
7
+ version = "0.2.1"
8
8
  authors = [
9
9
  { name="ProxyMesh", email="support@proxymesh.com" },
10
10
  ]
11
11
  description = "Add custom proxy headers to HTTPS requests in Scrapy"
12
12
  readme = "README.md"
13
13
  requires-python = ">=3.8"
14
+ dependencies = [
15
+ "scrapy>=2.14.2",
16
+ ]
14
17
  classifiers = [
15
18
  "Programming Language :: Python :: 3",
16
19
  "Operating System :: OS Independent",
@@ -1,7 +1,20 @@
1
1
  from urllib.parse import urlparse
2
- from scrapy.core.downloader.handlers.http11 import TunnelingAgent, TunnelingTCP4ClientEndpoint, ScrapyAgent
3
- from scrapy.utils.python import to_bytes
2
+
3
+ from scrapy.core.downloader.handlers import http11 as _http11
4
4
  from scrapy.http import Headers, Response
5
+ from scrapy.utils.httpobj import urlparse_cached
6
+ from scrapy.utils.python import to_bytes
7
+ from scrapy.utils.url import add_http_if_no_scheme
8
+
9
+ # Scrapy 2.18 made these HTTP11 internals private (leading underscore).
10
+ TunnelingTCP4ClientEndpoint = getattr(
11
+ _http11, "TunnelingTCP4ClientEndpoint", None
12
+ ) or getattr(_http11, "_TunnelingTCP4ClientEndpoint")
13
+ TunnelingAgent = getattr(_http11, "TunnelingAgent", None) or getattr(
14
+ _http11, "_TunnelingAgent"
15
+ )
16
+ ScrapyAgent = getattr(_http11, "ScrapyAgent", None) or getattr(_http11, "_ScrapyAgent")
17
+
5
18
 
6
19
  def tunnel_request_data_with_headers(host: str, port: int, **proxy_headers) -> bytes:
7
20
  r"""
@@ -18,13 +31,14 @@ def tunnel_request_data_with_headers(host: str, port: int, **proxy_headers) -> b
18
31
  host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port))
19
32
  tunnel_req = b"CONNECT " + host_value + b" HTTP/1.1\r\n"
20
33
  tunnel_req += b"Host: " + host_value + b"\r\n"
21
-
34
+
22
35
  for key, val in proxy_headers.items():
23
36
  tunnel_req += to_bytes(key) + b": " + to_bytes(val) + b"\r\n"
24
-
37
+
25
38
  tunnel_req += b"\r\n"
26
39
  return tunnel_req
27
40
 
41
+
28
42
  class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
29
43
  def __init__(
30
44
  self,
@@ -34,7 +48,7 @@ class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
34
48
  proxyConf,
35
49
  contextFactory,
36
50
  timeout: float = 30,
37
- bindAddress = None,
51
+ bindAddress=None,
38
52
  **proxy_headers
39
53
  ):
40
54
  super().__init__(reactor, host, port, proxyConf, contextFactory, timeout, bindAddress)
@@ -43,7 +57,7 @@ class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
43
57
  if self._proxyAuthHeader:
44
58
  self._proxy_headers['Proxy-Authorization'] = self._proxyAuthHeader
45
59
  self._proxy_headers.update(proxy_headers)
46
-
60
+
47
61
  def requestTunnel(self, protocol):
48
62
  """Asks the proxy to open a tunnel."""
49
63
  assert protocol.transport
@@ -55,7 +69,7 @@ class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
55
69
  protocol.dataReceived = self.processProxyResponse # type: ignore[method-assign]
56
70
  self._protocol = protocol
57
71
  return protocol
58
-
72
+
59
73
  def processProxyResponse(self, data: bytes):
60
74
  # data might have proxy headers, looks like
61
75
  # b'HTTP/1.1 200 Connection established\r\nProxy-Header: VALUE\r\n\r\n'
@@ -69,6 +83,7 @@ class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
69
83
  self._proxy_response_headers = Headers(response_headers)
70
84
  return super(TunnelingHeadersTCP4ClientEndpoint, self).processProxyResponse(data)
71
85
 
86
+
72
87
  class TunnelingHeadersAgent(TunnelingAgent):
73
88
  """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
74
89
  downloads. It may look strange that we have chosen to subclass Agent and not
@@ -79,7 +94,7 @@ class TunnelingHeadersAgent(TunnelingAgent):
79
94
  def __init__(self, *args, **kwargs):
80
95
  super().__init__(*args, **kwargs)
81
96
  self._proxy_headers = {}
82
-
97
+
83
98
  def set_proxy_headers(self, proxy_headers):
84
99
  self._proxy_headers = proxy_headers
85
100
 
@@ -97,32 +112,66 @@ class TunnelingHeadersAgent(TunnelingAgent):
97
112
  )
98
113
  return self._endpoint
99
114
 
115
+
100
116
  class ScrapyProxyHeadersAgent(ScrapyAgent):
101
117
  _TunnelingAgent = TunnelingHeadersAgent
102
118
 
103
119
  def __init__(self, *args, **kwargs):
104
120
  super().__init__(*args, **kwargs)
105
121
  self._agent = None
106
-
107
- def _get_agent(self, request, timeout: float):
108
- self._agent = super()._get_agent(request, timeout)
122
+ self.proxy_response_headers = None
109
123
 
124
+ def _get_agent(self, request, timeout: float):
125
+ # Scrapy 2.18+ instantiates _TunnelingAgent directly instead of
126
+ # self._TunnelingAgent, so always build our subclass for HTTPS proxies.
110
127
  proxy = request.meta.get("proxy")
111
- proxy_headers = request.meta.get('proxy_headers')
112
- if proxy and proxy_headers:
113
- scheme = urlparse(request.url).scheme
114
- if scheme == "https":
128
+ if proxy and urlparse_cached(request).scheme == "https":
129
+ from twisted.internet import reactor
130
+
131
+ proxy = add_http_if_no_scheme(proxy)
132
+ proxy_parsed = urlparse(proxy)
133
+ proxy_host = proxy_parsed.hostname
134
+ proxy_port = proxy_parsed.port
135
+ if not proxy_port:
136
+ proxy_port = 443 if proxy_parsed.scheme == "https" else 80
137
+ if proxy_parsed.scheme == "https":
138
+ raise NotImplementedError(
139
+ "HTTPS proxies for HTTPS destinations are not supported"
140
+ )
141
+ bindaddress = request.meta.get("bindaddress") or self._bindAddress
142
+ try:
143
+ from scrapy.utils._download_handlers import normalize_bind_address
144
+ bindaddress = normalize_bind_address(bindaddress)
145
+ except ImportError:
146
+ pass
147
+ proxyAuth = request.headers.get(b"Proxy-Authorization", None)
148
+ proxyConf = (proxy_host, proxy_port, proxyAuth)
149
+ self._agent = TunnelingHeadersAgent(
150
+ reactor=reactor,
151
+ proxyConf=proxyConf,
152
+ contextFactory=self._contextFactory,
153
+ connectTimeout=timeout,
154
+ bindAddress=bindaddress,
155
+ pool=self._pool,
156
+ )
157
+ proxy_headers = request.meta.get("proxy_headers")
158
+ if proxy_headers:
115
159
  self._agent.set_proxy_headers(proxy_headers)
116
-
160
+ return self._agent
161
+
162
+ self._agent = super()._get_agent(request, timeout)
163
+ proxy_headers = request.meta.get("proxy_headers")
164
+ if proxy and proxy_headers and hasattr(self._agent, "set_proxy_headers"):
165
+ self._agent.set_proxy_headers(proxy_headers)
117
166
  return self._agent
118
167
 
119
- def _cb_bodydone(self, result, request, url: str):
120
- r = super()._cb_bodydone(result, request, url)
168
+ def _cb_bodydone(self, result, *args):
169
+ # Scrapy 2.15+ changed signature from (result, request, url) to (result, url)
170
+ r = super()._cb_bodydone(result, *args)
121
171
  if isinstance(r, Response):
122
172
  if self._agent and hasattr(self._agent, '_endpoint'):
123
173
  proxy_response_headers = getattr(self._agent._endpoint, '_proxy_response_headers', None)
124
174
  if proxy_response_headers:
125
175
  r.headers.update(proxy_response_headers)
126
- # save this for download handler
127
- r._proxy_response_headers = proxy_response_headers
128
- return r
176
+ self.proxy_response_headers = proxy_response_headers
177
+ return r
@@ -0,0 +1,47 @@
1
+ from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
2
+ from scrapy.utils.defer import maybe_deferred_to_future
3
+ from scrapy_proxy_headers.agent import ScrapyProxyHeadersAgent
4
+
5
+
6
+ class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
7
+ def __init__(self, *args, **kwargs):
8
+ super().__init__(*args, **kwargs)
9
+ self._proxy_headers_by_proxy = {}
10
+
11
+ async def download_request(self, request, spider=None):
12
+ """Download the request and merge CONNECT proxy headers into the response."""
13
+ # Support both old Scrapy (spider param) and new Scrapy (self._crawler.spider)
14
+ if spider is None:
15
+ spider = self._crawler.spider
16
+
17
+ agent_kwargs = {
18
+ "contextFactory": self._contextFactory,
19
+ "pool": self._pool,
20
+ "maxsize": getattr(spider, "download_maxsize", self._default_maxsize),
21
+ "warnsize": getattr(spider, "download_warnsize", self._default_warnsize),
22
+ "fail_on_dataloss": self._fail_on_dataloss,
23
+ "crawler": self._crawler,
24
+ }
25
+ bind_address = getattr(self, "_bind_address", None)
26
+ if bind_address is not None:
27
+ agent_kwargs["bindAddress"] = bind_address
28
+
29
+ agent = ScrapyProxyHeadersAgent(**agent_kwargs)
30
+ deferred = agent.download_request(request)
31
+ proxy = request.meta.get("proxy")
32
+
33
+ if proxy:
34
+ # Proxy tunnels can get re-used; when that happens, proxy headers
35
+ # are not available in subsequent responses. Save proxy headers by
36
+ # proxy URL from the first tunnel response to add to later responses.
37
+ def callback(response):
38
+ if agent.proxy_response_headers:
39
+ self._proxy_headers_by_proxy[proxy] = agent.proxy_response_headers
40
+
41
+ if proxy in self._proxy_headers_by_proxy:
42
+ response.headers.update(self._proxy_headers_by_proxy[proxy])
43
+
44
+ return response
45
+
46
+ deferred.addCallback(callback)
47
+ return await maybe_deferred_to_future(deferred)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapy-proxy-headers
3
- Version: 0.1.6
3
+ Version: 0.2.1
4
4
  Summary: Add custom proxy headers to HTTPS requests in Scrapy
5
5
  Author-email: ProxyMesh <support@proxymesh.com>
6
6
  Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
@@ -17,6 +17,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
17
17
  Requires-Python: >=3.8
18
18
  Description-Content-Type: text/markdown
19
19
  License-File: LICENSE
20
+ Requires-Dist: scrapy>=2.14.2
20
21
  Dynamic: license-file
21
22
 
22
23
  # Scrapy Proxy Headers
@@ -200,7 +201,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
200
201
  ## Requirements
201
202
 
202
203
  - Python 3.8+
203
- - Scrapy 2.0+
204
+ - Scrapy 2.14.2+
204
205
 
205
206
  ## License
206
207
 
@@ -7,4 +7,5 @@ scrapy_proxy_headers/download_handler.py
7
7
  scrapy_proxy_headers.egg-info/PKG-INFO
8
8
  scrapy_proxy_headers.egg-info/SOURCES.txt
9
9
  scrapy_proxy_headers.egg-info/dependency_links.txt
10
+ scrapy_proxy_headers.egg-info/requires.txt
10
11
  scrapy_proxy_headers.egg-info/top_level.txt
@@ -1,37 +0,0 @@
1
- from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
2
- from scrapy_proxy_headers.agent import ScrapyProxyHeadersAgent
3
-
4
- class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
5
- def __init__(self, *args, **kwargs):
6
- super().__init__(*args, **kwargs)
7
- self._proxy_headers_by_proxy = {}
8
-
9
- def download_request(self, request, spider):
10
- """Return a deferred for the HTTP download"""
11
- agent = ScrapyProxyHeadersAgent(
12
- contextFactory=self._contextFactory,
13
- pool=self._pool,
14
- maxsize=getattr(spider, "download_maxsize", self._default_maxsize),
15
- warnsize=getattr(spider, "download_warnsize", self._default_warnsize),
16
- fail_on_dataloss=self._fail_on_dataloss,
17
- crawler=self._crawler,
18
- )
19
- response = agent.download_request(request)
20
- proxy = request.meta.get("proxy")
21
-
22
- if proxy:
23
- # we need to do all this because the proxy tunnels can get re-used
24
- # when that happens, the proxy headers are not available in subsequent responses
25
- # so we need to save the proxy headers by the proxy, from the first tunnel response
26
- # so we can add them to subsequent responses
27
- def callback(response):
28
- if hasattr(response, '_proxy_response_headers'):
29
- self._proxy_headers_by_proxy[proxy] = response._proxy_response_headers
30
-
31
- if proxy in self._proxy_headers_by_proxy:
32
- response.headers.update(self._proxy_headers_by_proxy[proxy])
33
-
34
- return response
35
-
36
- response.addCallback(callback)
37
- return response