scrapy-proxy-headers 0.1.5__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,11 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapy-proxy-headers
3
- Version: 0.1.5
3
+ Version: 0.2.0
4
4
  Summary: Add custom proxy headers to HTTPS requests in Scrapy
5
5
  Author-email: ProxyMesh <support@proxymesh.com>
6
6
  Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
7
7
  Project-URL: Changelog, https://github.com/proxymesh/scrapy-proxy-headers/commits/main/
8
8
  Project-URL: Issues, https://github.com/proxymesh/scrapy-proxy-headers/issues
9
+ Project-URL: Documentation, https://scrapy-proxy-headers.readthedocs.io/en/latest/
10
+ Project-URL: ProxyMesh, https://proxymesh.com
9
11
  Classifier: Programming Language :: Python :: 3
10
12
  Classifier: Operating System :: OS Independent
11
13
  Classifier: License :: OSI Approved :: BSD License
@@ -15,6 +17,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
15
17
  Requires-Python: >=3.8
16
18
  Description-Content-Type: text/markdown
17
19
  License-File: LICENSE
20
+ Requires-Dist: scrapy>=2.14.1
18
21
  Dynamic: license-file
19
22
 
20
23
  # Scrapy Proxy Headers
@@ -4,13 +4,16 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "scrapy-proxy-headers"
7
- version = "0.1.5"
7
+ version = "0.2.0"
8
8
  authors = [
9
9
  { name="ProxyMesh", email="support@proxymesh.com" },
10
10
  ]
11
11
  description = "Add custom proxy headers to HTTPS requests in Scrapy"
12
12
  readme = "README.md"
13
13
  requires-python = ">=3.8"
14
+ dependencies = [
15
+ "scrapy>=2.14.1",
16
+ ]
14
17
  classifiers = [
15
18
  "Programming Language :: Python :: 3",
16
19
  "Operating System :: OS Independent",
@@ -24,3 +27,5 @@ classifiers = [
24
27
  Homepage = "https://github.com/proxymesh/scrapy-proxy-headers"
25
28
  Changelog = "https://github.com/proxymesh/scrapy-proxy-headers/commits/main/"
26
29
  Issues = "https://github.com/proxymesh/scrapy-proxy-headers/issues"
30
+ Documentation = "https://scrapy-proxy-headers.readthedocs.io/en/latest/"
31
+ ProxyMesh = "https://proxymesh.com"
@@ -103,6 +103,7 @@ class ScrapyProxyHeadersAgent(ScrapyAgent):
103
103
  def __init__(self, *args, **kwargs):
104
104
  super().__init__(*args, **kwargs)
105
105
  self._agent = None
106
+ self.proxy_response_headers = None
106
107
 
107
108
  def _get_agent(self, request, timeout: float):
108
109
  self._agent = super()._get_agent(request, timeout)
@@ -116,13 +117,13 @@ class ScrapyProxyHeadersAgent(ScrapyAgent):
116
117
 
117
118
  return self._agent
118
119
 
119
- def _cb_bodydone(self, result, request, url: str):
120
- r = super()._cb_bodydone(result, request, url)
120
+ def _cb_bodydone(self, result, *args):
121
+ # Scrapy 2.15+ changed signature from (result, request, url) to (result, url)
122
+ r = super()._cb_bodydone(result, *args)
121
123
  if isinstance(r, Response):
122
124
  if self._agent and hasattr(self._agent, '_endpoint'):
123
125
  proxy_response_headers = getattr(self._agent._endpoint, '_proxy_response_headers', None)
124
126
  if proxy_response_headers:
125
127
  r.headers.update(proxy_response_headers)
126
- # save this for download handler
127
- r._proxy_response_headers = proxy_response_headers
128
+ self.proxy_response_headers = proxy_response_headers
128
129
  return r
@@ -1,13 +1,18 @@
1
1
  from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
2
2
  from scrapy_proxy_headers.agent import ScrapyProxyHeadersAgent
3
3
 
4
+
4
5
  class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
5
6
  def __init__(self, *args, **kwargs):
6
7
  super().__init__(*args, **kwargs)
7
8
  self._proxy_headers_by_proxy = {}
8
9
 
9
- def download_request(self, request, spider):
10
+ def download_request(self, request, spider=None):
10
11
  """Return a deferred for the HTTP download"""
12
+ # Support both old Scrapy (spider param) and new Scrapy (self._crawler.spider)
13
+ if spider is None:
14
+ spider = self._crawler.spider
15
+
11
16
  agent = ScrapyProxyHeadersAgent(
12
17
  contextFactory=self._contextFactory,
13
18
  pool=self._pool,
@@ -20,13 +25,12 @@ class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
20
25
  proxy = request.meta.get("proxy")
21
26
 
22
27
  if proxy:
23
- # we need to do all this because the proxy tunnels can get re-used
24
- # when that happens, the proxy headers are not available in subsequent responses
25
- # so we need to save the proxy headers by the proxy, from the first tunnel response
26
- # so we can add them to subsequent responses
28
+ # Proxy tunnels can get re-used; when that happens, proxy headers
29
+ # are not available in subsequent responses. Save proxy headers by
30
+ # proxy URL from the first tunnel response to add to later responses.
27
31
  def callback(response):
28
- if hasattr(response, '_proxy_response_headers'):
29
- self._proxy_headers_by_proxy[proxy] = response._proxy_response_headers
32
+ if agent.proxy_response_headers:
33
+ self._proxy_headers_by_proxy[proxy] = agent.proxy_response_headers
30
34
 
31
35
  if proxy in self._proxy_headers_by_proxy:
32
36
  response.headers.update(self._proxy_headers_by_proxy[proxy])
@@ -1,11 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scrapy-proxy-headers
3
- Version: 0.1.5
3
+ Version: 0.2.0
4
4
  Summary: Add custom proxy headers to HTTPS requests in Scrapy
5
5
  Author-email: ProxyMesh <support@proxymesh.com>
6
6
  Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
7
7
  Project-URL: Changelog, https://github.com/proxymesh/scrapy-proxy-headers/commits/main/
8
8
  Project-URL: Issues, https://github.com/proxymesh/scrapy-proxy-headers/issues
9
+ Project-URL: Documentation, https://scrapy-proxy-headers.readthedocs.io/en/latest/
10
+ Project-URL: ProxyMesh, https://proxymesh.com
9
11
  Classifier: Programming Language :: Python :: 3
10
12
  Classifier: Operating System :: OS Independent
11
13
  Classifier: License :: OSI Approved :: BSD License
@@ -15,6 +17,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
15
17
  Requires-Python: >=3.8
16
18
  Description-Content-Type: text/markdown
17
19
  License-File: LICENSE
20
+ Requires-Dist: scrapy>=2.14.1
18
21
  Dynamic: license-file
19
22
 
20
23
  # Scrapy Proxy Headers
@@ -7,4 +7,5 @@ scrapy_proxy_headers/download_handler.py
7
7
  scrapy_proxy_headers.egg-info/PKG-INFO
8
8
  scrapy_proxy_headers.egg-info/SOURCES.txt
9
9
  scrapy_proxy_headers.egg-info/dependency_links.txt
10
+ scrapy_proxy_headers.egg-info/requires.txt
10
11
  scrapy_proxy_headers.egg-info/top_level.txt