scrapy-proxy-headers 0.1.6__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scrapy_proxy_headers-0.1.6/scrapy_proxy_headers.egg-info → scrapy_proxy_headers-0.2.1}/PKG-INFO +3 -2
- {scrapy_proxy_headers-0.1.6 → scrapy_proxy_headers-0.2.1}/README.md +1 -1
- {scrapy_proxy_headers-0.1.6 → scrapy_proxy_headers-0.2.1}/pyproject.toml +4 -1
- {scrapy_proxy_headers-0.1.6 → scrapy_proxy_headers-0.2.1}/scrapy_proxy_headers/agent.py +70 -21
- scrapy_proxy_headers-0.2.1/scrapy_proxy_headers/download_handler.py +47 -0
- {scrapy_proxy_headers-0.1.6 → scrapy_proxy_headers-0.2.1/scrapy_proxy_headers.egg-info}/PKG-INFO +3 -2
- {scrapy_proxy_headers-0.1.6 → scrapy_proxy_headers-0.2.1}/scrapy_proxy_headers.egg-info/SOURCES.txt +1 -0
- scrapy_proxy_headers-0.2.1/scrapy_proxy_headers.egg-info/requires.txt +1 -0
- scrapy_proxy_headers-0.1.6/scrapy_proxy_headers/download_handler.py +0 -37
- {scrapy_proxy_headers-0.1.6 → scrapy_proxy_headers-0.2.1}/LICENSE +0 -0
- {scrapy_proxy_headers-0.1.6 → scrapy_proxy_headers-0.2.1}/scrapy_proxy_headers/__init__.py +0 -0
- {scrapy_proxy_headers-0.1.6 → scrapy_proxy_headers-0.2.1}/scrapy_proxy_headers.egg-info/dependency_links.txt +0 -0
- {scrapy_proxy_headers-0.1.6 → scrapy_proxy_headers-0.2.1}/scrapy_proxy_headers.egg-info/top_level.txt +0 -0
- {scrapy_proxy_headers-0.1.6 → scrapy_proxy_headers-0.2.1}/setup.cfg +0 -0
{scrapy_proxy_headers-0.1.6/scrapy_proxy_headers.egg-info → scrapy_proxy_headers-0.2.1}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scrapy-proxy-headers
|
|
3
|
-
Version: 0.1
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Add custom proxy headers to HTTPS requests in Scrapy
|
|
5
5
|
Author-email: ProxyMesh <support@proxymesh.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
|
|
@@ -17,6 +17,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
|
17
17
|
Requires-Python: >=3.8
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: LICENSE
|
|
20
|
+
Requires-Dist: scrapy>=2.14.2
|
|
20
21
|
Dynamic: license-file
|
|
21
22
|
|
|
22
23
|
# Scrapy Proxy Headers
|
|
@@ -200,7 +201,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
|
|
|
200
201
|
## Requirements
|
|
201
202
|
|
|
202
203
|
- Python 3.8+
|
|
203
|
-
- Scrapy 2.
|
|
204
|
+
- Scrapy 2.14.2+
|
|
204
205
|
|
|
205
206
|
## License
|
|
206
207
|
|
|
@@ -4,13 +4,16 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scrapy-proxy-headers"
|
|
7
|
-
version = "0.1
|
|
7
|
+
version = "0.2.1"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name="ProxyMesh", email="support@proxymesh.com" },
|
|
10
10
|
]
|
|
11
11
|
description = "Add custom proxy headers to HTTPS requests in Scrapy"
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
requires-python = ">=3.8"
|
|
14
|
+
dependencies = [
|
|
15
|
+
"scrapy>=2.14.2",
|
|
16
|
+
]
|
|
14
17
|
classifiers = [
|
|
15
18
|
"Programming Language :: Python :: 3",
|
|
16
19
|
"Operating System :: OS Independent",
|
|
@@ -1,7 +1,20 @@
|
|
|
1
1
|
from urllib.parse import urlparse
|
|
2
|
-
|
|
3
|
-
from scrapy.
|
|
2
|
+
|
|
3
|
+
from scrapy.core.downloader.handlers import http11 as _http11
|
|
4
4
|
from scrapy.http import Headers, Response
|
|
5
|
+
from scrapy.utils.httpobj import urlparse_cached
|
|
6
|
+
from scrapy.utils.python import to_bytes
|
|
7
|
+
from scrapy.utils.url import add_http_if_no_scheme
|
|
8
|
+
|
|
9
|
+
# Scrapy 2.18 made these HTTP11 internals private (leading underscore).
|
|
10
|
+
TunnelingTCP4ClientEndpoint = getattr(
|
|
11
|
+
_http11, "TunnelingTCP4ClientEndpoint", None
|
|
12
|
+
) or getattr(_http11, "_TunnelingTCP4ClientEndpoint")
|
|
13
|
+
TunnelingAgent = getattr(_http11, "TunnelingAgent", None) or getattr(
|
|
14
|
+
_http11, "_TunnelingAgent"
|
|
15
|
+
)
|
|
16
|
+
ScrapyAgent = getattr(_http11, "ScrapyAgent", None) or getattr(_http11, "_ScrapyAgent")
|
|
17
|
+
|
|
5
18
|
|
|
6
19
|
def tunnel_request_data_with_headers(host: str, port: int, **proxy_headers) -> bytes:
|
|
7
20
|
r"""
|
|
@@ -18,13 +31,14 @@ def tunnel_request_data_with_headers(host: str, port: int, **proxy_headers) -> b
|
|
|
18
31
|
host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port))
|
|
19
32
|
tunnel_req = b"CONNECT " + host_value + b" HTTP/1.1\r\n"
|
|
20
33
|
tunnel_req += b"Host: " + host_value + b"\r\n"
|
|
21
|
-
|
|
34
|
+
|
|
22
35
|
for key, val in proxy_headers.items():
|
|
23
36
|
tunnel_req += to_bytes(key) + b": " + to_bytes(val) + b"\r\n"
|
|
24
|
-
|
|
37
|
+
|
|
25
38
|
tunnel_req += b"\r\n"
|
|
26
39
|
return tunnel_req
|
|
27
40
|
|
|
41
|
+
|
|
28
42
|
class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
|
|
29
43
|
def __init__(
|
|
30
44
|
self,
|
|
@@ -34,7 +48,7 @@ class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
|
|
|
34
48
|
proxyConf,
|
|
35
49
|
contextFactory,
|
|
36
50
|
timeout: float = 30,
|
|
37
|
-
bindAddress
|
|
51
|
+
bindAddress=None,
|
|
38
52
|
**proxy_headers
|
|
39
53
|
):
|
|
40
54
|
super().__init__(reactor, host, port, proxyConf, contextFactory, timeout, bindAddress)
|
|
@@ -43,7 +57,7 @@ class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
|
|
|
43
57
|
if self._proxyAuthHeader:
|
|
44
58
|
self._proxy_headers['Proxy-Authorization'] = self._proxyAuthHeader
|
|
45
59
|
self._proxy_headers.update(proxy_headers)
|
|
46
|
-
|
|
60
|
+
|
|
47
61
|
def requestTunnel(self, protocol):
|
|
48
62
|
"""Asks the proxy to open a tunnel."""
|
|
49
63
|
assert protocol.transport
|
|
@@ -55,7 +69,7 @@ class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
|
|
|
55
69
|
protocol.dataReceived = self.processProxyResponse # type: ignore[method-assign]
|
|
56
70
|
self._protocol = protocol
|
|
57
71
|
return protocol
|
|
58
|
-
|
|
72
|
+
|
|
59
73
|
def processProxyResponse(self, data: bytes):
|
|
60
74
|
# data might have proxy headers, looks like
|
|
61
75
|
# b'HTTP/1.1 200 Connection established\r\nProxy-Header: VALUE\r\n\r\n'
|
|
@@ -69,6 +83,7 @@ class TunnelingHeadersTCP4ClientEndpoint(TunnelingTCP4ClientEndpoint):
|
|
|
69
83
|
self._proxy_response_headers = Headers(response_headers)
|
|
70
84
|
return super(TunnelingHeadersTCP4ClientEndpoint, self).processProxyResponse(data)
|
|
71
85
|
|
|
86
|
+
|
|
72
87
|
class TunnelingHeadersAgent(TunnelingAgent):
|
|
73
88
|
"""An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
|
|
74
89
|
downloads. It may look strange that we have chosen to subclass Agent and not
|
|
@@ -79,7 +94,7 @@ class TunnelingHeadersAgent(TunnelingAgent):
|
|
|
79
94
|
def __init__(self, *args, **kwargs):
|
|
80
95
|
super().__init__(*args, **kwargs)
|
|
81
96
|
self._proxy_headers = {}
|
|
82
|
-
|
|
97
|
+
|
|
83
98
|
def set_proxy_headers(self, proxy_headers):
|
|
84
99
|
self._proxy_headers = proxy_headers
|
|
85
100
|
|
|
@@ -97,32 +112,66 @@ class TunnelingHeadersAgent(TunnelingAgent):
|
|
|
97
112
|
)
|
|
98
113
|
return self._endpoint
|
|
99
114
|
|
|
115
|
+
|
|
100
116
|
class ScrapyProxyHeadersAgent(ScrapyAgent):
|
|
101
117
|
_TunnelingAgent = TunnelingHeadersAgent
|
|
102
118
|
|
|
103
119
|
def __init__(self, *args, **kwargs):
|
|
104
120
|
super().__init__(*args, **kwargs)
|
|
105
121
|
self._agent = None
|
|
106
|
-
|
|
107
|
-
def _get_agent(self, request, timeout: float):
|
|
108
|
-
self._agent = super()._get_agent(request, timeout)
|
|
122
|
+
self.proxy_response_headers = None
|
|
109
123
|
|
|
124
|
+
def _get_agent(self, request, timeout: float):
|
|
125
|
+
# Scrapy 2.18+ instantiates _TunnelingAgent directly instead of
|
|
126
|
+
# self._TunnelingAgent, so always build our subclass for HTTPS proxies.
|
|
110
127
|
proxy = request.meta.get("proxy")
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
128
|
+
if proxy and urlparse_cached(request).scheme == "https":
|
|
129
|
+
from twisted.internet import reactor
|
|
130
|
+
|
|
131
|
+
proxy = add_http_if_no_scheme(proxy)
|
|
132
|
+
proxy_parsed = urlparse(proxy)
|
|
133
|
+
proxy_host = proxy_parsed.hostname
|
|
134
|
+
proxy_port = proxy_parsed.port
|
|
135
|
+
if not proxy_port:
|
|
136
|
+
proxy_port = 443 if proxy_parsed.scheme == "https" else 80
|
|
137
|
+
if proxy_parsed.scheme == "https":
|
|
138
|
+
raise NotImplementedError(
|
|
139
|
+
"HTTPS proxies for HTTPS destinations are not supported"
|
|
140
|
+
)
|
|
141
|
+
bindaddress = request.meta.get("bindaddress") or self._bindAddress
|
|
142
|
+
try:
|
|
143
|
+
from scrapy.utils._download_handlers import normalize_bind_address
|
|
144
|
+
bindaddress = normalize_bind_address(bindaddress)
|
|
145
|
+
except ImportError:
|
|
146
|
+
pass
|
|
147
|
+
proxyAuth = request.headers.get(b"Proxy-Authorization", None)
|
|
148
|
+
proxyConf = (proxy_host, proxy_port, proxyAuth)
|
|
149
|
+
self._agent = TunnelingHeadersAgent(
|
|
150
|
+
reactor=reactor,
|
|
151
|
+
proxyConf=proxyConf,
|
|
152
|
+
contextFactory=self._contextFactory,
|
|
153
|
+
connectTimeout=timeout,
|
|
154
|
+
bindAddress=bindaddress,
|
|
155
|
+
pool=self._pool,
|
|
156
|
+
)
|
|
157
|
+
proxy_headers = request.meta.get("proxy_headers")
|
|
158
|
+
if proxy_headers:
|
|
115
159
|
self._agent.set_proxy_headers(proxy_headers)
|
|
116
|
-
|
|
160
|
+
return self._agent
|
|
161
|
+
|
|
162
|
+
self._agent = super()._get_agent(request, timeout)
|
|
163
|
+
proxy_headers = request.meta.get("proxy_headers")
|
|
164
|
+
if proxy and proxy_headers and hasattr(self._agent, "set_proxy_headers"):
|
|
165
|
+
self._agent.set_proxy_headers(proxy_headers)
|
|
117
166
|
return self._agent
|
|
118
167
|
|
|
119
|
-
def _cb_bodydone(self, result,
|
|
120
|
-
|
|
168
|
+
def _cb_bodydone(self, result, *args):
|
|
169
|
+
# Scrapy 2.15+ changed signature from (result, request, url) to (result, url)
|
|
170
|
+
r = super()._cb_bodydone(result, *args)
|
|
121
171
|
if isinstance(r, Response):
|
|
122
172
|
if self._agent and hasattr(self._agent, '_endpoint'):
|
|
123
173
|
proxy_response_headers = getattr(self._agent._endpoint, '_proxy_response_headers', None)
|
|
124
174
|
if proxy_response_headers:
|
|
125
175
|
r.headers.update(proxy_response_headers)
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
return r
|
|
176
|
+
self.proxy_response_headers = proxy_response_headers
|
|
177
|
+
return r
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
|
|
2
|
+
from scrapy.utils.defer import maybe_deferred_to_future
|
|
3
|
+
from scrapy_proxy_headers.agent import ScrapyProxyHeadersAgent
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
|
|
7
|
+
def __init__(self, *args, **kwargs):
|
|
8
|
+
super().__init__(*args, **kwargs)
|
|
9
|
+
self._proxy_headers_by_proxy = {}
|
|
10
|
+
|
|
11
|
+
async def download_request(self, request, spider=None):
|
|
12
|
+
"""Download the request and merge CONNECT proxy headers into the response."""
|
|
13
|
+
# Support both old Scrapy (spider param) and new Scrapy (self._crawler.spider)
|
|
14
|
+
if spider is None:
|
|
15
|
+
spider = self._crawler.spider
|
|
16
|
+
|
|
17
|
+
agent_kwargs = {
|
|
18
|
+
"contextFactory": self._contextFactory,
|
|
19
|
+
"pool": self._pool,
|
|
20
|
+
"maxsize": getattr(spider, "download_maxsize", self._default_maxsize),
|
|
21
|
+
"warnsize": getattr(spider, "download_warnsize", self._default_warnsize),
|
|
22
|
+
"fail_on_dataloss": self._fail_on_dataloss,
|
|
23
|
+
"crawler": self._crawler,
|
|
24
|
+
}
|
|
25
|
+
bind_address = getattr(self, "_bind_address", None)
|
|
26
|
+
if bind_address is not None:
|
|
27
|
+
agent_kwargs["bindAddress"] = bind_address
|
|
28
|
+
|
|
29
|
+
agent = ScrapyProxyHeadersAgent(**agent_kwargs)
|
|
30
|
+
deferred = agent.download_request(request)
|
|
31
|
+
proxy = request.meta.get("proxy")
|
|
32
|
+
|
|
33
|
+
if proxy:
|
|
34
|
+
# Proxy tunnels can get re-used; when that happens, proxy headers
|
|
35
|
+
# are not available in subsequent responses. Save proxy headers by
|
|
36
|
+
# proxy URL from the first tunnel response to add to later responses.
|
|
37
|
+
def callback(response):
|
|
38
|
+
if agent.proxy_response_headers:
|
|
39
|
+
self._proxy_headers_by_proxy[proxy] = agent.proxy_response_headers
|
|
40
|
+
|
|
41
|
+
if proxy in self._proxy_headers_by_proxy:
|
|
42
|
+
response.headers.update(self._proxy_headers_by_proxy[proxy])
|
|
43
|
+
|
|
44
|
+
return response
|
|
45
|
+
|
|
46
|
+
deferred.addCallback(callback)
|
|
47
|
+
return await maybe_deferred_to_future(deferred)
|
{scrapy_proxy_headers-0.1.6 → scrapy_proxy_headers-0.2.1/scrapy_proxy_headers.egg-info}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scrapy-proxy-headers
|
|
3
|
-
Version: 0.1
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Add custom proxy headers to HTTPS requests in Scrapy
|
|
5
5
|
Author-email: ProxyMesh <support@proxymesh.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
|
|
@@ -17,6 +17,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
|
17
17
|
Requires-Python: >=3.8
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: LICENSE
|
|
20
|
+
Requires-Dist: scrapy>=2.14.2
|
|
20
21
|
Dynamic: license-file
|
|
21
22
|
|
|
22
23
|
# Scrapy Proxy Headers
|
|
@@ -200,7 +201,7 @@ Full documentation is available at [scrapy-proxy-headers.readthedocs.io](https:/
|
|
|
200
201
|
## Requirements
|
|
201
202
|
|
|
202
203
|
- Python 3.8+
|
|
203
|
-
- Scrapy 2.
|
|
204
|
+
- Scrapy 2.14.2+
|
|
204
205
|
|
|
205
206
|
## License
|
|
206
207
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
scrapy>=2.14.2
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
|
|
2
|
-
from scrapy_proxy_headers.agent import ScrapyProxyHeadersAgent
|
|
3
|
-
|
|
4
|
-
class HTTP11ProxyDownloadHandler(HTTP11DownloadHandler):
|
|
5
|
-
def __init__(self, *args, **kwargs):
|
|
6
|
-
super().__init__(*args, **kwargs)
|
|
7
|
-
self._proxy_headers_by_proxy = {}
|
|
8
|
-
|
|
9
|
-
def download_request(self, request, spider):
|
|
10
|
-
"""Return a deferred for the HTTP download"""
|
|
11
|
-
agent = ScrapyProxyHeadersAgent(
|
|
12
|
-
contextFactory=self._contextFactory,
|
|
13
|
-
pool=self._pool,
|
|
14
|
-
maxsize=getattr(spider, "download_maxsize", self._default_maxsize),
|
|
15
|
-
warnsize=getattr(spider, "download_warnsize", self._default_warnsize),
|
|
16
|
-
fail_on_dataloss=self._fail_on_dataloss,
|
|
17
|
-
crawler=self._crawler,
|
|
18
|
-
)
|
|
19
|
-
response = agent.download_request(request)
|
|
20
|
-
proxy = request.meta.get("proxy")
|
|
21
|
-
|
|
22
|
-
if proxy:
|
|
23
|
-
# we need to do all this because the proxy tunnels can get re-used
|
|
24
|
-
# when that happens, the proxy headers are not available in subsequent responses
|
|
25
|
-
# so we need to save the proxy headers by the proxy, from the first tunnel response
|
|
26
|
-
# so we can add them to subsequent responses
|
|
27
|
-
def callback(response):
|
|
28
|
-
if hasattr(response, '_proxy_response_headers'):
|
|
29
|
-
self._proxy_headers_by_proxy[proxy] = response._proxy_response_headers
|
|
30
|
-
|
|
31
|
-
if proxy in self._proxy_headers_by_proxy:
|
|
32
|
-
response.headers.update(self._proxy_headers_by_proxy[proxy])
|
|
33
|
-
|
|
34
|
-
return response
|
|
35
|
-
|
|
36
|
-
response.addCallback(callback)
|
|
37
|
-
return response
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|