scrapy-proxy-headers 0.1.0__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapy_proxy_headers-0.1.2/PKG-INFO +41 -0
- {scrapy_proxy_headers-0.1.0 → scrapy_proxy_headers-0.1.2}/README.md +1 -1
- scrapy_proxy_headers-0.1.2/pyproject.toml +25 -0
- scrapy_proxy_headers-0.1.2/scrapy_proxy_headers/__init__.py +1 -0
- {scrapy_proxy_headers-0.1.0 → scrapy_proxy_headers-0.1.2}/scrapy_proxy_headers/agent.py +0 -2
- scrapy_proxy_headers-0.1.2/scrapy_proxy_headers.egg-info/PKG-INFO +41 -0
- {scrapy_proxy_headers-0.1.0 → scrapy_proxy_headers-0.1.2}/scrapy_proxy_headers.egg-info/SOURCES.txt +1 -3
- scrapy_proxy_headers-0.1.0/PKG-INFO +0 -6
- scrapy_proxy_headers-0.1.0/scrapy_proxy_headers/__init__.py +0 -13
- scrapy_proxy_headers-0.1.0/scrapy_proxy_headers.egg-info/PKG-INFO +0 -6
- scrapy_proxy_headers-0.1.0/scrapy_proxy_headers.egg-info/entry_points.txt +0 -2
- scrapy_proxy_headers-0.1.0/scrapy_proxy_headers.egg-info/requires.txt +0 -1
- scrapy_proxy_headers-0.1.0/setup.py +0 -20
- {scrapy_proxy_headers-0.1.0 → scrapy_proxy_headers-0.1.2}/LICENSE +0 -0
- {scrapy_proxy_headers-0.1.0 → scrapy_proxy_headers-0.1.2}/scrapy_proxy_headers/download_handler.py +0 -0
- {scrapy_proxy_headers-0.1.0 → scrapy_proxy_headers-0.1.2}/scrapy_proxy_headers.egg-info/dependency_links.txt +0 -0
- {scrapy_proxy_headers-0.1.0 → scrapy_proxy_headers-0.1.2}/scrapy_proxy_headers.egg-info/top_level.txt +0 -0
- {scrapy_proxy_headers-0.1.0 → scrapy_proxy_headers-0.1.2}/setup.cfg +0 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: scrapy-proxy-headers
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Add custom proxy headers to HTTPS requests in Scrapy
|
|
5
|
+
Author-email: ProxyMesh <support@proxymesh.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
|
|
7
|
+
Project-URL: Changelog, https://github.com/proxymesh/scrapy-proxy-headers/commits/main/
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
13
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
14
|
+
Requires-Python: >=3.8
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
|
|
18
|
+
The `scrapy-proxy-headers` package is designed for adding proxy headers to HTTPS requests.
|
|
19
|
+
|
|
20
|
+
In normal usage, custom headers put in `request.headers` cannot be read by a proxy when you make a HTTPS request, because the headers are encrypted and passed through the proxy tunnel, along with the rest of the request body. You can read more about this at [Proxy Server Requests over HTTPS](https://docs.proxymesh.com/article/145-proxy-server-requests-over-https).
|
|
21
|
+
|
|
22
|
+
Because Scrapy does not have a good way to pass custom headers to a proxy when you make HTTPS requests, we at [ProxyMesh](https://proxymesh.com) made this extension to support our customers that use Scrapy and want to use custom headers to control our proxy behavior. But this extension can work for any custom headers through a proxy.
|
|
23
|
+
|
|
24
|
+
To use this extension, do the following:
|
|
25
|
+
|
|
26
|
+
1. `pip install scrapy_proxy_headers`
|
|
27
|
+
2. In your Scrapy `settings.py`, add the following code:
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
DOWNLOAD_HANDLERS = {
|
|
31
|
+
"https": "scrapy_proxy_headers.HTTP11ProxyDownloadHandler"
|
|
32
|
+
}
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
3. When you want make a request with a custom proxy header, instead of using `request.headers`, use `request.meta["proxy_headers"]` like this:
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
request.meta["proxy_headers"] = {"X-ProxyMesh-Country": "US"}
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Currently this package does not yet support reading custom response headers from the proxy, but that feature is coming soon.
|
|
@@ -2,7 +2,7 @@ The `scrapy-proxy-headers` package is designed for adding proxy headers to HTTPS
|
|
|
2
2
|
|
|
3
3
|
In normal usage, custom headers put in `request.headers` cannot be read by a proxy when you make a HTTPS request, because the headers are encrypted and passed through the proxy tunnel, along with the rest of the request body. You can read more about this at [Proxy Server Requests over HTTPS](https://docs.proxymesh.com/article/145-proxy-server-requests-over-https).
|
|
4
4
|
|
|
5
|
-
Because Scrapy does not have a good way to pass custom headers to a proxy when you make HTTPS requests, we at ProxyMesh made this extension to support our customers that use Scrapy and want to use custom headers to control our proxy behavior. But this extension can work for any custom headers through a proxy.
|
|
5
|
+
Because Scrapy does not have a good way to pass custom headers to a proxy when you make HTTPS requests, we at [ProxyMesh](https://proxymesh.com) made this extension to support our customers that use Scrapy and want to use custom headers to control our proxy behavior. But this extension can work for any custom headers through a proxy.
|
|
6
6
|
|
|
7
7
|
To use this extension, do the following:
|
|
8
8
|
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "scrapy-proxy-headers"
|
|
7
|
+
version = "0.1.2"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name="ProxyMesh", email="support@proxymesh.com" },
|
|
10
|
+
]
|
|
11
|
+
description = "Add custom proxy headers to HTTPS requests in Scrapy"
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
requires-python = ">=3.8"
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Operating System :: OS Independent",
|
|
17
|
+
"License :: OSI Approved :: BSD License",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Topic :: Internet :: WWW/HTTP",
|
|
20
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.urls]
|
|
24
|
+
Homepage = "https://github.com/proxymesh/scrapy-proxy-headers"
|
|
25
|
+
Changelog = "https://github.com/proxymesh/scrapy-proxy-headers/commits/main/"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .download_handler import HTTP11ProxyDownloadHandler
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: scrapy-proxy-headers
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Add custom proxy headers to HTTPS requests in Scrapy
|
|
5
|
+
Author-email: ProxyMesh <support@proxymesh.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/proxymesh/scrapy-proxy-headers
|
|
7
|
+
Project-URL: Changelog, https://github.com/proxymesh/scrapy-proxy-headers/commits/main/
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
13
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
14
|
+
Requires-Python: >=3.8
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
|
|
18
|
+
The `scrapy-proxy-headers` package is designed for adding proxy headers to HTTPS requests.
|
|
19
|
+
|
|
20
|
+
In normal usage, custom headers put in `request.headers` cannot be read by a proxy when you make a HTTPS request, because the headers are encrypted and passed through the proxy tunnel, along with the rest of the request body. You can read more about this at [Proxy Server Requests over HTTPS](https://docs.proxymesh.com/article/145-proxy-server-requests-over-https).
|
|
21
|
+
|
|
22
|
+
Because Scrapy does not have a good way to pass custom headers to a proxy when you make HTTPS requests, we at [ProxyMesh](https://proxymesh.com) made this extension to support our customers that use Scrapy and want to use custom headers to control our proxy behavior. But this extension can work for any custom headers through a proxy.
|
|
23
|
+
|
|
24
|
+
To use this extension, do the following:
|
|
25
|
+
|
|
26
|
+
1. `pip install scrapy_proxy_headers`
|
|
27
|
+
2. In your Scrapy `settings.py`, add the following code:
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
DOWNLOAD_HANDLERS = {
|
|
31
|
+
"https": "scrapy_proxy_headers.HTTP11ProxyDownloadHandler"
|
|
32
|
+
}
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
3. When you want make a request with a custom proxy header, instead of using `request.headers`, use `request.meta["proxy_headers"]` like this:
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
request.meta["proxy_headers"] = {"X-ProxyMesh-Country": "US"}
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Currently this package does not yet support reading custom response headers from the proxy, but that feature is coming soon.
|
{scrapy_proxy_headers-0.1.0 → scrapy_proxy_headers-0.1.2}/scrapy_proxy_headers.egg-info/SOURCES.txt
RENAMED
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
LICENSE
|
|
2
2
|
README.md
|
|
3
|
-
|
|
3
|
+
pyproject.toml
|
|
4
4
|
scrapy_proxy_headers/__init__.py
|
|
5
5
|
scrapy_proxy_headers/agent.py
|
|
6
6
|
scrapy_proxy_headers/download_handler.py
|
|
7
7
|
scrapy_proxy_headers.egg-info/PKG-INFO
|
|
8
8
|
scrapy_proxy_headers.egg-info/SOURCES.txt
|
|
9
9
|
scrapy_proxy_headers.egg-info/dependency_links.txt
|
|
10
|
-
scrapy_proxy_headers.egg-info/entry_points.txt
|
|
11
|
-
scrapy_proxy_headers.egg-info/requires.txt
|
|
12
10
|
scrapy_proxy_headers.egg-info/top_level.txt
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
To use this, in your settings, do the following:
|
|
3
|
-
|
|
4
|
-
DOWNLOAD_HANDLERS = {
|
|
5
|
-
"https": "scrapy_proxy_headers.HTTP11ProxyDownloadHandler"
|
|
6
|
-
}
|
|
7
|
-
|
|
8
|
-
Then when you make a request with a custom proxy header, instead of using request.headers, use request.meta["proxy_headers"] like this:
|
|
9
|
-
|
|
10
|
-
request.meta["proxy_headers"] = {"X-ProxyMesh-Country": "US"}
|
|
11
|
-
"""
|
|
12
|
-
|
|
13
|
-
from .download_handler import HTTP11ProxyDownloadHandler
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
scrapy>=2.0
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
from setuptools import setup, find_packages
|
|
2
|
-
|
|
3
|
-
setup(
|
|
4
|
-
name="scrapy_proxy_headers",
|
|
5
|
-
version="0.1.0",
|
|
6
|
-
packages=find_packages(),
|
|
7
|
-
install_requires=[
|
|
8
|
-
"scrapy>=2.0",
|
|
9
|
-
],
|
|
10
|
-
entry_points={
|
|
11
|
-
"scrapy.downloader_handlers": [
|
|
12
|
-
"https = scrapy_proxy_headers.HTTP11ProxyDownloadHandler"
|
|
13
|
-
],
|
|
14
|
-
},
|
|
15
|
-
classifiers=[
|
|
16
|
-
"Programming Language :: Python :: 3",
|
|
17
|
-
#"License :: OSI Approved :: MIT License",
|
|
18
|
-
"Operating System :: OS Independent",
|
|
19
|
-
],
|
|
20
|
-
)
|
|
File without changes
|
{scrapy_proxy_headers-0.1.0 → scrapy_proxy_headers-0.1.2}/scrapy_proxy_headers/download_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|