proxyproviders 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- proxyproviders-0.1.0/LICENSE +21 -0
- proxyproviders-0.1.0/PKG-INFO +224 -0
- proxyproviders-0.1.0/README.md +180 -0
- proxyproviders-0.1.0/proxyproviders/models/__init__.py +0 -0
- proxyproviders-0.1.0/proxyproviders/models/proxy.py +35 -0
- proxyproviders-0.1.0/proxyproviders/providers/brightdata.py +217 -0
- proxyproviders-0.1.0/proxyproviders/providers/webshare.py +124 -0
- proxyproviders-0.1.0/proxyproviders/proxyproviders.egg-info/PKG-INFO +224 -0
- proxyproviders-0.1.0/proxyproviders/proxyproviders.egg-info/SOURCES.txt +14 -0
- proxyproviders-0.1.0/proxyproviders/proxyproviders.egg-info/dependency_links.txt +1 -0
- proxyproviders-0.1.0/proxyproviders/proxyproviders.egg-info/requires.txt +10 -0
- proxyproviders-0.1.0/proxyproviders/proxyproviders.egg-info/top_level.txt +2 -0
- proxyproviders-0.1.0/pyproject.toml +41 -0
- proxyproviders-0.1.0/setup.cfg +4 -0
- proxyproviders-0.1.0/tests/test_exceptions.py +20 -0
- proxyproviders-0.1.0/tests/test_proxy_provider.py +93 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 David Teather
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: proxyproviders
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A unified interface for different proxy providers
|
|
5
|
+
Author-email: David Teather <contact.davidteather@gmail.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2025 David Teather
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Classifier: Development Status :: 3 - Alpha
|
|
29
|
+
Classifier: Intended Audience :: Developers
|
|
30
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
31
|
+
Classifier: Programming Language :: Python :: 3
|
|
32
|
+
Requires-Python: >=3.9
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
License-File: LICENSE
|
|
35
|
+
Requires-Dist: requests
|
|
36
|
+
Provides-Extra: dev
|
|
37
|
+
Requires-Dist: pytest; extra == "dev"
|
|
38
|
+
Requires-Dist: sphinx; extra == "dev"
|
|
39
|
+
Requires-Dist: black; extra == "dev"
|
|
40
|
+
Requires-Dist: mypy; extra == "dev"
|
|
41
|
+
Requires-Dist: coverage; extra == "dev"
|
|
42
|
+
Requires-Dist: types-requests; extra == "dev"
|
|
43
|
+
Requires-Dist: toml; extra == "dev"
|
|
44
|
+
|
|
45
|
+
# ProxyProviders
|
|
46
|
+
|
|
47
|
+
The Unified Python Proxy API for managing proxies with support for BrightData, WebShare, and more.
|
|
48
|
+
|
|
49
|
+
[](https://www.linkedin.com/in/davidteather/) [](https://github.com/sponsors/davidteather) [](https://github.com/davidteather/proxyproviders/releases) [](https://github.com/davidteather/proxyproviders/blob/main/LICENSE) [](https://pypi.org/project/proxyproviders/)  [](https://discord.gg/yyPhbfma6f)
|
|
50
|
+
|
|
51
|
+
## Table of Contents
|
|
52
|
+
|
|
53
|
+
- [Documentation](#documentation)
|
|
54
|
+
- [Getting Started](#getting-started)
|
|
55
|
+
- [How to Support The Project](#how-to-support-the-project)
|
|
56
|
+
- [Installing](#installing)
|
|
57
|
+
- [Quick Start Guide](#quick-start-guide)
|
|
58
|
+
- [Examples](https://github.com/davidteather/proxyproviders/tree/main/examples)
|
|
59
|
+
|
|
60
|
+
## Documentation
|
|
61
|
+
|
|
62
|
+
You can find the full documentation [here](https://davidteather.github.io/proxyproviders)
|
|
63
|
+
|
|
64
|
+
**Note:** If you want to learn how to web scrape websites check my [free and open-source course for learning everything web scraping](https://github.com/davidteather/everything-web-scraping)
|
|
65
|
+
|
|
66
|
+
### How to Support The Project
|
|
67
|
+
|
|
68
|
+
- Star the repo 😎
|
|
69
|
+
- Consider [sponsoring](https://github.com/sponsors/davidteather) me on GitHub
|
|
70
|
+
- Send me an email or a [LinkedIn](https://www.linkedin.com/in/davidteather/) message about what you're doing with it :D
|
|
71
|
+
- Submit PRs for issues or any new providers/features :)
|
|
72
|
+
|
|
73
|
+
## Getting Started
|
|
74
|
+
|
|
75
|
+
To get started using this package follow the instructions below.
|
|
76
|
+
|
|
77
|
+
### Installation
|
|
78
|
+
|
|
79
|
+
This package is installable via pip.
|
|
80
|
+
```sh
|
|
81
|
+
python -m pip install proxyproviders
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Quick Start Guide
|
|
85
|
+
|
|
86
|
+
Here's a quick bit of code to get you started! There's also a few [examples in the folder](https://github.com/davidteather/proxyproviders/tree/main/examples).
|
|
87
|
+
|
|
88
|
+
**Note:** If you want to learn how to web scrape websites check my [free and open-source course for learning everything web scraping](https://github.com/davidteather/everything-web-scraping)
|
|
89
|
+
|
|
90
|
+
### Choose a Proxy Provider
|
|
91
|
+
|
|
92
|
+
If you already haven't, choose which proxy provider to use. You can find a [list in the documentation](https://davidteather.github.io/proxyproviders/#proxyproviders-supported-providers). After choosing, look at the documentation around the specific provider you've choosen. Where needed, we've laid out steps to get api keys in the documentation. These steps will vary slightly by provider. For this example I'll be using the [Webshare Provider](https://davidteather.github.io/proxyproviders/#proxyproviders.providers.webshare.Webshare) because it's both easy to setup and they give you 10 free data center proxies to test out.
|
|
93
|
+
|
|
94
|
+
You can create an account on webshare [here](https://www.webshare.io/?referral_code=3x5812idzzzp) (affiliate link), then head into the [API tab](https://dashboard.webshare.io/userapi/keys) on the side and generate a new token. Keep this API token safe and don't post it publically.
|
|
95
|
+
|
|
96
|
+
### Basic Example
|
|
97
|
+
|
|
98
|
+
After you can list out your proxies with
|
|
99
|
+
```py
|
|
100
|
+
from proxyproviders import Webshare
|
|
101
|
+
|
|
102
|
+
proxy_provider = Webshare(api_key="your-api-key")
|
|
103
|
+
|
|
104
|
+
proxies = proxy_provider.list_proxies()
|
|
105
|
+
|
|
106
|
+
print(proxies)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Each provider has their own custom options, the `Webshare` class lets you specify url params according to their [api spec](https://apidocs.webshare.io/proxy-list/list#parameters), here's an example which will only return proxies that are based in the US.
|
|
110
|
+
|
|
111
|
+
```py
|
|
112
|
+
proxy_provider = Webshare(api_key="your-api-key", params={"country_code_in": "US"})
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### Using ProxyConfig
|
|
116
|
+
|
|
117
|
+
For any shared logic across all types of proxy providers, we use the `ProxyConfig` data class to configure them. The full docs for [ProxyConfig are here](https://davidteather.github.io/proxyproviders/#proxyproviders.proxy_provider.ProxyConfig). In this example we will configure it to use a shorter `refresh_interval` than default.
|
|
118
|
+
|
|
119
|
+
```py
|
|
120
|
+
from proxyproviders import Webshare, ProxyConfig
|
|
121
|
+
import time
|
|
122
|
+
|
|
123
|
+
config = ProxyConfig(refresh_interval=3)
|
|
124
|
+
ws = Webshare(api_key="your-api-token", config=config)
|
|
125
|
+
|
|
126
|
+
proxies = ws.list_proxies() # calls API
|
|
127
|
+
|
|
128
|
+
ws.list_proxies() # cached
|
|
129
|
+
|
|
130
|
+
time.sleep(5)
|
|
131
|
+
ws.list_proxies() # calls API since it's more than 3s later
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### Function Using Generic ProxyProvider
|
|
135
|
+
|
|
136
|
+
Since all proxy providers implement the same interface, we can make a function that allows us to easily swap out and utilize different providers. This is the main appeal of having a unified interface. It allows other modules to be provider agnostic, like my [TikTokAPI](https://github.com/davidteather/TikTok-Api) package.
|
|
137
|
+
|
|
138
|
+
```py
|
|
139
|
+
from proxyproviders import Webshare, BrightData, ProxyProvider, ProxyConfig
|
|
140
|
+
|
|
141
|
+
def some_function(provider: ProxyProvider):
|
|
142
|
+
proxies = provider.list_proxies()
|
|
143
|
+
print(proxies)
|
|
144
|
+
|
|
145
|
+
webshare = Webshare(api_key="your_api_key")
|
|
146
|
+
brightdata = BrightData(api_key="your_api_key", zone="my_zone")
|
|
147
|
+
|
|
148
|
+
some_function(webshare)
|
|
149
|
+
some_function(brightdata)
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Here's a more meaningful example that takes the `Proxy` class and uses it to create a python requests http proxy.
|
|
153
|
+
|
|
154
|
+
```py
|
|
155
|
+
from proxyproviders import Webshare, BrightData, ProxyProvider
|
|
156
|
+
import requests
|
|
157
|
+
import os
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def request_with_proxy(provider: ProxyProvider):
|
|
161
|
+
requests_proxy = None
|
|
162
|
+
|
|
163
|
+
if provider:
|
|
164
|
+
proxies = provider.list_proxies()
|
|
165
|
+
|
|
166
|
+
requests_proxy = {
|
|
167
|
+
"http": f"http://{proxies[0].username}:{proxies[0].password}@{proxies[0].proxy_address}:{proxies[0].port}",
|
|
168
|
+
"https": f"http://{proxies[0].username}:{proxies[0].password}@{proxies[0].proxy_address}:{proxies[0].port}",
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
r = requests.get("https://httpbin.org/ip", proxies=requests_proxy)
|
|
172
|
+
return r.json()
|
|
173
|
+
|
|
174
|
+
webshare = Webshare(api_key="your_api_key")
|
|
175
|
+
brightdata = BrightData(api_key="your_api_key", zone="your_zone")
|
|
176
|
+
|
|
177
|
+
print(f"Your IP: {request_with_proxy(None)}")
|
|
178
|
+
print(f"Webshare: {request_with_proxy(webshare)}")
|
|
179
|
+
print(f"BrightData: {request_with_proxy(brightdata)}")
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### Making Your Own Proxy Provider
|
|
183
|
+
|
|
184
|
+
Here's a skeleton of how you can make your very own `ProxyProvider` class. You'll need to implemenet all the required functions of the `ProxyProvider` which may be more than what's here at the time of writing.
|
|
185
|
+
|
|
186
|
+
If you do find yourself making one of these, consider contributing it back to the repository so everyone can use them :D
|
|
187
|
+
|
|
188
|
+
```py
|
|
189
|
+
from proxyproviders import ProxyProvider, ProxyConfig, Proxy
|
|
190
|
+
from typing import List, Optional
|
|
191
|
+
|
|
192
|
+
class MyProxyProvider(ProxyProvider):
|
|
193
|
+
def __init__(self, config: Optional[ProxyConfig] = None):
|
|
194
|
+
super().__init__(config=config)
|
|
195
|
+
|
|
196
|
+
def _fetch_proxies(self):
|
|
197
|
+
proxies: List[Proxy] = []
|
|
198
|
+
|
|
199
|
+
for i in range(10):
|
|
200
|
+
# TODO: your real proxy fetching logic
|
|
201
|
+
|
|
202
|
+
# There are required fields on the Proxy class, be sure that these are filled out properly
|
|
203
|
+
# especially if you're using it with another library.
|
|
204
|
+
proxy = Proxy(
|
|
205
|
+
id=str(i),
|
|
206
|
+
username="username",
|
|
207
|
+
password="password",
|
|
208
|
+
proxy_address="192.168.1.1",
|
|
209
|
+
port=80,
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
proxies.append(proxy)
|
|
213
|
+
|
|
214
|
+
return proxies
|
|
215
|
+
|
|
216
|
+
def some_function(provider: ProxyProvider):
|
|
217
|
+
proxies = provider.list_proxies()
|
|
218
|
+
for proxy in proxies:
|
|
219
|
+
print(proxy)
|
|
220
|
+
|
|
221
|
+
provider = MyProxyProvider()
|
|
222
|
+
some_function(provider) # calls the function with the provider
|
|
223
|
+
```
|
|
224
|
+
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# ProxyProviders
|
|
2
|
+
|
|
3
|
+
The Unified Python Proxy API for managing proxies with support for BrightData, WebShare, and more.
|
|
4
|
+
|
|
5
|
+
[](https://www.linkedin.com/in/davidteather/) [](https://github.com/sponsors/davidteather) [](https://github.com/davidteather/proxyproviders/releases) [](https://github.com/davidteather/proxyproviders/blob/main/LICENSE) [](https://pypi.org/project/proxyproviders/)  [](https://discord.gg/yyPhbfma6f)
|
|
6
|
+
|
|
7
|
+
## Table of Contents
|
|
8
|
+
|
|
9
|
+
- [Documentation](#documentation)
|
|
10
|
+
- [Getting Started](#getting-started)
|
|
11
|
+
- [How to Support The Project](#how-to-support-the-project)
|
|
12
|
+
- [Installing](#installing)
|
|
13
|
+
- [Quick Start Guide](#quick-start-guide)
|
|
14
|
+
- [Examples](https://github.com/davidteather/proxyproviders/tree/main/examples)
|
|
15
|
+
|
|
16
|
+
## Documentation
|
|
17
|
+
|
|
18
|
+
You can find the full documentation [here](https://davidteather.github.io/proxyproviders)
|
|
19
|
+
|
|
20
|
+
**Note:** If you want to learn how to web scrape websites check my [free and open-source course for learning everything web scraping](https://github.com/davidteather/everything-web-scraping)
|
|
21
|
+
|
|
22
|
+
### How to Support The Project
|
|
23
|
+
|
|
24
|
+
- Star the repo 😎
|
|
25
|
+
- Consider [sponsoring](https://github.com/sponsors/davidteather) me on GitHub
|
|
26
|
+
- Send me an email or a [LinkedIn](https://www.linkedin.com/in/davidteather/) message about what you're doing with it :D
|
|
27
|
+
- Submit PRs for issues or any new providers/features :)
|
|
28
|
+
|
|
29
|
+
## Getting Started
|
|
30
|
+
|
|
31
|
+
To get started using this package follow the instructions below.
|
|
32
|
+
|
|
33
|
+
### Installation
|
|
34
|
+
|
|
35
|
+
This package is installable via pip.
|
|
36
|
+
```sh
|
|
37
|
+
python -m pip install proxyproviders
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Quick Start Guide
|
|
41
|
+
|
|
42
|
+
Here's a quick bit of code to get you started! There's also a few [examples in the folder](https://github.com/davidteather/proxyproviders/tree/main/examples).
|
|
43
|
+
|
|
44
|
+
**Note:** If you want to learn how to web scrape websites check my [free and open-source course for learning everything web scraping](https://github.com/davidteather/everything-web-scraping)
|
|
45
|
+
|
|
46
|
+
### Choose a Proxy Provider
|
|
47
|
+
|
|
48
|
+
If you already haven't, choose which proxy provider to use. You can find a [list in the documentation](https://davidteather.github.io/proxyproviders/#proxyproviders-supported-providers). After choosing, look at the documentation around the specific provider you've choosen. Where needed, we've laid out steps to get api keys in the documentation. These steps will vary slightly by provider. For this example I'll be using the [Webshare Provider](https://davidteather.github.io/proxyproviders/#proxyproviders.providers.webshare.Webshare) because it's both easy to setup and they give you 10 free data center proxies to test out.
|
|
49
|
+
|
|
50
|
+
You can create an account on webshare [here](https://www.webshare.io/?referral_code=3x5812idzzzp) (affiliate link), then head into the [API tab](https://dashboard.webshare.io/userapi/keys) on the side and generate a new token. Keep this API token safe and don't post it publically.
|
|
51
|
+
|
|
52
|
+
### Basic Example
|
|
53
|
+
|
|
54
|
+
After you can list out your proxies with
|
|
55
|
+
```py
|
|
56
|
+
from proxyproviders import Webshare
|
|
57
|
+
|
|
58
|
+
proxy_provider = Webshare(api_key="your-api-key")
|
|
59
|
+
|
|
60
|
+
proxies = proxy_provider.list_proxies()
|
|
61
|
+
|
|
62
|
+
print(proxies)
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Each provider has their own custom options, the `Webshare` class lets you specify url params according to their [api spec](https://apidocs.webshare.io/proxy-list/list#parameters), here's an example which will only return proxies that are based in the US.
|
|
66
|
+
|
|
67
|
+
```py
|
|
68
|
+
proxy_provider = Webshare(api_key="your-api-key", params={"country_code_in": "US"})
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
### Using ProxyConfig
|
|
72
|
+
|
|
73
|
+
For any shared logic across all types of proxy providers, we use the `ProxyConfig` data class to configure them. The full docs for [ProxyConfig are here](https://davidteather.github.io/proxyproviders/#proxyproviders.proxy_provider.ProxyConfig). In this example we will configure it to use a shorter `refresh_interval` than default.
|
|
74
|
+
|
|
75
|
+
```py
|
|
76
|
+
from proxyproviders import Webshare, ProxyConfig
|
|
77
|
+
import time
|
|
78
|
+
|
|
79
|
+
config = ProxyConfig(refresh_interval=3)
|
|
80
|
+
ws = Webshare(api_key="your-api-token", config=config)
|
|
81
|
+
|
|
82
|
+
proxies = ws.list_proxies() # calls API
|
|
83
|
+
|
|
84
|
+
ws.list_proxies() # cached
|
|
85
|
+
|
|
86
|
+
time.sleep(5)
|
|
87
|
+
ws.list_proxies() # calls API since it's more than 3s later
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### Function Using Generic ProxyProvider
|
|
91
|
+
|
|
92
|
+
Since all proxy providers implement the same interface, we can make a function that allows us to easily swap out and utilize different providers. This is the main appeal of having a unified interface. It allows other modules to be provider agnostic, like my [TikTokAPI](https://github.com/davidteather/TikTok-Api) package.
|
|
93
|
+
|
|
94
|
+
```py
|
|
95
|
+
from proxyproviders import Webshare, BrightData, ProxyProvider, ProxyConfig
|
|
96
|
+
|
|
97
|
+
def some_function(provider: ProxyProvider):
|
|
98
|
+
proxies = provider.list_proxies()
|
|
99
|
+
print(proxies)
|
|
100
|
+
|
|
101
|
+
webshare = Webshare(api_key="your_api_key")
|
|
102
|
+
brightdata = BrightData(api_key="your_api_key", zone="my_zone")
|
|
103
|
+
|
|
104
|
+
some_function(webshare)
|
|
105
|
+
some_function(brightdata)
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Here's a more meaningful example that takes the `Proxy` class and uses it to create a python requests http proxy.
|
|
109
|
+
|
|
110
|
+
```py
|
|
111
|
+
from proxyproviders import Webshare, BrightData, ProxyProvider
|
|
112
|
+
import requests
|
|
113
|
+
import os
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def request_with_proxy(provider: ProxyProvider):
|
|
117
|
+
requests_proxy = None
|
|
118
|
+
|
|
119
|
+
if provider:
|
|
120
|
+
proxies = provider.list_proxies()
|
|
121
|
+
|
|
122
|
+
requests_proxy = {
|
|
123
|
+
"http": f"http://{proxies[0].username}:{proxies[0].password}@{proxies[0].proxy_address}:{proxies[0].port}",
|
|
124
|
+
"https": f"http://{proxies[0].username}:{proxies[0].password}@{proxies[0].proxy_address}:{proxies[0].port}",
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
r = requests.get("https://httpbin.org/ip", proxies=requests_proxy)
|
|
128
|
+
return r.json()
|
|
129
|
+
|
|
130
|
+
webshare = Webshare(api_key="your_api_key")
|
|
131
|
+
brightdata = BrightData(api_key="your_api_key", zone="your_zone")
|
|
132
|
+
|
|
133
|
+
print(f"Your IP: {request_with_proxy(None)}")
|
|
134
|
+
print(f"Webshare: {request_with_proxy(webshare)}")
|
|
135
|
+
print(f"BrightData: {request_with_proxy(brightdata)}")
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### Making Your Own Proxy Provider
|
|
139
|
+
|
|
140
|
+
Here's a skeleton of how you can make your very own `ProxyProvider` class. You'll need to implemenet all the required functions of the `ProxyProvider` which may be more than what's here at the time of writing.
|
|
141
|
+
|
|
142
|
+
If you do find yourself making one of these, consider contributing it back to the repository so everyone can use them :D
|
|
143
|
+
|
|
144
|
+
```py
|
|
145
|
+
from proxyproviders import ProxyProvider, ProxyConfig, Proxy
|
|
146
|
+
from typing import List, Optional
|
|
147
|
+
|
|
148
|
+
class MyProxyProvider(ProxyProvider):
|
|
149
|
+
def __init__(self, config: Optional[ProxyConfig] = None):
|
|
150
|
+
super().__init__(config=config)
|
|
151
|
+
|
|
152
|
+
def _fetch_proxies(self):
|
|
153
|
+
proxies: List[Proxy] = []
|
|
154
|
+
|
|
155
|
+
for i in range(10):
|
|
156
|
+
# TODO: your real proxy fetching logic
|
|
157
|
+
|
|
158
|
+
# There are required fields on the Proxy class, be sure that these are filled out properly
|
|
159
|
+
# especially if you're using it with another library.
|
|
160
|
+
proxy = Proxy(
|
|
161
|
+
id=str(i),
|
|
162
|
+
username="username",
|
|
163
|
+
password="password",
|
|
164
|
+
proxy_address="192.168.1.1",
|
|
165
|
+
port=80,
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
proxies.append(proxy)
|
|
169
|
+
|
|
170
|
+
return proxies
|
|
171
|
+
|
|
172
|
+
def some_function(provider: ProxyProvider):
|
|
173
|
+
proxies = provider.list_proxies()
|
|
174
|
+
for proxy in proxies:
|
|
175
|
+
print(proxy)
|
|
176
|
+
|
|
177
|
+
provider = MyProxyProvider()
|
|
178
|
+
some_function(provider) # calls the function with the provider
|
|
179
|
+
```
|
|
180
|
+
|
|
File without changes
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from typing import Optional, List
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class Proxy:
|
|
8
|
+
"""Our shared data model for a proxy object across all providers."""
|
|
9
|
+
|
|
10
|
+
id: str
|
|
11
|
+
"""A unique identifier for the proxy"""
|
|
12
|
+
|
|
13
|
+
username: str
|
|
14
|
+
"""The username required for authenticating with the proxy"""
|
|
15
|
+
|
|
16
|
+
password: str
|
|
17
|
+
"""The password required for authenticating with the proxy"""
|
|
18
|
+
|
|
19
|
+
proxy_address: str
|
|
20
|
+
"""The IP address or domain name of the proxy"""
|
|
21
|
+
|
|
22
|
+
port: int
|
|
23
|
+
"""The port number through which the proxy connection is established"""
|
|
24
|
+
|
|
25
|
+
country_code: Optional[str] = None
|
|
26
|
+
"""The country code where the proxy is located, e.g., 'US', 'FR'. Optional"""
|
|
27
|
+
|
|
28
|
+
city_name: Optional[str] = None
|
|
29
|
+
"""The city name where the proxy is located, e.g., 'New York', 'Paris'. Optional"""
|
|
30
|
+
|
|
31
|
+
created_at: Optional[datetime] = None
|
|
32
|
+
"""The timestamp when the proxy was created. Optional"""
|
|
33
|
+
|
|
34
|
+
protocols: Optional[List[str]] = None
|
|
35
|
+
"""A list of connection protocols supported by the proxy, e.g., ['http', 'https']"""
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
from typing import List, Dict, Optional
|
|
3
|
+
from ..proxy_provider import ProxyProvider, ProxyConfig
|
|
4
|
+
from ..exceptions import ProxyFetchException, ProxyInvalidResponseException
|
|
5
|
+
from ..models.proxy import Proxy
|
|
6
|
+
import threading
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class BrightData(ProxyProvider):
|
|
10
|
+
"""BrightData (formerly luminati) is a proxy provider that offers residential and datacenter proxies.
|
|
11
|
+
|
|
12
|
+
Create an account `here <https://get.brightdata.com/davidteather>`_ (affiliate link).
|
|
13
|
+
|
|
14
|
+
You can find your API key in the account settings `here <https://brightdata.com/cp/setting/users>`_ then create "add token" with scope "limit" (`BrightData article <https://docs.brightdata.com/general/account/api-token>`_ for more info)
|
|
15
|
+
|
|
16
|
+
The BrightData API documentation is `here <https://docs.brightdata.com/api-reference/account-management-api/Get_active_Zones?playground=open>`_
|
|
17
|
+
|
|
18
|
+
:param api_key: Your BrightData API key
|
|
19
|
+
:param zone: The zone ID/name to fetch proxies for (you can get this from the BrightData dashboard)
|
|
20
|
+
:param username_suffix: Optional suffix to append to the username `more info <https://docs.brightdata.com/proxy-networks/config-options>`_ allows you to target region, city, etc. (requires use_super_proxy=True)
|
|
21
|
+
:param use_super_proxy: Optional flag to use super proxy instead of targeting specific IPs, this is enabled by default. If you want to target specific IPs or have consistent IPs for a session, set this to False.
|
|
22
|
+
:param config: Configuration for the proxy provider. View the ProxyConfig class docs for more information.
|
|
23
|
+
|
|
24
|
+
Example:
|
|
25
|
+
|
|
26
|
+
.. code-block:: python
|
|
27
|
+
|
|
28
|
+
from proxyproviders import BrightData
|
|
29
|
+
|
|
30
|
+
# Initialize the BrightData API client with an API key and a zone
|
|
31
|
+
proxy_provider = BrightData(api_key="your-api-key", zone="my_zone")
|
|
32
|
+
|
|
33
|
+
# Fetch proxies
|
|
34
|
+
proxies = proxy_provider.list_proxies() # returns one proxy for super proxy by default
|
|
35
|
+
|
|
36
|
+
# If you want to manage specific IPs
|
|
37
|
+
proxy_provider = BrightData(api_key="your-api-key", zone="my_zone", use_super_proxy=False)
|
|
38
|
+
proxies = proxy_provider.list_proxies() # returns multiple proxies for each IP in the zone (potentially thousands)
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
_BASE_URL = "https://api.brightdata.com"
|
|
42
|
+
_SUPER_PROXY_ADDRESS = "brd.superproxy.io"
|
|
43
|
+
_SUPER_PROXY_PORT = 33335
|
|
44
|
+
_PROTOCOLS: List[str] = ["http", "https"] # BrightData supports both HTTP and HTTPS
|
|
45
|
+
|
|
46
|
+
def __init__(
|
|
47
|
+
self,
|
|
48
|
+
api_key: str,
|
|
49
|
+
zone: str,
|
|
50
|
+
username_suffix: Optional[str] = None,
|
|
51
|
+
use_super_proxy: Optional[bool] = True,
|
|
52
|
+
config: Optional[ProxyConfig] = None,
|
|
53
|
+
):
|
|
54
|
+
super().__init__(config)
|
|
55
|
+
self.api_key = api_key
|
|
56
|
+
self.zone = zone
|
|
57
|
+
self.username_suffix = username_suffix
|
|
58
|
+
self.use_super_proxy = use_super_proxy
|
|
59
|
+
|
|
60
|
+
def _fetch_proxies(self) -> List[Proxy]:
|
|
61
|
+
username = self.get_zone_username(self.zone)
|
|
62
|
+
if self.username_suffix:
|
|
63
|
+
username += self.username_suffix
|
|
64
|
+
|
|
65
|
+
passwords = self.get_zone_passwords(self.zone)
|
|
66
|
+
|
|
67
|
+
if self.use_super_proxy:
|
|
68
|
+
# Let the super proxy handle the IP rotation
|
|
69
|
+
return [
|
|
70
|
+
Proxy(
|
|
71
|
+
id="super",
|
|
72
|
+
username=username,
|
|
73
|
+
password=passwords["passwords"][0],
|
|
74
|
+
proxy_address=self._SUPER_PROXY_ADDRESS,
|
|
75
|
+
port=self._SUPER_PROXY_PORT,
|
|
76
|
+
protocols=self._PROTOCOLS,
|
|
77
|
+
)
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
proxies = []
|
|
81
|
+
|
|
82
|
+
# Fetch all IPs in the zone, and create a proxy for each
|
|
83
|
+
# Brightdata doesn't let us target directly so we tell the superproxy what to do
|
|
84
|
+
ips = self.list_all_ips_in_zone(self.zone)
|
|
85
|
+
|
|
86
|
+
for ip in ips:
|
|
87
|
+
ip_targeted_username = username + f"-ip-{ip['ip']}"
|
|
88
|
+
|
|
89
|
+
proxies.append(
|
|
90
|
+
Proxy(
|
|
91
|
+
id=ip["ip"],
|
|
92
|
+
username=ip_targeted_username,
|
|
93
|
+
password=passwords["passwords"][0],
|
|
94
|
+
proxy_address=self._SUPER_PROXY_ADDRESS,
|
|
95
|
+
port=self._SUPER_PROXY_PORT,
|
|
96
|
+
country_code=ip["country"],
|
|
97
|
+
protocols=self._PROTOCOLS,
|
|
98
|
+
)
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
return proxies
|
|
102
|
+
|
|
103
|
+
def get_active_zones(self) -> Dict:
|
|
104
|
+
"""Fetches active zones from BrightData API.
|
|
105
|
+
|
|
106
|
+
Response:
|
|
107
|
+
|
|
108
|
+
.. code-block:: json
|
|
109
|
+
|
|
110
|
+
[
|
|
111
|
+
{
|
|
112
|
+
"name": "zone1",
|
|
113
|
+
"type": "dc",
|
|
114
|
+
}
|
|
115
|
+
]
|
|
116
|
+
|
|
117
|
+
"""
|
|
118
|
+
return self._make_request("/zone/get_active_zones", "GET")
|
|
119
|
+
|
|
120
|
+
def get_zone_username(self, zone: str) -> str:
|
|
121
|
+
"""Fetches zone username for the given zone ID from BrightData API.
|
|
122
|
+
|
|
123
|
+
Note: this isn't directly an API endpoint, I'm sort of reconstructing some things here and it seems to behave a little weird.
|
|
124
|
+
|
|
125
|
+
:param zone: The zone ID to fetch username for
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
data = self._make_request(f"/status", "GET", params={"zone": zone})
|
|
129
|
+
|
|
130
|
+
customer = data.get("customer")
|
|
131
|
+
if not customer:
|
|
132
|
+
raise ProxyInvalidResponseException("Failed to fetch customer data")
|
|
133
|
+
|
|
134
|
+
return f"brd-customer-{customer}-zone-{zone}"
|
|
135
|
+
|
|
136
|
+
def get_zone_passwords(self, zone: str) -> Dict:
|
|
137
|
+
"""Fetches zone passwords from BrightData API.
|
|
138
|
+
|
|
139
|
+
:param zone: The zone ID to fetch passwords for
|
|
140
|
+
|
|
141
|
+
Response:
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
.. code-block:: json
|
|
145
|
+
|
|
146
|
+
{
|
|
147
|
+
"passwords": [
|
|
148
|
+
"password1",
|
|
149
|
+
"password2",
|
|
150
|
+
]
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
"""
|
|
154
|
+
return self._make_request(f"/zone/passwords", "GET", params={"zone": zone})
|
|
155
|
+
|
|
156
|
+
def list_all_ips_in_zone(self, zone: str, country: Optional[str] = None) -> Dict:
|
|
157
|
+
"""Fetches all IPs in a zone from BrightData API.
|
|
158
|
+
|
|
159
|
+
:param zone: The zone ID to fetch IPs for
|
|
160
|
+
:param country: Optional 2-letter country code to filter IPs by
|
|
161
|
+
|
|
162
|
+
Response:
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
.. code-block:: json
|
|
166
|
+
|
|
167
|
+
[
|
|
168
|
+
{
|
|
169
|
+
"ip": "192.168.1.1",
|
|
170
|
+
"country": "US",
|
|
171
|
+
}
|
|
172
|
+
]
|
|
173
|
+
|
|
174
|
+
"""
|
|
175
|
+
return self._make_request(
|
|
176
|
+
f"/zone/route_ips",
|
|
177
|
+
"GET",
|
|
178
|
+
params={"zone": zone, "list_countries": True, "country": country},
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
def _make_request(
|
|
182
|
+
self,
|
|
183
|
+
path: str,
|
|
184
|
+
method: str,
|
|
185
|
+
params: Optional[Dict] = None,
|
|
186
|
+
json: Optional[Dict] = None,
|
|
187
|
+
) -> Dict:
|
|
188
|
+
"""Makes a request to the BrightData API.
|
|
189
|
+
|
|
190
|
+
:param path: The path to the endpoint
|
|
191
|
+
:param method: The HTTP method to use
|
|
192
|
+
:param params: Optional parameters to include in the request
|
|
193
|
+
:param json: Optional JSON data to include in the request
|
|
194
|
+
"""
|
|
195
|
+
|
|
196
|
+
headers = {
|
|
197
|
+
"Authorization": f"Bearer {self.api_key}",
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
url = f"{self._BASE_URL}{path}"
|
|
201
|
+
response = requests.request(
|
|
202
|
+
method, url, headers=headers, params=params, json=json
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
if response.status_code != 200:
|
|
206
|
+
raise ProxyFetchException(
|
|
207
|
+
f"Failed to fetch from BrightData, got status code {response.status_code}, text: {response.text}"
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
try:
|
|
211
|
+
data = response.json()
|
|
212
|
+
except Exception as e:
|
|
213
|
+
raise ProxyInvalidResponseException(
|
|
214
|
+
f"Failed to parse response: {str(e)}"
|
|
215
|
+
) from e
|
|
216
|
+
|
|
217
|
+
return data
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from typing import List, Dict, Optional
|
|
4
|
+
from ..proxy_provider import ProxyProvider, ProxyConfig
|
|
5
|
+
from ..exceptions import (
|
|
6
|
+
ProxyFetchException,
|
|
7
|
+
ProxyConversionException,
|
|
8
|
+
ProxyInvalidResponseException,
|
|
9
|
+
)
|
|
10
|
+
from ..models.proxy import Proxy
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Webshare(ProxyProvider):
|
|
14
|
+
"""Webshare is a proxy provider that offers residential and datacenter proxies.
|
|
15
|
+
|
|
16
|
+
Create an account for webshare `here <https://www.webshare.io/?referral_code=3x5812idzzzp>`_ (affiliate link) to get started with 10 free data center proxies.
|
|
17
|
+
|
|
18
|
+
You can find your API key in the Webshare dashboard `here <https://dashboard.webshare.io/userapi/keys>`_
|
|
19
|
+
|
|
20
|
+
You can find a list of all available parameters in the Webshare API documentation `here <https://apidocs.webshare.io/proxy-list/list#parameters>`_
|
|
21
|
+
|
|
22
|
+
:param api_key: Your Webshare API key
|
|
23
|
+
:param search_params: Optional parameters to include in the API requests
|
|
24
|
+
:param config: Configuration for the proxy provider. View the ProxyConfig class docs for more information.
|
|
25
|
+
|
|
26
|
+
Example:
|
|
27
|
+
|
|
28
|
+
.. code-block:: python
|
|
29
|
+
|
|
30
|
+
from proxyproviders import Webshare
|
|
31
|
+
|
|
32
|
+
# Initialize the Webshare API client with an API key and optional parameters
|
|
33
|
+
proxy_provider = Webshare(api_key="your-api-key", params={"country_code_in": "US"})
|
|
34
|
+
|
|
35
|
+
# Fetch proxies
|
|
36
|
+
proxies = proxy_provider.list_proxies()
|
|
37
|
+
|
|
38
|
+
# With config
|
|
39
|
+
from proxyproviders import ProxyConfig
|
|
40
|
+
|
|
41
|
+
config = ProxyConfig(refresh_interval=60)
|
|
42
|
+
proxy_provider = Webshare(api_key="your-api-key", params={"country_code_in": "US"}, config=config)
|
|
43
|
+
|
|
44
|
+
# Fetch proxies
|
|
45
|
+
proxies = proxy_provider.list_proxies()
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
_BASE_URL = "https://proxy.webshare.io/api/v2"
|
|
49
|
+
_PROTOCOLS = ["http", "https"]
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
api_key: str,
|
|
54
|
+
search_params: Optional[dict] = None,
|
|
55
|
+
config: Optional[ProxyConfig] = None,
|
|
56
|
+
):
|
|
57
|
+
super().__init__(config)
|
|
58
|
+
self.api_key = api_key
|
|
59
|
+
self.search_params = search_params or {}
|
|
60
|
+
|
|
61
|
+
def _fetch_proxies(self) -> List[Proxy]:
|
|
62
|
+
"""Fetches proxies from Webshare API and converts them to the standardized Proxy format."""
|
|
63
|
+
headers = {"Authorization": f"Token {self.api_key}"}
|
|
64
|
+
|
|
65
|
+
all_proxies = []
|
|
66
|
+
page = 0
|
|
67
|
+
|
|
68
|
+
while True:
|
|
69
|
+
page += 1
|
|
70
|
+
default_params = {
|
|
71
|
+
"mode": "direct",
|
|
72
|
+
"page": page,
|
|
73
|
+
"page_size": 100,
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
params = {**default_params, **self.search_params}
|
|
77
|
+
try:
|
|
78
|
+
response = requests.get(
|
|
79
|
+
f"{self._BASE_URL}/proxy/list", params=params, headers=headers
|
|
80
|
+
)
|
|
81
|
+
response.raise_for_status() # Raises HTTPError for bad responses
|
|
82
|
+
except requests.exceptions.RequestException as e:
|
|
83
|
+
raise ProxyFetchException(f"Failed to fetch proxies: {str(e)}") from e
|
|
84
|
+
|
|
85
|
+
data = response.json()
|
|
86
|
+
|
|
87
|
+
proxy_data = data.get("results")
|
|
88
|
+
if not proxy_data:
|
|
89
|
+
raise ProxyInvalidResponseException(response.text)
|
|
90
|
+
|
|
91
|
+
all_proxies.extend([self._convert_to_proxy(proxy) for proxy in proxy_data])
|
|
92
|
+
|
|
93
|
+
if data.get("next") is None: # no more pages
|
|
94
|
+
break
|
|
95
|
+
|
|
96
|
+
return all_proxies
|
|
97
|
+
|
|
98
|
+
def _convert_to_proxy(self, data: Dict) -> Proxy:
|
|
99
|
+
"""Converts Webshare's proxy data to the shared Proxy format."""
|
|
100
|
+
try:
|
|
101
|
+
return Proxy(
|
|
102
|
+
id=data["id"],
|
|
103
|
+
username=data["username"],
|
|
104
|
+
password=data["password"],
|
|
105
|
+
proxy_address=data["proxy_address"],
|
|
106
|
+
port=int(data["port"]),
|
|
107
|
+
country_code=data.get("country_code"),
|
|
108
|
+
city_name=data.get("city_name"),
|
|
109
|
+
created_at=self._parse_timestamp(data.get("created_at")),
|
|
110
|
+
protocols=self._PROTOCOLS,
|
|
111
|
+
)
|
|
112
|
+
except (KeyError, ValueError) as e:
|
|
113
|
+
raise ProxyConversionException(
|
|
114
|
+
f"Failed to convert proxy data: {str(e)}"
|
|
115
|
+
) from e
|
|
116
|
+
|
|
117
|
+
def _parse_timestamp(self, timestamp: Optional[str]) -> Optional[datetime]:
|
|
118
|
+
"""Parses an ISO 8601 timestamp string into a datetime object."""
|
|
119
|
+
if timestamp:
|
|
120
|
+
try:
|
|
121
|
+
return datetime.fromisoformat(timestamp)
|
|
122
|
+
except ValueError:
|
|
123
|
+
return None
|
|
124
|
+
return None
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: proxyproviders
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A unified interface for different proxy providers
|
|
5
|
+
Author-email: David Teather <contact.davidteather@gmail.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2025 David Teather
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Classifier: Development Status :: 3 - Alpha
|
|
29
|
+
Classifier: Intended Audience :: Developers
|
|
30
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
31
|
+
Classifier: Programming Language :: Python :: 3
|
|
32
|
+
Requires-Python: >=3.9
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
License-File: LICENSE
|
|
35
|
+
Requires-Dist: requests
|
|
36
|
+
Provides-Extra: dev
|
|
37
|
+
Requires-Dist: pytest; extra == "dev"
|
|
38
|
+
Requires-Dist: sphinx; extra == "dev"
|
|
39
|
+
Requires-Dist: black; extra == "dev"
|
|
40
|
+
Requires-Dist: mypy; extra == "dev"
|
|
41
|
+
Requires-Dist: coverage; extra == "dev"
|
|
42
|
+
Requires-Dist: types-requests; extra == "dev"
|
|
43
|
+
Requires-Dist: toml; extra == "dev"
|
|
44
|
+
|
|
45
|
+
# ProxyProviders
|
|
46
|
+
|
|
47
|
+
The Unified Python Proxy API for managing proxies with support for BrightData, WebShare, and more.
|
|
48
|
+
|
|
49
|
+
[](https://www.linkedin.com/in/davidteather/) [](https://github.com/sponsors/davidteather) [](https://github.com/davidteather/proxyproviders/releases) [](https://github.com/davidteather/proxyproviders/blob/main/LICENSE) [](https://pypi.org/project/proxyproviders/)  [](https://discord.gg/yyPhbfma6f)
|
|
50
|
+
|
|
51
|
+
## Table of Contents
|
|
52
|
+
|
|
53
|
+
- [Documentation](#documentation)
|
|
54
|
+
- [Getting Started](#getting-started)
|
|
55
|
+
- [How to Support The Project](#how-to-support-the-project)
|
|
56
|
+
- [Installing](#installing)
|
|
57
|
+
- [Quick Start Guide](#quick-start-guide)
|
|
58
|
+
- [Examples](https://github.com/davidteather/proxyproviders/tree/main/examples)
|
|
59
|
+
|
|
60
|
+
## Documentation
|
|
61
|
+
|
|
62
|
+
You can find the full documentation [here](https://davidteather.github.io/proxyproviders)
|
|
63
|
+
|
|
64
|
+
**Note:** If you want to learn how to web scrape websites check my [free and open-source course for learning everything web scraping](https://github.com/davidteather/everything-web-scraping)
|
|
65
|
+
|
|
66
|
+
### How to Support The Project
|
|
67
|
+
|
|
68
|
+
- Star the repo 😎
|
|
69
|
+
- Consider [sponsoring](https://github.com/sponsors/davidteather) me on GitHub
|
|
70
|
+
- Send me an email or a [LinkedIn](https://www.linkedin.com/in/davidteather/) message about what you're doing with it :D
|
|
71
|
+
- Submit PRs for issues or any new providers/features :)
|
|
72
|
+
|
|
73
|
+
## Getting Started
|
|
74
|
+
|
|
75
|
+
To get started using this package follow the instructions below.
|
|
76
|
+
|
|
77
|
+
### Installation
|
|
78
|
+
|
|
79
|
+
This package is installable via pip.
|
|
80
|
+
```sh
|
|
81
|
+
python -m pip install proxyproviders
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Quick Start Guide
|
|
85
|
+
|
|
86
|
+
Here's a quick bit of code to get you started! There's also a few [examples in the folder](https://github.com/davidteather/proxyproviders/tree/main/examples).
|
|
87
|
+
|
|
88
|
+
**Note:** If you want to learn how to web scrape websites check my [free and open-source course for learning everything web scraping](https://github.com/davidteather/everything-web-scraping)
|
|
89
|
+
|
|
90
|
+
### Choose a Proxy Provider
|
|
91
|
+
|
|
92
|
+
If you already haven't, choose which proxy provider to use. You can find a [list in the documentation](https://davidteather.github.io/proxyproviders/#proxyproviders-supported-providers). After choosing, look at the documentation around the specific provider you've choosen. Where needed, we've laid out steps to get api keys in the documentation. These steps will vary slightly by provider. For this example I'll be using the [Webshare Provider](https://davidteather.github.io/proxyproviders/#proxyproviders.providers.webshare.Webshare) because it's both easy to setup and they give you 10 free data center proxies to test out.
|
|
93
|
+
|
|
94
|
+
You can create an account on webshare [here](https://www.webshare.io/?referral_code=3x5812idzzzp) (affiliate link), then head into the [API tab](https://dashboard.webshare.io/userapi/keys) on the side and generate a new token. Keep this API token safe and don't post it publically.
|
|
95
|
+
|
|
96
|
+
### Basic Example
|
|
97
|
+
|
|
98
|
+
After you can list out your proxies with
|
|
99
|
+
```py
|
|
100
|
+
from proxyproviders import Webshare
|
|
101
|
+
|
|
102
|
+
proxy_provider = Webshare(api_key="your-api-key")
|
|
103
|
+
|
|
104
|
+
proxies = proxy_provider.list_proxies()
|
|
105
|
+
|
|
106
|
+
print(proxies)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Each provider has their own custom options, the `Webshare` class lets you specify url params according to their [api spec](https://apidocs.webshare.io/proxy-list/list#parameters), here's an example which will only return proxies that are based in the US.
|
|
110
|
+
|
|
111
|
+
```py
|
|
112
|
+
proxy_provider = Webshare(api_key="your-api-key", params={"country_code_in": "US"})
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### Using ProxyConfig
|
|
116
|
+
|
|
117
|
+
For any shared logic across all types of proxy providers, we use the `ProxyConfig` data class to configure them. The full docs for [ProxyConfig are here](https://davidteather.github.io/proxyproviders/#proxyproviders.proxy_provider.ProxyConfig). In this example we will configure it to use a shorter `refresh_interval` than default.
|
|
118
|
+
|
|
119
|
+
```py
|
|
120
|
+
from proxyproviders import Webshare, ProxyConfig
|
|
121
|
+
import time
|
|
122
|
+
|
|
123
|
+
config = ProxyConfig(refresh_interval=3)
|
|
124
|
+
ws = Webshare(api_key="your-api-token", config=config)
|
|
125
|
+
|
|
126
|
+
proxies = ws.list_proxies() # calls API
|
|
127
|
+
|
|
128
|
+
ws.list_proxies() # cached
|
|
129
|
+
|
|
130
|
+
time.sleep(5)
|
|
131
|
+
ws.list_proxies() # calls API since it's more than 3s later
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### Function Using Generic ProxyProvider
|
|
135
|
+
|
|
136
|
+
Since all proxy providers implement the same interface, we can make a function that allows us to easily swap out and utilize different providers. This is the main appeal of having a unified interface. It allows other modules to be provider agnostic, like my [TikTokAPI](https://github.com/davidteather/TikTok-Api) package.
|
|
137
|
+
|
|
138
|
+
```py
|
|
139
|
+
from proxyproviders import Webshare, BrightData, ProxyProvider, ProxyConfig
|
|
140
|
+
|
|
141
|
+
def some_function(provider: ProxyProvider):
|
|
142
|
+
proxies = provider.list_proxies()
|
|
143
|
+
print(proxies)
|
|
144
|
+
|
|
145
|
+
webshare = Webshare(api_key="your_api_key")
|
|
146
|
+
brightdata = BrightData(api_key="your_api_key", zone="my_zone")
|
|
147
|
+
|
|
148
|
+
some_function(webshare)
|
|
149
|
+
some_function(brightdata)
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Here's a more meaningful example that takes the `Proxy` class and uses it to create a python requests http proxy.
|
|
153
|
+
|
|
154
|
+
```py
|
|
155
|
+
from proxyproviders import Webshare, BrightData, ProxyProvider
|
|
156
|
+
import requests
|
|
157
|
+
import os
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def request_with_proxy(provider: ProxyProvider):
|
|
161
|
+
requests_proxy = None
|
|
162
|
+
|
|
163
|
+
if provider:
|
|
164
|
+
proxies = provider.list_proxies()
|
|
165
|
+
|
|
166
|
+
requests_proxy = {
|
|
167
|
+
"http": f"http://{proxies[0].username}:{proxies[0].password}@{proxies[0].proxy_address}:{proxies[0].port}",
|
|
168
|
+
"https": f"http://{proxies[0].username}:{proxies[0].password}@{proxies[0].proxy_address}:{proxies[0].port}",
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
r = requests.get("https://httpbin.org/ip", proxies=requests_proxy)
|
|
172
|
+
return r.json()
|
|
173
|
+
|
|
174
|
+
webshare = Webshare(api_key="your_api_key")
|
|
175
|
+
brightdata = BrightData(api_key="your_api_key", zone="your_zone")
|
|
176
|
+
|
|
177
|
+
print(f"Your IP: {request_with_proxy(None)}")
|
|
178
|
+
print(f"Webshare: {request_with_proxy(webshare)}")
|
|
179
|
+
print(f"BrightData: {request_with_proxy(brightdata)}")
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### Making Your Own Proxy Provider
|
|
183
|
+
|
|
184
|
+
Here's a skeleton of how you can make your very own `ProxyProvider` class. You'll need to implemenet all the required functions of the `ProxyProvider` which may be more than what's here at the time of writing.
|
|
185
|
+
|
|
186
|
+
If you do find yourself making one of these, consider contributing it back to the repository so everyone can use them :D
|
|
187
|
+
|
|
188
|
+
```py
|
|
189
|
+
from proxyproviders import ProxyProvider, ProxyConfig, Proxy
|
|
190
|
+
from typing import List, Optional
|
|
191
|
+
|
|
192
|
+
class MyProxyProvider(ProxyProvider):
|
|
193
|
+
def __init__(self, config: Optional[ProxyConfig] = None):
|
|
194
|
+
super().__init__(config=config)
|
|
195
|
+
|
|
196
|
+
def _fetch_proxies(self):
|
|
197
|
+
proxies: List[Proxy] = []
|
|
198
|
+
|
|
199
|
+
for i in range(10):
|
|
200
|
+
# TODO: your real proxy fetching logic
|
|
201
|
+
|
|
202
|
+
# There are required fields on the Proxy class, be sure that these are filled out properly
|
|
203
|
+
# especially if you're using it with another library.
|
|
204
|
+
proxy = Proxy(
|
|
205
|
+
id=str(i),
|
|
206
|
+
username="username",
|
|
207
|
+
password="password",
|
|
208
|
+
proxy_address="192.168.1.1",
|
|
209
|
+
port=80,
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
proxies.append(proxy)
|
|
213
|
+
|
|
214
|
+
return proxies
|
|
215
|
+
|
|
216
|
+
def some_function(provider: ProxyProvider):
|
|
217
|
+
proxies = provider.list_proxies()
|
|
218
|
+
for proxy in proxies:
|
|
219
|
+
print(proxy)
|
|
220
|
+
|
|
221
|
+
provider = MyProxyProvider()
|
|
222
|
+
some_function(provider) # calls the function with the provider
|
|
223
|
+
```
|
|
224
|
+
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
proxyproviders/models/__init__.py
|
|
5
|
+
proxyproviders/models/proxy.py
|
|
6
|
+
proxyproviders/providers/brightdata.py
|
|
7
|
+
proxyproviders/providers/webshare.py
|
|
8
|
+
proxyproviders/proxyproviders.egg-info/PKG-INFO
|
|
9
|
+
proxyproviders/proxyproviders.egg-info/SOURCES.txt
|
|
10
|
+
proxyproviders/proxyproviders.egg-info/dependency_links.txt
|
|
11
|
+
proxyproviders/proxyproviders.egg-info/requires.txt
|
|
12
|
+
proxyproviders/proxyproviders.egg-info/top_level.txt
|
|
13
|
+
tests/test_exceptions.py
|
|
14
|
+
tests/test_proxy_provider.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "proxyproviders"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "A unified interface for different proxy providers"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [{ name = "David Teather", email = "contact.davidteather@gmail.com" }]
|
|
11
|
+
license = { file = "LICENSE" }
|
|
12
|
+
dependencies = ["requests"]
|
|
13
|
+
requires-python = ">=3.9"
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.optional-dependencies]
|
|
22
|
+
dev = ["pytest", "sphinx", "black", "mypy", "coverage", "types-requests", "toml"]
|
|
23
|
+
|
|
24
|
+
[tool.setuptools.packages.find]
|
|
25
|
+
where = ["proxyproviders"]
|
|
26
|
+
|
|
27
|
+
[tool.coverage.run]
|
|
28
|
+
branch = true
|
|
29
|
+
source = ["proxyproviders"]
|
|
30
|
+
omit = ["tests/*"]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
[tool.poetry.group.dev.dependencies]
|
|
34
|
+
mypy = "^1.15.0"
|
|
35
|
+
black = "^25.1.0"
|
|
36
|
+
pytest = "^8.3.4"
|
|
37
|
+
coverage = "^7.6.12"
|
|
38
|
+
types-requests = "^2.32.0.20241016"
|
|
39
|
+
toml = "^0.10.2"
|
|
40
|
+
responses = "^0.25.6"
|
|
41
|
+
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from proxyproviders.exceptions import ProxyFetchException, ProxyInvalidResponseException
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def test_proxy_fetch_exception():
|
|
6
|
+
"""
|
|
7
|
+
Test that ProxyFetchException stores the correct message and status code.
|
|
8
|
+
"""
|
|
9
|
+
ex = ProxyFetchException("Fetch error", status_code=500)
|
|
10
|
+
assert "Fetch error" in str(ex)
|
|
11
|
+
assert ex.status_code == 500
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_proxy_invalid_response_exception():
|
|
15
|
+
"""
|
|
16
|
+
Test that ProxyInvalidResponseException correctly formats its message.
|
|
17
|
+
"""
|
|
18
|
+
response = '{"error": "Invalid API Key"}'
|
|
19
|
+
ex = ProxyInvalidResponseException(response)
|
|
20
|
+
assert "Invalid response received" in str(ex)
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from proxyproviders.proxy_provider import ProxyProvider, ProxyConfig
|
|
3
|
+
from datetime import timedelta
|
|
4
|
+
from typing import List
|
|
5
|
+
from proxyproviders.models.proxy import Proxy
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class MockProxyProvider(ProxyProvider):
|
|
9
|
+
"""Mock implementation of ProxyProvider for testing."""
|
|
10
|
+
|
|
11
|
+
def _fetch_proxies(self):
|
|
12
|
+
return []
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_should_refresh():
|
|
16
|
+
"""
|
|
17
|
+
Test that the provider correctly determines whether it should refresh proxies.
|
|
18
|
+
"""
|
|
19
|
+
provider = MockProxyProvider(config=ProxyConfig(refresh_interval=10))
|
|
20
|
+
# Initially, no proxies exist so it should refresh.
|
|
21
|
+
assert provider.should_refresh() is True
|
|
22
|
+
# Simulate a fetch to set proxies and update the refresh timestamp.
|
|
23
|
+
provider._set_proxies([])
|
|
24
|
+
# Immediately afterward, it should not refresh because the interval has not passed.
|
|
25
|
+
assert provider.should_refresh() is False
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_list_proxies():
|
|
29
|
+
"""
|
|
30
|
+
Test that list_proxies returns a list even if no proxies are available.
|
|
31
|
+
"""
|
|
32
|
+
provider = MockProxyProvider()
|
|
33
|
+
assert isinstance(provider.list_proxies(), list)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_force_refresh():
|
|
37
|
+
"""
|
|
38
|
+
Test that list_proxies fetches new proxies when force_refresh is True.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
# Directly subclassing the ProxyProvider for this test
|
|
42
|
+
# for simplicity and to avoid the need for a real provider.
|
|
43
|
+
class TestProxyProvider(MockProxyProvider):
|
|
44
|
+
def _fetch_proxies(self) -> List[Proxy]:
|
|
45
|
+
proxies = [1, 2, 3]
|
|
46
|
+
return proxies
|
|
47
|
+
|
|
48
|
+
provider = TestProxyProvider()
|
|
49
|
+
|
|
50
|
+
print(provider._proxies)
|
|
51
|
+
assert provider._proxies is None
|
|
52
|
+
|
|
53
|
+
proxies = provider.list_proxies(force_refresh=True)
|
|
54
|
+
print(proxies)
|
|
55
|
+
|
|
56
|
+
assert proxies == [1, 2, 3]
|
|
57
|
+
|
|
58
|
+
assert provider.list_proxies() == [1, 2, 3]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_refresh_interval():
|
|
62
|
+
"""
|
|
63
|
+
Tests around the refresh_interval configuration.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
# When refresh_interval is 0, it should never refresh automatically
|
|
67
|
+
provider = MockProxyProvider(config=ProxyConfig(refresh_interval=0))
|
|
68
|
+
provider._fetch_proxies = lambda: [1, 2, 3]
|
|
69
|
+
|
|
70
|
+
# Simulate a fetch and set proxies to check refresh
|
|
71
|
+
provider._set_proxies([1, 2, 3])
|
|
72
|
+
assert provider.list_proxies() == [1, 2, 3]
|
|
73
|
+
|
|
74
|
+
# Should not refresh since refresh_interval is 0
|
|
75
|
+
assert provider.should_refresh() is False
|
|
76
|
+
|
|
77
|
+
# Simulate a time lapse, but the proxies should not refresh
|
|
78
|
+
provider._last_refresh = provider._last_refresh - timedelta(seconds=1)
|
|
79
|
+
assert provider.should_refresh() is False
|
|
80
|
+
|
|
81
|
+
# When refresh_interval is set to a value, refresh should happen after the specified interval
|
|
82
|
+
provider = MockProxyProvider(config=ProxyConfig(refresh_interval=10))
|
|
83
|
+
provider._fetch_proxies = lambda: [1, 2, 3]
|
|
84
|
+
provider._set_proxies([1, 2, 3])
|
|
85
|
+
assert provider.list_proxies() == [1, 2, 3]
|
|
86
|
+
|
|
87
|
+
# Simulate the interval passing (10 seconds)
|
|
88
|
+
provider._last_refresh = provider._last_refresh - timedelta(seconds=11)
|
|
89
|
+
assert provider.should_refresh() is True
|
|
90
|
+
|
|
91
|
+
# Fetch new proxies after interval has passed
|
|
92
|
+
provider._set_proxies([4, 5, 6])
|
|
93
|
+
assert provider.list_proxies() == [4, 5, 6]
|