proxyproviders 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {proxyproviders-0.1.0/proxyproviders/proxyproviders.egg-info → proxyproviders-0.2.0}/PKG-INFO +126 -53
- proxyproviders-0.1.0/PKG-INFO → proxyproviders-0.2.0/README.md +90 -62
- proxyproviders-0.2.0/proxyproviders/__init__.py +14 -0
- proxyproviders-0.2.0/proxyproviders/algorithms.py +112 -0
- proxyproviders-0.2.0/proxyproviders/exceptions.py +35 -0
- proxyproviders-0.2.0/proxyproviders/models/proxy.py +118 -0
- {proxyproviders-0.1.0 → proxyproviders-0.2.0}/proxyproviders/providers/brightdata.py +4 -3
- {proxyproviders-0.1.0 → proxyproviders-0.2.0}/proxyproviders/providers/webshare.py +7 -5
- proxyproviders-0.2.0/proxyproviders/proxy_provider.py +130 -0
- {proxyproviders-0.1.0 → proxyproviders-0.2.0}/pyproject.toml +3 -4
- proxyproviders-0.1.0/README.md +0 -180
- proxyproviders-0.1.0/proxyproviders/models/proxy.py +0 -35
- proxyproviders-0.1.0/proxyproviders/proxyproviders.egg-info/SOURCES.txt +0 -14
- proxyproviders-0.1.0/proxyproviders/proxyproviders.egg-info/dependency_links.txt +0 -1
- proxyproviders-0.1.0/proxyproviders/proxyproviders.egg-info/requires.txt +0 -10
- proxyproviders-0.1.0/proxyproviders/proxyproviders.egg-info/top_level.txt +0 -2
- proxyproviders-0.1.0/setup.cfg +0 -4
- proxyproviders-0.1.0/tests/test_exceptions.py +0 -20
- proxyproviders-0.1.0/tests/test_proxy_provider.py +0 -93
- {proxyproviders-0.1.0 → proxyproviders-0.2.0}/LICENSE +0 -0
- {proxyproviders-0.1.0 → proxyproviders-0.2.0}/proxyproviders/models/__init__.py +0 -0
{proxyproviders-0.1.0/proxyproviders/proxyproviders.egg-info → proxyproviders-0.2.0}/PKG-INFO
RENAMED
|
@@ -1,52 +1,52 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: proxyproviders
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A unified interface for different proxy providers
|
|
5
|
-
Author-email: David Teather <contact.davidteather@gmail.com>
|
|
6
5
|
License: MIT License
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
6
|
+
|
|
7
|
+
Copyright (c) 2025 David Teather
|
|
8
|
+
|
|
9
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
10
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
11
|
+
in the Software without restriction, including without limitation the rights
|
|
12
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
13
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
14
|
+
furnished to do so, subject to the following conditions:
|
|
15
|
+
|
|
16
|
+
The above copyright notice and this permission notice shall be included in all
|
|
17
|
+
copies or substantial portions of the Software.
|
|
18
|
+
|
|
19
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
20
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
21
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
22
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
23
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
24
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
|
+
SOFTWARE.
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Author: David Teather
|
|
28
|
+
Author-email: contact.davidteather@gmail.com
|
|
29
|
+
Requires-Python: >=3.9
|
|
28
30
|
Classifier: Development Status :: 3 - Alpha
|
|
29
31
|
Classifier: Intended Audience :: Developers
|
|
30
32
|
Classifier: License :: OSI Approved :: MIT License
|
|
31
33
|
Classifier: Programming Language :: Python :: 3
|
|
32
|
-
Requires-Python: >=3.9
|
|
33
|
-
Description-Content-Type: text/markdown
|
|
34
|
-
License-File: LICENSE
|
|
35
|
-
Requires-Dist: requests
|
|
36
34
|
Provides-Extra: dev
|
|
37
|
-
Requires-Dist:
|
|
38
|
-
Requires-Dist:
|
|
39
|
-
Requires-Dist:
|
|
40
|
-
Requires-Dist:
|
|
41
|
-
Requires-Dist:
|
|
42
|
-
Requires-Dist:
|
|
43
|
-
Requires-Dist: toml; extra == "dev"
|
|
35
|
+
Requires-Dist: black ; extra == "dev"
|
|
36
|
+
Requires-Dist: coverage ; extra == "dev"
|
|
37
|
+
Requires-Dist: mypy ; extra == "dev"
|
|
38
|
+
Requires-Dist: pytest ; extra == "dev"
|
|
39
|
+
Requires-Dist: requests
|
|
40
|
+
Requires-Dist: sphinx ; extra == "dev"
|
|
41
|
+
Requires-Dist: toml ; extra == "dev"
|
|
42
|
+
Requires-Dist: types-requests ; extra == "dev"
|
|
43
|
+
Description-Content-Type: text/markdown
|
|
44
44
|
|
|
45
45
|
# ProxyProviders
|
|
46
46
|
|
|
47
47
|
The Unified Python Proxy API for managing proxies with support for BrightData, WebShare, and more.
|
|
48
48
|
|
|
49
|
-
[](https://codecov.io/gh/davidteather/proxyproviders) [](https://github.com/sponsors/davidteather) [](https://github.com/davidteather/proxyproviders/releases) [](https://github.com/davidteather/proxyproviders/blob/main/LICENSE) [](https://pypi.org/project/proxyproviders/)  [](https://discord.gg/yyPhbfma6f) [](https://www.linkedin.com/in/davidteather/)
|
|
50
50
|
|
|
51
51
|
## Table of Contents
|
|
52
52
|
|
|
@@ -106,6 +106,24 @@ proxies = proxy_provider.list_proxies()
|
|
|
106
106
|
print(proxies)
|
|
107
107
|
```
|
|
108
108
|
|
|
109
|
+
For simple usage, you can get a proxy and use it immediately:
|
|
110
|
+
```py
|
|
111
|
+
from proxyproviders import Webshare
|
|
112
|
+
import requests
|
|
113
|
+
|
|
114
|
+
provider = Webshare(api_key="your-api-key")
|
|
115
|
+
|
|
116
|
+
# Get a proxy (uses RoundRobin by default) and use it with requests
|
|
117
|
+
from proxyproviders.models.proxy import ProxyFormat
|
|
118
|
+
|
|
119
|
+
proxy = provider.get_proxy()
|
|
120
|
+
response = requests.get("https://httpbin.org/ip", proxies=proxy.format(ProxyFormat.REQUESTS))
|
|
121
|
+
print(response.json())
|
|
122
|
+
|
|
123
|
+
# Or in one-line
|
|
124
|
+
response = requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
125
|
+
```
|
|
126
|
+
|
|
109
127
|
Each provider has their own custom options, the `Webshare` class lets you specify url params according to their [api spec](https://apidocs.webshare.io/proxy-list/list#parameters), here's an example which will only return proxies that are based in the US.
|
|
110
128
|
|
|
111
129
|
```py
|
|
@@ -151,32 +169,86 @@ some_function(brightdata)
|
|
|
151
169
|
|
|
152
170
|
Here's a more meaningful example that takes the `Proxy` class and uses it to create a python requests http proxy.
|
|
153
171
|
|
|
172
|
+
#### Simple Usage
|
|
154
173
|
```py
|
|
155
|
-
from proxyproviders import Webshare
|
|
174
|
+
from proxyproviders import Webshare
|
|
175
|
+
from proxyproviders.algorithms import Random, RoundRobin
|
|
176
|
+
from proxyproviders.models.proxy import ProxyFormat
|
|
156
177
|
import requests
|
|
157
|
-
import os
|
|
158
178
|
|
|
179
|
+
provider = Webshare(api_key="your_api_key")
|
|
159
180
|
|
|
160
|
-
|
|
161
|
-
|
|
181
|
+
# Get proxy using default RoundRobin and make request
|
|
182
|
+
proxy = provider.get_proxy()
|
|
183
|
+
response = requests.get("https://httpbin.org/ip", proxies=proxy.format(ProxyFormat.REQUESTS))
|
|
162
184
|
|
|
163
|
-
|
|
164
|
-
|
|
185
|
+
# Or in one-line
|
|
186
|
+
response = requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
187
|
+
```
|
|
165
188
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
189
|
+
#### Built-in Algorithms
|
|
190
|
+
```py
|
|
191
|
+
from proxyproviders import Webshare
|
|
192
|
+
from proxyproviders.algorithms import Random, RoundRobin, First
|
|
170
193
|
|
|
171
|
-
|
|
172
|
-
return r.json()
|
|
194
|
+
provider = Webshare(api_key="your_api_key")
|
|
173
195
|
|
|
174
|
-
|
|
175
|
-
|
|
196
|
+
# Default: RoundRobin (cycles through proxies for load balancing)
|
|
197
|
+
proxy = provider.get_proxy()
|
|
198
|
+
|
|
199
|
+
# Random selection
|
|
200
|
+
proxy = provider.get_proxy(Random())
|
|
201
|
+
|
|
202
|
+
# Always first proxy (deterministic)
|
|
203
|
+
proxy = provider.get_proxy(First())
|
|
204
|
+
|
|
205
|
+
# Algorithm can maintain state when reused
|
|
206
|
+
round_robin = RoundRobin()
|
|
207
|
+
proxy1 = provider.get_proxy(round_robin)
|
|
208
|
+
proxy2 = provider.get_proxy(round_robin) # Next in sequence
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
#### Algorithm State Management
|
|
212
|
+
```py
|
|
213
|
+
from proxyproviders import Webshare
|
|
214
|
+
from proxyproviders.algorithms import RoundRobin, Random
|
|
215
|
+
|
|
216
|
+
provider = Webshare(api_key="your_api_key")
|
|
217
|
+
|
|
218
|
+
# Create reusable algorithm for state management
|
|
219
|
+
round_robin = RoundRobin() # Maintains state across calls
|
|
220
|
+
random_algo = Random() # Stateless but reusable
|
|
221
|
+
|
|
222
|
+
# Each call to round_robin will cycle to next proxy
|
|
223
|
+
for i in range(3):
|
|
224
|
+
proxy = provider.get_proxy(round_robin)
|
|
225
|
+
print(f"RoundRobin {i}: {proxy.proxy_address}")
|
|
176
226
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
227
|
+
# Provider also maintains its own default RoundRobin state when not specified
|
|
228
|
+
for i in range(3):
|
|
229
|
+
proxy = provider.get_proxy() # Uses provider's default RoundRobin
|
|
230
|
+
print(f"Default {i}: {proxy.proxy_address}")
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
#### Custom Algorithms
|
|
234
|
+
```py
|
|
235
|
+
from proxyproviders.algorithms import Algorithm
|
|
236
|
+
from typing import List
|
|
237
|
+
from proxyproviders.models.proxy import Proxy
|
|
238
|
+
|
|
239
|
+
class USProxyAlgorithm(Algorithm):
|
|
240
|
+
"""Prefers US proxies, falls back to first available."""
|
|
241
|
+
|
|
242
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
243
|
+
# Try to find a US proxy
|
|
244
|
+
for proxy in proxies:
|
|
245
|
+
if proxy.country_code == "US":
|
|
246
|
+
return proxy
|
|
247
|
+
# Fall back to first proxy
|
|
248
|
+
return proxies[0]
|
|
249
|
+
|
|
250
|
+
# Use your custom algorithm
|
|
251
|
+
proxy = provider.get_proxy(USProxyAlgorithm())
|
|
180
252
|
```
|
|
181
253
|
|
|
182
254
|
### Making Your Own Proxy Provider
|
|
@@ -222,3 +294,4 @@ provider = MyProxyProvider()
|
|
|
222
294
|
some_function(provider) # calls the function with the provider
|
|
223
295
|
```
|
|
224
296
|
|
|
297
|
+
|
|
@@ -1,52 +1,8 @@
|
|
|
1
|
-
Metadata-Version: 2.2
|
|
2
|
-
Name: proxyproviders
|
|
3
|
-
Version: 0.1.0
|
|
4
|
-
Summary: A unified interface for different proxy providers
|
|
5
|
-
Author-email: David Teather <contact.davidteather@gmail.com>
|
|
6
|
-
License: MIT License
|
|
7
|
-
|
|
8
|
-
Copyright (c) 2025 David Teather
|
|
9
|
-
|
|
10
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
-
in the Software without restriction, including without limitation the rights
|
|
13
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
-
furnished to do so, subject to the following conditions:
|
|
16
|
-
|
|
17
|
-
The above copyright notice and this permission notice shall be included in all
|
|
18
|
-
copies or substantial portions of the Software.
|
|
19
|
-
|
|
20
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
-
SOFTWARE.
|
|
27
|
-
|
|
28
|
-
Classifier: Development Status :: 3 - Alpha
|
|
29
|
-
Classifier: Intended Audience :: Developers
|
|
30
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
31
|
-
Classifier: Programming Language :: Python :: 3
|
|
32
|
-
Requires-Python: >=3.9
|
|
33
|
-
Description-Content-Type: text/markdown
|
|
34
|
-
License-File: LICENSE
|
|
35
|
-
Requires-Dist: requests
|
|
36
|
-
Provides-Extra: dev
|
|
37
|
-
Requires-Dist: pytest; extra == "dev"
|
|
38
|
-
Requires-Dist: sphinx; extra == "dev"
|
|
39
|
-
Requires-Dist: black; extra == "dev"
|
|
40
|
-
Requires-Dist: mypy; extra == "dev"
|
|
41
|
-
Requires-Dist: coverage; extra == "dev"
|
|
42
|
-
Requires-Dist: types-requests; extra == "dev"
|
|
43
|
-
Requires-Dist: toml; extra == "dev"
|
|
44
|
-
|
|
45
1
|
# ProxyProviders
|
|
46
2
|
|
|
47
3
|
The Unified Python Proxy API for managing proxies with support for BrightData, WebShare, and more.
|
|
48
4
|
|
|
49
|
-
[](https://codecov.io/gh/davidteather/proxyproviders) [](https://github.com/sponsors/davidteather) [](https://github.com/davidteather/proxyproviders/releases) [](https://github.com/davidteather/proxyproviders/blob/main/LICENSE) [](https://pypi.org/project/proxyproviders/)  [](https://discord.gg/yyPhbfma6f) [](https://www.linkedin.com/in/davidteather/)
|
|
50
6
|
|
|
51
7
|
## Table of Contents
|
|
52
8
|
|
|
@@ -106,6 +62,24 @@ proxies = proxy_provider.list_proxies()
|
|
|
106
62
|
print(proxies)
|
|
107
63
|
```
|
|
108
64
|
|
|
65
|
+
For simple usage, you can get a proxy and use it immediately:
|
|
66
|
+
```py
|
|
67
|
+
from proxyproviders import Webshare
|
|
68
|
+
import requests
|
|
69
|
+
|
|
70
|
+
provider = Webshare(api_key="your-api-key")
|
|
71
|
+
|
|
72
|
+
# Get a proxy (uses RoundRobin by default) and use it with requests
|
|
73
|
+
from proxyproviders.models.proxy import ProxyFormat
|
|
74
|
+
|
|
75
|
+
proxy = provider.get_proxy()
|
|
76
|
+
response = requests.get("https://httpbin.org/ip", proxies=proxy.format(ProxyFormat.REQUESTS))
|
|
77
|
+
print(response.json())
|
|
78
|
+
|
|
79
|
+
# Or in one-line
|
|
80
|
+
response = requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
81
|
+
```
|
|
82
|
+
|
|
109
83
|
Each provider has their own custom options, the `Webshare` class lets you specify url params according to their [api spec](https://apidocs.webshare.io/proxy-list/list#parameters), here's an example which will only return proxies that are based in the US.
|
|
110
84
|
|
|
111
85
|
```py
|
|
@@ -151,32 +125,86 @@ some_function(brightdata)
|
|
|
151
125
|
|
|
152
126
|
Here's a more meaningful example that takes the `Proxy` class and uses it to create a python requests http proxy.
|
|
153
127
|
|
|
128
|
+
#### Simple Usage
|
|
154
129
|
```py
|
|
155
|
-
from proxyproviders import Webshare
|
|
130
|
+
from proxyproviders import Webshare
|
|
131
|
+
from proxyproviders.algorithms import Random, RoundRobin
|
|
132
|
+
from proxyproviders.models.proxy import ProxyFormat
|
|
156
133
|
import requests
|
|
157
|
-
import os
|
|
158
134
|
|
|
135
|
+
provider = Webshare(api_key="your_api_key")
|
|
136
|
+
|
|
137
|
+
# Get proxy using default RoundRobin and make request
|
|
138
|
+
proxy = provider.get_proxy()
|
|
139
|
+
response = requests.get("https://httpbin.org/ip", proxies=proxy.format(ProxyFormat.REQUESTS))
|
|
140
|
+
|
|
141
|
+
# Or in one-line
|
|
142
|
+
response = requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
#### Built-in Algorithms
|
|
146
|
+
```py
|
|
147
|
+
from proxyproviders import Webshare
|
|
148
|
+
from proxyproviders.algorithms import Random, RoundRobin, First
|
|
159
149
|
|
|
160
|
-
|
|
161
|
-
requests_proxy = None
|
|
150
|
+
provider = Webshare(api_key="your_api_key")
|
|
162
151
|
|
|
163
|
-
|
|
164
|
-
|
|
152
|
+
# Default: RoundRobin (cycles through proxies for load balancing)
|
|
153
|
+
proxy = provider.get_proxy()
|
|
165
154
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
"https": f"http://{proxies[0].username}:{proxies[0].password}@{proxies[0].proxy_address}:{proxies[0].port}",
|
|
169
|
-
}
|
|
155
|
+
# Random selection
|
|
156
|
+
proxy = provider.get_proxy(Random())
|
|
170
157
|
|
|
171
|
-
|
|
172
|
-
|
|
158
|
+
# Always first proxy (deterministic)
|
|
159
|
+
proxy = provider.get_proxy(First())
|
|
173
160
|
|
|
174
|
-
|
|
175
|
-
|
|
161
|
+
# Algorithm can maintain state when reused
|
|
162
|
+
round_robin = RoundRobin()
|
|
163
|
+
proxy1 = provider.get_proxy(round_robin)
|
|
164
|
+
proxy2 = provider.get_proxy(round_robin) # Next in sequence
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
#### Algorithm State Management
|
|
168
|
+
```py
|
|
169
|
+
from proxyproviders import Webshare
|
|
170
|
+
from proxyproviders.algorithms import RoundRobin, Random
|
|
171
|
+
|
|
172
|
+
provider = Webshare(api_key="your_api_key")
|
|
173
|
+
|
|
174
|
+
# Create reusable algorithm for state management
|
|
175
|
+
round_robin = RoundRobin() # Maintains state across calls
|
|
176
|
+
random_algo = Random() # Stateless but reusable
|
|
177
|
+
|
|
178
|
+
# Each call to round_robin will cycle to next proxy
|
|
179
|
+
for i in range(3):
|
|
180
|
+
proxy = provider.get_proxy(round_robin)
|
|
181
|
+
print(f"RoundRobin {i}: {proxy.proxy_address}")
|
|
176
182
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
183
|
+
# Provider also maintains its own default RoundRobin state when not specified
|
|
184
|
+
for i in range(3):
|
|
185
|
+
proxy = provider.get_proxy() # Uses provider's default RoundRobin
|
|
186
|
+
print(f"Default {i}: {proxy.proxy_address}")
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
#### Custom Algorithms
|
|
190
|
+
```py
|
|
191
|
+
from proxyproviders.algorithms import Algorithm
|
|
192
|
+
from typing import List
|
|
193
|
+
from proxyproviders.models.proxy import Proxy
|
|
194
|
+
|
|
195
|
+
class USProxyAlgorithm(Algorithm):
|
|
196
|
+
"""Prefers US proxies, falls back to first available."""
|
|
197
|
+
|
|
198
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
199
|
+
# Try to find a US proxy
|
|
200
|
+
for proxy in proxies:
|
|
201
|
+
if proxy.country_code == "US":
|
|
202
|
+
return proxy
|
|
203
|
+
# Fall back to first proxy
|
|
204
|
+
return proxies[0]
|
|
205
|
+
|
|
206
|
+
# Use your custom algorithm
|
|
207
|
+
proxy = provider.get_proxy(USProxyAlgorithm())
|
|
180
208
|
```
|
|
181
209
|
|
|
182
210
|
### Making Your Own Proxy Provider
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from . import algorithms
|
|
2
|
+
from .models.proxy import Proxy
|
|
3
|
+
from .providers.brightdata import BrightData
|
|
4
|
+
from .providers.webshare import Webshare
|
|
5
|
+
from .proxy_provider import ProxyConfig, ProxyProvider
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"ProxyProvider",
|
|
9
|
+
"ProxyConfig",
|
|
10
|
+
"Proxy",
|
|
11
|
+
"Webshare",
|
|
12
|
+
"BrightData",
|
|
13
|
+
"algorithms",
|
|
14
|
+
]
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Selection algorithms for choosing proxies from a provider."""
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
import threading
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from typing import Dict, List
|
|
7
|
+
|
|
8
|
+
from .models.proxy import Proxy
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Algorithm(ABC):
|
|
12
|
+
"""Base class for proxy selection algorithms.
|
|
13
|
+
|
|
14
|
+
Users can extend this class to create custom proxy selection logic.
|
|
15
|
+
|
|
16
|
+
Example:
|
|
17
|
+
>>> class MyAlgorithm(Algorithm):
|
|
18
|
+
... def select(self, proxies):
|
|
19
|
+
... return proxies[0] # Always return first proxy
|
|
20
|
+
...
|
|
21
|
+
>>> algorithm = MyAlgorithm()
|
|
22
|
+
>>> proxy = provider.get_proxy(algorithm)
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
@abstractmethod
|
|
26
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
27
|
+
"""Select a proxy from the list.
|
|
28
|
+
|
|
29
|
+
:param proxies: List of available proxies
|
|
30
|
+
:return: Selected proxy
|
|
31
|
+
:raises ValueError: If proxy list is empty
|
|
32
|
+
"""
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Random(Algorithm):
|
|
37
|
+
"""Random proxy selection algorithm.
|
|
38
|
+
|
|
39
|
+
Selects a random proxy from the available list on each call.
|
|
40
|
+
|
|
41
|
+
Example:
|
|
42
|
+
>>> algorithm = Random()
|
|
43
|
+
>>> proxy = provider.get_proxy(algorithm)
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
47
|
+
"""Select a random proxy from the list.
|
|
48
|
+
|
|
49
|
+
:param proxies: List of available proxies
|
|
50
|
+
:return: Randomly selected proxy
|
|
51
|
+
:raises ValueError: If proxy list is empty
|
|
52
|
+
"""
|
|
53
|
+
if not proxies:
|
|
54
|
+
raise ValueError("Cannot select from empty proxy list")
|
|
55
|
+
return random.choice(proxies)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class RoundRobin(Algorithm):
|
|
59
|
+
"""Round-robin proxy selection algorithm with state tracking.
|
|
60
|
+
|
|
61
|
+
Cycles through proxies in order, maintaining state across calls.
|
|
62
|
+
Each RoundRobin instance maintains its own independent state.
|
|
63
|
+
|
|
64
|
+
Example:
|
|
65
|
+
>>> algorithm = RoundRobin()
|
|
66
|
+
>>> proxy1 = provider.get_proxy(algorithm)
|
|
67
|
+
>>> proxy2 = provider.get_proxy(algorithm) # Next proxy in sequence
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
def __init__(self):
|
|
71
|
+
"""Initialize round-robin algorithm with state tracking."""
|
|
72
|
+
self._current_index = 0
|
|
73
|
+
self._lock = threading.Lock()
|
|
74
|
+
|
|
75
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
76
|
+
"""Select next proxy in round-robin sequence.
|
|
77
|
+
|
|
78
|
+
:param proxies: List of available proxies
|
|
79
|
+
:return: Next proxy in round-robin sequence
|
|
80
|
+
:raises ValueError: If proxy list is empty
|
|
81
|
+
"""
|
|
82
|
+
if not proxies:
|
|
83
|
+
raise ValueError("Cannot select from empty proxy list")
|
|
84
|
+
|
|
85
|
+
with self._lock:
|
|
86
|
+
selected_proxy = proxies[self._current_index % len(proxies)]
|
|
87
|
+
self._current_index = (self._current_index + 1) % len(proxies)
|
|
88
|
+
|
|
89
|
+
return selected_proxy
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class First(Algorithm):
|
|
93
|
+
"""First proxy selection algorithm.
|
|
94
|
+
|
|
95
|
+
Always selects the first proxy from the list.
|
|
96
|
+
Useful for deterministic behavior or when proxies are pre-sorted.
|
|
97
|
+
|
|
98
|
+
Example:
|
|
99
|
+
>>> algorithm = First()
|
|
100
|
+
>>> proxy = provider.get_proxy(algorithm) # Always first proxy
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
104
|
+
"""Select the first proxy from the list.
|
|
105
|
+
|
|
106
|
+
:param proxies: List of available proxies
|
|
107
|
+
:return: First proxy in the list
|
|
108
|
+
:raises ValueError: If proxy list is empty
|
|
109
|
+
"""
|
|
110
|
+
if not proxies:
|
|
111
|
+
raise ValueError("Cannot select from empty proxy list")
|
|
112
|
+
return proxies[0]
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
All custom exceptions for the ProxyProvider classes are defined here.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ProxyProviderException(Exception):
|
|
9
|
+
"""Base class for all ProxyProvider errors."""
|
|
10
|
+
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ProxyFetchException(ProxyProviderException):
|
|
15
|
+
"""Raised when there is an error fetching proxies from the provider."""
|
|
16
|
+
|
|
17
|
+
def __init__(self, message: str, status_code: Optional[int] = None):
|
|
18
|
+
self.message = message
|
|
19
|
+
self.status_code = status_code
|
|
20
|
+
super().__init__(self.message)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class ProxyConversionException(ProxyProviderException):
|
|
24
|
+
"""Raised when there is an error converting proxy data to a standardized format."""
|
|
25
|
+
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ProxyInvalidResponseException(ProxyProviderException):
|
|
30
|
+
"""Raised when the provider returns an invalid response."""
|
|
31
|
+
|
|
32
|
+
def __init__(self, response: str):
|
|
33
|
+
self.response = response
|
|
34
|
+
self.message = f"Invalid response received: {response}"
|
|
35
|
+
super().__init__(self.message)
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from enum import Enum
|
|
4
|
+
from typing import Dict, List, Optional, Union
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ProxyFormat(Enum):
|
|
8
|
+
"""Supported proxy output formats."""
|
|
9
|
+
|
|
10
|
+
REQUESTS = "requests"
|
|
11
|
+
"""Requests format, for use in requests library HTTP calls"""
|
|
12
|
+
|
|
13
|
+
CURL = "curl"
|
|
14
|
+
"""CURL format, for use in curl commands"""
|
|
15
|
+
|
|
16
|
+
HTTPX = "httpx"
|
|
17
|
+
"""HTTPX format, for use in httpx library HTTP calls"""
|
|
18
|
+
|
|
19
|
+
AIOHTTP = "aiohttp"
|
|
20
|
+
"""AIOHTTP format, for use in aiohttp library HTTP calls"""
|
|
21
|
+
|
|
22
|
+
URL = "url"
|
|
23
|
+
"""URL format, for use in URL strings"""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class Proxy:
|
|
28
|
+
"""Our shared data model for a proxy object across all providers."""
|
|
29
|
+
|
|
30
|
+
id: str
|
|
31
|
+
"""A unique identifier for the proxy"""
|
|
32
|
+
|
|
33
|
+
username: str
|
|
34
|
+
"""The username required for authenticating with the proxy"""
|
|
35
|
+
|
|
36
|
+
password: str
|
|
37
|
+
"""The password required for authenticating with the proxy"""
|
|
38
|
+
|
|
39
|
+
proxy_address: str
|
|
40
|
+
"""The IP address or domain name of the proxy"""
|
|
41
|
+
|
|
42
|
+
port: int
|
|
43
|
+
"""The port number through which the proxy connection is established"""
|
|
44
|
+
|
|
45
|
+
country_code: Optional[str] = None
|
|
46
|
+
"""The country code where the proxy is located, e.g., 'US', 'FR'. Optional"""
|
|
47
|
+
|
|
48
|
+
city_name: Optional[str] = None
|
|
49
|
+
"""The city name where the proxy is located, e.g., 'New York', 'Paris'. Optional"""
|
|
50
|
+
|
|
51
|
+
created_at: Optional[datetime] = None
|
|
52
|
+
"""The timestamp when the proxy was created. Optional"""
|
|
53
|
+
|
|
54
|
+
protocols: Optional[List[str]] = None
|
|
55
|
+
"""A list of connection protocols supported by the proxy, e.g., ['http', 'https']"""
|
|
56
|
+
|
|
57
|
+
def to_url(self, protocol: str = "http") -> str:
|
|
58
|
+
"""Convert proxy to URL format for use with HTTP clients.
|
|
59
|
+
|
|
60
|
+
:param protocol: The protocol to use in the URL (default: 'http')
|
|
61
|
+
:return: Proxy URL in format 'protocol://username:password@host:port'
|
|
62
|
+
|
|
63
|
+
Example:
|
|
64
|
+
>>> proxy.to_url()
|
|
65
|
+
'http://user:pass@192.168.1.1:8080'
|
|
66
|
+
>>> proxy.to_url('https')
|
|
67
|
+
'https://user:pass@192.168.1.1:8080'
|
|
68
|
+
"""
|
|
69
|
+
return f"{protocol}://{self.username}:{self.password}@{self.proxy_address}:{self.port}"
|
|
70
|
+
|
|
71
|
+
def format(
|
|
72
|
+
self, format_type: Union[ProxyFormat, str] = ProxyFormat.URL, **kwargs
|
|
73
|
+
) -> Union[str, Dict[str, str], List[str]]:
|
|
74
|
+
"""Format proxy for different HTTP clients and tools.
|
|
75
|
+
|
|
76
|
+
:param format_type: Output format (default: URL string)
|
|
77
|
+
:param kwargs: Format-specific options
|
|
78
|
+
:return: Formatted proxy data
|
|
79
|
+
|
|
80
|
+
Examples:
|
|
81
|
+
>>> proxy.format() # Default URL string
|
|
82
|
+
'http://user:pass@192.168.1.1:8080'
|
|
83
|
+
|
|
84
|
+
>>> proxy.format(ProxyFormat.REQUESTS)
|
|
85
|
+
{'http': 'http://user:pass@192.168.1.1:8080', 'https': 'http://user:pass@192.168.1.1:8080'}
|
|
86
|
+
|
|
87
|
+
>>> proxy.format('curl') # String also works
|
|
88
|
+
['-x', 'http://user:pass@192.168.1.1:8080']
|
|
89
|
+
|
|
90
|
+
>>> proxy.format(ProxyFormat.HTTPX)
|
|
91
|
+
{'http://': 'http://user:pass@192.168.1.1:8080', 'https://': 'http://user:pass@192.168.1.1:8080'}
|
|
92
|
+
"""
|
|
93
|
+
if isinstance(format_type, str):
|
|
94
|
+
format_type = ProxyFormat(format_type)
|
|
95
|
+
|
|
96
|
+
if format_type == ProxyFormat.URL:
|
|
97
|
+
protocol = kwargs.get("protocol", "http")
|
|
98
|
+
return self.to_url(protocol)
|
|
99
|
+
|
|
100
|
+
elif format_type == ProxyFormat.REQUESTS:
|
|
101
|
+
protocols = kwargs.get("protocols", self.protocols or ["http", "https"])
|
|
102
|
+
proxy_url = self.to_url("http")
|
|
103
|
+
return {protocol: proxy_url for protocol in protocols}
|
|
104
|
+
|
|
105
|
+
elif format_type == ProxyFormat.CURL:
|
|
106
|
+
return ["-x", self.to_url("http")]
|
|
107
|
+
|
|
108
|
+
elif format_type == ProxyFormat.HTTPX:
|
|
109
|
+
# httpx uses 'http://' and 'https://' as keys
|
|
110
|
+
proxy_url = self.to_url("http")
|
|
111
|
+
return {"http://": proxy_url, "https://": proxy_url}
|
|
112
|
+
|
|
113
|
+
elif format_type == ProxyFormat.AIOHTTP:
|
|
114
|
+
# aiohttp takes a single URL string
|
|
115
|
+
return self.to_url("http")
|
|
116
|
+
|
|
117
|
+
else:
|
|
118
|
+
raise ValueError(f"Unsupported format: {format_type}")
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
from typing import Dict, List, Optional
|
|
2
|
+
|
|
1
3
|
import requests
|
|
2
|
-
|
|
3
|
-
from ..proxy_provider import ProxyProvider, ProxyConfig
|
|
4
|
+
|
|
4
5
|
from ..exceptions import ProxyFetchException, ProxyInvalidResponseException
|
|
5
6
|
from ..models.proxy import Proxy
|
|
6
|
-
import
|
|
7
|
+
from ..proxy_provider import ProxyConfig, ProxyProvider
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
class BrightData(ProxyProvider):
|
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
import requests
|
|
2
1
|
from datetime import datetime
|
|
3
|
-
from typing import
|
|
4
|
-
|
|
2
|
+
from typing import Dict, List, Optional
|
|
3
|
+
|
|
4
|
+
import requests
|
|
5
|
+
|
|
5
6
|
from ..exceptions import (
|
|
6
|
-
ProxyFetchException,
|
|
7
7
|
ProxyConversionException,
|
|
8
|
+
ProxyFetchException,
|
|
8
9
|
ProxyInvalidResponseException,
|
|
9
10
|
)
|
|
10
11
|
from ..models.proxy import Proxy
|
|
12
|
+
from ..proxy_provider import ProxyConfig, ProxyProvider
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
class Webshare(ProxyProvider):
|
|
@@ -85,7 +87,7 @@ class Webshare(ProxyProvider):
|
|
|
85
87
|
data = response.json()
|
|
86
88
|
|
|
87
89
|
proxy_data = data.get("results")
|
|
88
|
-
if
|
|
90
|
+
if proxy_data is None:
|
|
89
91
|
raise ProxyInvalidResponseException(response.text)
|
|
90
92
|
|
|
91
93
|
all_proxies.extend([self._convert_to_proxy(proxy) for proxy in proxy_data])
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import threading
|
|
2
|
+
from abc import ABC, abstractmethod
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
from typing import TYPE_CHECKING, List, Optional
|
|
6
|
+
|
|
7
|
+
from .models.proxy import Proxy
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from .algorithms import Algorithm
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class ProxyConfig:
|
|
15
|
+
"""Provides standard shared configuration options for all proxy providers.
|
|
16
|
+
|
|
17
|
+
Example:
|
|
18
|
+
|
|
19
|
+
.. code-block:: python
|
|
20
|
+
|
|
21
|
+
from proxyproviders import Webshare, ProxyConfig
|
|
22
|
+
|
|
23
|
+
config = ProxyConfig(refresh_interval=5)
|
|
24
|
+
proxy_provider = Webshare(api_key="your-api-key", config=config)
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
refresh_interval: float = 600
|
|
28
|
+
"""Duration in seconds after which the proxy list is refreshed to fetch new proxies.
|
|
29
|
+
Set to 0 to disable automatic refreshing after the initial fetch."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class ProxyProvider(ABC):
|
|
33
|
+
"""
|
|
34
|
+
Base class for all proxy providers that provides shared functionality across all providers.
|
|
35
|
+
|
|
36
|
+
You'll need to use a specific provider to fetch proxies and this shouldn't directly be initialized. However, this class is useful for
|
|
37
|
+
type hinting and for creating custom providers that are not included in the library, although consider contributing them to the library if you are making them.
|
|
38
|
+
|
|
39
|
+
:param config: Configuration for the proxy provider. View the ProxyConfig class docs for more information.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(self, config: Optional[ProxyConfig] = None):
|
|
43
|
+
if config is None:
|
|
44
|
+
config = ProxyConfig()
|
|
45
|
+
|
|
46
|
+
self.config: ProxyConfig = config
|
|
47
|
+
self._proxies: Optional[List[Proxy]] = None
|
|
48
|
+
self._last_refresh: Optional[datetime] = None
|
|
49
|
+
self._lock = threading.Lock()
|
|
50
|
+
|
|
51
|
+
# Initialize default algorithm (RoundRobin for state persistence)
|
|
52
|
+
from .algorithms import RoundRobin
|
|
53
|
+
|
|
54
|
+
self._default_algorithm = RoundRobin()
|
|
55
|
+
|
|
56
|
+
def list_proxies(self, force_refresh: bool = False) -> List[Proxy]:
|
|
57
|
+
"""
|
|
58
|
+
Returns the stored proxies.
|
|
59
|
+
If a format function is provided, it applies that function to each proxy.
|
|
60
|
+
|
|
61
|
+
:param force_refresh: If True, fetches new proxies even if the list is not considered stale.
|
|
62
|
+
"""
|
|
63
|
+
if force_refresh or self.should_refresh() or not self._proxies:
|
|
64
|
+
proxies = self._fetch_proxies()
|
|
65
|
+
self._set_proxies(proxies)
|
|
66
|
+
|
|
67
|
+
with self._lock:
|
|
68
|
+
return list(self._proxies) if self._proxies else []
|
|
69
|
+
|
|
70
|
+
def get_proxy(self, algorithm: Optional["Algorithm"] = None) -> Proxy:
|
|
71
|
+
"""Get a single proxy using the specified selection algorithm.
|
|
72
|
+
|
|
73
|
+
:param algorithm: Selection algorithm to use. If None, uses the provider's default RoundRobin algorithm.
|
|
74
|
+
:return: Selected proxy
|
|
75
|
+
:raises ValueError: If no proxies are available
|
|
76
|
+
|
|
77
|
+
Example:
|
|
78
|
+
>>> from proxyproviders import Webshare
|
|
79
|
+
>>> from proxyproviders.algorithms import Random, RoundRobin
|
|
80
|
+
>>>
|
|
81
|
+
>>> provider = Webshare(api_key="your-key")
|
|
82
|
+
>>>
|
|
83
|
+
>>> # Uses default RoundRobin (cycles through proxies)
|
|
84
|
+
>>> proxy = provider.get_proxy()
|
|
85
|
+
>>>
|
|
86
|
+
>>> # Uses Random selection
|
|
87
|
+
>>> proxy = provider.get_proxy(Random())
|
|
88
|
+
>>>
|
|
89
|
+
>>> # One-liner with requests
|
|
90
|
+
>>> import requests
|
|
91
|
+
>>> from proxyproviders.models.proxy import ProxyFormat
|
|
92
|
+
>>> requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
93
|
+
"""
|
|
94
|
+
proxies = self.list_proxies()
|
|
95
|
+
if not proxies:
|
|
96
|
+
raise ValueError("No proxies available from provider")
|
|
97
|
+
|
|
98
|
+
if algorithm is None:
|
|
99
|
+
algorithm = self._default_algorithm
|
|
100
|
+
|
|
101
|
+
return algorithm.select(proxies)
|
|
102
|
+
|
|
103
|
+
#
|
|
104
|
+
# Internal Methods
|
|
105
|
+
#
|
|
106
|
+
@abstractmethod
|
|
107
|
+
def _fetch_proxies(self) -> List[Proxy]:
|
|
108
|
+
"""Fetch proxies from the provider implementation.
|
|
109
|
+
This is not meant to be called directly, as it will always pull from the API even if not stale.
|
|
110
|
+
"""
|
|
111
|
+
pass
|
|
112
|
+
|
|
113
|
+
def should_refresh(self) -> bool:
|
|
114
|
+
"""Returns True if the proxy list should be refreshed based on the refresh_interval."""
|
|
115
|
+
if self._proxies is None or self._last_refresh is None:
|
|
116
|
+
return True
|
|
117
|
+
|
|
118
|
+
if self.config.refresh_interval > 0:
|
|
119
|
+
with self._lock:
|
|
120
|
+
return (
|
|
121
|
+
datetime.now() - self._last_refresh
|
|
122
|
+
).total_seconds() >= self.config.refresh_interval
|
|
123
|
+
|
|
124
|
+
return False
|
|
125
|
+
|
|
126
|
+
def _set_proxies(self, proxies: List[Proxy]):
|
|
127
|
+
"""Sets the list of proxies and updates the last refresh time."""
|
|
128
|
+
with self._lock:
|
|
129
|
+
self._proxies = proxies
|
|
130
|
+
self._last_refresh = datetime.now()
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["
|
|
3
|
-
build-backend = "
|
|
2
|
+
requires = ["poetry-core>=1.0.0"]
|
|
3
|
+
build-backend = "poetry.core.masonry.api"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "proxyproviders"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "A unified interface for different proxy providers"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [{ name = "David Teather", email = "contact.davidteather@gmail.com" }]
|
|
@@ -38,4 +38,3 @@ coverage = "^7.6.12"
|
|
|
38
38
|
types-requests = "^2.32.0.20241016"
|
|
39
39
|
toml = "^0.10.2"
|
|
40
40
|
responses = "^0.25.6"
|
|
41
|
-
|
proxyproviders-0.1.0/README.md
DELETED
|
@@ -1,180 +0,0 @@
|
|
|
1
|
-
# ProxyProviders
|
|
2
|
-
|
|
3
|
-
The Unified Python Proxy API for managing proxies with support for BrightData, WebShare, and more.
|
|
4
|
-
|
|
5
|
-
[](https://www.linkedin.com/in/davidteather/) [](https://github.com/sponsors/davidteather) [](https://github.com/davidteather/proxyproviders/releases) [](https://github.com/davidteather/proxyproviders/blob/main/LICENSE) [](https://pypi.org/project/proxyproviders/)  [](https://discord.gg/yyPhbfma6f)
|
|
6
|
-
|
|
7
|
-
## Table of Contents
|
|
8
|
-
|
|
9
|
-
- [Documentation](#documentation)
|
|
10
|
-
- [Getting Started](#getting-started)
|
|
11
|
-
- [How to Support The Project](#how-to-support-the-project)
|
|
12
|
-
- [Installing](#installing)
|
|
13
|
-
- [Quick Start Guide](#quick-start-guide)
|
|
14
|
-
- [Examples](https://github.com/davidteather/proxyproviders/tree/main/examples)
|
|
15
|
-
|
|
16
|
-
## Documentation
|
|
17
|
-
|
|
18
|
-
You can find the full documentation [here](https://davidteather.github.io/proxyproviders)
|
|
19
|
-
|
|
20
|
-
**Note:** If you want to learn how to web scrape websites check my [free and open-source course for learning everything web scraping](https://github.com/davidteather/everything-web-scraping)
|
|
21
|
-
|
|
22
|
-
### How to Support The Project
|
|
23
|
-
|
|
24
|
-
- Star the repo 😎
|
|
25
|
-
- Consider [sponsoring](https://github.com/sponsors/davidteather) me on GitHub
|
|
26
|
-
- Send me an email or a [LinkedIn](https://www.linkedin.com/in/davidteather/) message about what you're doing with it :D
|
|
27
|
-
- Submit PRs for issues or any new providers/features :)
|
|
28
|
-
|
|
29
|
-
## Getting Started
|
|
30
|
-
|
|
31
|
-
To get started using this package follow the instructions below.
|
|
32
|
-
|
|
33
|
-
### Installation
|
|
34
|
-
|
|
35
|
-
This package is installable via pip.
|
|
36
|
-
```sh
|
|
37
|
-
python -m pip install proxyproviders
|
|
38
|
-
```
|
|
39
|
-
|
|
40
|
-
## Quick Start Guide
|
|
41
|
-
|
|
42
|
-
Here's a quick bit of code to get you started! There's also a few [examples in the folder](https://github.com/davidteather/proxyproviders/tree/main/examples).
|
|
43
|
-
|
|
44
|
-
**Note:** If you want to learn how to web scrape websites check my [free and open-source course for learning everything web scraping](https://github.com/davidteather/everything-web-scraping)
|
|
45
|
-
|
|
46
|
-
### Choose a Proxy Provider
|
|
47
|
-
|
|
48
|
-
If you already haven't, choose which proxy provider to use. You can find a [list in the documentation](https://davidteather.github.io/proxyproviders/#proxyproviders-supported-providers). After choosing, look at the documentation around the specific provider you've choosen. Where needed, we've laid out steps to get api keys in the documentation. These steps will vary slightly by provider. For this example I'll be using the [Webshare Provider](https://davidteather.github.io/proxyproviders/#proxyproviders.providers.webshare.Webshare) because it's both easy to setup and they give you 10 free data center proxies to test out.
|
|
49
|
-
|
|
50
|
-
You can create an account on webshare [here](https://www.webshare.io/?referral_code=3x5812idzzzp) (affiliate link), then head into the [API tab](https://dashboard.webshare.io/userapi/keys) on the side and generate a new token. Keep this API token safe and don't post it publically.
|
|
51
|
-
|
|
52
|
-
### Basic Example
|
|
53
|
-
|
|
54
|
-
After you can list out your proxies with
|
|
55
|
-
```py
|
|
56
|
-
from proxyproviders import Webshare
|
|
57
|
-
|
|
58
|
-
proxy_provider = Webshare(api_key="your-api-key")
|
|
59
|
-
|
|
60
|
-
proxies = proxy_provider.list_proxies()
|
|
61
|
-
|
|
62
|
-
print(proxies)
|
|
63
|
-
```
|
|
64
|
-
|
|
65
|
-
Each provider has their own custom options, the `Webshare` class lets you specify url params according to their [api spec](https://apidocs.webshare.io/proxy-list/list#parameters), here's an example which will only return proxies that are based in the US.
|
|
66
|
-
|
|
67
|
-
```py
|
|
68
|
-
proxy_provider = Webshare(api_key="your-api-key", params={"country_code_in": "US"})
|
|
69
|
-
```
|
|
70
|
-
|
|
71
|
-
### Using ProxyConfig
|
|
72
|
-
|
|
73
|
-
For any shared logic across all types of proxy providers, we use the `ProxyConfig` data class to configure them. The full docs for [ProxyConfig are here](https://davidteather.github.io/proxyproviders/#proxyproviders.proxy_provider.ProxyConfig). In this example we will configure it to use a shorter `refresh_interval` than default.
|
|
74
|
-
|
|
75
|
-
```py
|
|
76
|
-
from proxyproviders import Webshare, ProxyConfig
|
|
77
|
-
import time
|
|
78
|
-
|
|
79
|
-
config = ProxyConfig(refresh_interval=3)
|
|
80
|
-
ws = Webshare(api_key="your-api-token", config=config)
|
|
81
|
-
|
|
82
|
-
proxies = ws.list_proxies() # calls API
|
|
83
|
-
|
|
84
|
-
ws.list_proxies() # cached
|
|
85
|
-
|
|
86
|
-
time.sleep(5)
|
|
87
|
-
ws.list_proxies() # calls API since it's more than 3s later
|
|
88
|
-
```
|
|
89
|
-
|
|
90
|
-
### Function Using Generic ProxyProvider
|
|
91
|
-
|
|
92
|
-
Since all proxy providers implement the same interface, we can make a function that allows us to easily swap out and utilize different providers. This is the main appeal of having a unified interface. It allows other modules to be provider agnostic, like my [TikTokAPI](https://github.com/davidteather/TikTok-Api) package.
|
|
93
|
-
|
|
94
|
-
```py
|
|
95
|
-
from proxyproviders import Webshare, BrightData, ProxyProvider, ProxyConfig
|
|
96
|
-
|
|
97
|
-
def some_function(provider: ProxyProvider):
|
|
98
|
-
proxies = provider.list_proxies()
|
|
99
|
-
print(proxies)
|
|
100
|
-
|
|
101
|
-
webshare = Webshare(api_key="your_api_key")
|
|
102
|
-
brightdata = BrightData(api_key="your_api_key", zone="my_zone")
|
|
103
|
-
|
|
104
|
-
some_function(webshare)
|
|
105
|
-
some_function(brightdata)
|
|
106
|
-
```
|
|
107
|
-
|
|
108
|
-
Here's a more meaningful example that takes the `Proxy` class and uses it to create a python requests http proxy.
|
|
109
|
-
|
|
110
|
-
```py
|
|
111
|
-
from proxyproviders import Webshare, BrightData, ProxyProvider
|
|
112
|
-
import requests
|
|
113
|
-
import os
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
def request_with_proxy(provider: ProxyProvider):
|
|
117
|
-
requests_proxy = None
|
|
118
|
-
|
|
119
|
-
if provider:
|
|
120
|
-
proxies = provider.list_proxies()
|
|
121
|
-
|
|
122
|
-
requests_proxy = {
|
|
123
|
-
"http": f"http://{proxies[0].username}:{proxies[0].password}@{proxies[0].proxy_address}:{proxies[0].port}",
|
|
124
|
-
"https": f"http://{proxies[0].username}:{proxies[0].password}@{proxies[0].proxy_address}:{proxies[0].port}",
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
r = requests.get("https://httpbin.org/ip", proxies=requests_proxy)
|
|
128
|
-
return r.json()
|
|
129
|
-
|
|
130
|
-
webshare = Webshare(api_key="your_api_key")
|
|
131
|
-
brightdata = BrightData(api_key="your_api_key", zone="your_zone")
|
|
132
|
-
|
|
133
|
-
print(f"Your IP: {request_with_proxy(None)}")
|
|
134
|
-
print(f"Webshare: {request_with_proxy(webshare)}")
|
|
135
|
-
print(f"BrightData: {request_with_proxy(brightdata)}")
|
|
136
|
-
```
|
|
137
|
-
|
|
138
|
-
### Making Your Own Proxy Provider
|
|
139
|
-
|
|
140
|
-
Here's a skeleton of how you can make your very own `ProxyProvider` class. You'll need to implemenet all the required functions of the `ProxyProvider` which may be more than what's here at the time of writing.
|
|
141
|
-
|
|
142
|
-
If you do find yourself making one of these, consider contributing it back to the repository so everyone can use them :D
|
|
143
|
-
|
|
144
|
-
```py
|
|
145
|
-
from proxyproviders import ProxyProvider, ProxyConfig, Proxy
|
|
146
|
-
from typing import List, Optional
|
|
147
|
-
|
|
148
|
-
class MyProxyProvider(ProxyProvider):
|
|
149
|
-
def __init__(self, config: Optional[ProxyConfig] = None):
|
|
150
|
-
super().__init__(config=config)
|
|
151
|
-
|
|
152
|
-
def _fetch_proxies(self):
|
|
153
|
-
proxies: List[Proxy] = []
|
|
154
|
-
|
|
155
|
-
for i in range(10):
|
|
156
|
-
# TODO: your real proxy fetching logic
|
|
157
|
-
|
|
158
|
-
# There are required fields on the Proxy class, be sure that these are filled out properly
|
|
159
|
-
# especially if you're using it with another library.
|
|
160
|
-
proxy = Proxy(
|
|
161
|
-
id=str(i),
|
|
162
|
-
username="username",
|
|
163
|
-
password="password",
|
|
164
|
-
proxy_address="192.168.1.1",
|
|
165
|
-
port=80,
|
|
166
|
-
)
|
|
167
|
-
|
|
168
|
-
proxies.append(proxy)
|
|
169
|
-
|
|
170
|
-
return proxies
|
|
171
|
-
|
|
172
|
-
def some_function(provider: ProxyProvider):
|
|
173
|
-
proxies = provider.list_proxies()
|
|
174
|
-
for proxy in proxies:
|
|
175
|
-
print(proxy)
|
|
176
|
-
|
|
177
|
-
provider = MyProxyProvider()
|
|
178
|
-
some_function(provider) # calls the function with the provider
|
|
179
|
-
```
|
|
180
|
-
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
from dataclasses import dataclass
|
|
2
|
-
from typing import Optional, List
|
|
3
|
-
from datetime import datetime
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
@dataclass
|
|
7
|
-
class Proxy:
|
|
8
|
-
"""Our shared data model for a proxy object across all providers."""
|
|
9
|
-
|
|
10
|
-
id: str
|
|
11
|
-
"""A unique identifier for the proxy"""
|
|
12
|
-
|
|
13
|
-
username: str
|
|
14
|
-
"""The username required for authenticating with the proxy"""
|
|
15
|
-
|
|
16
|
-
password: str
|
|
17
|
-
"""The password required for authenticating with the proxy"""
|
|
18
|
-
|
|
19
|
-
proxy_address: str
|
|
20
|
-
"""The IP address or domain name of the proxy"""
|
|
21
|
-
|
|
22
|
-
port: int
|
|
23
|
-
"""The port number through which the proxy connection is established"""
|
|
24
|
-
|
|
25
|
-
country_code: Optional[str] = None
|
|
26
|
-
"""The country code where the proxy is located, e.g., 'US', 'FR'. Optional"""
|
|
27
|
-
|
|
28
|
-
city_name: Optional[str] = None
|
|
29
|
-
"""The city name where the proxy is located, e.g., 'New York', 'Paris'. Optional"""
|
|
30
|
-
|
|
31
|
-
created_at: Optional[datetime] = None
|
|
32
|
-
"""The timestamp when the proxy was created. Optional"""
|
|
33
|
-
|
|
34
|
-
protocols: Optional[List[str]] = None
|
|
35
|
-
"""A list of connection protocols supported by the proxy, e.g., ['http', 'https']"""
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
LICENSE
|
|
2
|
-
README.md
|
|
3
|
-
pyproject.toml
|
|
4
|
-
proxyproviders/models/__init__.py
|
|
5
|
-
proxyproviders/models/proxy.py
|
|
6
|
-
proxyproviders/providers/brightdata.py
|
|
7
|
-
proxyproviders/providers/webshare.py
|
|
8
|
-
proxyproviders/proxyproviders.egg-info/PKG-INFO
|
|
9
|
-
proxyproviders/proxyproviders.egg-info/SOURCES.txt
|
|
10
|
-
proxyproviders/proxyproviders.egg-info/dependency_links.txt
|
|
11
|
-
proxyproviders/proxyproviders.egg-info/requires.txt
|
|
12
|
-
proxyproviders/proxyproviders.egg-info/top_level.txt
|
|
13
|
-
tests/test_exceptions.py
|
|
14
|
-
tests/test_proxy_provider.py
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
|
proxyproviders-0.1.0/setup.cfg
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import pytest
|
|
2
|
-
from proxyproviders.exceptions import ProxyFetchException, ProxyInvalidResponseException
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
def test_proxy_fetch_exception():
|
|
6
|
-
"""
|
|
7
|
-
Test that ProxyFetchException stores the correct message and status code.
|
|
8
|
-
"""
|
|
9
|
-
ex = ProxyFetchException("Fetch error", status_code=500)
|
|
10
|
-
assert "Fetch error" in str(ex)
|
|
11
|
-
assert ex.status_code == 500
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def test_proxy_invalid_response_exception():
|
|
15
|
-
"""
|
|
16
|
-
Test that ProxyInvalidResponseException correctly formats its message.
|
|
17
|
-
"""
|
|
18
|
-
response = '{"error": "Invalid API Key"}'
|
|
19
|
-
ex = ProxyInvalidResponseException(response)
|
|
20
|
-
assert "Invalid response received" in str(ex)
|
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
import pytest
|
|
2
|
-
from proxyproviders.proxy_provider import ProxyProvider, ProxyConfig
|
|
3
|
-
from datetime import timedelta
|
|
4
|
-
from typing import List
|
|
5
|
-
from proxyproviders.models.proxy import Proxy
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
class MockProxyProvider(ProxyProvider):
|
|
9
|
-
"""Mock implementation of ProxyProvider for testing."""
|
|
10
|
-
|
|
11
|
-
def _fetch_proxies(self):
|
|
12
|
-
return []
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def test_should_refresh():
|
|
16
|
-
"""
|
|
17
|
-
Test that the provider correctly determines whether it should refresh proxies.
|
|
18
|
-
"""
|
|
19
|
-
provider = MockProxyProvider(config=ProxyConfig(refresh_interval=10))
|
|
20
|
-
# Initially, no proxies exist so it should refresh.
|
|
21
|
-
assert provider.should_refresh() is True
|
|
22
|
-
# Simulate a fetch to set proxies and update the refresh timestamp.
|
|
23
|
-
provider._set_proxies([])
|
|
24
|
-
# Immediately afterward, it should not refresh because the interval has not passed.
|
|
25
|
-
assert provider.should_refresh() is False
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def test_list_proxies():
|
|
29
|
-
"""
|
|
30
|
-
Test that list_proxies returns a list even if no proxies are available.
|
|
31
|
-
"""
|
|
32
|
-
provider = MockProxyProvider()
|
|
33
|
-
assert isinstance(provider.list_proxies(), list)
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def test_force_refresh():
|
|
37
|
-
"""
|
|
38
|
-
Test that list_proxies fetches new proxies when force_refresh is True.
|
|
39
|
-
"""
|
|
40
|
-
|
|
41
|
-
# Directly subclassing the ProxyProvider for this test
|
|
42
|
-
# for simplicity and to avoid the need for a real provider.
|
|
43
|
-
class TestProxyProvider(MockProxyProvider):
|
|
44
|
-
def _fetch_proxies(self) -> List[Proxy]:
|
|
45
|
-
proxies = [1, 2, 3]
|
|
46
|
-
return proxies
|
|
47
|
-
|
|
48
|
-
provider = TestProxyProvider()
|
|
49
|
-
|
|
50
|
-
print(provider._proxies)
|
|
51
|
-
assert provider._proxies is None
|
|
52
|
-
|
|
53
|
-
proxies = provider.list_proxies(force_refresh=True)
|
|
54
|
-
print(proxies)
|
|
55
|
-
|
|
56
|
-
assert proxies == [1, 2, 3]
|
|
57
|
-
|
|
58
|
-
assert provider.list_proxies() == [1, 2, 3]
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def test_refresh_interval():
|
|
62
|
-
"""
|
|
63
|
-
Tests around the refresh_interval configuration.
|
|
64
|
-
"""
|
|
65
|
-
|
|
66
|
-
# When refresh_interval is 0, it should never refresh automatically
|
|
67
|
-
provider = MockProxyProvider(config=ProxyConfig(refresh_interval=0))
|
|
68
|
-
provider._fetch_proxies = lambda: [1, 2, 3]
|
|
69
|
-
|
|
70
|
-
# Simulate a fetch and set proxies to check refresh
|
|
71
|
-
provider._set_proxies([1, 2, 3])
|
|
72
|
-
assert provider.list_proxies() == [1, 2, 3]
|
|
73
|
-
|
|
74
|
-
# Should not refresh since refresh_interval is 0
|
|
75
|
-
assert provider.should_refresh() is False
|
|
76
|
-
|
|
77
|
-
# Simulate a time lapse, but the proxies should not refresh
|
|
78
|
-
provider._last_refresh = provider._last_refresh - timedelta(seconds=1)
|
|
79
|
-
assert provider.should_refresh() is False
|
|
80
|
-
|
|
81
|
-
# When refresh_interval is set to a value, refresh should happen after the specified interval
|
|
82
|
-
provider = MockProxyProvider(config=ProxyConfig(refresh_interval=10))
|
|
83
|
-
provider._fetch_proxies = lambda: [1, 2, 3]
|
|
84
|
-
provider._set_proxies([1, 2, 3])
|
|
85
|
-
assert provider.list_proxies() == [1, 2, 3]
|
|
86
|
-
|
|
87
|
-
# Simulate the interval passing (10 seconds)
|
|
88
|
-
provider._last_refresh = provider._last_refresh - timedelta(seconds=11)
|
|
89
|
-
assert provider.should_refresh() is True
|
|
90
|
-
|
|
91
|
-
# Fetch new proxies after interval has passed
|
|
92
|
-
provider._set_proxies([4, 5, 6])
|
|
93
|
-
assert provider.list_proxies() == [4, 5, 6]
|
|
File without changes
|
|
File without changes
|