proxyproviders 0.1.1__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {proxyproviders-0.1.1 → proxyproviders-0.2.1}/PKG-INFO +95 -20
- {proxyproviders-0.1.1 → proxyproviders-0.2.1}/README.md +92 -18
- proxyproviders-0.2.1/proxyproviders/__init__.py +16 -0
- proxyproviders-0.2.1/proxyproviders/algorithms.py +112 -0
- proxyproviders-0.2.1/proxyproviders/models/proxy.py +149 -0
- {proxyproviders-0.1.1 → proxyproviders-0.2.1}/proxyproviders/providers/brightdata.py +4 -3
- {proxyproviders-0.1.1 → proxyproviders-0.2.1}/proxyproviders/providers/webshare.py +7 -5
- {proxyproviders-0.1.1 → proxyproviders-0.2.1}/proxyproviders/proxy_provider.py +45 -3
- proxyproviders-0.2.1/proxyproviders/version.py +3 -0
- {proxyproviders-0.1.1 → proxyproviders-0.2.1}/pyproject.toml +1 -1
- proxyproviders-0.1.1/proxyproviders/__init__.py +0 -6
- proxyproviders-0.1.1/proxyproviders/models/proxy.py +0 -35
- {proxyproviders-0.1.1 → proxyproviders-0.2.1}/LICENSE +0 -0
- {proxyproviders-0.1.1 → proxyproviders-0.2.1}/proxyproviders/exceptions.py +0 -0
- {proxyproviders-0.1.1 → proxyproviders-0.2.1}/proxyproviders/models/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: proxyproviders
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: A unified interface for different proxy providers
|
|
5
5
|
License: MIT License
|
|
6
6
|
|
|
@@ -23,6 +23,7 @@ License: MIT License
|
|
|
23
23
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
24
24
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
25
|
SOFTWARE.
|
|
26
|
+
License-File: LICENSE
|
|
26
27
|
Author: David Teather
|
|
27
28
|
Author-email: contact.davidteather@gmail.com
|
|
28
29
|
Requires-Python: >=3.9
|
|
@@ -45,7 +46,7 @@ Description-Content-Type: text/markdown
|
|
|
45
46
|
|
|
46
47
|
The Unified Python Proxy API for managing proxies with support for BrightData, WebShare, and more.
|
|
47
48
|
|
|
48
|
-
[](https://codecov.io/gh/davidteather/proxyproviders) [](https://github.com/sponsors/davidteather) [](https://github.com/davidteather/proxyproviders/releases) [](https://github.com/davidteather/proxyproviders/blob/main/LICENSE) [](https://pypi.org/project/proxyproviders/)  [](https://discord.gg/yyPhbfma6f) [](https://www.linkedin.com/in/davidteather/)
|
|
49
50
|
|
|
50
51
|
## Table of Contents
|
|
51
52
|
|
|
@@ -84,6 +85,8 @@ python -m pip install proxyproviders
|
|
|
84
85
|
|
|
85
86
|
Here's a quick bit of code to get you started! There's also a few [examples in the folder](https://github.com/davidteather/proxyproviders/tree/main/examples).
|
|
86
87
|
|
|
88
|
+
**📺 Video Tutorial:** [Watch the quick start guide on YouTube](https://youtu.be/h-JouO_orYo) for a walkthrough of getting started with ProxyProviders.
|
|
89
|
+
|
|
87
90
|
**Note:** If you want to learn how to web scrape websites check my [free and open-source course for learning everything web scraping](https://github.com/davidteather/everything-web-scraping)
|
|
88
91
|
|
|
89
92
|
### Choose a Proxy Provider
|
|
@@ -105,6 +108,24 @@ proxies = proxy_provider.list_proxies()
|
|
|
105
108
|
print(proxies)
|
|
106
109
|
```
|
|
107
110
|
|
|
111
|
+
For simple usage, you can get a proxy and use it immediately:
|
|
112
|
+
```py
|
|
113
|
+
from proxyproviders import Webshare
|
|
114
|
+
import requests
|
|
115
|
+
|
|
116
|
+
provider = Webshare(api_key="your-api-key")
|
|
117
|
+
|
|
118
|
+
# Get a proxy (uses RoundRobin by default) and use it with requests
|
|
119
|
+
from proxyproviders.models.proxy import ProxyFormat
|
|
120
|
+
|
|
121
|
+
proxy = provider.get_proxy()
|
|
122
|
+
response = requests.get("https://httpbin.org/ip", proxies=proxy.format(ProxyFormat.REQUESTS))
|
|
123
|
+
print(response.json())
|
|
124
|
+
|
|
125
|
+
# Or in one-line
|
|
126
|
+
response = requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
127
|
+
```
|
|
128
|
+
|
|
108
129
|
Each provider has their own custom options, the `Webshare` class lets you specify url params according to their [api spec](https://apidocs.webshare.io/proxy-list/list#parameters), here's an example which will only return proxies that are based in the US.
|
|
109
130
|
|
|
110
131
|
```py
|
|
@@ -150,32 +171,86 @@ some_function(brightdata)
|
|
|
150
171
|
|
|
151
172
|
Here's a more meaningful example that takes the `Proxy` class and uses it to create a python requests http proxy.
|
|
152
173
|
|
|
174
|
+
#### Simple Usage
|
|
153
175
|
```py
|
|
154
|
-
from proxyproviders import Webshare
|
|
176
|
+
from proxyproviders import Webshare
|
|
177
|
+
from proxyproviders.algorithms import Random, RoundRobin
|
|
178
|
+
from proxyproviders.models.proxy import ProxyFormat
|
|
155
179
|
import requests
|
|
156
|
-
import os
|
|
157
180
|
|
|
181
|
+
provider = Webshare(api_key="your_api_key")
|
|
182
|
+
|
|
183
|
+
# Get proxy using default RoundRobin and make request
|
|
184
|
+
proxy = provider.get_proxy()
|
|
185
|
+
response = requests.get("https://httpbin.org/ip", proxies=proxy.format(ProxyFormat.REQUESTS))
|
|
186
|
+
|
|
187
|
+
# Or in one-line
|
|
188
|
+
response = requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
#### Built-in Algorithms
|
|
192
|
+
```py
|
|
193
|
+
from proxyproviders import Webshare
|
|
194
|
+
from proxyproviders.algorithms import Random, RoundRobin, First
|
|
158
195
|
|
|
159
|
-
|
|
160
|
-
requests_proxy = None
|
|
196
|
+
provider = Webshare(api_key="your_api_key")
|
|
161
197
|
|
|
162
|
-
|
|
163
|
-
|
|
198
|
+
# Default: RoundRobin (cycles through proxies for load balancing)
|
|
199
|
+
proxy = provider.get_proxy()
|
|
164
200
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
"https": f"http://{proxies[0].username}:{proxies[0].password}@{proxies[0].proxy_address}:{proxies[0].port}",
|
|
168
|
-
}
|
|
201
|
+
# Random selection
|
|
202
|
+
proxy = provider.get_proxy(Random())
|
|
169
203
|
|
|
170
|
-
|
|
171
|
-
|
|
204
|
+
# Always first proxy (deterministic)
|
|
205
|
+
proxy = provider.get_proxy(First())
|
|
172
206
|
|
|
173
|
-
|
|
174
|
-
|
|
207
|
+
# Algorithm can maintain state when reused
|
|
208
|
+
round_robin = RoundRobin()
|
|
209
|
+
proxy1 = provider.get_proxy(round_robin)
|
|
210
|
+
proxy2 = provider.get_proxy(round_robin) # Next in sequence
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
#### Algorithm State Management
|
|
214
|
+
```py
|
|
215
|
+
from proxyproviders import Webshare
|
|
216
|
+
from proxyproviders.algorithms import RoundRobin, Random
|
|
217
|
+
|
|
218
|
+
provider = Webshare(api_key="your_api_key")
|
|
219
|
+
|
|
220
|
+
# Create reusable algorithm for state management
|
|
221
|
+
round_robin = RoundRobin() # Maintains state across calls
|
|
222
|
+
random_algo = Random() # Stateless but reusable
|
|
223
|
+
|
|
224
|
+
# Each call to round_robin will cycle to next proxy
|
|
225
|
+
for i in range(3):
|
|
226
|
+
proxy = provider.get_proxy(round_robin)
|
|
227
|
+
print(f"RoundRobin {i}: {proxy.proxy_address}")
|
|
175
228
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
229
|
+
# Provider also maintains its own default RoundRobin state when not specified
|
|
230
|
+
for i in range(3):
|
|
231
|
+
proxy = provider.get_proxy() # Uses provider's default RoundRobin
|
|
232
|
+
print(f"Default {i}: {proxy.proxy_address}")
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
#### Custom Algorithms
|
|
236
|
+
```py
|
|
237
|
+
from proxyproviders.algorithms import Algorithm
|
|
238
|
+
from typing import List
|
|
239
|
+
from proxyproviders.models.proxy import Proxy
|
|
240
|
+
|
|
241
|
+
class USProxyAlgorithm(Algorithm):
|
|
242
|
+
"""Prefers US proxies, falls back to first available."""
|
|
243
|
+
|
|
244
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
245
|
+
# Try to find a US proxy
|
|
246
|
+
for proxy in proxies:
|
|
247
|
+
if proxy.country_code == "US":
|
|
248
|
+
return proxy
|
|
249
|
+
# Fall back to first proxy
|
|
250
|
+
return proxies[0]
|
|
251
|
+
|
|
252
|
+
# Use your custom algorithm
|
|
253
|
+
proxy = provider.get_proxy(USProxyAlgorithm())
|
|
179
254
|
```
|
|
180
255
|
|
|
181
256
|
### Making Your Own Proxy Provider
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
The Unified Python Proxy API for managing proxies with support for BrightData, WebShare, and more.
|
|
4
4
|
|
|
5
|
-
[](https://codecov.io/gh/davidteather/proxyproviders) [](https://github.com/sponsors/davidteather) [](https://github.com/davidteather/proxyproviders/releases) [](https://github.com/davidteather/proxyproviders/blob/main/LICENSE) [](https://pypi.org/project/proxyproviders/)  [](https://discord.gg/yyPhbfma6f) [](https://www.linkedin.com/in/davidteather/)
|
|
6
6
|
|
|
7
7
|
## Table of Contents
|
|
8
8
|
|
|
@@ -41,6 +41,8 @@ python -m pip install proxyproviders
|
|
|
41
41
|
|
|
42
42
|
Here's a quick bit of code to get you started! There's also a few [examples in the folder](https://github.com/davidteather/proxyproviders/tree/main/examples).
|
|
43
43
|
|
|
44
|
+
**📺 Video Tutorial:** [Watch the quick start guide on YouTube](https://youtu.be/h-JouO_orYo) for a walkthrough of getting started with ProxyProviders.
|
|
45
|
+
|
|
44
46
|
**Note:** If you want to learn how to web scrape websites check my [free and open-source course for learning everything web scraping](https://github.com/davidteather/everything-web-scraping)
|
|
45
47
|
|
|
46
48
|
### Choose a Proxy Provider
|
|
@@ -62,6 +64,24 @@ proxies = proxy_provider.list_proxies()
|
|
|
62
64
|
print(proxies)
|
|
63
65
|
```
|
|
64
66
|
|
|
67
|
+
For simple usage, you can get a proxy and use it immediately:
|
|
68
|
+
```py
|
|
69
|
+
from proxyproviders import Webshare
|
|
70
|
+
import requests
|
|
71
|
+
|
|
72
|
+
provider = Webshare(api_key="your-api-key")
|
|
73
|
+
|
|
74
|
+
# Get a proxy (uses RoundRobin by default) and use it with requests
|
|
75
|
+
from proxyproviders.models.proxy import ProxyFormat
|
|
76
|
+
|
|
77
|
+
proxy = provider.get_proxy()
|
|
78
|
+
response = requests.get("https://httpbin.org/ip", proxies=proxy.format(ProxyFormat.REQUESTS))
|
|
79
|
+
print(response.json())
|
|
80
|
+
|
|
81
|
+
# Or in one-line
|
|
82
|
+
response = requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
83
|
+
```
|
|
84
|
+
|
|
65
85
|
Each provider has their own custom options, the `Webshare` class lets you specify url params according to their [api spec](https://apidocs.webshare.io/proxy-list/list#parameters), here's an example which will only return proxies that are based in the US.
|
|
66
86
|
|
|
67
87
|
```py
|
|
@@ -107,32 +127,86 @@ some_function(brightdata)
|
|
|
107
127
|
|
|
108
128
|
Here's a more meaningful example that takes the `Proxy` class and uses it to create a python requests http proxy.
|
|
109
129
|
|
|
130
|
+
#### Simple Usage
|
|
110
131
|
```py
|
|
111
|
-
from proxyproviders import Webshare
|
|
132
|
+
from proxyproviders import Webshare
|
|
133
|
+
from proxyproviders.algorithms import Random, RoundRobin
|
|
134
|
+
from proxyproviders.models.proxy import ProxyFormat
|
|
112
135
|
import requests
|
|
113
|
-
import os
|
|
114
136
|
|
|
137
|
+
provider = Webshare(api_key="your_api_key")
|
|
138
|
+
|
|
139
|
+
# Get proxy using default RoundRobin and make request
|
|
140
|
+
proxy = provider.get_proxy()
|
|
141
|
+
response = requests.get("https://httpbin.org/ip", proxies=proxy.format(ProxyFormat.REQUESTS))
|
|
142
|
+
|
|
143
|
+
# Or in one-line
|
|
144
|
+
response = requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
#### Built-in Algorithms
|
|
148
|
+
```py
|
|
149
|
+
from proxyproviders import Webshare
|
|
150
|
+
from proxyproviders.algorithms import Random, RoundRobin, First
|
|
115
151
|
|
|
116
|
-
|
|
117
|
-
requests_proxy = None
|
|
152
|
+
provider = Webshare(api_key="your_api_key")
|
|
118
153
|
|
|
119
|
-
|
|
120
|
-
|
|
154
|
+
# Default: RoundRobin (cycles through proxies for load balancing)
|
|
155
|
+
proxy = provider.get_proxy()
|
|
121
156
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
"https": f"http://{proxies[0].username}:{proxies[0].password}@{proxies[0].proxy_address}:{proxies[0].port}",
|
|
125
|
-
}
|
|
157
|
+
# Random selection
|
|
158
|
+
proxy = provider.get_proxy(Random())
|
|
126
159
|
|
|
127
|
-
|
|
128
|
-
|
|
160
|
+
# Always first proxy (deterministic)
|
|
161
|
+
proxy = provider.get_proxy(First())
|
|
129
162
|
|
|
130
|
-
|
|
131
|
-
|
|
163
|
+
# Algorithm can maintain state when reused
|
|
164
|
+
round_robin = RoundRobin()
|
|
165
|
+
proxy1 = provider.get_proxy(round_robin)
|
|
166
|
+
proxy2 = provider.get_proxy(round_robin) # Next in sequence
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
#### Algorithm State Management
|
|
170
|
+
```py
|
|
171
|
+
from proxyproviders import Webshare
|
|
172
|
+
from proxyproviders.algorithms import RoundRobin, Random
|
|
173
|
+
|
|
174
|
+
provider = Webshare(api_key="your_api_key")
|
|
175
|
+
|
|
176
|
+
# Create reusable algorithm for state management
|
|
177
|
+
round_robin = RoundRobin() # Maintains state across calls
|
|
178
|
+
random_algo = Random() # Stateless but reusable
|
|
179
|
+
|
|
180
|
+
# Each call to round_robin will cycle to next proxy
|
|
181
|
+
for i in range(3):
|
|
182
|
+
proxy = provider.get_proxy(round_robin)
|
|
183
|
+
print(f"RoundRobin {i}: {proxy.proxy_address}")
|
|
132
184
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
185
|
+
# Provider also maintains its own default RoundRobin state when not specified
|
|
186
|
+
for i in range(3):
|
|
187
|
+
proxy = provider.get_proxy() # Uses provider's default RoundRobin
|
|
188
|
+
print(f"Default {i}: {proxy.proxy_address}")
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
#### Custom Algorithms
|
|
192
|
+
```py
|
|
193
|
+
from proxyproviders.algorithms import Algorithm
|
|
194
|
+
from typing import List
|
|
195
|
+
from proxyproviders.models.proxy import Proxy
|
|
196
|
+
|
|
197
|
+
class USProxyAlgorithm(Algorithm):
|
|
198
|
+
"""Prefers US proxies, falls back to first available."""
|
|
199
|
+
|
|
200
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
201
|
+
# Try to find a US proxy
|
|
202
|
+
for proxy in proxies:
|
|
203
|
+
if proxy.country_code == "US":
|
|
204
|
+
return proxy
|
|
205
|
+
# Fall back to first proxy
|
|
206
|
+
return proxies[0]
|
|
207
|
+
|
|
208
|
+
# Use your custom algorithm
|
|
209
|
+
proxy = provider.get_proxy(USProxyAlgorithm())
|
|
136
210
|
```
|
|
137
211
|
|
|
138
212
|
### Making Your Own Proxy Provider
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from . import algorithms
|
|
2
|
+
from .models.proxy import Proxy, ProxyFormat
|
|
3
|
+
from .providers.brightdata import BrightData
|
|
4
|
+
from .providers.webshare import Webshare
|
|
5
|
+
from .proxy_provider import ProxyConfig, ProxyProvider
|
|
6
|
+
from .version import __version__
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"ProxyProvider",
|
|
10
|
+
"ProxyConfig",
|
|
11
|
+
"Proxy",
|
|
12
|
+
"ProxyFormat",
|
|
13
|
+
"Webshare",
|
|
14
|
+
"BrightData",
|
|
15
|
+
"algorithms",
|
|
16
|
+
]
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Selection algorithms for choosing proxies from a provider."""
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
import threading
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from typing import Dict, List
|
|
7
|
+
|
|
8
|
+
from .models.proxy import Proxy
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Algorithm(ABC):
|
|
12
|
+
"""Base class for proxy selection algorithms.
|
|
13
|
+
|
|
14
|
+
Users can extend this class to create custom proxy selection logic.
|
|
15
|
+
|
|
16
|
+
Example:
|
|
17
|
+
>>> class MyAlgorithm(Algorithm):
|
|
18
|
+
... def select(self, proxies):
|
|
19
|
+
... return proxies[0] # Always return first proxy
|
|
20
|
+
...
|
|
21
|
+
>>> algorithm = MyAlgorithm()
|
|
22
|
+
>>> proxy = provider.get_proxy(algorithm)
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
@abstractmethod
|
|
26
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
27
|
+
"""Select a proxy from the list.
|
|
28
|
+
|
|
29
|
+
:param proxies: List of available proxies
|
|
30
|
+
:return: Selected proxy
|
|
31
|
+
:raises ValueError: If proxy list is empty
|
|
32
|
+
"""
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Random(Algorithm):
|
|
37
|
+
"""Random proxy selection algorithm.
|
|
38
|
+
|
|
39
|
+
Selects a random proxy from the available list on each call.
|
|
40
|
+
|
|
41
|
+
Example:
|
|
42
|
+
>>> algorithm = Random()
|
|
43
|
+
>>> proxy = provider.get_proxy(algorithm)
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
47
|
+
"""Select a random proxy from the list.
|
|
48
|
+
|
|
49
|
+
:param proxies: List of available proxies
|
|
50
|
+
:return: Randomly selected proxy
|
|
51
|
+
:raises ValueError: If proxy list is empty
|
|
52
|
+
"""
|
|
53
|
+
if not proxies:
|
|
54
|
+
raise ValueError("Cannot select from empty proxy list")
|
|
55
|
+
return random.choice(proxies)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class RoundRobin(Algorithm):
|
|
59
|
+
"""Round-robin proxy selection algorithm with state tracking.
|
|
60
|
+
|
|
61
|
+
Cycles through proxies in order, maintaining state across calls.
|
|
62
|
+
Each RoundRobin instance maintains its own independent state.
|
|
63
|
+
|
|
64
|
+
Example:
|
|
65
|
+
>>> algorithm = RoundRobin()
|
|
66
|
+
>>> proxy1 = provider.get_proxy(algorithm)
|
|
67
|
+
>>> proxy2 = provider.get_proxy(algorithm) # Next proxy in sequence
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
def __init__(self):
|
|
71
|
+
"""Initialize round-robin algorithm with state tracking."""
|
|
72
|
+
self._current_index = 0
|
|
73
|
+
self._lock = threading.Lock()
|
|
74
|
+
|
|
75
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
76
|
+
"""Select next proxy in round-robin sequence.
|
|
77
|
+
|
|
78
|
+
:param proxies: List of available proxies
|
|
79
|
+
:return: Next proxy in round-robin sequence
|
|
80
|
+
:raises ValueError: If proxy list is empty
|
|
81
|
+
"""
|
|
82
|
+
if not proxies:
|
|
83
|
+
raise ValueError("Cannot select from empty proxy list")
|
|
84
|
+
|
|
85
|
+
with self._lock:
|
|
86
|
+
selected_proxy = proxies[self._current_index % len(proxies)]
|
|
87
|
+
self._current_index = (self._current_index + 1) % len(proxies)
|
|
88
|
+
|
|
89
|
+
return selected_proxy
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class First(Algorithm):
|
|
93
|
+
"""First proxy selection algorithm.
|
|
94
|
+
|
|
95
|
+
Always selects the first proxy from the list.
|
|
96
|
+
Useful for deterministic behavior or when proxies are pre-sorted.
|
|
97
|
+
|
|
98
|
+
Example:
|
|
99
|
+
>>> algorithm = First()
|
|
100
|
+
>>> proxy = provider.get_proxy(algorithm) # Always first proxy
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
104
|
+
"""Select the first proxy from the list.
|
|
105
|
+
|
|
106
|
+
:param proxies: List of available proxies
|
|
107
|
+
:return: First proxy in the list
|
|
108
|
+
:raises ValueError: If proxy list is empty
|
|
109
|
+
"""
|
|
110
|
+
if not proxies:
|
|
111
|
+
raise ValueError("Cannot select from empty proxy list")
|
|
112
|
+
return proxies[0]
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from enum import Enum
|
|
4
|
+
from typing import Dict, List, Optional, Union
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ProxyFormat(Enum):
|
|
8
|
+
"""Supported proxy output formats."""
|
|
9
|
+
|
|
10
|
+
REQUESTS = "requests"
|
|
11
|
+
"""Requests format, for use in requests library HTTP calls"""
|
|
12
|
+
|
|
13
|
+
CURL = "curl"
|
|
14
|
+
"""CURL format, for use in curl commands"""
|
|
15
|
+
|
|
16
|
+
HTTPX = "httpx"
|
|
17
|
+
"""HTTPX format, for use in httpx library HTTP calls"""
|
|
18
|
+
|
|
19
|
+
AIOHTTP = "aiohttp"
|
|
20
|
+
"""AIOHTTP format, for use in aiohttp library HTTP calls"""
|
|
21
|
+
|
|
22
|
+
PLAYWRIGHT = "playwright"
|
|
23
|
+
"""Playwright format, for use in Playwright browser automation"""
|
|
24
|
+
|
|
25
|
+
URL = "url"
|
|
26
|
+
"""URL format, for use in URL strings"""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class Proxy:
|
|
31
|
+
"""Our shared data model for a proxy object across all providers."""
|
|
32
|
+
|
|
33
|
+
id: str
|
|
34
|
+
"""A unique identifier for the proxy"""
|
|
35
|
+
|
|
36
|
+
username: str
|
|
37
|
+
"""The username required for authenticating with the proxy"""
|
|
38
|
+
|
|
39
|
+
password: str
|
|
40
|
+
"""The password required for authenticating with the proxy"""
|
|
41
|
+
|
|
42
|
+
proxy_address: str
|
|
43
|
+
"""The IP address or domain name of the proxy"""
|
|
44
|
+
|
|
45
|
+
port: int
|
|
46
|
+
"""The port number through which the proxy connection is established"""
|
|
47
|
+
|
|
48
|
+
country_code: Optional[str] = None
|
|
49
|
+
"""The country code where the proxy is located, e.g., 'US', 'FR'. Optional"""
|
|
50
|
+
|
|
51
|
+
city_name: Optional[str] = None
|
|
52
|
+
"""The city name where the proxy is located, e.g., 'New York', 'Paris'. Optional"""
|
|
53
|
+
|
|
54
|
+
created_at: Optional[datetime] = None
|
|
55
|
+
"""The timestamp when the proxy was created. Optional"""
|
|
56
|
+
|
|
57
|
+
protocols: Optional[List[str]] = None
|
|
58
|
+
"""A list of connection protocols supported by the proxy, e.g., ['http', 'https']"""
|
|
59
|
+
|
|
60
|
+
def to_url(self, protocol: str = "http") -> str:
|
|
61
|
+
"""Convert proxy to URL format for use with HTTP clients.
|
|
62
|
+
|
|
63
|
+
:param protocol: The protocol to use in the URL (default: 'http')
|
|
64
|
+
:return: Proxy URL in format 'protocol://username:password@host:port'
|
|
65
|
+
|
|
66
|
+
Example:
|
|
67
|
+
>>> proxy.to_url()
|
|
68
|
+
'http://user:pass@192.168.1.1:8080'
|
|
69
|
+
>>> proxy.to_url('https')
|
|
70
|
+
'https://user:pass@192.168.1.1:8080'
|
|
71
|
+
"""
|
|
72
|
+
return f"{protocol}://{self.username}:{self.password}@{self.proxy_address}:{self.port}"
|
|
73
|
+
|
|
74
|
+
def format(
|
|
75
|
+
self, format_type: Union[ProxyFormat, str] = ProxyFormat.URL, **kwargs
|
|
76
|
+
) -> Union[str, Dict[str, str], List[str]]:
|
|
77
|
+
"""Format proxy for different HTTP clients and tools.
|
|
78
|
+
|
|
79
|
+
:param format_type: Output format (default: URL string)
|
|
80
|
+
:param kwargs: Format-specific options
|
|
81
|
+
:return: Formatted proxy data
|
|
82
|
+
|
|
83
|
+
Examples:
|
|
84
|
+
>>> proxy.format() # Default URL string
|
|
85
|
+
'http://user:pass@192.168.1.1:8080'
|
|
86
|
+
|
|
87
|
+
>>> proxy.format(ProxyFormat.REQUESTS)
|
|
88
|
+
{'http': 'http://user:pass@192.168.1.1:8080', 'https': 'http://user:pass@192.168.1.1:8080'}
|
|
89
|
+
|
|
90
|
+
>>> proxy.format('curl') # String also works
|
|
91
|
+
['-x', 'http://user:pass@192.168.1.1:8080']
|
|
92
|
+
|
|
93
|
+
>>> proxy.format(ProxyFormat.HTTPX)
|
|
94
|
+
{'http://': 'http://user:pass@192.168.1.1:8080', 'https://': 'http://user:pass@192.168.1.1:8080'}
|
|
95
|
+
"""
|
|
96
|
+
# Convert to ProxyFormat enum, handling both string and enum inputs
|
|
97
|
+
if isinstance(format_type, str):
|
|
98
|
+
try:
|
|
99
|
+
format_type = ProxyFormat(format_type)
|
|
100
|
+
except ValueError:
|
|
101
|
+
raise ValueError(
|
|
102
|
+
f"Invalid format type: '{format_type}'. Valid options are: {[f.value for f in ProxyFormat]}"
|
|
103
|
+
)
|
|
104
|
+
elif not isinstance(format_type, ProxyFormat):
|
|
105
|
+
raise ValueError( # pyright: ignore[reportUnreachable] this is actually reachable
|
|
106
|
+
f"Invalid format type: {type(format_type).__name__}. Expected ProxyFormat enum or string."
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
format_handlers = {
|
|
110
|
+
ProxyFormat.URL: lambda: self.to_url(kwargs.get("protocol", "http")),
|
|
111
|
+
ProxyFormat.REQUESTS: lambda: self._format_requests(kwargs),
|
|
112
|
+
ProxyFormat.CURL: lambda: ["-x", self.to_url("http")],
|
|
113
|
+
ProxyFormat.HTTPX: lambda: self._format_httpx(),
|
|
114
|
+
ProxyFormat.AIOHTTP: lambda: self.to_url("http"),
|
|
115
|
+
ProxyFormat.PLAYWRIGHT: lambda: self._format_playwright(**kwargs),
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
handler = format_handlers[format_type]
|
|
119
|
+
return handler()
|
|
120
|
+
|
|
121
|
+
def _format_requests(self, kwargs):
|
|
122
|
+
"""Format proxy for requests library."""
|
|
123
|
+
protocols = kwargs.get("protocols", self.protocols or ["http", "https"])
|
|
124
|
+
proxy_url = self.to_url("http")
|
|
125
|
+
return {protocol: proxy_url for protocol in protocols}
|
|
126
|
+
|
|
127
|
+
def _format_httpx(self):
|
|
128
|
+
"""Format proxy for httpx library."""
|
|
129
|
+
proxy_url = self.to_url("http")
|
|
130
|
+
return {"http://": proxy_url, "https://": proxy_url}
|
|
131
|
+
|
|
132
|
+
def _format_playwright(self, **kwargs):
|
|
133
|
+
"""Format proxy for Playwright."""
|
|
134
|
+
# Playwright expects server with protocol (e.g., 'http://ip:port', 'socks5://ip:port')
|
|
135
|
+
# Allow protocol selection via kwargs, default to http
|
|
136
|
+
protocol = kwargs.get("protocol", "http")
|
|
137
|
+
|
|
138
|
+
# Validate protocol is supported by the proxy
|
|
139
|
+
if self.protocols and protocol not in self.protocols:
|
|
140
|
+
# If proxy has specific protocols, use the first available one
|
|
141
|
+
protocol = self.protocols[0]
|
|
142
|
+
|
|
143
|
+
playwright_proxy = {"server": f"{protocol}://{self.proxy_address}:{self.port}"}
|
|
144
|
+
|
|
145
|
+
if self.username and self.password:
|
|
146
|
+
playwright_proxy["username"] = self.username
|
|
147
|
+
playwright_proxy["password"] = self.password
|
|
148
|
+
|
|
149
|
+
return playwright_proxy
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
from typing import Dict, List, Optional
|
|
2
|
+
|
|
1
3
|
import requests
|
|
2
|
-
|
|
3
|
-
from ..proxy_provider import ProxyProvider, ProxyConfig
|
|
4
|
+
|
|
4
5
|
from ..exceptions import ProxyFetchException, ProxyInvalidResponseException
|
|
5
6
|
from ..models.proxy import Proxy
|
|
6
|
-
import
|
|
7
|
+
from ..proxy_provider import ProxyConfig, ProxyProvider
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
class BrightData(ProxyProvider):
|
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
import requests
|
|
2
1
|
from datetime import datetime
|
|
3
|
-
from typing import
|
|
4
|
-
|
|
2
|
+
from typing import Dict, List, Optional
|
|
3
|
+
|
|
4
|
+
import requests
|
|
5
|
+
|
|
5
6
|
from ..exceptions import (
|
|
6
|
-
ProxyFetchException,
|
|
7
7
|
ProxyConversionException,
|
|
8
|
+
ProxyFetchException,
|
|
8
9
|
ProxyInvalidResponseException,
|
|
9
10
|
)
|
|
10
11
|
from ..models.proxy import Proxy
|
|
12
|
+
from ..proxy_provider import ProxyConfig, ProxyProvider
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
class Webshare(ProxyProvider):
|
|
@@ -85,7 +87,7 @@ class Webshare(ProxyProvider):
|
|
|
85
87
|
data = response.json()
|
|
86
88
|
|
|
87
89
|
proxy_data = data.get("results")
|
|
88
|
-
if
|
|
90
|
+
if proxy_data is None:
|
|
89
91
|
raise ProxyInvalidResponseException(response.text)
|
|
90
92
|
|
|
91
93
|
all_proxies.extend([self._convert_to_proxy(proxy) for proxy in proxy_data])
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
+
import threading
|
|
1
2
|
from abc import ABC, abstractmethod
|
|
3
|
+
from dataclasses import dataclass
|
|
2
4
|
from datetime import datetime
|
|
3
|
-
from typing import List, Optional
|
|
5
|
+
from typing import TYPE_CHECKING, List, Optional
|
|
6
|
+
|
|
4
7
|
from .models.proxy import Proxy
|
|
5
|
-
|
|
6
|
-
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from .algorithms import Algorithm
|
|
7
11
|
|
|
8
12
|
|
|
9
13
|
@dataclass
|
|
@@ -44,6 +48,11 @@ class ProxyProvider(ABC):
|
|
|
44
48
|
self._last_refresh: Optional[datetime] = None
|
|
45
49
|
self._lock = threading.Lock()
|
|
46
50
|
|
|
51
|
+
# Initialize default algorithm (RoundRobin for state persistence)
|
|
52
|
+
from .algorithms import RoundRobin
|
|
53
|
+
|
|
54
|
+
self._default_algorithm = RoundRobin()
|
|
55
|
+
|
|
47
56
|
def list_proxies(self, force_refresh: bool = False) -> List[Proxy]:
|
|
48
57
|
"""
|
|
49
58
|
Returns the stored proxies.
|
|
@@ -58,6 +67,39 @@ class ProxyProvider(ABC):
|
|
|
58
67
|
with self._lock:
|
|
59
68
|
return list(self._proxies) if self._proxies else []
|
|
60
69
|
|
|
70
|
+
def get_proxy(self, algorithm: Optional["Algorithm"] = None) -> Proxy:
|
|
71
|
+
"""Get a single proxy using the specified selection algorithm.
|
|
72
|
+
|
|
73
|
+
:param algorithm: Selection algorithm to use. If None, uses the provider's default RoundRobin algorithm.
|
|
74
|
+
:return: Selected proxy
|
|
75
|
+
:raises ValueError: If no proxies are available
|
|
76
|
+
|
|
77
|
+
Example:
|
|
78
|
+
>>> from proxyproviders import Webshare
|
|
79
|
+
>>> from proxyproviders.algorithms import Random, RoundRobin
|
|
80
|
+
>>>
|
|
81
|
+
>>> provider = Webshare(api_key="your-key")
|
|
82
|
+
>>>
|
|
83
|
+
>>> # Uses default RoundRobin (cycles through proxies)
|
|
84
|
+
>>> proxy = provider.get_proxy()
|
|
85
|
+
>>>
|
|
86
|
+
>>> # Uses Random selection
|
|
87
|
+
>>> proxy = provider.get_proxy(Random())
|
|
88
|
+
>>>
|
|
89
|
+
>>> # One-liner with requests
|
|
90
|
+
>>> import requests
|
|
91
|
+
>>> from proxyproviders.models.proxy import ProxyFormat
|
|
92
|
+
>>> requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
93
|
+
"""
|
|
94
|
+
proxies = self.list_proxies()
|
|
95
|
+
if not proxies:
|
|
96
|
+
raise ValueError("No proxies available from provider")
|
|
97
|
+
|
|
98
|
+
if algorithm is None:
|
|
99
|
+
algorithm = self._default_algorithm
|
|
100
|
+
|
|
101
|
+
return algorithm.select(proxies)
|
|
102
|
+
|
|
61
103
|
#
|
|
62
104
|
# Internal Methods
|
|
63
105
|
#
|
|
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "proxyproviders"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.1"
|
|
8
8
|
description = "A unified interface for different proxy providers"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [{ name = "David Teather", email = "contact.davidteather@gmail.com" }]
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
from dataclasses import dataclass
|
|
2
|
-
from typing import Optional, List
|
|
3
|
-
from datetime import datetime
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
@dataclass
|
|
7
|
-
class Proxy:
|
|
8
|
-
"""Our shared data model for a proxy object across all providers."""
|
|
9
|
-
|
|
10
|
-
id: str
|
|
11
|
-
"""A unique identifier for the proxy"""
|
|
12
|
-
|
|
13
|
-
username: str
|
|
14
|
-
"""The username required for authenticating with the proxy"""
|
|
15
|
-
|
|
16
|
-
password: str
|
|
17
|
-
"""The password required for authenticating with the proxy"""
|
|
18
|
-
|
|
19
|
-
proxy_address: str
|
|
20
|
-
"""The IP address or domain name of the proxy"""
|
|
21
|
-
|
|
22
|
-
port: int
|
|
23
|
-
"""The port number through which the proxy connection is established"""
|
|
24
|
-
|
|
25
|
-
country_code: Optional[str] = None
|
|
26
|
-
"""The country code where the proxy is located, e.g., 'US', 'FR'. Optional"""
|
|
27
|
-
|
|
28
|
-
city_name: Optional[str] = None
|
|
29
|
-
"""The city name where the proxy is located, e.g., 'New York', 'Paris'. Optional"""
|
|
30
|
-
|
|
31
|
-
created_at: Optional[datetime] = None
|
|
32
|
-
"""The timestamp when the proxy was created. Optional"""
|
|
33
|
-
|
|
34
|
-
protocols: Optional[List[str]] = None
|
|
35
|
-
"""A list of connection protocols supported by the proxy, e.g., ['http', 'https']"""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|