proxyproviders 0.1.1__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {proxyproviders-0.1.1 → proxyproviders-0.2.0}/PKG-INFO +93 -20
- {proxyproviders-0.1.1 → proxyproviders-0.2.0}/README.md +90 -18
- proxyproviders-0.2.0/proxyproviders/__init__.py +14 -0
- proxyproviders-0.2.0/proxyproviders/algorithms.py +112 -0
- proxyproviders-0.2.0/proxyproviders/models/proxy.py +118 -0
- {proxyproviders-0.1.1 → proxyproviders-0.2.0}/proxyproviders/providers/brightdata.py +4 -3
- {proxyproviders-0.1.1 → proxyproviders-0.2.0}/proxyproviders/providers/webshare.py +7 -5
- {proxyproviders-0.1.1 → proxyproviders-0.2.0}/proxyproviders/proxy_provider.py +45 -3
- {proxyproviders-0.1.1 → proxyproviders-0.2.0}/pyproject.toml +1 -1
- proxyproviders-0.1.1/proxyproviders/__init__.py +0 -6
- proxyproviders-0.1.1/proxyproviders/models/proxy.py +0 -35
- {proxyproviders-0.1.1 → proxyproviders-0.2.0}/LICENSE +0 -0
- {proxyproviders-0.1.1 → proxyproviders-0.2.0}/proxyproviders/exceptions.py +0 -0
- {proxyproviders-0.1.1 → proxyproviders-0.2.0}/proxyproviders/models/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: proxyproviders
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A unified interface for different proxy providers
|
|
5
5
|
License: MIT License
|
|
6
6
|
|
|
@@ -23,6 +23,7 @@ License: MIT License
|
|
|
23
23
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
24
24
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
25
|
SOFTWARE.
|
|
26
|
+
License-File: LICENSE
|
|
26
27
|
Author: David Teather
|
|
27
28
|
Author-email: contact.davidteather@gmail.com
|
|
28
29
|
Requires-Python: >=3.9
|
|
@@ -45,7 +46,7 @@ Description-Content-Type: text/markdown
|
|
|
45
46
|
|
|
46
47
|
The Unified Python Proxy API for managing proxies with support for BrightData, WebShare, and more.
|
|
47
48
|
|
|
48
|
-
[](https://codecov.io/gh/davidteather/proxyproviders) [](https://github.com/sponsors/davidteather) [](https://github.com/davidteather/proxyproviders/releases) [](https://github.com/davidteather/proxyproviders/blob/main/LICENSE) [](https://pypi.org/project/proxyproviders/)  [](https://discord.gg/yyPhbfma6f) [](https://www.linkedin.com/in/davidteather/)
|
|
49
50
|
|
|
50
51
|
## Table of Contents
|
|
51
52
|
|
|
@@ -105,6 +106,24 @@ proxies = proxy_provider.list_proxies()
|
|
|
105
106
|
print(proxies)
|
|
106
107
|
```
|
|
107
108
|
|
|
109
|
+
For simple usage, you can get a proxy and use it immediately:
|
|
110
|
+
```py
|
|
111
|
+
from proxyproviders import Webshare
|
|
112
|
+
import requests
|
|
113
|
+
|
|
114
|
+
provider = Webshare(api_key="your-api-key")
|
|
115
|
+
|
|
116
|
+
# Get a proxy (uses RoundRobin by default) and use it with requests
|
|
117
|
+
from proxyproviders.models.proxy import ProxyFormat
|
|
118
|
+
|
|
119
|
+
proxy = provider.get_proxy()
|
|
120
|
+
response = requests.get("https://httpbin.org/ip", proxies=proxy.format(ProxyFormat.REQUESTS))
|
|
121
|
+
print(response.json())
|
|
122
|
+
|
|
123
|
+
# Or in one-line
|
|
124
|
+
response = requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
125
|
+
```
|
|
126
|
+
|
|
108
127
|
Each provider has their own custom options, the `Webshare` class lets you specify url params according to their [api spec](https://apidocs.webshare.io/proxy-list/list#parameters), here's an example which will only return proxies that are based in the US.
|
|
109
128
|
|
|
110
129
|
```py
|
|
@@ -150,32 +169,86 @@ some_function(brightdata)
|
|
|
150
169
|
|
|
151
170
|
Here's a more meaningful example that takes the `Proxy` class and uses it to create a python requests http proxy.
|
|
152
171
|
|
|
172
|
+
#### Simple Usage
|
|
153
173
|
```py
|
|
154
|
-
from proxyproviders import Webshare
|
|
174
|
+
from proxyproviders import Webshare
|
|
175
|
+
from proxyproviders.algorithms import Random, RoundRobin
|
|
176
|
+
from proxyproviders.models.proxy import ProxyFormat
|
|
155
177
|
import requests
|
|
156
|
-
import os
|
|
157
178
|
|
|
179
|
+
provider = Webshare(api_key="your_api_key")
|
|
158
180
|
|
|
159
|
-
|
|
160
|
-
|
|
181
|
+
# Get proxy using default RoundRobin and make request
|
|
182
|
+
proxy = provider.get_proxy()
|
|
183
|
+
response = requests.get("https://httpbin.org/ip", proxies=proxy.format(ProxyFormat.REQUESTS))
|
|
161
184
|
|
|
162
|
-
|
|
163
|
-
|
|
185
|
+
# Or in one-line
|
|
186
|
+
response = requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
187
|
+
```
|
|
164
188
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
189
|
+
#### Built-in Algorithms
|
|
190
|
+
```py
|
|
191
|
+
from proxyproviders import Webshare
|
|
192
|
+
from proxyproviders.algorithms import Random, RoundRobin, First
|
|
169
193
|
|
|
170
|
-
|
|
171
|
-
return r.json()
|
|
194
|
+
provider = Webshare(api_key="your_api_key")
|
|
172
195
|
|
|
173
|
-
|
|
174
|
-
|
|
196
|
+
# Default: RoundRobin (cycles through proxies for load balancing)
|
|
197
|
+
proxy = provider.get_proxy()
|
|
175
198
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
199
|
+
# Random selection
|
|
200
|
+
proxy = provider.get_proxy(Random())
|
|
201
|
+
|
|
202
|
+
# Always first proxy (deterministic)
|
|
203
|
+
proxy = provider.get_proxy(First())
|
|
204
|
+
|
|
205
|
+
# Algorithm can maintain state when reused
|
|
206
|
+
round_robin = RoundRobin()
|
|
207
|
+
proxy1 = provider.get_proxy(round_robin)
|
|
208
|
+
proxy2 = provider.get_proxy(round_robin) # Next in sequence
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
#### Algorithm State Management
|
|
212
|
+
```py
|
|
213
|
+
from proxyproviders import Webshare
|
|
214
|
+
from proxyproviders.algorithms import RoundRobin, Random
|
|
215
|
+
|
|
216
|
+
provider = Webshare(api_key="your_api_key")
|
|
217
|
+
|
|
218
|
+
# Create reusable algorithm for state management
|
|
219
|
+
round_robin = RoundRobin() # Maintains state across calls
|
|
220
|
+
random_algo = Random() # Stateless but reusable
|
|
221
|
+
|
|
222
|
+
# Each call to round_robin will cycle to next proxy
|
|
223
|
+
for i in range(3):
|
|
224
|
+
proxy = provider.get_proxy(round_robin)
|
|
225
|
+
print(f"RoundRobin {i}: {proxy.proxy_address}")
|
|
226
|
+
|
|
227
|
+
# Provider also maintains its own default RoundRobin state when not specified
|
|
228
|
+
for i in range(3):
|
|
229
|
+
proxy = provider.get_proxy() # Uses provider's default RoundRobin
|
|
230
|
+
print(f"Default {i}: {proxy.proxy_address}")
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
#### Custom Algorithms
|
|
234
|
+
```py
|
|
235
|
+
from proxyproviders.algorithms import Algorithm
|
|
236
|
+
from typing import List
|
|
237
|
+
from proxyproviders.models.proxy import Proxy
|
|
238
|
+
|
|
239
|
+
class USProxyAlgorithm(Algorithm):
|
|
240
|
+
"""Prefers US proxies, falls back to first available."""
|
|
241
|
+
|
|
242
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
243
|
+
# Try to find a US proxy
|
|
244
|
+
for proxy in proxies:
|
|
245
|
+
if proxy.country_code == "US":
|
|
246
|
+
return proxy
|
|
247
|
+
# Fall back to first proxy
|
|
248
|
+
return proxies[0]
|
|
249
|
+
|
|
250
|
+
# Use your custom algorithm
|
|
251
|
+
proxy = provider.get_proxy(USProxyAlgorithm())
|
|
179
252
|
```
|
|
180
253
|
|
|
181
254
|
### Making Your Own Proxy Provider
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
The Unified Python Proxy API for managing proxies with support for BrightData, WebShare, and more.
|
|
4
4
|
|
|
5
|
-
[](https://codecov.io/gh/davidteather/proxyproviders) [](https://github.com/sponsors/davidteather) [](https://github.com/davidteather/proxyproviders/releases) [](https://github.com/davidteather/proxyproviders/blob/main/LICENSE) [](https://pypi.org/project/proxyproviders/)  [](https://discord.gg/yyPhbfma6f) [](https://www.linkedin.com/in/davidteather/)
|
|
6
6
|
|
|
7
7
|
## Table of Contents
|
|
8
8
|
|
|
@@ -62,6 +62,24 @@ proxies = proxy_provider.list_proxies()
|
|
|
62
62
|
print(proxies)
|
|
63
63
|
```
|
|
64
64
|
|
|
65
|
+
For simple usage, you can get a proxy and use it immediately:
|
|
66
|
+
```py
|
|
67
|
+
from proxyproviders import Webshare
|
|
68
|
+
import requests
|
|
69
|
+
|
|
70
|
+
provider = Webshare(api_key="your-api-key")
|
|
71
|
+
|
|
72
|
+
# Get a proxy (uses RoundRobin by default) and use it with requests
|
|
73
|
+
from proxyproviders.models.proxy import ProxyFormat
|
|
74
|
+
|
|
75
|
+
proxy = provider.get_proxy()
|
|
76
|
+
response = requests.get("https://httpbin.org/ip", proxies=proxy.format(ProxyFormat.REQUESTS))
|
|
77
|
+
print(response.json())
|
|
78
|
+
|
|
79
|
+
# Or in one-line
|
|
80
|
+
response = requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
81
|
+
```
|
|
82
|
+
|
|
65
83
|
Each provider has their own custom options, the `Webshare` class lets you specify url params according to their [api spec](https://apidocs.webshare.io/proxy-list/list#parameters), here's an example which will only return proxies that are based in the US.
|
|
66
84
|
|
|
67
85
|
```py
|
|
@@ -107,32 +125,86 @@ some_function(brightdata)
|
|
|
107
125
|
|
|
108
126
|
Here's a more meaningful example that takes the `Proxy` class and uses it to create a python requests http proxy.
|
|
109
127
|
|
|
128
|
+
#### Simple Usage
|
|
110
129
|
```py
|
|
111
|
-
from proxyproviders import Webshare
|
|
130
|
+
from proxyproviders import Webshare
|
|
131
|
+
from proxyproviders.algorithms import Random, RoundRobin
|
|
132
|
+
from proxyproviders.models.proxy import ProxyFormat
|
|
112
133
|
import requests
|
|
113
|
-
import os
|
|
114
134
|
|
|
135
|
+
provider = Webshare(api_key="your_api_key")
|
|
115
136
|
|
|
116
|
-
|
|
117
|
-
|
|
137
|
+
# Get proxy using default RoundRobin and make request
|
|
138
|
+
proxy = provider.get_proxy()
|
|
139
|
+
response = requests.get("https://httpbin.org/ip", proxies=proxy.format(ProxyFormat.REQUESTS))
|
|
118
140
|
|
|
119
|
-
|
|
120
|
-
|
|
141
|
+
# Or in one-line
|
|
142
|
+
response = requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
143
|
+
```
|
|
121
144
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
145
|
+
#### Built-in Algorithms
|
|
146
|
+
```py
|
|
147
|
+
from proxyproviders import Webshare
|
|
148
|
+
from proxyproviders.algorithms import Random, RoundRobin, First
|
|
126
149
|
|
|
127
|
-
|
|
128
|
-
return r.json()
|
|
150
|
+
provider = Webshare(api_key="your_api_key")
|
|
129
151
|
|
|
130
|
-
|
|
131
|
-
|
|
152
|
+
# Default: RoundRobin (cycles through proxies for load balancing)
|
|
153
|
+
proxy = provider.get_proxy()
|
|
132
154
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
155
|
+
# Random selection
|
|
156
|
+
proxy = provider.get_proxy(Random())
|
|
157
|
+
|
|
158
|
+
# Always first proxy (deterministic)
|
|
159
|
+
proxy = provider.get_proxy(First())
|
|
160
|
+
|
|
161
|
+
# Algorithm can maintain state when reused
|
|
162
|
+
round_robin = RoundRobin()
|
|
163
|
+
proxy1 = provider.get_proxy(round_robin)
|
|
164
|
+
proxy2 = provider.get_proxy(round_robin) # Next in sequence
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
#### Algorithm State Management
|
|
168
|
+
```py
|
|
169
|
+
from proxyproviders import Webshare
|
|
170
|
+
from proxyproviders.algorithms import RoundRobin, Random
|
|
171
|
+
|
|
172
|
+
provider = Webshare(api_key="your_api_key")
|
|
173
|
+
|
|
174
|
+
# Create reusable algorithm for state management
|
|
175
|
+
round_robin = RoundRobin() # Maintains state across calls
|
|
176
|
+
random_algo = Random() # Stateless but reusable
|
|
177
|
+
|
|
178
|
+
# Each call to round_robin will cycle to next proxy
|
|
179
|
+
for i in range(3):
|
|
180
|
+
proxy = provider.get_proxy(round_robin)
|
|
181
|
+
print(f"RoundRobin {i}: {proxy.proxy_address}")
|
|
182
|
+
|
|
183
|
+
# Provider also maintains its own default RoundRobin state when not specified
|
|
184
|
+
for i in range(3):
|
|
185
|
+
proxy = provider.get_proxy() # Uses provider's default RoundRobin
|
|
186
|
+
print(f"Default {i}: {proxy.proxy_address}")
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
#### Custom Algorithms
|
|
190
|
+
```py
|
|
191
|
+
from proxyproviders.algorithms import Algorithm
|
|
192
|
+
from typing import List
|
|
193
|
+
from proxyproviders.models.proxy import Proxy
|
|
194
|
+
|
|
195
|
+
class USProxyAlgorithm(Algorithm):
|
|
196
|
+
"""Prefers US proxies, falls back to first available."""
|
|
197
|
+
|
|
198
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
199
|
+
# Try to find a US proxy
|
|
200
|
+
for proxy in proxies:
|
|
201
|
+
if proxy.country_code == "US":
|
|
202
|
+
return proxy
|
|
203
|
+
# Fall back to first proxy
|
|
204
|
+
return proxies[0]
|
|
205
|
+
|
|
206
|
+
# Use your custom algorithm
|
|
207
|
+
proxy = provider.get_proxy(USProxyAlgorithm())
|
|
136
208
|
```
|
|
137
209
|
|
|
138
210
|
### Making Your Own Proxy Provider
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from . import algorithms
|
|
2
|
+
from .models.proxy import Proxy
|
|
3
|
+
from .providers.brightdata import BrightData
|
|
4
|
+
from .providers.webshare import Webshare
|
|
5
|
+
from .proxy_provider import ProxyConfig, ProxyProvider
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"ProxyProvider",
|
|
9
|
+
"ProxyConfig",
|
|
10
|
+
"Proxy",
|
|
11
|
+
"Webshare",
|
|
12
|
+
"BrightData",
|
|
13
|
+
"algorithms",
|
|
14
|
+
]
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Selection algorithms for choosing proxies from a provider."""
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
import threading
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from typing import Dict, List
|
|
7
|
+
|
|
8
|
+
from .models.proxy import Proxy
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Algorithm(ABC):
|
|
12
|
+
"""Base class for proxy selection algorithms.
|
|
13
|
+
|
|
14
|
+
Users can extend this class to create custom proxy selection logic.
|
|
15
|
+
|
|
16
|
+
Example:
|
|
17
|
+
>>> class MyAlgorithm(Algorithm):
|
|
18
|
+
... def select(self, proxies):
|
|
19
|
+
... return proxies[0] # Always return first proxy
|
|
20
|
+
...
|
|
21
|
+
>>> algorithm = MyAlgorithm()
|
|
22
|
+
>>> proxy = provider.get_proxy(algorithm)
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
@abstractmethod
|
|
26
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
27
|
+
"""Select a proxy from the list.
|
|
28
|
+
|
|
29
|
+
:param proxies: List of available proxies
|
|
30
|
+
:return: Selected proxy
|
|
31
|
+
:raises ValueError: If proxy list is empty
|
|
32
|
+
"""
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Random(Algorithm):
|
|
37
|
+
"""Random proxy selection algorithm.
|
|
38
|
+
|
|
39
|
+
Selects a random proxy from the available list on each call.
|
|
40
|
+
|
|
41
|
+
Example:
|
|
42
|
+
>>> algorithm = Random()
|
|
43
|
+
>>> proxy = provider.get_proxy(algorithm)
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
47
|
+
"""Select a random proxy from the list.
|
|
48
|
+
|
|
49
|
+
:param proxies: List of available proxies
|
|
50
|
+
:return: Randomly selected proxy
|
|
51
|
+
:raises ValueError: If proxy list is empty
|
|
52
|
+
"""
|
|
53
|
+
if not proxies:
|
|
54
|
+
raise ValueError("Cannot select from empty proxy list")
|
|
55
|
+
return random.choice(proxies)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class RoundRobin(Algorithm):
|
|
59
|
+
"""Round-robin proxy selection algorithm with state tracking.
|
|
60
|
+
|
|
61
|
+
Cycles through proxies in order, maintaining state across calls.
|
|
62
|
+
Each RoundRobin instance maintains its own independent state.
|
|
63
|
+
|
|
64
|
+
Example:
|
|
65
|
+
>>> algorithm = RoundRobin()
|
|
66
|
+
>>> proxy1 = provider.get_proxy(algorithm)
|
|
67
|
+
>>> proxy2 = provider.get_proxy(algorithm) # Next proxy in sequence
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
def __init__(self):
|
|
71
|
+
"""Initialize round-robin algorithm with state tracking."""
|
|
72
|
+
self._current_index = 0
|
|
73
|
+
self._lock = threading.Lock()
|
|
74
|
+
|
|
75
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
76
|
+
"""Select next proxy in round-robin sequence.
|
|
77
|
+
|
|
78
|
+
:param proxies: List of available proxies
|
|
79
|
+
:return: Next proxy in round-robin sequence
|
|
80
|
+
:raises ValueError: If proxy list is empty
|
|
81
|
+
"""
|
|
82
|
+
if not proxies:
|
|
83
|
+
raise ValueError("Cannot select from empty proxy list")
|
|
84
|
+
|
|
85
|
+
with self._lock:
|
|
86
|
+
selected_proxy = proxies[self._current_index % len(proxies)]
|
|
87
|
+
self._current_index = (self._current_index + 1) % len(proxies)
|
|
88
|
+
|
|
89
|
+
return selected_proxy
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class First(Algorithm):
|
|
93
|
+
"""First proxy selection algorithm.
|
|
94
|
+
|
|
95
|
+
Always selects the first proxy from the list.
|
|
96
|
+
Useful for deterministic behavior or when proxies are pre-sorted.
|
|
97
|
+
|
|
98
|
+
Example:
|
|
99
|
+
>>> algorithm = First()
|
|
100
|
+
>>> proxy = provider.get_proxy(algorithm) # Always first proxy
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
def select(self, proxies: List[Proxy]) -> Proxy:
|
|
104
|
+
"""Select the first proxy from the list.
|
|
105
|
+
|
|
106
|
+
:param proxies: List of available proxies
|
|
107
|
+
:return: First proxy in the list
|
|
108
|
+
:raises ValueError: If proxy list is empty
|
|
109
|
+
"""
|
|
110
|
+
if not proxies:
|
|
111
|
+
raise ValueError("Cannot select from empty proxy list")
|
|
112
|
+
return proxies[0]
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from enum import Enum
|
|
4
|
+
from typing import Dict, List, Optional, Union
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ProxyFormat(Enum):
|
|
8
|
+
"""Supported proxy output formats."""
|
|
9
|
+
|
|
10
|
+
REQUESTS = "requests"
|
|
11
|
+
"""Requests format, for use in requests library HTTP calls"""
|
|
12
|
+
|
|
13
|
+
CURL = "curl"
|
|
14
|
+
"""CURL format, for use in curl commands"""
|
|
15
|
+
|
|
16
|
+
HTTPX = "httpx"
|
|
17
|
+
"""HTTPX format, for use in httpx library HTTP calls"""
|
|
18
|
+
|
|
19
|
+
AIOHTTP = "aiohttp"
|
|
20
|
+
"""AIOHTTP format, for use in aiohttp library HTTP calls"""
|
|
21
|
+
|
|
22
|
+
URL = "url"
|
|
23
|
+
"""URL format, for use in URL strings"""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class Proxy:
|
|
28
|
+
"""Our shared data model for a proxy object across all providers."""
|
|
29
|
+
|
|
30
|
+
id: str
|
|
31
|
+
"""A unique identifier for the proxy"""
|
|
32
|
+
|
|
33
|
+
username: str
|
|
34
|
+
"""The username required for authenticating with the proxy"""
|
|
35
|
+
|
|
36
|
+
password: str
|
|
37
|
+
"""The password required for authenticating with the proxy"""
|
|
38
|
+
|
|
39
|
+
proxy_address: str
|
|
40
|
+
"""The IP address or domain name of the proxy"""
|
|
41
|
+
|
|
42
|
+
port: int
|
|
43
|
+
"""The port number through which the proxy connection is established"""
|
|
44
|
+
|
|
45
|
+
country_code: Optional[str] = None
|
|
46
|
+
"""The country code where the proxy is located, e.g., 'US', 'FR'. Optional"""
|
|
47
|
+
|
|
48
|
+
city_name: Optional[str] = None
|
|
49
|
+
"""The city name where the proxy is located, e.g., 'New York', 'Paris'. Optional"""
|
|
50
|
+
|
|
51
|
+
created_at: Optional[datetime] = None
|
|
52
|
+
"""The timestamp when the proxy was created. Optional"""
|
|
53
|
+
|
|
54
|
+
protocols: Optional[List[str]] = None
|
|
55
|
+
"""A list of connection protocols supported by the proxy, e.g., ['http', 'https']"""
|
|
56
|
+
|
|
57
|
+
def to_url(self, protocol: str = "http") -> str:
|
|
58
|
+
"""Convert proxy to URL format for use with HTTP clients.
|
|
59
|
+
|
|
60
|
+
:param protocol: The protocol to use in the URL (default: 'http')
|
|
61
|
+
:return: Proxy URL in format 'protocol://username:password@host:port'
|
|
62
|
+
|
|
63
|
+
Example:
|
|
64
|
+
>>> proxy.to_url()
|
|
65
|
+
'http://user:pass@192.168.1.1:8080'
|
|
66
|
+
>>> proxy.to_url('https')
|
|
67
|
+
'https://user:pass@192.168.1.1:8080'
|
|
68
|
+
"""
|
|
69
|
+
return f"{protocol}://{self.username}:{self.password}@{self.proxy_address}:{self.port}"
|
|
70
|
+
|
|
71
|
+
def format(
|
|
72
|
+
self, format_type: Union[ProxyFormat, str] = ProxyFormat.URL, **kwargs
|
|
73
|
+
) -> Union[str, Dict[str, str], List[str]]:
|
|
74
|
+
"""Format proxy for different HTTP clients and tools.
|
|
75
|
+
|
|
76
|
+
:param format_type: Output format (default: URL string)
|
|
77
|
+
:param kwargs: Format-specific options
|
|
78
|
+
:return: Formatted proxy data
|
|
79
|
+
|
|
80
|
+
Examples:
|
|
81
|
+
>>> proxy.format() # Default URL string
|
|
82
|
+
'http://user:pass@192.168.1.1:8080'
|
|
83
|
+
|
|
84
|
+
>>> proxy.format(ProxyFormat.REQUESTS)
|
|
85
|
+
{'http': 'http://user:pass@192.168.1.1:8080', 'https': 'http://user:pass@192.168.1.1:8080'}
|
|
86
|
+
|
|
87
|
+
>>> proxy.format('curl') # String also works
|
|
88
|
+
['-x', 'http://user:pass@192.168.1.1:8080']
|
|
89
|
+
|
|
90
|
+
>>> proxy.format(ProxyFormat.HTTPX)
|
|
91
|
+
{'http://': 'http://user:pass@192.168.1.1:8080', 'https://': 'http://user:pass@192.168.1.1:8080'}
|
|
92
|
+
"""
|
|
93
|
+
if isinstance(format_type, str):
|
|
94
|
+
format_type = ProxyFormat(format_type)
|
|
95
|
+
|
|
96
|
+
if format_type == ProxyFormat.URL:
|
|
97
|
+
protocol = kwargs.get("protocol", "http")
|
|
98
|
+
return self.to_url(protocol)
|
|
99
|
+
|
|
100
|
+
elif format_type == ProxyFormat.REQUESTS:
|
|
101
|
+
protocols = kwargs.get("protocols", self.protocols or ["http", "https"])
|
|
102
|
+
proxy_url = self.to_url("http")
|
|
103
|
+
return {protocol: proxy_url for protocol in protocols}
|
|
104
|
+
|
|
105
|
+
elif format_type == ProxyFormat.CURL:
|
|
106
|
+
return ["-x", self.to_url("http")]
|
|
107
|
+
|
|
108
|
+
elif format_type == ProxyFormat.HTTPX:
|
|
109
|
+
# httpx uses 'http://' and 'https://' as keys
|
|
110
|
+
proxy_url = self.to_url("http")
|
|
111
|
+
return {"http://": proxy_url, "https://": proxy_url}
|
|
112
|
+
|
|
113
|
+
elif format_type == ProxyFormat.AIOHTTP:
|
|
114
|
+
# aiohttp takes a single URL string
|
|
115
|
+
return self.to_url("http")
|
|
116
|
+
|
|
117
|
+
else:
|
|
118
|
+
raise ValueError(f"Unsupported format: {format_type}")
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
from typing import Dict, List, Optional
|
|
2
|
+
|
|
1
3
|
import requests
|
|
2
|
-
|
|
3
|
-
from ..proxy_provider import ProxyProvider, ProxyConfig
|
|
4
|
+
|
|
4
5
|
from ..exceptions import ProxyFetchException, ProxyInvalidResponseException
|
|
5
6
|
from ..models.proxy import Proxy
|
|
6
|
-
import
|
|
7
|
+
from ..proxy_provider import ProxyConfig, ProxyProvider
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
class BrightData(ProxyProvider):
|
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
import requests
|
|
2
1
|
from datetime import datetime
|
|
3
|
-
from typing import
|
|
4
|
-
|
|
2
|
+
from typing import Dict, List, Optional
|
|
3
|
+
|
|
4
|
+
import requests
|
|
5
|
+
|
|
5
6
|
from ..exceptions import (
|
|
6
|
-
ProxyFetchException,
|
|
7
7
|
ProxyConversionException,
|
|
8
|
+
ProxyFetchException,
|
|
8
9
|
ProxyInvalidResponseException,
|
|
9
10
|
)
|
|
10
11
|
from ..models.proxy import Proxy
|
|
12
|
+
from ..proxy_provider import ProxyConfig, ProxyProvider
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
class Webshare(ProxyProvider):
|
|
@@ -85,7 +87,7 @@ class Webshare(ProxyProvider):
|
|
|
85
87
|
data = response.json()
|
|
86
88
|
|
|
87
89
|
proxy_data = data.get("results")
|
|
88
|
-
if
|
|
90
|
+
if proxy_data is None:
|
|
89
91
|
raise ProxyInvalidResponseException(response.text)
|
|
90
92
|
|
|
91
93
|
all_proxies.extend([self._convert_to_proxy(proxy) for proxy in proxy_data])
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
+
import threading
|
|
1
2
|
from abc import ABC, abstractmethod
|
|
3
|
+
from dataclasses import dataclass
|
|
2
4
|
from datetime import datetime
|
|
3
|
-
from typing import List, Optional
|
|
5
|
+
from typing import TYPE_CHECKING, List, Optional
|
|
6
|
+
|
|
4
7
|
from .models.proxy import Proxy
|
|
5
|
-
|
|
6
|
-
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from .algorithms import Algorithm
|
|
7
11
|
|
|
8
12
|
|
|
9
13
|
@dataclass
|
|
@@ -44,6 +48,11 @@ class ProxyProvider(ABC):
|
|
|
44
48
|
self._last_refresh: Optional[datetime] = None
|
|
45
49
|
self._lock = threading.Lock()
|
|
46
50
|
|
|
51
|
+
# Initialize default algorithm (RoundRobin for state persistence)
|
|
52
|
+
from .algorithms import RoundRobin
|
|
53
|
+
|
|
54
|
+
self._default_algorithm = RoundRobin()
|
|
55
|
+
|
|
47
56
|
def list_proxies(self, force_refresh: bool = False) -> List[Proxy]:
|
|
48
57
|
"""
|
|
49
58
|
Returns the stored proxies.
|
|
@@ -58,6 +67,39 @@ class ProxyProvider(ABC):
|
|
|
58
67
|
with self._lock:
|
|
59
68
|
return list(self._proxies) if self._proxies else []
|
|
60
69
|
|
|
70
|
+
def get_proxy(self, algorithm: Optional["Algorithm"] = None) -> Proxy:
|
|
71
|
+
"""Get a single proxy using the specified selection algorithm.
|
|
72
|
+
|
|
73
|
+
:param algorithm: Selection algorithm to use. If None, uses the provider's default RoundRobin algorithm.
|
|
74
|
+
:return: Selected proxy
|
|
75
|
+
:raises ValueError: If no proxies are available
|
|
76
|
+
|
|
77
|
+
Example:
|
|
78
|
+
>>> from proxyproviders import Webshare
|
|
79
|
+
>>> from proxyproviders.algorithms import Random, RoundRobin
|
|
80
|
+
>>>
|
|
81
|
+
>>> provider = Webshare(api_key="your-key")
|
|
82
|
+
>>>
|
|
83
|
+
>>> # Uses default RoundRobin (cycles through proxies)
|
|
84
|
+
>>> proxy = provider.get_proxy()
|
|
85
|
+
>>>
|
|
86
|
+
>>> # Uses Random selection
|
|
87
|
+
>>> proxy = provider.get_proxy(Random())
|
|
88
|
+
>>>
|
|
89
|
+
>>> # One-liner with requests
|
|
90
|
+
>>> import requests
|
|
91
|
+
>>> from proxyproviders.models.proxy import ProxyFormat
|
|
92
|
+
>>> requests.get("https://httpbin.org/ip", proxies=provider.get_proxy().format(ProxyFormat.REQUESTS))
|
|
93
|
+
"""
|
|
94
|
+
proxies = self.list_proxies()
|
|
95
|
+
if not proxies:
|
|
96
|
+
raise ValueError("No proxies available from provider")
|
|
97
|
+
|
|
98
|
+
if algorithm is None:
|
|
99
|
+
algorithm = self._default_algorithm
|
|
100
|
+
|
|
101
|
+
return algorithm.select(proxies)
|
|
102
|
+
|
|
61
103
|
#
|
|
62
104
|
# Internal Methods
|
|
63
105
|
#
|
|
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "proxyproviders"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "A unified interface for different proxy providers"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [{ name = "David Teather", email = "contact.davidteather@gmail.com" }]
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
from dataclasses import dataclass
|
|
2
|
-
from typing import Optional, List
|
|
3
|
-
from datetime import datetime
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
@dataclass
|
|
7
|
-
class Proxy:
|
|
8
|
-
"""Our shared data model for a proxy object across all providers."""
|
|
9
|
-
|
|
10
|
-
id: str
|
|
11
|
-
"""A unique identifier for the proxy"""
|
|
12
|
-
|
|
13
|
-
username: str
|
|
14
|
-
"""The username required for authenticating with the proxy"""
|
|
15
|
-
|
|
16
|
-
password: str
|
|
17
|
-
"""The password required for authenticating with the proxy"""
|
|
18
|
-
|
|
19
|
-
proxy_address: str
|
|
20
|
-
"""The IP address or domain name of the proxy"""
|
|
21
|
-
|
|
22
|
-
port: int
|
|
23
|
-
"""The port number through which the proxy connection is established"""
|
|
24
|
-
|
|
25
|
-
country_code: Optional[str] = None
|
|
26
|
-
"""The country code where the proxy is located, e.g., 'US', 'FR'. Optional"""
|
|
27
|
-
|
|
28
|
-
city_name: Optional[str] = None
|
|
29
|
-
"""The city name where the proxy is located, e.g., 'New York', 'Paris'. Optional"""
|
|
30
|
-
|
|
31
|
-
created_at: Optional[datetime] = None
|
|
32
|
-
"""The timestamp when the proxy was created. Optional"""
|
|
33
|
-
|
|
34
|
-
protocols: Optional[List[str]] = None
|
|
35
|
-
"""A list of connection protocols supported by the proxy, e.g., ['http', 'https']"""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|