free-proxy 1.2.2__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {free_proxy-1.2.2/free_proxy.egg-info → free_proxy-1.3.0}/PKG-INFO +60 -4
- {free_proxy-1.2.2 → free_proxy-1.3.0}/README.md +55 -2
- free_proxy-1.3.0/fp/fp.py +196 -0
- {free_proxy-1.2.2 → free_proxy-1.3.0/free_proxy.egg-info}/PKG-INFO +60 -4
- {free_proxy-1.2.2 → free_proxy-1.3.0}/free_proxy.egg-info/requires.txt +3 -0
- {free_proxy-1.2.2 → free_proxy-1.3.0}/setup.py +6 -3
- free_proxy-1.2.2/fp/fp.py +0 -97
- {free_proxy-1.2.2 → free_proxy-1.3.0}/LICENSE +0 -0
- {free_proxy-1.2.2 → free_proxy-1.3.0}/fp/__init__.py +0 -0
- {free_proxy-1.2.2 → free_proxy-1.3.0}/fp/errors.py +0 -0
- {free_proxy-1.2.2 → free_proxy-1.3.0}/free_proxy.egg-info/SOURCES.txt +0 -0
- {free_proxy-1.2.2 → free_proxy-1.3.0}/free_proxy.egg-info/dependency_links.txt +0 -0
- {free_proxy-1.2.2 → free_proxy-1.3.0}/free_proxy.egg-info/top_level.txt +0 -0
- {free_proxy-1.2.2 → free_proxy-1.3.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: free_proxy
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Proxy scraper for further use
|
|
5
5
|
Home-page: https://github.com/jundymek/free-proxy
|
|
6
6
|
Author: jundymek
|
|
@@ -8,11 +8,13 @@ Author-email: jundymek@gmail.com
|
|
|
8
8
|
Classifier: Programming Language :: Python :: 3
|
|
9
9
|
Classifier: License :: OSI Approved :: MIT License
|
|
10
10
|
Classifier: Operating System :: OS Independent
|
|
11
|
-
Requires-Python: >=3.
|
|
11
|
+
Requires-Python: >=3.9
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
Requires-Dist: lxml<7,>=5.0
|
|
15
15
|
Requires-Dist: requests<3,>=2.31
|
|
16
|
+
Provides-Extra: async
|
|
17
|
+
Requires-Dist: aiohttp<4,>=3.8; extra == "async"
|
|
16
18
|
Dynamic: author
|
|
17
19
|
Dynamic: author-email
|
|
18
20
|
Dynamic: classifier
|
|
@@ -20,13 +22,14 @@ Dynamic: description
|
|
|
20
22
|
Dynamic: description-content-type
|
|
21
23
|
Dynamic: home-page
|
|
22
24
|
Dynamic: license-file
|
|
25
|
+
Dynamic: provides-extra
|
|
23
26
|
Dynamic: requires-dist
|
|
24
27
|
Dynamic: requires-python
|
|
25
28
|
Dynamic: summary
|
|
26
29
|
|
|
27
30
|
# Free-proxy
|
|
28
31
|
|
|
29
|
-

|
|
30
33
|
|
|
31
34
|
## Get free working proxies from <https://www.sslproxies.org/>, <https://www.us-proxy.org/>, <https://free-proxy-list.net/uk-proxy.html> and <https://free-proxy-list.net> and use them in your script
|
|
32
35
|
|
|
@@ -43,9 +46,10 @@ Returns proxy as string:
|
|
|
43
46
|
|
|
44
47
|
### Requirements
|
|
45
48
|
|
|
46
|
-
- [Python 3](https://www.python.org/downloads/)
|
|
49
|
+
- [Python 3](https://www.python.org/downloads/) (3.9+)
|
|
47
50
|
- [Requests library](https://github.com/psf/requests)
|
|
48
51
|
- [Lxml library](https://github.com/lxml/lxml)
|
|
52
|
+
- [Aiohttp library](https://github.com/aio-libs/aiohttp) (optional, only for async usage)
|
|
49
53
|
|
|
50
54
|
### Installation
|
|
51
55
|
|
|
@@ -53,6 +57,12 @@ Returns proxy as string:
|
|
|
53
57
|
pip install free-proxy
|
|
54
58
|
```
|
|
55
59
|
|
|
60
|
+
With async support (installs aiohttp):
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
pip install "free-proxy[async]"
|
|
64
|
+
```
|
|
65
|
+
|
|
56
66
|
[](https://asciinema.org/a/Xolpn3eD2tyJl8Y8HE9zolgex)
|
|
57
67
|
|
|
58
68
|
### Usage examples
|
|
@@ -177,9 +187,55 @@ proxy = FreeProxy(url='http://httpbin.org/get').get()
|
|
|
177
187
|
proxy = FreeProxy(request_timeout=5).get()
|
|
178
188
|
```
|
|
179
189
|
|
|
190
|
+
## Async usage
|
|
191
|
+
|
|
192
|
+
`AsyncFreeProxy` is an asyncio variant of `FreeProxy`. It accepts the same options,
|
|
193
|
+
but checks proxies **concurrently** (up to `max_concurrent` at a time, default 20)
|
|
194
|
+
and returns the first working one, which is usually much faster than the
|
|
195
|
+
synchronous version. Remaining checks are cancelled as soon as a winner is found.
|
|
196
|
+
|
|
197
|
+
Async support requires the `async` extra:
|
|
198
|
+
|
|
199
|
+
```python
|
|
200
|
+
pip install "free-proxy[async]"
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
In a plain script, run it with `asyncio.run`:
|
|
204
|
+
|
|
205
|
+
```python
|
|
206
|
+
import asyncio
|
|
207
|
+
|
|
208
|
+
from fp.fp import AsyncFreeProxy
|
|
209
|
+
|
|
210
|
+
proxy = asyncio.run(AsyncFreeProxy().get())
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Inside your own async application (or in a Jupyter cell), just await it:
|
|
214
|
+
|
|
215
|
+
```python
|
|
216
|
+
proxy = await AsyncFreeProxy(country_id=['US'], max_concurrent=50).get()
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
- **`max_concurrent` parameter** (`AsyncFreeProxy` only)
|
|
220
|
+
Maximum number of proxies checked at the same time. Defaults to `max_concurrent=20`.
|
|
221
|
+
Raise it for faster scanning on a strong connection, lower it to be gentler on
|
|
222
|
+
your network and the test URL.
|
|
223
|
+
|
|
180
224
|
## CHANGELOG
|
|
181
225
|
|
|
182
226
|
---
|
|
227
|
+
## [1.3.0] - 2026-09-06
|
|
228
|
+
|
|
229
|
+
- Added `AsyncFreeProxy` — an asyncio variant of `FreeProxy` that checks proxies concurrently and returns the first working one (based on the idea from #39, thanks @NerdzzyDev)
|
|
230
|
+
- Async support is an optional extra: `pip install "free-proxy[async]"` (installs aiohttp)
|
|
231
|
+
- Added `max_concurrent` parameter to `AsyncFreeProxy` (default: 20) limiting how many proxies are checked at once
|
|
232
|
+
- Raised minimum Python version to 3.9
|
|
233
|
+
|
|
234
|
+
## [1.2.3] - 2026-08-24
|
|
235
|
+
|
|
236
|
+
- Fixed the proxy check silently bypassing the proxy when the test URL scheme did not match the `https` setting. With default settings (HTTP proxy mapping + HTTPS test URL) `requests` selected no proxy at all, the peername verification then rejected every candidate and `get()` always failed. The proxies mapping now covers both `http` and `https` schemes. Broken since 1.2.1. (#61)
|
|
237
|
+
- Added a CI workflow running the test suite on Python 3.9–3.13
|
|
238
|
+
|
|
183
239
|
## [1.2.2] - 2026-07-04
|
|
184
240
|
|
|
185
241
|
- Fixed `get_proxy_list` hanging indefinitely when a source website stalls — the proxy list request now has a timeout (#58)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Free-proxy
|
|
2
2
|
|
|
3
|
-

|
|
4
4
|
|
|
5
5
|
## Get free working proxies from <https://www.sslproxies.org/>, <https://www.us-proxy.org/>, <https://free-proxy-list.net/uk-proxy.html> and <https://free-proxy-list.net> and use them in your script
|
|
6
6
|
|
|
@@ -17,9 +17,10 @@ Returns proxy as string:
|
|
|
17
17
|
|
|
18
18
|
### Requirements
|
|
19
19
|
|
|
20
|
-
- [Python 3](https://www.python.org/downloads/)
|
|
20
|
+
- [Python 3](https://www.python.org/downloads/) (3.9+)
|
|
21
21
|
- [Requests library](https://github.com/psf/requests)
|
|
22
22
|
- [Lxml library](https://github.com/lxml/lxml)
|
|
23
|
+
- [Aiohttp library](https://github.com/aio-libs/aiohttp) (optional, only for async usage)
|
|
23
24
|
|
|
24
25
|
### Installation
|
|
25
26
|
|
|
@@ -27,6 +28,12 @@ Returns proxy as string:
|
|
|
27
28
|
pip install free-proxy
|
|
28
29
|
```
|
|
29
30
|
|
|
31
|
+
With async support (installs aiohttp):
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
pip install "free-proxy[async]"
|
|
35
|
+
```
|
|
36
|
+
|
|
30
37
|
[](https://asciinema.org/a/Xolpn3eD2tyJl8Y8HE9zolgex)
|
|
31
38
|
|
|
32
39
|
### Usage examples
|
|
@@ -151,9 +158,55 @@ proxy = FreeProxy(url='http://httpbin.org/get').get()
|
|
|
151
158
|
proxy = FreeProxy(request_timeout=5).get()
|
|
152
159
|
```
|
|
153
160
|
|
|
161
|
+
## Async usage
|
|
162
|
+
|
|
163
|
+
`AsyncFreeProxy` is an asyncio variant of `FreeProxy`. It accepts the same options,
|
|
164
|
+
but checks proxies **concurrently** (up to `max_concurrent` at a time, default 20)
|
|
165
|
+
and returns the first working one, which is usually much faster than the
|
|
166
|
+
synchronous version. Remaining checks are cancelled as soon as a winner is found.
|
|
167
|
+
|
|
168
|
+
Async support requires the `async` extra:
|
|
169
|
+
|
|
170
|
+
```python
|
|
171
|
+
pip install "free-proxy[async]"
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
In a plain script, run it with `asyncio.run`:
|
|
175
|
+
|
|
176
|
+
```python
|
|
177
|
+
import asyncio
|
|
178
|
+
|
|
179
|
+
from fp.fp import AsyncFreeProxy
|
|
180
|
+
|
|
181
|
+
proxy = asyncio.run(AsyncFreeProxy().get())
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
Inside your own async application (or in a Jupyter cell), just await it:
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
proxy = await AsyncFreeProxy(country_id=['US'], max_concurrent=50).get()
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
- **`max_concurrent` parameter** (`AsyncFreeProxy` only)
|
|
191
|
+
Maximum number of proxies checked at the same time. Defaults to `max_concurrent=20`.
|
|
192
|
+
Raise it for faster scanning on a strong connection, lower it to be gentler on
|
|
193
|
+
your network and the test URL.
|
|
194
|
+
|
|
154
195
|
## CHANGELOG
|
|
155
196
|
|
|
156
197
|
---
|
|
198
|
+
## [1.3.0] - 2026-09-06
|
|
199
|
+
|
|
200
|
+
- Added `AsyncFreeProxy` — an asyncio variant of `FreeProxy` that checks proxies concurrently and returns the first working one (based on the idea from #39, thanks @NerdzzyDev)
|
|
201
|
+
- Async support is an optional extra: `pip install "free-proxy[async]"` (installs aiohttp)
|
|
202
|
+
- Added `max_concurrent` parameter to `AsyncFreeProxy` (default: 20) limiting how many proxies are checked at once
|
|
203
|
+
- Raised minimum Python version to 3.9
|
|
204
|
+
|
|
205
|
+
## [1.2.3] - 2026-08-24
|
|
206
|
+
|
|
207
|
+
- Fixed the proxy check silently bypassing the proxy when the test URL scheme did not match the `https` setting. With default settings (HTTP proxy mapping + HTTPS test URL) `requests` selected no proxy at all, the peername verification then rejected every candidate and `get()` always failed. The proxies mapping now covers both `http` and `https` schemes. Broken since 1.2.1. (#61)
|
|
208
|
+
- Added a CI workflow running the test suite on Python 3.9–3.13
|
|
209
|
+
|
|
157
210
|
## [1.2.2] - 2026-07-04
|
|
158
211
|
|
|
159
212
|
- Fixed `get_proxy_list` hanging indefinitely when a source website stalls — the proxy list request now has a timeout (#58)
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import random
|
|
5
|
+
|
|
6
|
+
import lxml.html as lh
|
|
7
|
+
import requests
|
|
8
|
+
|
|
9
|
+
from fp.errors import FreeProxyException
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
import aiohttp
|
|
13
|
+
except ImportError:
|
|
14
|
+
# Optional dependency -- install with: pip install "free-proxy[async]"
|
|
15
|
+
aiohttp = None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class FreeProxy:
|
|
19
|
+
'''
|
|
20
|
+
FreeProxy class scrapes proxies from <https://www.sslproxies.org/>,
|
|
21
|
+
<https://www.us-proxy.org/>, <https://free-proxy-list.net/uk-proxy.html>,
|
|
22
|
+
and <https://free-proxy-list.net> and checks if proxy is working.
|
|
23
|
+
There is possibility to filter proxies by country and acceptable timeout.
|
|
24
|
+
You can also randomize list of proxies from where script would get first
|
|
25
|
+
working proxy.
|
|
26
|
+
'''
|
|
27
|
+
|
|
28
|
+
def __init__(self, country_id=None, timeout=0.5, rand=False, anonym=False, elite=False, google=None, https=False, url='https://www.google.com', request_timeout=10):
|
|
29
|
+
self.country_id = country_id
|
|
30
|
+
self.timeout = timeout
|
|
31
|
+
self.request_timeout = request_timeout
|
|
32
|
+
self.random = rand
|
|
33
|
+
self.anonym = anonym
|
|
34
|
+
self.elite = elite
|
|
35
|
+
self.google = google
|
|
36
|
+
self.schema = 'https' if https else 'http'
|
|
37
|
+
self.url = url
|
|
38
|
+
|
|
39
|
+
def get_proxy_list(self, repeat):
|
|
40
|
+
try:
|
|
41
|
+
page = requests.get(self._website(repeat), timeout=self.request_timeout)
|
|
42
|
+
except requests.exceptions.RequestException as e:
|
|
43
|
+
raise FreeProxyException(
|
|
44
|
+
f'Request to {self._website(repeat)} failed') from e
|
|
45
|
+
return self._parse_proxy_list(page.content)
|
|
46
|
+
|
|
47
|
+
def _parse_proxy_list(self, content):
|
|
48
|
+
try:
|
|
49
|
+
doc = lh.fromstring(content)
|
|
50
|
+
tr_elements = doc.xpath('//*[@id="list"]//tr')
|
|
51
|
+
return [f'{tr_elements[i][0].text_content()}:{tr_elements[i][1].text_content()}'
|
|
52
|
+
for i in range(1, len(tr_elements)) if self._criteria(tr_elements[i])]
|
|
53
|
+
except Exception as e:
|
|
54
|
+
raise FreeProxyException('Failed to get list of proxies') from e
|
|
55
|
+
|
|
56
|
+
def _website(self, repeat):
|
|
57
|
+
if repeat:
|
|
58
|
+
return "https://free-proxy-list.net"
|
|
59
|
+
elif self.country_id == ['US']:
|
|
60
|
+
return 'https://www.us-proxy.org'
|
|
61
|
+
elif self.country_id == ['GB']:
|
|
62
|
+
return 'https://free-proxy-list.net/uk-proxy.html'
|
|
63
|
+
else:
|
|
64
|
+
return 'https://www.sslproxies.org'
|
|
65
|
+
|
|
66
|
+
def _criteria(self, row_elements):
|
|
67
|
+
country_criteria = True if not self.country_id else row_elements[2].text_content(
|
|
68
|
+
) in self.country_id
|
|
69
|
+
elite_criteria = True if not self.elite else 'elite' in row_elements[4].text_content(
|
|
70
|
+
)
|
|
71
|
+
anonym_criteria = True if (
|
|
72
|
+
not self.anonym) or self.elite else 'anonymous' == row_elements[4].text_content()
|
|
73
|
+
switch = {'yes': True, 'no': False}
|
|
74
|
+
google_criteria = True if self.google is None else self.google == switch.get(
|
|
75
|
+
row_elements[5].text_content())
|
|
76
|
+
https_criteria = True if self.schema == 'http' else row_elements[6].text_content(
|
|
77
|
+
).lower() == 'yes'
|
|
78
|
+
return country_criteria and elite_criteria and anonym_criteria and google_criteria and https_criteria
|
|
79
|
+
|
|
80
|
+
def get(self, repeat=False):
|
|
81
|
+
'''Returns a working proxy that matches the specified parameters.'''
|
|
82
|
+
proxy_list = self.get_proxy_list(repeat)
|
|
83
|
+
if self.random:
|
|
84
|
+
random.shuffle(proxy_list)
|
|
85
|
+
working_proxy = None
|
|
86
|
+
for proxy_address in proxy_list:
|
|
87
|
+
# requests selects the proxy by the URL scheme, so the mapping
|
|
88
|
+
# must cover both -- otherwise e.g. an https test URL with the
|
|
89
|
+
# default http schema bypasses the proxy entirely.
|
|
90
|
+
proxies = {'http': f'http://{proxy_address}',
|
|
91
|
+
'https': f'http://{proxy_address}'}
|
|
92
|
+
try:
|
|
93
|
+
working_proxy = self.__check_if_proxy_is_working(proxies)
|
|
94
|
+
if working_proxy:
|
|
95
|
+
return working_proxy
|
|
96
|
+
except requests.exceptions.RequestException:
|
|
97
|
+
continue
|
|
98
|
+
if not working_proxy and not repeat:
|
|
99
|
+
if self.country_id is not None:
|
|
100
|
+
self.country_id = None
|
|
101
|
+
return self.get(repeat=True)
|
|
102
|
+
raise FreeProxyException(
|
|
103
|
+
'There are no working proxies at this time.')
|
|
104
|
+
|
|
105
|
+
def __check_if_proxy_is_working(self, proxies):
|
|
106
|
+
url = self.url
|
|
107
|
+
ip = proxies[self.schema].split(':')[1][2:]
|
|
108
|
+
with requests.get(url, proxies=proxies, timeout=self.timeout, stream=True) as r:
|
|
109
|
+
if r.raw.connection.sock and r.raw.connection.sock.getpeername()[0] == ip:
|
|
110
|
+
return proxies[self.schema]
|
|
111
|
+
return
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class AsyncFreeProxy(FreeProxy):
|
|
115
|
+
'''
|
|
116
|
+
Asynchronous variant of FreeProxy built on aiohttp. Requires the "async"
|
|
117
|
+
extra: pip install "free-proxy[async]".
|
|
118
|
+
|
|
119
|
+
Public methods mirror FreeProxy but are coroutines: call them with await
|
|
120
|
+
from your own event loop, or via asyncio.run() in a plain script:
|
|
121
|
+
|
|
122
|
+
proxy = asyncio.run(AsyncFreeProxy().get())
|
|
123
|
+
|
|
124
|
+
Proxies are checked concurrently (up to max_concurrent at a time) and the
|
|
125
|
+
first working one wins; remaining checks are cancelled.
|
|
126
|
+
'''
|
|
127
|
+
|
|
128
|
+
def __init__(self, *args, max_concurrent=20, **kwargs):
|
|
129
|
+
if aiohttp is None:
|
|
130
|
+
raise FreeProxyException(
|
|
131
|
+
'aiohttp is not installed. Install async support with: pip install "free-proxy[async]"')
|
|
132
|
+
if max_concurrent < 1:
|
|
133
|
+
# 0 would park every task on the semaphore forever -- the
|
|
134
|
+
# request timeout never starts while waiting for a slot.
|
|
135
|
+
raise ValueError('max_concurrent must be a positive integer')
|
|
136
|
+
super().__init__(*args, **kwargs)
|
|
137
|
+
self.max_concurrent = max_concurrent
|
|
138
|
+
|
|
139
|
+
async def get_proxy_list(self, repeat):
|
|
140
|
+
timeout = aiohttp.ClientTimeout(total=self.request_timeout)
|
|
141
|
+
try:
|
|
142
|
+
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
143
|
+
async with session.get(self._website(repeat)) as response:
|
|
144
|
+
content = await response.read()
|
|
145
|
+
except (aiohttp.ClientError, asyncio.TimeoutError) as e:
|
|
146
|
+
raise FreeProxyException(
|
|
147
|
+
f'Request to {self._website(repeat)} failed') from e
|
|
148
|
+
return self._parse_proxy_list(content)
|
|
149
|
+
|
|
150
|
+
async def get(self, repeat=False):
|
|
151
|
+
'''Returns a working proxy that matches the specified parameters.'''
|
|
152
|
+
proxy_list = await self.get_proxy_list(repeat)
|
|
153
|
+
if self.random:
|
|
154
|
+
random.shuffle(proxy_list)
|
|
155
|
+
working_proxy = await self._find_working_proxy(proxy_list)
|
|
156
|
+
if working_proxy:
|
|
157
|
+
return working_proxy
|
|
158
|
+
if not repeat:
|
|
159
|
+
if self.country_id is not None:
|
|
160
|
+
self.country_id = None
|
|
161
|
+
return await self.get(repeat=True)
|
|
162
|
+
raise FreeProxyException(
|
|
163
|
+
'There are no working proxies at this time.')
|
|
164
|
+
|
|
165
|
+
async def _find_working_proxy(self, proxy_list):
|
|
166
|
+
semaphore = asyncio.Semaphore(self.max_concurrent)
|
|
167
|
+
timeout = aiohttp.ClientTimeout(total=self.timeout)
|
|
168
|
+
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
169
|
+
tasks = [asyncio.create_task(self._check_if_proxy_is_working(session, semaphore, proxy_address))
|
|
170
|
+
for proxy_address in proxy_list]
|
|
171
|
+
try:
|
|
172
|
+
for finished in asyncio.as_completed(tasks):
|
|
173
|
+
working_proxy = await finished
|
|
174
|
+
if working_proxy:
|
|
175
|
+
return working_proxy
|
|
176
|
+
return None
|
|
177
|
+
finally:
|
|
178
|
+
# Cancel unfinished checks (no-op for finished tasks) and let
|
|
179
|
+
# the cancellations settle before the session closes under them.
|
|
180
|
+
for task in tasks:
|
|
181
|
+
task.cancel()
|
|
182
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
183
|
+
|
|
184
|
+
async def _check_if_proxy_is_working(self, session, semaphore, proxy_address):
|
|
185
|
+
async with semaphore:
|
|
186
|
+
try:
|
|
187
|
+
async with session.get(self.url, proxy=f'http://{proxy_address}') as response:
|
|
188
|
+
connection = response.connection
|
|
189
|
+
if connection is None or connection.transport is None:
|
|
190
|
+
return None
|
|
191
|
+
peername = connection.transport.get_extra_info('peername')
|
|
192
|
+
if peername and peername[0] == proxy_address.split(':')[0]:
|
|
193
|
+
return f'http://{proxy_address}'
|
|
194
|
+
except (aiohttp.ClientError, asyncio.TimeoutError, OSError):
|
|
195
|
+
pass
|
|
196
|
+
return None
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: free_proxy
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Proxy scraper for further use
|
|
5
5
|
Home-page: https://github.com/jundymek/free-proxy
|
|
6
6
|
Author: jundymek
|
|
@@ -8,11 +8,13 @@ Author-email: jundymek@gmail.com
|
|
|
8
8
|
Classifier: Programming Language :: Python :: 3
|
|
9
9
|
Classifier: License :: OSI Approved :: MIT License
|
|
10
10
|
Classifier: Operating System :: OS Independent
|
|
11
|
-
Requires-Python: >=3.
|
|
11
|
+
Requires-Python: >=3.9
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
14
|
Requires-Dist: lxml<7,>=5.0
|
|
15
15
|
Requires-Dist: requests<3,>=2.31
|
|
16
|
+
Provides-Extra: async
|
|
17
|
+
Requires-Dist: aiohttp<4,>=3.8; extra == "async"
|
|
16
18
|
Dynamic: author
|
|
17
19
|
Dynamic: author-email
|
|
18
20
|
Dynamic: classifier
|
|
@@ -20,13 +22,14 @@ Dynamic: description
|
|
|
20
22
|
Dynamic: description-content-type
|
|
21
23
|
Dynamic: home-page
|
|
22
24
|
Dynamic: license-file
|
|
25
|
+
Dynamic: provides-extra
|
|
23
26
|
Dynamic: requires-dist
|
|
24
27
|
Dynamic: requires-python
|
|
25
28
|
Dynamic: summary
|
|
26
29
|
|
|
27
30
|
# Free-proxy
|
|
28
31
|
|
|
29
|
-

|
|
30
33
|
|
|
31
34
|
## Get free working proxies from <https://www.sslproxies.org/>, <https://www.us-proxy.org/>, <https://free-proxy-list.net/uk-proxy.html> and <https://free-proxy-list.net> and use them in your script
|
|
32
35
|
|
|
@@ -43,9 +46,10 @@ Returns proxy as string:
|
|
|
43
46
|
|
|
44
47
|
### Requirements
|
|
45
48
|
|
|
46
|
-
- [Python 3](https://www.python.org/downloads/)
|
|
49
|
+
- [Python 3](https://www.python.org/downloads/) (3.9+)
|
|
47
50
|
- [Requests library](https://github.com/psf/requests)
|
|
48
51
|
- [Lxml library](https://github.com/lxml/lxml)
|
|
52
|
+
- [Aiohttp library](https://github.com/aio-libs/aiohttp) (optional, only for async usage)
|
|
49
53
|
|
|
50
54
|
### Installation
|
|
51
55
|
|
|
@@ -53,6 +57,12 @@ Returns proxy as string:
|
|
|
53
57
|
pip install free-proxy
|
|
54
58
|
```
|
|
55
59
|
|
|
60
|
+
With async support (installs aiohttp):
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
pip install "free-proxy[async]"
|
|
64
|
+
```
|
|
65
|
+
|
|
56
66
|
[](https://asciinema.org/a/Xolpn3eD2tyJl8Y8HE9zolgex)
|
|
57
67
|
|
|
58
68
|
### Usage examples
|
|
@@ -177,9 +187,55 @@ proxy = FreeProxy(url='http://httpbin.org/get').get()
|
|
|
177
187
|
proxy = FreeProxy(request_timeout=5).get()
|
|
178
188
|
```
|
|
179
189
|
|
|
190
|
+
## Async usage
|
|
191
|
+
|
|
192
|
+
`AsyncFreeProxy` is an asyncio variant of `FreeProxy`. It accepts the same options,
|
|
193
|
+
but checks proxies **concurrently** (up to `max_concurrent` at a time, default 20)
|
|
194
|
+
and returns the first working one, which is usually much faster than the
|
|
195
|
+
synchronous version. Remaining checks are cancelled as soon as a winner is found.
|
|
196
|
+
|
|
197
|
+
Async support requires the `async` extra:
|
|
198
|
+
|
|
199
|
+
```python
|
|
200
|
+
pip install "free-proxy[async]"
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
In a plain script, run it with `asyncio.run`:
|
|
204
|
+
|
|
205
|
+
```python
|
|
206
|
+
import asyncio
|
|
207
|
+
|
|
208
|
+
from fp.fp import AsyncFreeProxy
|
|
209
|
+
|
|
210
|
+
proxy = asyncio.run(AsyncFreeProxy().get())
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Inside your own async application (or in a Jupyter cell), just await it:
|
|
214
|
+
|
|
215
|
+
```python
|
|
216
|
+
proxy = await AsyncFreeProxy(country_id=['US'], max_concurrent=50).get()
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
- **`max_concurrent` parameter** (`AsyncFreeProxy` only)
|
|
220
|
+
Maximum number of proxies checked at the same time. Defaults to `max_concurrent=20`.
|
|
221
|
+
Raise it for faster scanning on a strong connection, lower it to be gentler on
|
|
222
|
+
your network and the test URL.
|
|
223
|
+
|
|
180
224
|
## CHANGELOG
|
|
181
225
|
|
|
182
226
|
---
|
|
227
|
+
## [1.3.0] - 2026-09-06
|
|
228
|
+
|
|
229
|
+
- Added `AsyncFreeProxy` — an asyncio variant of `FreeProxy` that checks proxies concurrently and returns the first working one (based on the idea from #39, thanks @NerdzzyDev)
|
|
230
|
+
- Async support is an optional extra: `pip install "free-proxy[async]"` (installs aiohttp)
|
|
231
|
+
- Added `max_concurrent` parameter to `AsyncFreeProxy` (default: 20) limiting how many proxies are checked at once
|
|
232
|
+
- Raised minimum Python version to 3.9
|
|
233
|
+
|
|
234
|
+
## [1.2.3] - 2026-08-24
|
|
235
|
+
|
|
236
|
+
- Fixed the proxy check silently bypassing the proxy when the test URL scheme did not match the `https` setting. With default settings (HTTP proxy mapping + HTTPS test URL) `requests` selected no proxy at all, the peername verification then rejected every candidate and `get()` always failed. The proxies mapping now covers both `http` and `https` schemes. Broken since 1.2.1. (#61)
|
|
237
|
+
- Added a CI workflow running the test suite on Python 3.9–3.13
|
|
238
|
+
|
|
183
239
|
## [1.2.2] - 2026-07-04
|
|
184
240
|
|
|
185
241
|
- Fixed `get_proxy_list` hanging indefinitely when a source website stalls — the proxy list request now has a timeout (#58)
|
|
@@ -4,7 +4,7 @@ with open("README.md", "r") as fh:
|
|
|
4
4
|
long_description = fh.read()
|
|
5
5
|
setuptools.setup(
|
|
6
6
|
name='free_proxy',
|
|
7
|
-
version='1.
|
|
7
|
+
version='1.3.0',
|
|
8
8
|
author="jundymek",
|
|
9
9
|
author_email="jundymek@gmail.com",
|
|
10
10
|
description="Proxy scraper for further use",
|
|
@@ -17,6 +17,9 @@ setuptools.setup(
|
|
|
17
17
|
"License :: OSI Approved :: MIT License",
|
|
18
18
|
"Operating System :: OS Independent",
|
|
19
19
|
],
|
|
20
|
-
python_requires='>=3.
|
|
21
|
-
install_requires=['lxml>=5.0,<7', 'requests>=2.31,<3']
|
|
20
|
+
python_requires='>=3.9',
|
|
21
|
+
install_requires=['lxml>=5.0,<7', 'requests>=2.31,<3'],
|
|
22
|
+
extras_require={
|
|
23
|
+
'async': ['aiohttp>=3.8,<4'],
|
|
24
|
+
}
|
|
22
25
|
)
|
free_proxy-1.2.2/fp/fp.py
DELETED
|
@@ -1,97 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
|
|
3
|
-
import random
|
|
4
|
-
|
|
5
|
-
import lxml.html as lh
|
|
6
|
-
import requests
|
|
7
|
-
|
|
8
|
-
from fp.errors import FreeProxyException
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class FreeProxy:
|
|
12
|
-
'''
|
|
13
|
-
FreeProxy class scrapes proxies from <https://www.sslproxies.org/>,
|
|
14
|
-
<https://www.us-proxy.org/>, <https://free-proxy-list.net/uk-proxy.html>,
|
|
15
|
-
and <https://free-proxy-list.net> and checks if proxy is working.
|
|
16
|
-
There is possibility to filter proxies by country and acceptable timeout.
|
|
17
|
-
You can also randomize list of proxies from where script would get first
|
|
18
|
-
working proxy.
|
|
19
|
-
'''
|
|
20
|
-
|
|
21
|
-
def __init__(self, country_id=None, timeout=0.5, rand=False, anonym=False, elite=False, google=None, https=False, url='https://www.google.com', request_timeout=10):
|
|
22
|
-
self.country_id = country_id
|
|
23
|
-
self.timeout = timeout
|
|
24
|
-
self.request_timeout = request_timeout
|
|
25
|
-
self.random = rand
|
|
26
|
-
self.anonym = anonym
|
|
27
|
-
self.elite = elite
|
|
28
|
-
self.google = google
|
|
29
|
-
self.schema = 'https' if https else 'http'
|
|
30
|
-
self.url = url
|
|
31
|
-
|
|
32
|
-
def get_proxy_list(self, repeat):
|
|
33
|
-
try:
|
|
34
|
-
page = requests.get(self.__website(repeat), timeout=self.request_timeout)
|
|
35
|
-
doc = lh.fromstring(page.content)
|
|
36
|
-
except requests.exceptions.RequestException as e:
|
|
37
|
-
raise FreeProxyException(
|
|
38
|
-
f'Request to {self.__website(repeat)} failed') from e
|
|
39
|
-
try:
|
|
40
|
-
tr_elements = doc.xpath('//*[@id="list"]//tr')
|
|
41
|
-
return [f'{tr_elements[i][0].text_content()}:{tr_elements[i][1].text_content()}'
|
|
42
|
-
for i in range(1, len(tr_elements)) if self.__criteria(tr_elements[i])]
|
|
43
|
-
except Exception as e:
|
|
44
|
-
raise FreeProxyException('Failed to get list of proxies') from e
|
|
45
|
-
|
|
46
|
-
def __website(self, repeat):
|
|
47
|
-
if repeat:
|
|
48
|
-
return "https://free-proxy-list.net"
|
|
49
|
-
elif self.country_id == ['US']:
|
|
50
|
-
return 'https://www.us-proxy.org'
|
|
51
|
-
elif self.country_id == ['GB']:
|
|
52
|
-
return 'https://free-proxy-list.net/uk-proxy.html'
|
|
53
|
-
else:
|
|
54
|
-
return 'https://www.sslproxies.org'
|
|
55
|
-
|
|
56
|
-
def __criteria(self, row_elements):
|
|
57
|
-
country_criteria = True if not self.country_id else row_elements[2].text_content(
|
|
58
|
-
) in self.country_id
|
|
59
|
-
elite_criteria = True if not self.elite else 'elite' in row_elements[4].text_content(
|
|
60
|
-
)
|
|
61
|
-
anonym_criteria = True if (
|
|
62
|
-
not self.anonym) or self.elite else 'anonymous' == row_elements[4].text_content()
|
|
63
|
-
switch = {'yes': True, 'no': False}
|
|
64
|
-
google_criteria = True if self.google is None else self.google == switch.get(
|
|
65
|
-
row_elements[5].text_content())
|
|
66
|
-
https_criteria = True if self.schema == 'http' else row_elements[6].text_content(
|
|
67
|
-
).lower() == 'yes'
|
|
68
|
-
return country_criteria and elite_criteria and anonym_criteria and google_criteria and https_criteria
|
|
69
|
-
|
|
70
|
-
def get(self, repeat=False):
|
|
71
|
-
'''Returns a working proxy that matches the specified parameters.'''
|
|
72
|
-
proxy_list = self.get_proxy_list(repeat)
|
|
73
|
-
if self.random:
|
|
74
|
-
random.shuffle(proxy_list)
|
|
75
|
-
working_proxy = None
|
|
76
|
-
for proxy_address in proxy_list:
|
|
77
|
-
proxies = {self.schema: f'http://{proxy_address}'}
|
|
78
|
-
try:
|
|
79
|
-
working_proxy = self.__check_if_proxy_is_working(proxies)
|
|
80
|
-
if working_proxy:
|
|
81
|
-
return working_proxy
|
|
82
|
-
except requests.exceptions.RequestException:
|
|
83
|
-
continue
|
|
84
|
-
if not working_proxy and not repeat:
|
|
85
|
-
if self.country_id is not None:
|
|
86
|
-
self.country_id = None
|
|
87
|
-
return self.get(repeat=True)
|
|
88
|
-
raise FreeProxyException(
|
|
89
|
-
'There are no working proxies at this time.')
|
|
90
|
-
|
|
91
|
-
def __check_if_proxy_is_working(self, proxies):
|
|
92
|
-
url = self.url
|
|
93
|
-
ip = proxies[self.schema].split(':')[1][2:]
|
|
94
|
-
with requests.get(url, proxies=proxies, timeout=self.timeout, stream=True) as r:
|
|
95
|
-
if r.raw.connection.sock and r.raw.connection.sock.getpeername()[0] == ip:
|
|
96
|
-
return proxies[self.schema]
|
|
97
|
-
return
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|