proxmint 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- proxmint-0.1.0/LICENSE +21 -0
- proxmint-0.1.0/PKG-INFO +118 -0
- proxmint-0.1.0/README.md +93 -0
- proxmint-0.1.0/pyproject.toml +42 -0
- proxmint-0.1.0/src/proxmint/__init__.py +141 -0
- proxmint-0.1.0/src/proxmint/aiohttp.py +52 -0
- proxmint-0.1.0/src/proxmint/httpx.py +88 -0
- proxmint-0.1.0/tests/test_proxmint.py +112 -0
proxmint-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Proxmint
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
proxmint-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: proxmint
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Free proxy list re-checked every 30 minutes, with rotating-proxy middleware for aiohttp and a rotating transport for httpx.
|
|
5
|
+
Project-URL: Homepage, https://proxmint.com/docs/python
|
|
6
|
+
Project-URL: Repository, https://github.com/proxmint/free-proxy-list
|
|
7
|
+
Project-URL: Issues, https://github.com/proxmint/free-proxy-list/issues
|
|
8
|
+
Author-email: Proxmint <support@proxmint.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: aiohttp,free-proxy,httpx,proxies,proxy,proxy-list,rotating-proxy,socks5,web-scraping
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Framework :: AsyncIO
|
|
14
|
+
Classifier: Framework :: aiohttp
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Topic :: Internet :: Proxy Servers
|
|
18
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Provides-Extra: aiohttp
|
|
21
|
+
Requires-Dist: aiohttp>=3.12; extra == 'aiohttp'
|
|
22
|
+
Provides-Extra: httpx
|
|
23
|
+
Requires-Dist: httpx>=0.27; extra == 'httpx'
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# proxmint
|
|
27
|
+
|
|
28
|
+
Free proxy list for Python, re-checked every 30 minutes, plus proxy rotation for aiohttp and httpx.
|
|
29
|
+
The client has no dependencies and needs no API key or signup.
|
|
30
|
+
|
|
31
|
+
Every proxy the API returns was proven alive by a real request made *through it* within the last half hour,
|
|
32
|
+
not pinged and not assumed. Dead entries are dropped rather than left to rot.
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install proxmint # the client, standard library only
|
|
36
|
+
pip install "proxmint[aiohttp]" # + rotating-proxy middleware for aiohttp
|
|
37
|
+
pip install "proxmint[httpx]" # + rotating transports for httpx
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Free proxy list
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from proxmint import free_proxies, proxy_urls
|
|
44
|
+
|
|
45
|
+
proxies = free_proxies(protocol="socks5", country="de", limit=50)
|
|
46
|
+
# [{"ip": "203.0.113.7", "port": 1080, "protocol": "socks5", "countryCode": "de",
|
|
47
|
+
# "countryName": "Germany", "anonymity": "elite", "latencyMs": 412, "uptimePct": 97.5,
|
|
48
|
+
# "score": 91, "lastCheckedAt": "2026-10-28T10:56:12.583Z"}, ...]
|
|
49
|
+
|
|
50
|
+
proxy_urls(sort="latency", limit=10)
|
|
51
|
+
# ["http://198.51.100.4:3128", "socks5://203.0.113.7:1080", ...]
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
| Filter | |
|
|
55
|
+
|---|---|
|
|
56
|
+
| `protocol` | `http` · `https` · `socks4` · `socks5` |
|
|
57
|
+
| `country` | ISO 3166-1 alpha-2, e.g. `de` |
|
|
58
|
+
| `anonymity` | `elite` · `anonymous` · `transparent` |
|
|
59
|
+
| `sort` | `score` (default) · `latency` · `uptime` · `checked` |
|
|
60
|
+
| `limit` | rows wanted (default 20); paged past the API's 200-row cap |
|
|
61
|
+
|
|
62
|
+
An `https` proxy is an HTTP proxy that can also tunnel TLS, so its URL starts with `http://`. `proxy_url(row)` builds
|
|
63
|
+
the URL for one row. Failures raise `ProxmintError` with a `status`. The API allows 60 requests a minute per IP.
|
|
64
|
+
|
|
65
|
+
## Rotating proxies with aiohttp
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
import aiohttp
|
|
69
|
+
from proxmint.aiohttp import RotatingProxy
|
|
70
|
+
|
|
71
|
+
async with aiohttp.ClientSession(middlewares=(RotatingProxy(),)) as session:
|
|
72
|
+
async with session.get("https://example.com") as resp:
|
|
73
|
+
print(resp.status)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Each request goes out through the next proxy in the pool. A proxy that refuses the connection, times out, drops it or
|
|
77
|
+
answers `407` is removed, and the request is retried through another, up to `tries=3` proxies. Each proxy gets
|
|
78
|
+
`timeout=10` seconds to connect and send back headers. A session-wide `ClientTimeout(total=...)` covers all the tries
|
|
79
|
+
together, so keep it loose.
|
|
80
|
+
|
|
81
|
+
Without a list of your own, the pool is the 50 best `http` proxies on the free list, fetched again whenever it runs
|
|
82
|
+
dry. The filters above narrow it: `RotatingProxy(country="us", sort="latency")`. To rotate through your own proxies,
|
|
83
|
+
pass them: `RotatingProxy(["http://user:pass@203.0.113.7:3128", ...])`.
|
|
84
|
+
|
|
85
|
+
Needs aiohttp 3.12 or later (client middleware). aiohttp speaks HTTP proxies only.
|
|
86
|
+
|
|
87
|
+
## Rotating proxies with httpx
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
import httpx
|
|
91
|
+
from proxmint.httpx import AsyncRotatingTransport, RotatingTransport
|
|
92
|
+
|
|
93
|
+
with httpx.Client(transport=RotatingTransport(country="us")) as client:
|
|
94
|
+
print(client.get("https://example.com").status_code)
|
|
95
|
+
|
|
96
|
+
async with httpx.AsyncClient(transport=AsyncRotatingTransport()) as client:
|
|
97
|
+
print((await client.get("https://example.com")).status_code)
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Same rotation and the same arguments. For SOCKS, `pip install "httpx[socks]"` and pass `protocol="socks5"` or your
|
|
101
|
+
own `socks5://` URLs.
|
|
102
|
+
|
|
103
|
+
Retries resend the request body, so a streamed upload can't be retried.
|
|
104
|
+
|
|
105
|
+
## Warning
|
|
106
|
+
|
|
107
|
+
These are free public proxies run by strangers. They are slow, short-lived, and some are honeypots that log or tamper
|
|
108
|
+
with traffic. Use them for testing and research. **Never send logged-in or sensitive traffic through one.** We verify
|
|
109
|
+
that a proxy forwards traffic, nothing more. We do not operate, audit or endorse any of them.
|
|
110
|
+
|
|
111
|
+
Needing proxies that stay up is a different problem, and the one [Proxmint](https://proxmint.com) sells.
|
|
112
|
+
|
|
113
|
+
## Also
|
|
114
|
+
|
|
115
|
+
- Docs: <https://proxmint.com/docs/python>
|
|
116
|
+
- Plain text and JSON files, committed every 30 minutes: [github.com/proxmint/free-proxy-list](https://github.com/proxmint/free-proxy-list)
|
|
117
|
+
- HTTP API: <https://proxmint.com/free-proxies#api>
|
|
118
|
+
- The same list from the command line: `npx proxmint`, or `sudo snap install proxmint`
|
proxmint-0.1.0/README.md
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# proxmint
|
|
2
|
+
|
|
3
|
+
Free proxy list for Python, re-checked every 30 minutes, plus proxy rotation for aiohttp and httpx.
|
|
4
|
+
The client has no dependencies and needs no API key or signup.
|
|
5
|
+
|
|
6
|
+
Every proxy the API returns was proven alive by a real request made *through it* within the last half hour,
|
|
7
|
+
not pinged and not assumed. Dead entries are dropped rather than left to rot.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install proxmint # the client, standard library only
|
|
11
|
+
pip install "proxmint[aiohttp]" # + rotating-proxy middleware for aiohttp
|
|
12
|
+
pip install "proxmint[httpx]" # + rotating transports for httpx
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Free proxy list
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
from proxmint import free_proxies, proxy_urls
|
|
19
|
+
|
|
20
|
+
proxies = free_proxies(protocol="socks5", country="de", limit=50)
|
|
21
|
+
# [{"ip": "203.0.113.7", "port": 1080, "protocol": "socks5", "countryCode": "de",
|
|
22
|
+
# "countryName": "Germany", "anonymity": "elite", "latencyMs": 412, "uptimePct": 97.5,
|
|
23
|
+
# "score": 91, "lastCheckedAt": "2026-10-28T10:56:12.583Z"}, ...]
|
|
24
|
+
|
|
25
|
+
proxy_urls(sort="latency", limit=10)
|
|
26
|
+
# ["http://198.51.100.4:3128", "socks5://203.0.113.7:1080", ...]
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
| Filter | |
|
|
30
|
+
|---|---|
|
|
31
|
+
| `protocol` | `http` · `https` · `socks4` · `socks5` |
|
|
32
|
+
| `country` | ISO 3166-1 alpha-2, e.g. `de` |
|
|
33
|
+
| `anonymity` | `elite` · `anonymous` · `transparent` |
|
|
34
|
+
| `sort` | `score` (default) · `latency` · `uptime` · `checked` |
|
|
35
|
+
| `limit` | rows wanted (default 20); paged past the API's 200-row cap |
|
|
36
|
+
|
|
37
|
+
An `https` proxy is an HTTP proxy that can also tunnel TLS, so its URL starts with `http://`. `proxy_url(row)` builds
|
|
38
|
+
the URL for one row. Failures raise `ProxmintError` with a `status`. The API allows 60 requests a minute per IP.
|
|
39
|
+
|
|
40
|
+
## Rotating proxies with aiohttp
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
import aiohttp
|
|
44
|
+
from proxmint.aiohttp import RotatingProxy
|
|
45
|
+
|
|
46
|
+
async with aiohttp.ClientSession(middlewares=(RotatingProxy(),)) as session:
|
|
47
|
+
async with session.get("https://example.com") as resp:
|
|
48
|
+
print(resp.status)
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Each request goes out through the next proxy in the pool. A proxy that refuses the connection, times out, drops it or
|
|
52
|
+
answers `407` is removed, and the request is retried through another, up to `tries=3` proxies. Each proxy gets
|
|
53
|
+
`timeout=10` seconds to connect and send back headers. A session-wide `ClientTimeout(total=...)` covers all the tries
|
|
54
|
+
together, so keep it loose.
|
|
55
|
+
|
|
56
|
+
Without a list of your own, the pool is the 50 best `http` proxies on the free list, fetched again whenever it runs
|
|
57
|
+
dry. The filters above narrow it: `RotatingProxy(country="us", sort="latency")`. To rotate through your own proxies,
|
|
58
|
+
pass them: `RotatingProxy(["http://user:pass@203.0.113.7:3128", ...])`.
|
|
59
|
+
|
|
60
|
+
Needs aiohttp 3.12 or later (client middleware). aiohttp speaks HTTP proxies only.
|
|
61
|
+
|
|
62
|
+
## Rotating proxies with httpx
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
import httpx
|
|
66
|
+
from proxmint.httpx import AsyncRotatingTransport, RotatingTransport
|
|
67
|
+
|
|
68
|
+
with httpx.Client(transport=RotatingTransport(country="us")) as client:
|
|
69
|
+
print(client.get("https://example.com").status_code)
|
|
70
|
+
|
|
71
|
+
async with httpx.AsyncClient(transport=AsyncRotatingTransport()) as client:
|
|
72
|
+
print((await client.get("https://example.com")).status_code)
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Same rotation and the same arguments. For SOCKS, `pip install "httpx[socks]"` and pass `protocol="socks5"` or your
|
|
76
|
+
own `socks5://` URLs.
|
|
77
|
+
|
|
78
|
+
Retries resend the request body, so a streamed upload can't be retried.
|
|
79
|
+
|
|
80
|
+
## Warning
|
|
81
|
+
|
|
82
|
+
These are free public proxies run by strangers. They are slow, short-lived, and some are honeypots that log or tamper
|
|
83
|
+
with traffic. Use them for testing and research. **Never send logged-in or sensitive traffic through one.** We verify
|
|
84
|
+
that a proxy forwards traffic, nothing more. We do not operate, audit or endorse any of them.
|
|
85
|
+
|
|
86
|
+
Needing proxies that stay up is a different problem, and the one [Proxmint](https://proxmint.com) sells.
|
|
87
|
+
|
|
88
|
+
## Also
|
|
89
|
+
|
|
90
|
+
- Docs: <https://proxmint.com/docs/python>
|
|
91
|
+
- Plain text and JSON files, committed every 30 minutes: [github.com/proxmint/free-proxy-list](https://github.com/proxmint/free-proxy-list)
|
|
92
|
+
- HTTP API: <https://proxmint.com/free-proxies#api>
|
|
93
|
+
- The same list from the command line: `npx proxmint`, or `sudo snap install proxmint`
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "proxmint"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Free proxy list re-checked every 30 minutes, with rotating-proxy middleware for aiohttp and a rotating transport for httpx."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.10"
|
|
9
|
+
authors = [{ name = "Proxmint", email = "support@proxmint.com" }]
|
|
10
|
+
keywords = ["proxy", "proxies", "free-proxy", "proxy-list", "rotating-proxy", "aiohttp", "httpx", "socks5", "web-scraping"]
|
|
11
|
+
classifiers = [
|
|
12
|
+
"Development Status :: 4 - Beta",
|
|
13
|
+
"Intended Audience :: Developers",
|
|
14
|
+
"Framework :: AsyncIO",
|
|
15
|
+
"Framework :: aiohttp",
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Topic :: Internet :: Proxy Servers",
|
|
18
|
+
"Topic :: Internet :: WWW/HTTP",
|
|
19
|
+
]
|
|
20
|
+
dependencies = []
|
|
21
|
+
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
aiohttp = ["aiohttp>=3.12"]
|
|
24
|
+
httpx = ["httpx>=0.27"]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://proxmint.com/docs/python"
|
|
28
|
+
Repository = "https://github.com/proxmint/free-proxy-list"
|
|
29
|
+
Issues = "https://github.com/proxmint/free-proxy-list/issues"
|
|
30
|
+
|
|
31
|
+
[dependency-groups]
|
|
32
|
+
test = ["pytest>=8", "aiohttp>=3.12", "httpx>=0.27"]
|
|
33
|
+
|
|
34
|
+
[build-system]
|
|
35
|
+
requires = ["hatchling>=1.27"]
|
|
36
|
+
build-backend = "hatchling.build"
|
|
37
|
+
|
|
38
|
+
[tool.hatch.build.targets.wheel]
|
|
39
|
+
packages = ["src/proxmint"]
|
|
40
|
+
|
|
41
|
+
[tool.pytest.ini_options]
|
|
42
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Client for the Proxmint free proxy list, plus proxy rotation for aiohttp and httpx.
|
|
2
|
+
|
|
3
|
+
Every proxy the API returns was proven alive by a real request made through it
|
|
4
|
+
within the last 30 minutes: https://proxmint.com/free-proxies. No key, no signup.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import asyncio
|
|
10
|
+
import json
|
|
11
|
+
import threading
|
|
12
|
+
import urllib.error
|
|
13
|
+
import urllib.parse
|
|
14
|
+
import urllib.request
|
|
15
|
+
|
|
16
|
+
__all__ = ["ProxmintError", "free_proxies", "proxy_url", "proxy_urls"]
|
|
17
|
+
__version__ = "0.1.0"
|
|
18
|
+
|
|
19
|
+
API = "https://proxmint.com/api/free-proxies"
|
|
20
|
+
MAX_PAGE_SIZE = 200 # the server caps a page here, so larger requests are paged
|
|
21
|
+
UA = f"proxmint-python/{__version__} (+https://proxmint.com/free-proxies)"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ProxmintError(Exception):
|
|
25
|
+
"""The API refused or could not be reached; `status` is the HTTP status, if any."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, message: str, status: int | None = None):
|
|
28
|
+
super().__init__(message)
|
|
29
|
+
self.status = status
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def free_proxies(
|
|
33
|
+
protocol: str | None = None,
|
|
34
|
+
country: str | None = None,
|
|
35
|
+
anonymity: str | None = None,
|
|
36
|
+
sort: str | None = None,
|
|
37
|
+
limit: int = 20,
|
|
38
|
+
timeout: float = 30,
|
|
39
|
+
) -> list[dict]:
|
|
40
|
+
"""Live proxies, best first, as dicts with ip, port, protocol, countryCode,
|
|
41
|
+
countryName, anonymity, latencyMs, uptimePct, score and lastCheckedAt.
|
|
42
|
+
|
|
43
|
+
protocol: "http", "https", "socks4" or "socks5". country: ISO 3166-1 alpha-2,
|
|
44
|
+
such as "de". anonymity: "elite", "anonymous" or "transparent". sort: "score"
|
|
45
|
+
(the default), "latency", "uptime" or "checked". limit is paged past the API's
|
|
46
|
+
200-row cap.
|
|
47
|
+
"""
|
|
48
|
+
if not isinstance(limit, int) or isinstance(limit, bool) or limit < 1:
|
|
49
|
+
raise ValueError(f"limit must be a positive integer, got {limit!r}")
|
|
50
|
+
out: list[dict] = []
|
|
51
|
+
page = 1
|
|
52
|
+
while len(out) < limit:
|
|
53
|
+
query = {
|
|
54
|
+
"protocol": protocol,
|
|
55
|
+
"country": country.lower() if country else None,
|
|
56
|
+
"anonymity": anonymity,
|
|
57
|
+
"sort": sort,
|
|
58
|
+
"page": page,
|
|
59
|
+
"pageSize": min(limit - len(out), MAX_PAGE_SIZE),
|
|
60
|
+
}
|
|
61
|
+
url = f"{API}?{urllib.parse.urlencode({k: v for k, v in query.items() if v is not None})}"
|
|
62
|
+
req = urllib.request.Request(url, headers={"Accept": "application/json", "User-Agent": UA})
|
|
63
|
+
try:
|
|
64
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
65
|
+
body = json.load(resp)
|
|
66
|
+
except urllib.error.HTTPError as e:
|
|
67
|
+
if e.code == 429:
|
|
68
|
+
raise ProxmintError("rate limited: the API allows 60 requests a minute per IP", 429) from None
|
|
69
|
+
raise ProxmintError(f"Proxmint API returned {e.code} {e.reason}", e.code) from None
|
|
70
|
+
except (urllib.error.URLError, TimeoutError) as e:
|
|
71
|
+
raise ProxmintError(f"could not reach the Proxmint API: {e}") from None
|
|
72
|
+
rows = body.get("rows") or []
|
|
73
|
+
out.extend(rows)
|
|
74
|
+
# A short page means the pool ended, not that the caller got what they asked for.
|
|
75
|
+
if not rows or len(out) >= (body.get("total") or 0):
|
|
76
|
+
break
|
|
77
|
+
page += 1
|
|
78
|
+
return out[:limit]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def proxy_url(proxy: dict) -> str:
|
|
82
|
+
"""The URL a client connects to for one row. An "https" proxy is an HTTP proxy
|
|
83
|
+
that can tunnel TLS with CONNECT, so it is reached over http://; an https:// URL
|
|
84
|
+
would make the client attempt TLS to the proxy itself."""
|
|
85
|
+
scheme = "http" if proxy["protocol"] == "https" else proxy["protocol"]
|
|
86
|
+
return f"{scheme}://{proxy['ip']}:{proxy['port']}"
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def proxy_urls(**filters) -> list[str]:
|
|
90
|
+
"""`free_proxies(**filters)` as proxy URLs, best first."""
|
|
91
|
+
return [proxy_url(p) for p in free_proxies(**filters)]
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class _Pool:
|
|
95
|
+
"""Round-robin over proxy URLs, shared by the aiohttp and httpx rotators. A proxy
|
|
96
|
+
that fails is dropped; an empty pool refills from the caller's list or, without
|
|
97
|
+
one, from a fresh fetch of the free list."""
|
|
98
|
+
|
|
99
|
+
def __init__(self, proxies, filters: dict):
|
|
100
|
+
self._fixed = None if proxies is None else [str(p) for p in proxies]
|
|
101
|
+
self._filters = {"protocol": "http", "limit": 50, **filters}
|
|
102
|
+
self._urls: list[str] = []
|
|
103
|
+
self._i = 0
|
|
104
|
+
self._lock = threading.Lock()
|
|
105
|
+
self._refilling = asyncio.Lock()
|
|
106
|
+
|
|
107
|
+
def refill(self) -> None:
|
|
108
|
+
urls = list(self._fixed) if self._fixed is not None else proxy_urls(**self._filters)
|
|
109
|
+
if not urls:
|
|
110
|
+
raise ProxmintError("no proxies to rotate through")
|
|
111
|
+
with self._lock:
|
|
112
|
+
self._urls, self._i = urls, 0
|
|
113
|
+
|
|
114
|
+
def _next(self) -> str | None:
|
|
115
|
+
with self._lock:
|
|
116
|
+
if not self._urls:
|
|
117
|
+
return None
|
|
118
|
+
self._i %= len(self._urls)
|
|
119
|
+
url = self._urls[self._i]
|
|
120
|
+
self._i += 1
|
|
121
|
+
return url
|
|
122
|
+
|
|
123
|
+
def take(self) -> str:
|
|
124
|
+
while (url := self._next()) is None:
|
|
125
|
+
self.refill()
|
|
126
|
+
return url
|
|
127
|
+
|
|
128
|
+
async def take_async(self) -> str:
|
|
129
|
+
while (url := self._next()) is None:
|
|
130
|
+
async with self._refilling: # one fetch, however many requests found the pool empty
|
|
131
|
+
if not self._urls:
|
|
132
|
+
await asyncio.to_thread(self.refill)
|
|
133
|
+
return url
|
|
134
|
+
|
|
135
|
+
def drop(self, url: str) -> None:
|
|
136
|
+
with self._lock:
|
|
137
|
+
if url in self._urls:
|
|
138
|
+
k = self._urls.index(url)
|
|
139
|
+
del self._urls[k]
|
|
140
|
+
if k < self._i: # keep the cursor on the proxy that was next
|
|
141
|
+
self._i -= 1
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""aiohttp client middleware that sends each request through the next proxy in a pool."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
|
|
7
|
+
import aiohttp
|
|
8
|
+
from yarl import URL
|
|
9
|
+
|
|
10
|
+
from . import _Pool
|
|
11
|
+
|
|
12
|
+
# What a dead, blocked or tampering proxy looks like from aiohttp's side.
|
|
13
|
+
PROXY_FAILURES = (aiohttp.ClientConnectionError, aiohttp.ClientHttpProxyError, asyncio.TimeoutError)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class RotatingProxy:
|
|
17
|
+
"""Client middleware (aiohttp 3.12+) that sends each request through the next
|
|
18
|
+
proxy in a pool. When a proxy fails, it is dropped and the request is retried
|
|
19
|
+
through another, up to `tries` proxies. Each proxy gets `timeout` seconds to
|
|
20
|
+
connect and answer with headers; a session-wide ClientTimeout(total=...) would
|
|
21
|
+
cover all the tries together, so one hanging proxy could use it up.
|
|
22
|
+
|
|
23
|
+
async with aiohttp.ClientSession(middlewares=(RotatingProxy(),)) as session:
|
|
24
|
+
async with session.get("https://example.com") as resp: ...
|
|
25
|
+
|
|
26
|
+
proxies: your own http:// proxy URLs, credentials included if they need them
|
|
27
|
+
(aiohttp has no SOCKS support). Without them the pool is the Proxmint free list,
|
|
28
|
+
filtered by `filters` (see `proxmint.free_proxies`; protocol defaults to "http")
|
|
29
|
+
and fetched again whenever it runs dry.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(self, proxies=None, *, tries: int = 3, timeout: float | None = 10, **filters):
|
|
33
|
+
self._pool = _Pool(proxies, filters)
|
|
34
|
+
self._tries = tries
|
|
35
|
+
self._timeout = timeout
|
|
36
|
+
|
|
37
|
+
async def __call__(self, req: aiohttp.ClientRequest, handler) -> aiohttp.ClientResponse:
|
|
38
|
+
for attempt in range(1, self._tries + 1):
|
|
39
|
+
proxy = await self._pool.take_async()
|
|
40
|
+
req.update_proxy(URL(proxy), None, req.proxy_headers)
|
|
41
|
+
try:
|
|
42
|
+
resp = await asyncio.wait_for(handler(req), self._timeout)
|
|
43
|
+
except PROXY_FAILURES:
|
|
44
|
+
self._pool.drop(proxy)
|
|
45
|
+
if attempt == self._tries:
|
|
46
|
+
raise
|
|
47
|
+
continue
|
|
48
|
+
if resp.status != 407 or attempt == self._tries:
|
|
49
|
+
return resp
|
|
50
|
+
resp.release() # the proxy wants credentials it was not given
|
|
51
|
+
self._pool.drop(proxy)
|
|
52
|
+
raise AssertionError("unreachable")
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""httpx transports that send each request through the next proxy in a pool."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
|
|
7
|
+
from . import _Pool
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class RotatingTransport(httpx.BaseTransport):
|
|
11
|
+
"""Sends each request through the next proxy in a pool. When a proxy fails, it
|
|
12
|
+
is dropped and the request is retried through another, up to `tries` proxies.
|
|
13
|
+
|
|
14
|
+
client = httpx.Client(transport=RotatingTransport())
|
|
15
|
+
|
|
16
|
+
proxies: your own proxy URLs, credentials included if they need them (socks5://
|
|
17
|
+
needs `httpx[socks]`). Without them the pool is the Proxmint free list, filtered
|
|
18
|
+
by `filters` (see `proxmint.free_proxies`; protocol defaults to "http") and
|
|
19
|
+
fetched again whenever it runs dry.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self, proxies=None, *, tries: int = 3, **filters):
|
|
23
|
+
self._pool = _Pool(proxies, filters)
|
|
24
|
+
self._tries = tries
|
|
25
|
+
self._transports: dict[str, httpx.HTTPTransport] = {}
|
|
26
|
+
|
|
27
|
+
def handle_request(self, request: httpx.Request) -> httpx.Response:
|
|
28
|
+
for attempt in range(1, self._tries + 1):
|
|
29
|
+
proxy = self._pool.take()
|
|
30
|
+
transport = self._transports.get(proxy) or self._transports.setdefault(proxy, httpx.HTTPTransport(proxy=proxy))
|
|
31
|
+
try:
|
|
32
|
+
response = transport.handle_request(request)
|
|
33
|
+
except httpx.TransportError:
|
|
34
|
+
self._drop(proxy)
|
|
35
|
+
if attempt == self._tries:
|
|
36
|
+
raise
|
|
37
|
+
continue
|
|
38
|
+
if response.status_code != 407 or attempt == self._tries:
|
|
39
|
+
return response
|
|
40
|
+
response.close() # the proxy wants credentials it was not given
|
|
41
|
+
self._drop(proxy)
|
|
42
|
+
raise AssertionError("unreachable")
|
|
43
|
+
|
|
44
|
+
def _drop(self, proxy: str) -> None:
|
|
45
|
+
self._pool.drop(proxy)
|
|
46
|
+
if (transport := self._transports.pop(proxy, None)) is not None:
|
|
47
|
+
transport.close()
|
|
48
|
+
|
|
49
|
+
def close(self) -> None:
|
|
50
|
+
for transport in self._transports.values():
|
|
51
|
+
transport.close()
|
|
52
|
+
self._transports.clear()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class AsyncRotatingTransport(httpx.AsyncBaseTransport):
|
|
56
|
+
"""`RotatingTransport` for `httpx.AsyncClient`."""
|
|
57
|
+
|
|
58
|
+
def __init__(self, proxies=None, *, tries: int = 3, **filters):
|
|
59
|
+
self._pool = _Pool(proxies, filters)
|
|
60
|
+
self._tries = tries
|
|
61
|
+
self._transports: dict[str, httpx.AsyncHTTPTransport] = {}
|
|
62
|
+
|
|
63
|
+
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
|
64
|
+
for attempt in range(1, self._tries + 1):
|
|
65
|
+
proxy = await self._pool.take_async()
|
|
66
|
+
transport = self._transports.get(proxy) or self._transports.setdefault(proxy, httpx.AsyncHTTPTransport(proxy=proxy))
|
|
67
|
+
try:
|
|
68
|
+
response = await transport.handle_async_request(request)
|
|
69
|
+
except httpx.TransportError:
|
|
70
|
+
await self._drop(proxy)
|
|
71
|
+
if attempt == self._tries:
|
|
72
|
+
raise
|
|
73
|
+
continue
|
|
74
|
+
if response.status_code != 407 or attempt == self._tries:
|
|
75
|
+
return response
|
|
76
|
+
await response.aclose()
|
|
77
|
+
await self._drop(proxy)
|
|
78
|
+
raise AssertionError("unreachable")
|
|
79
|
+
|
|
80
|
+
async def _drop(self, proxy: str) -> None:
|
|
81
|
+
self._pool.drop(proxy)
|
|
82
|
+
if (transport := self._transports.pop(proxy, None)) is not None:
|
|
83
|
+
await transport.aclose()
|
|
84
|
+
|
|
85
|
+
async def aclose(self) -> None:
|
|
86
|
+
for transport in self._transports.values():
|
|
87
|
+
await transport.aclose()
|
|
88
|
+
self._transports.clear()
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Offline checks: paging, URL schemes, and that both rotators drop a failing proxy
|
|
2
|
+
and retry through the next one."""
|
|
3
|
+
|
|
4
|
+
import asyncio
|
|
5
|
+
import io
|
|
6
|
+
import json
|
|
7
|
+
import types
|
|
8
|
+
import urllib.error
|
|
9
|
+
import urllib.parse
|
|
10
|
+
|
|
11
|
+
import aiohttp
|
|
12
|
+
import httpx
|
|
13
|
+
import pytest
|
|
14
|
+
|
|
15
|
+
import proxmint
|
|
16
|
+
import proxmint.aiohttp
|
|
17
|
+
import proxmint.httpx
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def stub_api(monkeypatch, total, status=200):
|
|
21
|
+
"""Serve a pool of `total` rows through urlopen; returns the queries made."""
|
|
22
|
+
calls = []
|
|
23
|
+
|
|
24
|
+
def urlopen(req, timeout):
|
|
25
|
+
q = dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(req.full_url).query))
|
|
26
|
+
calls.append(q)
|
|
27
|
+
if status != 200:
|
|
28
|
+
raise urllib.error.HTTPError(req.full_url, status, "Nope", {}, None)
|
|
29
|
+
page, size = int(q["page"]), int(q["pageSize"])
|
|
30
|
+
start = (page - 1) * size
|
|
31
|
+
rows = [{"ip": f"10.0.0.{n % 256}", "port": 8080, "protocol": "http"} for n in range(start, min(start + size, total))]
|
|
32
|
+
return io.BytesIO(json.dumps({"rows": rows, "total": total}).encode())
|
|
33
|
+
|
|
34
|
+
monkeypatch.setattr(proxmint.urllib.request, "urlopen", urlopen)
|
|
35
|
+
return calls
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_pages_past_the_200_row_cap(monkeypatch):
|
|
39
|
+
calls = stub_api(monkeypatch, total=1000)
|
|
40
|
+
assert len(proxmint.free_proxies(limit=350)) == 350
|
|
41
|
+
assert [(c["page"], c["pageSize"]) for c in calls] == [("1", "200"), ("2", "150")]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_stops_at_the_end_of_the_pool(monkeypatch):
|
|
45
|
+
calls = stub_api(monkeypatch, total=30)
|
|
46
|
+
assert len(proxmint.free_proxies(limit=500)) == 30
|
|
47
|
+
assert len(calls) == 1
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_bad_input_and_http_errors(monkeypatch):
|
|
51
|
+
calls = stub_api(monkeypatch, total=10, status=429)
|
|
52
|
+
with pytest.raises(ValueError):
|
|
53
|
+
proxmint.free_proxies(limit=0)
|
|
54
|
+
assert calls == []
|
|
55
|
+
with pytest.raises(proxmint.ProxmintError) as e:
|
|
56
|
+
proxmint.free_proxies(country="DE")
|
|
57
|
+
assert e.value.status == 429 and calls[0]["country"] == "de"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_https_proxies_are_reached_over_http():
|
|
61
|
+
assert proxmint.proxy_url({"protocol": "https", "ip": "1.2.3.4", "port": 8080}) == "http://1.2.3.4:8080"
|
|
62
|
+
assert proxmint.proxy_url({"protocol": "socks5", "ip": "1.2.3.4", "port": 1080}) == "socks5://1.2.3.4:1080"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
DEAD, GOOD, AUTH, SLOW = "http://10.0.0.1:1", "http://10.0.0.2:2", "http://10.0.0.3:3", "http://10.0.0.4:4"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class FakeTransport(httpx.BaseTransport):
|
|
69
|
+
def __init__(self, proxy):
|
|
70
|
+
self.proxy = str(proxy)
|
|
71
|
+
|
|
72
|
+
def handle_request(self, request):
|
|
73
|
+
if self.proxy == DEAD:
|
|
74
|
+
raise httpx.ConnectError("refused", request=request)
|
|
75
|
+
return httpx.Response(407 if self.proxy == AUTH else 200, text=self.proxy)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_httpx_drops_failing_proxies_and_retries(monkeypatch):
|
|
79
|
+
monkeypatch.setattr(proxmint.httpx.httpx, "HTTPTransport", lambda proxy: FakeTransport(proxy))
|
|
80
|
+
with httpx.Client(transport=proxmint.httpx.RotatingTransport([DEAD, AUTH, GOOD])) as client:
|
|
81
|
+
assert client.get("http://example.com").text == GOOD # dead, then 407, then good
|
|
82
|
+
assert client.get("http://example.com").text == GOOD # only the good one is left
|
|
83
|
+
with httpx.Client(transport=proxmint.httpx.RotatingTransport([DEAD], tries=2)) as client:
|
|
84
|
+
with pytest.raises(httpx.ConnectError):
|
|
85
|
+
client.get("http://example.com")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class FakeRequest:
|
|
89
|
+
proxy_headers = None
|
|
90
|
+
|
|
91
|
+
def update_proxy(self, proxy, auth, headers):
|
|
92
|
+
self.proxy = str(proxy)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def test_aiohttp_drops_failing_proxies_and_retries():
|
|
96
|
+
async def handler(req):
|
|
97
|
+
if req.proxy == DEAD:
|
|
98
|
+
raise aiohttp.ServerDisconnectedError()
|
|
99
|
+
if req.proxy == SLOW:
|
|
100
|
+
await asyncio.sleep(5)
|
|
101
|
+
return types.SimpleNamespace(status=407 if req.proxy == AUTH else 200, proxy=req.proxy, release=lambda: None)
|
|
102
|
+
|
|
103
|
+
async def run():
|
|
104
|
+
mw = proxmint.aiohttp.RotatingProxy([DEAD, AUTH, GOOD])
|
|
105
|
+
assert (await mw(FakeRequest(), handler)).proxy == GOOD # dead, then 407, then good
|
|
106
|
+
assert [(await mw(FakeRequest(), handler)).proxy for _ in range(2)] == [GOOD, GOOD]
|
|
107
|
+
slow_first = proxmint.aiohttp.RotatingProxy([SLOW, GOOD], timeout=0.05)
|
|
108
|
+
assert (await slow_first(FakeRequest(), handler)).proxy == GOOD # a hang costs one timeout
|
|
109
|
+
with pytest.raises(aiohttp.ServerDisconnectedError):
|
|
110
|
+
await proxmint.aiohttp.RotatingProxy([DEAD], tries=2)(FakeRequest(), handler)
|
|
111
|
+
|
|
112
|
+
asyncio.run(run())
|