proxy-scraper-cli 1.22.0__tar.gz → 1.23.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/PKG-INFO +42 -11
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/README.md +39 -9
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxy_scraper_cli.egg-info/PKG-INFO +42 -11
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxy_scraper_cli.egg-info/SOURCES.txt +6 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxy_scraper_cli.egg-info/requires.txt +1 -1
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/__init__.py +1 -1
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/agent.py +2 -2
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/api.py +8 -6
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/exporters.py +48 -0
- proxy_scraper_cli-1.23.0/proxyscraper/mirror.py +348 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/pages.py +17 -12
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/publish.py +11 -3
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ranking.py +26 -2
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/core.py +17 -7
- proxy_scraper_cli-1.23.0/proxyscraper/server/dashboard.html +351 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/status.py +59 -4
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/site/index.html +3 -3
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/sources.json +1 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/serve.py +5 -4
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/widgets.py +6 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/pyproject.toml +3 -2
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_agent.py +5 -4
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_live_api.py +6 -5
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_mcp_server.py +2 -1
- proxy_scraper_cli-1.23.0/tests/test_mirror.py +75 -0
- proxy_scraper_cli-1.23.0/tests/test_pac.py +116 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_pick.py +2 -2
- proxy_scraper_cli-1.23.0/tests/test_serve_dashboard.py +234 -0
- proxy_scraper_cli-1.23.0/tests/test_survival_ranking.py +58 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/LICENSE +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxy_scraper_cli.egg-info/dependency_links.txt +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxy_scraper_cli.egg-info/entry_points.txt +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxy_scraper_cli.egg-info/top_level.txt +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/__main__.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/action.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/app.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/asndb.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/blocklist.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/charts.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/checker.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/cli.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/compat.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/completion.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/fetchcache.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/geo.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/geodb.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/handshake.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/history.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/judges.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/mcp_entry.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/mcp_server.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/netio.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/options.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/output.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/parsing.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/paths.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/pipeline.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/preferences.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/proxypages.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/report.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/runsteps.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/__init__.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/api.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/http.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/pool.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/socks.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/upstream.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/site/apple-touch-icon.png +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/site/googleaac1161b7853c5b5.html +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/site/logo.png +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/site/og.png +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/sites.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/sources.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/speed.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/targets.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/__init__.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/dashboard.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/keys.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/report.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/wizard.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/setup.cfg +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_action.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_api.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_app.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_asndb.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_auth.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_blocklist.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_charts.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_checker.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_compat.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_completion.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_discovery_growth.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_exporters.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_fetchcache.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_geodb.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_judges.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_learning.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_mine_sources.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_options.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_output_stdout.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_own_sources.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_packaging.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_pages.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_parsing.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_parsing_fuzz.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_paths.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_pipeline.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_preferences.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_proxy_pages.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_proxy_sources.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_publish.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_report.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_robustness.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_rotator.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_serve_api.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_serve_password.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_serve_refill.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_serve_v2.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_server.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_server_closed_writer.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_server_fixes.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_sites.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_sources_json.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_speed.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_speed_ranking.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_tamper.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_targets.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_ui.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_uptime.py +0 -0
- {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_wizard.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: proxy-scraper-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.23.0
|
|
4
4
|
Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
|
|
5
5
|
Author: Maximilian Feix
|
|
6
6
|
License-Expression: MIT
|
|
@@ -22,6 +22,7 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
22
22
|
Classifier: Programming Language :: Python :: 3.11
|
|
23
23
|
Classifier: Programming Language :: Python :: 3.12
|
|
24
24
|
Classifier: Programming Language :: Python :: 3.13
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
25
26
|
Classifier: Topic :: Internet :: Proxy Servers
|
|
26
27
|
Classifier: Topic :: System :: Networking
|
|
27
28
|
Requires-Python: >=3.9
|
|
@@ -36,7 +37,7 @@ Requires-Dist: mcp<3,>=2.2; python_version >= "3.10" and extra == "mcp"
|
|
|
36
37
|
Provides-Extra: dev
|
|
37
38
|
Requires-Dist: pytest; extra == "dev"
|
|
38
39
|
Requires-Dist: hypothesis; extra == "dev"
|
|
39
|
-
Requires-Dist: ruff==0.16.
|
|
40
|
+
Requires-Dist: ruff==0.16.9; extra == "dev"
|
|
40
41
|
Dynamic: license-file
|
|
41
42
|
|
|
42
43
|
<div align="center">
|
|
@@ -51,16 +52,19 @@ Dynamic: license-file
|
|
|
51
52
|
[](https://github.com/maximilianfeix/proxy-scraper/releases/latest)
|
|
52
53
|
[](#live-list)
|
|
53
54
|
[](https://pypi.org/project/proxy-scraper-cli/)
|
|
54
|
-
[](pyproject.toml)
|
|
55
56
|
[](https://github.com/maximilianfeix/proxy-scraper/stargazers)
|
|
56
57
|
[](LICENSE)
|
|
57
58
|
|
|
58
59
|
<a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://img.shields.io/badge/Browse_the_live_list-D4F77A?style=for-the-badge&labelColor=121113" alt="Browse the live list"></a>
|
|
60
|
+
<a href="https://github.com/maximilianfeix/free-proxy-list"><img src="https://img.shields.io/badge/Just_the_lists-121113?style=for-the-badge" alt="Just the lists: free-proxy-list"></a>
|
|
59
61
|
<a href="#install"><img src="https://img.shields.io/badge/Install-121113?style=for-the-badge" alt="Install"></a>
|
|
60
62
|
<a href="#from-python"><img src="https://img.shields.io/badge/Python_API-121113?style=for-the-badge" alt="Python API"></a>
|
|
61
63
|
<a href="#mcp"><img src="https://img.shields.io/badge/MCP_server-121113?style=for-the-badge" alt="MCP server for AI agents"></a>
|
|
62
64
|
<a href="bot/"><img src="https://img.shields.io/badge/Discord_bot-121113?style=for-the-badge" alt="Discord bot"></a>
|
|
63
65
|
|
|
66
|
+
**English** · [简体中文](README.zh-CN.md)
|
|
67
|
+
|
|
64
68
|
[Install](#install) · [Live list](#live-list) · [For AI agents](#mcp) · [Proxy server](#proxy-server) · [How it works](#how-it-works) · [Recipes](#recipes) · [Options](#options) · [FAQ](#faq)
|
|
65
69
|
|
|
66
70
|
</div>
|
|
@@ -114,6 +118,14 @@ proxy-scraper
|
|
|
114
118
|
|
|
115
119
|
The package is called `proxy-scraper-cli` on PyPI (the plain name is taken), the command is `proxy-scraper`.
|
|
116
120
|
|
|
121
|
+
**With [Homebrew](https://brew.sh)** on macOS and Linux:
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
brew install maximilianfeix/tap/proxy-scraper
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
The formula follows each release within about a day.
|
|
128
|
+
|
|
117
129
|
<details>
|
|
118
130
|
<summary><b>Other ways: Docker, pip, a faster event loop, or straight from the repo</b></summary>
|
|
119
131
|
<br>
|
|
@@ -178,7 +190,9 @@ Keys: <kbd>↑</kbd><kbd>↓</kbd> select · <kbd>Space</kbd> toggle · <kbd>1</
|
|
|
178
190
|
|
|
179
191
|
## Live proxy list
|
|
180
192
|
|
|
181
|
-
Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and publishes the hits to the [`proxy-list`](../../tree/proxy-list) branch – every entry worked in the last run,
|
|
193
|
+
Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and publishes the hits to the [`proxy-list`](../../tree/proxy-list) branch – every entry worked in the last run, best first: fast ones that are likely to still be up come on top.
|
|
194
|
+
|
|
195
|
+
> **Just want the lists?** They also live in a repository of their own, **[maximilianfeix/free-proxy-list](https://github.com/maximilianfeix/free-proxy-list)** – one list per protocol, per country and per site, updated every hour, with the numbers of the last run on the front page. Watch or star it to keep it at hand.
|
|
182
196
|
|
|
183
197
|
<a href="https://maximilianfeix.github.io/proxy-scraper/"><picture>
|
|
184
198
|
<source media="(prefers-color-scheme: dark)" srcset="https://maximilianfeix.github.io/proxy-scraper/chart-dark.svg">
|
|
@@ -319,7 +333,7 @@ Started without arguments, the tool asks what you need using the arrow keys –
|
|
|
319
333
|
<td valign="top">
|
|
320
334
|
|
|
321
335
|
**Real verification**<br>
|
|
322
|
-
Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests
|
|
336
|
+
Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests: over 3.6 million checks, 54 % of the proxies that answered the first request failed the second. A third request catches proxies that **tamper with content**: about one in 40 working proxies changed a known page, usually by injecting a script ([the numbers](docs/free-proxies-in-numbers.md)). Plus: HTTPS through a tunnel with **verified TLS**, anonymity level *elite / anonymous / transparent* and the country of the exit IP.
|
|
323
337
|
|
|
324
338
|
</td>
|
|
325
339
|
<td valign="top">
|
|
@@ -361,13 +375,13 @@ Speed chart, latency histogram, protocols, countries and the latest hits in real
|
|
|
361
375
|
<td valign="top">
|
|
362
376
|
|
|
363
377
|
**Runs everywhere**<br>
|
|
364
|
-
macOS, Linux and Windows, Python 3.9 to 3.
|
|
378
|
+
macOS, Linux and Windows, Python 3.9 to 3.14. Only two dependencies: `rich` and `certifi`. It even notices when a firewall blocks proxies.
|
|
365
379
|
|
|
366
380
|
</td>
|
|
367
381
|
<td valign="top">
|
|
368
382
|
|
|
369
383
|
**Thoroughly tested**<br>
|
|
370
|
-
|
|
384
|
+
760+ tests run offline against real mini proxies and honeypots on `localhost` – on Linux, macOS and Windows with Python 3.9, 3.11, 3.13 and 3.14.
|
|
371
385
|
|
|
372
386
|
</td>
|
|
373
387
|
</tr>
|
|
@@ -515,7 +529,7 @@ Without a password, `--serve-host` means **anyone who reaches the port can use i
|
|
|
515
529
|
|
|
516
530
|
- every connection goes through a different proxy (unless sticky); fast and proven ones are preferred
|
|
517
531
|
- `CONNECT` for HTTPS and plain HTTP requests; HTTP, SOCKS4 and SOCKS5 proxies can sit behind it (SOCKS5 with DNS through the proxy)
|
|
518
|
-
- HTTPS
|
|
532
|
+
- HTTPS prefers proxies that passed the test with **verified TLS**; when no such proxy fits right now – none matches a wish like `country-de`, or they have all dropped out – it uses the others – your client's own certificate check still catches a proxy that breaks the encryption, so leave it on
|
|
519
533
|
- if a proxy stays silent inside the tunnel or returns an error page instead of TLS, the same first packet quietly goes to the next one
|
|
520
534
|
- three failures in a row and a proxy leaves the rotation – every 5 minutes those get re-checked and come back if they work again
|
|
521
535
|
- `--serve-refill 6` checks fresh proxies every 6 hours in the background (the live list with `--recheck live`, otherwise the last run + history) with the same checks and filters, and adds the hits – a server that runs for days doesn't run dry
|
|
@@ -523,6 +537,10 @@ Without a password, `--serve-host` means **anyone who reaches the port can use i
|
|
|
523
537
|
|
|
524
538
|
In testing: 20 of 20 HTTPS requests succeeded, over 15 different exit IPs. In the wizard this is **Proxy server right away**.
|
|
525
539
|
|
|
540
|
+
**Live dashboard:** open `http://127.0.0.1:8899/__proxy-scraper/` in a browser – the pool, success rate, traffic, a live chart of requests, countries, the best proxies and the latest connections, refreshed every two seconds. It needs nothing from the internet, and `--serve-password` protects it like the rest.
|
|
541
|
+
|
|
542
|
+
<div align="center"><img src="docs/dashboard-live.png" alt="The live dashboard of the rotating proxy server: usable proxies, success rate, requests per second, countries and the latest connections" width="860"></div>
|
|
543
|
+
|
|
526
544
|
<a id="pool-api"></a>
|
|
527
545
|
|
|
528
546
|
### Proxy pool API
|
|
@@ -741,6 +759,18 @@ proxychains4 -f results/latest/proxychains.conf curl https://api.ipify.org
|
|
|
741
759
|
|
|
742
760
|
`clash.yaml` has all HTTP and SOCKS5 proxies plus a `url-test` group that always picks the fastest. `singbox.json` does the same for sing-box, SOCKS4 included, and opens a local proxy: `sing-box run -c results/latest/singbox.json`, then use `127.0.0.1:2080` as HTTP or SOCKS5 proxy. All three leave out HTTP proxies that can't tunnel (`CONNECT`), because these tools tunnel everything.
|
|
743
761
|
|
|
762
|
+
**Browsers and the OS** – `--export pac` writes `proxy.pac`: point Firefox, Chrome (via the system settings), FoxyProxy or macOS/Windows proxy settings at it and every request goes through the best 30 HTTP and SOCKS5 proxies in order, the browser moving on by itself when one fails. Local names and private IP addresses stay direct (a host name like `10.example.com` doesn't count as private). The file itself has no `DIRECT` fallback and resolves no host names, and SOCKS4 is left out because it would make the browser resolve names with your own DNS. Two Firefox settings matter: turn on *Proxy DNS when using SOCKS v5*, and set `network.proxy.failover_direct` to `false` in `about:config` – otherwise Firefox goes direct once every proxy has failed. No install at all: the live list publishes one every hour at `https://maximilianfeix.github.io/proxy-scraper/proxy.pac`.
|
|
763
|
+
|
|
764
|
+
**Burp Suite, OWASP ZAP, mitmproxy** – put the rotating server behind your intercepting proxy, so a scan or a brute-force test comes from many IPs while you still see every request. Start it with `proxy-scraper --recheck live --serve`, then:
|
|
765
|
+
|
|
766
|
+
| Tool | Where | Set |
|
|
767
|
+
|---|---|---|
|
|
768
|
+
| Burp Suite | *Settings → Network → Connections → Upstream proxy servers* | a rule for destination host `*`, proxy host `127.0.0.1`, port `8899`; with `--serve-password` add Basic auth (user `any` or a wish like `country-de`, the password) |
|
|
769
|
+
| OWASP ZAP | *Options → Network → Connection → HTTP Proxy* | host `127.0.0.1`, port `8899`, and under its authentication the user (`any` or a wish) and the password – use the HTTP setting, not SOCKS: ZAP never sends loopback addresses through its SOCKS proxy |
|
|
770
|
+
| mitmproxy | command line | `mitmproxy --mode upstream:http://127.0.0.1:8899 --upstream-auth country-us:x` |
|
|
771
|
+
|
|
772
|
+
Username wishes – `country-us` sends everything out through US proxies, `session-NAME` keeps one exit IP for a login flow – reach the server as the proxy login. mitmproxy sends it right away; Burp and ZAP may wait to be asked, so start the server with `--serve-password` when you rely on a wish there. **Keep your tool's upstream certificate checks on** (no `--ssl-insecure` in mitmproxy, no "ignore certificate errors" upstream in Burp or ZAP): free proxies are strangers, and that check is what stops one from reading or changing your HTTPS traffic. Tested end to end with mitmproxy 12, certificate checks on: HTTP and HTTPS through mitmproxy and the rotating server, every request out through a US proxy, and a site with an expired certificate refused. Only test what you're allowed to.
|
|
773
|
+
|
|
744
774
|
**In a pipe** – `-o -` prints the hits to stdout, the interface moves to stderr
|
|
745
775
|
|
|
746
776
|
```bash
|
|
@@ -834,7 +864,7 @@ curl.exe -x (Get-Content "$run\all.txt" -TotalCount 1) http://api.ipify.org
|
|
|
834
864
|
| `--serve-refill HOURS` | while serving, check fresh proxies every HOURS and add the hits to the pool |
|
|
835
865
|
| `--serve [PORT]` | afterwards serve as a rotating proxy on `127.0.0.1:PORT` (default: 8899) |
|
|
836
866
|
| `-o FILE` | also write all hits to this file; `-o -` prints them to stdout |
|
|
837
|
-
| `--export FORMATS` | extra files for other tools: `proxychains`, `clash`, `singbox`, `curl` or `all` |
|
|
867
|
+
| `--export FORMATS` | extra files for other tools: `proxychains`, `clash`, `singbox`, `curl`, `pac` or `all` |
|
|
838
868
|
| `-V`, `--version` | print the version |
|
|
839
869
|
| `--completion SHELL` | print the tab completion script for bash, zsh, fish or PowerShell |
|
|
840
870
|
|
|
@@ -920,7 +950,7 @@ Only for things that don't matter. Public proxies are run by strangers who can r
|
|
|
920
950
|
|
|
921
951
|
What's next is in the [open issues](../../issues) – ideas and wishes are welcome as an [issue](../../issues/new/choose). During Hacktoberfest there are [beginner-friendly issues](../../issues?q=is%3Aopen+label%3Ahacktoberfest) with pointers on where to start.
|
|
922
952
|
|
|
923
|
-
|
|
953
|
+
Shipped in v1.22 and v1.23: a live dashboard for the rotating server, `--export pac` and an hourly `proxy.pac`, "best first" that counts how likely a proxy is still up, the lists in a repository of their own ([free-proxy-list](https://github.com/maximilianfeix/free-proxy-list)), daily snapshots on Hugging Face, `brew install`, a Chinese README, Python 3.14, and a recipe for Burp Suite, ZAP and mitmproxy.
|
|
924
954
|
|
|
925
955
|
Shipped in v1.10 to v1.21: download speed per proxy and "best first" everywhere, daily snapshots, a rotating proxy URL for requests, httpx and Playwright, `--pick` for working proxies without a scan, a proxy pool API compatible with jhao104/proxy_pool, your own lists with `--source`, a GitHub Action, a Telegram bot and an Atom feed for the weekly report.
|
|
926
956
|
|
|
@@ -969,12 +999,13 @@ proxyscraper/
|
|
|
969
999
|
├── geodb.py DB-IP country database (monthly, binary search)
|
|
970
1000
|
├── targets.py target sites for --target
|
|
971
1001
|
├── output.py result files
|
|
972
|
-
├── exporters.py proxychains, Clash, sing-box and
|
|
1002
|
+
├── exporters.py proxychains, Clash, sing-box, curl and PAC formats (--export)
|
|
973
1003
|
├── server/ rotating proxy server (--serve): pool · http · upstream · socks · status · api · core
|
|
974
1004
|
├── api.py find_proxies() / check_proxies() for Python
|
|
975
1005
|
├── agent.py MCP tools without the SDK: live list, filters, fetch through proxies
|
|
976
1006
|
├── mcp_server.py MCP server (proxy-scraper-mcp) for AI agents
|
|
977
1007
|
├── publish.py live list for GitHub Actions
|
|
1008
|
+
├── mirror.py the same lists in their own repo (free-proxy-list)
|
|
978
1009
|
├── paths.py where state and results are stored
|
|
979
1010
|
├── compat.py differences between Unix and Windows
|
|
980
1011
|
├── netio.py small HTTP client on asyncio
|
|
@@ -10,16 +10,19 @@
|
|
|
10
10
|
[](https://github.com/maximilianfeix/proxy-scraper/releases/latest)
|
|
11
11
|
[](#live-list)
|
|
12
12
|
[](https://pypi.org/project/proxy-scraper-cli/)
|
|
13
|
-
[](pyproject.toml)
|
|
14
14
|
[](https://github.com/maximilianfeix/proxy-scraper/stargazers)
|
|
15
15
|
[](LICENSE)
|
|
16
16
|
|
|
17
17
|
<a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://img.shields.io/badge/Browse_the_live_list-D4F77A?style=for-the-badge&labelColor=121113" alt="Browse the live list"></a>
|
|
18
|
+
<a href="https://github.com/maximilianfeix/free-proxy-list"><img src="https://img.shields.io/badge/Just_the_lists-121113?style=for-the-badge" alt="Just the lists: free-proxy-list"></a>
|
|
18
19
|
<a href="#install"><img src="https://img.shields.io/badge/Install-121113?style=for-the-badge" alt="Install"></a>
|
|
19
20
|
<a href="#from-python"><img src="https://img.shields.io/badge/Python_API-121113?style=for-the-badge" alt="Python API"></a>
|
|
20
21
|
<a href="#mcp"><img src="https://img.shields.io/badge/MCP_server-121113?style=for-the-badge" alt="MCP server for AI agents"></a>
|
|
21
22
|
<a href="bot/"><img src="https://img.shields.io/badge/Discord_bot-121113?style=for-the-badge" alt="Discord bot"></a>
|
|
22
23
|
|
|
24
|
+
**English** · [简体中文](README.zh-CN.md)
|
|
25
|
+
|
|
23
26
|
[Install](#install) · [Live list](#live-list) · [For AI agents](#mcp) · [Proxy server](#proxy-server) · [How it works](#how-it-works) · [Recipes](#recipes) · [Options](#options) · [FAQ](#faq)
|
|
24
27
|
|
|
25
28
|
</div>
|
|
@@ -73,6 +76,14 @@ proxy-scraper
|
|
|
73
76
|
|
|
74
77
|
The package is called `proxy-scraper-cli` on PyPI (the plain name is taken), the command is `proxy-scraper`.
|
|
75
78
|
|
|
79
|
+
**With [Homebrew](https://brew.sh)** on macOS and Linux:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
brew install maximilianfeix/tap/proxy-scraper
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
The formula follows each release within about a day.
|
|
86
|
+
|
|
76
87
|
<details>
|
|
77
88
|
<summary><b>Other ways: Docker, pip, a faster event loop, or straight from the repo</b></summary>
|
|
78
89
|
<br>
|
|
@@ -137,7 +148,9 @@ Keys: <kbd>↑</kbd><kbd>↓</kbd> select · <kbd>Space</kbd> toggle · <kbd>1</
|
|
|
137
148
|
|
|
138
149
|
## Live proxy list
|
|
139
150
|
|
|
140
|
-
Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and publishes the hits to the [`proxy-list`](../../tree/proxy-list) branch – every entry worked in the last run,
|
|
151
|
+
Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and publishes the hits to the [`proxy-list`](../../tree/proxy-list) branch – every entry worked in the last run, best first: fast ones that are likely to still be up come on top.
|
|
152
|
+
|
|
153
|
+
> **Just want the lists?** They also live in a repository of their own, **[maximilianfeix/free-proxy-list](https://github.com/maximilianfeix/free-proxy-list)** – one list per protocol, per country and per site, updated every hour, with the numbers of the last run on the front page. Watch or star it to keep it at hand.
|
|
141
154
|
|
|
142
155
|
<a href="https://maximilianfeix.github.io/proxy-scraper/"><picture>
|
|
143
156
|
<source media="(prefers-color-scheme: dark)" srcset="https://maximilianfeix.github.io/proxy-scraper/chart-dark.svg">
|
|
@@ -278,7 +291,7 @@ Started without arguments, the tool asks what you need using the arrow keys –
|
|
|
278
291
|
<td valign="top">
|
|
279
292
|
|
|
280
293
|
**Real verification**<br>
|
|
281
|
-
Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests
|
|
294
|
+
Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests: over 3.6 million checks, 54 % of the proxies that answered the first request failed the second. A third request catches proxies that **tamper with content**: about one in 40 working proxies changed a known page, usually by injecting a script ([the numbers](docs/free-proxies-in-numbers.md)). Plus: HTTPS through a tunnel with **verified TLS**, anonymity level *elite / anonymous / transparent* and the country of the exit IP.
|
|
282
295
|
|
|
283
296
|
</td>
|
|
284
297
|
<td valign="top">
|
|
@@ -320,13 +333,13 @@ Speed chart, latency histogram, protocols, countries and the latest hits in real
|
|
|
320
333
|
<td valign="top">
|
|
321
334
|
|
|
322
335
|
**Runs everywhere**<br>
|
|
323
|
-
macOS, Linux and Windows, Python 3.9 to 3.
|
|
336
|
+
macOS, Linux and Windows, Python 3.9 to 3.14. Only two dependencies: `rich` and `certifi`. It even notices when a firewall blocks proxies.
|
|
324
337
|
|
|
325
338
|
</td>
|
|
326
339
|
<td valign="top">
|
|
327
340
|
|
|
328
341
|
**Thoroughly tested**<br>
|
|
329
|
-
|
|
342
|
+
760+ tests run offline against real mini proxies and honeypots on `localhost` – on Linux, macOS and Windows with Python 3.9, 3.11, 3.13 and 3.14.
|
|
330
343
|
|
|
331
344
|
</td>
|
|
332
345
|
</tr>
|
|
@@ -474,7 +487,7 @@ Without a password, `--serve-host` means **anyone who reaches the port can use i
|
|
|
474
487
|
|
|
475
488
|
- every connection goes through a different proxy (unless sticky); fast and proven ones are preferred
|
|
476
489
|
- `CONNECT` for HTTPS and plain HTTP requests; HTTP, SOCKS4 and SOCKS5 proxies can sit behind it (SOCKS5 with DNS through the proxy)
|
|
477
|
-
- HTTPS
|
|
490
|
+
- HTTPS prefers proxies that passed the test with **verified TLS**; when no such proxy fits right now – none matches a wish like `country-de`, or they have all dropped out – it uses the others – your client's own certificate check still catches a proxy that breaks the encryption, so leave it on
|
|
478
491
|
- if a proxy stays silent inside the tunnel or returns an error page instead of TLS, the same first packet quietly goes to the next one
|
|
479
492
|
- three failures in a row and a proxy leaves the rotation – every 5 minutes those get re-checked and come back if they work again
|
|
480
493
|
- `--serve-refill 6` checks fresh proxies every 6 hours in the background (the live list with `--recheck live`, otherwise the last run + history) with the same checks and filters, and adds the hits – a server that runs for days doesn't run dry
|
|
@@ -482,6 +495,10 @@ Without a password, `--serve-host` means **anyone who reaches the port can use i
|
|
|
482
495
|
|
|
483
496
|
In testing: 20 of 20 HTTPS requests succeeded, over 15 different exit IPs. In the wizard this is **Proxy server right away**.
|
|
484
497
|
|
|
498
|
+
**Live dashboard:** open `http://127.0.0.1:8899/__proxy-scraper/` in a browser – the pool, success rate, traffic, a live chart of requests, countries, the best proxies and the latest connections, refreshed every two seconds. It needs nothing from the internet, and `--serve-password` protects it like the rest.
|
|
499
|
+
|
|
500
|
+
<div align="center"><img src="docs/dashboard-live.png" alt="The live dashboard of the rotating proxy server: usable proxies, success rate, requests per second, countries and the latest connections" width="860"></div>
|
|
501
|
+
|
|
485
502
|
<a id="pool-api"></a>
|
|
486
503
|
|
|
487
504
|
### Proxy pool API
|
|
@@ -700,6 +717,18 @@ proxychains4 -f results/latest/proxychains.conf curl https://api.ipify.org
|
|
|
700
717
|
|
|
701
718
|
`clash.yaml` has all HTTP and SOCKS5 proxies plus a `url-test` group that always picks the fastest. `singbox.json` does the same for sing-box, SOCKS4 included, and opens a local proxy: `sing-box run -c results/latest/singbox.json`, then use `127.0.0.1:2080` as HTTP or SOCKS5 proxy. All three leave out HTTP proxies that can't tunnel (`CONNECT`), because these tools tunnel everything.
|
|
702
719
|
|
|
720
|
+
**Browsers and the OS** – `--export pac` writes `proxy.pac`: point Firefox, Chrome (via the system settings), FoxyProxy or macOS/Windows proxy settings at it and every request goes through the best 30 HTTP and SOCKS5 proxies in order, the browser moving on by itself when one fails. Local names and private IP addresses stay direct (a host name like `10.example.com` doesn't count as private). The file itself has no `DIRECT` fallback and resolves no host names, and SOCKS4 is left out because it would make the browser resolve names with your own DNS. Two Firefox settings matter: turn on *Proxy DNS when using SOCKS v5*, and set `network.proxy.failover_direct` to `false` in `about:config` – otherwise Firefox goes direct once every proxy has failed. No install at all: the live list publishes one every hour at `https://maximilianfeix.github.io/proxy-scraper/proxy.pac`.
|
|
721
|
+
|
|
722
|
+
**Burp Suite, OWASP ZAP, mitmproxy** – put the rotating server behind your intercepting proxy, so a scan or a brute-force test comes from many IPs while you still see every request. Start it with `proxy-scraper --recheck live --serve`, then:
|
|
723
|
+
|
|
724
|
+
| Tool | Where | Set |
|
|
725
|
+
|---|---|---|
|
|
726
|
+
| Burp Suite | *Settings → Network → Connections → Upstream proxy servers* | a rule for destination host `*`, proxy host `127.0.0.1`, port `8899`; with `--serve-password` add Basic auth (user `any` or a wish like `country-de`, the password) |
|
|
727
|
+
| OWASP ZAP | *Options → Network → Connection → HTTP Proxy* | host `127.0.0.1`, port `8899`, and under its authentication the user (`any` or a wish) and the password – use the HTTP setting, not SOCKS: ZAP never sends loopback addresses through its SOCKS proxy |
|
|
728
|
+
| mitmproxy | command line | `mitmproxy --mode upstream:http://127.0.0.1:8899 --upstream-auth country-us:x` |
|
|
729
|
+
|
|
730
|
+
Username wishes – `country-us` sends everything out through US proxies, `session-NAME` keeps one exit IP for a login flow – reach the server as the proxy login. mitmproxy sends it right away; Burp and ZAP may wait to be asked, so start the server with `--serve-password` when you rely on a wish there. **Keep your tool's upstream certificate checks on** (no `--ssl-insecure` in mitmproxy, no "ignore certificate errors" upstream in Burp or ZAP): free proxies are strangers, and that check is what stops one from reading or changing your HTTPS traffic. Tested end to end with mitmproxy 12, certificate checks on: HTTP and HTTPS through mitmproxy and the rotating server, every request out through a US proxy, and a site with an expired certificate refused. Only test what you're allowed to.
|
|
731
|
+
|
|
703
732
|
**In a pipe** – `-o -` prints the hits to stdout, the interface moves to stderr
|
|
704
733
|
|
|
705
734
|
```bash
|
|
@@ -793,7 +822,7 @@ curl.exe -x (Get-Content "$run\all.txt" -TotalCount 1) http://api.ipify.org
|
|
|
793
822
|
| `--serve-refill HOURS` | while serving, check fresh proxies every HOURS and add the hits to the pool |
|
|
794
823
|
| `--serve [PORT]` | afterwards serve as a rotating proxy on `127.0.0.1:PORT` (default: 8899) |
|
|
795
824
|
| `-o FILE` | also write all hits to this file; `-o -` prints them to stdout |
|
|
796
|
-
| `--export FORMATS` | extra files for other tools: `proxychains`, `clash`, `singbox`, `curl` or `all` |
|
|
825
|
+
| `--export FORMATS` | extra files for other tools: `proxychains`, `clash`, `singbox`, `curl`, `pac` or `all` |
|
|
797
826
|
| `-V`, `--version` | print the version |
|
|
798
827
|
| `--completion SHELL` | print the tab completion script for bash, zsh, fish or PowerShell |
|
|
799
828
|
|
|
@@ -879,7 +908,7 @@ Only for things that don't matter. Public proxies are run by strangers who can r
|
|
|
879
908
|
|
|
880
909
|
What's next is in the [open issues](../../issues) – ideas and wishes are welcome as an [issue](../../issues/new/choose). During Hacktoberfest there are [beginner-friendly issues](../../issues?q=is%3Aopen+label%3Ahacktoberfest) with pointers on where to start.
|
|
881
910
|
|
|
882
|
-
|
|
911
|
+
Shipped in v1.22 and v1.23: a live dashboard for the rotating server, `--export pac` and an hourly `proxy.pac`, "best first" that counts how likely a proxy is still up, the lists in a repository of their own ([free-proxy-list](https://github.com/maximilianfeix/free-proxy-list)), daily snapshots on Hugging Face, `brew install`, a Chinese README, Python 3.14, and a recipe for Burp Suite, ZAP and mitmproxy.
|
|
883
912
|
|
|
884
913
|
Shipped in v1.10 to v1.21: download speed per proxy and "best first" everywhere, daily snapshots, a rotating proxy URL for requests, httpx and Playwright, `--pick` for working proxies without a scan, a proxy pool API compatible with jhao104/proxy_pool, your own lists with `--source`, a GitHub Action, a Telegram bot and an Atom feed for the weekly report.
|
|
885
914
|
|
|
@@ -928,12 +957,13 @@ proxyscraper/
|
|
|
928
957
|
├── geodb.py DB-IP country database (monthly, binary search)
|
|
929
958
|
├── targets.py target sites for --target
|
|
930
959
|
├── output.py result files
|
|
931
|
-
├── exporters.py proxychains, Clash, sing-box and
|
|
960
|
+
├── exporters.py proxychains, Clash, sing-box, curl and PAC formats (--export)
|
|
932
961
|
├── server/ rotating proxy server (--serve): pool · http · upstream · socks · status · api · core
|
|
933
962
|
├── api.py find_proxies() / check_proxies() for Python
|
|
934
963
|
├── agent.py MCP tools without the SDK: live list, filters, fetch through proxies
|
|
935
964
|
├── mcp_server.py MCP server (proxy-scraper-mcp) for AI agents
|
|
936
965
|
├── publish.py live list for GitHub Actions
|
|
966
|
+
├── mirror.py the same lists in their own repo (free-proxy-list)
|
|
937
967
|
├── paths.py where state and results are stored
|
|
938
968
|
├── compat.py differences between Unix and Windows
|
|
939
969
|
├── netio.py small HTTP client on asyncio
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: proxy-scraper-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.23.0
|
|
4
4
|
Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
|
|
5
5
|
Author: Maximilian Feix
|
|
6
6
|
License-Expression: MIT
|
|
@@ -22,6 +22,7 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
22
22
|
Classifier: Programming Language :: Python :: 3.11
|
|
23
23
|
Classifier: Programming Language :: Python :: 3.12
|
|
24
24
|
Classifier: Programming Language :: Python :: 3.13
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
25
26
|
Classifier: Topic :: Internet :: Proxy Servers
|
|
26
27
|
Classifier: Topic :: System :: Networking
|
|
27
28
|
Requires-Python: >=3.9
|
|
@@ -36,7 +37,7 @@ Requires-Dist: mcp<3,>=2.2; python_version >= "3.10" and extra == "mcp"
|
|
|
36
37
|
Provides-Extra: dev
|
|
37
38
|
Requires-Dist: pytest; extra == "dev"
|
|
38
39
|
Requires-Dist: hypothesis; extra == "dev"
|
|
39
|
-
Requires-Dist: ruff==0.16.
|
|
40
|
+
Requires-Dist: ruff==0.16.9; extra == "dev"
|
|
40
41
|
Dynamic: license-file
|
|
41
42
|
|
|
42
43
|
<div align="center">
|
|
@@ -51,16 +52,19 @@ Dynamic: license-file
|
|
|
51
52
|
[](https://github.com/maximilianfeix/proxy-scraper/releases/latest)
|
|
52
53
|
[](#live-list)
|
|
53
54
|
[](https://pypi.org/project/proxy-scraper-cli/)
|
|
54
|
-
[](pyproject.toml)
|
|
55
56
|
[](https://github.com/maximilianfeix/proxy-scraper/stargazers)
|
|
56
57
|
[](LICENSE)
|
|
57
58
|
|
|
58
59
|
<a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://img.shields.io/badge/Browse_the_live_list-D4F77A?style=for-the-badge&labelColor=121113" alt="Browse the live list"></a>
|
|
60
|
+
<a href="https://github.com/maximilianfeix/free-proxy-list"><img src="https://img.shields.io/badge/Just_the_lists-121113?style=for-the-badge" alt="Just the lists: free-proxy-list"></a>
|
|
59
61
|
<a href="#install"><img src="https://img.shields.io/badge/Install-121113?style=for-the-badge" alt="Install"></a>
|
|
60
62
|
<a href="#from-python"><img src="https://img.shields.io/badge/Python_API-121113?style=for-the-badge" alt="Python API"></a>
|
|
61
63
|
<a href="#mcp"><img src="https://img.shields.io/badge/MCP_server-121113?style=for-the-badge" alt="MCP server for AI agents"></a>
|
|
62
64
|
<a href="bot/"><img src="https://img.shields.io/badge/Discord_bot-121113?style=for-the-badge" alt="Discord bot"></a>
|
|
63
65
|
|
|
66
|
+
**English** · [简体中文](README.zh-CN.md)
|
|
67
|
+
|
|
64
68
|
[Install](#install) · [Live list](#live-list) · [For AI agents](#mcp) · [Proxy server](#proxy-server) · [How it works](#how-it-works) · [Recipes](#recipes) · [Options](#options) · [FAQ](#faq)
|
|
65
69
|
|
|
66
70
|
</div>
|
|
@@ -114,6 +118,14 @@ proxy-scraper
|
|
|
114
118
|
|
|
115
119
|
The package is called `proxy-scraper-cli` on PyPI (the plain name is taken), the command is `proxy-scraper`.
|
|
116
120
|
|
|
121
|
+
**With [Homebrew](https://brew.sh)** on macOS and Linux:
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
brew install maximilianfeix/tap/proxy-scraper
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
The formula follows each release within about a day.
|
|
128
|
+
|
|
117
129
|
<details>
|
|
118
130
|
<summary><b>Other ways: Docker, pip, a faster event loop, or straight from the repo</b></summary>
|
|
119
131
|
<br>
|
|
@@ -178,7 +190,9 @@ Keys: <kbd>↑</kbd><kbd>↓</kbd> select · <kbd>Space</kbd> toggle · <kbd>1</
|
|
|
178
190
|
|
|
179
191
|
## Live proxy list
|
|
180
192
|
|
|
181
|
-
Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and publishes the hits to the [`proxy-list`](../../tree/proxy-list) branch – every entry worked in the last run,
|
|
193
|
+
Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and publishes the hits to the [`proxy-list`](../../tree/proxy-list) branch – every entry worked in the last run, best first: fast ones that are likely to still be up come on top.
|
|
194
|
+
|
|
195
|
+
> **Just want the lists?** They also live in a repository of their own, **[maximilianfeix/free-proxy-list](https://github.com/maximilianfeix/free-proxy-list)** – one list per protocol, per country and per site, updated every hour, with the numbers of the last run on the front page. Watch or star it to keep it at hand.
|
|
182
196
|
|
|
183
197
|
<a href="https://maximilianfeix.github.io/proxy-scraper/"><picture>
|
|
184
198
|
<source media="(prefers-color-scheme: dark)" srcset="https://maximilianfeix.github.io/proxy-scraper/chart-dark.svg">
|
|
@@ -319,7 +333,7 @@ Started without arguments, the tool asks what you need using the arrow keys –
|
|
|
319
333
|
<td valign="top">
|
|
320
334
|
|
|
321
335
|
**Real verification**<br>
|
|
322
|
-
Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests
|
|
336
|
+
Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests: over 3.6 million checks, 54 % of the proxies that answered the first request failed the second. A third request catches proxies that **tamper with content**: about one in 40 working proxies changed a known page, usually by injecting a script ([the numbers](docs/free-proxies-in-numbers.md)). Plus: HTTPS through a tunnel with **verified TLS**, anonymity level *elite / anonymous / transparent* and the country of the exit IP.
|
|
323
337
|
|
|
324
338
|
</td>
|
|
325
339
|
<td valign="top">
|
|
@@ -361,13 +375,13 @@ Speed chart, latency histogram, protocols, countries and the latest hits in real
|
|
|
361
375
|
<td valign="top">
|
|
362
376
|
|
|
363
377
|
**Runs everywhere**<br>
|
|
364
|
-
macOS, Linux and Windows, Python 3.9 to 3.
|
|
378
|
+
macOS, Linux and Windows, Python 3.9 to 3.14. Only two dependencies: `rich` and `certifi`. It even notices when a firewall blocks proxies.
|
|
365
379
|
|
|
366
380
|
</td>
|
|
367
381
|
<td valign="top">
|
|
368
382
|
|
|
369
383
|
**Thoroughly tested**<br>
|
|
370
|
-
|
|
384
|
+
760+ tests run offline against real mini proxies and honeypots on `localhost` – on Linux, macOS and Windows with Python 3.9, 3.11, 3.13 and 3.14.
|
|
371
385
|
|
|
372
386
|
</td>
|
|
373
387
|
</tr>
|
|
@@ -515,7 +529,7 @@ Without a password, `--serve-host` means **anyone who reaches the port can use i
|
|
|
515
529
|
|
|
516
530
|
- every connection goes through a different proxy (unless sticky); fast and proven ones are preferred
|
|
517
531
|
- `CONNECT` for HTTPS and plain HTTP requests; HTTP, SOCKS4 and SOCKS5 proxies can sit behind it (SOCKS5 with DNS through the proxy)
|
|
518
|
-
- HTTPS
|
|
532
|
+
- HTTPS prefers proxies that passed the test with **verified TLS**; when no such proxy fits right now – none matches a wish like `country-de`, or they have all dropped out – it uses the others – your client's own certificate check still catches a proxy that breaks the encryption, so leave it on
|
|
519
533
|
- if a proxy stays silent inside the tunnel or returns an error page instead of TLS, the same first packet quietly goes to the next one
|
|
520
534
|
- three failures in a row and a proxy leaves the rotation – every 5 minutes those get re-checked and come back if they work again
|
|
521
535
|
- `--serve-refill 6` checks fresh proxies every 6 hours in the background (the live list with `--recheck live`, otherwise the last run + history) with the same checks and filters, and adds the hits – a server that runs for days doesn't run dry
|
|
@@ -523,6 +537,10 @@ Without a password, `--serve-host` means **anyone who reaches the port can use i
|
|
|
523
537
|
|
|
524
538
|
In testing: 20 of 20 HTTPS requests succeeded, over 15 different exit IPs. In the wizard this is **Proxy server right away**.
|
|
525
539
|
|
|
540
|
+
**Live dashboard:** open `http://127.0.0.1:8899/__proxy-scraper/` in a browser – the pool, success rate, traffic, a live chart of requests, countries, the best proxies and the latest connections, refreshed every two seconds. It needs nothing from the internet, and `--serve-password` protects it like the rest.
|
|
541
|
+
|
|
542
|
+
<div align="center"><img src="docs/dashboard-live.png" alt="The live dashboard of the rotating proxy server: usable proxies, success rate, requests per second, countries and the latest connections" width="860"></div>
|
|
543
|
+
|
|
526
544
|
<a id="pool-api"></a>
|
|
527
545
|
|
|
528
546
|
### Proxy pool API
|
|
@@ -741,6 +759,18 @@ proxychains4 -f results/latest/proxychains.conf curl https://api.ipify.org
|
|
|
741
759
|
|
|
742
760
|
`clash.yaml` has all HTTP and SOCKS5 proxies plus a `url-test` group that always picks the fastest. `singbox.json` does the same for sing-box, SOCKS4 included, and opens a local proxy: `sing-box run -c results/latest/singbox.json`, then use `127.0.0.1:2080` as HTTP or SOCKS5 proxy. All three leave out HTTP proxies that can't tunnel (`CONNECT`), because these tools tunnel everything.
|
|
743
761
|
|
|
762
|
+
**Browsers and the OS** – `--export pac` writes `proxy.pac`: point Firefox, Chrome (via the system settings), FoxyProxy or macOS/Windows proxy settings at it and every request goes through the best 30 HTTP and SOCKS5 proxies in order, the browser moving on by itself when one fails. Local names and private IP addresses stay direct (a host name like `10.example.com` doesn't count as private). The file itself has no `DIRECT` fallback and resolves no host names, and SOCKS4 is left out because it would make the browser resolve names with your own DNS. Two Firefox settings matter: turn on *Proxy DNS when using SOCKS v5*, and set `network.proxy.failover_direct` to `false` in `about:config` – otherwise Firefox goes direct once every proxy has failed. No install at all: the live list publishes one every hour at `https://maximilianfeix.github.io/proxy-scraper/proxy.pac`.
|
|
763
|
+
|
|
764
|
+
**Burp Suite, OWASP ZAP, mitmproxy** – put the rotating server behind your intercepting proxy, so a scan or a brute-force test comes from many IPs while you still see every request. Start it with `proxy-scraper --recheck live --serve`, then:
|
|
765
|
+
|
|
766
|
+
| Tool | Where | Set |
|
|
767
|
+
|---|---|---|
|
|
768
|
+
| Burp Suite | *Settings → Network → Connections → Upstream proxy servers* | a rule for destination host `*`, proxy host `127.0.0.1`, port `8899`; with `--serve-password` add Basic auth (user `any` or a wish like `country-de`, the password) |
|
|
769
|
+
| OWASP ZAP | *Options → Network → Connection → HTTP Proxy* | host `127.0.0.1`, port `8899`, and under its authentication the user (`any` or a wish) and the password – use the HTTP setting, not SOCKS: ZAP never sends loopback addresses through its SOCKS proxy |
|
|
770
|
+
| mitmproxy | command line | `mitmproxy --mode upstream:http://127.0.0.1:8899 --upstream-auth country-us:x` |
|
|
771
|
+
|
|
772
|
+
Username wishes – `country-us` sends everything out through US proxies, `session-NAME` keeps one exit IP for a login flow – reach the server as the proxy login. mitmproxy sends it right away; Burp and ZAP may wait to be asked, so start the server with `--serve-password` when you rely on a wish there. **Keep your tool's upstream certificate checks on** (no `--ssl-insecure` in mitmproxy, no "ignore certificate errors" upstream in Burp or ZAP): free proxies are strangers, and that check is what stops one from reading or changing your HTTPS traffic. Tested end to end with mitmproxy 12, certificate checks on: HTTP and HTTPS through mitmproxy and the rotating server, every request out through a US proxy, and a site with an expired certificate refused. Only test what you're allowed to.
|
|
773
|
+
|
|
744
774
|
**In a pipe** – `-o -` prints the hits to stdout, the interface moves to stderr
|
|
745
775
|
|
|
746
776
|
```bash
|
|
@@ -834,7 +864,7 @@ curl.exe -x (Get-Content "$run\all.txt" -TotalCount 1) http://api.ipify.org
|
|
|
834
864
|
| `--serve-refill HOURS` | while serving, check fresh proxies every HOURS and add the hits to the pool |
|
|
835
865
|
| `--serve [PORT]` | afterwards serve as a rotating proxy on `127.0.0.1:PORT` (default: 8899) |
|
|
836
866
|
| `-o FILE` | also write all hits to this file; `-o -` prints them to stdout |
|
|
837
|
-
| `--export FORMATS` | extra files for other tools: `proxychains`, `clash`, `singbox`, `curl` or `all` |
|
|
867
|
+
| `--export FORMATS` | extra files for other tools: `proxychains`, `clash`, `singbox`, `curl`, `pac` or `all` |
|
|
838
868
|
| `-V`, `--version` | print the version |
|
|
839
869
|
| `--completion SHELL` | print the tab completion script for bash, zsh, fish or PowerShell |
|
|
840
870
|
|
|
@@ -920,7 +950,7 @@ Only for things that don't matter. Public proxies are run by strangers who can r
|
|
|
920
950
|
|
|
921
951
|
What's next is in the [open issues](../../issues) – ideas and wishes are welcome as an [issue](../../issues/new/choose). During Hacktoberfest there are [beginner-friendly issues](../../issues?q=is%3Aopen+label%3Ahacktoberfest) with pointers on where to start.
|
|
922
952
|
|
|
923
|
-
|
|
953
|
+
Shipped in v1.22 and v1.23: a live dashboard for the rotating server, `--export pac` and an hourly `proxy.pac`, "best first" that counts how likely a proxy is still up, the lists in a repository of their own ([free-proxy-list](https://github.com/maximilianfeix/free-proxy-list)), daily snapshots on Hugging Face, `brew install`, a Chinese README, Python 3.14, and a recipe for Burp Suite, ZAP and mitmproxy.
|
|
924
954
|
|
|
925
955
|
Shipped in v1.10 to v1.21: download speed per proxy and "best first" everywhere, daily snapshots, a rotating proxy URL for requests, httpx and Playwright, `--pick` for working proxies without a scan, a proxy pool API compatible with jhao104/proxy_pool, your own lists with `--source`, a GitHub Action, a Telegram bot and an Atom feed for the weekly report.
|
|
926
956
|
|
|
@@ -969,12 +999,13 @@ proxyscraper/
|
|
|
969
999
|
├── geodb.py DB-IP country database (monthly, binary search)
|
|
970
1000
|
├── targets.py target sites for --target
|
|
971
1001
|
├── output.py result files
|
|
972
|
-
├── exporters.py proxychains, Clash, sing-box and
|
|
1002
|
+
├── exporters.py proxychains, Clash, sing-box, curl and PAC formats (--export)
|
|
973
1003
|
├── server/ rotating proxy server (--serve): pool · http · upstream · socks · status · api · core
|
|
974
1004
|
├── api.py find_proxies() / check_proxies() for Python
|
|
975
1005
|
├── agent.py MCP tools without the SDK: live list, filters, fetch through proxies
|
|
976
1006
|
├── mcp_server.py MCP server (proxy-scraper-mcp) for AI agents
|
|
977
1007
|
├── publish.py live list for GitHub Actions
|
|
1008
|
+
├── mirror.py the same lists in their own repo (free-proxy-list)
|
|
978
1009
|
├── paths.py where state and results are stored
|
|
979
1010
|
├── compat.py differences between Unix and Windows
|
|
980
1011
|
├── netio.py small HTTP client on asyncio
|
{proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxy_scraper_cli.egg-info/SOURCES.txt
RENAMED
|
@@ -29,6 +29,7 @@ proxyscraper/history.py
|
|
|
29
29
|
proxyscraper/judges.py
|
|
30
30
|
proxyscraper/mcp_entry.py
|
|
31
31
|
proxyscraper/mcp_server.py
|
|
32
|
+
proxyscraper/mirror.py
|
|
32
33
|
proxyscraper/netio.py
|
|
33
34
|
proxyscraper/options.py
|
|
34
35
|
proxyscraper/output.py
|
|
@@ -50,6 +51,7 @@ proxyscraper/targets.py
|
|
|
50
51
|
proxyscraper/server/__init__.py
|
|
51
52
|
proxyscraper/server/api.py
|
|
52
53
|
proxyscraper/server/core.py
|
|
54
|
+
proxyscraper/server/dashboard.html
|
|
53
55
|
proxyscraper/server/http.py
|
|
54
56
|
proxyscraper/server/pool.py
|
|
55
57
|
proxyscraper/server/socks.py
|
|
@@ -87,9 +89,11 @@ tests/test_learning.py
|
|
|
87
89
|
tests/test_live_api.py
|
|
88
90
|
tests/test_mcp_server.py
|
|
89
91
|
tests/test_mine_sources.py
|
|
92
|
+
tests/test_mirror.py
|
|
90
93
|
tests/test_options.py
|
|
91
94
|
tests/test_output_stdout.py
|
|
92
95
|
tests/test_own_sources.py
|
|
96
|
+
tests/test_pac.py
|
|
93
97
|
tests/test_packaging.py
|
|
94
98
|
tests/test_pages.py
|
|
95
99
|
tests/test_parsing.py
|
|
@@ -105,6 +109,7 @@ tests/test_report.py
|
|
|
105
109
|
tests/test_robustness.py
|
|
106
110
|
tests/test_rotator.py
|
|
107
111
|
tests/test_serve_api.py
|
|
112
|
+
tests/test_serve_dashboard.py
|
|
108
113
|
tests/test_serve_password.py
|
|
109
114
|
tests/test_serve_refill.py
|
|
110
115
|
tests/test_serve_v2.py
|
|
@@ -115,6 +120,7 @@ tests/test_sites.py
|
|
|
115
120
|
tests/test_sources_json.py
|
|
116
121
|
tests/test_speed.py
|
|
117
122
|
tests/test_speed_ranking.py
|
|
123
|
+
tests/test_survival_ranking.py
|
|
118
124
|
tests/test_tamper.py
|
|
119
125
|
tests/test_targets.py
|
|
120
126
|
tests/test_ui.py
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""proxy-scraper: collects public HTTP/SOCKS4/SOCKS5 proxies from hundreds of sources, checks them
|
|
2
2
|
in parallel with its own protocol handshakes and learns which sources deliver good proxies."""
|
|
3
3
|
|
|
4
|
-
__version__ = "1.
|
|
4
|
+
__version__ = "1.23.0"
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
def __getattr__(name):
|
|
@@ -181,8 +181,8 @@ def select(rows: Iterable[dict], protocol: str = "any", countries: Iterable[str]
|
|
|
181
181
|
elite_only: bool = False, exclude_datacenter: bool = False, exclude_blocklisted: bool = False,
|
|
182
182
|
stable_only: bool = False, max_latency_ms: int = 0, run_hours: int = 1, min_uptime: int = 0,
|
|
183
183
|
works_on: Iterable[str] = (), min_speed_kbps: int = 0) -> List[dict]:
|
|
184
|
-
"""The rows that pass every filter, best first (answer plus download speed
|
|
185
|
-
website."""
|
|
184
|
+
"""The rows that pass every filter, best first (answer plus download speed, and how likely it's still up) – the
|
|
185
|
+
same filters as on the website."""
|
|
186
186
|
return best_first(matching(rows, protocol, countries, https_only, elite_only, exclude_datacenter,
|
|
187
187
|
exclude_blocklisted, stable_only, max_latency_ms, run_hours, min_uptime, works_on,
|
|
188
188
|
min_speed_kbps))
|
|
@@ -208,9 +208,9 @@ async def live_proxies_async(*, types: Iterable[str] = PROXY_TYPES, countries: I
|
|
|
208
208
|
anonymity: str = "", max_latency: int = 0, no_datacenter: bool = False,
|
|
209
209
|
no_blocklisted: bool = False, min_uptime: int = 0, works_on: Iterable[str] = (),
|
|
210
210
|
min_speed: int = 0, limit: int = 0) -> List[LiveProxy]:
|
|
211
|
-
"""The proxies from the hourly list that pass the filters,
|
|
212
|
-
download speed. Nothing is checked here: they worked from GitHub's servers
|
|
213
|
-
ago). find_proxies checks from your network.
|
|
211
|
+
"""The proxies from the hourly list that pass the filters, best first – by first answer plus measured
|
|
212
|
+
download speed, and how likely each one is still up. Nothing is checked here: they worked from GitHub's servers
|
|
213
|
+
in the last run (at most an hour ago). find_proxies checks from your network.
|
|
214
214
|
|
|
215
215
|
Same filters as find_proxies, plus
|
|
216
216
|
min_uptime only proxies listed in at least this share (percent) of the week's runs, e.g. 90
|
|
@@ -220,7 +220,7 @@ async def live_proxies_async(*, types: Iterable[str] = PROXY_TYPES, countries: I
|
|
|
220
220
|
limit at most this many (0 = all)
|
|
221
221
|
"""
|
|
222
222
|
from .agent import AgentError, LiveSource
|
|
223
|
-
from .ranking import
|
|
223
|
+
from .ranking import expected_ms, typical_speed
|
|
224
224
|
from .sites import SITE
|
|
225
225
|
|
|
226
226
|
types = _types(types)
|
|
@@ -245,9 +245,11 @@ async def live_proxies_async(*, types: Iterable[str] = PROXY_TYPES, countries: I
|
|
|
245
245
|
raise ConnectionError(str(e)) from None
|
|
246
246
|
found = [_live_proxy(r, data.run_hours) for r in data.rows
|
|
247
247
|
if r.get("ptype") in types and isinstance(r.get("proxy"), str)]
|
|
248
|
-
#
|
|
248
|
+
# best first – by how quickly a page comes through when the list has download speeds (#186), and how likely the
|
|
249
|
+
# proxy is still up
|
|
249
250
|
typical = typical_speed(p.speed_kbps for p in found)
|
|
250
|
-
|
|
251
|
+
run_hours = data.run_hours or 1
|
|
252
|
+
found.sort(key=lambda p: expected_ms(p.latency, p.speed_kbps, p.https, typical, p.up_for_hours // run_hours))
|
|
251
253
|
found = [p for p in found if opts.filters.accepts(p) and (not min_uptime or (p.uptime_7d or 0) >= min_uptime)
|
|
252
254
|
and all(p.sites.get(s) for s in works_on) and (not min_speed or (p.speed_kbps or 0) >= min_speed)]
|
|
253
255
|
return found[:limit] if limit else found
|