proxy-scraper-cli 1.26.0__tar.gz → 1.27.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/PKG-INFO +27 -25
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/README.md +26 -24
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxy_scraper_cli.egg-info/PKG-INFO +27 -25
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxy_scraper_cli.egg-info/SOURCES.txt +2 -1
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/__init__.py +1 -1
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_serve_api.py +24 -1
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_serve_dashboard.py +2 -1
- proxy_scraper_cli-1.27.0/tests/test_workflow_pip.py +85 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/LICENSE +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxy_scraper_cli.egg-info/dependency_links.txt +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxy_scraper_cli.egg-info/entry_points.txt +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxy_scraper_cli.egg-info/requires.txt +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxy_scraper_cli.egg-info/top_level.txt +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/__main__.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/action.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/agent.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/api.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/app.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/asndb.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/blocklist.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/charts.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/checker.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/cli.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/compat.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/completion.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/exporters.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/fetchcache.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/geo.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/geodb.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/handshake.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/history.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/judges.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/mcp_entry.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/mcp_server.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/mirror.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/netio.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/options.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/output.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/pages.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/parsing.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/paths.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/pipeline.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/preferences.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/proxypages.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/publish.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/ranking.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/report.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/runsteps.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/server/__init__.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/server/api.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/server/core.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/server/dashboard.html +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/server/http.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/server/pool.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/server/socks.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/server/status.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/server/upstream.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/site/apple-touch-icon.png +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/site/googleaac1161b7853c5b5.html +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/site/index.html +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/site/logo.png +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/site/og.png +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/sites.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/sources.json +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/sources.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/speed.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/targets.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/ui/__init__.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/ui/dashboard.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/ui/keys.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/ui/report.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/ui/serve.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/ui/widgets.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/proxyscraper/ui/wizard.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/pyproject.toml +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/setup.cfg +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_action.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_agent.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_api.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_app.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_asndb.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_auth.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_blocklist.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_charts.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_checker.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_citation.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_colon_logins.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_compat.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_completion.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_discovery_growth.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_exporters.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_fetchcache.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_geodb.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_hf_card.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_hf_parquet.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_judges.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_learning.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_live_api.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_mcp_server.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_mine_sources.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_mirror.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_options.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_output_stdout.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_own_sources.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_pac.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_packaging.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_pages.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_parsing.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_parsing_fuzz.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_paths.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_pick.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_pipeline.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_preferences.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_proxy_pages.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_proxy_sources.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_publish.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_pypi_readme.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_recheck_stdin.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_report.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_robustness.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_rotator.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_serve_password.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_serve_refill.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_serve_v2.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_server.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_server_closed_writer.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_server_fixes.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_sites.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_sources_json.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_speed.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_speed_ranking.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_survival_ranking.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_tamper.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_targets.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_ui.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_uptime.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_wizard.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_workflow_permissions.py +0 -0
- {proxy_scraper_cli-1.26.0 → proxy_scraper_cli-1.27.0}/tests/test_workflow_pins.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: proxy-scraper-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.27.0
|
|
4
4
|
Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
|
|
5
5
|
Author: Maximilian Feix
|
|
6
6
|
License-Expression: MIT
|
|
@@ -45,9 +45,9 @@ Dynamic: license-file
|
|
|
45
45
|
<div align="center">
|
|
46
46
|
|
|
47
47
|
<picture>
|
|
48
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
49
|
-
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
50
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
48
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/banner-dark.svg">
|
|
49
|
+
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/banner-light.svg">
|
|
50
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/banner-dark.svg" alt="proxy-scraper – free proxies that actually work" width="100%">
|
|
51
51
|
</picture>
|
|
52
52
|
|
|
53
53
|
[](https://github.com/maximilianfeix/proxy-scraper/actions/workflows/tests.yml)
|
|
@@ -55,18 +55,18 @@ Dynamic: license-file
|
|
|
55
55
|
[](#live-list)
|
|
56
56
|
[](https://pypi.org/project/proxy-scraper-cli/)
|
|
57
57
|
[](https://pepy.tech/projects/proxy-scraper-cli)
|
|
58
|
-
[](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
58
|
+
[](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/pyproject.toml)
|
|
59
59
|
[](https://github.com/maximilianfeix/proxy-scraper/stargazers)
|
|
60
|
-
[](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
60
|
+
[](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/LICENSE)
|
|
61
61
|
|
|
62
62
|
<a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://img.shields.io/badge/Browse_the_live_list-D4F77A?style=for-the-badge&labelColor=121113" alt="Browse the live list"></a>
|
|
63
63
|
<a href="https://github.com/maximilianfeix/free-proxy-list"><img src="https://img.shields.io/badge/Just_the_lists-121113?style=for-the-badge" alt="Just the lists: free-proxy-list"></a>
|
|
64
64
|
<a href="#install"><img src="https://img.shields.io/badge/Install-121113?style=for-the-badge" alt="Install"></a>
|
|
65
65
|
<a href="#from-python"><img src="https://img.shields.io/badge/Python_API-121113?style=for-the-badge" alt="Python API"></a>
|
|
66
66
|
<a href="#mcp"><img src="https://img.shields.io/badge/MCP_server-121113?style=for-the-badge" alt="MCP server for AI agents"></a>
|
|
67
|
-
<a href="https://github.com/maximilianfeix/proxy-scraper/tree/v1.
|
|
67
|
+
<a href="https://github.com/maximilianfeix/proxy-scraper/tree/v1.27.0/bot/"><img src="https://img.shields.io/badge/Discord_bot-121113?style=for-the-badge" alt="Discord bot"></a>
|
|
68
68
|
|
|
69
|
-
**English** · [简体中文](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
69
|
+
**English** · [简体中文](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/README.zh-CN.md)
|
|
70
70
|
|
|
71
71
|
[Install](#install) · [Live list](#live-list) · [For AI agents](#mcp) · [Proxy server](#proxy-server) · [How it works](#how-it-works) · [Recipes](#recipes) · [Options](#options) · [FAQ](#faq)
|
|
72
72
|
|
|
@@ -76,7 +76,7 @@ Dynamic: license-file
|
|
|
76
76
|
|
|
77
77
|
Most free proxy lists are 95 % dead, and a good part of the rest are honeypots or proxies that inject scripts into your pages. **proxy-scraper** collects public HTTP, SOCKS4 and SOCKS5 proxies from 700+ sources and keeps only the ones that pass every check. It learns with every run which sources are worth it, and it can turn the result into a single rotating proxy.
|
|
78
78
|
|
|
79
|
-
**In numbers:** in one day it checked 3.6 million free proxies – 1.1 % worked, and more than half of the ones that answered failed a second request. [What's actually out there →](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
79
|
+
**In numbers:** in one day it checked 3.6 million free proxies – 1.1 % worked, and more than half of the ones that answered failed a second request. [What's actually out there →](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/docs/free-proxies-in-numbers.md)
|
|
80
80
|
|
|
81
81
|
**Used by:** the hourly list is a built-in source of [monosans/proxy-scraper-checker](https://github.com/monosans/proxy-scraper-checker) and [gfpcom/free-proxy-list](https://github.com/gfpcom/free-proxy-list).
|
|
82
82
|
|
|
@@ -92,7 +92,7 @@ curl -s https://maximilianfeix.github.io/proxy-scraper/socks5.txt | head
|
|
|
92
92
|
`uvx` comes with [uv](https://docs.astral.sh/uv/); `pipx run proxy-scraper-cli --pick 5` does the same.
|
|
93
93
|
|
|
94
94
|
<div align="center">
|
|
95
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
95
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/demo.svg" alt="Animated demo: the live dashboard during a check, the report, the setup wizard and collecting" width="880">
|
|
96
96
|
</div>
|
|
97
97
|
|
|
98
98
|
> **Who runs free proxies – and is it safe to use them?** Mostly nobody on purpose: misconfigured servers with an open port, a few deliberately open ones, infected machines, and traps that log or rewrite your traffic. That's why every proxy here has to pass [five checks](#how-it-works), and why they're for public data and testing – never for logins, payments or anything personal. [More on where they come from →](#who-runs-free-proxies)
|
|
@@ -181,7 +181,7 @@ Installed, the learned state lives in your user data folder (`~/Library/Applicat
|
|
|
181
181
|
Started without arguments, a wizard asks what you are looking for:
|
|
182
182
|
|
|
183
183
|
<div align="center">
|
|
184
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
184
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/wizard.svg" alt="Setup wizard" width="760">
|
|
185
185
|
</div>
|
|
186
186
|
|
|
187
187
|
| Preset | What it does |
|
|
@@ -216,7 +216,7 @@ Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and pub
|
|
|
216
216
|
|
|
217
217
|
Every protocol and country also has its own page with a plain download, e.g. [SOCKS5](https://maximilianfeix.github.io/proxy-scraper/socks5/) or [Germany](https://maximilianfeix.github.io/proxy-scraper/country/de/) (`country/de/proxies.txt`).
|
|
218
218
|
|
|
219
|
-
<div align="center"><a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
219
|
+
<div align="center"><a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/website.png" alt="The live list website" width="860"></a></div>
|
|
220
220
|
|
|
221
221
|
<div align="center">
|
|
222
222
|
|
|
@@ -262,6 +262,8 @@ Or let the tool start from it: `proxy-scraper --recheck live` downloads the list
|
|
|
262
262
|
|
|
263
263
|
`proxy-scraper-mcp` is an [MCP](https://modelcontextprotocol.io) server: Claude Code, Claude Desktop, Cursor, VS Code, Codex and any other MCP client can ask for working proxies and load pages through them.
|
|
264
264
|
|
|
265
|
+
[](https://glama.ai/mcp/servers/maximilianfeix/proxy-scraper)
|
|
266
|
+
|
|
265
267
|
| Tool | What it does |
|
|
266
268
|
|---|---|
|
|
267
269
|
| `get_proxies` | working proxies right now, from the hourly list – filter by protocol, country, HTTPS, elite, no datacenter, not blocklisted, stable, uptime, latency, gets through to Google/Reddit/Amazon |
|
|
@@ -358,7 +360,7 @@ Started without arguments, the tool asks what you need using the arrow keys –
|
|
|
358
360
|
<td valign="top">
|
|
359
361
|
|
|
360
362
|
**Real verification**<br>
|
|
361
|
-
Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests: over 3.6 million checks, 54 % of the proxies that answered the first request failed the second. A third request catches proxies that **tamper with content**: about one in 40 working proxies changed a known page, usually by injecting a script ([the numbers](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
363
|
+
Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests: over 3.6 million checks, 54 % of the proxies that answered the first request failed the second. A third request catches proxies that **tamper with content**: about one in 40 working proxies changed a known page, usually by injecting a script ([the numbers](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/docs/free-proxies-in-numbers.md)). Plus: HTTPS through a tunnel with **verified TLS**, anonymity level *elite / anonymous / transparent* and the country of the exit IP.
|
|
362
364
|
|
|
363
365
|
</td>
|
|
364
366
|
<td valign="top">
|
|
@@ -543,7 +545,7 @@ proxy-scraper --recheck --serve --serve-host 0.0.0.0
|
|
|
543
545
|
curl -x "http://country-de:$PROXY_SCRAPER_SERVE_PASSWORD@your-server:8899" https://api.ipify.org
|
|
544
546
|
```
|
|
545
547
|
|
|
546
|
-
Without a password, `--serve-host` means **anyone who reaches the port can use it**. The ready-made [`compose.yaml`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
548
|
+
Without a password, `--serve-host` means **anyone who reaches the port can use it**. The ready-made [`compose.yaml`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/compose.yaml) starts the server in Docker from the live list, with a password, a health check and learned state in a volume: put `PROXY_PASSWORD=…` into `.env`, then `docker compose up -d`.
|
|
547
549
|
|
|
548
550
|
| Option | What it does |
|
|
549
551
|
|---|---|
|
|
@@ -564,7 +566,7 @@ In testing: 20 of 20 HTTPS requests succeeded, over 15 different exit IPs. In th
|
|
|
564
566
|
|
|
565
567
|
**Live dashboard:** open `http://127.0.0.1:8899/__proxy-scraper/` in a browser – the pool, success rate, traffic, a live chart of requests, countries, the best proxies and the latest connections, refreshed every two seconds. It needs nothing from the internet, and `--serve-password` protects it like the rest.
|
|
566
568
|
|
|
567
|
-
<div align="center"><img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
569
|
+
<div align="center"><img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/dashboard-live.png" alt="The live dashboard of the rotating proxy server: usable proxies, success rate, requests per second, countries and the latest connections" width="860"></div>
|
|
568
570
|
|
|
569
571
|
<a id="pool-api"></a>
|
|
570
572
|
|
|
@@ -602,16 +604,16 @@ requests.get("https://example.com", proxies={"http": proxy, "https": proxy})
|
|
|
602
604
|
|
|
603
605
|
## Discord and Telegram bot
|
|
604
606
|
|
|
605
|
-
The live list can also come to you: [`bot/`](https://github.com/maximilianfeix/proxy-scraper/tree/v1.
|
|
607
|
+
The live list can also come to you: [`bot/`](https://github.com/maximilianfeix/proxy-scraper/tree/v1.27.0/bot/) is a Discord bot that posts every run into a server – a summary with the fastest proxies, the full lists per protocol as files, and slash commands like `/proxies type:socks5 country:DE https:true`. It sets up its own read-only channels when you invite it, and a GitHub Action deploys it to a server as a systemd service. The same process answers on Telegram too: `/proxy de socks5` for one proxy with a curl line, `/proxies 10 us https` for a short list. Setup in [bot/README.md](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/bot/README.md).
|
|
606
608
|
|
|
607
609
|
<a id="how-it-works"></a>
|
|
608
610
|
|
|
609
611
|
## How it works
|
|
610
612
|
|
|
611
613
|
<picture>
|
|
612
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
613
|
-
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
614
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
614
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/checks-dark.svg">
|
|
615
|
+
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/checks-light.svg">
|
|
616
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/checks-dark.svg" alt="The five checks: 700+ lists, a real handshake, two sites with one IP, nothing injected, the details" width="100%">
|
|
615
617
|
</picture>
|
|
616
618
|
|
|
617
619
|
|
|
@@ -626,7 +628,7 @@ flowchart LR
|
|
|
626
628
|
G -. next run .-> C
|
|
627
629
|
```
|
|
628
630
|
|
|
629
|
-
1. **Sources** – the curated list in [`sources.json`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
631
|
+
1. **Sources** – the curated list in [`sources.json`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/proxyscraper/sources.json), meta sources (other projects that maintain lists of proxy sources) and, once a day, a GitHub search for actively maintained repos. Every search adds to what earlier ones found; a list the search hasn't seen for three weeks drops out.
|
|
630
632
|
2. **Collect** – plain text, HTML tables, JSON APIs and `type://ip:port` lines are recognized; private and reserved address ranges are dropped. Lists that haven't changed since the last run answer `304` and come from a local cache – a second run right after the first loads 0 MB instead of ~160 MB.
|
|
631
633
|
3. **Prioritize** – known working proxies first, then by the learned hit rate of their sources.
|
|
632
634
|
4. **Check** – every proxy has to fetch its exit IP from a check target (`checkip.amazonaws.com`, with `ifconfig.me`, `ipinfo.io`, `wtfismyip.com` and `ident.me` as reserves – none of them behind Cloudflare) and return a valid, *foreign* IP. If the target goes down mid-run, the tool switches and re-checks the proxies that were affected, so the statistics don't learn from an outage. Anyone passing your own IP through is out. Then comes the **confirmation** via `httpbin.org`: fake proxies that only answer the first check with “200 + IP” fail here. Finally a static HTML page has to arrive byte for byte as it does without a proxy – anyone injecting ads or scripts is out.
|
|
@@ -638,9 +640,9 @@ flowchart LR
|
|
|
638
640
|
<summary><b>📸 See the live dashboard and final report</b></summary>
|
|
639
641
|
<br>
|
|
640
642
|
<div align="center">
|
|
641
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
643
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/dashboard.svg" alt="Live dashboard while checking" width="860">
|
|
642
644
|
<br><br>
|
|
643
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
645
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/summary.svg" alt="Final report after a run" width="860">
|
|
644
646
|
</div>
|
|
645
647
|
</details>
|
|
646
648
|
|
|
@@ -1027,7 +1029,7 @@ Shipped in [v1.5](https://github.com/maximilianfeix/proxy-scraper/milestone/5):
|
|
|
1027
1029
|
|
|
1028
1030
|
## Contributing
|
|
1029
1031
|
|
|
1030
|
-
Bug reports, new sources and pull requests are very welcome – see [CONTRIBUTING.md](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
1032
|
+
Bug reports, new sources and pull requests are very welcome – see [CONTRIBUTING.md](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/CONTRIBUTING.md), and [docs/ARCHITECTURE.md](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/docs/ARCHITECTURE.md) for how the pieces fit together. The short version:
|
|
1031
1033
|
|
|
1032
1034
|
```bash
|
|
1033
1035
|
pip install -e ".[dev]"
|
|
@@ -1101,7 +1103,7 @@ Want to join? [Beginner-friendly issues](https://github.com/maximilianfeix/proxy
|
|
|
1101
1103
|
|
|
1102
1104
|
## Acknowledgements
|
|
1103
1105
|
|
|
1104
|
-
proxy-scraper stands on the work of the people who publish free proxy lists. Thanks to everyone listed in [`sources.json`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
1106
|
+
proxy-scraper stands on the work of the people who publish free proxy lists. Thanks to everyone listed in [`sources.json`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/proxyscraper/sources.json), and especially to
|
|
1105
1107
|
|
|
1106
1108
|
- [monosans/proxy-scraper-checker](https://github.com/monosans/proxy-scraper-checker) and [gfpcom/free-proxy-list](https://github.com/gfpcom/free-proxy-list), whose curated source collections are read as meta sources
|
|
1107
1109
|
- [Textualize/rich](https://github.com/Textualize/rich), which draws the whole terminal UI
|
|
@@ -1116,7 +1118,7 @@ This tool only collects publicly listed proxies and checks whether they work. Yo
|
|
|
1116
1118
|
|
|
1117
1119
|
---
|
|
1118
1120
|
|
|
1119
|
-
<sub>Made in Germany by <a href="https://github.com/maximilianfeix">@maximilianfeix</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
1121
|
+
<sub>Made in Germany by <a href="https://github.com/maximilianfeix">@maximilianfeix</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/LICENSE">MIT License</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/CHANGELOG.md">Changelog</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/SECURITY.md">Security</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/CONTRIBUTING.md">Contributing</a></sub>
|
|
1120
1122
|
|
|
1121
1123
|
<sub>If proxy-scraper saves you time, a ⭐ helps others find it.</sub>
|
|
1122
1124
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
<div align="center">
|
|
2
2
|
|
|
3
3
|
<picture>
|
|
4
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
5
|
-
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
6
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/banner-dark.svg">
|
|
5
|
+
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/banner-light.svg">
|
|
6
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/banner-dark.svg" alt="proxy-scraper – free proxies that actually work" width="100%">
|
|
7
7
|
</picture>
|
|
8
8
|
|
|
9
9
|
[](https://github.com/maximilianfeix/proxy-scraper/actions/workflows/tests.yml)
|
|
@@ -11,18 +11,18 @@
|
|
|
11
11
|
[](#live-list)
|
|
12
12
|
[](https://pypi.org/project/proxy-scraper-cli/)
|
|
13
13
|
[](https://pepy.tech/projects/proxy-scraper-cli)
|
|
14
|
-
[](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
14
|
+
[](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/pyproject.toml)
|
|
15
15
|
[](https://github.com/maximilianfeix/proxy-scraper/stargazers)
|
|
16
|
-
[](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
16
|
+
[](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/LICENSE)
|
|
17
17
|
|
|
18
18
|
<a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://img.shields.io/badge/Browse_the_live_list-D4F77A?style=for-the-badge&labelColor=121113" alt="Browse the live list"></a>
|
|
19
19
|
<a href="https://github.com/maximilianfeix/free-proxy-list"><img src="https://img.shields.io/badge/Just_the_lists-121113?style=for-the-badge" alt="Just the lists: free-proxy-list"></a>
|
|
20
20
|
<a href="#install"><img src="https://img.shields.io/badge/Install-121113?style=for-the-badge" alt="Install"></a>
|
|
21
21
|
<a href="#from-python"><img src="https://img.shields.io/badge/Python_API-121113?style=for-the-badge" alt="Python API"></a>
|
|
22
22
|
<a href="#mcp"><img src="https://img.shields.io/badge/MCP_server-121113?style=for-the-badge" alt="MCP server for AI agents"></a>
|
|
23
|
-
<a href="https://github.com/maximilianfeix/proxy-scraper/tree/v1.
|
|
23
|
+
<a href="https://github.com/maximilianfeix/proxy-scraper/tree/v1.27.0/bot/"><img src="https://img.shields.io/badge/Discord_bot-121113?style=for-the-badge" alt="Discord bot"></a>
|
|
24
24
|
|
|
25
|
-
**English** · [简体中文](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
25
|
+
**English** · [简体中文](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/README.zh-CN.md)
|
|
26
26
|
|
|
27
27
|
[Install](#install) · [Live list](#live-list) · [For AI agents](#mcp) · [Proxy server](#proxy-server) · [How it works](#how-it-works) · [Recipes](#recipes) · [Options](#options) · [FAQ](#faq)
|
|
28
28
|
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
|
|
33
33
|
Most free proxy lists are 95 % dead, and a good part of the rest are honeypots or proxies that inject scripts into your pages. **proxy-scraper** collects public HTTP, SOCKS4 and SOCKS5 proxies from 700+ sources and keeps only the ones that pass every check. It learns with every run which sources are worth it, and it can turn the result into a single rotating proxy.
|
|
34
34
|
|
|
35
|
-
**In numbers:** in one day it checked 3.6 million free proxies – 1.1 % worked, and more than half of the ones that answered failed a second request. [What's actually out there →](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
35
|
+
**In numbers:** in one day it checked 3.6 million free proxies – 1.1 % worked, and more than half of the ones that answered failed a second request. [What's actually out there →](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/docs/free-proxies-in-numbers.md)
|
|
36
36
|
|
|
37
37
|
**Used by:** the hourly list is a built-in source of [monosans/proxy-scraper-checker](https://github.com/monosans/proxy-scraper-checker) and [gfpcom/free-proxy-list](https://github.com/gfpcom/free-proxy-list).
|
|
38
38
|
|
|
@@ -48,7 +48,7 @@ curl -s https://maximilianfeix.github.io/proxy-scraper/socks5.txt | head
|
|
|
48
48
|
`uvx` comes with [uv](https://docs.astral.sh/uv/); `pipx run proxy-scraper-cli --pick 5` does the same.
|
|
49
49
|
|
|
50
50
|
<div align="center">
|
|
51
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
51
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/demo.svg" alt="Animated demo: the live dashboard during a check, the report, the setup wizard and collecting" width="880">
|
|
52
52
|
</div>
|
|
53
53
|
|
|
54
54
|
> **Who runs free proxies – and is it safe to use them?** Mostly nobody on purpose: misconfigured servers with an open port, a few deliberately open ones, infected machines, and traps that log or rewrite your traffic. That's why every proxy here has to pass [five checks](#how-it-works), and why they're for public data and testing – never for logins, payments or anything personal. [More on where they come from →](#who-runs-free-proxies)
|
|
@@ -137,7 +137,7 @@ Installed, the learned state lives in your user data folder (`~/Library/Applicat
|
|
|
137
137
|
Started without arguments, a wizard asks what you are looking for:
|
|
138
138
|
|
|
139
139
|
<div align="center">
|
|
140
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
140
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/wizard.svg" alt="Setup wizard" width="760">
|
|
141
141
|
</div>
|
|
142
142
|
|
|
143
143
|
| Preset | What it does |
|
|
@@ -172,7 +172,7 @@ Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and pub
|
|
|
172
172
|
|
|
173
173
|
Every protocol and country also has its own page with a plain download, e.g. [SOCKS5](https://maximilianfeix.github.io/proxy-scraper/socks5/) or [Germany](https://maximilianfeix.github.io/proxy-scraper/country/de/) (`country/de/proxies.txt`).
|
|
174
174
|
|
|
175
|
-
<div align="center"><a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
175
|
+
<div align="center"><a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/website.png" alt="The live list website" width="860"></a></div>
|
|
176
176
|
|
|
177
177
|
<div align="center">
|
|
178
178
|
|
|
@@ -218,6 +218,8 @@ Or let the tool start from it: `proxy-scraper --recheck live` downloads the list
|
|
|
218
218
|
|
|
219
219
|
`proxy-scraper-mcp` is an [MCP](https://modelcontextprotocol.io) server: Claude Code, Claude Desktop, Cursor, VS Code, Codex and any other MCP client can ask for working proxies and load pages through them.
|
|
220
220
|
|
|
221
|
+
[](https://glama.ai/mcp/servers/maximilianfeix/proxy-scraper)
|
|
222
|
+
|
|
221
223
|
| Tool | What it does |
|
|
222
224
|
|---|---|
|
|
223
225
|
| `get_proxies` | working proxies right now, from the hourly list – filter by protocol, country, HTTPS, elite, no datacenter, not blocklisted, stable, uptime, latency, gets through to Google/Reddit/Amazon |
|
|
@@ -314,7 +316,7 @@ Started without arguments, the tool asks what you need using the arrow keys –
|
|
|
314
316
|
<td valign="top">
|
|
315
317
|
|
|
316
318
|
**Real verification**<br>
|
|
317
|
-
Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests: over 3.6 million checks, 54 % of the proxies that answered the first request failed the second. A third request catches proxies that **tamper with content**: about one in 40 working proxies changed a known page, usually by injecting a script ([the numbers](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
319
|
+
Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests: over 3.6 million checks, 54 % of the proxies that answered the first request failed the second. A third request catches proxies that **tamper with content**: about one in 40 working proxies changed a known page, usually by injecting a script ([the numbers](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/docs/free-proxies-in-numbers.md)). Plus: HTTPS through a tunnel with **verified TLS**, anonymity level *elite / anonymous / transparent* and the country of the exit IP.
|
|
318
320
|
|
|
319
321
|
</td>
|
|
320
322
|
<td valign="top">
|
|
@@ -499,7 +501,7 @@ proxy-scraper --recheck --serve --serve-host 0.0.0.0
|
|
|
499
501
|
curl -x "http://country-de:$PROXY_SCRAPER_SERVE_PASSWORD@your-server:8899" https://api.ipify.org
|
|
500
502
|
```
|
|
501
503
|
|
|
502
|
-
Without a password, `--serve-host` means **anyone who reaches the port can use it**. The ready-made [`compose.yaml`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
504
|
+
Without a password, `--serve-host` means **anyone who reaches the port can use it**. The ready-made [`compose.yaml`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/compose.yaml) starts the server in Docker from the live list, with a password, a health check and learned state in a volume: put `PROXY_PASSWORD=…` into `.env`, then `docker compose up -d`.
|
|
503
505
|
|
|
504
506
|
| Option | What it does |
|
|
505
507
|
|---|---|
|
|
@@ -520,7 +522,7 @@ In testing: 20 of 20 HTTPS requests succeeded, over 15 different exit IPs. In th
|
|
|
520
522
|
|
|
521
523
|
**Live dashboard:** open `http://127.0.0.1:8899/__proxy-scraper/` in a browser – the pool, success rate, traffic, a live chart of requests, countries, the best proxies and the latest connections, refreshed every two seconds. It needs nothing from the internet, and `--serve-password` protects it like the rest.
|
|
522
524
|
|
|
523
|
-
<div align="center"><img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
525
|
+
<div align="center"><img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/dashboard-live.png" alt="The live dashboard of the rotating proxy server: usable proxies, success rate, requests per second, countries and the latest connections" width="860"></div>
|
|
524
526
|
|
|
525
527
|
<a id="pool-api"></a>
|
|
526
528
|
|
|
@@ -558,16 +560,16 @@ requests.get("https://example.com", proxies={"http": proxy, "https": proxy})
|
|
|
558
560
|
|
|
559
561
|
## Discord and Telegram bot
|
|
560
562
|
|
|
561
|
-
The live list can also come to you: [`bot/`](https://github.com/maximilianfeix/proxy-scraper/tree/v1.
|
|
563
|
+
The live list can also come to you: [`bot/`](https://github.com/maximilianfeix/proxy-scraper/tree/v1.27.0/bot/) is a Discord bot that posts every run into a server – a summary with the fastest proxies, the full lists per protocol as files, and slash commands like `/proxies type:socks5 country:DE https:true`. It sets up its own read-only channels when you invite it, and a GitHub Action deploys it to a server as a systemd service. The same process answers on Telegram too: `/proxy de socks5` for one proxy with a curl line, `/proxies 10 us https` for a short list. Setup in [bot/README.md](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/bot/README.md).
|
|
562
564
|
|
|
563
565
|
<a id="how-it-works"></a>
|
|
564
566
|
|
|
565
567
|
## How it works
|
|
566
568
|
|
|
567
569
|
<picture>
|
|
568
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
569
|
-
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
570
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
570
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/checks-dark.svg">
|
|
571
|
+
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/checks-light.svg">
|
|
572
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/checks-dark.svg" alt="The five checks: 700+ lists, a real handshake, two sites with one IP, nothing injected, the details" width="100%">
|
|
571
573
|
</picture>
|
|
572
574
|
|
|
573
575
|
|
|
@@ -582,7 +584,7 @@ flowchart LR
|
|
|
582
584
|
G -. next run .-> C
|
|
583
585
|
```
|
|
584
586
|
|
|
585
|
-
1. **Sources** – the curated list in [`sources.json`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
587
|
+
1. **Sources** – the curated list in [`sources.json`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/proxyscraper/sources.json), meta sources (other projects that maintain lists of proxy sources) and, once a day, a GitHub search for actively maintained repos. Every search adds to what earlier ones found; a list the search hasn't seen for three weeks drops out.
|
|
586
588
|
2. **Collect** – plain text, HTML tables, JSON APIs and `type://ip:port` lines are recognized; private and reserved address ranges are dropped. Lists that haven't changed since the last run answer `304` and come from a local cache – a second run right after the first loads 0 MB instead of ~160 MB.
|
|
587
589
|
3. **Prioritize** – known working proxies first, then by the learned hit rate of their sources.
|
|
588
590
|
4. **Check** – every proxy has to fetch its exit IP from a check target (`checkip.amazonaws.com`, with `ifconfig.me`, `ipinfo.io`, `wtfismyip.com` and `ident.me` as reserves – none of them behind Cloudflare) and return a valid, *foreign* IP. If the target goes down mid-run, the tool switches and re-checks the proxies that were affected, so the statistics don't learn from an outage. Anyone passing your own IP through is out. Then comes the **confirmation** via `httpbin.org`: fake proxies that only answer the first check with “200 + IP” fail here. Finally a static HTML page has to arrive byte for byte as it does without a proxy – anyone injecting ads or scripts is out.
|
|
@@ -594,9 +596,9 @@ flowchart LR
|
|
|
594
596
|
<summary><b>📸 See the live dashboard and final report</b></summary>
|
|
595
597
|
<br>
|
|
596
598
|
<div align="center">
|
|
597
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
599
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/dashboard.svg" alt="Live dashboard while checking" width="860">
|
|
598
600
|
<br><br>
|
|
599
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
601
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/summary.svg" alt="Final report after a run" width="860">
|
|
600
602
|
</div>
|
|
601
603
|
</details>
|
|
602
604
|
|
|
@@ -983,7 +985,7 @@ Shipped in [v1.5](https://github.com/maximilianfeix/proxy-scraper/milestone/5):
|
|
|
983
985
|
|
|
984
986
|
## Contributing
|
|
985
987
|
|
|
986
|
-
Bug reports, new sources and pull requests are very welcome – see [CONTRIBUTING.md](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
988
|
+
Bug reports, new sources and pull requests are very welcome – see [CONTRIBUTING.md](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/CONTRIBUTING.md), and [docs/ARCHITECTURE.md](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/docs/ARCHITECTURE.md) for how the pieces fit together. The short version:
|
|
987
989
|
|
|
988
990
|
```bash
|
|
989
991
|
pip install -e ".[dev]"
|
|
@@ -1057,7 +1059,7 @@ Want to join? [Beginner-friendly issues](https://github.com/maximilianfeix/proxy
|
|
|
1057
1059
|
|
|
1058
1060
|
## Acknowledgements
|
|
1059
1061
|
|
|
1060
|
-
proxy-scraper stands on the work of the people who publish free proxy lists. Thanks to everyone listed in [`sources.json`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
1062
|
+
proxy-scraper stands on the work of the people who publish free proxy lists. Thanks to everyone listed in [`sources.json`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/proxyscraper/sources.json), and especially to
|
|
1061
1063
|
|
|
1062
1064
|
- [monosans/proxy-scraper-checker](https://github.com/monosans/proxy-scraper-checker) and [gfpcom/free-proxy-list](https://github.com/gfpcom/free-proxy-list), whose curated source collections are read as meta sources
|
|
1063
1065
|
- [Textualize/rich](https://github.com/Textualize/rich), which draws the whole terminal UI
|
|
@@ -1072,7 +1074,7 @@ This tool only collects publicly listed proxies and checks whether they work. Yo
|
|
|
1072
1074
|
|
|
1073
1075
|
---
|
|
1074
1076
|
|
|
1075
|
-
<sub>Made in Germany by <a href="https://github.com/maximilianfeix">@maximilianfeix</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
1077
|
+
<sub>Made in Germany by <a href="https://github.com/maximilianfeix">@maximilianfeix</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/LICENSE">MIT License</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/CHANGELOG.md">Changelog</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/SECURITY.md">Security</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/CONTRIBUTING.md">Contributing</a></sub>
|
|
1076
1078
|
|
|
1077
1079
|
<sub>If proxy-scraper saves you time, a ⭐ helps others find it.</sub>
|
|
1078
1080
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: proxy-scraper-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.27.0
|
|
4
4
|
Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
|
|
5
5
|
Author: Maximilian Feix
|
|
6
6
|
License-Expression: MIT
|
|
@@ -45,9 +45,9 @@ Dynamic: license-file
|
|
|
45
45
|
<div align="center">
|
|
46
46
|
|
|
47
47
|
<picture>
|
|
48
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
49
|
-
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
50
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
48
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/banner-dark.svg">
|
|
49
|
+
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/banner-light.svg">
|
|
50
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/banner-dark.svg" alt="proxy-scraper – free proxies that actually work" width="100%">
|
|
51
51
|
</picture>
|
|
52
52
|
|
|
53
53
|
[](https://github.com/maximilianfeix/proxy-scraper/actions/workflows/tests.yml)
|
|
@@ -55,18 +55,18 @@ Dynamic: license-file
|
|
|
55
55
|
[](#live-list)
|
|
56
56
|
[](https://pypi.org/project/proxy-scraper-cli/)
|
|
57
57
|
[](https://pepy.tech/projects/proxy-scraper-cli)
|
|
58
|
-
[](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
58
|
+
[](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/pyproject.toml)
|
|
59
59
|
[](https://github.com/maximilianfeix/proxy-scraper/stargazers)
|
|
60
|
-
[](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
60
|
+
[](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/LICENSE)
|
|
61
61
|
|
|
62
62
|
<a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://img.shields.io/badge/Browse_the_live_list-D4F77A?style=for-the-badge&labelColor=121113" alt="Browse the live list"></a>
|
|
63
63
|
<a href="https://github.com/maximilianfeix/free-proxy-list"><img src="https://img.shields.io/badge/Just_the_lists-121113?style=for-the-badge" alt="Just the lists: free-proxy-list"></a>
|
|
64
64
|
<a href="#install"><img src="https://img.shields.io/badge/Install-121113?style=for-the-badge" alt="Install"></a>
|
|
65
65
|
<a href="#from-python"><img src="https://img.shields.io/badge/Python_API-121113?style=for-the-badge" alt="Python API"></a>
|
|
66
66
|
<a href="#mcp"><img src="https://img.shields.io/badge/MCP_server-121113?style=for-the-badge" alt="MCP server for AI agents"></a>
|
|
67
|
-
<a href="https://github.com/maximilianfeix/proxy-scraper/tree/v1.
|
|
67
|
+
<a href="https://github.com/maximilianfeix/proxy-scraper/tree/v1.27.0/bot/"><img src="https://img.shields.io/badge/Discord_bot-121113?style=for-the-badge" alt="Discord bot"></a>
|
|
68
68
|
|
|
69
|
-
**English** · [简体中文](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
69
|
+
**English** · [简体中文](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/README.zh-CN.md)
|
|
70
70
|
|
|
71
71
|
[Install](#install) · [Live list](#live-list) · [For AI agents](#mcp) · [Proxy server](#proxy-server) · [How it works](#how-it-works) · [Recipes](#recipes) · [Options](#options) · [FAQ](#faq)
|
|
72
72
|
|
|
@@ -76,7 +76,7 @@ Dynamic: license-file
|
|
|
76
76
|
|
|
77
77
|
Most free proxy lists are 95 % dead, and a good part of the rest are honeypots or proxies that inject scripts into your pages. **proxy-scraper** collects public HTTP, SOCKS4 and SOCKS5 proxies from 700+ sources and keeps only the ones that pass every check. It learns with every run which sources are worth it, and it can turn the result into a single rotating proxy.
|
|
78
78
|
|
|
79
|
-
**In numbers:** in one day it checked 3.6 million free proxies – 1.1 % worked, and more than half of the ones that answered failed a second request. [What's actually out there →](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
79
|
+
**In numbers:** in one day it checked 3.6 million free proxies – 1.1 % worked, and more than half of the ones that answered failed a second request. [What's actually out there →](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/docs/free-proxies-in-numbers.md)
|
|
80
80
|
|
|
81
81
|
**Used by:** the hourly list is a built-in source of [monosans/proxy-scraper-checker](https://github.com/monosans/proxy-scraper-checker) and [gfpcom/free-proxy-list](https://github.com/gfpcom/free-proxy-list).
|
|
82
82
|
|
|
@@ -92,7 +92,7 @@ curl -s https://maximilianfeix.github.io/proxy-scraper/socks5.txt | head
|
|
|
92
92
|
`uvx` comes with [uv](https://docs.astral.sh/uv/); `pipx run proxy-scraper-cli --pick 5` does the same.
|
|
93
93
|
|
|
94
94
|
<div align="center">
|
|
95
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
95
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/demo.svg" alt="Animated demo: the live dashboard during a check, the report, the setup wizard and collecting" width="880">
|
|
96
96
|
</div>
|
|
97
97
|
|
|
98
98
|
> **Who runs free proxies – and is it safe to use them?** Mostly nobody on purpose: misconfigured servers with an open port, a few deliberately open ones, infected machines, and traps that log or rewrite your traffic. That's why every proxy here has to pass [five checks](#how-it-works), and why they're for public data and testing – never for logins, payments or anything personal. [More on where they come from →](#who-runs-free-proxies)
|
|
@@ -181,7 +181,7 @@ Installed, the learned state lives in your user data folder (`~/Library/Applicat
|
|
|
181
181
|
Started without arguments, a wizard asks what you are looking for:
|
|
182
182
|
|
|
183
183
|
<div align="center">
|
|
184
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
184
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/wizard.svg" alt="Setup wizard" width="760">
|
|
185
185
|
</div>
|
|
186
186
|
|
|
187
187
|
| Preset | What it does |
|
|
@@ -216,7 +216,7 @@ Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and pub
|
|
|
216
216
|
|
|
217
217
|
Every protocol and country also has its own page with a plain download, e.g. [SOCKS5](https://maximilianfeix.github.io/proxy-scraper/socks5/) or [Germany](https://maximilianfeix.github.io/proxy-scraper/country/de/) (`country/de/proxies.txt`).
|
|
218
218
|
|
|
219
|
-
<div align="center"><a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
219
|
+
<div align="center"><a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/website.png" alt="The live list website" width="860"></a></div>
|
|
220
220
|
|
|
221
221
|
<div align="center">
|
|
222
222
|
|
|
@@ -262,6 +262,8 @@ Or let the tool start from it: `proxy-scraper --recheck live` downloads the list
|
|
|
262
262
|
|
|
263
263
|
`proxy-scraper-mcp` is an [MCP](https://modelcontextprotocol.io) server: Claude Code, Claude Desktop, Cursor, VS Code, Codex and any other MCP client can ask for working proxies and load pages through them.
|
|
264
264
|
|
|
265
|
+
[](https://glama.ai/mcp/servers/maximilianfeix/proxy-scraper)
|
|
266
|
+
|
|
265
267
|
| Tool | What it does |
|
|
266
268
|
|---|---|
|
|
267
269
|
| `get_proxies` | working proxies right now, from the hourly list – filter by protocol, country, HTTPS, elite, no datacenter, not blocklisted, stable, uptime, latency, gets through to Google/Reddit/Amazon |
|
|
@@ -358,7 +360,7 @@ Started without arguments, the tool asks what you need using the arrow keys –
|
|
|
358
360
|
<td valign="top">
|
|
359
361
|
|
|
360
362
|
**Real verification**<br>
|
|
361
|
-
Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests: over 3.6 million checks, 54 % of the proxies that answered the first request failed the second. A third request catches proxies that **tamper with content**: about one in 40 working proxies changed a known page, usually by injecting a script ([the numbers](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
363
|
+
Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests: over 3.6 million checks, 54 % of the proxies that answered the first request failed the second. A third request catches proxies that **tamper with content**: about one in 40 working proxies changed a known page, usually by injecting a script ([the numbers](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/docs/free-proxies-in-numbers.md)). Plus: HTTPS through a tunnel with **verified TLS**, anonymity level *elite / anonymous / transparent* and the country of the exit IP.
|
|
362
364
|
|
|
363
365
|
</td>
|
|
364
366
|
<td valign="top">
|
|
@@ -543,7 +545,7 @@ proxy-scraper --recheck --serve --serve-host 0.0.0.0
|
|
|
543
545
|
curl -x "http://country-de:$PROXY_SCRAPER_SERVE_PASSWORD@your-server:8899" https://api.ipify.org
|
|
544
546
|
```
|
|
545
547
|
|
|
546
|
-
Without a password, `--serve-host` means **anyone who reaches the port can use it**. The ready-made [`compose.yaml`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
548
|
+
Without a password, `--serve-host` means **anyone who reaches the port can use it**. The ready-made [`compose.yaml`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/compose.yaml) starts the server in Docker from the live list, with a password, a health check and learned state in a volume: put `PROXY_PASSWORD=…` into `.env`, then `docker compose up -d`.
|
|
547
549
|
|
|
548
550
|
| Option | What it does |
|
|
549
551
|
|---|---|
|
|
@@ -564,7 +566,7 @@ In testing: 20 of 20 HTTPS requests succeeded, over 15 different exit IPs. In th
|
|
|
564
566
|
|
|
565
567
|
**Live dashboard:** open `http://127.0.0.1:8899/__proxy-scraper/` in a browser – the pool, success rate, traffic, a live chart of requests, countries, the best proxies and the latest connections, refreshed every two seconds. It needs nothing from the internet, and `--serve-password` protects it like the rest.
|
|
566
568
|
|
|
567
|
-
<div align="center"><img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
569
|
+
<div align="center"><img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/dashboard-live.png" alt="The live dashboard of the rotating proxy server: usable proxies, success rate, requests per second, countries and the latest connections" width="860"></div>
|
|
568
570
|
|
|
569
571
|
<a id="pool-api"></a>
|
|
570
572
|
|
|
@@ -602,16 +604,16 @@ requests.get("https://example.com", proxies={"http": proxy, "https": proxy})
|
|
|
602
604
|
|
|
603
605
|
## Discord and Telegram bot
|
|
604
606
|
|
|
605
|
-
The live list can also come to you: [`bot/`](https://github.com/maximilianfeix/proxy-scraper/tree/v1.
|
|
607
|
+
The live list can also come to you: [`bot/`](https://github.com/maximilianfeix/proxy-scraper/tree/v1.27.0/bot/) is a Discord bot that posts every run into a server – a summary with the fastest proxies, the full lists per protocol as files, and slash commands like `/proxies type:socks5 country:DE https:true`. It sets up its own read-only channels when you invite it, and a GitHub Action deploys it to a server as a systemd service. The same process answers on Telegram too: `/proxy de socks5` for one proxy with a curl line, `/proxies 10 us https` for a short list. Setup in [bot/README.md](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/bot/README.md).
|
|
606
608
|
|
|
607
609
|
<a id="how-it-works"></a>
|
|
608
610
|
|
|
609
611
|
## How it works
|
|
610
612
|
|
|
611
613
|
<picture>
|
|
612
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
613
|
-
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
614
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
614
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/checks-dark.svg">
|
|
615
|
+
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/checks-light.svg">
|
|
616
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/checks-dark.svg" alt="The five checks: 700+ lists, a real handshake, two sites with one IP, nothing injected, the details" width="100%">
|
|
615
617
|
</picture>
|
|
616
618
|
|
|
617
619
|
|
|
@@ -626,7 +628,7 @@ flowchart LR
|
|
|
626
628
|
G -. next run .-> C
|
|
627
629
|
```
|
|
628
630
|
|
|
629
|
-
1. **Sources** – the curated list in [`sources.json`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
631
|
+
1. **Sources** – the curated list in [`sources.json`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/proxyscraper/sources.json), meta sources (other projects that maintain lists of proxy sources) and, once a day, a GitHub search for actively maintained repos. Every search adds to what earlier ones found; a list the search hasn't seen for three weeks drops out.
|
|
630
632
|
2. **Collect** – plain text, HTML tables, JSON APIs and `type://ip:port` lines are recognized; private and reserved address ranges are dropped. Lists that haven't changed since the last run answer `304` and come from a local cache – a second run right after the first loads 0 MB instead of ~160 MB.
|
|
631
633
|
3. **Prioritize** – known working proxies first, then by the learned hit rate of their sources.
|
|
632
634
|
4. **Check** – every proxy has to fetch its exit IP from a check target (`checkip.amazonaws.com`, with `ifconfig.me`, `ipinfo.io`, `wtfismyip.com` and `ident.me` as reserves – none of them behind Cloudflare) and return a valid, *foreign* IP. If the target goes down mid-run, the tool switches and re-checks the proxies that were affected, so the statistics don't learn from an outage. Anyone passing your own IP through is out. Then comes the **confirmation** via `httpbin.org`: fake proxies that only answer the first check with “200 + IP” fail here. Finally a static HTML page has to arrive byte for byte as it does without a proxy – anyone injecting ads or scripts is out.
|
|
@@ -638,9 +640,9 @@ flowchart LR
|
|
|
638
640
|
<summary><b>📸 See the live dashboard and final report</b></summary>
|
|
639
641
|
<br>
|
|
640
642
|
<div align="center">
|
|
641
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
643
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/dashboard.svg" alt="Live dashboard while checking" width="860">
|
|
642
644
|
<br><br>
|
|
643
|
-
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.
|
|
645
|
+
<img src="https://raw.githubusercontent.com/maximilianfeix/proxy-scraper/v1.27.0/docs/summary.svg" alt="Final report after a run" width="860">
|
|
644
646
|
</div>
|
|
645
647
|
</details>
|
|
646
648
|
|
|
@@ -1027,7 +1029,7 @@ Shipped in [v1.5](https://github.com/maximilianfeix/proxy-scraper/milestone/5):
|
|
|
1027
1029
|
|
|
1028
1030
|
## Contributing
|
|
1029
1031
|
|
|
1030
|
-
Bug reports, new sources and pull requests are very welcome – see [CONTRIBUTING.md](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
1032
|
+
Bug reports, new sources and pull requests are very welcome – see [CONTRIBUTING.md](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/CONTRIBUTING.md), and [docs/ARCHITECTURE.md](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/docs/ARCHITECTURE.md) for how the pieces fit together. The short version:
|
|
1031
1033
|
|
|
1032
1034
|
```bash
|
|
1033
1035
|
pip install -e ".[dev]"
|
|
@@ -1101,7 +1103,7 @@ Want to join? [Beginner-friendly issues](https://github.com/maximilianfeix/proxy
|
|
|
1101
1103
|
|
|
1102
1104
|
## Acknowledgements
|
|
1103
1105
|
|
|
1104
|
-
proxy-scraper stands on the work of the people who publish free proxy lists. Thanks to everyone listed in [`sources.json`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
1106
|
+
proxy-scraper stands on the work of the people who publish free proxy lists. Thanks to everyone listed in [`sources.json`](https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/proxyscraper/sources.json), and especially to
|
|
1105
1107
|
|
|
1106
1108
|
- [monosans/proxy-scraper-checker](https://github.com/monosans/proxy-scraper-checker) and [gfpcom/free-proxy-list](https://github.com/gfpcom/free-proxy-list), whose curated source collections are read as meta sources
|
|
1107
1109
|
- [Textualize/rich](https://github.com/Textualize/rich), which draws the whole terminal UI
|
|
@@ -1116,7 +1118,7 @@ This tool only collects publicly listed proxies and checks whether they work. Yo
|
|
|
1116
1118
|
|
|
1117
1119
|
---
|
|
1118
1120
|
|
|
1119
|
-
<sub>Made in Germany by <a href="https://github.com/maximilianfeix">@maximilianfeix</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.
|
|
1121
|
+
<sub>Made in Germany by <a href="https://github.com/maximilianfeix">@maximilianfeix</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/LICENSE">MIT License</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/CHANGELOG.md">Changelog</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/SECURITY.md">Security</a> · <a href="https://github.com/maximilianfeix/proxy-scraper/blob/v1.27.0/CONTRIBUTING.md">Contributing</a></sub>
|
|
1120
1122
|
|
|
1121
1123
|
<sub>If proxy-scraper saves you time, a ⭐ helps others find it.</sub>
|
|
1122
1124
|
|