proxy-scraper-cli 1.22.0__tar.gz → 1.23.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/PKG-INFO +42 -11
  2. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/README.md +39 -9
  3. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxy_scraper_cli.egg-info/PKG-INFO +42 -11
  4. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxy_scraper_cli.egg-info/SOURCES.txt +6 -0
  5. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxy_scraper_cli.egg-info/requires.txt +1 -1
  6. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/__init__.py +1 -1
  7. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/agent.py +2 -2
  8. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/api.py +8 -6
  9. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/exporters.py +48 -0
  10. proxy_scraper_cli-1.23.0/proxyscraper/mirror.py +348 -0
  11. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/pages.py +17 -12
  12. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/publish.py +11 -3
  13. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ranking.py +26 -2
  14. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/core.py +17 -7
  15. proxy_scraper_cli-1.23.0/proxyscraper/server/dashboard.html +351 -0
  16. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/status.py +59 -4
  17. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/site/index.html +3 -3
  18. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/sources.json +1 -0
  19. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/serve.py +5 -4
  20. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/widgets.py +6 -0
  21. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/pyproject.toml +3 -2
  22. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_agent.py +5 -4
  23. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_live_api.py +6 -5
  24. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_mcp_server.py +2 -1
  25. proxy_scraper_cli-1.23.0/tests/test_mirror.py +75 -0
  26. proxy_scraper_cli-1.23.0/tests/test_pac.py +116 -0
  27. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_pick.py +2 -2
  28. proxy_scraper_cli-1.23.0/tests/test_serve_dashboard.py +234 -0
  29. proxy_scraper_cli-1.23.0/tests/test_survival_ranking.py +58 -0
  30. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/LICENSE +0 -0
  31. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxy_scraper_cli.egg-info/dependency_links.txt +0 -0
  32. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxy_scraper_cli.egg-info/entry_points.txt +0 -0
  33. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxy_scraper_cli.egg-info/top_level.txt +0 -0
  34. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/__main__.py +0 -0
  35. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/action.py +0 -0
  36. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/app.py +0 -0
  37. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/asndb.py +0 -0
  38. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/blocklist.py +0 -0
  39. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/charts.py +0 -0
  40. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/checker.py +0 -0
  41. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/cli.py +0 -0
  42. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/compat.py +0 -0
  43. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/completion.py +0 -0
  44. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/fetchcache.py +0 -0
  45. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/geo.py +0 -0
  46. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/geodb.py +0 -0
  47. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/handshake.py +0 -0
  48. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/history.py +0 -0
  49. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/judges.py +0 -0
  50. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/mcp_entry.py +0 -0
  51. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/mcp_server.py +0 -0
  52. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/netio.py +0 -0
  53. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/options.py +0 -0
  54. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/output.py +0 -0
  55. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/parsing.py +0 -0
  56. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/paths.py +0 -0
  57. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/pipeline.py +0 -0
  58. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/preferences.py +0 -0
  59. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/proxypages.py +0 -0
  60. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/report.py +0 -0
  61. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/runsteps.py +0 -0
  62. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/__init__.py +0 -0
  63. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/api.py +0 -0
  64. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/http.py +0 -0
  65. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/pool.py +0 -0
  66. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/socks.py +0 -0
  67. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/server/upstream.py +0 -0
  68. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/site/apple-touch-icon.png +0 -0
  69. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/site/googleaac1161b7853c5b5.html +0 -0
  70. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/site/logo.png +0 -0
  71. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/site/og.png +0 -0
  72. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/sites.py +0 -0
  73. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/sources.py +0 -0
  74. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/speed.py +0 -0
  75. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/targets.py +0 -0
  76. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/__init__.py +0 -0
  77. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/dashboard.py +0 -0
  78. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/keys.py +0 -0
  79. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/report.py +0 -0
  80. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/proxyscraper/ui/wizard.py +0 -0
  81. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/setup.cfg +0 -0
  82. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_action.py +0 -0
  83. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_api.py +0 -0
  84. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_app.py +0 -0
  85. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_asndb.py +0 -0
  86. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_auth.py +0 -0
  87. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_blocklist.py +0 -0
  88. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_charts.py +0 -0
  89. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_checker.py +0 -0
  90. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_compat.py +0 -0
  91. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_completion.py +0 -0
  92. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_discovery_growth.py +0 -0
  93. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_exporters.py +0 -0
  94. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_fetchcache.py +0 -0
  95. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_geodb.py +0 -0
  96. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_judges.py +0 -0
  97. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_learning.py +0 -0
  98. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_mine_sources.py +0 -0
  99. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_options.py +0 -0
  100. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_output_stdout.py +0 -0
  101. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_own_sources.py +0 -0
  102. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_packaging.py +0 -0
  103. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_pages.py +0 -0
  104. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_parsing.py +0 -0
  105. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_parsing_fuzz.py +0 -0
  106. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_paths.py +0 -0
  107. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_pipeline.py +0 -0
  108. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_preferences.py +0 -0
  109. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_proxy_pages.py +0 -0
  110. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_proxy_sources.py +0 -0
  111. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_publish.py +0 -0
  112. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_report.py +0 -0
  113. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_robustness.py +0 -0
  114. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_rotator.py +0 -0
  115. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_serve_api.py +0 -0
  116. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_serve_password.py +0 -0
  117. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_serve_refill.py +0 -0
  118. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_serve_v2.py +0 -0
  119. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_server.py +0 -0
  120. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_server_closed_writer.py +0 -0
  121. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_server_fixes.py +0 -0
  122. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_sites.py +0 -0
  123. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_sources_json.py +0 -0
  124. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_speed.py +0 -0
  125. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_speed_ranking.py +0 -0
  126. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_tamper.py +0 -0
  127. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_targets.py +0 -0
  128. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_ui.py +0 -0
  129. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_uptime.py +0 -0
  130. {proxy_scraper_cli-1.22.0 → proxy_scraper_cli-1.23.0}/tests/test_wizard.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: proxy-scraper-cli
3
- Version: 1.22.0
3
+ Version: 1.23.0
4
4
  Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
5
5
  Author: Maximilian Feix
6
6
  License-Expression: MIT
@@ -22,6 +22,7 @@ Classifier: Programming Language :: Python :: 3.10
22
22
  Classifier: Programming Language :: Python :: 3.11
23
23
  Classifier: Programming Language :: Python :: 3.12
24
24
  Classifier: Programming Language :: Python :: 3.13
25
+ Classifier: Programming Language :: Python :: 3.14
25
26
  Classifier: Topic :: Internet :: Proxy Servers
26
27
  Classifier: Topic :: System :: Networking
27
28
  Requires-Python: >=3.9
@@ -36,7 +37,7 @@ Requires-Dist: mcp<3,>=2.2; python_version >= "3.10" and extra == "mcp"
36
37
  Provides-Extra: dev
37
38
  Requires-Dist: pytest; extra == "dev"
38
39
  Requires-Dist: hypothesis; extra == "dev"
39
- Requires-Dist: ruff==0.16.8; extra == "dev"
40
+ Requires-Dist: ruff==0.16.9; extra == "dev"
40
41
  Dynamic: license-file
41
42
 
42
43
  <div align="center">
@@ -51,16 +52,19 @@ Dynamic: license-file
51
52
  [![Release](https://img.shields.io/github/v/release/maximilianfeix/proxy-scraper?style=flat-square&color=D4F77A&labelColor=121113)](https://github.com/maximilianfeix/proxy-scraper/releases/latest)
52
53
  [![Live proxies](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fmaximilianfeix%2Fproxy-scraper%2Fproxy-list%2Fbadges%2Ftotal.json&style=flat-square&labelColor=121113)](#live-list)
53
54
  [![PyPI](https://img.shields.io/pypi/v/proxy-scraper-cli?style=flat-square&color=D4F77A&labelColor=121113&label=pypi)](https://pypi.org/project/proxy-scraper-cli/)
54
- [![Python](https://img.shields.io/badge/python-3.9–3.13-D4F77A?style=flat-square&labelColor=121113)](pyproject.toml)
55
+ [![Python](https://img.shields.io/badge/python-3.9–3.14-D4F77A?style=flat-square&labelColor=121113)](pyproject.toml)
55
56
  [![Stars](https://img.shields.io/github/stars/maximilianfeix/proxy-scraper?style=flat-square&color=D4F77A&labelColor=121113)](https://github.com/maximilianfeix/proxy-scraper/stargazers)
56
57
  [![License](https://img.shields.io/badge/license-MIT-D4F77A?style=flat-square&labelColor=121113)](LICENSE)
57
58
 
58
59
  <a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://img.shields.io/badge/Browse_the_live_list-D4F77A?style=for-the-badge&labelColor=121113" alt="Browse the live list"></a>
60
+ <a href="https://github.com/maximilianfeix/free-proxy-list"><img src="https://img.shields.io/badge/Just_the_lists-121113?style=for-the-badge" alt="Just the lists: free-proxy-list"></a>
59
61
  <a href="#install"><img src="https://img.shields.io/badge/Install-121113?style=for-the-badge" alt="Install"></a>
60
62
  <a href="#from-python"><img src="https://img.shields.io/badge/Python_API-121113?style=for-the-badge" alt="Python API"></a>
61
63
  <a href="#mcp"><img src="https://img.shields.io/badge/MCP_server-121113?style=for-the-badge" alt="MCP server for AI agents"></a>
62
64
  <a href="bot/"><img src="https://img.shields.io/badge/Discord_bot-121113?style=for-the-badge" alt="Discord bot"></a>
63
65
 
66
+ **English** · [简体中文](README.zh-CN.md)
67
+
64
68
  [Install](#install) · [Live list](#live-list) · [For AI agents](#mcp) · [Proxy server](#proxy-server) · [How it works](#how-it-works) · [Recipes](#recipes) · [Options](#options) · [FAQ](#faq)
65
69
 
66
70
  </div>
@@ -114,6 +118,14 @@ proxy-scraper
114
118
 
115
119
  The package is called `proxy-scraper-cli` on PyPI (the plain name is taken), the command is `proxy-scraper`.
116
120
 
121
+ **With [Homebrew](https://brew.sh)** on macOS and Linux:
122
+
123
+ ```bash
124
+ brew install maximilianfeix/tap/proxy-scraper
125
+ ```
126
+
127
+ The formula follows each release within about a day.
128
+
117
129
  <details>
118
130
  <summary><b>Other ways: Docker, pip, a faster event loop, or straight from the repo</b></summary>
119
131
  <br>
@@ -178,7 +190,9 @@ Keys: <kbd>↑</kbd><kbd>↓</kbd> select · <kbd>Space</kbd> toggle · <kbd>1</
178
190
 
179
191
  ## Live proxy list
180
192
 
181
- Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and publishes the hits to the [`proxy-list`](../../tree/proxy-list) branch – every entry worked in the last run, fastest first.
193
+ Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and publishes the hits to the [`proxy-list`](../../tree/proxy-list) branch – every entry worked in the last run, best first: fast ones that are likely to still be up come on top.
194
+
195
+ > **Just want the lists?** They also live in a repository of their own, **[maximilianfeix/free-proxy-list](https://github.com/maximilianfeix/free-proxy-list)** – one list per protocol, per country and per site, updated every hour, with the numbers of the last run on the front page. Watch or star it to keep it at hand.
182
196
 
183
197
  <a href="https://maximilianfeix.github.io/proxy-scraper/"><picture>
184
198
  <source media="(prefers-color-scheme: dark)" srcset="https://maximilianfeix.github.io/proxy-scraper/chart-dark.svg">
@@ -319,7 +333,7 @@ Started without arguments, the tool asks what you need using the arrow keys –
319
333
  <td valign="top">
320
334
 
321
335
  **Real verification**<br>
322
- Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests (in some runs 5 out of 6 “hits”). A third request catches proxies that **tamper with content**: in our measurements one in five working proxies injected a script into a plain HTML page. Plus: HTTPS through a tunnel with **verified TLS**, anonymity level *elite / anonymous / transparent* and the country of the exit IP.
336
+ Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests: over 3.6 million checks, 54 % of the proxies that answered the first request failed the second. A third request catches proxies that **tamper with content**: about one in 40 working proxies changed a known page, usually by injecting a script ([the numbers](docs/free-proxies-in-numbers.md)). Plus: HTTPS through a tunnel with **verified TLS**, anonymity level *elite / anonymous / transparent* and the country of the exit IP.
323
337
 
324
338
  </td>
325
339
  <td valign="top">
@@ -361,13 +375,13 @@ Speed chart, latency histogram, protocols, countries and the latest hits in real
361
375
  <td valign="top">
362
376
 
363
377
  **Runs everywhere**<br>
364
- macOS, Linux and Windows, Python 3.9 to 3.13. Only two dependencies: `rich` and `certifi`. It even notices when a firewall blocks proxies.
378
+ macOS, Linux and Windows, Python 3.9 to 3.14. Only two dependencies: `rich` and `certifi`. It even notices when a firewall blocks proxies.
365
379
 
366
380
  </td>
367
381
  <td valign="top">
368
382
 
369
383
  **Thoroughly tested**<br>
370
- 450+ tests run offline against real mini proxies and honeypots on `localhost` – on Linux, macOS and Windows with Python 3.9, 3.11 and 3.13.
384
+ 760+ tests run offline against real mini proxies and honeypots on `localhost` – on Linux, macOS and Windows with Python 3.9, 3.11, 3.13 and 3.14.
371
385
 
372
386
  </td>
373
387
  </tr>
@@ -515,7 +529,7 @@ Without a password, `--serve-host` means **anyone who reaches the port can use i
515
529
 
516
530
  - every connection goes through a different proxy (unless sticky); fast and proven ones are preferred
517
531
  - `CONNECT` for HTTPS and plain HTTP requests; HTTP, SOCKS4 and SOCKS5 proxies can sit behind it (SOCKS5 with DNS through the proxy)
518
- - HTTPS only uses proxies that passed the test with **verified TLS** – no broken encryption
532
+ - HTTPS prefers proxies that passed the test with **verified TLS**; when no such proxy fits right now – none matches a wish like `country-de`, or they have all dropped out – it uses the others – your client's own certificate check still catches a proxy that breaks the encryption, so leave it on
519
533
  - if a proxy stays silent inside the tunnel or returns an error page instead of TLS, the same first packet quietly goes to the next one
520
534
  - three failures in a row and a proxy leaves the rotation – every 5 minutes those get re-checked and come back if they work again
521
535
  - `--serve-refill 6` checks fresh proxies every 6 hours in the background (the live list with `--recheck live`, otherwise the last run + history) with the same checks and filters, and adds the hits – a server that runs for days doesn't run dry
@@ -523,6 +537,10 @@ Without a password, `--serve-host` means **anyone who reaches the port can use i
523
537
 
524
538
  In testing: 20 of 20 HTTPS requests succeeded, over 15 different exit IPs. In the wizard this is **Proxy server right away**.
525
539
 
540
+ **Live dashboard:** open `http://127.0.0.1:8899/__proxy-scraper/` in a browser – the pool, success rate, traffic, a live chart of requests, countries, the best proxies and the latest connections, refreshed every two seconds. It needs nothing from the internet, and `--serve-password` protects it like the rest.
541
+
542
+ <div align="center"><img src="docs/dashboard-live.png" alt="The live dashboard of the rotating proxy server: usable proxies, success rate, requests per second, countries and the latest connections" width="860"></div>
543
+
526
544
  <a id="pool-api"></a>
527
545
 
528
546
  ### Proxy pool API
@@ -741,6 +759,18 @@ proxychains4 -f results/latest/proxychains.conf curl https://api.ipify.org
741
759
 
742
760
  `clash.yaml` has all HTTP and SOCKS5 proxies plus a `url-test` group that always picks the fastest. `singbox.json` does the same for sing-box, SOCKS4 included, and opens a local proxy: `sing-box run -c results/latest/singbox.json`, then use `127.0.0.1:2080` as HTTP or SOCKS5 proxy. All three leave out HTTP proxies that can't tunnel (`CONNECT`), because these tools tunnel everything.
743
761
 
762
+ **Browsers and the OS** – `--export pac` writes `proxy.pac`: point Firefox, Chrome (via the system settings), FoxyProxy or macOS/Windows proxy settings at it and every request goes through the best 30 HTTP and SOCKS5 proxies in order, the browser moving on by itself when one fails. Local names and private IP addresses stay direct (a host name like `10.example.com` doesn't count as private). The file itself has no `DIRECT` fallback and resolves no host names, and SOCKS4 is left out because it would make the browser resolve names with your own DNS. Two Firefox settings matter: turn on *Proxy DNS when using SOCKS v5*, and set `network.proxy.failover_direct` to `false` in `about:config` – otherwise Firefox goes direct once every proxy has failed. No install at all: the live list publishes one every hour at `https://maximilianfeix.github.io/proxy-scraper/proxy.pac`.
763
+
764
+ **Burp Suite, OWASP ZAP, mitmproxy** – put the rotating server behind your intercepting proxy, so a scan or a brute-force test comes from many IPs while you still see every request. Start it with `proxy-scraper --recheck live --serve`, then:
765
+
766
+ | Tool | Where | Set |
767
+ |---|---|---|
768
+ | Burp Suite | *Settings → Network → Connections → Upstream proxy servers* | a rule for destination host `*`, proxy host `127.0.0.1`, port `8899`; with `--serve-password` add Basic auth (user `any` or a wish like `country-de`, the password) |
769
+ | OWASP ZAP | *Options → Network → Connection → HTTP Proxy* | host `127.0.0.1`, port `8899`, and under its authentication the user (`any` or a wish) and the password – use the HTTP setting, not SOCKS: ZAP never sends loopback addresses through its SOCKS proxy |
770
+ | mitmproxy | command line | `mitmproxy --mode upstream:http://127.0.0.1:8899 --upstream-auth country-us:x` |
771
+
772
+ Username wishes – `country-us` sends everything out through US proxies, `session-NAME` keeps one exit IP for a login flow – reach the server as the proxy login. mitmproxy sends it right away; Burp and ZAP may wait to be asked, so start the server with `--serve-password` when you rely on a wish there. **Keep your tool's upstream certificate checks on** (no `--ssl-insecure` in mitmproxy, no "ignore certificate errors" upstream in Burp or ZAP): free proxies are strangers, and that check is what stops one from reading or changing your HTTPS traffic. Tested end to end with mitmproxy 12, certificate checks on: HTTP and HTTPS through mitmproxy and the rotating server, every request out through a US proxy, and a site with an expired certificate refused. Only test what you're allowed to.
773
+
744
774
  **In a pipe** – `-o -` prints the hits to stdout, the interface moves to stderr
745
775
 
746
776
  ```bash
@@ -834,7 +864,7 @@ curl.exe -x (Get-Content "$run\all.txt" -TotalCount 1) http://api.ipify.org
834
864
  | `--serve-refill HOURS` | while serving, check fresh proxies every HOURS and add the hits to the pool |
835
865
  | `--serve [PORT]` | afterwards serve as a rotating proxy on `127.0.0.1:PORT` (default: 8899) |
836
866
  | `-o FILE` | also write all hits to this file; `-o -` prints them to stdout |
837
- | `--export FORMATS` | extra files for other tools: `proxychains`, `clash`, `singbox`, `curl` or `all` |
867
+ | `--export FORMATS` | extra files for other tools: `proxychains`, `clash`, `singbox`, `curl`, `pac` or `all` |
838
868
  | `-V`, `--version` | print the version |
839
869
  | `--completion SHELL` | print the tab completion script for bash, zsh, fish or PowerShell |
840
870
 
@@ -920,7 +950,7 @@ Only for things that don't matter. Public proxies are run by strangers who can r
920
950
 
921
951
  What's next is in the [open issues](../../issues) – ideas and wishes are welcome as an [issue](../../issues/new/choose). During Hacktoberfest there are [beginner-friendly issues](../../issues?q=is%3Aopen+label%3Ahacktoberfest) with pointers on where to start.
922
952
 
923
- - [ ] [Mirror the daily snapshots to a Hugging Face dataset](../../issues/188)
953
+ Shipped in v1.22 and v1.23: a live dashboard for the rotating server, `--export pac` and an hourly `proxy.pac`, "best first" that counts how likely a proxy is still up, the lists in a repository of their own ([free-proxy-list](https://github.com/maximilianfeix/free-proxy-list)), daily snapshots on Hugging Face, `brew install`, a Chinese README, Python 3.14, and a recipe for Burp Suite, ZAP and mitmproxy.
924
954
 
925
955
  Shipped in v1.10 to v1.21: download speed per proxy and "best first" everywhere, daily snapshots, a rotating proxy URL for requests, httpx and Playwright, `--pick` for working proxies without a scan, a proxy pool API compatible with jhao104/proxy_pool, your own lists with `--source`, a GitHub Action, a Telegram bot and an Atom feed for the weekly report.
926
956
 
@@ -969,12 +999,13 @@ proxyscraper/
969
999
  ├── geodb.py DB-IP country database (monthly, binary search)
970
1000
  ├── targets.py target sites for --target
971
1001
  ├── output.py result files
972
- ├── exporters.py proxychains, Clash, sing-box and curl formats (--export)
1002
+ ├── exporters.py proxychains, Clash, sing-box, curl and PAC formats (--export)
973
1003
  ├── server/ rotating proxy server (--serve): pool · http · upstream · socks · status · api · core
974
1004
  ├── api.py find_proxies() / check_proxies() for Python
975
1005
  ├── agent.py MCP tools without the SDK: live list, filters, fetch through proxies
976
1006
  ├── mcp_server.py MCP server (proxy-scraper-mcp) for AI agents
977
1007
  ├── publish.py live list for GitHub Actions
1008
+ ├── mirror.py the same lists in their own repo (free-proxy-list)
978
1009
  ├── paths.py where state and results are stored
979
1010
  ├── compat.py differences between Unix and Windows
980
1011
  ├── netio.py small HTTP client on asyncio
@@ -10,16 +10,19 @@
10
10
  [![Release](https://img.shields.io/github/v/release/maximilianfeix/proxy-scraper?style=flat-square&color=D4F77A&labelColor=121113)](https://github.com/maximilianfeix/proxy-scraper/releases/latest)
11
11
  [![Live proxies](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fmaximilianfeix%2Fproxy-scraper%2Fproxy-list%2Fbadges%2Ftotal.json&style=flat-square&labelColor=121113)](#live-list)
12
12
  [![PyPI](https://img.shields.io/pypi/v/proxy-scraper-cli?style=flat-square&color=D4F77A&labelColor=121113&label=pypi)](https://pypi.org/project/proxy-scraper-cli/)
13
- [![Python](https://img.shields.io/badge/python-3.9–3.13-D4F77A?style=flat-square&labelColor=121113)](pyproject.toml)
13
+ [![Python](https://img.shields.io/badge/python-3.9–3.14-D4F77A?style=flat-square&labelColor=121113)](pyproject.toml)
14
14
  [![Stars](https://img.shields.io/github/stars/maximilianfeix/proxy-scraper?style=flat-square&color=D4F77A&labelColor=121113)](https://github.com/maximilianfeix/proxy-scraper/stargazers)
15
15
  [![License](https://img.shields.io/badge/license-MIT-D4F77A?style=flat-square&labelColor=121113)](LICENSE)
16
16
 
17
17
  <a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://img.shields.io/badge/Browse_the_live_list-D4F77A?style=for-the-badge&labelColor=121113" alt="Browse the live list"></a>
18
+ <a href="https://github.com/maximilianfeix/free-proxy-list"><img src="https://img.shields.io/badge/Just_the_lists-121113?style=for-the-badge" alt="Just the lists: free-proxy-list"></a>
18
19
  <a href="#install"><img src="https://img.shields.io/badge/Install-121113?style=for-the-badge" alt="Install"></a>
19
20
  <a href="#from-python"><img src="https://img.shields.io/badge/Python_API-121113?style=for-the-badge" alt="Python API"></a>
20
21
  <a href="#mcp"><img src="https://img.shields.io/badge/MCP_server-121113?style=for-the-badge" alt="MCP server for AI agents"></a>
21
22
  <a href="bot/"><img src="https://img.shields.io/badge/Discord_bot-121113?style=for-the-badge" alt="Discord bot"></a>
22
23
 
24
+ **English** · [简体中文](README.zh-CN.md)
25
+
23
26
  [Install](#install) · [Live list](#live-list) · [For AI agents](#mcp) · [Proxy server](#proxy-server) · [How it works](#how-it-works) · [Recipes](#recipes) · [Options](#options) · [FAQ](#faq)
24
27
 
25
28
  </div>
@@ -73,6 +76,14 @@ proxy-scraper
73
76
 
74
77
  The package is called `proxy-scraper-cli` on PyPI (the plain name is taken), the command is `proxy-scraper`.
75
78
 
79
+ **With [Homebrew](https://brew.sh)** on macOS and Linux:
80
+
81
+ ```bash
82
+ brew install maximilianfeix/tap/proxy-scraper
83
+ ```
84
+
85
+ The formula follows each release within about a day.
86
+
76
87
  <details>
77
88
  <summary><b>Other ways: Docker, pip, a faster event loop, or straight from the repo</b></summary>
78
89
  <br>
@@ -137,7 +148,9 @@ Keys: <kbd>↑</kbd><kbd>↓</kbd> select · <kbd>Space</kbd> toggle · <kbd>1</
137
148
 
138
149
  ## Live proxy list
139
150
 
140
- Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and publishes the hits to the [`proxy-list`](../../tree/proxy-list) branch – every entry worked in the last run, fastest first.
151
+ Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and publishes the hits to the [`proxy-list`](../../tree/proxy-list) branch – every entry worked in the last run, best first: fast ones that are likely to still be up come on top.
152
+
153
+ > **Just want the lists?** They also live in a repository of their own, **[maximilianfeix/free-proxy-list](https://github.com/maximilianfeix/free-proxy-list)** – one list per protocol, per country and per site, updated every hour, with the numbers of the last run on the front page. Watch or star it to keep it at hand.
141
154
 
142
155
  <a href="https://maximilianfeix.github.io/proxy-scraper/"><picture>
143
156
  <source media="(prefers-color-scheme: dark)" srcset="https://maximilianfeix.github.io/proxy-scraper/chart-dark.svg">
@@ -278,7 +291,7 @@ Started without arguments, the tool asks what you need using the arrow keys –
278
291
  <td valign="top">
279
292
 
280
293
  **Real verification**<br>
281
- Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests (in some runs 5 out of 6 “hits”). A third request catches proxies that **tamper with content**: in our measurements one in five working proxies injected a script into a plain HTML page. Plus: HTTPS through a tunnel with **verified TLS**, anonymity level *elite / anonymous / transparent* and the country of the exit IP.
294
+ Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests: over 3.6 million checks, 54 % of the proxies that answered the first request failed the second. A third request catches proxies that **tamper with content**: about one in 40 working proxies changed a known page, usually by injecting a script ([the numbers](docs/free-proxies-in-numbers.md)). Plus: HTTPS through a tunnel with **verified TLS**, anonymity level *elite / anonymous / transparent* and the country of the exit IP.
282
295
 
283
296
  </td>
284
297
  <td valign="top">
@@ -320,13 +333,13 @@ Speed chart, latency histogram, protocols, countries and the latest hits in real
320
333
  <td valign="top">
321
334
 
322
335
  **Runs everywhere**<br>
323
- macOS, Linux and Windows, Python 3.9 to 3.13. Only two dependencies: `rich` and `certifi`. It even notices when a firewall blocks proxies.
336
+ macOS, Linux and Windows, Python 3.9 to 3.14. Only two dependencies: `rich` and `certifi`. It even notices when a firewall blocks proxies.
324
337
 
325
338
  </td>
326
339
  <td valign="top">
327
340
 
328
341
  **Thoroughly tested**<br>
329
- 450+ tests run offline against real mini proxies and honeypots on `localhost` – on Linux, macOS and Windows with Python 3.9, 3.11 and 3.13.
342
+ 760+ tests run offline against real mini proxies and honeypots on `localhost` – on Linux, macOS and Windows with Python 3.9, 3.11, 3.13 and 3.14.
330
343
 
331
344
  </td>
332
345
  </tr>
@@ -474,7 +487,7 @@ Without a password, `--serve-host` means **anyone who reaches the port can use i
474
487
 
475
488
  - every connection goes through a different proxy (unless sticky); fast and proven ones are preferred
476
489
  - `CONNECT` for HTTPS and plain HTTP requests; HTTP, SOCKS4 and SOCKS5 proxies can sit behind it (SOCKS5 with DNS through the proxy)
477
- - HTTPS only uses proxies that passed the test with **verified TLS** – no broken encryption
490
+ - HTTPS prefers proxies that passed the test with **verified TLS**; when no such proxy fits right now – none matches a wish like `country-de`, or they have all dropped out – it uses the others – your client's own certificate check still catches a proxy that breaks the encryption, so leave it on
478
491
  - if a proxy stays silent inside the tunnel or returns an error page instead of TLS, the same first packet quietly goes to the next one
479
492
  - three failures in a row and a proxy leaves the rotation – every 5 minutes those get re-checked and come back if they work again
480
493
  - `--serve-refill 6` checks fresh proxies every 6 hours in the background (the live list with `--recheck live`, otherwise the last run + history) with the same checks and filters, and adds the hits – a server that runs for days doesn't run dry
@@ -482,6 +495,10 @@ Without a password, `--serve-host` means **anyone who reaches the port can use i
482
495
 
483
496
  In testing: 20 of 20 HTTPS requests succeeded, over 15 different exit IPs. In the wizard this is **Proxy server right away**.
484
497
 
498
+ **Live dashboard:** open `http://127.0.0.1:8899/__proxy-scraper/` in a browser – the pool, success rate, traffic, a live chart of requests, countries, the best proxies and the latest connections, refreshed every two seconds. It needs nothing from the internet, and `--serve-password` protects it like the rest.
499
+
500
+ <div align="center"><img src="docs/dashboard-live.png" alt="The live dashboard of the rotating proxy server: usable proxies, success rate, requests per second, countries and the latest connections" width="860"></div>
501
+
485
502
  <a id="pool-api"></a>
486
503
 
487
504
  ### Proxy pool API
@@ -700,6 +717,18 @@ proxychains4 -f results/latest/proxychains.conf curl https://api.ipify.org
700
717
 
701
718
  `clash.yaml` has all HTTP and SOCKS5 proxies plus a `url-test` group that always picks the fastest. `singbox.json` does the same for sing-box, SOCKS4 included, and opens a local proxy: `sing-box run -c results/latest/singbox.json`, then use `127.0.0.1:2080` as HTTP or SOCKS5 proxy. All three leave out HTTP proxies that can't tunnel (`CONNECT`), because these tools tunnel everything.
702
719
 
720
+ **Browsers and the OS** – `--export pac` writes `proxy.pac`: point Firefox, Chrome (via the system settings), FoxyProxy or macOS/Windows proxy settings at it and every request goes through the best 30 HTTP and SOCKS5 proxies in order, the browser moving on by itself when one fails. Local names and private IP addresses stay direct (a host name like `10.example.com` doesn't count as private). The file itself has no `DIRECT` fallback and resolves no host names, and SOCKS4 is left out because it would make the browser resolve names with your own DNS. Two Firefox settings matter: turn on *Proxy DNS when using SOCKS v5*, and set `network.proxy.failover_direct` to `false` in `about:config` – otherwise Firefox goes direct once every proxy has failed. No install at all: the live list publishes one every hour at `https://maximilianfeix.github.io/proxy-scraper/proxy.pac`.
721
+
722
+ **Burp Suite, OWASP ZAP, mitmproxy** – put the rotating server behind your intercepting proxy, so a scan or a brute-force test comes from many IPs while you still see every request. Start it with `proxy-scraper --recheck live --serve`, then:
723
+
724
+ | Tool | Where | Set |
725
+ |---|---|---|
726
+ | Burp Suite | *Settings → Network → Connections → Upstream proxy servers* | a rule for destination host `*`, proxy host `127.0.0.1`, port `8899`; with `--serve-password` add Basic auth (user `any` or a wish like `country-de`, the password) |
727
+ | OWASP ZAP | *Options → Network → Connection → HTTP Proxy* | host `127.0.0.1`, port `8899`, and under its authentication the user (`any` or a wish) and the password – use the HTTP setting, not SOCKS: ZAP never sends loopback addresses through its SOCKS proxy |
728
+ | mitmproxy | command line | `mitmproxy --mode upstream:http://127.0.0.1:8899 --upstream-auth country-us:x` |
729
+
730
+ Username wishes – `country-us` sends everything out through US proxies, `session-NAME` keeps one exit IP for a login flow – reach the server as the proxy login. mitmproxy sends it right away; Burp and ZAP may wait to be asked, so start the server with `--serve-password` when you rely on a wish there. **Keep your tool's upstream certificate checks on** (no `--ssl-insecure` in mitmproxy, no "ignore certificate errors" upstream in Burp or ZAP): free proxies are strangers, and that check is what stops one from reading or changing your HTTPS traffic. Tested end to end with mitmproxy 12, certificate checks on: HTTP and HTTPS through mitmproxy and the rotating server, every request out through a US proxy, and a site with an expired certificate refused. Only test what you're allowed to.
731
+
703
732
  **In a pipe** – `-o -` prints the hits to stdout, the interface moves to stderr
704
733
 
705
734
  ```bash
@@ -793,7 +822,7 @@ curl.exe -x (Get-Content "$run\all.txt" -TotalCount 1) http://api.ipify.org
793
822
  | `--serve-refill HOURS` | while serving, check fresh proxies every HOURS and add the hits to the pool |
794
823
  | `--serve [PORT]` | afterwards serve as a rotating proxy on `127.0.0.1:PORT` (default: 8899) |
795
824
  | `-o FILE` | also write all hits to this file; `-o -` prints them to stdout |
796
- | `--export FORMATS` | extra files for other tools: `proxychains`, `clash`, `singbox`, `curl` or `all` |
825
+ | `--export FORMATS` | extra files for other tools: `proxychains`, `clash`, `singbox`, `curl`, `pac` or `all` |
797
826
  | `-V`, `--version` | print the version |
798
827
  | `--completion SHELL` | print the tab completion script for bash, zsh, fish or PowerShell |
799
828
 
@@ -879,7 +908,7 @@ Only for things that don't matter. Public proxies are run by strangers who can r
879
908
 
880
909
  What's next is in the [open issues](../../issues) – ideas and wishes are welcome as an [issue](../../issues/new/choose). During Hacktoberfest there are [beginner-friendly issues](../../issues?q=is%3Aopen+label%3Ahacktoberfest) with pointers on where to start.
881
910
 
882
- - [ ] [Mirror the daily snapshots to a Hugging Face dataset](../../issues/188)
911
+ Shipped in v1.22 and v1.23: a live dashboard for the rotating server, `--export pac` and an hourly `proxy.pac`, "best first" that counts how likely a proxy is still up, the lists in a repository of their own ([free-proxy-list](https://github.com/maximilianfeix/free-proxy-list)), daily snapshots on Hugging Face, `brew install`, a Chinese README, Python 3.14, and a recipe for Burp Suite, ZAP and mitmproxy.
883
912
 
884
913
  Shipped in v1.10 to v1.21: download speed per proxy and "best first" everywhere, daily snapshots, a rotating proxy URL for requests, httpx and Playwright, `--pick` for working proxies without a scan, a proxy pool API compatible with jhao104/proxy_pool, your own lists with `--source`, a GitHub Action, a Telegram bot and an Atom feed for the weekly report.
885
914
 
@@ -928,12 +957,13 @@ proxyscraper/
928
957
  ├── geodb.py DB-IP country database (monthly, binary search)
929
958
  ├── targets.py target sites for --target
930
959
  ├── output.py result files
931
- ├── exporters.py proxychains, Clash, sing-box and curl formats (--export)
960
+ ├── exporters.py proxychains, Clash, sing-box, curl and PAC formats (--export)
932
961
  ├── server/ rotating proxy server (--serve): pool · http · upstream · socks · status · api · core
933
962
  ├── api.py find_proxies() / check_proxies() for Python
934
963
  ├── agent.py MCP tools without the SDK: live list, filters, fetch through proxies
935
964
  ├── mcp_server.py MCP server (proxy-scraper-mcp) for AI agents
936
965
  ├── publish.py live list for GitHub Actions
966
+ ├── mirror.py the same lists in their own repo (free-proxy-list)
937
967
  ├── paths.py where state and results are stored
938
968
  ├── compat.py differences between Unix and Windows
939
969
  ├── netio.py small HTTP client on asyncio
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: proxy-scraper-cli
3
- Version: 1.22.0
3
+ Version: 1.23.0
4
4
  Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
5
5
  Author: Maximilian Feix
6
6
  License-Expression: MIT
@@ -22,6 +22,7 @@ Classifier: Programming Language :: Python :: 3.10
22
22
  Classifier: Programming Language :: Python :: 3.11
23
23
  Classifier: Programming Language :: Python :: 3.12
24
24
  Classifier: Programming Language :: Python :: 3.13
25
+ Classifier: Programming Language :: Python :: 3.14
25
26
  Classifier: Topic :: Internet :: Proxy Servers
26
27
  Classifier: Topic :: System :: Networking
27
28
  Requires-Python: >=3.9
@@ -36,7 +37,7 @@ Requires-Dist: mcp<3,>=2.2; python_version >= "3.10" and extra == "mcp"
36
37
  Provides-Extra: dev
37
38
  Requires-Dist: pytest; extra == "dev"
38
39
  Requires-Dist: hypothesis; extra == "dev"
39
- Requires-Dist: ruff==0.16.8; extra == "dev"
40
+ Requires-Dist: ruff==0.16.9; extra == "dev"
40
41
  Dynamic: license-file
41
42
 
42
43
  <div align="center">
@@ -51,16 +52,19 @@ Dynamic: license-file
51
52
  [![Release](https://img.shields.io/github/v/release/maximilianfeix/proxy-scraper?style=flat-square&color=D4F77A&labelColor=121113)](https://github.com/maximilianfeix/proxy-scraper/releases/latest)
52
53
  [![Live proxies](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fmaximilianfeix%2Fproxy-scraper%2Fproxy-list%2Fbadges%2Ftotal.json&style=flat-square&labelColor=121113)](#live-list)
53
54
  [![PyPI](https://img.shields.io/pypi/v/proxy-scraper-cli?style=flat-square&color=D4F77A&labelColor=121113&label=pypi)](https://pypi.org/project/proxy-scraper-cli/)
54
- [![Python](https://img.shields.io/badge/python-3.9–3.13-D4F77A?style=flat-square&labelColor=121113)](pyproject.toml)
55
+ [![Python](https://img.shields.io/badge/python-3.9–3.14-D4F77A?style=flat-square&labelColor=121113)](pyproject.toml)
55
56
  [![Stars](https://img.shields.io/github/stars/maximilianfeix/proxy-scraper?style=flat-square&color=D4F77A&labelColor=121113)](https://github.com/maximilianfeix/proxy-scraper/stargazers)
56
57
  [![License](https://img.shields.io/badge/license-MIT-D4F77A?style=flat-square&labelColor=121113)](LICENSE)
57
58
 
58
59
  <a href="https://maximilianfeix.github.io/proxy-scraper/"><img src="https://img.shields.io/badge/Browse_the_live_list-D4F77A?style=for-the-badge&labelColor=121113" alt="Browse the live list"></a>
60
+ <a href="https://github.com/maximilianfeix/free-proxy-list"><img src="https://img.shields.io/badge/Just_the_lists-121113?style=for-the-badge" alt="Just the lists: free-proxy-list"></a>
59
61
  <a href="#install"><img src="https://img.shields.io/badge/Install-121113?style=for-the-badge" alt="Install"></a>
60
62
  <a href="#from-python"><img src="https://img.shields.io/badge/Python_API-121113?style=for-the-badge" alt="Python API"></a>
61
63
  <a href="#mcp"><img src="https://img.shields.io/badge/MCP_server-121113?style=for-the-badge" alt="MCP server for AI agents"></a>
62
64
  <a href="bot/"><img src="https://img.shields.io/badge/Discord_bot-121113?style=for-the-badge" alt="Discord bot"></a>
63
65
 
66
+ **English** · [简体中文](README.zh-CN.md)
67
+
64
68
  [Install](#install) · [Live list](#live-list) · [For AI agents](#mcp) · [Proxy server](#proxy-server) · [How it works](#how-it-works) · [Recipes](#recipes) · [Options](#options) · [FAQ](#faq)
65
69
 
66
70
  </div>
@@ -114,6 +118,14 @@ proxy-scraper
114
118
 
115
119
  The package is called `proxy-scraper-cli` on PyPI (the plain name is taken), the command is `proxy-scraper`.
116
120
 
121
+ **With [Homebrew](https://brew.sh)** on macOS and Linux:
122
+
123
+ ```bash
124
+ brew install maximilianfeix/tap/proxy-scraper
125
+ ```
126
+
127
+ The formula follows each release within about a day.
128
+
117
129
  <details>
118
130
  <summary><b>Other ways: Docker, pip, a faster event loop, or straight from the repo</b></summary>
119
131
  <br>
@@ -178,7 +190,9 @@ Keys: <kbd>↑</kbd><kbd>↓</kbd> select · <kbd>Space</kbd> toggle · <kbd>1</
178
190
 
179
191
  ## Live proxy list
180
192
 
181
- Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and publishes the hits to the [`proxy-list`](../../tree/proxy-list) branch – every entry worked in the last run, fastest first.
193
+ Don't want to scan yourself? Every hour **GitHub Actions** runs the tool and publishes the hits to the [`proxy-list`](../../tree/proxy-list) branch – every entry worked in the last run, best first: fast ones that are likely to still be up come on top.
194
+
195
+ > **Just want the lists?** They also live in a repository of their own, **[maximilianfeix/free-proxy-list](https://github.com/maximilianfeix/free-proxy-list)** – one list per protocol, per country and per site, updated every hour, with the numbers of the last run on the front page. Watch or star it to keep it at hand.
182
196
 
183
197
  <a href="https://maximilianfeix.github.io/proxy-scraper/"><picture>
184
198
  <source media="(prefers-color-scheme: dark)" srcset="https://maximilianfeix.github.io/proxy-scraper/chart-dark.svg">
@@ -319,7 +333,7 @@ Started without arguments, the tool asks what you need using the arrow keys –
319
333
  <td valign="top">
320
334
 
321
335
  **Real verification**<br>
322
- Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests (in some runs 5 out of 6 “hits”). A third request catches proxies that **tamper with content**: in our measurements one in five working proxies injected a script into a plain HTML page. Plus: HTTPS through a tunnel with **verified TLS**, anonymity level *elite / anonymous / transparent* and the country of the exit IP.
336
+ Every hit has to fetch two independent pages – that weeds out **honeypots** that only answer check requests: over 3.6 million checks, 54 % of the proxies that answered the first request failed the second. A third request catches proxies that **tamper with content**: about one in 40 working proxies changed a known page, usually by injecting a script ([the numbers](docs/free-proxies-in-numbers.md)). Plus: HTTPS through a tunnel with **verified TLS**, anonymity level *elite / anonymous / transparent* and the country of the exit IP.
323
337
 
324
338
  </td>
325
339
  <td valign="top">
@@ -361,13 +375,13 @@ Speed chart, latency histogram, protocols, countries and the latest hits in real
361
375
  <td valign="top">
362
376
 
363
377
  **Runs everywhere**<br>
364
- macOS, Linux and Windows, Python 3.9 to 3.13. Only two dependencies: `rich` and `certifi`. It even notices when a firewall blocks proxies.
378
+ macOS, Linux and Windows, Python 3.9 to 3.14. Only two dependencies: `rich` and `certifi`. It even notices when a firewall blocks proxies.
365
379
 
366
380
  </td>
367
381
  <td valign="top">
368
382
 
369
383
  **Thoroughly tested**<br>
370
- 450+ tests run offline against real mini proxies and honeypots on `localhost` – on Linux, macOS and Windows with Python 3.9, 3.11 and 3.13.
384
+ 760+ tests run offline against real mini proxies and honeypots on `localhost` – on Linux, macOS and Windows with Python 3.9, 3.11, 3.13 and 3.14.
371
385
 
372
386
  </td>
373
387
  </tr>
@@ -515,7 +529,7 @@ Without a password, `--serve-host` means **anyone who reaches the port can use i
515
529
 
516
530
  - every connection goes through a different proxy (unless sticky); fast and proven ones are preferred
517
531
  - `CONNECT` for HTTPS and plain HTTP requests; HTTP, SOCKS4 and SOCKS5 proxies can sit behind it (SOCKS5 with DNS through the proxy)
518
- - HTTPS only uses proxies that passed the test with **verified TLS** – no broken encryption
532
+ - HTTPS prefers proxies that passed the test with **verified TLS**; when no such proxy fits right now – none matches a wish like `country-de`, or they have all dropped out – it uses the others – your client's own certificate check still catches a proxy that breaks the encryption, so leave it on
519
533
  - if a proxy stays silent inside the tunnel or returns an error page instead of TLS, the same first packet quietly goes to the next one
520
534
  - three failures in a row and a proxy leaves the rotation – every 5 minutes those get re-checked and come back if they work again
521
535
  - `--serve-refill 6` checks fresh proxies every 6 hours in the background (the live list with `--recheck live`, otherwise the last run + history) with the same checks and filters, and adds the hits – a server that runs for days doesn't run dry
@@ -523,6 +537,10 @@ Without a password, `--serve-host` means **anyone who reaches the port can use i
523
537
 
524
538
  In testing: 20 of 20 HTTPS requests succeeded, over 15 different exit IPs. In the wizard this is **Proxy server right away**.
525
539
 
540
+ **Live dashboard:** open `http://127.0.0.1:8899/__proxy-scraper/` in a browser – the pool, success rate, traffic, a live chart of requests, countries, the best proxies and the latest connections, refreshed every two seconds. It needs nothing from the internet, and `--serve-password` protects it like the rest.
541
+
542
+ <div align="center"><img src="docs/dashboard-live.png" alt="The live dashboard of the rotating proxy server: usable proxies, success rate, requests per second, countries and the latest connections" width="860"></div>
543
+
526
544
  <a id="pool-api"></a>
527
545
 
528
546
  ### Proxy pool API
@@ -741,6 +759,18 @@ proxychains4 -f results/latest/proxychains.conf curl https://api.ipify.org
741
759
 
742
760
  `clash.yaml` has all HTTP and SOCKS5 proxies plus a `url-test` group that always picks the fastest. `singbox.json` does the same for sing-box, SOCKS4 included, and opens a local proxy: `sing-box run -c results/latest/singbox.json`, then use `127.0.0.1:2080` as HTTP or SOCKS5 proxy. All three leave out HTTP proxies that can't tunnel (`CONNECT`), because these tools tunnel everything.
743
761
 
762
+ **Browsers and the OS** – `--export pac` writes `proxy.pac`: point Firefox, Chrome (via the system settings), FoxyProxy or macOS/Windows proxy settings at it and every request goes through the best 30 HTTP and SOCKS5 proxies in order, the browser moving on by itself when one fails. Local names and private IP addresses stay direct (a host name like `10.example.com` doesn't count as private). The file itself has no `DIRECT` fallback and resolves no host names, and SOCKS4 is left out because it would make the browser resolve names with your own DNS. Two Firefox settings matter: turn on *Proxy DNS when using SOCKS v5*, and set `network.proxy.failover_direct` to `false` in `about:config` – otherwise Firefox goes direct once every proxy has failed. No install at all: the live list publishes one every hour at `https://maximilianfeix.github.io/proxy-scraper/proxy.pac`.
763
+
764
+ **Burp Suite, OWASP ZAP, mitmproxy** – put the rotating server behind your intercepting proxy, so a scan or a brute-force test comes from many IPs while you still see every request. Start it with `proxy-scraper --recheck live --serve`, then:
765
+
766
+ | Tool | Where | Set |
767
+ |---|---|---|
768
+ | Burp Suite | *Settings → Network → Connections → Upstream proxy servers* | a rule for destination host `*`, proxy host `127.0.0.1`, port `8899`; with `--serve-password` add Basic auth (user `any` or a wish like `country-de`, the password) |
769
+ | OWASP ZAP | *Options → Network → Connection → HTTP Proxy* | host `127.0.0.1`, port `8899`, and under its authentication the user (`any` or a wish) and the password – use the HTTP setting, not SOCKS: ZAP never sends loopback addresses through its SOCKS proxy |
770
+ | mitmproxy | command line | `mitmproxy --mode upstream:http://127.0.0.1:8899 --upstream-auth country-us:x` |
771
+
772
+ Username wishes – `country-us` sends everything out through US proxies, `session-NAME` keeps one exit IP for a login flow – reach the server as the proxy login. mitmproxy sends it right away; Burp and ZAP may wait to be asked, so start the server with `--serve-password` when you rely on a wish there. **Keep your tool's upstream certificate checks on** (no `--ssl-insecure` in mitmproxy, no "ignore certificate errors" upstream in Burp or ZAP): free proxies are strangers, and that check is what stops one from reading or changing your HTTPS traffic. Tested end to end with mitmproxy 12, certificate checks on: HTTP and HTTPS through mitmproxy and the rotating server, every request out through a US proxy, and a site with an expired certificate refused. Only test what you're allowed to.
773
+
744
774
  **In a pipe** – `-o -` prints the hits to stdout, the interface moves to stderr
745
775
 
746
776
  ```bash
@@ -834,7 +864,7 @@ curl.exe -x (Get-Content "$run\all.txt" -TotalCount 1) http://api.ipify.org
834
864
  | `--serve-refill HOURS` | while serving, check fresh proxies every HOURS and add the hits to the pool |
835
865
  | `--serve [PORT]` | afterwards serve as a rotating proxy on `127.0.0.1:PORT` (default: 8899) |
836
866
  | `-o FILE` | also write all hits to this file; `-o -` prints them to stdout |
837
- | `--export FORMATS` | extra files for other tools: `proxychains`, `clash`, `singbox`, `curl` or `all` |
867
+ | `--export FORMATS` | extra files for other tools: `proxychains`, `clash`, `singbox`, `curl`, `pac` or `all` |
838
868
  | `-V`, `--version` | print the version |
839
869
  | `--completion SHELL` | print the tab completion script for bash, zsh, fish or PowerShell |
840
870
 
@@ -920,7 +950,7 @@ Only for things that don't matter. Public proxies are run by strangers who can r
920
950
 
921
951
  What's next is in the [open issues](../../issues) – ideas and wishes are welcome as an [issue](../../issues/new/choose). During Hacktoberfest there are [beginner-friendly issues](../../issues?q=is%3Aopen+label%3Ahacktoberfest) with pointers on where to start.
922
952
 
923
- - [ ] [Mirror the daily snapshots to a Hugging Face dataset](../../issues/188)
953
+ Shipped in v1.22 and v1.23: a live dashboard for the rotating server, `--export pac` and an hourly `proxy.pac`, "best first" that counts how likely a proxy is still up, the lists in a repository of their own ([free-proxy-list](https://github.com/maximilianfeix/free-proxy-list)), daily snapshots on Hugging Face, `brew install`, a Chinese README, Python 3.14, and a recipe for Burp Suite, ZAP and mitmproxy.
924
954
 
925
955
  Shipped in v1.10 to v1.21: download speed per proxy and "best first" everywhere, daily snapshots, a rotating proxy URL for requests, httpx and Playwright, `--pick` for working proxies without a scan, a proxy pool API compatible with jhao104/proxy_pool, your own lists with `--source`, a GitHub Action, a Telegram bot and an Atom feed for the weekly report.
926
956
 
@@ -969,12 +999,13 @@ proxyscraper/
969
999
  ├── geodb.py DB-IP country database (monthly, binary search)
970
1000
  ├── targets.py target sites for --target
971
1001
  ├── output.py result files
972
- ├── exporters.py proxychains, Clash, sing-box and curl formats (--export)
1002
+ ├── exporters.py proxychains, Clash, sing-box, curl and PAC formats (--export)
973
1003
  ├── server/ rotating proxy server (--serve): pool · http · upstream · socks · status · api · core
974
1004
  ├── api.py find_proxies() / check_proxies() for Python
975
1005
  ├── agent.py MCP tools without the SDK: live list, filters, fetch through proxies
976
1006
  ├── mcp_server.py MCP server (proxy-scraper-mcp) for AI agents
977
1007
  ├── publish.py live list for GitHub Actions
1008
+ ├── mirror.py the same lists in their own repo (free-proxy-list)
978
1009
  ├── paths.py where state and results are stored
979
1010
  ├── compat.py differences between Unix and Windows
980
1011
  ├── netio.py small HTTP client on asyncio
@@ -29,6 +29,7 @@ proxyscraper/history.py
29
29
  proxyscraper/judges.py
30
30
  proxyscraper/mcp_entry.py
31
31
  proxyscraper/mcp_server.py
32
+ proxyscraper/mirror.py
32
33
  proxyscraper/netio.py
33
34
  proxyscraper/options.py
34
35
  proxyscraper/output.py
@@ -50,6 +51,7 @@ proxyscraper/targets.py
50
51
  proxyscraper/server/__init__.py
51
52
  proxyscraper/server/api.py
52
53
  proxyscraper/server/core.py
54
+ proxyscraper/server/dashboard.html
53
55
  proxyscraper/server/http.py
54
56
  proxyscraper/server/pool.py
55
57
  proxyscraper/server/socks.py
@@ -87,9 +89,11 @@ tests/test_learning.py
87
89
  tests/test_live_api.py
88
90
  tests/test_mcp_server.py
89
91
  tests/test_mine_sources.py
92
+ tests/test_mirror.py
90
93
  tests/test_options.py
91
94
  tests/test_output_stdout.py
92
95
  tests/test_own_sources.py
96
+ tests/test_pac.py
93
97
  tests/test_packaging.py
94
98
  tests/test_pages.py
95
99
  tests/test_parsing.py
@@ -105,6 +109,7 @@ tests/test_report.py
105
109
  tests/test_robustness.py
106
110
  tests/test_rotator.py
107
111
  tests/test_serve_api.py
112
+ tests/test_serve_dashboard.py
108
113
  tests/test_serve_password.py
109
114
  tests/test_serve_refill.py
110
115
  tests/test_serve_v2.py
@@ -115,6 +120,7 @@ tests/test_sites.py
115
120
  tests/test_sources_json.py
116
121
  tests/test_speed.py
117
122
  tests/test_speed_ranking.py
123
+ tests/test_survival_ranking.py
118
124
  tests/test_tamper.py
119
125
  tests/test_targets.py
120
126
  tests/test_ui.py
@@ -4,7 +4,7 @@ certifi
4
4
  [dev]
5
5
  pytest
6
6
  hypothesis
7
- ruff==0.16.8
7
+ ruff==0.16.9
8
8
 
9
9
  [fast]
10
10
 
@@ -1,7 +1,7 @@
1
1
  """proxy-scraper: collects public HTTP/SOCKS4/SOCKS5 proxies from hundreds of sources, checks them
2
2
  in parallel with its own protocol handshakes and learns which sources deliver good proxies."""
3
3
 
4
- __version__ = "1.22.0"
4
+ __version__ = "1.23.0"
5
5
 
6
6
 
7
7
  def __getattr__(name):
@@ -181,8 +181,8 @@ def select(rows: Iterable[dict], protocol: str = "any", countries: Iterable[str]
181
181
  elite_only: bool = False, exclude_datacenter: bool = False, exclude_blocklisted: bool = False,
182
182
  stable_only: bool = False, max_latency_ms: int = 0, run_hours: int = 1, min_uptime: int = 0,
183
183
  works_on: Iterable[str] = (), min_speed_kbps: int = 0) -> List[dict]:
184
- """The rows that pass every filter, best first (answer plus download speed) – the same filters as on the
185
- website."""
184
+ """The rows that pass every filter, best first (answer plus download speed, and how likely it's still up) – the
185
+ same filters as on the website."""
186
186
  return best_first(matching(rows, protocol, countries, https_only, elite_only, exclude_datacenter,
187
187
  exclude_blocklisted, stable_only, max_latency_ms, run_hours, min_uptime, works_on,
188
188
  min_speed_kbps))
@@ -208,9 +208,9 @@ async def live_proxies_async(*, types: Iterable[str] = PROXY_TYPES, countries: I
208
208
  anonymity: str = "", max_latency: int = 0, no_datacenter: bool = False,
209
209
  no_blocklisted: bool = False, min_uptime: int = 0, works_on: Iterable[str] = (),
210
210
  min_speed: int = 0, limit: int = 0) -> List[LiveProxy]:
211
- """The proxies from the hourly list that pass the filters, fastest first – by first answer plus measured
212
- download speed. Nothing is checked here: they worked from GitHub's servers in the last run (at most an hour
213
- ago). find_proxies checks from your network.
211
+ """The proxies from the hourly list that pass the filters, best first – by first answer plus measured
212
+ download speed, and how likely each one is still up. Nothing is checked here: they worked from GitHub's servers
213
+ in the last run (at most an hour ago). find_proxies checks from your network.
214
214
 
215
215
  Same filters as find_proxies, plus
216
216
  min_uptime only proxies listed in at least this share (percent) of the week's runs, e.g. 90
@@ -220,7 +220,7 @@ async def live_proxies_async(*, types: Iterable[str] = PROXY_TYPES, countries: I
220
220
  limit at most this many (0 = all)
221
221
  """
222
222
  from .agent import AgentError, LiveSource
223
- from .ranking import page_ms, typical_speed
223
+ from .ranking import expected_ms, typical_speed
224
224
  from .sites import SITE
225
225
 
226
226
  types = _types(types)
@@ -245,9 +245,11 @@ async def live_proxies_async(*, types: Iterable[str] = PROXY_TYPES, countries: I
245
245
  raise ConnectionError(str(e)) from None
246
246
  found = [_live_proxy(r, data.run_hours) for r in data.rows
247
247
  if r.get("ptype") in types and isinstance(r.get("proxy"), str)]
248
- # fastest first – by how quickly a page comes through when the list has download speeds (#186)
248
+ # best first – by how quickly a page comes through when the list has download speeds (#186), and how likely the
249
+ # proxy is still up
249
250
  typical = typical_speed(p.speed_kbps for p in found)
250
- found.sort(key=lambda p: page_ms(p.latency, p.speed_kbps, p.https, typical))
251
+ run_hours = data.run_hours or 1
252
+ found.sort(key=lambda p: expected_ms(p.latency, p.speed_kbps, p.https, typical, p.up_for_hours // run_hours))
251
253
  found = [p for p in found if opts.filters.accepts(p) and (not min_uptime or (p.uptime_7d or 0) >= min_uptime)
252
254
  and all(p.sites.get(s) for s in works_on) and (not min_speed or (p.speed_kbps or 0) >= min_speed)]
253
255
  return found[:limit] if limit else found