proxy-scraper-cli 1.13.0__tar.gz → 1.14.0__tar.gz

{proxy_scraper_cli-1.13.0 → proxy_scraper_cli-1.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.13.0
+Version: 1.14.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT
@@ -699,6 +699,13 @@ pip download requests   # git, pip, npm & co. now go through the pool
 
 `proxy` is the fastest one, `file` a text file with all of them (up to `limit`, default 20), `count` how many. Without a match the step fails, unless `fail-if-empty: false`.
 
+**In the shell, without a scan** – `--pick` takes proxies from the hourly list in about half a second
+
+```bash
+curl -x "$(proxy-scraper --pick --https-only --min-uptime 90)" https://api.ipify.org
+proxy-scraper --pick 10 --country DE --works-on google > de.txt
+```
+
 **Without installing anything** – straight from the live list
 
 ```bash
@@ -743,6 +750,7 @@ curl.exe -x (Get-Content "$run\all.txt" -TotalCount 1) http://api.ipify.org
 | `--discover` | search GitHub for new sources right now |
 | `--no-cache` | download every list again (unchanged ones are normally skipped via ETag) |
 | `--list-sources [N]` | show the source ranking (add `--json` for scripts: url, status, hit rate, checks, last change) |
+| `--pick [N]` | print N proxies (default 1) from the hourly checked list and exit – no scan, same filters, plus `--min-uptime PERCENT` and `--works-on google,reddit,…` |
 | `--serve-host ADDR` | where the proxy server listens (default `127.0.0.1`; `0.0.0.0` for Docker, with a warning) |
 | `--serve-password SECRET` | clients must send this password in the proxy login; better set `PROXY_SCRAPER_SERVE_PASSWORD` |
 | `--rotate STRATEGY` · `--sticky SEC` | how the proxy server picks proxies, see [above](#proxy-server) |

{proxy_scraper_cli-1.13.0 → proxy_scraper_cli-1.14.0}/README.md

@@ -658,6 +658,13 @@ pip download requests   # git, pip, npm & co. now go through the pool
 
 `proxy` is the fastest one, `file` a text file with all of them (up to `limit`, default 20), `count` how many. Without a match the step fails, unless `fail-if-empty: false`.
 
+**In the shell, without a scan** – `--pick` takes proxies from the hourly list in about half a second
+
+```bash
+curl -x "$(proxy-scraper --pick --https-only --min-uptime 90)" https://api.ipify.org
+proxy-scraper --pick 10 --country DE --works-on google > de.txt
+```
+
 **Without installing anything** – straight from the live list
 
 ```bash
@@ -702,6 +709,7 @@ curl.exe -x (Get-Content "$run\all.txt" -TotalCount 1) http://api.ipify.org
 | `--discover` | search GitHub for new sources right now |
 | `--no-cache` | download every list again (unchanged ones are normally skipped via ETag) |
 | `--list-sources [N]` | show the source ranking (add `--json` for scripts: url, status, hit rate, checks, last change) |
+| `--pick [N]` | print N proxies (default 1) from the hourly checked list and exit – no scan, same filters, plus `--min-uptime PERCENT` and `--works-on google,reddit,…` |
 | `--serve-host ADDR` | where the proxy server listens (default `127.0.0.1`; `0.0.0.0` for Docker, with a warning) |
 | `--serve-password SECRET` | clients must send this password in the proxy login; better set `PROXY_SCRAPER_SERVE_PASSWORD` |
 | `--rotate STRATEGY` · `--sticky SEC` | how the proxy server picks proxies, see [above](#proxy-server) |

{proxy_scraper_cli-1.13.0 → proxy_scraper_cli-1.14.0}/proxy_scraper_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.13.0
+Version: 1.14.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT
@@ -699,6 +699,13 @@ pip download requests   # git, pip, npm & co. now go through the pool
 
 `proxy` is the fastest one, `file` a text file with all of them (up to `limit`, default 20), `count` how many. Without a match the step fails, unless `fail-if-empty: false`.
 
+**In the shell, without a scan** – `--pick` takes proxies from the hourly list in about half a second
+
+```bash
+curl -x "$(proxy-scraper --pick --https-only --min-uptime 90)" https://api.ipify.org
+proxy-scraper --pick 10 --country DE --works-on google > de.txt
+```
+
 **Without installing anything** – straight from the live list
 
 ```bash
@@ -743,6 +750,7 @@ curl.exe -x (Get-Content "$run\all.txt" -TotalCount 1) http://api.ipify.org
 | `--discover` | search GitHub for new sources right now |
 | `--no-cache` | download every list again (unchanged ones are normally skipped via ETag) |
 | `--list-sources [N]` | show the source ranking (add `--json` for scripts: url, status, hit rate, checks, last change) |
+| `--pick [N]` | print N proxies (default 1) from the hourly checked list and exit – no scan, same filters, plus `--min-uptime PERCENT` and `--works-on google,reddit,…` |
 | `--serve-host ADDR` | where the proxy server listens (default `127.0.0.1`; `0.0.0.0` for Docker, with a warning) |
 | `--serve-password SECRET` | clients must send this password in the proxy login; better set `PROXY_SCRAPER_SERVE_PASSWORD` |
 | `--rotate STRATEGY` · `--sticky SEC` | how the proxy server picks proxies, see [above](#proxy-server) |

{proxy_scraper_cli-1.13.0 → proxy_scraper_cli-1.14.0}/proxy_scraper_cli.egg-info/SOURCES.txt

@@ -92,6 +92,7 @@ tests/test_pages.py
 tests/test_parsing.py
 tests/test_parsing_fuzz.py
 tests/test_paths.py
+tests/test_pick.py
 tests/test_pipeline.py
 tests/test_proxy_pages.py
 tests/test_proxy_sources.py

{proxy_scraper_cli-1.13.0 → proxy_scraper_cli-1.14.0}/proxyscraper/__init__.py

@@ -1,7 +1,7 @@
 """proxy-scraper: collects public HTTP/SOCKS4/SOCKS5 proxies from hundreds of sources, checks them
 in parallel with its own protocol handshakes and learns which sources deliver good proxies."""
 
-__version__ = "1.13.0"
+__version__ = "1.14.0"
 
 
 def __getattr__(name):

{proxy_scraper_cli-1.13.0 → proxy_scraper_cli-1.14.0}/proxyscraper/cli.py

@@ -34,6 +34,7 @@ from .output import has_latest_results
 from .parsing import PROXY_TYPES
 from .preferences import load_last_argv, save_last_argv
 from .server.pool import STRATEGIES
+from .sites import SITES
 from .targets import parse_target
 from .ui import ACCENT, BAD, MUTED, banner, note, render_source_ranking, widgets
 from .ui.keys import is_interactive
@@ -62,6 +63,36 @@ def list_sources(limit: int, as_json: bool = False) -> int:
     return 0
 
 
+def pick(args) -> int:
+    """--pick: proxies from the hourly list, nothing but their URLs on stdout (for $(…) in a shell)."""
+    from .api import live_proxies
+
+    try:
+        found = live_proxies(types=args.types, countries=args.country or "", https=args.https_only,
+                             anonymity=args.anonymity or "", max_latency=args.max_latency,
+                             no_datacenter=args.no_datacenter, no_blocklisted=args.no_blocklisted,
+                             min_uptime=args.min_uptime, works_on=args.works_on, limit=args.pick)
+    except (ConnectionError, ValueError) as e:
+        print(f"proxy-scraper: {e}", file=sys.stderr)
+        return 1
+    if not found:
+        print("proxy-scraper: no proxy in the live list matches these filters right now – loosen them, "
+              "or run a scan of your own", file=sys.stderr)
+        return 1
+    sys.stdout.write("".join(f"{p.url}\n" for p in found))
+    return 0
+
+
+def site_list(text: str) -> List[str]:
+    """argparse type for --works-on: google,reddit -> ["google", "reddit"]"""
+    names = [n.strip().lower() for n in text.split(",") if n.strip()]
+    known = [s.name for s in SITES]
+    for n in names:
+        if n not in known:
+            raise argparse.ArgumentTypeError(f"unknown site {n!r}, use {', '.join(known)}")
+    return names
+
+
 def _number(kind, minimum, strict=False, maximum=None):
     """argparse type for numbers with limits – with an understandable error message."""
     def parse(text: str):
@@ -96,6 +127,7 @@ def export_list(text: str) -> List[str]:
 positive_int = _number(int, 1)
 port_number = _number(int, 1, maximum=65535)
 non_negative_int = _number(int, 0)
+percent = _number(int, 0, maximum=100)
 positive_float = _number(float, 0, strict=True)
 non_negative_float = _number(float, 0)
 
@@ -166,6 +198,16 @@ def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
     f.add_argument("--target", action="append", type=target_url, metavar="URL",
                    help="only proxies that reach this site (repeatable), e.g. --target google.com")
 
+    live = p.add_argument_group("Live list (no scan)")
+    live.add_argument("--pick", nargs="?", const=1, type=positive_int, metavar="N",
+                      help="print N proxies (default 1) from the hourly checked list that match the filters above, "
+                           "one URL per line, and exit – e.g. curl -x \"$(proxy-scraper --pick --https-only)\" …")
+    live.add_argument("--min-uptime", type=percent, default=0, metavar="PERCENT",
+                      help="with --pick: only proxies on the list in at least this share of the week's checks")
+    live.add_argument("--works-on", type=site_list, default=[], metavar="SITES",
+                      help=f"with --pick: only proxies that got through to these sites in the last check "
+                           f"({', '.join(s.name for s in SITES)})")
+
     v = p.add_argument_group("Proxy server")
     v.add_argument("--serve", nargs="?", const=DEFAULT_SERVE_PORT, default=0, type=port_number, metavar="PORT",
                    help=f"serve the hits as a rotating proxy on 127.0.0.1:PORT after the run "
@@ -209,6 +251,14 @@ def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
     args = p.parse_args(argv)
     if args.json and args.list_sources is None:
         p.error("--json only works together with --list-sources")
+    if (args.min_uptime or args.works_on) and args.pick is None:
+        p.error("--min-uptime and --works-on only work together with --pick")
+    if args.pick is not None:  # --pick exits right away: options for a scan or the server would be silently ignored
+        scan_only = [flag for dest, flag in (("target", "--target"), ("serve", "--serve"), ("recheck", "--recheck"),
+                                             ("want", "--want"), ("output", "-o/--output"), ("export", "--export"))
+                     if getattr(args, dest) != p.get_default(dest)]
+        if scan_only:
+            p.error(f"--pick takes proxies from the live list and exits – {', '.join(scan_only)} belong to a scan")
     return args
 
 
@@ -260,6 +310,8 @@ def run(argv: Optional[List[str]] = None) -> int:
         return mcp_main()
     if args.list_sources is not None:
         return list_sources(args.list_sources, args.json)
+    if args.pick is not None:
+        return pick(args)
     opts = RunOptions.from_args(args)
     if opts.output == STDOUT:
         widgets.console = Console(highlight=False, stderr=True)  # stdout only carries the hits

proxy_scraper_cli-1.14.0/tests/test_pick.py

@@ -0,0 +1,44 @@
+"""proxy-scraper --pick: proxies from the hourly list on stdout, with the usual filters, no scan."""
+
+import pytest
+
+from proxyscraper import api, cli
+from tests.test_live_api import ROWS, STATS, fetch_from
+
+
+@pytest.fixture(autouse=True)
+def offline(monkeypatch):
+    rows = [dict(ROWS[0], sites={"google": True}), ROWS[1], ROWS[2]]
+    monkeypatch.setattr(api, "_live_fetch", fetch_from({"proxies.json": rows, "stats.json": STATS}))
+
+
+def test_one_proxy_by_default_and_nothing_else_on_stdout(capsys):
+    assert cli.run(["--pick"]) == 0
+    assert capsys.readouterr().out == "http://2.2.2.2:80\n"  # the fastest
+
+
+def test_the_usual_filters_apply(capsys):
+    assert cli.run(["--pick", "5", "--types", "socks4", "socks5", "--country", "DE", "--https-only"]) == 0
+    assert capsys.readouterr().out.splitlines() == ["socks4://3.3.3.3:4145", "socks5://1.1.1.1:1080"]
+    assert cli.run(["--pick", "--min-uptime", "90", "--works-on", "google"]) == 0
+    assert capsys.readouterr().out == "socks5://1.1.1.1:1080\n"
+
+
+def test_nothing_matching_is_exit_1_with_a_hint_on_stderr(capsys):
+    assert cli.run(["--pick", "--country", "JP"]) == 1
+    out = capsys.readouterr()
+    assert out.out == "" and "no proxy" in out.err.lower()
+
+
+def test_pick_only_filters_need_pick(capsys):
+    with pytest.raises(SystemExit):
+        cli.run(["--min-uptime", "90"])
+    assert "--pick" in capsys.readouterr().err
+
+
+@pytest.mark.parametrize("extra", [["--target", "discord.com"], ["--serve"], ["--recheck"], ["--want", "5"],
+                                   ["-o", "x.txt"], ["--export", "clash"]])
+def test_options_for_a_scan_are_an_error_not_ignored(extra, capsys):
+    with pytest.raises(SystemExit):
+        cli.run(["--pick", *extra])
+    assert "--pick" in capsys.readouterr().err