proxy-scraper-cli 1.15.0__tar.gz → 1.17.0__tar.gz

{proxy_scraper_cli-1.15.0 → proxy_scraper_cli-1.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.15.0
+Version: 1.17.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT
@@ -444,6 +444,18 @@ r = requests.get("https://api.ipify.org", proxies={"https": next(pool)}, timeout
 
 Same filters as `find_proxies`, plus `min_uptime`, `works_on` (`["google"]`, `"reddit"`, `"amazon"`, `"instagram"`, `"tiktok"`) and `limit`. Every result also has `uptime_24h`, `uptime_7d`, `first_seen`, `up_for_hours` and `sites`.
 
+Or let the library do the retrying: `ProxyRotator` loads a URL through the list and moves on to the next proxy when one fails – HTTPS only through proxies with verified TLS.
+
+```python
+from proxyscraper import ProxyRotator
+
+with ProxyRotator(country="DE") as rotator:
+    r = rotator.get("https://httpbin.org/ip")
+    print(r.status, r.text, "via", r.via)
+```
+
+Usually a few seconds; when many proxies in a row are down it can take a minute (`timeout=` is per attempt). There's `aget()` with `async with` for asyncio code.
+
 <a id="proxy-server"></a>
 
 ## Rotating proxy server

{proxy_scraper_cli-1.15.0 → proxy_scraper_cli-1.17.0}/README.md

@@ -403,6 +403,18 @@ r = requests.get("https://api.ipify.org", proxies={"https": next(pool)}, timeout
 
 Same filters as `find_proxies`, plus `min_uptime`, `works_on` (`["google"]`, `"reddit"`, `"amazon"`, `"instagram"`, `"tiktok"`) and `limit`. Every result also has `uptime_24h`, `uptime_7d`, `first_seen`, `up_for_hours` and `sites`.
 
+Or let the library do the retrying: `ProxyRotator` loads a URL through the list and moves on to the next proxy when one fails – HTTPS only through proxies with verified TLS.
+
+```python
+from proxyscraper import ProxyRotator
+
+with ProxyRotator(country="DE") as rotator:
+    r = rotator.get("https://httpbin.org/ip")
+    print(r.status, r.text, "via", r.via)
+```
+
+Usually a few seconds; when many proxies in a row are down it can take a minute (`timeout=` is per attempt). There's `aget()` with `async with` for asyncio code.
+
 <a id="proxy-server"></a>
 
 ## Rotating proxy server

{proxy_scraper_cli-1.15.0 → proxy_scraper_cli-1.17.0}/proxy_scraper_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.15.0
+Version: 1.17.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT
@@ -444,6 +444,18 @@ r = requests.get("https://api.ipify.org", proxies={"https": next(pool)}, timeout
 
 Same filters as `find_proxies`, plus `min_uptime`, `works_on` (`["google"]`, `"reddit"`, `"amazon"`, `"instagram"`, `"tiktok"`) and `limit`. Every result also has `uptime_24h`, `uptime_7d`, `first_seen`, `up_for_hours` and `sites`.
 
+Or let the library do the retrying: `ProxyRotator` loads a URL through the list and moves on to the next proxy when one fails – HTTPS only through proxies with verified TLS.
+
+```python
+from proxyscraper import ProxyRotator
+
+with ProxyRotator(country="DE") as rotator:
+    r = rotator.get("https://httpbin.org/ip")
+    print(r.status, r.text, "via", r.via)
+```
+
+Usually a few seconds; when many proxies in a row are down it can take a minute (`timeout=` is per attempt). There's `aget()` with `async with` for asyncio code.
+
 <a id="proxy-server"></a>
 
 ## Rotating proxy server

{proxy_scraper_cli-1.15.0 → proxy_scraper_cli-1.17.0}/proxy_scraper_cli.egg-info/SOURCES.txt

@@ -99,6 +99,7 @@ tests/test_proxy_sources.py
 tests/test_publish.py
 tests/test_report.py
 tests/test_robustness.py
+tests/test_rotator.py
 tests/test_serve_password.py
 tests/test_serve_refill.py
 tests/test_serve_v2.py

{proxy_scraper_cli-1.15.0 → proxy_scraper_cli-1.17.0}/proxyscraper/__init__.py

@@ -1,13 +1,13 @@
 """proxy-scraper: collects public HTTP/SOCKS4/SOCKS5 proxies from hundreds of sources, checks them
 in parallel with its own protocol handshakes and learns which sources deliver good proxies."""
 
-__version__ = "1.15.0"
+__version__ = "1.17.0"
 
 
 def __getattr__(name):
     # load the Python API only when needed – "import proxyscraper" should stay light (the CLI doesn't need it)
     if name in ("find_proxies", "find_proxies_async", "check_proxies", "check_proxies_async", "live_proxies",
-                "live_proxies_async", "LiveProxy"):
+                "live_proxies_async", "LiveProxy", "ProxyRotator", "ProxyResponse"):
         from . import api
         return getattr(api, name)
     raise AttributeError(f"module 'proxyscraper' has no attribute {name!r}")

{proxy_scraper_cli-1.15.0 → proxy_scraper_cli-1.17.0}/proxyscraper/agent.py

@@ -502,7 +502,8 @@ class PageFetcher:
         return live
 
     async def fetch(self, url: str, protocol: str = "any", country: str = "", max_chars: int = 20000,
-                    raw_html: bool = False, progress: Optional[Progress] = None, tick: float = 5.0) -> dict:
+                    raw_html: bool = False, progress: Optional[Progress] = None, tick: float = 5.0,
+                    with_body: bool = False) -> dict:
         url = await asyncio.to_thread(self.check, url)  # may look the name up
         protocol = _check_protocol(protocol)
         countries = normalize_countries([country] if country else [])
@@ -566,6 +567,9 @@ class PageFetcher:
             "text": text[:max_chars] if is_text else "",
             "truncated": truncated,
             "note": None if is_text else f"Binary content ({content_type}) isn't returned as text.",
+            # for the Python API (ProxyRotator): the bytes and every header line as they came
+            **({"body": body[:MAX_PAGE_BYTES], "header_items": list(headers.items()) if headers else [],
+                "body_truncated": len(body) > MAX_PAGE_BYTES} if with_body else {}),
         }
 
     async def _download(self, url: str, user: str, port: int, progress: Optional[Progress], tick: float,

{proxy_scraper_cli-1.15.0 → proxy_scraper_cli-1.17.0}/proxyscraper/api.py

@@ -25,13 +25,14 @@ from __future__ import annotations
 
 import asyncio
 import contextlib
+import email.message
 import io
 import os
 import re
 import tempfile
 import threading
 from dataclasses import dataclass, field
-from typing import Dict, Iterable, List, Optional
+from typing import Dict, Iterable, List, Optional, Tuple
 
 from rich.console import Console
 
@@ -41,8 +42,18 @@ from .parsing import PROXY_TYPES
 from .targets import parse_target
 from .ui import widgets
 
-__all__ = ["CheckResult", "LiveProxy", "check_proxies", "check_proxies_async", "find_proxies", "find_proxies_async",
-           "live_proxies", "live_proxies_async"]
+__all__ = [
+    "CheckResult",
+    "LiveProxy",
+    "ProxyResponse",
+    "ProxyRotator",
+    "check_proxies",
+    "check_proxies_async",
+    "find_proxies",
+    "find_proxies_async",
+    "live_proxies",
+    "live_proxies_async",
+]
 
 
 def _types(types: Iterable[str]) -> List[str]:
@@ -237,3 +248,154 @@ async def live_proxies_async(*, types: Iterable[str] = PROXY_TYPES, countries: I
 def live_proxies(**kwargs) -> List[LiveProxy]:
     """Like live_proxies_async, just synchronous (starts its own event loop)."""
     return asyncio.run(live_proxies_async(**kwargs))
+
+
+# --------------------------------------------------------------------------- fetching through the list
+
+class _Headers(dict):
+    """Response headers: looked up without regard to case, the first value per name as a plain dict entry,
+    and every value – Set-Cookie comes more than once – through get_all()."""
+
+    def __init__(self, items: Iterable[Tuple[str, str]] = ()):
+        self._items = list(items)
+        super().__init__()
+        for name, value in self._items:
+            super().setdefault(name.lower(), value)
+
+    def __getitem__(self, name: str) -> str:
+        return super().__getitem__(name.lower())
+
+    def get(self, name: str, default=None):
+        return super().get(name.lower(), default)
+
+    def __contains__(self, name: object) -> bool:
+        return isinstance(name, str) and super().__contains__(name.lower())
+
+    def get_all(self, name: str) -> List[str]:
+        return [value for key, value in self._items if key.lower() == name.lower()]
+
+
+@dataclass
+class ProxyResponse:
+    """What ProxyRotator.get() returns."""
+    status: int
+    url: str
+    final_url: str                     # after redirects
+    headers: _Headers                  # case doesn't matter; headers.get_all("Set-Cookie") for repeated ones
+    content: bytes
+    via: Optional[str]                 # the proxy that delivered it, e.g. socks5://1.2.3.4:1080
+    proxy_country: Optional[str]
+    elapsed_ms: int
+    truncated: bool = False            # the body was longer than 2 MB and is cut off
+
+    @property
+    def text(self) -> str:
+        message = email.message.Message()
+        message["Content-Type"] = self.headers.get("Content-Type", "")
+        charset = message.get_content_charset() or "utf-8"  # handles charset="…" and any case, like a browser
+        try:
+            return self.content.decode(charset, errors="replace")
+        except LookupError:  # a charset Python doesn't know
+            return self.content.decode("utf-8", errors="replace")
+
+    def raise_for_status(self) -> None:
+        if self.status >= 400:
+            raise ConnectionError(f"HTTP {self.status} from {self.final_url} (via {self.via})")
+        if self.truncated:
+            raise ConnectionError(f"the body from {self.final_url} was cut off at 2 MB")
+
+
+class ProxyRotator:
+    """Loads URLs through proxies from the hourly list and switches to the next one when a proxy fails –
+    the loop every scraper writes by hand. HTTPS only through proxies with verified TLS.
+
+        with ProxyRotator(country="DE") as rotator:
+            print(rotator.get("https://api.ipify.org").text)
+
+    Behind it runs the same rotating server as `proxy-scraper --serve`, on 127.0.0.1 with a random password.
+    Use it either sync (`get()`, safe from several threads) or async (`aget()` with `async with`) – not both on
+    the same rotator. Private and local addresses are refused. Bodies are cut at 2 MB (`truncated`).
+    """
+
+    def __init__(self, *, protocol: str = "any", country: str = "", timeout: float = 20.0,
+                 _source=None, _allow_private: bool = False):
+        self.protocol, self.country, self.timeout = protocol, country, timeout
+        self._source, self._allow_private = _source, _allow_private
+        self._fetcher = None
+        self._mode: Optional[str] = None  # "sync" or "async" – their proxy servers live on different loops
+        self._loop: Optional[asyncio.AbstractEventLoop] = None  # the sync API's own loop, in a thread
+        self._thread: Optional[threading.Thread] = None
+        self._start_lock = threading.Lock()
+
+    def _use(self, mode: str) -> None:
+        if self._mode is None:
+            self._mode = mode
+        elif self._mode != mode:
+            raise RuntimeError(f"this ProxyRotator is used {self._mode}hronously – make a second one for "
+                               f"{mode} code (sync: get()/close(), async: aget()/async with)")
+
+    def _make_fetcher(self):
+        from .agent import LiveSource, PageFetcher
+
+        return PageFetcher(self._source or LiveSource(fetch=_live_fetch), allow_private=self._allow_private,
+                           timeout=self.timeout)
+
+    async def aget(self, url: str) -> ProxyResponse:
+        self._use("async")
+        return await self._aget(url)
+
+    async def _aget(self, url: str) -> ProxyResponse:
+        from .agent import AgentError
+
+        if self._fetcher is None:
+            self._fetcher = self._make_fetcher()
+        try:
+            url = await asyncio.to_thread(self._fetcher.check, url)
+        except AgentError as e:
+            raise ValueError(str(e)) from None
+        try:
+            page = await self._fetcher.fetch(url, self.protocol, self.country, max_chars=0, raw_html=True,
+                                             with_body=True)
+        except AgentError as e:
+            raise ConnectionError(str(e)) from None
+        return ProxyResponse(status=page["status"], url=url, final_url=page["final_url"],
+                             headers=_Headers(page["header_items"]), content=page["body"], via=page["via"],
+                             proxy_country=page["proxy_country"], elapsed_ms=page["elapsed_ms"],
+                             truncated=page["body_truncated"])
+
+    async def aclose(self) -> None:
+        if self._fetcher is not None:
+            await self._fetcher.close()
+            self._fetcher = None
+
+    async def __aenter__(self) -> "ProxyRotator":
+        return self
+
+    async def __aexit__(self, *exc) -> None:
+        await self.aclose()
+
+    # sync: one event loop in a background thread keeps the proxy server alive between calls
+    def get(self, url: str) -> ProxyResponse:
+        with self._start_lock:  # several threads calling get() at once still share one loop
+            self._use("sync")
+            if self._loop is None:
+                self._loop = asyncio.new_event_loop()
+                self._thread = threading.Thread(target=self._loop.run_forever, name="proxy-rotator", daemon=True)
+                self._thread.start()
+        return asyncio.run_coroutine_threadsafe(self._aget(url), self._loop).result()
+
+    def close(self) -> None:
+        with self._start_lock:
+            if self._loop is None:
+                return
+            asyncio.run_coroutine_threadsafe(self.aclose(), self._loop).result()
+            self._loop.call_soon_threadsafe(self._loop.stop)
+            self._thread.join(timeout=5)
+            self._loop.close()
+            self._thread = None
+
+    def __enter__(self) -> "ProxyRotator":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()

{proxy_scraper_cli-1.15.0 → proxy_scraper_cli-1.17.0}/proxyscraper/pages.py

@@ -7,6 +7,8 @@ that answers exactly that, and every page links back to the full, filterable lis
 
 from __future__ import annotations
 
+import json
+import statistics
 from datetime import datetime
 from html import escape
 from pathlib import Path
@@ -100,6 +102,22 @@ footer { color: var(--muted); font-size: 14px; display: grid; gap: 6px; }
 """
 
 
+PAGE_CSS = """
+.wrap > * { min-width: 0; }  /* grid items default to their content's width: the wide table would push the page */
+.facts { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 150px), 1fr)); gap: 1px; margin: 0;
+         background: var(--line); border: 1px solid var(--line); border-radius: 16px; overflow: hidden; }
+.facts div { background: var(--panel); padding: 16px 18px; }
+.facts dt { font-size: 26px; font-weight: 600; letter-spacing: -.02em; font-variant-numeric: tabular-nums; }
+.facts dd { margin: 2px 0 0; color: var(--muted); font-size: 14px; }
+h2 { font-size: 24px; letter-spacing: -.02em; margin: 0 0 12px; font-weight: 600; }
+.use pre { background: var(--panel); border: 1px solid var(--line); border-radius: 12px; padding: 14px 16px;
+           overflow-x: auto; font-size: 14px; margin: 0 0 10px; }
+.note { color: var(--muted); font-size: 14px; }
+.faq h3 { font-size: 18px; font-weight: 500; margin: 22px 0 6px; }
+.faq p { margin: 0; color: var(--muted); max-width: 70ch; }
+"""
+
+
 def _list_query(path: str) -> str:
     """The same filter on the full list: the website reads its filters from the address."""
     kind = path.strip("/")
@@ -109,6 +127,61 @@ def _list_query(path: str) -> str:
             "elite": "?elite=1"}.get(kind, "")
 
 
+SITE_NAMES = {"tiktok": "TikTok"}
+
+
+def _pick_flags(path: str) -> str:
+    """The --pick filters that give the same selection as this page."""
+    kind = path.strip("/")
+    if kind.startswith("country/"):
+        return f"--country {kind.split('/')[1].upper()}"
+    return {"http": "--types http", "socks4": "--types socks4", "socks5": "--types socks5", "https": "--https-only",
+            "elite": "--anonymity elite"}.get(kind, "")
+
+
+def _short(path: str) -> str:
+    """How a page's proxies are called in a question: "proxies in Germany", "SOCKS5 proxies"."""
+    kind = path.strip("/")
+    if kind.startswith("country/"):
+        return f"proxies in {_short_name(COUNTRIES.get(kind.split('/')[1].upper(), kind.split('/')[1].upper()))}"
+    return {"https": "HTTPS proxies", "elite": "elite proxies"}.get(kind, f"{kind.upper()} proxies")
+
+
+def _facts(rows: List[dict]) -> Dict[str, object]:
+    speeds = [r["speed_kbps"] for r in rows if r.get("speed_kbps")]
+    return {"total": len(rows), "https": sum(1 for r in rows if r.get("https")),
+            "stable": sum(1 for r in rows if (r.get("uptime_7d") or 0) >= 90),
+            # None when none of them was tried: the site check only runs on HTTPS-capable ones, and not at all
+            # when it couldn't resolve the site – "0 get through" would blame proxies for a check that didn't run
+            "google": (sum(1 for r in rows if (r.get("sites") or {}).get("google"))
+                       if any("google" in (r.get("sites") or {}) for r in rows) else None),
+            "speed": round(statistics.median(speeds)) if speeds else None}
+
+
+def _faq(path: str, facts: Dict[str, object], when: str) -> List[Tuple[str, str]]:
+    short, flags = _short(path), _pick_flags(path)
+    if facts["google"] is None:
+        got = "These proxies weren't part of the last site check – it only tries the ones that tunnel HTTPS."
+    elif facts["google"]:
+        got = f"In the last check, {facts['google']:,} of them got through to Google search without a captcha."
+    else:
+        got = "In the last check, none of them got through to Google search without a captcha."
+    return [
+        (f"How many free {short} work right now?",
+         f"{facts['total']:,} passed every check in the last run ({when}): a real handshake, a honeypot check on two "
+         f"sites and a content check. {facts['https']:,} of them tunnel HTTPS with verified TLS, and "
+         f"{facts['stable']:,} were on the list in 90 % or more of this week's hourly checks. The list is checked "
+         "again every hour."),
+        (f"Which free {short} get through to Google?",
+         f"{got} The site check also tries Reddit, Amazon, Instagram and TikTok – the full list "
+         "on the website can be filtered by it."),
+        (f"How do I get working {short} in code or the terminal?",
+         f"Without installing anything, download proxies.txt from this page. With the tool: pipx install "
+         f"proxy-scraper-cli, then proxy-scraper --pick 5 {flags} prints five from the hourly list. From Python, "
+         "live_proxies() takes the same filters."),
+    ]
+
+
 def _render(path: str, title: str, what: str, rows: List[dict], updated: datetime, nav: str) -> str:
     depth = path.count("/")
     root = "../" * depth
@@ -120,19 +193,49 @@ def _render(path: str, title: str, what: str, rows: List[dict], updated: datetim
     else:
         lede = (f"No {what} passed every check in the last run ({when}). The list is checked again every hour, "
                 "so look again later or try the full list.")
-    body_rows = "\n".join(
-        f"<tr><td class=\"mono\"><a href=\"{root}proxy/{escape(r['proxy'].replace(':', '-'))}/\">"
-        f"{escape(r['proxy'])}</a></td><td>{escape(r['ptype'].upper())}</td>"
-        f"<td>{escape(r.get('country') or '–')}</td><td>{r['latency']:,} ms</td>"
-        f"<td>{'yes' if r.get('https') else 'no'}</td><td>{escape(r.get('anonymity') or '–')}</td>"
-        f"<td>{escape((r.get('org') or '–')[:40])}</td></tr>"
-        for r in rows[:ROWS_PER_PAGE])
+    def cells(r: dict) -> str:
+        through = ", ".join(SITE_NAMES.get(n, n.title()) for n, ok in (r.get("sites") or {}).items() if ok) or "–"
+        return (f"<tr><td class=\"mono\"><a href=\"{root}proxy/{escape(r['proxy'].replace(':', '-'))}/\">"
+                f"{escape(r['proxy'])}</a></td><td>{escape(r['ptype'].upper())}</td>"
+                f"<td>{escape(r.get('country') or '–')}</td><td>{r['latency']:,} ms</td>"
+                f"<td>{(str(r['speed_kbps']) + ' KB/s') if r.get('speed_kbps') else '–'}</td>"
+                f"<td>{(str(r['uptime_7d']) + ' %') if r.get('uptime_7d') is not None else '–'}</td>"
+                f"<td>{'yes' if r.get('https') else 'no'}</td><td>{escape(through)}</td>"
+                f"<td>{escape((r.get('org') or '–')[:40])}</td></tr>")
+    body_rows = "\n".join(cells(r) for r in rows[:ROWS_PER_PAGE])
     table = (f"<div class=\"table\"><table><caption>The {min(total, ROWS_PER_PAGE):,} fastest, as ip:port. "
              f"The download has all {total:,} as type://ip:port.</caption>"
              "<thead><tr><th scope=\"col\">Proxy</th><th scope=\"col\">Type</th><th scope=\"col\">Country</th>"
-             "<th scope=\"col\">Latency</th><th scope=\"col\">HTTPS</th><th scope=\"col\">Anonymity</th>"
+             "<th scope=\"col\">Latency</th><th scope=\"col\">Speed</th><th scope=\"col\">Uptime</th>"
+             "<th scope=\"col\">HTTPS</th><th scope=\"col\">Gets through</th>"
              f"<th scope=\"col\">Provider</th></tr></thead><tbody>{body_rows}</tbody></table></div>"
              if total else "")
+    facts = _facts(rows)
+    facts_html = "" if not total else (
+        "<dl class=\"facts\">"
+        f"<div><dt>{facts['total']:,}</dt><dd>working right now</dd></div>"
+        f"<div><dt>{facts['https']:,}</dt><dd>tunnel HTTPS</dd></div>"
+        f"<div><dt>{facts['stable']:,}</dt><dd>on the list 90 %+ of the week</dd></div>"
+        + (f"<div><dt>{facts['google']:,}</dt><dd>get through to Google</dd></div>"
+           if facts["google"] is not None else "")
+        + (f"<div><dt>{facts['speed']:,} KB/s</dt><dd>median download speed</dd></div>" if facts["speed"] else "")
+        + "</dl>")
+    flags = _pick_flags(path)
+    # curl goes to an https:// address, so only proxies that can tunnel it
+    curl_flags = flags if "--https-only" in flags else f"{flags} --https-only".strip()
+    commands = (
+        "<section class=\"use\"><h2>In the terminal or in code</h2>"
+        f"<pre class=\"mono\">pipx install proxy-scraper-cli\nproxy-scraper --pick 5 {escape(flags)}\n"
+        f"curl -x \"$(proxy-scraper --pick {escape(curl_flags)})\" https://api.ipify.org</pre>"
+        "<p class=\"note\">--pick takes proxies from the same hourly list in about half a second. "
+        f"From Python: <code>live_proxies()</code>, see the <a href=\"{REPO_URL}#from-python\">README</a>."
+        "</p></section>")
+    faq = _faq(path, facts, when)
+    faq_html = "<section class=\"faq\"><h2>Questions</h2>" + "".join(
+        f"<h3>{escape(q)}</h3><p>{escape(a)}</p>" for q, a in faq) + "</section>"
+    faq_ld = json.dumps({"@context": "https://schema.org", "@type": "FAQPage", "mainEntity": [
+        {"@type": "Question", "name": q, "acceptedAnswer": {"@type": "Answer", "text": a}} for q, a in faq]},
+        ensure_ascii=False).replace("</", "<\\/")
     description = (f"{total:,} free {what}, checked every hour: real handshake, honeypot and content check. "
                    "Download as text or filter the full list.")
     return f"""<!doctype html>
@@ -154,7 +257,8 @@ def _render(path: str, title: str, what: str, rows: List[dict], updated: datetim
 <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
 <link href="https://fonts.googleapis.com/css2?family=Geist:wght@400..700&family=Geist+Mono:wght@400;500&display=swap"
       rel="stylesheet">
-<style>{CSS}</style>
+<style>{CSS}{PAGE_CSS}</style>
+<script type="application/ld+json">{faq_ld}</script>
 </head>
 <body>
 <div class="wrap">
@@ -171,7 +275,10 @@ def _render(path: str, title: str, what: str, rows: List[dict], updated: datetim
       <a class="btn" href="{REPO_URL}">Check them from your network</a>
     </div>
   </main>
+  {facts_html}
   {table}
+  {commands}
+  {faq_html}
   {nav}
   <footer>
     <span>Free proxies are run by strangers. Never send passwords or personal data through them.</span>

{proxy_scraper_cli-1.15.0 → proxy_scraper_cli-1.17.0}/tests/test_pages.py

@@ -1,5 +1,6 @@
 """Static pages per protocol and country for search engines, plus the sitemap (#121)."""
 
+import html
 from datetime import datetime, timezone
 
 from proxyscraper import pages
@@ -58,3 +59,44 @@ def test_filter_button_opens_the_list_with_the_same_filter(tmp_path):
     for path, query in expected.items():
         html = (tmp_path / path / "index.html").read_text(encoding="utf-8")
         assert f'href="{"../" * (path.count("/") + 1)}{query}#list">Filter the full list' in html
+
+
+def test_a_country_page_has_the_numbers_commands_and_answers(tmp_path):
+    import json
+    import re
+    rows = [row(1, "socks5", https=True, uptime_7d=95, speed_kbps=400, sites={"google": True}),
+            row(2, "http", https=False, uptime_7d=20),
+            row(3, "http", country="US")]
+    pages.write_pages(rows, tmp_path, NOW)
+    de = (tmp_path / "country" / "de" / "index.html").read_text(encoding="utf-8")
+    facts = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", de))
+    assert "2 working right now" in facts and "1 tunnel HTTPS" in facts and "1 on the list 90 %+ of the week" in facts
+    assert "1 get through to Google" in facts and "400 KB/s" in facts
+    assert "proxy-scraper --pick 5 --country DE" in de and 'curl -x "$(proxy-scraper --pick --country DE' in de
+    assert ">Speed<" in de and ">Uptime<" in de
+    ld = [json.loads(b) for b in re.findall(r'<script type="application/ld\+json">(.*?)</script>', de, re.S)]
+    faq = next(d for d in ld if d["@type"] == "FAQPage")
+    assert any("Germany" in q["name"] for q in faq["mainEntity"])
+
+
+def test_protocol_pages_pick_by_protocol(tmp_path):
+    pages.write_pages([row(1, "socks5", https=True)], tmp_path, NOW)
+    assert "proxy-scraper --pick 5 --types socks5" in (tmp_path / "socks5" / "index.html").read_text()
+    assert "proxy-scraper --pick 5 --https-only" in (tmp_path / "https" / "index.html").read_text()
+    assert "proxy-scraper --pick 5 --anonymity elite" in (tmp_path / "elite" / "index.html").read_text()
+
+
+def test_an_empty_page_still_renders(tmp_path):
+    pages.write_pages([], tmp_path, NOW)
+    assert "No proxies in Germany" in (tmp_path / "country" / "de" / "index.html").read_text()
+
+
+def test_the_median_speed_is_the_real_median():
+    assert pages._facts([row(1, speed_kbps=100), row(2, speed_kbps=400)])["speed"] == 250
+
+
+def test_pages_dont_claim_google_blocked_proxies_that_were_never_checked(tmp_path):
+    pages.write_pages([row(1, "socks4"), row(2, "socks4")], tmp_path, NOW)
+    page = html.unescape((tmp_path / "socks4" / "index.html").read_text(encoding="utf-8"))
+    assert "none of them got through" not in page and "<dd>get through to Google</dd>" not in page
+    assert "weren't part of the last site check" in page

proxy_scraper_cli-1.17.0/tests/test_rotator.py

@@ -0,0 +1,129 @@
+"""ProxyRotator: get a URL through the live list, switching proxies until one delivers."""
+
+import asyncio
+
+import pytest
+
+from proxyscraper import ProxyRotator, agent
+from tests.fakes import http_forward_proxy, serve, target_server
+from tests.test_agent import STATS, row
+
+
+def source_for(rows):
+    live = agent.LiveList(rows=rows, stats=STATS, loaded_at=0.0)
+
+    class Source:
+        async def get(self):
+            return live
+    return Source()
+
+
+async def local_setup():
+    target_srv, target_port = await serve(target_server)
+    proxy_srv, proxy_port = await serve(http_forward_proxy)
+    rows = [row(1, proxy="127.0.0.1:1", url="http://127.0.0.1:1", latency=10),  # nothing listens there
+            row(2, proxy=f"127.0.0.1:{proxy_port}", url=f"http://127.0.0.1:{proxy_port}", latency=500)]
+    return target_srv, proxy_srv, target_port, rows
+
+
+def test_async_get_switches_past_a_dead_proxy():
+    async def go():
+        target_srv, proxy_srv, port, rows = await local_setup()
+        try:
+            async with ProxyRotator(_source=source_for(rows), _allow_private=True, timeout=5) as rotator:
+                return await rotator.aget(f"http://127.0.0.1:{port}/ok")
+        finally:
+            target_srv.close()
+            proxy_srv.close()
+    response = asyncio.run(go())
+    assert response.status == 200 and response.content == b"ok" and response.text == "ok"
+    assert response.via.endswith(":" + response.via.rsplit(":", 1)[1]) and "127.0.0.1:1" not in response.via
+
+
+def test_sync_get_works_from_plain_code():
+    # the sync version keeps its own event loop in a thread: the proxy server lives across calls
+    loop = asyncio.new_event_loop()
+    target_srv, proxy_srv, port, rows = loop.run_until_complete(local_setup())
+    import threading
+    thread = threading.Thread(target=loop.run_forever, daemon=True)
+    thread.start()
+    try:
+        with ProxyRotator(_source=source_for(rows), _allow_private=True, timeout=5) as rotator:
+            first = rotator.get(f"http://127.0.0.1:{port}/ok")
+            second = rotator.get(f"http://127.0.0.1:{port}/ok")
+        assert first.status == second.status == 200
+    finally:
+        loop.call_soon_threadsafe(target_srv.close)
+        loop.call_soon_threadsafe(proxy_srv.close)
+        loop.call_soon_threadsafe(loop.stop)
+
+
+def test_nothing_matching_is_a_clear_error():
+    async def go():
+        async with ProxyRotator(country="JP", _source=source_for([row(1)]), _allow_private=True) as rotator:
+            await rotator.aget("http://127.0.0.1:9/")
+    with pytest.raises(ConnectionError, match="JP"):
+        asyncio.run(go())
+
+
+def test_private_addresses_are_refused_by_default():
+    async def go():
+        async with ProxyRotator(_source=source_for([row(1)])) as rotator:
+            await rotator.aget("http://127.0.0.1/")
+    with pytest.raises(ValueError):
+        asyncio.run(go())
+
+
+def response(headers, content=b"x", truncated=False):
+    from proxyscraper.api import ProxyResponse, _Headers
+    return ProxyResponse(200, "u", "u", _Headers(headers), content, "http://p:1", None, 1, truncated=truncated)
+
+
+def test_headers_ignore_case_and_keep_every_value():
+    r = response([("content-type", "text/html"), ("Set-Cookie", "a=1"), ("set-cookie", "b=2")])
+    assert r.headers["Content-Type"] == "text/html" and r.headers.get("CONTENT-TYPE") == "text/html"
+    assert r.headers.get_all("Set-Cookie") == ["a=1", "b=2"]
+
+
+def test_the_charset_is_read_like_a_browser_would():
+    latin = "Grüße".encode("latin-1")
+    assert response([("Content-Type", 'text/html; charset="ISO-8859-1"')], latin).text == "Grüße"
+    assert response([("Content-Type", "text/html; Charset=latin-1")], latin).text == "Grüße"
+    assert response([("Content-Type", "text/html; charset=nonsense")], "ä".encode()).text == "ä"
+
+
+def test_a_cut_off_body_says_so():
+    r = response([], b"x" * 10, truncated=True)
+    assert r.truncated
+    with pytest.raises(ConnectionError, match="cut off"):
+        r.raise_for_status()
+
+
+def test_sync_get_from_many_threads_shares_one_loop(monkeypatch):
+    import concurrent.futures as cf
+
+    from proxyscraper import api
+
+    async def fake_aget(self, url):
+        return response([], url.encode())
+    monkeypatch.setattr(api.ProxyRotator, "_aget", fake_aget)
+    rotator = api.ProxyRotator()
+    with cf.ThreadPoolExecutor(8) as pool:
+        results = list(pool.map(rotator.get, [f"http://x/{i}" for i in range(32)]))
+    assert [r.content for r in results] == [f"http://x/{i}".encode() for i in range(32)]
+    loops = rotator._loop
+    rotator.close()
+    assert loops.is_closed()
+
+
+def test_mixing_sync_and_async_is_refused(monkeypatch):
+    from proxyscraper import api
+
+    async def fake_aget(self, url):
+        return response([])
+    monkeypatch.setattr(api.ProxyRotator, "_aget", fake_aget)
+    rotator = api.ProxyRotator()
+    rotator.get("http://x/")
+    with pytest.raises(RuntimeError, match="async"):
+        asyncio.run(rotator.aget("http://x/"))
+    rotator.close()