proxy-scraper-cli 1.18.0__tar.gz → 1.20.0__tar.gz

{proxy_scraper_cli-1.18.0 → proxy_scraper_cli-1.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.18.0
+Version: 1.20.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT
@@ -456,6 +456,21 @@ with ProxyRotator(country="DE") as rotator:
 
 Usually a few seconds; when many proxies in a row are down it can take a minute (`timeout=` is per attempt). There's `aget()` with `async with` for asyncio code.
 
+Already using requests, httpx, Playwright or Scrapy? `proxy_url()` gives you one local proxy address that rotates behind the scenes – every connection goes out through another proxy from the list, best first, dead ones skipped:
+
+```python
+import requests
+from proxyscraper import ProxyRotator
+
+with ProxyRotator(country="DE") as rotator:
+    proxy = rotator.proxy_url()          # http://country-de:…@127.0.0.1:…
+    requests.get("https://api.ipify.org", proxies={"http": proxy, "https": proxy})
+    # httpx.Client(proxy=proxy) · scrapy: meta={"proxy": proxy} · curl -x "$proxy"
+    # Playwright wants the login apart: chromium.launch(proxy=rotator.playwright_proxy())
+```
+
+It runs on 127.0.0.1 with a random password, takes over each new hourly list by itself and stops with the `with` block.
+
 <a id="proxy-server"></a>
 
 ## Rotating proxy server

{proxy_scraper_cli-1.18.0 → proxy_scraper_cli-1.20.0}/README.md

@@ -415,6 +415,21 @@ with ProxyRotator(country="DE") as rotator:
 
 Usually a few seconds; when many proxies in a row are down it can take a minute (`timeout=` is per attempt). There's `aget()` with `async with` for asyncio code.
 
+Already using requests, httpx, Playwright or Scrapy? `proxy_url()` gives you one local proxy address that rotates behind the scenes – every connection goes out through another proxy from the list, best first, dead ones skipped:
+
+```python
+import requests
+from proxyscraper import ProxyRotator
+
+with ProxyRotator(country="DE") as rotator:
+    proxy = rotator.proxy_url()          # http://country-de:…@127.0.0.1:…
+    requests.get("https://api.ipify.org", proxies={"http": proxy, "https": proxy})
+    # httpx.Client(proxy=proxy) · scrapy: meta={"proxy": proxy} · curl -x "$proxy"
+    # Playwright wants the login apart: chromium.launch(proxy=rotator.playwright_proxy())
+```
+
+It runs on 127.0.0.1 with a random password, takes over each new hourly list by itself and stops with the `with` block.
+
 <a id="proxy-server"></a>
 
 ## Rotating proxy server

{proxy_scraper_cli-1.18.0 → proxy_scraper_cli-1.20.0}/proxy_scraper_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.18.0
+Version: 1.20.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT
@@ -456,6 +456,21 @@ with ProxyRotator(country="DE") as rotator:
 
 Usually a few seconds; when many proxies in a row are down it can take a minute (`timeout=` is per attempt). There's `aget()` with `async with` for asyncio code.
 
+Already using requests, httpx, Playwright or Scrapy? `proxy_url()` gives you one local proxy address that rotates behind the scenes – every connection goes out through another proxy from the list, best first, dead ones skipped:
+
+```python
+import requests
+from proxyscraper import ProxyRotator
+
+with ProxyRotator(country="DE") as rotator:
+    proxy = rotator.proxy_url()          # http://country-de:…@127.0.0.1:…
+    requests.get("https://api.ipify.org", proxies={"http": proxy, "https": proxy})
+    # httpx.Client(proxy=proxy) · scrapy: meta={"proxy": proxy} · curl -x "$proxy"
+    # Playwright wants the login apart: chromium.launch(proxy=rotator.playwright_proxy())
+```
+
+It runs on 127.0.0.1 with a random password, takes over each new hourly list by itself and stops with the `with` block.
+
 <a id="proxy-server"></a>
 
 ## Rotating proxy server

{proxy_scraper_cli-1.18.0 → proxy_scraper_cli-1.20.0}/proxy_scraper_cli.egg-info/SOURCES.txt

@@ -39,6 +39,7 @@ proxyscraper/pipeline.py
 proxyscraper/preferences.py
 proxyscraper/proxypages.py
 proxyscraper/publish.py
+proxyscraper/ranking.py
 proxyscraper/report.py
 proxyscraper/runsteps.py
 proxyscraper/sites.py

{proxy_scraper_cli-1.18.0 → proxy_scraper_cli-1.20.0}/proxyscraper/__init__.py

@@ -1,7 +1,7 @@
 """proxy-scraper: collects public HTTP/SOCKS4/SOCKS5 proxies from hundreds of sources, checks them
 in parallel with its own protocol handshakes and learns which sources deliver good proxies."""
 
-__version__ = "1.18.0"
+__version__ = "1.20.0"
 
 
 def __getattr__(name):

{proxy_scraper_cli-1.18.0 → proxy_scraper_cli-1.20.0}/proxyscraper/agent.py

@@ -34,6 +34,7 @@ from .netio import http_get
 from .pages import SITE_URL
 from .parsing import PROXY_TYPES, make_key
 from .publish import RAW_BASE
+from .ranking import best_first
 from .server import ProxyPool, RotatingServer
 from .sites import SITES
 
@@ -180,11 +181,11 @@ def select(rows: Iterable[dict], protocol: str = "any", countries: Iterable[str]
            elite_only: bool = False, exclude_datacenter: bool = False, exclude_blocklisted: bool = False,
            stable_only: bool = False, max_latency_ms: int = 0, run_hours: int = 1, min_uptime: int = 0,
            works_on: Iterable[str] = (), min_speed_kbps: int = 0) -> List[dict]:
-    """The rows that pass every filter, fastest first – the same filters as on the website."""
-    return sorted(matching(rows, protocol, countries, https_only, elite_only, exclude_datacenter,
-                           exclude_blocklisted, stable_only, max_latency_ms, run_hours, min_uptime, works_on,
-                           min_speed_kbps),
-                  key=lambda r: r.get("latency") or 0)
+    """The rows that pass every filter, best first (answer plus download speed) – the same filters as on the
+    website."""
+    return best_first(matching(rows, protocol, countries, https_only, elite_only, exclude_datacenter,
+                               exclude_blocklisted, stable_only, max_latency_ms, run_hours, min_uptime, works_on,
+                               min_speed_kbps))
 
 
 def matching(rows: Iterable[dict], protocol: str = "any", countries: Iterable[str] = (), https_only: bool = False,
@@ -457,6 +458,22 @@ def _to_text(body: bytes, headers, content_type: str, raw_html: bool) -> str:
     return text if raw_html or "html" not in content_type.lower() else html_to_text(text)
 
 
+def _require_match(live: LiveList, protocol: str, countries: List[str], tls: bool) -> None:
+    if not any(True for _ in matching(live.rows, protocol, countries, https_only=tls, run_hours=live.run_hours)):
+        what = " ".join(x for x in (countries[0] if countries else "", "" if protocol == "any" else protocol,
+                                    "HTTPS-capable" if tls else "") if x)
+        raise AgentError(f"No {what} proxy in the live list right now."
+                         + (shortage_note(live.rows, 1, 0, countries) or ""))
+
+
+def _wishes(protocol: str, countries: List[str]) -> List[str]:
+    """What the rotating server reads from the proxy login: country-de, type-socks5."""
+    wishes = [f"country-{countries[0].lower()}"] if countries else []
+    if protocol != "any":
+        wishes.append(f"type-{protocol}")
+    return wishes
+
+
 class PageFetcher:
     """Loads pages through the rotating server of this package, running on 127.0.0.1 with a random password:
     the same failover as `--serve`, and HTTPS only through proxies that passed the verified-TLS test.
@@ -502,6 +519,19 @@ class PageFetcher:
                 self._list_id = list_id
         return live
 
+    async def local_proxy(self, protocol: str = "any", country: str = "") -> str:
+        """The URL of the rotating server behind this fetcher, with the login, for any HTTP client:
+        http://country-de:<password>@127.0.0.1:<port>. Every connection through it gets a proxy of its own."""
+        protocol = _check_protocol(protocol)
+        countries = normalize_countries([country] if country else [])
+        _require_match(await self._ready(), protocol, countries, tls=False)
+        user = "-".join(_wishes(protocol, countries)) or "any"
+        return f"http://{user}:{self._password}@127.0.0.1:{self.server.port}"
+
+    async def refresh(self) -> None:
+        """Take over a new run's list, if there is one (fetch does that by itself)."""
+        await self._ready()
+
     async def fetch(self, url: str, protocol: str = "any", country: str = "", max_chars: int = 20000,
                     raw_html: bool = False, progress: Optional[Progress] = None, tick: float = 5.0,
                     with_body: bool = False) -> dict:
@@ -510,15 +540,9 @@ class PageFetcher:
         countries = normalize_countries([country] if country else [])
         live = await self._ready()
         tls = urlsplit(url).scheme == "https"
-        if not any(True for _ in matching(live.rows, protocol, countries, https_only=tls, run_hours=live.run_hours)):
-            what = " ".join(x for x in (countries[0] if countries else "", "" if protocol == "any" else protocol,
-                                        "HTTPS-capable" if tls else "") if x)
-            raise AgentError(f"No {what} proxy in the live list right now."
-                             + (shortage_note(live.rows, 1, 0, countries) or ""))
+        _require_match(live, protocol, countries, tls)
         session = secrets.token_hex(6)
-        wishes = [f"country-{countries[0].lower()}"] if countries else []
-        if protocol != "any":
-            wishes.append(f"type-{protocol}")
+        wishes = _wishes(protocol, countries)
         pool, port = self.server.pool, self.server.port
         started = time.monotonic()
         result, faults, suspects = None, [], []

{proxy_scraper_cli-1.18.0 → proxy_scraper_cli-1.20.0}/proxyscraper/api.py

@@ -27,12 +27,14 @@ import asyncio
 import contextlib
 import email.message
 import io
+import logging
 import os
 import re
 import tempfile
 import threading
 from dataclasses import dataclass, field
 from typing import Dict, Iterable, List, Optional, Tuple
+from urllib.parse import urlsplit
 
 from rich.console import Console
 
@@ -42,6 +44,8 @@ from .parsing import PROXY_TYPES
 from .targets import parse_target
 from .ui import widgets
 
+log = logging.getLogger(__name__)
+
 __all__ = [
     "CheckResult",
     "LiveProxy",
@@ -216,7 +220,7 @@ async def live_proxies_async(*, types: Iterable[str] = PROXY_TYPES, countries: I
     limit       at most this many (0 = all)
     """
     from .agent import AgentError, LiveSource
-    from .server.pool import page_ms, typical_speed
+    from .ranking import page_ms, typical_speed
     from .sites import SITE
 
     types = _types(types)
@@ -242,8 +246,8 @@ async def live_proxies_async(*, types: Iterable[str] = PROXY_TYPES, countries: I
     found = [_live_proxy(r, data.run_hours) for r in data.rows
              if r.get("ptype") in types and isinstance(r.get("proxy"), str)]
     # fastest first – by how quickly a page comes through when the list has download speeds (#186)
-    typical = typical_speed(found)
-    found.sort(key=lambda p: page_ms(p, typical))
+    typical = typical_speed(p.speed_kbps for p in found)
+    found.sort(key=lambda p: page_ms(p.latency, p.speed_kbps, p.https, typical))
     found = [p for p in found if opts.filters.accepts(p) and (not min_uptime or (p.uptime_7d or 0) >= min_uptime)
              and all(p.sites.get(s) for s in works_on) and (not min_speed or (p.speed_kbps or 0) >= min_speed)]
     return found[:limit] if limit else found
@@ -309,6 +313,15 @@ class ProxyResponse:
             raise ConnectionError(f"the body from {self.final_url} was cut off at 2 MB")
 
 
+REFRESH_SECONDS = 300  # proxy_url(): how often the pool behind it looks for a new run's list
+
+
+def _playwright(url: str) -> Dict[str, str]:
+    parts = urlsplit(url)
+    return {"server": f"{parts.scheme}://{parts.hostname}:{parts.port}", "username": parts.username or "",
+            "password": parts.password or ""}
+
+
 class ProxyRotator:
     """Loads URLs through proxies from the hourly list and switches to the next one when a proxy fails –
     the loop every scraper writes by hand. HTTPS only through proxies with verified TLS.
@@ -330,6 +343,7 @@ class ProxyRotator:
         self._loop: Optional[asyncio.AbstractEventLoop] = None  # the sync API's own loop, in a thread
         self._thread: Optional[threading.Thread] = None
         self._start_lock = threading.Lock()
+        self._refresher: Optional[asyncio.Task] = None
 
     def _use(self, mode: str) -> None:
         if self._mode is None:
@@ -367,7 +381,43 @@ class ProxyRotator:
                              proxy_country=page["proxy_country"], elapsed_ms=page["elapsed_ms"],
                              truncated=page["body_truncated"])
 
+    async def aproxy_url(self) -> str:
+        self._use("async")
+        return await self._proxy_url()
+
+    async def _proxy_url(self) -> str:
+        from .agent import AgentError
+
+        if self._fetcher is None:
+            self._fetcher = self._make_fetcher()
+        try:
+            url = await self._fetcher.local_proxy(self.protocol, self.country)
+        except AgentError as e:
+            raise ConnectionError(str(e)) from None
+        if self._refresher is None:  # nobody calls get() on this path, so the pool refreshes by itself
+            self._refresher = asyncio.ensure_future(self._keep_fresh())
+        return url
+
+    async def aplaywright_proxy(self) -> Dict[str, str]:
+        self._use("async")
+        return _playwright(await self._proxy_url())
+
+    async def _keep_fresh(self) -> None:
+        from .agent import AgentError
+
+        while True:
+            await asyncio.sleep(REFRESH_SECONDS)
+            try:
+                await self._fetcher.refresh()
+            except AgentError:
+                pass  # GitHub not reachable right now: the pool keeps what it has, next try later
+            except Exception as e:  # a broken list must not end the refreshing for good
+                log.warning("proxy-scraper: could not refresh the proxy list: %s", e)
+
     async def aclose(self) -> None:
+        if self._refresher is not None:
+            self._refresher.cancel()
+            self._refresher = None
         if self._fetcher is not None:
             await self._fetcher.close()
             self._fetcher = None
@@ -380,13 +430,37 @@ class ProxyRotator:
 
     # sync: one event loop in a background thread keeps the proxy server alive between calls
     def get(self, url: str) -> ProxyResponse:
-        with self._start_lock:  # several threads calling get() at once still share one loop
-            self._use("sync")
+        return self._run_sync(self._aget(url))
+
+    def _run_sync(self, coro):
+        with self._start_lock:  # several threads calling at once still share one loop
+            try:
+                self._use("sync")
+            except RuntimeError:
+                coro.close()
+                raise
             if self._loop is None:
                 self._loop = asyncio.new_event_loop()
                 self._thread = threading.Thread(target=self._loop.run_forever, name="proxy-rotator", daemon=True)
                 self._thread.start()
-        return asyncio.run_coroutine_threadsafe(self._aget(url), self._loop).result()
+        return asyncio.run_coroutine_threadsafe(coro, self._loop).result()
+
+    def proxy_url(self) -> str:
+        """A local proxy URL for any library – requests, httpx, aiohttp, Playwright, Scrapy, curl:
+
+            with ProxyRotator(country="DE") as rotator:
+                proxy = rotator.proxy_url()
+                requests.get(url, proxies={"http": proxy, "https": proxy})
+
+        Every connection through it gets a proxy from the live list (best first, with failover), HTTPS only
+        through verified-TLS ones. It lives as long as the rotator and takes over each new run's list."""
+        return self._run_sync(self._proxy_url())
+
+    def playwright_proxy(self) -> Dict[str, str]:
+        """proxy_url() the way Playwright wants it – it ignores a login inside the server address:
+
+            browser = p.chromium.launch(proxy=rotator.playwright_proxy())"""
+        return _playwright(self.proxy_url())
 
     def close(self) -> None:
         with self._start_lock:

{proxy_scraper_cli-1.18.0 → proxy_scraper_cli-1.20.0}/proxyscraper/pages.py

@@ -14,6 +14,8 @@ from html import escape
 from pathlib import Path
 from typing import Callable, Dict, List, Sequence, Tuple
 
+from .ranking import best_first
+
 SITE_URL = "https://maximilianfeix.github.io/proxy-scraper/"
 REPO_URL = "https://github.com/maximilianfeix/proxy-scraper"
 ROWS_PER_PAGE = 200
@@ -203,8 +205,8 @@ def _render(path: str, title: str, what: str, rows: List[dict], updated: datetim
                 f"<td>{'yes' if r.get('https') else 'no'}</td><td>{escape(through)}</td>"
                 f"<td>{escape((r.get('org') or '–')[:40])}</td></tr>")
     body_rows = "\n".join(cells(r) for r in rows[:ROWS_PER_PAGE])
-    table = (f"<div class=\"table\"><table><caption>The {min(total, ROWS_PER_PAGE):,} fastest, as ip:port. "
-             f"The download has all {total:,} as type://ip:port.</caption>"
+    table = (f"<div class=\"table\"><table><caption>The {min(total, ROWS_PER_PAGE):,} fastest to load a page, "
+             f"as ip:port. The download has all {total:,} as type://ip:port.</caption>"
              "<thead><tr><th scope=\"col\">Proxy</th><th scope=\"col\">Type</th><th scope=\"col\">Country</th>"
              "<th scope=\"col\">Latency</th><th scope=\"col\">Speed</th><th scope=\"col\">Uptime</th>"
              "<th scope=\"col\">HTTPS</th><th scope=\"col\">Gets through</th>"
@@ -307,7 +309,7 @@ def _nav(specs, counts: Dict[str, int], current: str, root: str) -> str:
 def write_pages(rows: List[dict], out: Path, updated: datetime, extra: Sequence[str] = ()) -> List[str]:
     """Writes every page with its proxies.txt and the sitemap (plus the `extra` paths, e.g. the proxy pages)
     -> the paths listed in the sitemap."""
-    rows = sorted(rows, key=lambda r: r["latency"])
+    rows = best_first(rows)
     specs = page_specs()
     selected = {p: [r for r in rows if keep(r)] for p, _, _, keep in specs}
     counts = {p: len(v) for p, v in selected.items()}

{proxy_scraper_cli-1.18.0 → proxy_scraper_cli-1.20.0}/proxyscraper/publish.py

@@ -25,6 +25,7 @@ from .charts import THEMES, countries_svg, trend_svg
 from .pages import write_pages
 from .parsing import PROXY_TYPES
 from .proxypages import write_proxy_pages
+from .ranking import best_first
 from .report import weekly_facts, write_report
 from .sites import SITES
 
@@ -43,7 +44,7 @@ def is_public(row: dict) -> bool:
 
 def load_rows(run_dir: Path) -> List[dict]:
     rows = json.loads((run_dir / "proxies.json").read_text(encoding="utf-8"))
-    return sorted((r for r in rows if is_public(r)), key=lambda r: r["latency"])
+    return best_first(r for r in rows if is_public(r))
 
 
 def num(n: int) -> str:
@@ -116,7 +117,8 @@ def readme(stats: dict, counts: Dict[str, int]) -> str:
     return f"""# Live proxy list
 
 Generated automatically by [proxy-scraper](https://github.com/maximilianfeix/proxy-scraper) with GitHub Actions.
-Every proxy here really worked in the last run – sorted by latency, fastest first.
+Every proxy here really worked in the last run – fastest first, by first answer plus the download speed measured
+right after the run.
 Browse and filter it on the [website](https://maximilianfeix.github.io/proxy-scraper/).
 
 **Updated:** {updated} · **{num(stats["total"])} proxies** · median latency {num(stats["median_latency"])} ms
@@ -280,10 +282,10 @@ def publish(run_dir: Path, out: Path, minimum: int = 20, now: Optional[datetime]
         base = list(reader.fieldnames or ["proxy"])
         writer = csv.DictWriter(dst, fieldnames=[*base, *(k for k in extra if k not in base)])
         writer.writeheader()
-        by_url = {r["url"]: r for r in rows}
-        for r in reader:
-            if is_public(r):
-                listed_row = by_url.get(r.get("url", ""), {})
+        csv_rows = {r.get("url", ""): r for r in reader if is_public(r)}
+        for listed_row in rows:  # in the order of proxies.json: best first
+            r = csv_rows.get(listed_row.get("url", ""))
+            if r is not None:
                 works = " ".join(name for name, ok in (listed_row.get("sites") or {}).items() if ok)
                 writer.writerow({**r, **{k: listed_row.get(k, "") for k in extra}, "works_with": works})
     stats = stats_for(rows, now)

proxy_scraper_cli-1.20.0/proxyscraper/ranking.py

@@ -0,0 +1,44 @@
+"""Which proxy is "fastest": how quickly a page comes through, not just the first answer (#186).
+
+A quick answer to one tiny request says little about loading a page – in a test on the live list, the 25 HTTPS
+proxies quickest to answer loaded 4 of 50 real pages, the first 25 ranked by answer plus the download speed of the
+hourly speed step 26 of 50. Everything that orders proxies of the live list uses this: the published lists, the
+pages, --pick, live_proxies, the rotating pool.
+"""
+
+from __future__ import annotations
+
+import statistics
+from typing import Iterable, List, Optional
+
+PAGE_KB = 100               # roughly one page – the download the speed step measures
+UNMEASURED_KBPS = 20        # tried in the speed step, no download got through: counted as slow (it passed the checks)
+
+
+def page_ms(latency: float, speed_kbps: Optional[int], https: Optional[bool], typical_kbps: Optional[float]) -> float:
+    """Roughly how long a page takes through a proxy: the first answer plus the download.
+
+    typical_kbps is the median measured speed of the set, None when it has no speeds at all (an own scan) – then
+    latency decides alone. The speed step only tries HTTPS-capable proxies: one of those without a speed didn't
+    get the download through, the others were never tried and count as typical."""
+    if not typical_kbps:
+        return latency
+    kbps = speed_kbps or (UNMEASURED_KBPS if https else typical_kbps)
+    return latency + PAGE_KB * 1000 / kbps
+
+
+def typical_speed(speeds: Iterable[Optional[int]]) -> Optional[float]:
+    measured = [s for s in speeds if s]
+    return statistics.median(measured) if measured else None
+
+
+def best_first(rows: Iterable[dict]) -> List[dict]:
+    """Rows of a run (dicts as in proxies.json), the fastest to load a page first."""
+    rows = list(rows)
+    typical = typical_speed(_speed(r) for r in rows)
+    return sorted(rows, key=lambda r: page_ms(r["latency"], _speed(r), r.get("https") is True, typical))
+
+
+def _speed(row: dict) -> Optional[int]:
+    value = row.get("speed_kbps")
+    return value if type(value) is int and value > 0 else None

{proxy_scraper_cli-1.18.0 → proxy_scraper_cli-1.20.0}/proxyscraper/server/pool.py

@@ -13,36 +13,26 @@ from __future__ import annotations
 
 import random
 import re
-import statistics
 import time
 from dataclasses import dataclass
 from typing import Callable, Dict, List, Optional, Set, Tuple
 
+from .. import ranking
 from ..checker import CheckResult
 
 DISABLE_AFTER = 3           # this many failures in a row -> out of the rotation
 STRATEGIES = ("weighted", "random", "round-robin", "fastest")
 SESSION_SECONDS = 600       # how long a session (session-…) keeps its proxy when --sticky isn't set
 _TOKEN_RE = re.compile(r"(country|type|session)[-_]([A-Za-z0-9]+)")
-PAGE_KB = 100               # roughly one page – the download the speed step measures
-UNMEASURED_KBPS = 20        # tried in the speed step, no download got through: counted as slow (it passed the checks)
 
 
 def page_ms(result, typical_kbps: Optional[float]) -> float:
-    """Roughly how long a page takes through this proxy: the first answer plus the download.
-
-    typical_kbps is the median measured speed of the set, None when it has no speeds at all (an own scan) – then
-    latency decides alone. The speed step only tries HTTPS-capable proxies: one of those without a speed didn't
-    get the download through, the others were never tried and count as typical."""
-    if not typical_kbps:
-        return result.latency
-    kbps = result.speed_kbps or (UNMEASURED_KBPS if result.https else typical_kbps)
-    return result.latency + PAGE_KB * 1000 / kbps
+    """ranking.page_ms for a CheckResult (or anything with latency, speed_kbps and https)."""
+    return ranking.page_ms(result.latency, result.speed_kbps, result.https, typical_kbps)
 
 
 def typical_speed(results) -> Optional[float]:
-    speeds = [r.speed_kbps for r in results if r.speed_kbps]
-    return statistics.median(speeds) if speeds else None
+    return ranking.typical_speed(r.speed_kbps for r in results)
 
 
 @dataclass(frozen=True)

{proxy_scraper_cli-1.18.0 → proxy_scraper_cli-1.20.0}/proxyscraper/site/index.html

@@ -597,7 +597,7 @@
   <section class="block wrap" id="list" aria-labelledby="list-title">
     <div class="head-row">
       <h2 class="h2" id="list-title">Pick your proxies</h2>
-      <p class="intro-text">Filter, sort, then copy or download exactly the ones you need. Click any address to copy it.</p>
+      <p class="intro-text">Best first: the ones that answer quickly and download fast. Filter, sort, then copy or download exactly the ones you need. Click any address to copy it.</p>
     </div>
     <div class="filters" role="group" aria-label="Filters">
       <button class="chip" type="button" data-type="http" aria-pressed="true"><span class="dot http"></span>HTTP</button>
@@ -623,7 +623,7 @@
             <th scope="col" data-key="ptype"><button class="sort" type="button" aria-label="Type"><span class="name">Type</span></button></th>
             <th scope="col" data-key="proxy"><button class="sort" type="button">Proxy</button></th>
             <th scope="col" data-key="country"><button class="sort" type="button">Country</button></th>
-            <th scope="col" data-key="latency" aria-sort="ascending"><button class="sort" type="button">Latency</button></th>
+            <th scope="col" data-key="latency"><button class="sort" type="button">Latency</button></th>
             <th scope="col" data-key="speed_kbps" class="hide-sm"><button class="sort" type="button" title="Download speed in the last run – 100 KB through the proxy">Speed</button></th>
             <th scope="col" data-key="https" class="hide-sm"><button class="sort" type="button">HTTPS</button></th>
             <th scope="col" data-key="uptime_7d" class="hide-sm"><button class="sort" type="button" title="Share of this week's runs it was on the list">Uptime</button></th>
@@ -637,6 +637,7 @@
     <div class="list-foot">
       <span class="count" id="count" aria-live="polite"></span>
       <div class="actions">
+        <button class="btn small" id="best" type="button" hidden title="The ones that answer quickly and download fast">Best first</button>
         <button class="btn small" id="more" type="button" hidden>Show more</button>
         <button class="btn small" id="share" type="button" title="The filters are in the address, so the link opens this exact view">Share view</button>
         <button class="btn small" id="copy-all" type="button">Copy filtered</button>
@@ -871,7 +872,7 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
   const speed = (k) => !k ? '<span class="quiet">–</span>'
     : `<span class="${k >= 500 ? "fast" : k >= 100 ? "mid" : "slow"}">${k >= 1024 ? (k / 1024).toFixed(1) + " MB/s" : fmt(k) + " KB/s"}</span>`;
   const latency = (ms) => `<span class="${ms < 1000 ? "fast" : ms < 3000 ? "mid" : "slow"}">${fmt(ms)} ms</span>`;
-  const state = { rows: [], shown: PAGE, sort: "latency", dir: 1, types: new Set(["http", "socks4", "socks5"]),
+  const state = { rows: [], shown: PAGE, sort: "rank", dir: 1, types: new Set(["http", "socks4", "socks5"]),
                   https: false, elite: false, noDc: false, noBl: false, stable: false, country: "", site: "", maxLat: 8000, q: "" };
 
   // Theme
@@ -1037,6 +1038,7 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
       || '<tr><td colspan="9" class="empty">No proxy matches these filters. Try a higher latency limit or fewer filters.</td></tr>';
     $("#count").textContent = rows.length ? `Showing ${fmt(Math.min(state.shown, rows.length))} of ${fmt(rows.length)} matching proxies` : "";
     $("#more").hidden = rows.length <= state.shown;
+    $("#best").hidden = state.sort === "rank";
     saveView();
     $$("th[data-key]").forEach(th => th.dataset.key === state.sort ? th.setAttribute("aria-sort", state.dir > 0 ? "ascending" : "descending") : th.removeAttribute("aria-sort"));
   }
@@ -1052,7 +1054,7 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
     if (state.site) q.set("site", state.site);
     if (state.maxLat < 8000) q.set("latency", state.maxLat);
     if (state.q.trim()) q.set("q", state.q.trim());
-    if (state.sort !== "latency" || state.dir !== 1) q.set("sort", (state.dir < 0 ? "-" : "") + state.sort);
+    if (state.sort !== "rank") q.set("sort", (state.dir < 0 ? "-" : "") + state.sort);
     return location.pathname + (q.toString() ? `?${q}` : "") + location.hash;
   }
   // debounced: browsers limit replaceState (Safari throws after ~100 calls in 30 s, e.g. while dragging the slider)
@@ -1117,6 +1119,7 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
   $("#f-lat").oninput = (e) => { state.maxLat = +e.target.value; $("#f-lat-v").textContent = state.maxLat >= 8000 ? "any" : `${fmt(state.maxLat)} ms`; refresh(); };
   $("#f-q").oninput = (e) => { state.q = e.target.value; refresh(); };
   $("#more").onclick = () => { state.shown += PAGE * 4; render(); };
+  $("#best").onclick = () => { state.sort = "rank"; state.dir = 1; morph(render); };
   $$("th[data-key]").forEach(th => {
     th.querySelector(".sort").onclick = () => { const k = th.dataset.key; state.dir = state.sort === k ? -state.dir : 1; state.sort = k; morph(render); };
   });
@@ -1353,6 +1356,7 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
 
   let started = false;
   function apply(rows, st, history) {
+    rows.forEach((r, i) => { r.rank = i; });  // proxies.json comes best first: first answer plus download speed
     state.rows = rows;
     shownRun = st.updated;
     RUN_HOURS = st.run_hours || 6;  // lists from before run_hours existed ran every 6 hours

{proxy_scraper_cli-1.18.0 → proxy_scraper_cli-1.20.0}/tests/test_rotator.py

@@ -127,3 +127,95 @@ def test_mixing_sync_and_async_is_refused(monkeypatch):
     with pytest.raises(RuntimeError, match="async"):
         asyncio.run(rotator.aget("http://x/"))
     rotator.close()
+
+
+def get_through(proxy_url: str, url: str):
+    """A plain GET through an HTTP proxy URL with a login, the way requests or httpx send it."""
+    import base64
+    import http.client
+    from urllib.parse import urlsplit
+
+    p = urlsplit(proxy_url)
+    login = base64.b64encode(f"{p.username}:{p.password}".encode()).decode()
+    conn = http.client.HTTPConnection(p.hostname, p.port, timeout=10)
+    conn.request("GET", url, headers={"Proxy-Authorization": f"Basic {login}"})
+    response = conn.getresponse()
+    return response.status, response.read()
+
+
+def test_proxy_url_is_a_rotating_proxy_for_any_library():
+    loop = asyncio.new_event_loop()
+    target_srv, proxy_srv, port, rows = loop.run_until_complete(local_setup())
+    import threading
+    thread = threading.Thread(target=loop.run_forever, daemon=True)
+    thread.start()
+    try:
+        with ProxyRotator(_source=source_for(rows), _allow_private=True, timeout=5) as rotator:
+            url = rotator.proxy_url()
+            assert url.startswith("http://") and "@127.0.0.1:" in url
+            # the dead proxy is tried and skipped inside, the client just gets its page
+            assert get_through(url, f"http://127.0.0.1:{port}/ok") == (200, b"ok")
+            assert rotator.proxy_url() == url  # the same server for the rotator's lifetime
+    finally:
+        loop.call_soon_threadsafe(target_srv.close)
+        loop.call_soon_threadsafe(proxy_srv.close)
+        loop.call_soon_threadsafe(loop.stop)
+
+
+def test_proxy_url_carries_country_and_protocol_and_refuses_what_isnt_there():
+    async def go(**wishes):
+        rows = [row(1, country="DE", ptype="socks5")]
+        async with ProxyRotator(_source=source_for(rows), _allow_private=True, **wishes) as rotator:
+            return await rotator.aproxy_url()
+    assert "//country-de-type-socks5:" in asyncio.run(go(country="de", protocol="socks5"))
+
+    async def for_playwright():
+        async with ProxyRotator(_source=source_for([row(1)]), _allow_private=True) as rotator:
+            return await rotator.aproxy_url(), await rotator.aplaywright_proxy()
+    url, settings = asyncio.run(for_playwright())
+    # Playwright ignores a login inside "server": it wants the parts on their own
+    assert settings["server"] == "http://" + url.rsplit("@", 1)[1] and "@" not in settings["server"]
+    assert url == f"http://{settings['username']}:{settings['password']}@{settings['server'][7:]}"
+    with pytest.raises(ConnectionError, match="JP"):
+        asyncio.run(go(country="JP"))
+
+
+def test_proxy_url_keeps_the_pool_fresh(monkeypatch):
+    from proxyscraper import api
+
+    monkeypatch.setattr(api, "REFRESH_SECONDS", 0.05)
+    runs = iter([agent.LiveList(rows=[row(1)], stats={**STATS, "updated": "run 1"}, loaded_at=0.0),
+                 agent.LiveList(rows=[row(2)], stats={**STATS, "updated": "run 2"}, loaded_at=0.0)])
+
+    class Source:  # the first call sees run 1, every later one run 2
+        current = None
+
+        async def get(self):
+            self.current = next(runs, self.current)
+            return self.current
+
+    async def go():
+        async with ProxyRotator(_source=Source(), _allow_private=True) as rotator:
+            await rotator.aproxy_url()
+            await asyncio.sleep(0.3)
+            return {e.result.proxy for e in rotator._fetcher.server.pool.entries}
+    assert asyncio.run(go()) == {row(2)["proxy"]}
+
+
+def test_the_refresh_survives_a_surprise(monkeypatch):
+    from proxyscraper import api
+
+    monkeypatch.setattr(api, "REFRESH_SECONDS", 0.02)
+    calls = []
+
+    async def go():
+        async with ProxyRotator(_source=source_for([row(1)]), _allow_private=True) as rotator:
+            await rotator.aproxy_url()
+
+            async def broken():
+                calls.append(1)
+                raise KeyError("stats")
+            rotator._fetcher.refresh = broken
+            await asyncio.sleep(0.2)
+    asyncio.run(go())
+    assert len(calls) > 2  # kept trying after the first failure

{proxy_scraper_cli-1.18.0 → proxy_scraper_cli-1.20.0}/tests/test_speed_ranking.py

@@ -56,3 +56,53 @@ def test_proxies_the_speed_step_never_tried_count_as_typical_not_slow():
     pool = ProxyPool([plain_only, measured], strategy="fastest")
     assert pool.pick(set()).result is plain_only
     assert pool.pick(set(), tls=True).result is measured
+
+
+def published(n, latency, speed=None, https=True, ptype="http"):
+    row = {"proxy": f"10.0.0.{n}:80", "ptype": ptype, "url": f"{ptype}://10.0.0.{n}:80", "latency": latency,
+           "https": https, "country": "DE", "exit_ip": f"10.0.0.{n}", "anonymity": "elite"}
+    return {**row, "speed_kbps": speed} if speed else row
+
+
+def test_the_published_lists_start_with_the_ones_that_load_pages_fastest(tmp_path):
+    import json
+
+    from proxyscraper import publish
+
+    (tmp_path / "proxies.json").write_text(json.dumps([published(1, 100), published(2, 400, speed=500)]))
+    assert [r["proxy"] for r in publish.load_rows(tmp_path)] == ["10.0.0.2:80", "10.0.0.1:80"]
+
+
+def test_landing_pages_list_the_best_first(tmp_path):
+    from datetime import datetime, timezone
+
+    from proxyscraper import pages
+
+    pages.write_pages([published(1, 100), published(2, 400, speed=500)], tmp_path, datetime.now(timezone.utc))
+    assert (tmp_path / "country" / "de" / "proxies.txt").read_text().splitlines()[0] == "http://10.0.0.2:80"
+
+
+def test_the_csv_comes_in_the_same_order_as_the_json(tmp_path):
+    import csv
+    import json
+
+    from proxyscraper import publish
+    from proxyscraper.output import ResultWriter
+
+    results = [CheckResult(f"http 1.1.1.{i}:80", "http", f"1.1.1.{i}:80", 100 * (i + 1), "9.9.9.9", True, "elite", "DE")
+               for i in range(3)]
+    ResultWriter(run_dir=tmp_path / "run").finalize(results)
+    rows = json.loads((tmp_path / "run" / "proxies.json").read_text())
+    rows[2]["speed_kbps"] = 900  # the slowest to answer downloads fastest
+    (tmp_path / "run" / "proxies.json").write_text(json.dumps(rows))
+    publish.publish(tmp_path / "run", tmp_path / "out", minimum=1)
+    order = [r["proxy"] for r in json.loads((tmp_path / "out" / "proxies.json").read_text())]
+    with (tmp_path / "out" / "proxies.csv").open(newline="") as fh:
+        assert [r["proxy"] for r in csv.DictReader(fh)] == order and order[0] == "1.1.1.2:80"
+
+
+def test_the_mcp_tools_list_best_first_too():
+    from proxyscraper.agent import select
+
+    rows = [published(1, 100), published(2, 400, speed=500)]
+    assert [r["proxy"] for r in select(rows)] == ["10.0.0.2:80", "10.0.0.1:80"]