proxy-scraper-cli 1.7.1__py3-none-any.whl

proxyscraper/server/pool.py

@@ -0,0 +1,195 @@
+"""The pool of proxies that were found: selection, scoring and who drops out of the rotation.
+
+Selection per connection in three steps:
+  1. filters from the request (Selection): country, type – via the user name, e.g. "country-de-type-socks5"
+  2. sticky: the same session (or with --sticky the same target site) keeps the same proxy for a while
+  3. strategy for everything else: weighted (default), random, round-robin, fastest
+"""
+
+from __future__ import annotations
+
+import random
+import re
+import time
+from dataclasses import dataclass
+from typing import Callable, Dict, List, Optional, Set, Tuple
+
+from ..checker import CheckResult
+
+DISABLE_AFTER = 3           # this many failures in a row -> out of the rotation
+STRATEGIES = ("weighted", "random", "round-robin", "fastest")
+SESSION_SECONDS = 600       # how long a session (session-…) keeps its proxy when --sticky isn't set
+_TOKEN_RE = re.compile(r"(country|type|session)[-_]([A-Za-z0-9]+)")
+
+
+@dataclass(frozen=True)
+class Selection:
+    """The client's wishes for this connection – they come from the user name of the proxy login."""
+    country: str = ""
+    ptype: str = ""
+    session: str = ""
+
+    @classmethod
+    def from_username(cls, username: str) -> "Selection":
+        """'country-de-session-abc' -> Selection(country='DE', session='abc'). Unknown parts are ignored."""
+        found = dict(_TOKEN_RE.findall(username or ""))
+        ptype = found.get("type", "").lower()
+        return cls(country=found.get("country", "").upper()[:2],
+                   ptype=ptype if ptype in ("http", "socks4", "socks5") else "",
+                   session=found.get("session", "")[:64])
+
+    def __bool__(self) -> bool:
+        return bool(self.country or self.ptype or self.session)
+
+    def describe(self) -> str:
+        parts = [self.country, self.ptype, f"session {self.session}" if self.session else ""]
+        return " · ".join(p for p in parts if p)
+
+
+ANY = Selection()  # no special wishes
+
+
+@dataclass
+class PoolEntry:
+    result: CheckResult
+    ok: int = 0
+    fail: int = 0
+    fail_streak: int = 0
+    active: int = 0
+    disabled: bool = False
+
+    @property
+    def weight(self) -> float:
+        # prefer fast and proven ones, but give everyone a chance
+        reliability = (self.ok + 1) / (self.ok + self.fail + 2)
+        return reliability / (self.result.latency + 300)
+
+
+class ProxyPool:
+    def __init__(self, results: List[CheckResult], rng: Optional[random.Random] = None,
+                 strategy: str = "weighted", sticky_seconds: float = 0,
+                 clock: Callable[[], float] = time.monotonic, strict_tls: bool = False):
+        if strategy not in STRATEGIES:
+            raise ValueError(f"unknown strategy {strategy!r} (possible: {', '.join(STRATEGIES)})")
+        self.entries = [PoolEntry(r) for r in sorted(results, key=lambda r: r.latency)]
+        self.rng = rng or random.Random()
+        self.strategy = strategy
+        self.sticky_seconds = sticky_seconds
+        self.clock = clock
+        # TLS only through proxies that passed the verified-TLS test, even when all of those are down –
+        # --serve falls back to the rest then, a client that promised verified TLS (agent.PageFetcher) doesn't
+        self.strict_tls = strict_tls
+        self._sticky: Dict[str, Tuple[PoolEntry, float]] = {}  # session/target site -> (proxy, valid until)
+        self._next = 0  # for round-robin
+
+    @property
+    def usable(self) -> List[PoolEntry]:
+        return [e for e in self.entries if not e.disabled]
+
+    @property
+    def tls_capable(self) -> List[PoolEntry]:
+        return [e for e in self.usable if e.result.https]
+
+    def pick(self, exclude: Set[str], tls: bool = False, selection: Selection = ANY,
+             target: str = "") -> Optional[PoolEntry]:
+        """Next proxy for a connection to `target` (host:port). None = none fits (any more).
+
+        For TLS only proxies that passed the HTTPS test (verified TLS) – others often break up the
+        encryption. If there are none, all of them. Country and type wishes are strict, though:
+        whoever asks for "country-de" would rather get an error than a proxy from another country."""
+        candidates = [e for e in self.usable if e.result.key not in exclude and self._matches(e, selection)]
+        if tls and (self.strict_tls or any(e.result.https for e in self.usable if self._matches(e, selection))):
+            candidates = [e for e in candidates if e.result.https]
+        if not candidates:
+            return None
+        sticky_key = self._sticky_key(selection, target)
+        if sticky_key:
+            held = self._sticky.get(sticky_key)
+            if held and held[1] > self.clock() and held[0] in candidates:
+                return held[0]
+        entry = self._choose(candidates)
+        if sticky_key:
+            self._sticky[sticky_key] = (entry, self.clock() + (self.sticky_seconds or SESSION_SECONDS))
+            if len(self._sticky) > 10_000:  # don't collect old entries forever
+                now = self.clock()
+                self._sticky = {k: v for k, v in self._sticky.items() if v[1] > now}
+        return entry
+
+    def _sticky_key(self, selection: Selection, target: str) -> str:
+        if selection.session:
+            return f"session:{selection.session}"
+        if self.sticky_seconds and target:
+            return f"target:{target}"
+        return ""
+
+    @staticmethod
+    def _matches(entry: PoolEntry, selection: Selection) -> bool:
+        r = entry.result
+        return (not selection.country or r.country == selection.country) and \
+            (not selection.ptype or r.ptype == selection.ptype)
+
+    def _choose(self, candidates: List[PoolEntry]) -> PoolEntry:
+        if self.strategy == "random":
+            return self.rng.choice(candidates)
+        if self.strategy == "fastest":
+            # the fastest free one; if all are busy, the least busy one (then the faster one)
+            idle = [e for e in candidates if not e.active]
+            if idle:
+                return min(idle, key=lambda e: (e.result.latency, -e.weight))
+            return min(candidates, key=lambda e: (e.active, e.result.latency))
+        if self.strategy == "round-robin":
+            ordered = sorted(candidates, key=lambda e: e.result.key)
+            entry = ordered[self._next % len(ordered)]
+            self._next += 1
+            return entry
+        return self.rng.choices(candidates, weights=[e.weight for e in candidates])[0]
+
+    def report(self, entry: PoolEntry, ok: bool) -> None:
+        if ok:
+            entry.ok += 1
+            entry.fail_streak = 0
+        else:
+            entry.fail += 1
+            entry.fail_streak += 1
+            if entry.fail_streak >= DISABLE_AFTER:
+                entry.disabled = True
+                # if a session holds this proxy, it should get a different one next time
+                self._sticky = {k: v for k, v in self._sticky.items() if v[0] is not entry}
+
+    def session_entry(self, session: str) -> Optional[PoolEntry]:
+        """The proxy that currently serves a session (username session-NAME) – after failover, the one that worked."""
+        held = self._sticky.get(f"session:{session}")
+        return held[0] if held else None
+
+    def retire(self, entry: PoolEntry) -> None:
+        """Out of the rotation right away (e.g. it dropped a download halfway) – until a merge brings it back."""
+        entry.disabled = True
+        self._sticky = {k: v for k, v in self._sticky.items() if v[0] is not entry}
+
+    def merge(self, results: List[CheckResult], drop_missing: bool = False) -> int:
+        """Hits of a refill: new ones join the rotation, known ones that were disabled come back with the fresh
+        result, disabled ones that didn't pass again are dropped. Counters of known proxies stay. -> added.
+
+        drop_missing: `results` is the whole truth (a new live list) – whatever isn't in it goes, working or not.
+        A refill doesn't set it: it skips the proxies that are serving, so their absence means nothing."""
+        fresh = {r.key: r for r in results}
+        kept, added = [], 0
+        for entry in self.entries:
+            r = fresh.pop(entry.result.key, None)
+            if r is not None:
+                entry.result = r
+                if entry.disabled:
+                    self.revive(entry)
+            elif entry.disabled or drop_missing:
+                continue  # dead (or gone from the list) and not in the fresh results – make room
+            kept.append(entry)
+        for r in fresh.values():
+            kept.append(PoolEntry(r))
+            added += 1
+        self.entries = sorted(kept, key=lambda e: e.result.latency)
+        return added
+
+    def revive(self, entry: PoolEntry) -> None:
+        """A disabled proxy passed the recheck – back into the rotation."""
+        entry.disabled = False
+        entry.fail_streak = 0

proxyscraper/server/socks.py

@@ -0,0 +1,65 @@
+"""SOCKS5 on the client side: greeting, optional authentication (RFC 1929) and accepting CONNECT."""
+
+from __future__ import annotations
+
+import hmac
+import socket
+from typing import Tuple
+
+SOCKS5_VERSION = b"\x05"
+NO_AUTH, USER_PASS, NO_METHOD = 0x00, 0x02, 0xFF
+CMD_CONNECT = 0x01
+ATYP_IPV4, ATYP_DOMAIN, ATYP_IPV6 = 0x01, 0x03, 0x04
+
+
+class Socks5Refused(Exception):
+    """The client doesn't speak usable SOCKS5 or wants something other than CONNECT – the reply is already sent."""
+
+
+def socks5_reply(code: int) -> bytes:
+    """Reply to the CONNECT request. 0x00 = ok, 0x04 = host unreachable, 0x07 = command not supported."""
+    return bytes([5, code, 0, ATYP_IPV4]) + b"\x00" * 6
+
+
+async def socks5_accept(reader, writer, password: str = "") -> Tuple[str, int, str]:
+    """After the first byte (0x05): negotiate methods, authenticate if needed, read CONNECT.
+
+    -> (target host, target port, user name). The user name carries wishes like "country-de" (see
+    pool.Selection). With `password` set, username/password auth is mandatory and the password must match."""
+    count = (await reader.readexactly(1))[0]
+    methods = await reader.readexactly(count)
+    username = ""
+    if USER_PASS in methods:
+        writer.write(bytes([5, USER_PASS]))
+        await writer.drain()
+        await reader.readexactly(1)  # version of the sub-negotiation
+        username = (await reader.readexactly((await reader.readexactly(1))[0])).decode("utf-8", "replace")
+        given = await reader.readexactly((await reader.readexactly(1))[0])
+        if password and not hmac.compare_digest(given, password.encode()):
+            writer.write(b"\x01\x01")
+            await writer.drain()
+            raise Socks5Refused("wrong password")
+        writer.write(b"\x01\x00")
+    elif NO_AUTH in methods and not password:
+        writer.write(bytes([5, NO_AUTH]))
+    else:
+        writer.write(bytes([5, NO_METHOD]))
+        await writer.drain()
+        raise Socks5Refused("no supported method")
+    await writer.drain()
+
+    version, command, _, atyp = await reader.readexactly(4)
+    if atyp == ATYP_IPV4:
+        host = socket.inet_ntoa(await reader.readexactly(4))
+    elif atyp == ATYP_DOMAIN:
+        host = (await reader.readexactly((await reader.readexactly(1))[0])).decode("idna")
+    elif atyp == ATYP_IPV6:
+        host = socket.inet_ntop(socket.AF_INET6, await reader.readexactly(16))
+    else:
+        host = ""
+    port = int.from_bytes(await reader.readexactly(2), "big")
+    if version != 5 or command != CMD_CONNECT or not host or not port:
+        writer.write(socks5_reply(0x07))
+        await writer.drain()
+        raise Socks5Refused("CONNECT only")
+    return host, port, username

proxyscraper/server/status.py

@@ -0,0 +1,108 @@
+"""Status as JSON (/__proxy-scraper/status), metrics for Prometheus (/__proxy-scraper/metrics) and
+wishes from the proxy login."""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import hmac
+import json
+import time
+from statistics import median
+from typing import Any, Dict, List, Optional, Tuple
+
+from ..parsing import PROXY_TYPES
+from ..ui.widgets import shown_proxy
+from .pool import Selection
+
+STATUS_PREFIX = b"GET /__proxy-scraper/"
+STATUS_PATH = b"/__proxy-scraper/status"
+METRICS_PATH = b"/__proxy-scraper/metrics"
+METRICS_TYPE = b"text/plain; version=0.0.4; charset=utf-8"
+
+
+def basic_credentials(headers: List[Tuple[bytes, bytes]], header: bytes = b"proxy-authorization"
+                      ) -> Optional[Tuple[str, str]]:
+    """Basic auth from the given header -> (user, password), None if missing or broken."""
+    for name, value in headers:
+        if name.lower() == header and value[:6].lower() == b"basic ":
+            try:
+                decoded = base64.b64decode(value[6:].strip(), validate=True).decode("utf-8", "replace")
+            except (binascii.Error, ValueError):
+                return None
+            user, _, password = decoded.partition(":")
+            return user, password
+    return None
+
+
+def password_ok(headers: List[Tuple[bytes, bytes]], password: str, header: bytes = b"proxy-authorization") -> bool:
+    """No password configured, or the Basic auth in `header` carries exactly this password."""
+    if not password:
+        return True
+    creds = basic_credentials(headers, header)
+    return creds is not None and hmac.compare_digest(creds[1].encode(), password.encode())
+
+
+def selection_from_headers(headers: List[Tuple[bytes, bytes]]) -> Selection:
+    """Proxy-Authorization: Basic base64("country-de-session-abc:anything") -> Selection."""
+    creds = basic_credentials(headers)
+    return Selection.from_username(creds[0]) if creds else Selection()
+
+
+def status_json(server: Any) -> str:
+    """server: the RotatingServer (not imported, otherwise core and status would depend on each other in a circle)."""
+    st, pool = server.stats, server.pool
+    entries = sorted(pool.entries, key=lambda e: (e.disabled, -e.ok, e.result.latency))
+    payload = {
+        "listening": f"{server.host}:{server.port}",
+        "uptime_seconds": round(time.perf_counter() - st.started),
+        "strategy": pool.strategy,
+        "sticky_seconds": pool.sticky_seconds,
+        "requests": {"total": st.requests, "ok": st.ok, "failed": st.failed, "active": st.active},
+        "bytes": {"up": st.bytes_up, "down": st.bytes_down},
+        "pool": {"total": len(pool.entries), "usable": len(pool.usable), "https": len(pool.tls_capable),
+                 "revived": server.revived, "refilled": server.refilled,
+                 "last_refill": server.last_refill and round(server.last_refill)},
+        "proxies": [
+            {"url": f"{e.result.ptype}://{shown_proxy(e.result.proxy)}", "country": e.result.country,
+             "latency_ms": e.result.latency, "https": e.result.https, "ok": e.ok, "fail": e.fail,
+             "disabled": e.disabled, "active": e.active}
+            for e in entries[:100]
+        ],
+    }
+    return json.dumps(payload, indent=1, ensure_ascii=False)
+
+
+def metrics_text(server: Any) -> str:
+    """Prometheus text format, without a dependency – e.g. for Grafana when the server runs permanently."""
+    st, pool = server.stats, server.pool
+    out: List[str] = []
+
+    def metric(name: str, kind: str, help_text: str, samples: Dict[str, float]) -> None:
+        out.append(f"# HELP proxy_scraper_{name} {help_text}")
+        out.append(f"# TYPE proxy_scraper_{name} {kind}")
+        out.extend(f"proxy_scraper_{name}{labels} {value:g}" for labels, value in samples.items())
+
+    metric("uptime_seconds", "gauge", "Seconds since the server started.",
+           {"": round(time.perf_counter() - st.started)})
+    metric("requests_total", "counter", "Requests handled, by result.",
+           {'{result="ok"}': st.ok, '{result="failed"}': st.failed})
+    metric("requests_active", "gauge", "Requests in progress.", {"": st.active})
+    metric("bytes_total", "counter", "Bytes relayed, by direction.",
+           {'{direction="up"}': st.bytes_up, '{direction="down"}': st.bytes_down})
+    metric("revived_total", "counter", "Disabled proxies that passed a re-check.", {"": server.revived})
+
+    proxies, uses, latency = {}, {}, {}
+    for t in PROXY_TYPES:
+        entries = [e for e in pool.entries if e.result.ptype == t]
+        usable = [e for e in entries if not e.disabled]
+        proxies[f'{{type="{t}",state="usable"}}'] = len(usable)
+        proxies[f'{{type="{t}",state="disabled"}}'] = len(entries) - len(usable)
+        uses[f'{{type="{t}",result="ok"}}'] = sum(e.ok for e in entries)
+        uses[f'{{type="{t}",result="failed"}}'] = sum(e.fail for e in entries)
+        if usable:
+            latency[f'{{type="{t}"}}'] = median(e.result.latency for e in usable)
+    metric("pool_proxies", "gauge", "Proxies in the pool, by type and state.", proxies)
+    metric("proxy_uses_total", "counter", "Upstream attempts, by proxy type and result.", uses)
+    metric("pool_latency_median_ms", "gauge", "Median check latency of usable proxies.", latency)
+    return "\n".join(out) + "\n"

proxyscraper/server/upstream.py

@@ -0,0 +1,119 @@
+"""Connect to the target through a proxy from the pool (HTTP CONNECT, SOCKS4, SOCKS5)."""
+
+from __future__ import annotations
+
+import asyncio
+import ipaddress
+import socket
+
+from ..handshake import (
+    ProxyRefused,
+    parse_endpoint,
+    socks4_connect,
+    socks5_connect,
+    socks5_domain,
+    socks5_ipv4,
+    stream_io,
+    with_proxy_auth,
+)
+from .pool import PoolEntry
+
+
+class UpstreamError(Exception):
+    """The chosen proxy didn't establish the connection – next attempt."""
+
+
+class TargetError(UpstreamError):
+    """The proxy answered but couldn't open the connection to the target (refused tunnel, 502/504, target
+    not resolvable). That may be the target's fault, so it only counts against the proxy if another
+    proxy reaches the same target."""
+
+
+class Unsupported(UpstreamError):
+    """This proxy type can't do this target at all (SOCKS4 and IPv6). Never counts against the proxy."""
+
+
+GATEWAY_ERRORS = (b"502", b"503", b"504")
+
+
+async def open_upstream(entry: PoolEntry, host: str, port: int, timeout: float, tunnel: bool = True,
+                        public_only: bool = False):
+    """Connection through the proxy to host:port. tunnel=False means: HTTP upstream in forwarding mode
+    (classic proxy request without CONNECT). Raises UpstreamError if the proxy doesn't play along."""
+    r = entry.result
+    ep = parse_endpoint(r.proxy)
+    try:
+        reader, writer = await asyncio.wait_for(asyncio.open_connection(ep.host, ep.port), timeout)
+    except (OSError, asyncio.TimeoutError) as e:
+        raise UpstreamError(f"proxy unreachable: {e!r}") from None
+    if r.ptype == "http" and not tunnel:
+        return reader, writer
+    try:
+        await asyncio.wait_for(_handshake(r.ptype, r.proxy, reader, writer, host, port, public_only), timeout)
+    except BaseException as e:
+        writer.close()
+        if isinstance(e, UpstreamError):
+            raise
+        if isinstance(e, (OSError, asyncio.TimeoutError, asyncio.IncompleteReadError, ValueError)):
+            raise UpstreamError(f"tunnel failed: {e!r}") from None
+        raise
+    return reader, writer
+
+
+async def _handshake(ptype: str, proxy: str, reader, writer, host: str, port: int,
+                     public_only: bool = False) -> None:
+    try:
+        await _connect_through(ptype, proxy, reader, writer, host, port, public_only)
+    except ProxyRefused as e:  # the proxy speaks the protocol fine, only the CONNECT to the target failed
+        raise TargetError(str(e)) from None
+
+
+async def _connect_through(ptype: str, proxy: str, reader, writer, host: str, port: int,
+                           public_only: bool = False) -> None:
+    ep = parse_endpoint(proxy)
+    literal = _ip_literal(host)
+    if ptype == "http":
+        authority = f"[{host}]:{port}" if literal and literal.version == 6 else f"{host}:{port}"
+        connect = f"CONNECT {authority} HTTP/1.1\r\nHost: {authority}\r\n\r\n".encode()
+        writer.write(with_proxy_auth(connect, ep))
+        await writer.drain()
+        head = await reader.readuntil(b"\r\n\r\n")
+        first = head.split(b"\r\n", 1)[0]
+        parts = first.split()
+        # exactly 200 – "HTTP/1.1 2000" or similar is not an established tunnel
+        if len(parts) < 2 or not parts[0].startswith(b"HTTP/") or parts[1] != b"200":
+            error = TargetError if len(parts) > 1 and parts[1] in GATEWAY_ERRORS else UpstreamError
+            raise error(first.decode("latin-1"))
+    elif ptype == "socks4":
+        if literal and literal.version == 6:
+            raise Unsupported("SOCKS4 can't reach IPv6 targets")
+        ip = await _resolve(host, port, public_only)  # SOCKS4 only knows IPv4 addresses
+        if not await socks4_connect(*stream_io(reader, writer), ep, socket.inet_aton(ip), port):
+            raise UpstreamError("no SOCKS4 answer")
+    else:
+        if literal and literal.version == 6:
+            address = b"\x04" + literal.packed
+        elif literal:
+            address = socks5_ipv4(literal.packed)
+        else:
+            address = socks5_domain(host)  # host name instead of IP: resolved at the proxy (no DNS leak)
+        if not await socks5_connect(*stream_io(reader, writer), ep, address, port):
+            raise UpstreamError("SOCKS5 greeting or login failed")
+
+
+def _ip_literal(host: str):
+    try:
+        return ipaddress.ip_address(host)
+    except ValueError:
+        return None
+
+
+async def _resolve(host: str, port: int, public_only: bool = False) -> str:
+    try:
+        infos = await asyncio.get_running_loop().getaddrinfo(host, port, family=socket.AF_INET)
+    except OSError as e:  # no IPv4 address for the target – a SOCKS4 limit, not the proxy's fault
+        raise Unsupported(f"can't resolve {host} to IPv4: {e}") from None
+    ip = infos[0][4][0]
+    if public_only and not ipaddress.ip_address(ip).is_global:  # checked once more right before connecting
+        raise TargetError(f"{host} resolves to {ip}, a private address")
+    return ip

proxyscraper/site/googleaac1161b7853c5b5.html

@@ -0,0 +1 @@
+google-site-verification: googleaac1161b7853c5b5.html