proxy-scraper-cli 1.7.1__py3-none-any.whl

proxyscraper/fetchcache.py

@@ -0,0 +1,103 @@
+"""Don't reload unchanged lists every time.
+
+For every list the cache remembers the ETag or Last-Modified and the parsed proxy keys.
+On the next run the download asks with If-None-Match / If-Modified-Since; if 304 comes back,
+the keys come from the cache. They are stored unfiltered (all types), because --types can
+change between two runs.
+
+  data/fetch-cache/index.json      url -> {etag, modified, type, file, used}
+  data/fetch-cache/<sha1>.txt.gz   the keys, one per line
+"""
+
+from __future__ import annotations
+
+import gzip
+import hashlib
+import json
+import time
+import zlib
+from pathlib import Path
+from typing import Dict, Optional
+
+from .paths import DATA_DIR, atomic_write
+
+CACHE_DIR = DATA_DIR / "fetch-cache"
+FORGET_AFTER = 14 * 86400  # lists that haven't been loaded for this long are dropped
+
+
+def _valid_entry(entry) -> bool:
+    return (isinstance(entry, dict) and isinstance(entry.get("file"), str) and entry["file"].endswith(".txt.gz")
+            and "/" not in entry["file"] and "\\" not in entry["file"]
+            and isinstance(entry.get("etag", ""), str) and isinstance(entry.get("modified", ""), str)
+            and isinstance(entry.get("used", 0), (int, float)))
+
+
+class FetchCache:
+    def __init__(self, directory: Path = CACHE_DIR, enabled: bool = True):
+        self.dir = directory
+        self.enabled = enabled
+        self.entries: Dict[str, dict] = {}
+        self.hits = 0
+        if enabled:
+            try:
+                raw = json.loads((directory / "index.json").read_text(encoding="utf-8"))
+            except (OSError, ValueError):
+                raw = {}
+            # only entries of the expected shape – a broken index.json simply means "empty cache"
+            if isinstance(raw, dict):
+                self.entries = {url: e for url, e in raw.items() if _valid_entry(e)}
+
+    def conditional_headers(self, url: str, ptype: str) -> Dict[str, str]:
+        """Headers for a conditional request – empty if there's nothing (usable) in the cache.
+
+        ptype is the type the list is parsed with: if it changes (e.g. http -> auto),
+        the stored keys no longer fit and the list is loaded again."""
+        entry = self.entries.get(url) if self.enabled else None
+        if not entry or entry.get("type") != ptype or not (self.dir / entry["file"]).is_file():
+            return {}
+        headers = {}
+        if entry.get("etag"):
+            headers["If-None-Match"] = entry["etag"]
+        if entry.get("modified"):
+            headers["If-Modified-Since"] = entry["modified"]
+        return headers
+
+    def load(self, url: str) -> Optional[str]:
+        """Keys from the cache after a 304 response (None if the file is broken)."""
+        entry = self.entries.get(url)
+        if not entry:
+            return None
+        try:
+            keys = gzip.decompress((self.dir / entry["file"]).read_bytes()).decode("utf-8")
+        except (OSError, EOFError, UnicodeDecodeError, zlib.error):  # broken file = cache miss
+            return None
+        entry["used"] = time.time()
+        self.hits += 1
+        return keys
+
+    def store(self, url: str, headers: Dict[bytes, bytes], keys: str, ptype: str) -> None:
+        """After a normal download: remember it if the server sends an ETag or Last-Modified."""
+        if not self.enabled:
+            return
+        etag = headers.get(b"etag", b"").decode("latin-1").strip()
+        modified = headers.get(b"last-modified", b"").decode("latin-1").strip()
+        if not etag and not modified:
+            old = self.entries.pop(url, None)
+            if old:  # remove the file too, otherwise it would stay forever
+                (self.dir / old["file"]).unlink(missing_ok=True)
+            return
+        name = hashlib.sha1(url.encode()).hexdigest() + ".txt.gz"
+        self.dir.mkdir(parents=True, exist_ok=True)
+        tmp = self.dir / (name + ".tmp")
+        tmp.write_bytes(gzip.compress(keys.encode("utf-8"), compresslevel=5))
+        tmp.replace(self.dir / name)
+        self.entries[url] = {"etag": etag, "modified": modified, "type": ptype, "file": name, "used": time.time()}
+
+    def save(self, now: Optional[float] = None) -> None:
+        index = self.dir / "index.json"
+        if not self.enabled or (not self.entries and not index.exists()):
+            return  # an empty index is written too – otherwise a removed entry would come back
+        now = time.time() if now is None else now
+        for url in [u for u, e in self.entries.items() if now - e.get("used", 0) > FORGET_AFTER]:
+            (self.dir / self.entries.pop(url)["file"]).unlink(missing_ok=True)
+        atomic_write(index, json.dumps(self.entries, indent=0, sort_keys=True))

proxyscraper/geo.py

@@ -0,0 +1,150 @@
+"""Countries of the exit IPs.
+
+First offline from the DB-IP database (geodb.py) – instantly and without a limit. Only what's missing
+there (or when the database couldn't be loaded) goes to the batch API of ip-api.com
+(100 IPs per request, 15 requests/minute, runs in the background). API results are cached,
+so known IPs are never queried again.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+import time
+import warnings
+from typing import Callable, Dict, List, Optional
+
+from .geodb import CountryDB
+from .netio import http_request
+from .paths import DATA_DIR, atomic_write
+
+GEO_CACHE_FILE = DATA_DIR / "geo_cache.json"
+BATCH_URL = "http://ip-api.com/batch?fields=status,countryCode,query"
+BATCH_SIZE = 100
+MIN_INTERVAL = 4.2          # 60 s / 15 requests, plus a buffer
+CACHE_TTL = 30 * 86400.0
+
+
+def flag(country: str) -> str:
+    """'DE' -> 🇩🇪 (built from regional indicator characters)."""
+    if len(country) != 2 or not country.isalpha():
+        return "  "
+    return "".join(chr(0x1F1E6 + ord(c) - ord("A")) for c in country.upper())
+
+
+class GeoResolver:
+    def __init__(self, enabled: bool = True, on_resolved: Optional[Callable[[str, str], None]] = None,
+                 offline: Optional[CountryDB] = None):
+        self.enabled = enabled
+        self.offline = offline
+        self.offline_hits = 0
+        # per run every exit IP gets exactly one country – no matter whether it comes from the database, the cache
+        # or ip-api, and whether the database only arrives half-way through
+        self.assigned: Dict[str, str] = {}
+        self.on_resolved = on_resolved
+        self.cache: Dict[str, List] = {}  # ip -> [country, timestamp]
+        self.pending: List[str] = []
+        self._queued = set()
+        self._stop = asyncio.Event()
+        self.failed = False
+        if GEO_CACHE_FILE.exists():
+            try:
+                self.cache = json.loads(GEO_CACHE_FILE.read_text(encoding="utf-8"))
+            except (OSError, ValueError) as e:
+                warnings.warn(f"{GEO_CACHE_FILE.name} unreadable ({e})", stacklevel=2)
+
+    def lookup(self, ip: str) -> str:
+        hit = self.cache.get(ip)
+        return hit[0] if hit and time.time() - hit[1] < CACHE_TTL else ""
+
+    def use_offline(self, db: CountryDB) -> None:
+        """Take over a new database mid-run – also for IPs already waiting for ip-api,
+        so the same exit IP isn't classified one way once and another way the next time."""
+        self.offline = db
+        waiting, self.pending = self.pending, []
+        for ip in waiting:
+            country = db.lookup(ip)
+            if not country:
+                self.pending.append(ip)
+                continue
+            self.offline_hits += 1
+            self._assign(ip, country)
+
+    def _assign(self, ip: str, country: str) -> None:
+        if ip in self.assigned:
+            return
+        self.assigned[ip] = country
+        if self.on_resolved:
+            self.on_resolved(ip, country)
+
+    def request(self, ip: str) -> str:
+        """Country right away (offline or from the cache), otherwise queue it for the next batch request."""
+        if ip in self.assigned:
+            return self.assigned[ip]
+        country = ""
+        if self.enabled and self.offline:
+            country = self.offline.lookup(ip)
+            if country:
+                self.offline_hits += 1
+        country = country or self.lookup(ip)
+        if country:
+            self.assigned[ip] = country
+        elif self.enabled and ip not in self._queued:
+            self._queued.add(ip)
+            self.pending.append(ip)
+        return country
+
+    async def run(self) -> None:
+        """Background loop – ends after stop() as soon as nothing is pending."""
+        if not self.enabled:
+            return
+        while not (self._stop.is_set() and not self.pending):
+            if not self.pending:
+                with contextlib.suppress(asyncio.TimeoutError):
+                    await asyncio.wait_for(self._stop.wait(), 0.5)
+                continue
+            batch, self.pending = self.pending[:BATCH_SIZE], self.pending[BATCH_SIZE:]
+            started = time.monotonic()
+            wait = await self._resolve(batch)
+            if self.failed:
+                return
+            await asyncio.sleep(max(wait, MIN_INTERVAL - (time.monotonic() - started)))
+
+    async def _resolve(self, batch: List[str]) -> float:
+        """One batch request; returns the wait time needed before the next one."""
+        try:
+            status, headers, body = await http_request(
+                BATCH_URL, timeout=10, method="POST", body=json.dumps(batch).encode(),
+                headers={"Content-Type": "application/json"},
+            )
+        except Exception:  # API unreachable -> countries stay empty, the rest keeps running
+            self.failed = True
+            return 0.0
+        if status == 429:
+            self.pending = batch + self.pending
+            return float(headers.get(b"x-ttl", b"60") or 60)
+        if status != 200:
+            self.failed = True
+            return 0.0
+        now = time.time()
+        try:
+            rows = json.loads(body)
+        except ValueError:
+            return 0.0
+        for row in rows:
+            if row.get("status") == "success":
+                ip, country = row["query"], row["countryCode"]
+                self.cache[ip] = [country, now]
+                self._assign(ip, country)  # already classified otherwise (e.g. by the database)? keep it
+        # if ip-api reports the limit is almost reached, wait for the reset
+        if headers.get(b"x-rl", b"1") == b"0":
+            return float(headers.get(b"x-ttl", b"60") or 60)
+        return 0.0
+
+    def stop(self) -> None:
+        self._stop.set()
+
+    def save(self) -> None:
+        if self.cache:
+            atomic_write(GEO_CACHE_FILE, json.dumps(self.cache, separators=(",", ":")))

proxyscraper/geodb.py

@@ -0,0 +1,143 @@
+"""Countries offline – with the free country database from DB-IP (DB-IP Lite, CC BY 4.0).
+
+The CSV (≈ 4.5 MB gzip, new every month) is loaded once a month and turned into two number arrays
+plus country codes; a lookup is then a binary search and takes microseconds. That way the tool no
+longer depends on ip-api.com (15 requests/minute) – that's only a fallback for IPs missing here,
+or when the database can't be loaded.
+
+IP Geolocation by DB-IP: https://db-ip.com
+"""
+
+from __future__ import annotations
+
+import gzip
+import io
+import socket
+import struct
+from array import array
+from bisect import bisect_right
+from datetime import date
+from pathlib import Path
+from typing import Optional
+
+from .netio import http_get
+from .paths import DATA_DIR
+
+DB_FILE = DATA_DIR / "geo" / "dbip-country-ipv4.bin"
+URL = "https://download.db-ip.com/free/dbip-country-lite-{month}.csv.gz"
+MAGIC = b"PSGEO1"
+HEADER = len(MAGIC) + 7 + 4  # magic, month, count
+
+
+def _ip_to_int(ip: str) -> int:
+    return struct.unpack("!I", socket.inet_aton(ip))[0]
+
+
+class CountryDB:
+    def __init__(self, starts: array, ends: array, codes: bytes, month: str):
+        self.starts = starts
+        self.ends = ends
+        self.codes = codes  # 2 bytes per range
+        self.month = month
+
+    def __len__(self) -> int:
+        return len(self.starts)
+
+    def lookup(self, ip: str) -> str:
+        """'8.8.8.8' -> 'US'; '' if unknown or not a country (ZZ = reserved)."""
+        try:
+            n = _ip_to_int(ip)
+        except OSError:
+            return ""
+        i = bisect_right(self.starts, n) - 1
+        if i < 0 or n > self.ends[i]:
+            return ""
+        code = self.codes[2 * i:2 * i + 2].decode("ascii")
+        return "" if code == "ZZ" else code
+
+    @classmethod
+    def from_csv(cls, text: str, month: str) -> "CountryDB":
+        """DB-IP CSV ("start,end,country" for IPv4 and IPv6) -> only the IPv4 ranges."""
+        starts, ends, codes = array("I"), array("I"), bytearray()
+        for line in io.StringIO(text):
+            fields = line.rstrip("\r\n").split(",")
+            if len(fields) < 3 or ":" in fields[0]:
+                continue  # IPv6 or a broken line
+            start, end, code = fields[0], fields[1], fields[2].strip()  # evtl. weitere Spalten ignorieren
+            if len(code) != 2 or not (code.isascii() and code.isalpha()):
+                continue
+            try:
+                s, e = _ip_to_int(start), _ip_to_int(end)
+            except OSError:
+                continue  # broken address – skip just this line
+            if codes and codes[-2:] == code.encode() and s == ends[-1] + 1:
+                ends[-1] = e  # Nachbarbereich desselben Landes zusammenfassen
+                continue
+            starts.append(s)
+            ends.append(e)
+            codes += code.encode("ascii")
+        return cls(starts, ends, bytes(codes), month)
+
+    def save(self, path: Path = DB_FILE) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(".tmp")
+        with tmp.open("wb") as fh:
+            fh.write(MAGIC + self.month.encode("ascii").ljust(7) + struct.pack("!I", len(self.starts)))
+            self.starts.tofile(fh)
+            self.ends.tofile(fh)
+            fh.write(self.codes)
+        tmp.replace(path)
+
+    @classmethod
+    def load(cls, path: Path = DB_FILE) -> Optional["CountryDB"]:
+        try:
+            size = path.stat().st_size
+            with path.open("rb") as fh:
+                if fh.read(len(MAGIC)) != MAGIC:
+                    return None
+                month = fh.read(7).decode("ascii").strip()
+                (count,) = struct.unpack("!I", fh.read(4))
+                if size != HEADER + count * 10:  # 2 × 4 bytes + 2 bytes of country per range
+                    return None  # truncated or broken – don't blindly allocate memory
+                starts, ends = array("I"), array("I")
+                starts.fromfile(fh, count)
+                ends.fromfile(fh, count)
+                codes = fh.read(2 * count)
+        except (OSError, EOFError, ValueError, struct.error, UnicodeDecodeError):
+            return None
+        if len(codes) != 2 * count or not (codes.isascii() and codes.isalpha()):
+            return None  # broken country codes would only show up during a lookup
+        return cls(starts, ends, codes, month)
+
+
+def _months(today: date):
+    """Current and previous month as 'YYYY-MM' – at the start of a month the new file may not be there yet."""
+    yield today.strftime("%Y-%m")
+    prev = date(today.year - (today.month == 1), 12 if today.month == 1 else today.month - 1, 1)
+    yield prev.strftime("%Y-%m")
+
+
+def is_current(db: Optional[CountryDB], today: Optional[date] = None) -> bool:
+    return db is not None and db.month == (today or date.today()).strftime("%Y-%m")
+
+
+async def load_country_db(path: Path = DB_FILE, today: Optional[date] = None, fetch=http_get) -> Optional[CountryDB]:
+    """Database from data/ – new from DB-IP once a month. None if there is none at all.
+
+    May download (up to two attempts of 30 s each) – so call it in the background."""
+    today = today or date.today()
+    current = CountryDB.load(path)
+    if current and current.month == today.strftime("%Y-%m"):
+        return current
+    for month in _months(today):
+        if current and current.month >= month:
+            return current  # there's no newer one (yet)
+        try:
+            data = await fetch(URL.format(month=month), timeout=30)
+            db = CountryDB.from_csv(gzip.decompress(data).decode("ascii", "replace"), month)
+        except Exception:  # unreachable or broken – try the next month or keep the old file
+            continue
+        if len(db) > 1000:  # plausibility: the real file has more than 100,000 ranges
+            db.save(path)
+            return db
+    return current

proxyscraper/handshake.py

@@ -0,0 +1,153 @@
+"""Connecting through HTTP, SOCKS4 and SOCKS5 proxies – for the checker and the proxy server.
+
+Proxies with credentials appear as "user:pass@ip:port" in the key, user and password
+URL-encoded (otherwise ":" or "@" in the password would mess everything up).
+
+The functions get send/recv_exact instead of a stream because the HTTPS test works with
+bare sockets and everything else with asyncio streams.
+"""
+
+from __future__ import annotations
+
+import base64
+from dataclasses import dataclass
+from typing import Awaitable, Callable
+from urllib.parse import quote, unquote
+
+Send = Callable[[bytes], Awaitable[None]]
+RecvExact = Callable[[int], Awaitable[bytes]]
+
+SOCKS5_NO_AUTH = 0x00
+SOCKS5_USER_PASS = 0x02  # RFC 1929
+
+
+@dataclass(frozen=True)
+class Endpoint:
+    host: str
+    port: int
+    user: str = ""
+    password: str = ""
+
+    @property
+    def address(self) -> str:
+        return f"{self.host}:{self.port}"
+
+    @property
+    def has_auth(self) -> bool:
+        return bool(self.user)
+
+
+def parse_endpoint(proxy: str) -> Endpoint:
+    """'user:pass@1.2.3.4:1080' or '1.2.3.4:1080' -> Endpoint."""
+    auth, _, address = proxy.rpartition("@")
+    host, _, port = address.rpartition(":")
+    user, _, password = auth.partition(":")
+    return Endpoint(host, int(port), unquote(user), unquote(password))
+
+
+def format_auth(user: str, password: str) -> str:
+    """Encode user and password the way they appear in the key ('' without a user)."""
+    if not user:
+        return ""
+    return quote(user, safe="") + (":" + quote(password, safe="") if password else "")
+
+
+def proxy_authorization(ep: Endpoint) -> bytes:
+    """Header line for HTTP proxies (incl. \\r\\n), empty without credentials."""
+    if not ep.has_auth:
+        return b""
+    token = base64.b64encode(f"{ep.user}:{ep.password}".encode()).decode()
+    return f"Proxy-Authorization: Basic {token}\r\n".encode()
+
+
+def with_proxy_auth(request: bytes, ep: Endpoint) -> bytes:
+    """Appends Proxy-Authorization to the head of a finished HTTP request (ends with \\r\\n\\r\\n)."""
+    header = proxy_authorization(ep)
+    return request[:-2] + header + b"\r\n" if header else request
+
+
+class ProxyRefused(Exception):
+    """The proxy answered, but the CONNECT to the target was refused – it spoke the protocol fine."""
+
+
+async def socks4(send: Send, recv_exact: RecvExact, ep: Endpoint, ip_bytes: bytes, port: int) -> bool:
+    """SOCKS4 CONNECT; the user name goes into the user ID field (SOCKS4 has no password)."""
+    return await socks4_connect(send, recv_exact, ep, ip_bytes, port, strict=False)
+
+
+async def socks4_connect(send: Send, recv_exact: RecvExact, ep: Endpoint, ip_bytes: bytes, port: int,
+                         strict: bool = True) -> bool:
+    """strict: a proper SOCKS4 reply that refuses the CONNECT (0x5B–0x5D) raises ProxyRefused,
+    anything that isn't a SOCKS4 reply at all returns False."""
+    await send(b"\x04\x01" + port.to_bytes(2, "big") + ip_bytes + ep.user.encode()[:255] + b"\x00")
+    reply = await recv_exact(8)
+    if reply[1] == 0x5A:
+        return True
+    if strict and reply[0] == 0x00 and reply[1] in (0x5B, 0x5C, 0x5D):
+        raise ProxyRefused(f"SOCKS4 reply 0x{reply[1]:02x}")
+    return False
+
+
+async def socks5(send: Send, recv_exact: RecvExact, ep: Endpoint, address: bytes, port: int) -> bool:
+    """SOCKS5 CONNECT to address (ATYP + address, see socks5_ipv4/socks5_domain)."""
+    return await socks5_connect(send, recv_exact, ep, address, port, strict=False)
+
+
+async def socks5_connect(send: Send, recv_exact: RecvExact, ep: Endpoint, address: bytes, port: int,
+                         strict: bool = True) -> bool:
+    """strict: greeting and login are the proxy's part (False on failure); a refused CONNECT after a
+    successful greeting raises ProxyRefused, because then the proxy works and only the target failed."""
+    methods = bytes([SOCKS5_NO_AUTH, SOCKS5_USER_PASS]) if ep.has_auth else bytes([SOCKS5_NO_AUTH])
+    await send(b"\x05" + bytes([len(methods)]) + methods)
+    reply = await recv_exact(2)
+    if reply[0] != 0x05:
+        return False
+    if reply[1] == SOCKS5_USER_PASS and ep.has_auth:
+        user, password = ep.user.encode(), ep.password.encode()
+        if len(user) > 255 or len(password) > 255:
+            return False
+        await send(b"\x01" + bytes([len(user)]) + user + bytes([len(password)]) + password)
+        if (await recv_exact(2))[1] != 0x00:
+            return False  # login refused
+    elif reply[1] != SOCKS5_NO_AUTH:
+        return False
+    await send(b"\x05\x01\x00" + address + port.to_bytes(2, "big"))
+    if await socks5_reply_ok(recv_exact):
+        return True
+    if strict:
+        raise ProxyRefused("SOCKS5 refused the CONNECT")
+    return False
+
+
+def socks5_ipv4(ip_bytes: bytes) -> bytes:
+    return b"\x01" + ip_bytes
+
+
+def socks5_domain(host: str) -> bytes:
+    name = host.encode("idna")
+    return b"\x03" + bytes([len(name)]) + name
+
+
+async def socks5_reply_ok(recv_exact: RecvExact) -> bool:
+    resp = await recv_exact(4)
+    if resp[1] != 0x00:
+        return False
+    atyp = resp[3]
+    if atyp == 1:
+        await recv_exact(4 + 2)
+    elif atyp == 4:
+        await recv_exact(16 + 2)
+    elif atyp == 3:
+        ln = (await recv_exact(1))[0]
+        await recv_exact(ln + 2)
+    else:
+        return False
+    return True
+
+
+def stream_io(reader, writer):
+    """send/recv_exact for asyncio streams."""
+    async def send(data: bytes) -> None:
+        writer.write(data)
+        await writer.drain()
+    return send, reader.readexactly

proxyscraper/history.py

@@ -0,0 +1,106 @@
+"""History of working proxies across runs.
+
+A proxy that worked yesterday is much more likely to work today than any random list
+entry – that's why known proxies are checked first.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+import warnings
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from .paths import DATA_DIR, atomic_write
+
+HISTORY_FILE = DATA_DIR / "proxy_history.json"
+DAY = 86400.0
+FORGET_AFTER = 14 * DAY     # not working for this long -> forget it
+MAX_FAIL_STREAK = 4         # so oft hintereinander tot -> vergessen
+
+
+@dataclass
+class ProxyRecord:
+    first_ok: float = 0.0
+    last_ok: float = 0.0
+    ok: int = 0
+    fail: int = 0
+    fail_streak: int = 0
+    latency: int = 0
+    exit_ip: str = ""
+    country: str = ""
+    anonymity: str = ""
+    https: Optional[bool] = None
+
+    @property
+    def reliability(self) -> float:
+        """Share of successful checks, smoothed (1 out of 1 is not 100 %)."""
+        return (self.ok + 1) / (self.ok + self.fail + 2)
+
+
+class ProxyHistory:
+    def __init__(self, path: Path = HISTORY_FILE):
+        self.path = path
+        self.records: Dict[str, ProxyRecord] = {}
+        if path.exists():
+            try:
+                raw = json.loads(path.read_text(encoding="utf-8"))
+                self.records = {k: ProxyRecord(**v) for k, v in raw.items()}
+            except (OSError, ValueError, TypeError) as e:
+                warnings.warn(f"{path.name} unreadable ({e}), starting without proxy history", stacklevel=2)
+
+    @staticmethod
+    def exists(path: Path = HISTORY_FILE) -> bool:
+        """Is there a non-empty history? Without loading all of it."""
+        try:
+            return path.stat().st_size > 2  # "{}" = leer
+        except OSError:
+            return False
+
+    def __len__(self) -> int:
+        return len(self.records)
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.records
+
+    def get(self, key: str) -> Optional[ProxyRecord]:
+        return self.records.get(key)
+
+    def ranked_keys(self) -> List[str]:
+        """Known proxies, the most reliable and most recently successful first."""
+        return sorted(self.records, key=lambda k: (-self.records[k].reliability, -self.records[k].last_ok))
+
+    def record_ok(self, key: str, latency: int, exit_ip: str, now: Optional[float] = None, **details) -> None:
+        now = time.time() if now is None else now
+        rec = self.records.setdefault(key, ProxyRecord(first_ok=now))
+        rec.last_ok = now
+        rec.ok += 1
+        rec.fail_streak = 0
+        rec.latency = latency
+        rec.exit_ip = exit_ip
+        for name, value in details.items():
+            if value not in (None, ""):
+                setattr(rec, name, value)
+
+    def record_fail(self, key: str) -> None:
+        """Only for proxies that are already known – otherwise the history fills up with millions of dead ones."""
+        rec = self.records.get(key)
+        if rec:
+            rec.fail += 1
+            rec.fail_streak += 1
+
+    def prune(self, now: Optional[float] = None) -> int:
+        now = time.time() if now is None else now
+        dead = [
+            k for k, r in self.records.items()
+            if r.fail_streak >= MAX_FAIL_STREAK or now - r.last_ok > FORGET_AFTER
+        ]
+        for k in dead:
+            del self.records[k]
+        return len(dead)
+
+    def save(self) -> None:
+        payload = {k: asdict(r) for k, r in self.records.items()}
+        atomic_write(self.path, json.dumps(payload, separators=(",", ":")))