proxy-scraper-cli 1.7.1__py3-none-any.whl

proxyscraper/parsing.py

@@ -0,0 +1,192 @@
+"""Finds proxies in any list format and normalizes them.
+
+Proxies are kept everywhere as the key "type ip:port" (one string) – with more than a million
+entries that saves a lot of memory compared to tuples.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import re
+from typing import Iterable, List, Optional, Set, Tuple
+from urllib.parse import unquote
+
+from .handshake import format_auth
+
+PROXY_TYPES = ("http", "socks4", "socks5")
+TYPE_ALIASES = {
+    "http": "http", "https": "http",
+    "socks4": "socks4", "socks4a": "socks4",
+    "socks5": "socks5", "socks5h": "socks5", "socks5a": "socks5",
+    "auto": "auto", "mixed": "auto",
+}
+
+_IP = rb"((?:\d{1,3}\.){3}\d{1,3})"
+# Schneller Standardfall: ip:port
+PLAIN_RE = re.compile(rb"(?<![\d.])" + _IP + rb"\s*:\s*(\d{2,5})(?![\d.])")
+# complete: additionally "ip port" and HTML tables like <td>1.2.3.4</td><td>8080</td>
+PROXY_RE = re.compile(
+    rb"(?<![\d.])" + _IP + rb"(?:\s*:\s*|[ \t]+|\s*(?:<[^<>]{0,100}>\s*){1,6})(\d{2,5})(?![\d.])"
+)
+# JSON APIs like {"ip": "1.2.3.4", ..., "port": "8080"} – in both orders
+JSON_IP_PORT_RE = re.compile(rb'"ip"\s*:\s*"' + _IP + rb'"[^{}]{0,500}?"port"\s*:\s*"?(\d{2,5})')
+JSON_PORT_IP_RE = re.compile(rb'"port"\s*:\s*"?(\d{2,5})"?[^{}]{0,500}?"ip"\s*:\s*"' + _IP + rb'"')
+# type://[user:pass@]ip:port – here the type is in the line itself
+SCHEME_RE = re.compile(
+    rb"(?i)(?<![a-z0-9])(https?|socks4a?|socks5h?)://(?:([^\s@/]{1,100})@)?" + _IP + rb":(\d{2,5})(?!\d)"
+)
+SCHEME_TYPES = {k.encode(): v for k, v in TYPE_ALIASES.items() if v != "auto"}
+
+RawCandidate = Tuple[str, bytes, bytes, bytes]  # type, ip, port, credentials (b"" without)
+
+
+_SPACE_SEPARATED_RE = re.compile(rb"\d\.\d{1,3}[ \t]+\d{2,5}(?![\d.])")
+
+
+def _needs_full_regex(data: bytes) -> bool:
+    """Only space/HTML lists need the slower regex (sample from the start of the file)."""
+    sample = data[:8192]
+    return b"<" in sample or _SPACE_SEPARATED_RE.search(sample) is not None
+
+
+def extract_candidates(data: bytes, default_type: str) -> Set[RawCandidate]:
+    """Finds proxies in text, HTML tables and JSON.
+
+    Lines with a type:// prefix keep their own type; everything else gets the type of the source.
+    For sources of type "auto" only lines with a prefix count.
+    """
+    out: Set[RawCandidate] = set()
+    with_scheme = set()
+    if b"://" in data:
+        for scheme, auth, ip, port in SCHEME_RE.findall(data):
+            ptype = SCHEME_TYPES.get(scheme.lower())
+            if ptype:
+                out.add((ptype, ip, port, auth))
+                with_scheme.add((ip, port))
+    if default_type == "auto":
+        return out
+    regex = PROXY_RE if _needs_full_regex(data) else PLAIN_RE
+    pairs = set(regex.findall(data))
+    if b'"ip"' in data:
+        pairs.update(JSON_IP_PORT_RE.findall(data))
+        pairs.update((ip, port) for port, ip in JSON_PORT_IP_RE.findall(data))
+    out.update((default_type, ip, port, b"") for ip, port in pairs - with_scheme)
+    return out
+
+
+# networks that aren't publicly reachable (private, loopback, CGNAT, documentation, multicast, reserved …).
+# ipaddress.is_private and friends are very slow per address, hence a table by first octet:
+# True = the whole /8 is blocked, list = only check these subnets, empty = everything public.
+_BLOCKED_BY_FIRST_OCTET: list = [[] for _ in range(256)]
+for _net in map(ipaddress.IPv4Network, (
+    "0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16",
+    "172.16.0.0/12", "192.0.0.0/24", "192.0.2.0/24", "192.168.0.0/16", "198.18.0.0/15",
+    "198.51.100.0/24", "203.0.113.0/24", "224.0.0.0/4", "240.0.0.0/4",
+)):
+    _first = int(_net.network_address) >> 24
+    if _net.prefixlen <= 8:
+        for _o in range(_first, _first + 2 ** (8 - _net.prefixlen)):
+            _BLOCKED_BY_FIRST_OCTET[_o] = True
+    else:
+        _BLOCKED_BY_FIRST_OCTET[_first].append((int(_net.network_address), int(_net.netmask)))
+
+
+def normalize_public_ip(ip: bytes) -> Optional[str]:
+    """'001.2.3.4' -> '1.2.3.4'; None for invalid or non-public addresses."""
+    a, b, c, d = map(int, ip.split(b"."))
+    if a > 255 or b > 255 or c > 255 or d > 255:
+        return None
+    rules = _BLOCKED_BY_FIRST_OCTET[a]
+    if rules is True:
+        return None
+    if rules:
+        n = a << 24 | b << 16 | c << 8 | d
+        if any(n & mask == net for net, mask in rules):
+            return None
+    return f"{a}.{b}.{c}.{d}"
+
+
+def normalize_proxy(ip: bytes, port: bytes) -> Optional[str]:
+    """'001.2.3.4', b'080' -> '1.2.3.4:80'; None for an invalid port or a non-public IP."""
+    p = int(port)
+    if not 0 < p < 65536:
+        return None
+    ip_s = normalize_public_ip(ip)
+    return f"{ip_s}:{p}" if ip_s else None
+
+
+def make_key(ptype: str, proxy: str) -> str:
+    return f"{ptype} {proxy}"
+
+
+def split_key(key: str) -> Tuple[str, str]:
+    ptype, _, proxy = key.partition(" ")
+    return ptype, proxy
+
+
+def normalize_auth(auth: str) -> str:
+    """'user:p%40ss' or 'user:p@ss' -> encoded uniformly ('user:p%40ss'); '' without a user."""
+    user, _, password = auth.partition(":")
+    return format_auth(unquote(user), unquote(password))
+
+
+def validate_candidates(candidates: Iterable[RawCandidate]) -> Set[str]:
+    """{(type, ip, port, auth)} -> {"type [auth@]ip:port"} only for valid, public addresses."""
+    out = set()
+    for ptype, ip, port, auth in candidates:
+        proxy = normalize_proxy(ip, port)
+        if not proxy:
+            continue
+        if auth:
+            creds = normalize_auth(auth.decode("utf-8", "replace"))
+            if creds:
+                proxy = f"{creds}@{proxy}"
+        out.add(make_key(ptype, proxy))
+    return out
+
+
+def parse_blob(data: bytes, default_type: str, wanted: Tuple[str, ...]) -> str:
+    """Complete parse step of a source for the process pool.
+
+    Returns the keys as a single string separated by \\n – one object can be passed
+    between processes much faster than hundreds of thousands of small ones.
+    """
+    keys = [k for k in validate_candidates(extract_candidates(data, default_type)) if k.split(" ", 1)[0] in wanted]
+    return "\n".join(keys)
+
+
+def parse_proxy_line(line: str, default_type: Optional[str] = None) -> Optional[str]:
+    """A line from a result file ('socks5://user:pass@1.2.3.4:1080' or '1.2.3.4:80') -> key."""
+    line = line.strip()
+    if not line or line.startswith("#"):
+        return None
+    ptype = default_type
+    if "://" in line:
+        scheme, _, line = line.partition("://")
+        ptype = TYPE_ALIASES.get(scheme.lower())
+    if ptype not in PROXY_TYPES:
+        return None
+    parts = line.split()
+    if not parts:  # "http://" with nothing after it
+        return None
+    line = parts[0]
+    auth, _, line = line.rpartition("@")
+    ip, _, port = line.partition(":")
+    port = port.rstrip("/")
+    if not port.isdigit() or ip.count(".") != 3 or not all(p.isdigit() for p in ip.split(".")):
+        return None
+    proxy = normalize_proxy(ip.encode(), port.encode())
+    if not proxy:
+        return None
+    creds = normalize_auth(auth) if auth else ""
+    return make_key(ptype, f"{creds}@{proxy}" if creds else proxy)
+
+
+def parse_keys(lines: Iterable[str], default_type: Optional[str] = None) -> List[str]:
+    seen, out = set(), []
+    for line in lines:
+        key = parse_proxy_line(line, default_type)
+        if key and key not in seen:
+            seen.add(key)
+            out.append(key)
+    return out

proxyscraper/paths.py

@@ -0,0 +1,55 @@
+"""Where things are stored.
+
+- source list: in the package (proxyscraper/sources.json)
+- from a cloned repo: learned state in data/, results in results/ – both inside the project
+- installed (pip/pipx): learned state in the system's user data folder, results in the current folder
+- PROXY_SCRAPER_HOME overrides the folder for the learned state
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+from typing import Optional
+
+PACKAGE_DIR = Path(__file__).resolve().parent
+PROJECT_DIR = PACKAGE_DIR.parent
+APP_NAME = "proxy-scraper"
+
+
+def is_checkout(project_dir: Path = PROJECT_DIR) -> bool:
+    """Is the program running straight from the repo (instead of an installation)?"""
+    return (project_dir / "pyproject.toml").is_file() and (project_dir / "proxy_scraper.py").is_file()
+
+
+def user_data_dir(platform: str = sys.platform, env=os.environ, home: Optional[Path] = None) -> Path:
+    """The usual folder for application data on this system."""
+    home = home or Path.home()
+    if platform == "darwin":
+        return home / "Library" / "Application Support" / APP_NAME
+    if platform == "win32":
+        return Path(env.get("LOCALAPPDATA") or home / "AppData" / "Local") / APP_NAME
+    return Path(env.get("XDG_DATA_HOME") or home / ".local" / "share") / APP_NAME
+
+
+def data_dir(env=os.environ) -> Path:
+    if env.get("PROXY_SCRAPER_HOME"):
+        return Path(env["PROXY_SCRAPER_HOME"]).expanduser()
+    return PROJECT_DIR / "data" if is_checkout() else user_data_dir(env=env)
+
+
+def results_dir() -> Path:
+    return PROJECT_DIR / "results" if is_checkout() else Path.cwd() / "results"
+
+
+DATA_DIR = data_dir()
+RESULTS_DIR = results_dir()
+
+
+def atomic_write(path: Path, text: str) -> None:
+    # write to a temporary file first, then rename – an interruption leaves no half-written file
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(text, encoding="utf-8")
+    os.replace(tmp, path)

proxyscraper/pipeline.py

@@ -0,0 +1,434 @@
+"""Flow: assemble sources, load & parse in parallel, prioritize, check."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import os
+import random
+from collections import Counter
+from concurrent.futures import ProcessPoolExecutor
+from dataclasses import dataclass, field
+from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple
+
+from . import sources as srcs
+from .asndb import ProviderLookup
+from .blocklist import Blocklist
+from .checker import Checker, CheckResult
+from .compat import on_interrupt
+from .fetchcache import FetchCache
+from .geo import GeoResolver
+from .history import ProxyHistory
+from .judges import JudgeWatch
+from .netio import http_get, http_request
+from .options import RunOptions
+from .output import ResultWriter
+from .parsing import PROXY_TYPES, parse_blob, split_key
+from .ui import ACCENT, CheckDashboard, CollectView, fmt, widgets
+
+AUTO_DISCOVER_AFTER_DAYS = 1.0  # daily: found sources are kept, so every search adds to the pile
+# parse small lists directly – the detour through another process costs more than it saves
+INLINE_PARSE_BYTES = 256 * 1024
+DOWNLOAD_CONCURRENCY = 64
+
+
+# --------------------------------------------------------------------------- #
+# Sources
+# --------------------------------------------------------------------------- #
+
+@dataclass
+class SourcePlan:
+    sources: Dict[str, str]
+    skipped: Counter
+    n_curated: int
+    n_meta: int
+    meta_ok: int
+    meta_total: int
+    n_discovered: int
+    discovery_ran: bool = False
+    discovery_token: bool = False
+
+
+async def collect_sources(opts: RunOptions, quality: srcs.SourceStats) -> SourcePlan:
+    """Curated + meta + discovered sources, minus the ones learned to be bad."""
+    sources, meta = srcs.load_source_file()
+    n_curated = len(sources)
+
+    age = srcs.discovered_age_days()
+    due = not opts.no_discover and (age is None or age > AUTO_DISCOVER_AFTER_DAYS)
+    # asking gh for a token takes up to 5 s – only when discovery can actually run, and off the event loop
+    token = await asyncio.to_thread(srcs.github_token) if (opts.discover or due) else None
+    run_discovery = opts.discover or (due and token is not None)
+    if run_discovery:
+        max_repos = opts.discover_repos if token else min(opts.discover_repos, 40)
+        with widgets.console.status(f"[bold {ACCENT}]Searching GitHub for new proxy lists …", spinner="dots") as status:
+            found = await srcs.discover_github(
+                http_get, token, max_repos,
+                on_progress=lambda msg: status.update(f"[bold {ACCENT}]GitHub-Discovery:[/] {msg}"),
+            )
+        if found:
+            srcs.save_discovered(found)
+    discovered = srcs.load_discovered()
+
+    with widgets.console.status(f"[bold {ACCENT}]Loading meta sources …", spinner="dots"):
+        meta_found, meta_ok = await srcs.resolve_meta(meta, http_get)
+    for extra in (meta_found, discovered):
+        for url, ptype in extra.items():
+            sources.setdefault(url, ptype)
+
+    wanted = set(opts.types)
+    sources = {u: t for u, t in sources.items() if t == "auto" or t in wanted}
+    skipped: Counter = Counter()
+    if not opts.all_sources:
+        active = {}
+        for url, ptype in sources.items():
+            reason = quality.skip_now(url)
+            if reason:
+                skipped[reason] += 1
+            else:
+                active[url] = ptype
+        sources = active
+    return SourcePlan(
+        sources, skipped, n_curated, len(meta_found), meta_ok, len(meta), len(discovered),
+        discovery_ran=run_discovery, discovery_token=token is not None,
+    )
+
+
+@dataclass
+class ScrapeResult:
+    urls: List[str]
+    # proxy key -> indices of the sources that list it (for priority and statistics)
+    index: Dict[str, List[int]] = field(default_factory=dict)
+    ok_sources: int = 0
+
+
+async def scrape(sources: Dict[str, str], types, quality: srcs.SourceStats, view: CollectView,
+                 cache: Optional[FetchCache] = None) -> ScrapeResult:
+    """Loads every source and parses large lists in parallel on all CPU cores.
+
+    That keeps the event loop free for more downloads instead of working through regexes for minutes.
+    With the cache, unchanged lists are skipped via ETag (see fetchcache.py).
+    """
+    res = ScrapeResult(list(sources))
+    cache = cache or FetchCache(enabled=False)
+    # always parse every type, filtering only happens when sorting them in: the cache stores all types, and
+    # the statistics must be able to tell "no proxies at all" from "none of the requested types"
+    parse_types = PROXY_TYPES
+    prefixes = tuple(f"{t} " for t in types)
+    loop = asyncio.get_running_loop()
+    sem = asyncio.Semaphore(DOWNLOAD_CONCURRENCY)
+    index = res.index
+
+    with ProcessPoolExecutor(max_workers=max(1, (os.cpu_count() or 2) - 1)) as pool:
+
+        async def download(url: str):
+            """-> (data, keys from the cache). Exactly one of the two is set."""
+            conditional = cache.conditional_headers(url, sources[url])
+            async with sem:
+                status, headers, body = await http_request(url, timeout=30, headers=conditional or None)
+            if status == 304 and conditional:
+                cached = cache.load(url)
+                if cached is not None:
+                    return None, headers, cached
+                async with sem:  # cache file broken -> just load it again
+                    status, headers, body = await http_request(url, timeout=30)
+            if status != 200:
+                raise ConnectionError(f"HTTP {status}")
+            return body, headers, None
+
+        async def fetch(i: int, url: str) -> None:
+            data: Optional[bytes] = None
+            keys = ""
+            unchanged = False
+            try:
+                data, headers, cached = await download(url)
+                if cached is not None:
+                    keys, unchanged = cached, True
+                    view.cached += 1
+                else:
+                    view.bytes += len(data)
+                    if len(data) <= INLINE_PARSE_BYTES:
+                        keys = parse_blob(data, sources[url], parse_types)
+                    else:
+                        keys = await loop.run_in_executor(pool, parse_blob, data, sources[url], parse_types)
+                    cache.store(url, headers, keys, sources[url])
+            except Exception:  # source unreachable/broken – counts as a failure in the statistics
+                pass
+            n = parsed = 0
+            if keys:
+                parsed = keys.count("\n") + 1  # every type, also the ones this run doesn't want
+                for key in keys.split("\n"):
+                    if not key.startswith(prefixes):
+                        continue
+                    owners = index.get(key)
+                    if owners is None:
+                        index[key] = [i]
+                    else:
+                        owners.append(i)
+                    n += 1
+            quality.record_fetch(url, data, n, unchanged=unchanged, parsed=parsed)
+            if n:
+                res.ok_sources += 1
+                view.ok += 1
+            else:
+                view.failed += 1
+            view.unique = len(index)
+            view.advance()
+
+        await asyncio.gather(*(fetch(i, u) for i, u in enumerate(res.urls)))
+    return res
+
+
+def prioritize(res: ScrapeResult, quality: srcs.SourceStats, history: ProxyHistory, types) -> List[str]:
+    """Order of checking: known working proxies first, then by quality of the sources.
+
+    Within the same source quality, proxies that appear in more lists win.
+    """
+    wanted = set(types)
+    known = [k for k in history.ranked_keys() if split_key(k)[0] in wanted]
+    known_set = set(known)
+
+    scores = [quality.score(u) for u in res.urls]
+    keys = [k for k in res.index if k not in known_set]
+    random.shuffle(keys)  # shuffle real ties so every type makes progress
+    index = res.index
+
+    def priority(key: str) -> float:
+        owners = index[key]
+        if len(owners) == 1:
+            return scores[owners[0]]
+        return max(map(scores.__getitem__, owners)) + 0.002 * min(len(owners), 10)
+
+    prio = list(map(priority, keys))
+    order = sorted(range(len(keys)), key=prio.__getitem__, reverse=True)
+    return known + [keys[i] for i in order]
+
+
+def attribute_results(res: ScrapeResult, checked: List[str], working: Set[str]) -> Dict[str, Tuple[int, int]]:
+    """Attribute results to the sources -> url -> (checked, working)."""
+    n_checked = [0] * len(res.urls)
+    n_working = [0] * len(res.urls)
+    for key in checked:
+        ok = key in working
+        for i in res.index.get(key, ()):
+            n_checked[i] += 1
+            if ok:
+                n_working[i] += 1
+    return {u: (n_checked[i], n_working[i]) for i, u in enumerate(res.urls) if n_checked[i]}
+
+
+def best_sources(per_source: Dict[str, Tuple[int, int]], limit: int = 10, min_checked: int = 30):
+    return sorted(
+        ((w / c, w, c, u) for u, (c, w) in per_source.items() if c >= min_checked and w),
+        reverse=True,
+    )[:limit]
+
+
+# --------------------------------------------------------------------------- #
+# Checking
+# --------------------------------------------------------------------------- #
+
+@dataclass
+class CheckRun:
+    results: List[CheckResult] = field(default_factory=list)
+    checked: List[str] = field(default_factory=list)
+    working: Set[str] = field(default_factory=set)
+    interrupted: bool = False
+    reached_goal: bool = False
+    judge_switches: List[str] = field(default_factory=list)  # "old → new"
+    rechecked: int = 0  # checks that were repeated because a check target went down
+
+
+class JobQueue:
+    """Iterator over the jobs that still accepts stragglers after running empty.
+
+    A generator would be exhausted forever after the first StopIteration – a proxy that a still
+    running worker puts back after a check-target switch would then never be checked again."""
+
+    def __init__(self, jobs: Iterable[str]):
+        self.jobs = iter(jobs)
+        self.retry: List[str] = []
+
+    def __iter__(self):
+        return self
+
+    def __next__(self) -> str:
+        for key in self.jobs:
+            return key
+        if self.retry:
+            return self.retry.pop()
+        raise StopIteration
+
+
+async def run_checks(
+    jobs: List[str],
+    checker: Checker,
+    opts: RunOptions,
+    dashboard: CheckDashboard,
+    writer: ResultWriter,
+    geo: GeoResolver,
+    live_factory: Callable,
+    watch: Optional[JudgeWatch] = None,
+    providers: Optional[ProviderLookup] = None,
+    blocklist: Optional[Blocklist] = None,
+    quiet: bool = False,
+) -> CheckRun:
+    """Checks `jobs` with `opts.concurrency` parallel workers until everything is done, the goal is
+    reached or Ctrl+C is pressed.
+
+    With `watch` the check target is monitored: if it goes down, the checks since the last successful
+    probe are repeated and don't count for the source statistics.
+
+    quiet: runs in the background next to another live view (--serve-refill) – no spinner, and Ctrl+C
+    belongs to whoever runs in the foreground."""
+    loop = asyncio.get_running_loop()
+    stats = dashboard.s
+    filters, details, want = opts.filters, opts.details, opts.want
+    run = CheckRun()
+    written: Set[str] = set()
+    enriched: Set[str] = set()  # hits with a finished detail check (HTTPS may stay open)
+    by_exit_ip: Dict[str, List[CheckResult]] = {}
+    pending = JobQueue(jobs)  # all workers pull from the same queue – safe in asyncio without a lock
+
+    # Check target outages: every check remembers which target ("generation") it started with and its number.
+    # On a switch the old generation becomes suspicious from the last good probe on – failures from it
+    # are repeated, including those that only finish after the switch. Hits are never suspicious
+    # (the proxy did work), so every check is counted exactly once.
+    generation = 0
+    # Order instead of clock time: every check gets a running number when it starts. The event loop clock
+    # is only accurate to ~15 ms on Windows – two events could easily get the same timestamp there.
+    started_count = 0
+    last_ok = 0  # this many checks had started at the last good probe (0 = start of the run)
+    suspect_since: Dict[int, int] = {}       # replaced generation -> checks from this number on are suspicious
+    recent_failures: List[Tuple[int, int, str]] = []  # failures since the last good probe
+
+    def is_suspect(gen: int, started: int) -> bool:
+        return gen in suspect_since and started > suspect_since[gen]
+
+    def requeue(keys: List[str]) -> None:
+        pending.retry.extend(keys)
+        run.rechecked += len(keys)
+        dashboard.add_rechecks(keys)
+
+    def judge_ok() -> None:
+        nonlocal last_ok
+        last_ok = started_count
+        recent_failures.clear()
+
+    def judge_switched(old, new) -> None:
+        nonlocal generation, last_ok
+        suspect_since[generation] = last_ok
+        generation += 1
+        last_ok = started_count  # the new target was just probed successfully – the baseline from here on
+        failed = {key for gen, started, key in recent_failures if is_suspect(gen, started)}
+        recent_failures.clear()
+        if failed:
+            run.checked[:] = [k for k in run.checked if k not in failed]
+        requeue(sorted(failed))
+        run.judge_switches.append(f"{old.judge.host} → {new.judge.host}")
+        dashboard.judge_changed(new.judge.host)
+
+    if watch:
+        watch.on_ok, watch.on_switch = judge_ok, judge_switched
+    all_workers: Optional[asyncio.Future] = None
+
+    def consider(r: CheckResult) -> None:
+        """Live file & goal counter, as soon as all the info the filters need is there."""
+        if r.key in written or (details and r.key not in enriched):
+            return
+        if filters.countries and not r.country:
+            return  # country still coming – on_country calls again
+        if filters.accepts(r):
+            written.add(r.key)
+            stats.passing += 1
+            writer.add_live(r)
+            if want and stats.passing >= want and all_workers and not all_workers.done():
+                run.reached_goal = True
+                all_workers.cancel()
+
+    def on_country(ip: str, cc: str) -> None:
+        for r in by_exit_ip.get(ip, ()):
+            if not r.country:
+                r.country = cc
+                stats.countries[cc] += 1
+                consider(r)
+
+    geo.on_resolved = on_country
+    geo_task = asyncio.ensure_future(geo.run())
+    watch_task = asyncio.ensure_future(watch.run()) if watch else None
+
+    async def worker() -> None:
+        nonlocal started_count
+        for key in pending:
+            started_count += 1
+            gen, started = generation, started_count
+            r = await checker.check(key)
+            stats.add_checked(key.split(" ", 1)[0])
+            dashboard.advance()
+            if r is None and is_suspect(gen, started):
+                requeue([key])  # only finished after the switch – still a victim of the outage
+                continue
+            if r is None:
+                run.checked.append(key)
+                if watch:
+                    recent_failures.append((gen, started, key))
+                continue
+            # A hit only counts as checked once the verdict is in: a worker cancelled during the next requests
+            # (--want reached, Ctrl+C) must not record a proxy that just worked as a failure.
+            # second, independent request – honeypots often pass the first check by chance
+            if not await checker.confirm(r):
+                run.checked.append(key)
+                stats.fakes += 1
+                continue
+            # third request: does a known page arrive unchanged? Otherwise the proxy injects something
+            if await checker.tampers(r):
+                run.checked.append(key)
+                stats.tampered += 1
+                continue
+            if providers:
+                providers.annotate(r)
+            if blocklist:
+                r.blocklisted = await blocklist.lookup(r.exit_ip)
+            run.checked.append(key)
+            run.results.append(r)
+            run.working.add(key)
+            by_exit_ip.setdefault(r.exit_ip, []).append(r)
+            r.country = geo.request(r.exit_ip)
+            if r.country:
+                stats.countries[r.country] += 1
+            stats.add_working(r)
+            if details:
+                # HTTPS test only if the proxy can still pass the filters at all
+                if filters.may_pass(r):
+                    await checker.enrich(r)
+                    enriched.add(r.key)
+                    stats.add_details(r)
+                else:
+                    stats.details_saved += 1
+            consider(r)
+
+    with live_factory(dashboard):
+        workers = [asyncio.ensure_future(worker()) for _ in range(min(opts.concurrency, len(jobs)))]
+        all_workers = asyncio.gather(*workers)
+        # Ctrl+C stops cleanly so the results still get saved (on Windows too)
+        with contextlib.nullcontext() if quiet else on_interrupt(loop, all_workers.cancel):
+            try:
+                await all_workers
+            except asyncio.CancelledError:
+                run.interrupted = not run.reached_goal
+
+    if watch_task:
+        watch_task.cancel()
+    if quiet and run.interrupted:
+        # nobody pressed Ctrl+C here – the task running us was cancelled (the server stops), so pass it on
+        geo_task.cancel()
+        raise asyncio.CancelledError
+    # wait for open country lookups (briefly at most)
+    geo.stop()
+    if geo.pending and not geo.failed:
+        message = f"[bold {ACCENT}]Looking up countries for {fmt(len(geo.pending))} exit IPs …"
+        spinner = contextlib.nullcontext() if quiet else widgets.console.status(message, spinner="dots")
+        with spinner, contextlib.suppress(asyncio.TimeoutError):
+            await asyncio.wait_for(asyncio.shield(geo_task), 30)
+    geo_task.cancel()
+    return run