proxy-scraper-cli 1.7.1__py3-none-any.whl

proxyscraper/options.py

@@ -0,0 +1,274 @@
+"""All settings of a run in one place – whether they come from the command line or
+from the setup wizard."""
+
+from __future__ import annotations
+
+import shlex
+from dataclasses import dataclass, field
+from typing import List, Optional, Set
+
+from .checker import ANONYMITY_RANK, CheckResult
+from .exporters import EXPORTERS
+from .parsing import PROXY_TYPES
+from .paths import is_checkout
+from .server.pool import STRATEGIES
+from .targets import target_label
+
+DEFAULT_CONCURRENCY = 2000
+DEFAULT_TIMEOUT = 8.0
+DEFAULT_CONNECT_TIMEOUT = 4.0
+DEFAULT_DISCOVER_REPOS = 400
+DEFAULT_SERVE_PORT = 8899
+STDOUT = "-"  # -o -: hits to stdout
+
+
+def parse_countries(value: Optional[str]) -> Set[str]:
+    """'de, at,CH' -> {'DE', 'AT', 'CH'}"""
+    return {c.strip().upper() for c in (value or "").split(",") if c.strip()}
+
+
+@dataclass
+class Filters:
+    countries: Set[str] = field(default_factory=set)
+    https_only: bool = False
+    min_anonymity: str = ""
+    max_latency: int = 0
+    targets: List[str] = field(default_factory=list)  # target sites every proxy has to reach
+    no_datacenter: bool = False  # only exits that are not (recognizably) in a datacenter
+    no_blocklisted: bool = False  # only exits that are not on the SpamCop blocklist
+
+    @property
+    def needs_details(self) -> bool:
+        # anonymity comes from the confirmation (always runs) – HTTPS and target sites need the detail test
+        return self.https_only or bool(self.targets)
+
+    @property
+    def active(self) -> bool:
+        return bool(self.countries or self.https_only or self.min_anonymity or self.max_latency or self.targets
+                    or self.no_datacenter or self.no_blocklisted)
+
+    def accepts(self, r: CheckResult) -> bool:
+        if self.max_latency and r.latency > self.max_latency:
+            return False
+        if self.https_only and r.https is not True:
+            return False
+        if self.min_anonymity and ANONYMITY_RANK.get(r.anonymity, -1) < ANONYMITY_RANK[self.min_anonymity]:
+            return False
+        if self.countries and r.country not in self.countries:
+            return False
+        if self.no_datacenter and r.hosting:
+            return False
+        if self.no_blocklisted and r.blocklisted:
+            return False
+        return all(r.targets.get(url) for url in self.targets)  # reached every requested target site
+
+    def may_pass(self, r: CheckResult) -> bool:
+        """Can `r` still pass the filters? HTTPS is still unknown at this point, possibly the country too.
+
+        Whatever is sure to fail already doesn't need an expensive HTTPS test any more.
+        """
+        if self.max_latency and r.latency > self.max_latency:
+            return False
+        if self.min_anonymity and ANONYMITY_RANK.get(r.anonymity, -1) < ANONYMITY_RANK[self.min_anonymity]:
+            return False
+        if self.no_datacenter and r.hosting:
+            return False
+        if self.no_blocklisted and r.blocklisted:
+            return False
+        # country still unknown -> may still match
+        return not self.countries or not r.country or r.country in self.countries
+
+    def describe(self) -> str:
+        parts = []
+        if self.countries:
+            parts.append("country " + ",".join(sorted(self.countries)))
+        if self.https_only:
+            parts.append("HTTPS only")
+        if self.min_anonymity:
+            parts.append(f"min. {self.min_anonymity}")
+        if self.max_latency:
+            parts.append(f"≤ {self.max_latency} ms")
+        if self.targets:
+            parts.append("target " + ", ".join(target_label(u, self.targets) for u in self.targets))
+        if self.no_datacenter:
+            parts.append("no datacenters")
+        if self.no_blocklisted:
+            parts.append("not blocklisted")
+        return " · ".join(parts)
+
+
+@dataclass
+class RunOptions:
+    types: List[str] = field(default_factory=lambda: list(PROXY_TYPES))
+    filters: Filters = field(default_factory=Filters)
+    want: int = 0
+    limit: int = 0
+    fast: bool = False
+    no_geo: bool = False
+    recheck: Optional[str] = None
+    output: Optional[str] = None
+    exports: List[str] = field(default_factory=list)  # extra formats, see exporters.py
+    concurrency: int = DEFAULT_CONCURRENCY
+    timeout: float = DEFAULT_TIMEOUT
+    connect_timeout: float = DEFAULT_CONNECT_TIMEOUT
+    discover: bool = False
+    no_discover: bool = False
+    discover_repos: int = DEFAULT_DISCOVER_REPOS
+    all_sources: bool = False
+    no_cache: bool = False  # reload every list instead of taking unchanged ones from the cache
+    no_dnsbl: bool = False  # skip the blocklist lookup of the exit IPs
+    serve: int = 0  # port of the rotating proxy server after the run, 0 = off
+    rotate: str = "weighted"  # strategy of the proxy server, see server/pool.py
+    sticky: int = 0  # seconds a target site keeps the same proxy (0 = new one for every connection)
+    serve_host: str = "127.0.0.1"  # address of the proxy server; anything else is reachable from outside
+    serve_password: str = field(default="", repr=False)  # required from clients if set; never written to argv
+    serve_refill: float = 0  # hours between background refills of the proxy server's pool, 0 = off
+
+    def __post_init__(self) -> None:
+        unknown = set(self.types) - set(PROXY_TYPES)
+        if unknown:
+            raise ValueError(f"unknown proxy types: {', '.join(sorted(unknown))}")
+        # the types are really a set: fixed order, no duplicates.
+        # Otherwise "--types socks5 http" and "--types http socks5" would give different settings.
+        self.types = [t for t in PROXY_TYPES if t in self.types]
+        if not self.types:
+            raise ValueError("at least one proxy type needed")  # else to_argv() gives "--types" without a value
+        if self.concurrency < 1:
+            raise ValueError("concurrency must be at least 1")
+        if self.timeout <= 0 or self.connect_timeout <= 0:
+            raise ValueError("timeouts must be greater than 0")
+        if min(self.want, self.limit, self.discover_repos, self.filters.max_latency) < 0:
+            raise ValueError("amounts and latency must not be negative")
+        unknown_exports = set(self.exports) - set(EXPORTERS)
+        if unknown_exports:
+            raise ValueError(f"unknown export formats: {', '.join(sorted(unknown_exports))}")
+        self.exports = [e for e in EXPORTERS if e in self.exports]
+        if self.rotate not in STRATEGIES:
+            raise ValueError(f"unknown strategy: {self.rotate}")
+        if self.sticky < 0:
+            raise ValueError("--sticky must not be negative")
+        if self.serve_refill < 0:
+            raise ValueError("--serve-refill must not be negative")
+        if not 0 <= self.serve <= 65535:
+            raise ValueError("port must be between 1 and 65535")
+
+    @property
+    def details(self) -> bool:
+        """HTTPS test – can be switched off (--fast), unless the HTTPS filter needs it."""
+        return not self.fast or self.filters.needs_details
+
+    @property
+    def check_timeout(self) -> float:
+        """Timeout of the basic check: whatever exceeds the latency limit drops out anyway –
+        nobody needs to wait that long. With 2000 parallel slots that's a lot more throughput."""
+        if self.filters.max_latency:
+            return min(self.timeout, self.filters.max_latency / 1000)
+        return self.timeout
+
+    @property
+    def check_connect_timeout(self) -> float:
+        return min(self.connect_timeout, self.check_timeout)
+
+    @property
+    def geo(self) -> bool:
+        return not self.no_geo or bool(self.filters.countries)
+
+    @classmethod
+    def from_args(cls, args) -> "RunOptions":
+        return cls(
+            types=list(args.types),
+            filters=Filters(
+                countries=parse_countries(args.country),
+                https_only=args.https_only,
+                min_anonymity=args.anonymity or "",
+                max_latency=args.max_latency,
+                targets=list(dict.fromkeys(args.target or [])),
+                no_datacenter=args.no_datacenter,
+                no_blocklisted=args.no_blocklisted,
+            ),
+            want=args.want,
+            limit=args.limit,
+            fast=args.fast,
+            no_geo=args.no_geo,
+            recheck=args.recheck,
+            output=args.output,
+            exports=list(args.export or []),
+            concurrency=args.concurrency,
+            timeout=args.timeout,
+            connect_timeout=args.connect_timeout,
+            discover=args.discover,
+            no_discover=args.no_discover,
+            discover_repos=args.discover_repos,
+            all_sources=args.all_sources,
+            no_cache=args.no_cache,
+            no_dnsbl=args.no_dnsbl,
+            serve=args.serve,
+            rotate=args.rotate,
+            sticky=args.sticky,
+            serve_host=args.serve_host,
+            serve_password=args.serve_password,
+            serve_refill=args.serve_refill,
+        )
+
+    def to_argv(self) -> List[str]:
+        """Command line arguments that produce exactly these settings (only deviations from the defaults)."""
+        argv: List[str] = []
+        if self.types != list(PROXY_TYPES):
+            argv += ["--types", *self.types]
+        f = self.filters
+        if f.countries:
+            argv += ["--country", ",".join(sorted(f.countries))]
+        if f.https_only:
+            argv.append("--https-only")
+        if f.min_anonymity:
+            argv += ["--anonymity", f.min_anonymity]
+        if f.max_latency:
+            argv += ["--max-latency", str(f.max_latency)]
+        for url in f.targets:
+            argv += ["--target", url]
+        _flag(argv, "--no-datacenter", f.no_datacenter)
+        _flag(argv, "--no-blocklisted", f.no_blocklisted)
+        _opt(argv, "--want", self.want, 0)
+        _opt(argv, "--limit", self.limit, 0)
+        _flag(argv, "--fast", self.fast)
+        _flag(argv, "--no-geo", self.no_geo)
+        if self.recheck is not None:
+            argv += ["--recheck", self.recheck] if self.recheck else ["--recheck"]
+        if self.output:
+            argv += ["--output", self.output]
+        if self.exports:
+            argv += ["--export", ",".join(self.exports)]
+        _opt(argv, "--concurrency", self.concurrency, DEFAULT_CONCURRENCY)
+        _opt(argv, "--timeout", self.timeout, DEFAULT_TIMEOUT)
+        _opt(argv, "--connect-timeout", self.connect_timeout, DEFAULT_CONNECT_TIMEOUT)
+        _flag(argv, "--discover", self.discover)
+        _flag(argv, "--no-discover", self.no_discover)
+        _opt(argv, "--discover-repos", self.discover_repos, DEFAULT_DISCOVER_REPOS)
+        _flag(argv, "--all-sources", self.all_sources)
+        _flag(argv, "--no-cache", self.no_cache)
+        _flag(argv, "--no-dnsbl", self.no_dnsbl)
+        if self.serve:
+            argv += ["--serve"] if self.serve == DEFAULT_SERVE_PORT else ["--serve", str(self.serve)]
+        if self.rotate != "weighted":
+            argv += ["--rotate", self.rotate]
+        if self.serve_host != "127.0.0.1":
+            argv += ["--serve-host", self.serve_host]
+        _opt(argv, "--sticky", self.sticky, 0)
+        _opt(argv, "--serve-refill", float(self.serve_refill), 0.0)
+        return argv
+
+    def to_command(self, program: Optional[str] = None) -> str:
+        if program is None:  # started from the repo or installed?
+            program = "python3 proxy_scraper.py" if is_checkout() else "proxy-scraper"
+        return " ".join([program, *map(shlex.quote, self.to_argv())])
+
+
+def _opt(argv: List[str], name: str, value, default) -> None:
+    if value != default:
+        argv += [name, f"{value:g}" if isinstance(value, float) else str(value)]
+
+
+def _flag(argv: List[str], name: str, value: bool) -> None:
+    if value:
+        argv.append(name)
+

proxyscraper/output.py

@@ -0,0 +1,181 @@
+"""Result files.
+
+Every run gets its own folder results/<date>/ with
+  all.txt         type://ip:port, fastest first (also live during the run)
+  http.txt …      ip:port per protocol – ready for tools that just want a list
+  proxies.json    every detail (latency, country, HTTPS, anonymity, exit IP)
+  proxies.csv     the same as a table
+and results/latest.txt (or the symlink results/latest) always points to the newest run.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import csv
+import json
+import os
+from dataclasses import asdict
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional, Sequence, TextIO
+
+from . import paths
+from .checker import CheckResult
+from .exporters import EXPORTERS
+from .parsing import PROXY_TYPES
+from .paths import atomic_write
+from .targets import target_label
+
+
+def new_run_dir(base: Path) -> Path:
+    """Folder with a timestamp; two runs in the same second get -2, -3 … instead of overwriting each other."""
+    stamp = f"{datetime.now():%Y-%m-%d_%H-%M-%S}"
+    base.mkdir(parents=True, exist_ok=True)
+    for n in range(1, 1000):
+        path = base / (stamp if n == 1 else f"{stamp}-{n}")
+        try:
+            path.mkdir()
+        except FileExistsError:
+            continue
+        return path
+    raise FileExistsError(f"too many runs in one second: {stamp}")
+
+
+class ResultWriter:
+    def __init__(self, run_dir: Optional[Path] = None, extra_file: Optional[Path] = None,
+                 exports: Sequence[str] = (), stdout: Optional[TextIO] = None):
+        self.run_dir = run_dir or new_run_dir(paths.RESULTS_DIR)  # read now: the MCP server moves it
+        self.run_dir.mkdir(parents=True, exist_ok=True)
+        self.extra_file = extra_file
+        self.stdout = stdout  # -o -: the hits also go here, one type://ip:port per line
+        self.exports = list(exports)
+        self.live_path = self.run_dir / "all.txt"
+        self._live = self.live_path.open("w", encoding="utf-8")
+
+    def add_live(self, r: CheckResult) -> None:
+        self._live.write(f"{r.ptype}://{r.proxy}\n")
+        self._live.flush()
+
+    def close(self) -> None:
+        """Done without writing the result files (a refill in a scratch folder)."""
+        self._live.close()
+
+    def finalize(self, results: Iterable[CheckResult]) -> Dict[str, Path]:
+        self._live.close()
+        rows = sorted(results, key=lambda r: r.latency)
+        files: Dict[str, Path] = {}
+
+        lines = "".join(f"{r.ptype}://{r.proxy}\n" for r in rows)
+        self.live_path.write_text(lines, encoding="utf-8")
+        files["All (type://ip:port)"] = self.live_path
+        for t in PROXY_TYPES:
+            of_type = [r.proxy for r in rows if r.ptype == t]
+            if of_type:
+                path = self.run_dir / f"{t}.txt"
+                path.write_text("\n".join(of_type) + "\n", encoding="utf-8")
+                files[f"{t} (ip:port)"] = path
+
+        json_path = self.run_dir / "proxies.json"
+        json_path.write_text(json.dumps([_row(r) for r in rows], indent=1, ensure_ascii=False), encoding="utf-8")
+        files["Details (JSON)"] = json_path
+
+        csv_path = self.run_dir / "proxies.csv"
+        with csv_path.open("w", newline="", encoding="utf-8") as fh:
+            writer = csv.DictWriter(fh, fieldnames=list(_csv_row(rows[0]).keys()) if rows else ["proxy"])
+            writer.writeheader()
+            writer.writerows(_csv_row(r) for r in rows)
+        files["Details (CSV)"] = csv_path
+
+        now = datetime.now()
+        for name in self.exports:
+            filename, render = EXPORTERS[name]
+            path = self.run_dir / filename
+            path.write_text(render(rows, now), encoding="utf-8")
+            files[f"{name} ({filename})"] = path
+
+        if self.extra_file:
+            self.extra_file.parent.mkdir(parents=True, exist_ok=True)
+            self.extra_file.write_text(lines, encoding="utf-8")
+            files["Extra file (-o)"] = self.extra_file
+        if self.stdout is not None:
+            try:
+                self.stdout.write(lines)
+                self.stdout.flush()
+            except OSError:  # `| head` stopped reading early – BrokenPipeError, on Windows EINVAL
+                _silence(self.stdout)
+
+        _point_latest(self.run_dir)
+        return files
+
+
+def _silence(stream: TextIO) -> None:
+    """Point a closed pipe at devnull, so the interpreter doesn't fail flushing it again on exit."""
+    with contextlib.suppress(OSError, ValueError, AttributeError):
+        os.dup2(os.open(os.devnull, os.O_WRONLY), stream.fileno())
+
+
+def _row(r: CheckResult) -> dict:
+    d = asdict(r)
+    d.pop("key")
+    d["url"] = f"{r.ptype}://{r.proxy}"
+    return d
+
+
+def _csv_row(r: CheckResult) -> dict:
+    """Like _row, but flat: target sites as "google.com:ok;discord.com:no"."""
+    d = _row(r)
+    d["targets"] = ";".join(f"{target_label(u, r.targets)}:{'ok' if ok else 'no'}" for u, ok in r.targets.items())
+    return d
+
+
+LATEST_POINTER = "latest.txt"
+
+
+def _point_latest(run_dir: Path) -> None:
+    """results/latest.txt always names the newest run; results/latest is also a symlink.
+
+    The symlink is handy for looking around, but on Windows it needs admin or developer
+    rights – the pointer file works everywhere.
+    """
+    atomic_write(run_dir.parent / LATEST_POINTER, run_dir.name + "\n")
+    latest = run_dir.parent / "latest"
+    try:
+        if latest.is_symlink():
+            latest.unlink()
+        if not latest.exists():
+            os.symlink(run_dir.name, latest, target_is_directory=True)
+    except OSError:
+        pass  # no symlinks allowed – latest.txt is enough
+
+
+def latest_run_dir(results_dir: Optional[Path] = None) -> Optional[Path]:
+    results_dir = results_dir or paths.RESULTS_DIR
+    pointer = results_dir / LATEST_POINTER
+    try:
+        name = pointer.read_text(encoding="utf-8").strip()
+    except (OSError, UnicodeDecodeError):  # missing, no permission or broken -> try the symlink
+        name = ""
+    # only a folder name directly under results/ – empty, "..", or paths like "a/../.." would
+    # otherwise point to results/ itself or outside of it
+    if name and name not in (".", "..") and Path(name).name == name:
+        run_dir = results_dir / name
+        if run_dir.is_dir():
+            return run_dir
+    link = results_dir / "latest"
+    return link if link.is_dir() else None  # runs from older versions without latest.txt
+
+
+def has_latest_results(results_dir: Optional[Path] = None) -> bool:
+    """Is there a last run with hits? Doesn't read the whole file for that."""
+    run_dir = latest_run_dir(results_dir)
+    try:
+        return run_dir is not None and (run_dir / "all.txt").stat().st_size > 0
+    except OSError:
+        return False
+
+
+def latest_results(results_dir: Optional[Path] = None) -> List[str]:
+    """Proxies of the last run for --recheck without a file."""
+    run_dir = latest_run_dir(results_dir)
+    path = run_dir / "all.txt" if run_dir else None
+    return path.read_text(encoding="utf-8").splitlines() if path and path.exists() else []

proxyscraper/pages.py

@@ -0,0 +1,212 @@
+"""Static pages for search engines: one per protocol, per filter and per country, plus a sitemap.
+
+The website itself loads everything with JavaScript, so a search engine mostly sees an empty shell. These
+pages carry the proxies as plain HTML: "free socks5 proxy list" or "free proxies germany" land on a page
+that answers exactly that, and every page links back to the full, filterable list.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from html import escape
+from pathlib import Path
+from typing import Callable, Dict, List, Tuple
+
+SITE_URL = "https://maximilianfeix.github.io/proxy-scraper/"
+REPO_URL = "https://github.com/maximilianfeix/proxy-scraper"
+ROWS_PER_PAGE = 200
+SITEMAP_MIN = 3  # countries with fewer proxies still get their page, but aren't advertised in the sitemap
+
+# every country here always gets a page, so its URL never disappears between runs (empty ones say so)
+COUNTRIES = {
+    "AE": "the United Arab Emirates", "AR": "Argentina", "AT": "Austria", "AU": "Australia", "BD": "Bangladesh",
+    "BE": "Belgium", "BG": "Bulgaria", "BR": "Brazil", "CA": "Canada", "CH": "Switzerland", "CL": "Chile",
+    "CN": "China", "CO": "Colombia", "CZ": "Czechia", "DE": "Germany", "DK": "Denmark", "EC": "Ecuador",
+    "EE": "Estonia", "EG": "Egypt", "ES": "Spain", "FI": "Finland", "FR": "France", "GB": "the United Kingdom",
+    "GR": "Greece", "HK": "Hong Kong", "HR": "Croatia", "HU": "Hungary", "ID": "Indonesia", "IE": "Ireland",
+    "IL": "Israel", "IN": "India", "IQ": "Iraq", "IR": "Iran", "IT": "Italy", "JP": "Japan", "KE": "Kenya",
+    "KH": "Cambodia", "KR": "South Korea", "KZ": "Kazakhstan", "LT": "Lithuania", "LV": "Latvia", "MA": "Morocco",
+    "MX": "Mexico", "MY": "Malaysia", "NG": "Nigeria", "NL": "the Netherlands", "NO": "Norway", "NP": "Nepal",
+    "NZ": "New Zealand", "PE": "Peru", "PH": "the Philippines", "PK": "Pakistan", "PL": "Poland", "PT": "Portugal",
+    "RO": "Romania", "RS": "Serbia", "RU": "Russia", "SA": "Saudi Arabia", "SE": "Sweden", "SG": "Singapore",
+    "TH": "Thailand", "TR": "Turkey", "TW": "Taiwan", "UA": "Ukraine", "US": "the United States",
+    "VE": "Venezuela", "VN": "Vietnam", "ZA": "South Africa",
+}
+
+Filter = Callable[[dict], bool]
+
+
+def _short_name(country: str) -> str:
+    return country[4:] if country.startswith("the ") else country
+
+
+def page_specs() -> List[Tuple[str, str, str, Filter]]:
+    """(path, title, what one proxy is called in the lede, filter) for every page."""
+    specs: List[Tuple[str, str, str, Filter]] = [
+        ("http/", "Free HTTP proxy list", "HTTP proxies", lambda r: r["ptype"] == "http"),
+        ("socks4/", "Free SOCKS4 proxy list", "SOCKS4 proxies", lambda r: r["ptype"] == "socks4"),
+        ("socks5/", "Free SOCKS5 proxy list", "SOCKS5 proxies", lambda r: r["ptype"] == "socks5"),
+        ("https/", "Free HTTPS proxy list", "proxies that tunnel HTTPS with verified TLS",
+         lambda r: bool(r.get("https"))),
+        ("elite/", "Free elite proxy list", "elite proxies (no forwarded IP, no Via header)",
+         lambda r: r.get("anonymity") == "elite"),
+    ]
+    for cc, name in sorted(COUNTRIES.items(), key=lambda kv: _short_name(kv[1])):
+        specs.append((f"country/{cc.lower()}/", f"Free proxies in {_short_name(name)}",
+                      f"proxies in {name}", lambda r, cc=cc: r.get("country") == cc))
+    return specs
+
+
+CSS = """
+:root { --bg: #121113; --panel: #1A191C; --line: #2B292F; --text: #EDEBE6; --muted: #9A97A0; --signal: #D4F77A;
+        --signal-ink: #D4F77A; --on-signal: #121113; color-scheme: dark; }
+@media (prefers-color-scheme: light) {
+  :root { --bg: #EFEFEC; --panel: #FFFFFF; --line: #DAD9D4; --text: #121113; --muted: #5F5C66;
+          --signal-ink: #3F5A00; color-scheme: light; }
+}
+* { box-sizing: border-box; }
+body { margin: 0; background: var(--bg); color: var(--text); font: 400 16px/1.55 "Geist", ui-sans-serif, system-ui,
+       sans-serif; padding-inline: 16px; }
+.wrap { max-width: 1080px; margin: 0 auto; padding-block: 28px 64px; display: grid; gap: 36px; }
+a { color: var(--signal-ink); }
+:focus-visible { outline: 2px solid var(--signal-ink); outline-offset: 3px; border-radius: 6px; }
+.mono { font-family: "Geist Mono", ui-monospace, Menlo, monospace; }
+header { display: flex; justify-content: space-between; align-items: center; gap: 16px; flex-wrap: wrap; }
+.brand { display: flex; align-items: center; gap: 10px; color: var(--text); text-decoration: none; font-weight: 600;
+         letter-spacing: -.03em; font-size: 18px; }
+.brand img { width: 30px; height: 30px; border-radius: 9px; }
+h1 { margin: 0; font-size: clamp(36px, 6vw, 72px); line-height: 1; letter-spacing: -.05em; font-weight: 600;
+     text-wrap: balance; }
+.lede { margin: 16px 0 0; color: var(--muted); font-size: 18px; max-width: 62ch; }
+.actions { display: flex; gap: 10px; flex-wrap: wrap; margin-top: 22px; }
+.btn { display: inline-flex; align-items: center; padding: 10px 18px; border-radius: 99px; text-decoration: none;
+       font-weight: 500; border: 1px solid var(--line); color: var(--text); }
+.btn.primary { background: var(--signal); color: var(--on-signal); border-color: var(--signal); }
+.table { overflow-x: auto; background: var(--panel); border: 1px solid var(--line); border-radius: 16px; }
+table { border-collapse: collapse; width: 100%; min-width: 640px; font-size: 14px; }
+caption { text-align: left; padding: 14px 16px 0; color: var(--muted); font-size: 13px; }
+th { text-align: left; font-weight: 500; color: var(--muted); padding: 12px 16px;
+     border-bottom: 1px solid var(--line); }
+td { padding: 10px 16px; border-bottom: 1px solid var(--line); font-variant-numeric: tabular-nums;
+     white-space: nowrap; }
+tr:last-child td { border-bottom: 0; }
+.empty { padding: 28px 16px; color: var(--muted); }
+nav h2 { font-size: 15px; font-weight: 500; color: var(--muted); margin: 0 0 10px; }
+nav ul { list-style: none; margin: 0; padding: 0; display: flex; flex-wrap: wrap; gap: 8px; }
+nav a { display: inline-block; padding: 5px 12px; border: 1px solid var(--line); border-radius: 99px;
+        text-decoration: none; color: var(--text); font-size: 14px; }
+nav a[aria-current] { background: var(--signal); color: var(--on-signal); border-color: var(--signal); }
+footer { color: var(--muted); font-size: 14px; display: grid; gap: 6px; }
+"""
+
+
+def _render(path: str, title: str, what: str, rows: List[dict], updated: datetime, nav: str) -> str:
+    depth = path.count("/")
+    root = "../" * depth
+    total = len(rows)
+    when = updated.strftime("%d %b %Y, %H:%M UTC")
+    if total:
+        lede = (f"{total:,} {what} that worked in the last check ({when}). Each one passed a real handshake, "
+                "a honeypot check on two sites and a content check, so none of them rewrite the pages you load.")
+    else:
+        lede = (f"No {what} passed every check in the last run ({when}). The list is checked again every hour, "
+                "so look again later or try the full list.")
+    body_rows = "\n".join(
+        f"<tr><td class=\"mono\">{escape(r['proxy'])}</td><td>{escape(r['ptype'].upper())}</td>"
+        f"<td>{escape(r.get('country') or '–')}</td><td>{r['latency']:,} ms</td>"
+        f"<td>{'yes' if r.get('https') else 'no'}</td><td>{escape(r.get('anonymity') or '–')}</td>"
+        f"<td>{escape((r.get('org') or '–')[:40])}</td></tr>"
+        for r in rows[:ROWS_PER_PAGE])
+    table = (f"<div class=\"table\"><table><caption>The {min(total, ROWS_PER_PAGE):,} fastest, as ip:port. "
+             f"The download has all {total:,} as type://ip:port.</caption>"
+             "<thead><tr><th scope=\"col\">Proxy</th><th scope=\"col\">Type</th><th scope=\"col\">Country</th>"
+             "<th scope=\"col\">Latency</th><th scope=\"col\">HTTPS</th><th scope=\"col\">Anonymity</th>"
+             f"<th scope=\"col\">Provider</th></tr></thead><tbody>{body_rows}</tbody></table></div>"
+             if total else "")
+    description = (f"{total:,} free {what}, checked every hour: real handshake, honeypot and content check. "
+                   "Download as text or filter the full list.")
+    return f"""<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>{escape(title)} – checked every hour | proxy-scraper</title>
+<meta name="description" content="{escape(description)}">
+<link rel="canonical" href="{SITE_URL}{path}">
+<meta property="og:title" content="{escape(title)}">
+<meta property="og:description" content="{escape(description)}">
+<meta property="og:image" content="{SITE_URL}og.png">
+<meta property="og:url" content="{SITE_URL}{path}">
+<meta name="twitter:card" content="summary_large_image">
+<link rel="icon" href="{root}logo.png">
+<link rel="apple-touch-icon" href="{root}apple-touch-icon.png">
+<link rel="preconnect" href="https://fonts.googleapis.com">
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+<link href="https://fonts.googleapis.com/css2?family=Geist:wght@400..700&family=Geist+Mono:wght@400;500&display=swap"
+      rel="stylesheet">
+<style>{CSS}</style>
+</head>
+<body>
+<div class="wrap">
+  <header>
+    <a class="brand" href="{root}"><img src="{root}logo.png" alt="">proxy-scraper</a>
+    <a href="{root}">All proxies</a>
+  </header>
+  <main>
+    <h1>{escape(title)}</h1>
+    <p class="lede">{escape(lede)}</p>
+    <div class="actions">
+      <a class="btn primary" href="proxies.txt" download>Download {total:,} as .txt</a>
+      <a class="btn" href="{root}#list">Filter the full list</a>
+      <a class="btn" href="{REPO_URL}">Check them from your network</a>
+    </div>
+  </main>
+  {table}
+  {nav}
+  <footer>
+    <span>Free proxies are run by strangers. Never send passwords or personal data through them.</span>
+    <span>Collected and checked by <a href="{REPO_URL}">proxy-scraper</a>, open source, MIT licensed.</span>
+  </footer>
+</div>
+</body>
+</html>
+"""
+
+
+def _nav(specs, counts: Dict[str, int], current: str, root: str) -> str:
+    def links(items):
+        here = ' aria-current="page"'
+        return "".join(f'<li><a href="{root}{p}"{here if p == current else ""}>{escape(label)} ({counts[p]:,})</a></li>'
+                       for p, label in items)
+
+    kinds = [(p, t.replace("Free ", "").replace(" proxy list", ""))
+             for p, t, _, _ in specs if not p.startswith("country/")]
+    countries = sorted(((p, t.replace("Free proxies in ", "")) for p, t, _, _ in specs
+                        if p.startswith("country/") and counts[p]), key=lambda pt: -counts[pt[0]])
+    return (f"<nav aria-label=\"More lists\"><h2>By protocol</h2><ul>{links(kinds)}</ul></nav>"
+            f"<nav aria-label=\"By country\"><h2>By country</h2><ul>{links(countries)}</ul></nav>")
+
+
+def write_pages(rows: List[dict], out: Path, updated: datetime) -> List[str]:
+    """Writes every page with its proxies.txt and the sitemap. -> the paths listed in the sitemap."""
+    rows = sorted(rows, key=lambda r: r["latency"])
+    specs = page_specs()
+    selected = {p: [r for r in rows if keep(r)] for p, _, _, keep in specs}
+    counts = {p: len(v) for p, v in selected.items()}
+    listed = [""]
+    for path, title, what, _ in specs:
+        page_rows = selected[path]
+        folder = out / path
+        folder.mkdir(parents=True, exist_ok=True)
+        nav = _nav(specs, counts, path, "../" * path.count("/"))
+        (folder / "index.html").write_text(_render(path, title, what, page_rows, updated, nav), encoding="utf-8")
+        (folder / "proxies.txt").write_text("".join(f"{r['url']}\n" for r in page_rows), encoding="utf-8")
+        if not path.startswith("country/") or counts[path] >= SITEMAP_MIN:
+            listed.append(path)
+    lastmod = updated.strftime("%Y-%m-%dT%H:%M:%S+00:00")
+    urls = "".join(f"<url><loc>{SITE_URL}{p}</loc><lastmod>{lastmod}</lastmod><changefreq>hourly</changefreq></url>"
+                   for p in listed)
+    (out / "sitemap.xml").write_text('<?xml version="1.0" encoding="UTF-8"?>\n'
+                                     f'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">{urls}</urlset>\n',
+                                     encoding="utf-8")
+    return listed