proxy-scraper-cli 1.8.1__tar.gz → 1.9.0__tar.gz

{proxy_scraper_cli-1.8.1 → proxy_scraper_cli-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.8.1
+Version: 1.9.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT
@@ -578,6 +578,84 @@ for p in proxies:
         continue  # free proxies come and go – just take the next one
 ```
 
+**httpx** (`pip install "httpx[socks]"`) – straight from the hourly list, no scan
+
+```python
+import httpx
+from proxyscraper import live_proxies
+
+for p in live_proxies(types=["http", "socks5"], https=True, min_uptime=90, limit=10):  # httpx can't do SOCKS4
+    try:
+        with httpx.Client(proxy=p.url, timeout=10) as client:
+            print(p.url, "→", client.get("https://api.ipify.org").text)
+        break
+    except httpx.HTTPError:
+        continue  # next one
+```
+
+**aiohttp** (`pip install aiohttp aiohttp-socks` – aiohttp alone can't do SOCKS)
+
+```python
+import asyncio
+
+import aiohttp
+from aiohttp_socks import ProxyConnector, ProxyError
+from proxyscraper import live_proxies_async
+
+
+async def main():
+    for p in await live_proxies_async(https=True, min_uptime=90, limit=10):
+        try:
+            async with aiohttp.ClientSession(connector=ProxyConnector.from_url(p.url)) as session:
+                async with session.get("https://api.ipify.org", timeout=aiohttp.ClientTimeout(total=10)) as r:
+                    print(p.url, "→", await r.text())
+                    return
+        except (aiohttp.ClientError, ProxyError, asyncio.TimeoutError, OSError):
+            continue  # free proxies come and go – just take the next one
+
+asyncio.run(main())
+```
+
+**Scrapy** – a downloader middleware that sends every request, retries included, through the next proxy. Scrapy only speaks HTTP proxies, `https=True` picks the ones that can tunnel `https://` pages
+
+```python
+import itertools
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from proxyscraper import live_proxies
+
+PROXIES = [p.url for p in live_proxies(types="http", https=True, min_uptime=50)]
+if not PROXIES:
+    raise SystemExit("No proxy matches right now – loosen the filters (e.g. min_uptime)")
+POOL = itertools.cycle(PROXIES)
+
+
+class RotatingProxy:
+    def process_request(self, request, spider=None):  # newer Scrapy leaves out spider
+        request.meta["proxy"] = next(POOL)
+
+
+class IpSpider(scrapy.Spider):
+    name = "ip"
+    start_urls = [f"https://api.ipify.org/?n={i}" for i in range(3)]
+    custom_settings = {
+        "DOWNLOADER_MIDDLEWARES": {f"{__name__}.RotatingProxy": 350},
+        "RETRY_TIMES": 5, "DOWNLOAD_TIMEOUT": 15,
+    }
+
+    def parse(self, response):
+        print(response.meta["proxy"], "→", response.text)
+
+
+if __name__ == "__main__":
+    process = CrawlerProcess()
+    process.crawl(IpSpider)
+    process.start()
+```
+
+In a Scrapy project, put `RotatingProxy` in `middlewares.py` and add it to `DOWNLOADER_MIDDLEWARES` in `settings.py`. For long crawls, reload the pool now and then – the list changes every hour.
+
 **proxychains, Clash / Mihomo** – ready-made configs with `--export`
 
 ```bash

{proxy_scraper_cli-1.8.1 → proxy_scraper_cli-1.9.0}/README.md

@@ -538,6 +538,84 @@ for p in proxies:
         continue  # free proxies come and go – just take the next one
 ```
 
+**httpx** (`pip install "httpx[socks]"`) – straight from the hourly list, no scan
+
+```python
+import httpx
+from proxyscraper import live_proxies
+
+for p in live_proxies(types=["http", "socks5"], https=True, min_uptime=90, limit=10):  # httpx can't do SOCKS4
+    try:
+        with httpx.Client(proxy=p.url, timeout=10) as client:
+            print(p.url, "→", client.get("https://api.ipify.org").text)
+        break
+    except httpx.HTTPError:
+        continue  # next one
+```
+
+**aiohttp** (`pip install aiohttp aiohttp-socks` – aiohttp alone can't do SOCKS)
+
+```python
+import asyncio
+
+import aiohttp
+from aiohttp_socks import ProxyConnector, ProxyError
+from proxyscraper import live_proxies_async
+
+
+async def main():
+    for p in await live_proxies_async(https=True, min_uptime=90, limit=10):
+        try:
+            async with aiohttp.ClientSession(connector=ProxyConnector.from_url(p.url)) as session:
+                async with session.get("https://api.ipify.org", timeout=aiohttp.ClientTimeout(total=10)) as r:
+                    print(p.url, "→", await r.text())
+                    return
+        except (aiohttp.ClientError, ProxyError, asyncio.TimeoutError, OSError):
+            continue  # free proxies come and go – just take the next one
+
+asyncio.run(main())
+```
+
+**Scrapy** – a downloader middleware that sends every request, retries included, through the next proxy. Scrapy only speaks HTTP proxies, `https=True` picks the ones that can tunnel `https://` pages
+
+```python
+import itertools
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from proxyscraper import live_proxies
+
+PROXIES = [p.url for p in live_proxies(types="http", https=True, min_uptime=50)]
+if not PROXIES:
+    raise SystemExit("No proxy matches right now – loosen the filters (e.g. min_uptime)")
+POOL = itertools.cycle(PROXIES)
+
+
+class RotatingProxy:
+    def process_request(self, request, spider=None):  # newer Scrapy leaves out spider
+        request.meta["proxy"] = next(POOL)
+
+
+class IpSpider(scrapy.Spider):
+    name = "ip"
+    start_urls = [f"https://api.ipify.org/?n={i}" for i in range(3)]
+    custom_settings = {
+        "DOWNLOADER_MIDDLEWARES": {f"{__name__}.RotatingProxy": 350},
+        "RETRY_TIMES": 5, "DOWNLOAD_TIMEOUT": 15,
+    }
+
+    def parse(self, response):
+        print(response.meta["proxy"], "→", response.text)
+
+
+if __name__ == "__main__":
+    process = CrawlerProcess()
+    process.crawl(IpSpider)
+    process.start()
+```
+
+In a Scrapy project, put `RotatingProxy` in `middlewares.py` and add it to `DOWNLOADER_MIDDLEWARES` in `settings.py`. For long crawls, reload the pool now and then – the list changes every hour.
+
 **proxychains, Clash / Mihomo** – ready-made configs with `--export`
 
 ```bash

{proxy_scraper_cli-1.8.1 → proxy_scraper_cli-1.9.0}/proxy_scraper_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.8.1
+Version: 1.9.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT
@@ -578,6 +578,84 @@ for p in proxies:
         continue  # free proxies come and go – just take the next one
 ```
 
+**httpx** (`pip install "httpx[socks]"`) – straight from the hourly list, no scan
+
+```python
+import httpx
+from proxyscraper import live_proxies
+
+for p in live_proxies(types=["http", "socks5"], https=True, min_uptime=90, limit=10):  # httpx can't do SOCKS4
+    try:
+        with httpx.Client(proxy=p.url, timeout=10) as client:
+            print(p.url, "→", client.get("https://api.ipify.org").text)
+        break
+    except httpx.HTTPError:
+        continue  # next one
+```
+
+**aiohttp** (`pip install aiohttp aiohttp-socks` – aiohttp alone can't do SOCKS)
+
+```python
+import asyncio
+
+import aiohttp
+from aiohttp_socks import ProxyConnector, ProxyError
+from proxyscraper import live_proxies_async
+
+
+async def main():
+    for p in await live_proxies_async(https=True, min_uptime=90, limit=10):
+        try:
+            async with aiohttp.ClientSession(connector=ProxyConnector.from_url(p.url)) as session:
+                async with session.get("https://api.ipify.org", timeout=aiohttp.ClientTimeout(total=10)) as r:
+                    print(p.url, "→", await r.text())
+                    return
+        except (aiohttp.ClientError, ProxyError, asyncio.TimeoutError, OSError):
+            continue  # free proxies come and go – just take the next one
+
+asyncio.run(main())
+```
+
+**Scrapy** – a downloader middleware that sends every request, retries included, through the next proxy. Scrapy only speaks HTTP proxies, `https=True` picks the ones that can tunnel `https://` pages
+
+```python
+import itertools
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from proxyscraper import live_proxies
+
+PROXIES = [p.url for p in live_proxies(types="http", https=True, min_uptime=50)]
+if not PROXIES:
+    raise SystemExit("No proxy matches right now – loosen the filters (e.g. min_uptime)")
+POOL = itertools.cycle(PROXIES)
+
+
+class RotatingProxy:
+    def process_request(self, request, spider=None):  # newer Scrapy leaves out spider
+        request.meta["proxy"] = next(POOL)
+
+
+class IpSpider(scrapy.Spider):
+    name = "ip"
+    start_urls = [f"https://api.ipify.org/?n={i}" for i in range(3)]
+    custom_settings = {
+        "DOWNLOADER_MIDDLEWARES": {f"{__name__}.RotatingProxy": 350},
+        "RETRY_TIMES": 5, "DOWNLOAD_TIMEOUT": 15,
+    }
+
+    def parse(self, response):
+        print(response.meta["proxy"], "→", response.text)
+
+
+if __name__ == "__main__":
+    process = CrawlerProcess()
+    process.crawl(IpSpider)
+    process.start()
+```
+
+In a Scrapy project, put `RotatingProxy` in `middlewares.py` and add it to `DOWNLOADER_MIDDLEWARES` in `settings.py`. For long crawls, reload the pool now and then – the list changes every hour.
+
 **proxychains, Clash / Mihomo** – ready-made configs with `--export`
 
 ```bash

{proxy_scraper_cli-1.8.1 → proxy_scraper_cli-1.9.0}/proxy_scraper_cli.egg-info/SOURCES.txt

@@ -36,7 +36,9 @@ proxyscraper/parsing.py
 proxyscraper/paths.py
 proxyscraper/pipeline.py
 proxyscraper/preferences.py
+proxyscraper/proxypages.py
 proxyscraper/publish.py
+proxyscraper/report.py
 proxyscraper/sites.py
 proxyscraper/sources.json
 proxyscraper/sources.py
@@ -86,8 +88,10 @@ tests/test_pages.py
 tests/test_parsing.py
 tests/test_paths.py
 tests/test_pipeline.py
+tests/test_proxy_pages.py
 tests/test_proxy_sources.py
 tests/test_publish.py
+tests/test_report.py
 tests/test_robustness.py
 tests/test_serve_password.py
 tests/test_serve_refill.py

{proxy_scraper_cli-1.8.1 → proxy_scraper_cli-1.9.0}/proxyscraper/__init__.py

@@ -1,7 +1,7 @@
 """proxy-scraper: collects public HTTP/SOCKS4/SOCKS5 proxies from hundreds of sources, checks them
 in parallel with its own protocol handshakes and learns which sources deliver good proxies."""
 
-__version__ = "1.8.1"
+__version__ = "1.9.0"
 
 
 def __getattr__(name):

{proxy_scraper_cli-1.8.1 → proxy_scraper_cli-1.9.0}/proxyscraper/pages.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 from datetime import datetime
 from html import escape
 from pathlib import Path
-from typing import Callable, Dict, List, Tuple
+from typing import Callable, Dict, List, Sequence, Tuple
 
 SITE_URL = "https://maximilianfeix.github.io/proxy-scraper/"
 REPO_URL = "https://github.com/maximilianfeix/proxy-scraper"
@@ -121,7 +121,8 @@ def _render(path: str, title: str, what: str, rows: List[dict], updated: datetim
         lede = (f"No {what} passed every check in the last run ({when}). The list is checked again every hour, "
                 "so look again later or try the full list.")
     body_rows = "\n".join(
-        f"<tr><td class=\"mono\">{escape(r['proxy'])}</td><td>{escape(r['ptype'].upper())}</td>"
+        f"<tr><td class=\"mono\"><a href=\"{root}proxy/{escape(r['proxy'].replace(':', '-'))}/\">"
+        f"{escape(r['proxy'])}</a></td><td>{escape(r['ptype'].upper())}</td>"
         f"<td>{escape(r.get('country') or '–')}</td><td>{r['latency']:,} ms</td>"
         f"<td>{'yes' if r.get('https') else 'no'}</td><td>{escape(r.get('anonymity') or '–')}</td>"
         f"<td>{escape((r.get('org') or '–')[:40])}</td></tr>"
@@ -196,8 +197,9 @@ def _nav(specs, counts: Dict[str, int], current: str, root: str) -> str:
             f"<nav aria-label=\"By country\"><h2>By country</h2><ul>{links(countries)}</ul></nav>")
 
 
-def write_pages(rows: List[dict], out: Path, updated: datetime) -> List[str]:
-    """Writes every page with its proxies.txt and the sitemap. -> the paths listed in the sitemap."""
+def write_pages(rows: List[dict], out: Path, updated: datetime, extra: Sequence[str] = ()) -> List[str]:
+    """Writes every page with its proxies.txt and the sitemap (plus the `extra` paths, e.g. the proxy pages)
+    -> the paths listed in the sitemap."""
     rows = sorted(rows, key=lambda r: r["latency"])
     specs = page_specs()
     selected = {p: [r for r in rows if keep(r)] for p, _, _, keep in specs}
@@ -212,6 +214,7 @@ def write_pages(rows: List[dict], out: Path, updated: datetime) -> List[str]:
         (folder / "proxies.txt").write_text("".join(f"{r['url']}\n" for r in page_rows), encoding="utf-8")
         if not path.startswith("country/") or counts[path] >= SITEMAP_MIN:
             listed.append(path)
+    listed.extend(extra)
     lastmod = updated.strftime("%Y-%m-%dT%H:%M:%S+00:00")
     urls = "".join(f"<url><loc>{SITE_URL}{p}</loc><lastmod>{lastmod}</lastmod><changefreq>hourly</changefreq></url>"
                    for p in listed)

{proxy_scraper_cli-1.8.1 → proxy_scraper_cli-1.9.0}/proxyscraper/parsing.py

@@ -44,8 +44,8 @@ _SPACE_SEPARATED_RE = re.compile(rb"\d\.\d{1,3}[ \t]+\d{2,5}(?![\d.])")
 
 
 # Bump whenever parsing finds different proxies in the same bytes: the fetch cache stores parsed keys and
-# reloads every list once when this changes (2: plain lists of "auto" sources are read as http)
-PARSER_VERSION = 2
+# reloads every list once when this changes (2: plain lists of "auto" sources are read, 3: as http and socks5)
+PARSER_VERSION = 3
 
 
 def _needs_full_regex(data: bytes) -> bool:
@@ -59,8 +59,9 @@ def extract_candidates(data: bytes, default_type: str) -> Set[RawCandidate]:
 
     Lines with a type:// prefix keep their own type; everything else gets the type of the source.
     For sources of type "auto" the plain lines only count when they are the majority: a generic proxies.txt of
-    plain ip:port lines is read as HTTP, the most common kind (the checks sort out the rest). In a list of mostly
-    prefixed lines, the odd plain one has an unknown type and is left out.
+    plain ip:port lines doesn't say its protocol, so they're tried as HTTP and as SOCKS5 (measured on 20,000 such
+    entries: 44 worked as http, 32 as socks5, none as both; an unreachable address fails fast for both). In a list
+    of mostly prefixed lines, the odd plain one has an unknown type and is left out.
     """
     out: Set[RawCandidate] = set()
     with_scheme = set()
@@ -71,8 +72,6 @@ def extract_candidates(data: bytes, default_type: str) -> Set[RawCandidate]:
                 out.add((ptype, ip, port, auth))
                 with_scheme.add((ip, port))
     auto = default_type == "auto"
-    if auto:
-        default_type = "http"
     regex = PROXY_RE if _needs_full_regex(data) else PLAIN_RE
     pairs = set(regex.findall(data))
     if b'"ip"' in data:
@@ -81,7 +80,8 @@ def extract_candidates(data: bytes, default_type: str) -> Set[RawCandidate]:
     plain = pairs - with_scheme
     if auto and len(plain) <= len(out):
         return out
-    out.update((default_type, ip, port, b"") for ip, port in plain)
+    for ptype in ("http", "socks5") if auto else (default_type,):
+        out.update((ptype, ip, port, b"") for ip, port in plain)
     return out
 
 

proxy_scraper_cli-1.9.0/proxyscraper/proxypages.py

@@ -0,0 +1,241 @@
+"""A page per proxy on the live list: what it is, how reliable it has been this week, which big sites let it
+through, and the commands to try it yourself.
+
+Only proxies that were on the list for at least half of the week's runs are indexed – thousands of pages that
+come and go every hour would look like spam to search engines. The others get a page too, with noindex.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from datetime import datetime
+from html import escape
+from pathlib import Path
+from typing import Dict, List, Optional, Sequence, Tuple
+
+from .pages import COUNTRIES, CSS, SITE_URL, _short_name
+
+INDEX_MIN_UPTIME = 50  # percent of the week's runs
+SITE_NAMES = {"google": "Google", "reddit": "Reddit", "amazon": "Amazon"}
+
+EXTRA_CSS = """
+.wrap > * { min-width: 0; }
+h1.mono { font-size: clamp(28px, 7vw, 64px); letter-spacing: -.03em; overflow-wrap: anywhere; }
+.facts { display: grid; grid-template-columns: repeat(auto-fill, minmax(min(100%, 160px), 1fr)); gap: 1px;
+         background: var(--line);
+         border: 1px solid var(--line); border-radius: 16px; overflow: hidden; margin: 0; }
+.facts div { background: var(--panel); padding: 14px 16px; }
+.facts dt { color: var(--muted); font-size: 13px; }
+.facts dd { margin: 4px 0 0; font-size: 17px; }
+h2 { font-size: 22px; letter-spacing: -.02em; margin: 0 0 12px; font-weight: 600; }
+.timeline { display: flex; gap: 2px; align-items: stretch; height: 34px; }
+.cell { flex: 1; min-width: 2px; border-radius: 2px; background: var(--line); }
+.cell.on { background: var(--signal-ink); }
+.axis { display: flex; justify-content: space-between; color: var(--muted); font-size: 13px; margin-top: 6px; }
+.cmd code { white-space: pre-wrap; word-break: break-all; }
+.cmd { position: relative; background: var(--panel); border: 1px solid var(--line); border-radius: 12px;
+       padding: 12px 56px 12px 14px; margin: 0 0 10px; overflow-x: auto; font-size: 14px; }
+.cmd button { position: absolute; top: 8px; right: 8px; font: inherit; font-size: 13px; padding: 4px 10px;
+              border-radius: 99px; border: 1px solid var(--line); background: var(--bg); color: var(--text);
+              cursor: pointer; }
+.lede { overflow-wrap: anywhere; }
+.note { color: var(--muted); font-size: 14px; }
+.status { display: inline-flex; align-items: center; gap: 8px; color: var(--muted); }
+.status::before { content: ""; width: 9px; height: 9px; border-radius: 50%; background: var(--signal-ink); }
+.status.off::before { background: var(--muted); }
+"""
+
+COPY_JS = """<script>
+document.querySelectorAll(".cmd button").forEach(b => b.onclick = () => {
+  navigator.clipboard.writeText(b.previousElementSibling.textContent).then(() => {
+    b.textContent = "Copied"; setTimeout(() => { b.textContent = "Copy"; }, 1500);
+  });
+});
+</script>"""
+
+
+def page_path(proxy: str) -> str:
+    """'1.2.3.4:1080' -> 'proxy/1.2.3.4-1080/'"""
+    return "proxy/" + proxy.replace(":", "-") + "/"
+
+
+def _country(cc: str) -> str:
+    return _short_name(COUNTRIES[cc]) if cc in COUNTRIES else (cc or "unknown")
+
+
+def _commands(row: dict) -> List[str]:
+    target = "https://api.ipify.org" if row.get("https") else "http://api.ipify.org"
+    scheme = "socks5h" if row["ptype"] == "socks5" else row["ptype"]  # socks5h: DNS through the proxy too
+    return [f"curl -x {scheme}://{row['proxy']} {target}",
+            f'python -c "import requests; print(requests.get(\'{target}\', '
+            f'proxies={{\'http\': \'{row["url"]}\', \'https\': \'{row["url"]}\'}}, timeout=10).text)"']
+
+
+def _cmd(command: str) -> str:
+    return f'<div class="cmd"><code class="mono">{escape(command)}</code><button type="button">Copy</button></div>'
+
+
+def _timeline(bits: Optional[int], runs: Sequence[datetime]) -> str:
+    """One cell per run of the week, oldest left – filled where the proxy was on the list."""
+    if bits is None or not runs:
+        return ""
+    bits &= (1 << len(runs)) - 1  # bits seeded from a streak can reach further back than the history
+    cells = "".join(f'<span class="cell{" on" if bits >> k & 1 else ""}"></span>'
+                    for k in range(len(runs) - 1, -1, -1))  # bit 0 = the newest run
+    first = runs[0].strftime("%b %d, %H:%M UTC")
+    return (f'<section aria-label="Timeline"><h2>Every check this week</h2>'
+            f'<div class="timeline" role="img" aria-label="On the list in {bin(bits).count("1")} '
+            f'of the last {len(runs)} hourly checks">{cells}</div>'
+            f'<div class="axis"><span>{escape(first)}</span><span>now</span></div></section>')
+
+
+def _summary(entries: List[dict]) -> dict:
+    """One address, several protocols: the best of each value (the page is indexed by the best uptime too)."""
+    def first(key):
+        return next((r[key] for r in entries if r.get(key) not in (None, "")), None)
+    uptimes = [r["uptime_7d"] for r in entries if r.get("uptime_7d") is not None]
+    https = [r.get("https") for r in entries]
+    rank = {"elite": 3, "anonymous": 2, "transparent": 1}
+    sites = {k for r in entries for k, ok in (r.get("sites") or {}).items() if ok}
+    return {
+        "uptime": max(uptimes) if uptimes else None,
+        "latency": min(r["latency"] for r in entries) if all("latency" in r for r in entries) else None,
+        "https": True if True in https else False if False in https else None,
+        "anonymity": max((r.get("anonymity") or "" for r in entries), key=lambda a: rank.get(a, 0)) or None,
+        "sites": [SITE_NAMES[k] for k in SITE_NAMES if k in sites],
+        "country": first("country") or "", "org": first("org"), "asn": first("asn"), "exit_ip": first("exit_ip"),
+        "hosting": first("hosting"), "blocklisted": first("blocklisted"),
+        "first_seen": min((r["first_seen"] for r in entries if r.get("first_seen")), default=None),
+    }
+
+
+def _yes_no(value, yes: str, no: str) -> str:
+    return yes if value else no if value is False else "unknown"
+
+
+def _render(proxy: str, entries: List[dict], updated: datetime, bits: Optional[int], runs: Sequence[datetime],
+            indexed: bool, gone: bool = False) -> str:
+    root = "../../"
+    info = _summary(entries)
+    urls = " · ".join(escape(r["url"]) for r in entries)
+    cc = info["country"]
+    uptime = info["uptime"]
+    first_seen = (info["first_seen"][:16].replace("T", " ") + " UTC") if info["first_seen"] else "unknown"
+    facts = [("Protocol", ", ".join(r["ptype"].upper() for r in entries)),
+             ("Uptime this week", f"{uptime} %" if uptime is not None else "not measured yet")]
+    if not gone:
+        facts[1:1] = [
+            ("Country", _country(cc)),
+            ("Provider", f"{info['org'] or 'unknown'}" + (f" (AS{info['asn']})" if info["asn"] else "")),
+            ("Latency", f"{info['latency']:,} ms" if info["latency"] is not None else "unknown"),
+            ("HTTPS", _yes_no(info["https"], "yes, verified TLS", "no")),
+            ("Anonymity", info["anonymity"] or "unknown"),
+        ]
+        facts += [
+            ("Gets through to", ", ".join(info["sites"]) if info["sites"] else "none of the checked sites"),
+            ("Datacenter exit", _yes_no(info["hosting"], "yes", "no")),
+            ("Spam blocklist", _yes_no(info["blocklisted"], "listed", "not listed")),
+            ("First seen", first_seen),
+            ("Exit IP", info["exit_ip"] or "unknown"),
+        ]
+    else:
+        facts.append(("First seen", first_seen))
+    facts_html = "".join(f"<div><dt>{escape(k)}</dt><dd>{escape(v)}</dd></div>" for k, v in facts)
+    fastest = min(entries, key=lambda r: r.get("latency") or 0)
+    commands = "".join(_cmd(_commands(r)[0]) for r in entries)
+    python = _commands(fastest)[1]
+    where = f" ({_country(cc)})" if cc else ""
+    title = f"{'/'.join(r['ptype'].upper() for r in entries)} proxy {proxy}{where}"
+    reliability = f", on the list in {uptime} % of this week's hourly checks" if uptime is not None else ""
+    description = (f"Free {', '.join(r['ptype'].upper() for r in entries)} proxy {proxy}"
+                   f"{' in ' + _country(cc) if cc else ''}{reliability}. "
+                   "Latency, provider, HTTPS support and commands to test it.")
+    robots = "" if indexed else '<meta name="robots" content="noindex">\n'
+    checked = updated.strftime("%Y-%m-%d %H:%M UTC")
+    if gone:
+        status = (f'<span class="status off">Didn\'t pass the last check ({escape(checked)})</span><br>'
+                  "It was on the list for most of the week, so it may well come back.")
+        back = f'<a href="{root}#list">All working proxies</a>'
+    else:
+        status = f'<span class="status">Worked in the check at {escape(checked)}</span>'
+        back = f'<a href="{root}?country={escape(cc)}#list">All proxies in {escape(_country(cc))}</a>'
+    return f"""<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>{escape(title)} | proxy-scraper</title>
+<meta name="description" content="{escape(description)}">
+{robots}<link rel="canonical" href="{SITE_URL}{page_path(proxy)}">
+<link rel="icon" href="{root}logo.png">
+<link rel="preconnect" href="https://fonts.googleapis.com">
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+<link href="https://fonts.googleapis.com/css2?family=Geist:wght@400..700&family=Geist+Mono:wght@400;500&display=swap"
+      rel="stylesheet">
+<link rel="stylesheet" href="../style.css">
+</head>
+<body>
+<div class="wrap">
+  <header>
+    <a class="brand" href="{root}"><img src="{root}logo.png" alt="">proxy-scraper</a>
+    {back}
+  </header>
+  <main>
+    <h1 class="mono">{escape(proxy)}</h1>
+    <p class="lede">{status}<br>{urls}</p>
+  </main>
+  <dl class="facts">{facts_html}</dl>
+  {_timeline(bits, runs)}
+  <section aria-label="Test it"><h2>Test it from your machine</h2>
+    <p class="note">Free proxies come and go, and some behave differently from your network. This prints the IP
+    a website sees – the proxy's, if it works.</p>
+    {commands}
+    {_cmd(python)}
+    <p class="note">Never send passwords or personal data through a public proxy. Its operator can read everything
+    that isn't encrypted.</p>
+  </section>
+  <footer><span>Checked every hour by <a href="https://github.com/maximilianfeix/proxy-scraper">proxy-scraper</a>,
+  open source.</span></footer>
+</div>
+{COPY_JS}
+</body>
+</html>
+"""
+
+
+def write_proxy_pages(rows: List[dict], out: Path, updated: datetime, timelines: Dict[str, int],
+                      runs: Sequence[datetime], gone: Optional[Dict[str, Tuple[int, int, str]]] = None) -> List[str]:
+    """A page per ip:port -> the paths that should be in the sitemap (the reliable ones).
+
+    timelines: url -> bits (bit 0 = this run), runs: the times of the week's runs, oldest first.
+    gone: url -> (bits, uptime, first_seen) of reliable proxies that missed this run – their page stays, so a
+    link from the sitemap doesn't turn into a 404 the first time a proxy misses a check."""
+    (out / "proxy").mkdir(parents=True, exist_ok=True)
+    (out / "proxy" / "style.css").write_text(CSS + EXTRA_CSS, encoding="utf-8")  # shared: ~1,500 pages
+    by_address: Dict[str, List[dict]] = defaultdict(list)
+    for r in rows:
+        by_address[r["proxy"]].append(r)
+    missing: Dict[str, List[dict]] = defaultdict(list)
+    for url, (bits, uptime, first_seen) in (gone or {}).items():
+        ptype, _, proxy = url.partition("://")
+        if proxy and proxy not in by_address and uptime >= INDEX_MIN_UPTIME:
+            missing[proxy].append({"url": url, "ptype": ptype, "proxy": proxy, "uptime_7d": uptime,
+                                   "first_seen": first_seen, "_bits": bits})
+    indexed = []
+    for proxy, entries, is_gone in [*((p, e, False) for p, e in by_address.items()),
+                                    *((p, e, True) for p, e in missing.items())]:
+        entries.sort(key=lambda r: r["ptype"])
+        uptime = _summary(entries)["uptime"]
+        is_indexed = uptime is not None and uptime >= INDEX_MIN_UPTIME
+        bits = None
+        for r in entries:
+            b = r.get("_bits", timelines.get(r["url"]))
+            if b is not None:
+                bits = (bits or 0) | b
+        folder = out / page_path(proxy)
+        folder.mkdir(parents=True, exist_ok=True)
+        (folder / "index.html").write_text(_render(proxy, entries, updated, bits, runs, is_indexed, is_gone),
+                                           encoding="utf-8")
+        if is_indexed:
+            indexed.append(page_path(proxy))
+    return sorted(indexed)

{proxy_scraper_cli-1.8.1 → proxy_scraper_cli-1.9.0}/proxyscraper/publish.py

@@ -24,6 +24,8 @@ from typing import Dict, List, Optional
 from .charts import THEMES, countries_svg, trend_svg
 from .pages import write_pages
 from .parsing import PROXY_TYPES
+from .proxypages import write_proxy_pages
+from .report import weekly_facts, write_report
 from .sites import SITES
 
 SKIP_EXIT_CODE = 78
@@ -294,8 +296,20 @@ def publish(run_dir: Path, out: Path, minimum: int = 20, now: Optional[datetime]
         if asset.suffix in (".html", ".png"):
             (out / asset.name).write_bytes(asset.read_bytes())
     (out / ".nojekyll").write_text("", encoding="utf-8")  # Pages should serve the files unchanged
-    write_pages(rows, out, now)  # static pages per protocol and country plus sitemap.xml, for search engines
+    # a page per proxy (with its week as a timeline), then the pages per protocol and country plus sitemap.xml
+    try:
+        run_times = [datetime.fromisoformat(run["updated"]) for run in runs[-UPTIME_RUNS:]]
+    except (KeyError, TypeError, ValueError):
+        run_times = []  # a broken entry: no timeline rather than one whose cells don't match the times
+    timelines = {url: e["bits"] for url, e in listed.items()} if run_times else {}
+    current = {r["url"] for r in rows}
+    gone = {url: (e["bits"], uptime(e["bits"], runs, now, 7 * 24), e["first_seen"])
+            for url, e in listed.items() if url not in current} if run_times else {}
+    proxy_pages = write_proxy_pages(rows, out, now, timelines, run_times, gone=gone)
+    write_pages(rows, out, now, extra=proxy_pages)
     write_json(out / "history.json", runs)
+    # the weekly report: lifetimes, sites, countries – a page, markdown and a short post (report.py)
+    write_report(weekly_facts(rows, runs, {url: e["bits"] for url, e in listed.items()}, now), out)
     for theme in THEMES:  # charts for the README, which embeds them from GitHub Pages
         (out / f"chart-{theme}.svg").write_text(trend_svg(runs, now, theme), encoding="utf-8")
         (out / f"countries-{theme}.svg").write_text(countries_svg(stats["countries"], theme), encoding="utf-8")