proxy-scraper-cli 1.11.0__tar.gz → 1.13.0__tar.gz

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.11.0
+Version: 1.13.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.13.0}/proxy_scraper_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.11.0
+Version: 1.13.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.13.0}/proxy_scraper_cli.egg-info/SOURCES.txt

@@ -40,6 +40,7 @@ proxyscraper/preferences.py
 proxyscraper/proxypages.py
 proxyscraper/publish.py
 proxyscraper/report.py
+proxyscraper/runsteps.py
 proxyscraper/sites.py
 proxyscraper/sources.json
 proxyscraper/sources.py

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.13.0}/proxyscraper/__init__.py

@@ -1,7 +1,7 @@
 """proxy-scraper: collects public HTTP/SOCKS4/SOCKS5 proxies from hundreds of sources, checks them
 in parallel with its own protocol handshakes and learns which sources deliver good proxies."""
 
-__version__ = "1.11.0"
+__version__ = "1.13.0"
 
 
 def __getattr__(name):

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.13.0}/proxyscraper/checker.py

@@ -334,6 +334,10 @@ class Checker:
             writer.close()
         return 200 <= status < 400
 
+    async def tls_tunnel(self, ptype: str, proxy: str, host: str, ip: str, port: int = 443):
+        """Public side of _tls_tunnel for the steps after a run (sites, speed): ip as text."""
+        return await self._tls_tunnel(ptype, proxy, host, socket.inet_aton(ip), port)
+
     async def _tls_tunnel(self, ptype: str, proxy: str, host: str, ip_bytes: bytes, port: int):
         """Tunnel through the proxy to host:port with verified TLS inside. None = tunnel refused or MITM."""
         loop = asyncio.get_running_loop()

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.13.0}/proxyscraper/publish.py

@@ -70,7 +70,9 @@ def stats_for(rows: List[dict], now: datetime) -> dict:
         if any(r.get("blocklisted") is not None for r in rows) else None,
         "stable": sum(1 for r in rows if r.get("streak", 0) >= STABLE_RUNS),
         # proxies that got through to Google, Reddit, ... in the site check after the run (sites.py)
-        "sites": {s.name: sum(1 for r in rows if (r.get("sites") or {}).get(s.name)) for s in SITES},
+        # None when a site wasn't checked at all (the step failed, its DNS lookup didn't work) – not a measured 0
+        "sites": {s.name: sum(1 for r in rows if (r.get("sites") or {}).get(s.name))
+                  if any(s.name in (r.get("sites") or {}) for r in rows) else None for s in SITES},
         "run_hours": RUN_HOURS,
         "countries": dict(Counter(r["country"] for r in rows if r.get("country")).most_common(15)),
         "median_latency": round(statistics.median(r["latency"] for r in rows)) if rows else 0,

proxy_scraper_cli-1.13.0/proxyscraper/runsteps.py

@@ -0,0 +1,74 @@
+"""Shared plumbing for the extra steps that run on a finished run's proxies.json in the hourly workflow –
+which sites let a proxy through (sites.py), how fast it downloads (speed.py).
+
+Each step loads the run's rows, tries the HTTPS-capable proxies through a verified TLS tunnel with a limit on how
+many at once, and writes the file back in one go: if the workflow kills the step at its timeout, the publish step
+still finds the old file intact.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import socket
+from pathlib import Path
+from typing import Awaitable, Callable, Iterable, List, Tuple
+
+from .paths import atomic_write
+
+# what a probe through a free proxy can run into – all of it just means "no answer from this one"
+PROBE_ERRORS = (OSError, asyncio.TimeoutError, asyncio.IncompleteReadError, asyncio.LimitOverrunError, ValueError)
+
+
+def load_rows(run_dir: Path) -> Tuple[Path, List[dict]]:
+    path = run_dir / "proxies.json"
+    return path, json.loads(path.read_text(encoding="utf-8"))
+
+
+def save_rows(path: Path, rows: List[dict]) -> None:
+    atomic_write(path, json.dumps(rows, indent=1, ensure_ascii=False))
+
+
+def https_rows(rows: Iterable[dict]) -> List[dict]:
+    """Only these can tunnel to an https:// site – the others failed the HTTPS test in the run."""
+    return [r for r in rows if r.get("https") is True]
+
+
+def resolve_ipv4(host: str, port: int = 443) -> str:
+    """Looked up once per step: SOCKS4 needs the IP, and a thousand lookups of the same host help nobody."""
+    return socket.getaddrinfo(host, port, family=socket.AF_INET)[0][4][0]
+
+
+async def run_limited(jobs: Iterable[Callable[[], Awaitable[None]]], concurrency: int) -> None:
+    """Run the jobs with at most `concurrency` at a time."""
+    slots = asyncio.Semaphore(concurrency)
+
+    async def one(job: Callable[[], Awaitable[None]]) -> None:
+        async with slots:
+            await job()
+
+    await asyncio.gather(*(one(job) for job in jobs))
+
+
+class Tunnels:
+    """Verified TLS tunnels to a host through a proxy – the same handshake code the checks use."""
+
+    def __init__(self, timeout: float):
+        from .checker import Checker  # only here: the checker pulls in more than these steps need
+
+        self.checker = Checker(judge_ip="127.0.0.1", own_ips=(), timeout=timeout, connect_timeout=timeout / 2)
+
+    async def open(self, row: dict, host: str, ip: str, port: int = 443):
+        """(reader, writer) of the TLS connection, or None if the proxy refused the tunnel or broke up the TLS."""
+        return await self.checker.tls_tunnel(row["ptype"], row["proxy"], host, ip, port)
+
+
+def browser_get(host: str, path: str, *extra: str) -> bytes:
+    """A plain GET the way a browser would send it (so sites answer as they would to a person)."""
+    from .netio import USER_AGENT
+
+    lines = [f"GET {path} HTTP/1.1", f"Host: {host}", f"User-Agent: {USER_AGENT}",
+             "Accept: text/html,application/xhtml+xml,*/*;q=0.8", "Accept-Language: en-US,en;q=0.8", *extra,
+             "Connection: close"]
+    return ("\r\n".join(lines) + "\r\n\r\n").encode()
+

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.13.0}/proxyscraper/site/index.html

@@ -22,6 +22,9 @@
 <meta property="og:description" content="Every proxy on this list passed a real handshake, a honeypot check and a content check. Re-checked every hour.">
 <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 64 64'%3E%3Crect width='64' height='64' rx='18' fill='%23D4F77A'/%3E%3Cpath d='M13 31.5 26.5 44.5 51 18' fill='none' stroke='%23121113' stroke-width='5.5' stroke-linecap='round' stroke-linejoin='round'/%3E%3Ccircle cx='13' cy='31.5' r='4.6' fill='%23121113'/%3E%3Ccircle cx='51' cy='18' r='4.6' fill='%23121113'/%3E%3Ccircle cx='26.5' cy='44.5' r='7.6' fill='%23D4F77A' stroke='%23121113' stroke-width='5'/%3E%3C/svg%3E">
 <script type="application/ld+json">
+{"@context": "https://schema.org", "@type": "FAQPage", "mainEntity": [{"@type": "Question", "name": "Are free proxies safe to use?", "acceptedAnswer": {"@type": "Answer", "text": "They're run by strangers, so treat them as untrusted. Anything that isn't encrypted can be read by whoever runs the proxy – never send passwords or personal data through one. Every proxy on this list has passed a content check (it didn't change a known page), and the HTTPS ones tunnel TLS that's verified end to end, so an https:// site can't be read or altered. That makes them fine for scraping public pages or seeing a site from another country, not for your bank."}}, {"@type": "Question", "name": "Why do most free proxies not work?", "acceptedAnswer": {"@type": "Answer", "text": "They don't live long. Of the proxies that drop off this list again, most are gone by the next hourly check, and only a few last a whole day – the weekly report tracks it. Many lists never re-check, so a large share of what they hold is long dead, and some of what still answers are honeypots or proxies that inject ads."}}, {"@type": "Question", "name": "How often is the list updated?", "acceptedAnswer": {"@type": "Answer", "text": "Every hour. GitHub Actions collects proxies from 700+ public lists, checks them and publishes whatever passed. This page loads the new list by itself when a run lands."}}, {"@type": "Question", "name": "Can I use it in my own projects?", "acceptedAnswer": {"@type": "Answer", "text": "Yes. The code is open source under the MIT license, and the lists are free to use, including in commercial projects. The proxies themselves belong to other people, so stick to the terms of the sites you visit through them."}}, {"@type": "Question", "name": "How do I use it from Python or an AI agent?", "acceptedAnswer": {"@type": "Answer", "text": "From Python: pip install proxy-scraper-cli, then live_proxies() returns the checked list with filters for country, protocol, uptime and the sites a proxy gets through to. AI agents like Claude Code or Cursor can use the MCP server, which gives them working proxies and can load pages through them."}}, {"@type": "Question", "name": "A proxy from the list doesn't work for me – why?", "acceptedAnswer": {"@type": "Answer", "text": "It worked from GitHub's servers in the last run, but free proxies come and go, and some block certain networks. proxy-scraper --recheck live checks the list again from your own connection in about 30 seconds and keeps what works from there."}}]}
+</script>
+<script type="application/ld+json">
 {"@context": "https://schema.org", "@type": "Dataset",
  "name": "Free proxy list, checked every hour",
  "description": "HTTP, SOCKS4 and SOCKS5 proxies from 700+ public lists that passed a real handshake, a honeypot check on two sites and a content check. With country, provider, latency, anonymity and HTTPS support.",
@@ -341,6 +344,27 @@
   .map-wait { display: grid; place-items: center; height: 100%; margin: 0; }
   .status .next { color: var(--muted); }
   .status .next:not(:empty)::before { content: " · "; }
+  .bookmarklet .btn { cursor: grab; margin: 6px 0 4px; }
+  .link-btn { font: inherit; color: var(--text); background: none; border: 0; padding: 0; text-decoration: underline; cursor: pointer; }
+  /* Why this list: one question per row, what a typical list says against what was measured here */
+  .ledger { border-top: 1px solid var(--line); }
+  .ledger-row { display: grid; grid-template-columns: minmax(0, 1.1fr) minmax(0, .9fr) minmax(0, 1.6fr); gap: 8px 32px; padding: 20px 0;
+                border-bottom: 1px solid var(--line); align-items: baseline; }
+  .ledger-head { padding: 14px 0; color: var(--muted); font-size: 14px; }
+  .ledger-row .q { font-size: 18px; font-weight: 500; letter-spacing: -.01em; }
+  .ledger-row .them { color: var(--muted); }
+  .ledger-row .us b { color: var(--signal-ink); font-weight: 600; font-variant-numeric: tabular-nums; }
+  /* FAQ: opens on click, nothing moves on its own */
+  .faq-list { border-top: 1px solid var(--line); }
+  .faq-item { border-bottom: 1px solid var(--line); }
+  .faq-item summary { list-style: none; cursor: pointer; display: flex; justify-content: space-between; align-items: center; gap: 24px; padding: 22px 0; }
+  .faq-item summary::-webkit-details-marker { display: none; }
+  .faq-item summary h3 { margin: 0; font-size: clamp(18px, 2vw, 21px); font-weight: 500; letter-spacing: -.01em; }
+  .faq-item summary::after { content: ""; flex: none; width: 12px; height: 12px; border-right: 1.5px solid var(--muted); border-bottom: 1.5px solid var(--muted);
+                             transform: translateY(-3px) rotate(45deg); transition: transform .3s var(--ease); }
+  .faq-item[open] summary::after { transform: translateY(3px) rotate(225deg); }
+  .faq-item summary:focus-visible { outline: 2px solid var(--signal-ink); outline-offset: 4px; border-radius: 6px; }
+  .faq-item p { margin: 0 0 24px; max-width: 70ch; color: var(--muted); font-size: 17px; }
   .panel { background: var(--panel); border: 1px solid var(--line); border-radius: 22px; padding: clamp(22px, 3vw, 34px); }
 
   /* For AI agents: a real Claude Code exchange, revealed line by line once it scrolls in */
@@ -379,8 +403,10 @@
   .swatch { width: 16px; height: 3px; border-radius: 2px; background: var(--signal-ink); }
   .swatch.dashed { background: repeating-linear-gradient(90deg, var(--muted) 0 4px, transparent 4px 7px); }
   .bars { display: grid; gap: 12px; }
-  .bar-row { display: grid; grid-template-columns: 76px minmax(0, 1fr) 44px; align-items: center; gap: 12px; font-size: 14px; }
-  .bar-row .k { color: var(--muted); }
+  .bar-row { display: grid; grid-template-columns: minmax(76px, max-content) minmax(40px, 1fr) 44px; align-items: center; gap: 12px; font-size: 14px; }
+  .bars { display: grid; grid-template-columns: minmax(76px, max-content) minmax(40px, 1fr) 44px; gap: 10px 12px; }
+  .bars .bar-row { display: contents; }  /* one grid for all rows: the labels share a column as wide as the longest */
+  .bar-row .k { color: var(--muted); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; max-width: 150px; }
   .bar-row .v { text-align: right; font-variant-numeric: tabular-nums; }
   .bar { height: 10px; border-radius: 99px; background: var(--raised); overflow: hidden; }
   .bar i { display: block; height: 100%; width: 0; border-radius: 99px; background: var(--text); transition: width 1.2s var(--ease); }
@@ -436,6 +462,9 @@
     .ctas { justify-content: flex-start; }
     .figures { grid-template-columns: minmax(0, 1fr); }
     .head-row, .agent, .closing { grid-template-columns: minmax(0, 1fr); }
+    .ledger-row { grid-template-columns: minmax(0, 1fr); gap: 4px; }
+    .ledger-head { display: none; }
+    .ledger-row .them::before { content: "Typical list: "; }
     .tools li { grid-template-columns: minmax(0, 1fr); gap: 2px; }
     .transcript p { grid-template-columns: minmax(0, 1fr); gap: 4px; }
     .head-row .intro-text { justify-self: start; }
@@ -503,7 +532,7 @@
   <div class="wrap">
     <a class="brand" href="#top" aria-label="proxy-scraper, back to top"><svg class="mark" aria-hidden="true"><use href="#mark"/></svg>proxy-scraper</a>
     <nav class="nav-links" aria-label="Sections">
-      <a href="#list">The list</a><a href="#agents">AI agents</a><a href="#checks">How it's checked</a><a href="#runs">Last runs</a><a href="#use">Use it</a>
+      <a href="#list">The list</a><a href="#agents">AI agents</a><a href="#checks">How it's checked</a><a href="#why">Why this one</a><a href="#runs">Last runs</a><a href="#use">Use it</a>
     </nav>
     <div class="nav-actions">
       <button class="toggle" id="theme" type="button" aria-label="Switch to light theme">
@@ -666,6 +695,23 @@
       </div>
     </div>
   </section>
+  <section class="block wrap" id="why" aria-labelledby="why-title">
+    <div class="head-row">
+      <h2 class="h2" id="why-title">Why this list, not another one</h2>
+      <p class="intro-text">Most free proxy lists are a dump of addresses that answered at some point. This is what you'd want to know before you use one – and what this list measured in the last run.</p>
+    </div>
+    <div class="ledger" role="table" aria-label="What a typical free proxy list tells you, and what this one measures">
+      <div class="ledger-row ledger-head" role="row"><span role="columnheader">What you'd want to know</span><span role="columnheader">A typical free list</span><span role="columnheader">Here, in the last run</span></div>
+        <div class="ledger-row" role="row"><span class="q" role="rowheader">Is it a proxy at all?</span><span class="them" role="cell">That the port answered once</span><span class="us" role="cell">A real HTTP/SOCKS handshake and a page fetched through it: <b data-live="total">–</b> passed in the last run</span></div>
+        <div class="ledger-row" role="row"><span class="q" role="rowheader">Is it a honeypot?</span><span class="them" role="cell">Often not checked</span><span class="us" role="cell">The exit IP is checked on two different sites and has to match both times</span></div>
+        <div class="ledger-row" role="row"><span class="q" role="rowheader">Does it change the pages you load?</span><span class="them" role="cell">Not checked</span><span class="us" role="cell">A known page has to arrive byte for byte. Proxies that inject scripts or ads are dropped</span></div>
+        <div class="ledger-row" role="row"><span class="q" role="rowheader">Does HTTPS work through it?</span><span class="them" role="cell">Sometimes a flag in the list</span><span class="us" role="cell">A verified TLS handshake through the tunnel: <b data-live="https">–</b> of them</span></div>
+        <div class="ledger-row" role="row"><span class="q" role="rowheader">Will it still be there tomorrow?</span><span class="them" role="cell">Unknown</span><span class="us" role="cell">Every proxy's week of hourly checks. <b data-live="stable">–</b> were on the list for 90&nbsp;%+ of the week</span></div>
+        <div class="ledger-row" role="row"><span class="q" role="rowheader">Does Google let it through?</span><span class="them" role="cell">Unknown</span><span class="us" role="cell">Tried on Google, Reddit, Amazon, Instagram and TikTok: <b data-live="google">–</b> got through to Google</span></div>
+        <div class="ledger-row" role="row"><span class="q" role="rowheader">Is it fast enough for real pages?</span><span class="them" role="cell">A latency number, if any</span><span class="us" role="cell">100&nbsp;KB downloaded through every HTTPS-capable one: half of them manage <b data-live="speed">–</b> or more</span></div>
+        <div class="ledger-row" role="row"><span class="q" role="rowheader">How fresh is it?</span><span class="them" role="cell">Scraped once, rarely checked again</span><span class="us" role="cell">Checked again every hour, the last run <b data-live="ago">–</b></span></div>
+    </div>
+  </section>
 
   <section class="block wrap" id="runs" aria-labelledby="runs-title">
     <div class="head-row">
@@ -703,6 +749,9 @@
 proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button" data-copy-pre aria-label="Copy the install commands"><svg aria-hidden="true"><use href="#i-copy"/></svg></button></div></div>
       <div class="panel"><h3>Recheck from your network</h3><p>Starts from this list and checks it again from where you are. Takes about 30 seconds, then serves a rotating proxy on port 8899.</p>
         <div class="cmd"><pre>proxy-scraper --recheck live --serve</pre><button class="icon-btn" type="button" data-copy-pre aria-label="Copy the recheck command"><svg aria-hidden="true"><use href="#i-copy"/></svg></button></div></div>
+      <div class="panel bookmarklet"><h3>One click, on any site</h3><p>Drag this button to your bookmarks bar. Clicking it copies a proxy that worked in 90&nbsp;%+ of this week's checks – on whatever page you're on, nothing to install.</p>
+        <a class="btn primary" id="bookmarklet" href="javascript:(async()=>{const%20b=&quot;https://maximilianfeix.github.io/proxy-scraper/&quot;,g=async%20f=>{const%20r=await%20fetch(b+f,{cache:&quot;no-cache&quot;});if(!r.ok)throw%200;return(await%20r.text()).split(&quot;%5Cn&quot;).filter(l=>/%5E[a-z0-9]+:%5C/%5C/%5CS+%24/.test(l))};try{let%20l=await%20g(&quot;stable.txt&quot;);if(!l.length)l=(await%20g(&quot;https.txt&quot;)).slice(0,30);if(!l.length)throw%200;const%20p=l[Math.floor(Math.random()*l.length)];try{await%20navigator.clipboard.writeText(p);alert(&quot;Copied%20a%20working%20proxy:%5Cn&quot;+p)}catch(e){prompt(&quot;A%20working%20proxy%20%E2%80%93%20copy%20it:&quot;,p)}}catch(e){alert(&quot;proxy-scraper:%20the%20list%20could%20not%20be%20loaded&quot;)}})()" title="Drag me to your bookmarks bar">Get a proxy</a>
+        <p class="note">Or right here: <button class="link-btn" type="button" id="pick-one">copy a reliable one now</button>.</p></div>
       </div>
       <div class="panel"><h3>Plain lists</h3><p>One proxy per line, fastest first, updated with every run.</p><div class="raw" id="raw"></div></div>
     </div>
@@ -741,6 +790,20 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
 }</pre><button class="icon-btn" type="button" data-copy-pre aria-label="Copy the example"><svg aria-hidden="true"><use href="#i-copy"/></svg></button></div>
     </div>
   </section>
+  <section class="block wrap faq" id="faq" aria-labelledby="faq-title">
+    <div class="head-row">
+      <h2 class="h2" id="faq-title">Questions people ask</h2>
+      <p class="intro-text">Short answers. The <a href="https://github.com/maximilianfeix/proxy-scraper#faq">README</a> has more.</p>
+    </div>
+    <div class="faq-list">
+      <details class="faq-item"><summary><h3>Are free proxies safe to use?</h3></summary><p>They're run by strangers, so treat them as untrusted. Anything that isn't encrypted can be read by whoever runs the proxy – never send passwords or personal data through one. Every proxy on this list has passed a content check (it didn't change a known page), and the HTTPS ones tunnel TLS that's verified end to end, so an https:// site can't be read or altered. That makes them fine for scraping public pages or seeing a site from another country, not for your bank.</p></details>
+      <details class="faq-item"><summary><h3>Why do most free proxies not work?</h3></summary><p>They don't live long. Of the proxies that drop off this list again, most are gone by the next hourly check, and only a few last a whole day – the weekly report tracks it. Many lists never re-check, so a large share of what they hold is long dead, and some of what still answers are honeypots or proxies that inject ads.</p></details>
+      <details class="faq-item"><summary><h3>How often is the list updated?</h3></summary><p>Every hour. GitHub Actions collects proxies from 700+ public lists, checks them and publishes whatever passed. This page loads the new list by itself when a run lands.</p></details>
+      <details class="faq-item"><summary><h3>Can I use it in my own projects?</h3></summary><p>Yes. The code is open source under the MIT license, and the lists are free to use, including in commercial projects. The proxies themselves belong to other people, so stick to the terms of the sites you visit through them.</p></details>
+      <details class="faq-item"><summary><h3>How do I use it from Python or an AI agent?</h3></summary><p>From Python: pip install proxy-scraper-cli, then live_proxies() returns the checked list with filters for country, protocol, uptime and the sites a proxy gets through to. AI agents like Claude Code or Cursor can use the MCP server, which gives them working proxies and can load pages through them.</p></details>
+      <details class="faq-item"><summary><h3>A proxy from the list doesn't work for me – why?</h3></summary><p>It worked from GitHub's servers in the last run, but free proxies come and go, and some block certain networks. proxy-scraper --recheck live checks the list again from your own connection in about 30 seconds and keeps what works from there.</p></details>
+    </div>
+  </section>
   <section class="block wrap closing" aria-labelledby="star-title">
     <div>
       <h2 class="h2" id="star-title">Saved you an afternoon?</h2>
@@ -1024,6 +1087,15 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
   }
   loadView();
   $("#share").onclick = () => copy(new URL(viewUrl(), location.href).href, "Link to this view copied");
+  // the same as the bookmarklet, without the round trip: a random proxy from the reliable ones in the list
+  $("#pick-one").onclick = () => {
+    const pool = state.rows.filter(r => (r.uptime_7d || 0) >= 90);  // exactly what stable.txt holds
+    const from = pool.length ? pool : state.rows.filter(r => r.https).slice(0, 30);
+    if (!from.length) return toast("The list isn't loaded yet");
+    const p = from[Math.floor(Math.random() * from.length)];
+    copy(p.url, `Copied ${p.url}`);
+  };
+  $("#bookmarklet").onclick = (e) => { e.preventDefault(); toast("Drag it to your bookmarks bar – it works on any site from there"); };
   // chips, the country and sorting let the rows settle in; typing and the slider stay instant
   const morph = (fn) => {
     fn();
@@ -1295,6 +1367,10 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
     sieve(rows, st.total);
     started = true;
     $("#final-n").textContent = fmt(st.total);
+    const live = { total: fmt(st.total), https: fmt(st.https), stable: fmt(rows.filter(r => (r.uptime_7d || 0) >= 90).length),
+                   google: st.sites && st.sites.google != null ? fmt(st.sites.google) : "–",
+                   speed: st.median_speed_kbps ? `${fmt(st.median_speed_kbps)} KB/s` : "–", ago: ago(st.updated) };
+    $$("[data-live]").forEach((el) => { el.textContent = live[el.dataset.live] ?? "–"; });
     const dc = st.datacenter != null ? ` ${pct(st.datacenter, st.total)} % exit from a datacenter` : "";
     const bl = st.blocklisted != null ? `${dc ? "," : ""} ${pct(st.blocklisted, st.total)} % from an IP on a spam blocklist` : "";
     $("#step5").textContent = `HTTPS with verified TLS, anonymity, country, provider and spam blocklists. This time ${fmt(st.https)} tunnel HTTPS and ${fmt(st.elite)} are elite.${dc}${bl}${dc || bl ? "." : ""}`;

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.13.0}/proxyscraper/sites.py

@@ -12,15 +12,12 @@ from __future__ import annotations
 
 import argparse
 import asyncio
-import json
-import socket
 import sys
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Awaitable, Callable, Dict, List, Optional, Tuple
 
-from .netio import USER_AGENT
-from .paths import atomic_write
+from .runsteps import PROBE_ERRORS, Tunnels, browser_get, https_rows, load_rows, resolve_ipv4, run_limited, save_rows
 
 TIMEOUT = 10.0
 CONCURRENCY = 300
@@ -113,20 +110,16 @@ def page_complete(head: bytes, body: int, tail: bytes) -> bool:
 
 def real_probe(timeout: float = TIMEOUT) -> Probe:
     """Tunnel to the site through the proxy with verified TLS, send a browser-like GET, judge the answer."""
-    from .checker import Checker
-
-    checker = Checker(judge_ip="127.0.0.1", own_ips=(), timeout=timeout, connect_timeout=timeout / 2)
+    tunnels = Tunnels(timeout)
 
     async def probe(row: dict, site: Site, ip: str) -> Optional[bool]:
         async def go() -> Optional[bool]:
-            opened = await checker._tls_tunnel(row["ptype"], row["proxy"], site.host, socket.inet_aton(ip), 443)
+            opened = await tunnels.open(row, site.host, ip)
             if opened is None:
                 return None
             reader, writer = opened
             try:
-                writer.write(f"GET {site.path} HTTP/1.1\r\nHost: {site.host}\r\nUser-Agent: {USER_AGENT}\r\n"
-                             "Accept: text/html,application/xhtml+xml,*/*;q=0.8\r\nAccept-Language: en-US,en;q=0.8\r\n"
-                             "Connection: close\r\n\r\n".encode())
+                writer.write(browser_get(site.host, site.path))
                 await writer.drain()
                 head = await reader.readuntil(b"\r\n\r\n")
                 status, location = parse_head(head)
@@ -145,46 +138,37 @@ def real_probe(timeout: float = TIMEOUT) -> Probe:
             return site.verdict(status, location, body)
         try:
             return await asyncio.wait_for(go(), probe_timeout(site, timeout))
-        except (OSError, asyncio.TimeoutError, asyncio.IncompleteReadError, asyncio.LimitOverrunError, ValueError):
+        except PROBE_ERRORS:
             return None
     return probe
 
 
-def _resolve(host: str) -> str:
-    return socket.getaddrinfo(host, 443, family=socket.AF_INET)[0][4][0]
-
-
-async def fill_run(run_dir: Path, probe: Optional[Probe] = None, resolve: Callable[[str], str] = _resolve,
+async def fill_run(run_dir: Path, probe: Optional[Probe] = None, resolve: Callable[[str], str] = resolve_ipv4,
                    concurrency: int = CONCURRENCY) -> Dict[str, int]:
     """Probe every HTTPS-capable proxy of a run against every site; writes "sites" into its proxies.json.
     Returns how many proxies got through to each site."""
     probe = probe or real_probe()
-    path = run_dir / "proxies.json"
-    rows: List[dict] = json.loads(path.read_text(encoding="utf-8"))
+    path, rows = load_rows(run_dir)
     ips = {}
     for site in SITES:
         try:
             ips[site.name] = resolve(site.host)
         except OSError:
             continue  # this site can't be looked up right now – the others still can
-    slots = asyncio.Semaphore(concurrency)
 
-    async def one(row: dict, site: Site) -> None:
-        async with slots:
+    def job(row: dict, site: Site):
+        async def run() -> None:
             verdict = await probe(row, site, ips[site.name])
-        if verdict is not None:
-            row["sites"][site.name] = verdict
+            if verdict is not None:
+                row["sites"][site.name] = verdict
+        return run
 
-    jobs = []
     for row in rows:
         row["sites"] = {}
-        if row.get("https") is True:
-            jobs += [one(row, site) for site in SITES if site.name in ips]
-    await asyncio.gather(*jobs)
+    await run_limited((job(row, site) for row in https_rows(rows) for site in SITES if site.name in ips), concurrency)
     for row in rows:  # always in the same order, whichever answer came first
         row["sites"] = {s.name: row["sites"][s.name] for s in SITES if s.name in row["sites"]}
-    # in one go: if the step gets killed (it has a timeout), the publish step still finds the old file intact
-    atomic_write(path, json.dumps(rows, indent=1, ensure_ascii=False))
+    save_rows(path, rows)
     return {s.name: sum(1 for r in rows if r["sites"].get(s.name)) for s in SITES}
 
 

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.13.0}/proxyscraper/speed.py

@@ -13,16 +13,13 @@ from __future__ import annotations
 
 import argparse
 import asyncio
-import json
-import socket
 import statistics
 import sys
 import time
 from pathlib import Path
 from typing import Awaitable, Callable, List, Optional
 
-from .netio import USER_AGENT
-from .paths import atomic_write
+from .runsteps import PROBE_ERRORS, Tunnels, browser_get, https_rows, load_rows, resolve_ipv4, run_limited, save_rows
 
 HOST = "speed.cloudflare.com"
 BYTES = 100_000
@@ -42,9 +39,7 @@ def kbps(received: int, seconds: float) -> Optional[int]:
 
 
 def real_probe(deadline: float = DEADLINE) -> Probe:
-    from .checker import Checker
-
-    checker = Checker(judge_ip="127.0.0.1", own_ips=(), timeout=deadline, connect_timeout=deadline / 2)
+    tunnels = Tunnels(deadline)
 
     async def probe(row: dict, ip: str) -> Optional[int]:
         end = time.monotonic() + deadline
@@ -53,7 +48,7 @@ def real_probe(deadline: float = DEADLINE) -> Probe:
             return await asyncio.wait_for(coro, max(0.01, end - time.monotonic()))
 
         try:
-            opened = await within(checker._tls_tunnel(row["ptype"], row["proxy"], HOST, socket.inet_aton(ip), 443))
+            opened = await within(tunnels.open(row, HOST, ip))
             if opened is None:
                 return None
             reader, writer = opened
@@ -61,8 +56,7 @@ def real_probe(deadline: float = DEADLINE) -> Probe:
                 # the clock runs from the request to the last byte: a start at the first body byte would miss
                 # whatever already sat in the buffer with the head, and fast proxies would come out absurdly fast
                 sent = time.monotonic()
-                writer.write(f"GET {PATH} HTTP/1.1\r\nHost: {HOST}\r\nUser-Agent: {USER_AGENT}\r\n"
-                             "Accept-Encoding: identity\r\nConnection: close\r\n\r\n".encode())
+                writer.write(browser_get(HOST, PATH, "Accept-Encoding: identity"))
                 await within(writer.drain())
                 head = await within(reader.readuntil(b"\r\n\r\n"))
                 if b" 200 " not in head.split(b"\r\n", 1)[0] + b" ":
@@ -79,38 +73,33 @@ def real_probe(deadline: float = DEADLINE) -> Probe:
                 return kbps(received, time.monotonic() - sent)
             finally:
                 writer.close()
-        except (OSError, asyncio.TimeoutError, asyncio.IncompleteReadError, asyncio.LimitOverrunError, ValueError):
+        except PROBE_ERRORS:
             return None
     return probe
 
 
-def _resolve(host: str) -> str:
-    return socket.getaddrinfo(host, 443, family=socket.AF_INET)[0][4][0]
-
-
-async def fill_run(run_dir: Path, probe: Optional[Probe] = None, resolve: Callable[[str], str] = _resolve,
+async def fill_run(run_dir: Path, probe: Optional[Probe] = None, resolve: Callable[[str], str] = resolve_ipv4,
                    concurrency: int = CONCURRENCY) -> Optional[int]:
     """Measure every HTTPS-capable proxy of a run, write speed_kbps into its proxies.json -> the median."""
     probe = probe or real_probe()
-    path = run_dir / "proxies.json"
-    rows: List[dict] = json.loads(path.read_text(encoding="utf-8"))
+    path, rows = load_rows(run_dir)
     try:
         ip = resolve(HOST)
     except OSError:
         return None
-    slots = asyncio.Semaphore(concurrency)
 
-    async def one(row: dict) -> None:
-        async with slots:
+    def job(row: dict):
+        async def run() -> None:
             try:  # the probe keeps its own deadline – this only catches one that doesn't
                 value = await asyncio.wait_for(probe(row, ip), DEADLINE * 1.5)
             except asyncio.TimeoutError:
                 value = None
-        if value:
-            row["speed_kbps"] = value
+            if value:
+                row["speed_kbps"] = value
+        return run
 
-    await asyncio.gather(*(one(r) for r in rows if r.get("https") is True))
-    atomic_write(path, json.dumps(rows, indent=1, ensure_ascii=False))
+    await run_limited((job(r) for r in https_rows(rows)), concurrency)
+    save_rows(path, rows)
     speeds = [r["speed_kbps"] for r in rows if r.get("speed_kbps")]
     return round(statistics.median(speeds)) if speeds else None
 

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.13.0}/tests/test_sites.py

@@ -119,7 +119,8 @@ def test_publish_writes_a_list_per_site_and_counts(tmp_path):
     assert (out / "works-with" / "reddit.txt").read_text() == ""
     assert (out / "works-with" / "amazon.txt").read_text() == ""
     stats = json.loads((out / "stats.json").read_text())
-    assert stats["sites"] == {"google": 1, "reddit": 0, "amazon": 0, "instagram": 0, "tiktok": 0}
+    # amazon, instagram and tiktok weren't checked at all in this run: that's "not measured", not zero
+    assert stats["sites"] == {"google": 1, "reddit": 0, "amazon": None, "instagram": None, "tiktok": None}
     assert (out / "works-with" / "tiktok.txt").read_text() == ""
     import csv
     with (out / "proxies.csv").open(newline="", encoding="utf-8") as fh: