proxy-scraper-cli 1.11.0__tar.gz → 1.12.0__tar.gz

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.11.0
+Version: 1.12.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.12.0}/proxy_scraper_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.11.0
+Version: 1.12.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.12.0}/proxy_scraper_cli.egg-info/SOURCES.txt

@@ -40,6 +40,7 @@ proxyscraper/preferences.py
 proxyscraper/proxypages.py
 proxyscraper/publish.py
 proxyscraper/report.py
+proxyscraper/runsteps.py
 proxyscraper/sites.py
 proxyscraper/sources.json
 proxyscraper/sources.py

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.12.0}/proxyscraper/__init__.py

@@ -1,7 +1,7 @@
 """proxy-scraper: collects public HTTP/SOCKS4/SOCKS5 proxies from hundreds of sources, checks them
 in parallel with its own protocol handshakes and learns which sources deliver good proxies."""
 
-__version__ = "1.11.0"
+__version__ = "1.12.0"
 
 
 def __getattr__(name):

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.12.0}/proxyscraper/checker.py

@@ -334,6 +334,10 @@ class Checker:
             writer.close()
         return 200 <= status < 400
 
+    async def tls_tunnel(self, ptype: str, proxy: str, host: str, ip: str, port: int = 443):
+        """Public side of _tls_tunnel for the steps after a run (sites, speed): ip as text."""
+        return await self._tls_tunnel(ptype, proxy, host, socket.inet_aton(ip), port)
+
     async def _tls_tunnel(self, ptype: str, proxy: str, host: str, ip_bytes: bytes, port: int):
         """Tunnel through the proxy to host:port with verified TLS inside. None = tunnel refused or MITM."""
         loop = asyncio.get_running_loop()

proxy_scraper_cli-1.12.0/proxyscraper/runsteps.py

@@ -0,0 +1,74 @@
+"""Shared plumbing for the extra steps that run on a finished run's proxies.json in the hourly workflow –
+which sites let a proxy through (sites.py), how fast it downloads (speed.py).
+
+Each step loads the run's rows, tries the HTTPS-capable proxies through a verified TLS tunnel with a limit on how
+many at once, and writes the file back in one go: if the workflow kills the step at its timeout, the publish step
+still finds the old file intact.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import socket
+from pathlib import Path
+from typing import Awaitable, Callable, Iterable, List, Tuple
+
+from .paths import atomic_write
+
+# what a probe through a free proxy can run into – all of it just means "no answer from this one"
+PROBE_ERRORS = (OSError, asyncio.TimeoutError, asyncio.IncompleteReadError, asyncio.LimitOverrunError, ValueError)
+
+
+def load_rows(run_dir: Path) -> Tuple[Path, List[dict]]:
+    path = run_dir / "proxies.json"
+    return path, json.loads(path.read_text(encoding="utf-8"))
+
+
+def save_rows(path: Path, rows: List[dict]) -> None:
+    atomic_write(path, json.dumps(rows, indent=1, ensure_ascii=False))
+
+
+def https_rows(rows: Iterable[dict]) -> List[dict]:
+    """Only these can tunnel to an https:// site – the others failed the HTTPS test in the run."""
+    return [r for r in rows if r.get("https") is True]
+
+
+def resolve_ipv4(host: str, port: int = 443) -> str:
+    """Looked up once per step: SOCKS4 needs the IP, and a thousand lookups of the same host help nobody."""
+    return socket.getaddrinfo(host, port, family=socket.AF_INET)[0][4][0]
+
+
+async def run_limited(jobs: Iterable[Callable[[], Awaitable[None]]], concurrency: int) -> None:
+    """Run the jobs with at most `concurrency` at a time."""
+    slots = asyncio.Semaphore(concurrency)
+
+    async def one(job: Callable[[], Awaitable[None]]) -> None:
+        async with slots:
+            await job()
+
+    await asyncio.gather(*(one(job) for job in jobs))
+
+
+class Tunnels:
+    """Verified TLS tunnels to a host through a proxy – the same handshake code the checks use."""
+
+    def __init__(self, timeout: float):
+        from .checker import Checker  # only here: the checker pulls in more than these steps need
+
+        self.checker = Checker(judge_ip="127.0.0.1", own_ips=(), timeout=timeout, connect_timeout=timeout / 2)
+
+    async def open(self, row: dict, host: str, ip: str, port: int = 443):
+        """(reader, writer) of the TLS connection, or None if the proxy refused the tunnel or broke up the TLS."""
+        return await self.checker.tls_tunnel(row["ptype"], row["proxy"], host, ip, port)
+
+
+def browser_get(host: str, path: str, *extra: str) -> bytes:
+    """A plain GET the way a browser would send it (so sites answer as they would to a person)."""
+    from .netio import USER_AGENT
+
+    lines = [f"GET {path} HTTP/1.1", f"Host: {host}", f"User-Agent: {USER_AGENT}",
+             "Accept: text/html,application/xhtml+xml,*/*;q=0.8", "Accept-Language: en-US,en;q=0.8", *extra,
+             "Connection: close"]
+    return ("\r\n".join(lines) + "\r\n\r\n").encode()
+

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.12.0}/proxyscraper/site/index.html

@@ -341,6 +341,8 @@
   .map-wait { display: grid; place-items: center; height: 100%; margin: 0; }
   .status .next { color: var(--muted); }
   .status .next:not(:empty)::before { content: " · "; }
+  .bookmarklet .btn { cursor: grab; margin: 6px 0 4px; }
+  .link-btn { font: inherit; color: var(--text); background: none; border: 0; padding: 0; text-decoration: underline; cursor: pointer; }
   .panel { background: var(--panel); border: 1px solid var(--line); border-radius: 22px; padding: clamp(22px, 3vw, 34px); }
 
   /* For AI agents: a real Claude Code exchange, revealed line by line once it scrolls in */
@@ -379,8 +381,10 @@
   .swatch { width: 16px; height: 3px; border-radius: 2px; background: var(--signal-ink); }
   .swatch.dashed { background: repeating-linear-gradient(90deg, var(--muted) 0 4px, transparent 4px 7px); }
   .bars { display: grid; gap: 12px; }
-  .bar-row { display: grid; grid-template-columns: 76px minmax(0, 1fr) 44px; align-items: center; gap: 12px; font-size: 14px; }
-  .bar-row .k { color: var(--muted); }
+  .bar-row { display: grid; grid-template-columns: minmax(76px, max-content) minmax(40px, 1fr) 44px; align-items: center; gap: 12px; font-size: 14px; }
+  .bars { display: grid; grid-template-columns: minmax(76px, max-content) minmax(40px, 1fr) 44px; gap: 10px 12px; }
+  .bars .bar-row { display: contents; }  /* one grid for all rows: the labels share a column as wide as the longest */
+  .bar-row .k { color: var(--muted); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; max-width: 150px; }
   .bar-row .v { text-align: right; font-variant-numeric: tabular-nums; }
   .bar { height: 10px; border-radius: 99px; background: var(--raised); overflow: hidden; }
   .bar i { display: block; height: 100%; width: 0; border-radius: 99px; background: var(--text); transition: width 1.2s var(--ease); }
@@ -703,6 +707,9 @@
 proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button" data-copy-pre aria-label="Copy the install commands"><svg aria-hidden="true"><use href="#i-copy"/></svg></button></div></div>
       <div class="panel"><h3>Recheck from your network</h3><p>Starts from this list and checks it again from where you are. Takes about 30 seconds, then serves a rotating proxy on port 8899.</p>
         <div class="cmd"><pre>proxy-scraper --recheck live --serve</pre><button class="icon-btn" type="button" data-copy-pre aria-label="Copy the recheck command"><svg aria-hidden="true"><use href="#i-copy"/></svg></button></div></div>
+      <div class="panel bookmarklet"><h3>One click, on any site</h3><p>Drag this button to your bookmarks bar. Clicking it copies a proxy that worked in 90&nbsp;%+ of this week's checks – on whatever page you're on, nothing to install.</p>
+        <a class="btn primary" id="bookmarklet" href="javascript:(async()=>{const%20b=&quot;https://maximilianfeix.github.io/proxy-scraper/&quot;,g=async%20f=>{const%20r=await%20fetch(b+f,{cache:&quot;no-cache&quot;});if(!r.ok)throw%200;return(await%20r.text()).split(&quot;%5Cn&quot;).filter(l=>/%5E[a-z0-9]+:%5C/%5C/%5CS+%24/.test(l))};try{let%20l=await%20g(&quot;stable.txt&quot;);if(!l.length)l=(await%20g(&quot;https.txt&quot;)).slice(0,30);if(!l.length)throw%200;const%20p=l[Math.floor(Math.random()*l.length)];try{await%20navigator.clipboard.writeText(p);alert(&quot;Copied%20a%20working%20proxy:%5Cn&quot;+p)}catch(e){prompt(&quot;A%20working%20proxy%20%E2%80%93%20copy%20it:&quot;,p)}}catch(e){alert(&quot;proxy-scraper:%20the%20list%20could%20not%20be%20loaded&quot;)}})()" title="Drag me to your bookmarks bar">Get a proxy</a>
+        <p class="note">Or right here: <button class="link-btn" type="button" id="pick-one">copy a reliable one now</button>.</p></div>
       </div>
       <div class="panel"><h3>Plain lists</h3><p>One proxy per line, fastest first, updated with every run.</p><div class="raw" id="raw"></div></div>
     </div>
@@ -1024,6 +1031,15 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
   }
   loadView();
   $("#share").onclick = () => copy(new URL(viewUrl(), location.href).href, "Link to this view copied");
+  // the same as the bookmarklet, without the round trip: a random proxy from the reliable ones in the list
+  $("#pick-one").onclick = () => {
+    const pool = state.rows.filter(r => (r.uptime_7d || 0) >= 90);  // exactly what stable.txt holds
+    const from = pool.length ? pool : state.rows.filter(r => r.https).slice(0, 30);
+    if (!from.length) return toast("The list isn't loaded yet");
+    const p = from[Math.floor(Math.random() * from.length)];
+    copy(p.url, `Copied ${p.url}`);
+  };
+  $("#bookmarklet").onclick = (e) => { e.preventDefault(); toast("Drag it to your bookmarks bar – it works on any site from there"); };
   // chips, the country and sorting let the rows settle in; typing and the slider stay instant
   const morph = (fn) => {
     fn();

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.12.0}/proxyscraper/sites.py

@@ -12,15 +12,12 @@ from __future__ import annotations
 
 import argparse
 import asyncio
-import json
-import socket
 import sys
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Awaitable, Callable, Dict, List, Optional, Tuple
 
-from .netio import USER_AGENT
-from .paths import atomic_write
+from .runsteps import PROBE_ERRORS, Tunnels, browser_get, https_rows, load_rows, resolve_ipv4, run_limited, save_rows
 
 TIMEOUT = 10.0
 CONCURRENCY = 300
@@ -113,20 +110,16 @@ def page_complete(head: bytes, body: int, tail: bytes) -> bool:
 
 def real_probe(timeout: float = TIMEOUT) -> Probe:
     """Tunnel to the site through the proxy with verified TLS, send a browser-like GET, judge the answer."""
-    from .checker import Checker
-
-    checker = Checker(judge_ip="127.0.0.1", own_ips=(), timeout=timeout, connect_timeout=timeout / 2)
+    tunnels = Tunnels(timeout)
 
     async def probe(row: dict, site: Site, ip: str) -> Optional[bool]:
         async def go() -> Optional[bool]:
-            opened = await checker._tls_tunnel(row["ptype"], row["proxy"], site.host, socket.inet_aton(ip), 443)
+            opened = await tunnels.open(row, site.host, ip)
             if opened is None:
                 return None
             reader, writer = opened
             try:
-                writer.write(f"GET {site.path} HTTP/1.1\r\nHost: {site.host}\r\nUser-Agent: {USER_AGENT}\r\n"
-                             "Accept: text/html,application/xhtml+xml,*/*;q=0.8\r\nAccept-Language: en-US,en;q=0.8\r\n"
-                             "Connection: close\r\n\r\n".encode())
+                writer.write(browser_get(site.host, site.path))
                 await writer.drain()
                 head = await reader.readuntil(b"\r\n\r\n")
                 status, location = parse_head(head)
@@ -145,46 +138,37 @@ def real_probe(timeout: float = TIMEOUT) -> Probe:
             return site.verdict(status, location, body)
         try:
             return await asyncio.wait_for(go(), probe_timeout(site, timeout))
-        except (OSError, asyncio.TimeoutError, asyncio.IncompleteReadError, asyncio.LimitOverrunError, ValueError):
+        except PROBE_ERRORS:
             return None
     return probe
 
 
-def _resolve(host: str) -> str:
-    return socket.getaddrinfo(host, 443, family=socket.AF_INET)[0][4][0]
-
-
-async def fill_run(run_dir: Path, probe: Optional[Probe] = None, resolve: Callable[[str], str] = _resolve,
+async def fill_run(run_dir: Path, probe: Optional[Probe] = None, resolve: Callable[[str], str] = resolve_ipv4,
                    concurrency: int = CONCURRENCY) -> Dict[str, int]:
     """Probe every HTTPS-capable proxy of a run against every site; writes "sites" into its proxies.json.
     Returns how many proxies got through to each site."""
     probe = probe or real_probe()
-    path = run_dir / "proxies.json"
-    rows: List[dict] = json.loads(path.read_text(encoding="utf-8"))
+    path, rows = load_rows(run_dir)
     ips = {}
     for site in SITES:
         try:
             ips[site.name] = resolve(site.host)
         except OSError:
             continue  # this site can't be looked up right now – the others still can
-    slots = asyncio.Semaphore(concurrency)
 
-    async def one(row: dict, site: Site) -> None:
-        async with slots:
+    def job(row: dict, site: Site):
+        async def run() -> None:
             verdict = await probe(row, site, ips[site.name])
-        if verdict is not None:
-            row["sites"][site.name] = verdict
+            if verdict is not None:
+                row["sites"][site.name] = verdict
+        return run
 
-    jobs = []
     for row in rows:
         row["sites"] = {}
-        if row.get("https") is True:
-            jobs += [one(row, site) for site in SITES if site.name in ips]
-    await asyncio.gather(*jobs)
+    await run_limited((job(row, site) for row in https_rows(rows) for site in SITES if site.name in ips), concurrency)
     for row in rows:  # always in the same order, whichever answer came first
         row["sites"] = {s.name: row["sites"][s.name] for s in SITES if s.name in row["sites"]}
-    # in one go: if the step gets killed (it has a timeout), the publish step still finds the old file intact
-    atomic_write(path, json.dumps(rows, indent=1, ensure_ascii=False))
+    save_rows(path, rows)
     return {s.name: sum(1 for r in rows if r["sites"].get(s.name)) for s in SITES}
 
 

{proxy_scraper_cli-1.11.0 → proxy_scraper_cli-1.12.0}/proxyscraper/speed.py

@@ -13,16 +13,13 @@ from __future__ import annotations
 
 import argparse
 import asyncio
-import json
-import socket
 import statistics
 import sys
 import time
 from pathlib import Path
 from typing import Awaitable, Callable, List, Optional
 
-from .netio import USER_AGENT
-from .paths import atomic_write
+from .runsteps import PROBE_ERRORS, Tunnels, browser_get, https_rows, load_rows, resolve_ipv4, run_limited, save_rows
 
 HOST = "speed.cloudflare.com"
 BYTES = 100_000
@@ -42,9 +39,7 @@ def kbps(received: int, seconds: float) -> Optional[int]:
 
 
 def real_probe(deadline: float = DEADLINE) -> Probe:
-    from .checker import Checker
-
-    checker = Checker(judge_ip="127.0.0.1", own_ips=(), timeout=deadline, connect_timeout=deadline / 2)
+    tunnels = Tunnels(deadline)
 
     async def probe(row: dict, ip: str) -> Optional[int]:
         end = time.monotonic() + deadline
@@ -53,7 +48,7 @@ def real_probe(deadline: float = DEADLINE) -> Probe:
             return await asyncio.wait_for(coro, max(0.01, end - time.monotonic()))
 
         try:
-            opened = await within(checker._tls_tunnel(row["ptype"], row["proxy"], HOST, socket.inet_aton(ip), 443))
+            opened = await within(tunnels.open(row, HOST, ip))
             if opened is None:
                 return None
             reader, writer = opened
@@ -61,8 +56,7 @@ def real_probe(deadline: float = DEADLINE) -> Probe:
                 # the clock runs from the request to the last byte: a start at the first body byte would miss
                 # whatever already sat in the buffer with the head, and fast proxies would come out absurdly fast
                 sent = time.monotonic()
-                writer.write(f"GET {PATH} HTTP/1.1\r\nHost: {HOST}\r\nUser-Agent: {USER_AGENT}\r\n"
-                             "Accept-Encoding: identity\r\nConnection: close\r\n\r\n".encode())
+                writer.write(browser_get(HOST, PATH, "Accept-Encoding: identity"))
                 await within(writer.drain())
                 head = await within(reader.readuntil(b"\r\n\r\n"))
                 if b" 200 " not in head.split(b"\r\n", 1)[0] + b" ":
@@ -79,38 +73,33 @@ def real_probe(deadline: float = DEADLINE) -> Probe:
                 return kbps(received, time.monotonic() - sent)
             finally:
                 writer.close()
-        except (OSError, asyncio.TimeoutError, asyncio.IncompleteReadError, asyncio.LimitOverrunError, ValueError):
+        except PROBE_ERRORS:
             return None
     return probe
 
 
-def _resolve(host: str) -> str:
-    return socket.getaddrinfo(host, 443, family=socket.AF_INET)[0][4][0]
-
-
-async def fill_run(run_dir: Path, probe: Optional[Probe] = None, resolve: Callable[[str], str] = _resolve,
+async def fill_run(run_dir: Path, probe: Optional[Probe] = None, resolve: Callable[[str], str] = resolve_ipv4,
                    concurrency: int = CONCURRENCY) -> Optional[int]:
     """Measure every HTTPS-capable proxy of a run, write speed_kbps into its proxies.json -> the median."""
     probe = probe or real_probe()
-    path = run_dir / "proxies.json"
-    rows: List[dict] = json.loads(path.read_text(encoding="utf-8"))
+    path, rows = load_rows(run_dir)
     try:
         ip = resolve(HOST)
     except OSError:
         return None
-    slots = asyncio.Semaphore(concurrency)
 
-    async def one(row: dict) -> None:
-        async with slots:
+    def job(row: dict):
+        async def run() -> None:
             try:  # the probe keeps its own deadline – this only catches one that doesn't
                 value = await asyncio.wait_for(probe(row, ip), DEADLINE * 1.5)
             except asyncio.TimeoutError:
                 value = None
-        if value:
-            row["speed_kbps"] = value
+            if value:
+                row["speed_kbps"] = value
+        return run
 
-    await asyncio.gather(*(one(r) for r in rows if r.get("https") is True))
-    atomic_write(path, json.dumps(rows, indent=1, ensure_ascii=False))
+    await run_limited((job(r) for r in https_rows(rows)), concurrency)
+    save_rows(path, rows)
     speeds = [r["speed_kbps"] for r in rows if r.get("speed_kbps")]
     return round(statistics.median(speeds)) if speeds else None