proxy-scraper-cli 1.8.0__tar.gz → 1.9.0__tar.gz

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.8.0
+Version: 1.9.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT
@@ -578,6 +578,84 @@ for p in proxies:
         continue  # free proxies come and go – just take the next one
 ```
 
+**httpx** (`pip install "httpx[socks]"`) – straight from the hourly list, no scan
+
+```python
+import httpx
+from proxyscraper import live_proxies
+
+for p in live_proxies(types=["http", "socks5"], https=True, min_uptime=90, limit=10):  # httpx can't do SOCKS4
+    try:
+        with httpx.Client(proxy=p.url, timeout=10) as client:
+            print(p.url, "→", client.get("https://api.ipify.org").text)
+        break
+    except httpx.HTTPError:
+        continue  # next one
+```
+
+**aiohttp** (`pip install aiohttp aiohttp-socks` – aiohttp alone can't do SOCKS)
+
+```python
+import asyncio
+
+import aiohttp
+from aiohttp_socks import ProxyConnector, ProxyError
+from proxyscraper import live_proxies_async
+
+
+async def main():
+    for p in await live_proxies_async(https=True, min_uptime=90, limit=10):
+        try:
+            async with aiohttp.ClientSession(connector=ProxyConnector.from_url(p.url)) as session:
+                async with session.get("https://api.ipify.org", timeout=aiohttp.ClientTimeout(total=10)) as r:
+                    print(p.url, "→", await r.text())
+                    return
+        except (aiohttp.ClientError, ProxyError, asyncio.TimeoutError, OSError):
+            continue  # free proxies come and go – just take the next one
+
+asyncio.run(main())
+```
+
+**Scrapy** – a downloader middleware that sends every request, retries included, through the next proxy. Scrapy only speaks HTTP proxies, `https=True` picks the ones that can tunnel `https://` pages
+
+```python
+import itertools
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from proxyscraper import live_proxies
+
+PROXIES = [p.url for p in live_proxies(types="http", https=True, min_uptime=50)]
+if not PROXIES:
+    raise SystemExit("No proxy matches right now – loosen the filters (e.g. min_uptime)")
+POOL = itertools.cycle(PROXIES)
+
+
+class RotatingProxy:
+    def process_request(self, request, spider=None):  # newer Scrapy leaves out spider
+        request.meta["proxy"] = next(POOL)
+
+
+class IpSpider(scrapy.Spider):
+    name = "ip"
+    start_urls = [f"https://api.ipify.org/?n={i}" for i in range(3)]
+    custom_settings = {
+        "DOWNLOADER_MIDDLEWARES": {f"{__name__}.RotatingProxy": 350},
+        "RETRY_TIMES": 5, "DOWNLOAD_TIMEOUT": 15,
+    }
+
+    def parse(self, response):
+        print(response.meta["proxy"], "→", response.text)
+
+
+if __name__ == "__main__":
+    process = CrawlerProcess()
+    process.crawl(IpSpider)
+    process.start()
+```
+
+In a Scrapy project, put `RotatingProxy` in `middlewares.py` and add it to `DOWNLOADER_MIDDLEWARES` in `settings.py`. For long crawls, reload the pool now and then – the list changes every hour.
+
 **proxychains, Clash / Mihomo** – ready-made configs with `--export`
 
 ```bash

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.9.0}/README.md

@@ -538,6 +538,84 @@ for p in proxies:
         continue  # free proxies come and go – just take the next one
 ```
 
+**httpx** (`pip install "httpx[socks]"`) – straight from the hourly list, no scan
+
+```python
+import httpx
+from proxyscraper import live_proxies
+
+for p in live_proxies(types=["http", "socks5"], https=True, min_uptime=90, limit=10):  # httpx can't do SOCKS4
+    try:
+        with httpx.Client(proxy=p.url, timeout=10) as client:
+            print(p.url, "→", client.get("https://api.ipify.org").text)
+        break
+    except httpx.HTTPError:
+        continue  # next one
+```
+
+**aiohttp** (`pip install aiohttp aiohttp-socks` – aiohttp alone can't do SOCKS)
+
+```python
+import asyncio
+
+import aiohttp
+from aiohttp_socks import ProxyConnector, ProxyError
+from proxyscraper import live_proxies_async
+
+
+async def main():
+    for p in await live_proxies_async(https=True, min_uptime=90, limit=10):
+        try:
+            async with aiohttp.ClientSession(connector=ProxyConnector.from_url(p.url)) as session:
+                async with session.get("https://api.ipify.org", timeout=aiohttp.ClientTimeout(total=10)) as r:
+                    print(p.url, "→", await r.text())
+                    return
+        except (aiohttp.ClientError, ProxyError, asyncio.TimeoutError, OSError):
+            continue  # free proxies come and go – just take the next one
+
+asyncio.run(main())
+```
+
+**Scrapy** – a downloader middleware that sends every request, retries included, through the next proxy. Scrapy only speaks HTTP proxies, `https=True` picks the ones that can tunnel `https://` pages
+
+```python
+import itertools
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from proxyscraper import live_proxies
+
+PROXIES = [p.url for p in live_proxies(types="http", https=True, min_uptime=50)]
+if not PROXIES:
+    raise SystemExit("No proxy matches right now – loosen the filters (e.g. min_uptime)")
+POOL = itertools.cycle(PROXIES)
+
+
+class RotatingProxy:
+    def process_request(self, request, spider=None):  # newer Scrapy leaves out spider
+        request.meta["proxy"] = next(POOL)
+
+
+class IpSpider(scrapy.Spider):
+    name = "ip"
+    start_urls = [f"https://api.ipify.org/?n={i}" for i in range(3)]
+    custom_settings = {
+        "DOWNLOADER_MIDDLEWARES": {f"{__name__}.RotatingProxy": 350},
+        "RETRY_TIMES": 5, "DOWNLOAD_TIMEOUT": 15,
+    }
+
+    def parse(self, response):
+        print(response.meta["proxy"], "→", response.text)
+
+
+if __name__ == "__main__":
+    process = CrawlerProcess()
+    process.crawl(IpSpider)
+    process.start()
+```
+
+In a Scrapy project, put `RotatingProxy` in `middlewares.py` and add it to `DOWNLOADER_MIDDLEWARES` in `settings.py`. For long crawls, reload the pool now and then – the list changes every hour.
+
 **proxychains, Clash / Mihomo** – ready-made configs with `--export`
 
 ```bash

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.9.0}/proxy_scraper_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.8.0
+Version: 1.9.0
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT
@@ -578,6 +578,84 @@ for p in proxies:
         continue  # free proxies come and go – just take the next one
 ```
 
+**httpx** (`pip install "httpx[socks]"`) – straight from the hourly list, no scan
+
+```python
+import httpx
+from proxyscraper import live_proxies
+
+for p in live_proxies(types=["http", "socks5"], https=True, min_uptime=90, limit=10):  # httpx can't do SOCKS4
+    try:
+        with httpx.Client(proxy=p.url, timeout=10) as client:
+            print(p.url, "→", client.get("https://api.ipify.org").text)
+        break
+    except httpx.HTTPError:
+        continue  # next one
+```
+
+**aiohttp** (`pip install aiohttp aiohttp-socks` – aiohttp alone can't do SOCKS)
+
+```python
+import asyncio
+
+import aiohttp
+from aiohttp_socks import ProxyConnector, ProxyError
+from proxyscraper import live_proxies_async
+
+
+async def main():
+    for p in await live_proxies_async(https=True, min_uptime=90, limit=10):
+        try:
+            async with aiohttp.ClientSession(connector=ProxyConnector.from_url(p.url)) as session:
+                async with session.get("https://api.ipify.org", timeout=aiohttp.ClientTimeout(total=10)) as r:
+                    print(p.url, "→", await r.text())
+                    return
+        except (aiohttp.ClientError, ProxyError, asyncio.TimeoutError, OSError):
+            continue  # free proxies come and go – just take the next one
+
+asyncio.run(main())
+```
+
+**Scrapy** – a downloader middleware that sends every request, retries included, through the next proxy. Scrapy only speaks HTTP proxies, `https=True` picks the ones that can tunnel `https://` pages
+
+```python
+import itertools
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from proxyscraper import live_proxies
+
+PROXIES = [p.url for p in live_proxies(types="http", https=True, min_uptime=50)]
+if not PROXIES:
+    raise SystemExit("No proxy matches right now – loosen the filters (e.g. min_uptime)")
+POOL = itertools.cycle(PROXIES)
+
+
+class RotatingProxy:
+    def process_request(self, request, spider=None):  # newer Scrapy leaves out spider
+        request.meta["proxy"] = next(POOL)
+
+
+class IpSpider(scrapy.Spider):
+    name = "ip"
+    start_urls = [f"https://api.ipify.org/?n={i}" for i in range(3)]
+    custom_settings = {
+        "DOWNLOADER_MIDDLEWARES": {f"{__name__}.RotatingProxy": 350},
+        "RETRY_TIMES": 5, "DOWNLOAD_TIMEOUT": 15,
+    }
+
+    def parse(self, response):
+        print(response.meta["proxy"], "→", response.text)
+
+
+if __name__ == "__main__":
+    process = CrawlerProcess()
+    process.crawl(IpSpider)
+    process.start()
+```
+
+In a Scrapy project, put `RotatingProxy` in `middlewares.py` and add it to `DOWNLOADER_MIDDLEWARES` in `settings.py`. For long crawls, reload the pool now and then – the list changes every hour.
+
 **proxychains, Clash / Mihomo** – ready-made configs with `--export`
 
 ```bash

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.9.0}/proxy_scraper_cli.egg-info/SOURCES.txt

@@ -36,7 +36,9 @@ proxyscraper/parsing.py
 proxyscraper/paths.py
 proxyscraper/pipeline.py
 proxyscraper/preferences.py
+proxyscraper/proxypages.py
 proxyscraper/publish.py
+proxyscraper/report.py
 proxyscraper/sites.py
 proxyscraper/sources.json
 proxyscraper/sources.py
@@ -78,6 +80,7 @@ tests/test_judges.py
 tests/test_learning.py
 tests/test_live_api.py
 tests/test_mcp_server.py
+tests/test_mine_sources.py
 tests/test_options.py
 tests/test_output_stdout.py
 tests/test_packaging.py
@@ -85,8 +88,10 @@ tests/test_pages.py
 tests/test_parsing.py
 tests/test_paths.py
 tests/test_pipeline.py
+tests/test_proxy_pages.py
 tests/test_proxy_sources.py
 tests/test_publish.py
+tests/test_report.py
 tests/test_robustness.py
 tests/test_serve_password.py
 tests/test_serve_refill.py

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.9.0}/proxyscraper/__init__.py

@@ -1,7 +1,7 @@
 """proxy-scraper: collects public HTTP/SOCKS4/SOCKS5 proxies from hundreds of sources, checks them
 in parallel with its own protocol handshakes and learns which sources deliver good proxies."""
 
-__version__ = "1.8.0"
+__version__ = "1.9.0"
 
 
 def __getattr__(name):

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.9.0}/proxyscraper/fetchcache.py

@@ -5,7 +5,7 @@ On the next run the download asks with If-None-Match / If-Modified-Since; if 304
 the keys come from the cache. They are stored unfiltered (all types), because --types can
 change between two runs.
 
-  data/fetch-cache/index.json      url -> {etag, modified, type, file, used}
+  data/fetch-cache/index.json      url -> {etag, modified, type, file, used, parser}
   data/fetch-cache/<sha1>.txt.gz   the keys, one per line
 """
 
@@ -19,6 +19,7 @@ import zlib
 from pathlib import Path
 from typing import Dict, Optional
 
+from . import parsing
 from .paths import DATA_DIR, atomic_write
 
 CACHE_DIR = DATA_DIR / "fetch-cache"
@@ -53,7 +54,9 @@ class FetchCache:
         ptype is the type the list is parsed with: if it changes (e.g. http -> auto),
         the stored keys no longer fit and the list is loaded again."""
         entry = self.entries.get(url) if self.enabled else None
-        if not entry or entry.get("type") != ptype or not (self.dir / entry["file"]).is_file():
+        # keys from an older parser may miss proxies the current one finds -> load the list again
+        if (not entry or entry.get("type") != ptype or entry.get("parser") != parsing.PARSER_VERSION
+                or not (self.dir / entry["file"]).is_file()):
             return {}
         headers = {}
         if entry.get("etag"):
@@ -91,7 +94,8 @@ class FetchCache:
         tmp = self.dir / (name + ".tmp")
         tmp.write_bytes(gzip.compress(keys.encode("utf-8"), compresslevel=5))
         tmp.replace(self.dir / name)
-        self.entries[url] = {"etag": etag, "modified": modified, "type": ptype, "file": name, "used": time.time()}
+        self.entries[url] = {"etag": etag, "modified": modified, "type": ptype, "file": name, "used": time.time(),
+                             "parser": parsing.PARSER_VERSION}
 
     def save(self, now: Optional[float] = None) -> None:
         index = self.dir / "index.json"

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.9.0}/proxyscraper/pages.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 from datetime import datetime
 from html import escape
 from pathlib import Path
-from typing import Callable, Dict, List, Tuple
+from typing import Callable, Dict, List, Sequence, Tuple
 
 SITE_URL = "https://maximilianfeix.github.io/proxy-scraper/"
 REPO_URL = "https://github.com/maximilianfeix/proxy-scraper"
@@ -100,6 +100,15 @@ footer { color: var(--muted); font-size: 14px; display: grid; gap: 6px; }
 """
 
 
+def _list_query(path: str) -> str:
+    """The same filter on the full list: the website reads its filters from the address."""
+    kind = path.strip("/")
+    if kind.startswith("country/"):
+        return f"?country={kind.split('/')[1].upper()}"
+    return {"http": "?types=http", "socks4": "?types=socks4", "socks5": "?types=socks5", "https": "?https=1",
+            "elite": "?elite=1"}.get(kind, "")
+
+
 def _render(path: str, title: str, what: str, rows: List[dict], updated: datetime, nav: str) -> str:
     depth = path.count("/")
     root = "../" * depth
@@ -112,7 +121,8 @@ def _render(path: str, title: str, what: str, rows: List[dict], updated: datetim
         lede = (f"No {what} passed every check in the last run ({when}). The list is checked again every hour, "
                 "so look again later or try the full list.")
     body_rows = "\n".join(
-        f"<tr><td class=\"mono\">{escape(r['proxy'])}</td><td>{escape(r['ptype'].upper())}</td>"
+        f"<tr><td class=\"mono\"><a href=\"{root}proxy/{escape(r['proxy'].replace(':', '-'))}/\">"
+        f"{escape(r['proxy'])}</a></td><td>{escape(r['ptype'].upper())}</td>"
         f"<td>{escape(r.get('country') or '–')}</td><td>{r['latency']:,} ms</td>"
         f"<td>{'yes' if r.get('https') else 'no'}</td><td>{escape(r.get('anonymity') or '–')}</td>"
         f"<td>{escape((r.get('org') or '–')[:40])}</td></tr>"
@@ -157,7 +167,7 @@ def _render(path: str, title: str, what: str, rows: List[dict], updated: datetim
     <p class="lede">{escape(lede)}</p>
     <div class="actions">
       <a class="btn primary" href="proxies.txt" download>Download {total:,} as .txt</a>
-      <a class="btn" href="{root}#list">Filter the full list</a>
+      <a class="btn" href="{root}{_list_query(path)}#list">Filter the full list</a>
       <a class="btn" href="{REPO_URL}">Check them from your network</a>
     </div>
   </main>
@@ -187,8 +197,9 @@ def _nav(specs, counts: Dict[str, int], current: str, root: str) -> str:
             f"<nav aria-label=\"By country\"><h2>By country</h2><ul>{links(countries)}</ul></nav>")
 
 
-def write_pages(rows: List[dict], out: Path, updated: datetime) -> List[str]:
-    """Writes every page with its proxies.txt and the sitemap. -> the paths listed in the sitemap."""
+def write_pages(rows: List[dict], out: Path, updated: datetime, extra: Sequence[str] = ()) -> List[str]:
+    """Writes every page with its proxies.txt and the sitemap (plus the `extra` paths, e.g. the proxy pages)
+    -> the paths listed in the sitemap."""
     rows = sorted(rows, key=lambda r: r["latency"])
     specs = page_specs()
     selected = {p: [r for r in rows if keep(r)] for p, _, _, keep in specs}
@@ -203,6 +214,7 @@ def write_pages(rows: List[dict], out: Path, updated: datetime) -> List[str]:
         (folder / "proxies.txt").write_text("".join(f"{r['url']}\n" for r in page_rows), encoding="utf-8")
         if not path.startswith("country/") or counts[path] >= SITEMAP_MIN:
             listed.append(path)
+    listed.extend(extra)
     lastmod = updated.strftime("%Y-%m-%dT%H:%M:%S+00:00")
     urls = "".join(f"<url><loc>{SITE_URL}{p}</loc><lastmod>{lastmod}</lastmod><changefreq>hourly</changefreq></url>"
                    for p in listed)

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.9.0}/proxyscraper/parsing.py

@@ -43,6 +43,11 @@ RawCandidate = Tuple[str, bytes, bytes, bytes]  # type, ip, port, credentials (b
 _SPACE_SEPARATED_RE = re.compile(rb"\d\.\d{1,3}[ \t]+\d{2,5}(?![\d.])")
 
 
+# Bump whenever parsing finds different proxies in the same bytes: the fetch cache stores parsed keys and
+# reloads every list once when this changes (2: plain lists of "auto" sources are read, 3: as http and socks5)
+PARSER_VERSION = 3
+
+
 def _needs_full_regex(data: bytes) -> bool:
     """Only space/HTML lists need the slower regex (sample from the start of the file)."""
     sample = data[:8192]
@@ -53,7 +58,10 @@ def extract_candidates(data: bytes, default_type: str) -> Set[RawCandidate]:
     """Finds proxies in text, HTML tables and JSON.
 
     Lines with a type:// prefix keep their own type; everything else gets the type of the source.
-    For sources of type "auto" only lines with a prefix count.
+    For sources of type "auto" the plain lines only count when they are the majority: a generic proxies.txt of
+    plain ip:port lines doesn't say its protocol, so they're tried as HTTP and as SOCKS5 (measured on 20,000 such
+    entries: 44 worked as http, 32 as socks5, none as both; an unreachable address fails fast for both). In a list
+    of mostly prefixed lines, the odd plain one has an unknown type and is left out.
     """
     out: Set[RawCandidate] = set()
     with_scheme = set()
@@ -63,14 +71,17 @@ def extract_candidates(data: bytes, default_type: str) -> Set[RawCandidate]:
             if ptype:
                 out.add((ptype, ip, port, auth))
                 with_scheme.add((ip, port))
-    if default_type == "auto":
-        return out
+    auto = default_type == "auto"
     regex = PROXY_RE if _needs_full_regex(data) else PLAIN_RE
     pairs = set(regex.findall(data))
     if b'"ip"' in data:
         pairs.update(JSON_IP_PORT_RE.findall(data))
         pairs.update((ip, port) for port, ip in JSON_PORT_IP_RE.findall(data))
-    out.update((default_type, ip, port, b"") for ip, port in pairs - with_scheme)
+    plain = pairs - with_scheme
+    if auto and len(plain) <= len(out):
+        return out
+    for ptype in ("http", "socks5") if auto else (default_type,):
+        out.update((ptype, ip, port, b"") for ip, port in plain)
     return out
 
 

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.9.0}/proxyscraper/pipeline.py

@@ -65,6 +65,7 @@ async def collect_sources(opts: RunOptions, quality: srcs.SourceStats) -> Source
             found = await srcs.discover_github(
                 http_get, token, max_repos,
                 on_progress=lambda msg: status.update(f"[bold {ACCENT}]GitHub-Discovery:[/] {msg}"),
+                known=sources,  # curated ones needn't be tried; found ones are, so they stay fresh
             )
         if found:
             srcs.save_discovered(found)