proxy-scraper-cli 1.8.0__tar.gz → 1.8.1__tar.gz

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.8.0
+Version: 1.8.1
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/proxy_scraper_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy-scraper-cli
-Version: 1.8.0
+Version: 1.8.1
 Summary: Finds free HTTP, SOCKS4 and SOCKS5 proxies that actually work: 700+ sources, real checks, honeypot filtering and a rotating local proxy server.
 Author: Maximilian Feix
 License-Expression: MIT

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/proxy_scraper_cli.egg-info/SOURCES.txt

@@ -78,6 +78,7 @@ tests/test_judges.py
 tests/test_learning.py
 tests/test_live_api.py
 tests/test_mcp_server.py
+tests/test_mine_sources.py
 tests/test_options.py
 tests/test_output_stdout.py
 tests/test_packaging.py

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/proxyscraper/__init__.py

@@ -1,7 +1,7 @@
 """proxy-scraper: collects public HTTP/SOCKS4/SOCKS5 proxies from hundreds of sources, checks them
 in parallel with its own protocol handshakes and learns which sources deliver good proxies."""
 
-__version__ = "1.8.0"
+__version__ = "1.8.1"
 
 
 def __getattr__(name):

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/proxyscraper/fetchcache.py

@@ -5,7 +5,7 @@ On the next run the download asks with If-None-Match / If-Modified-Since; if 304
 the keys come from the cache. They are stored unfiltered (all types), because --types can
 change between two runs.
 
-  data/fetch-cache/index.json      url -> {etag, modified, type, file, used}
+  data/fetch-cache/index.json      url -> {etag, modified, type, file, used, parser}
   data/fetch-cache/<sha1>.txt.gz   the keys, one per line
 """
 
@@ -19,6 +19,7 @@ import zlib
 from pathlib import Path
 from typing import Dict, Optional
 
+from . import parsing
 from .paths import DATA_DIR, atomic_write
 
 CACHE_DIR = DATA_DIR / "fetch-cache"
@@ -53,7 +54,9 @@ class FetchCache:
         ptype is the type the list is parsed with: if it changes (e.g. http -> auto),
         the stored keys no longer fit and the list is loaded again."""
         entry = self.entries.get(url) if self.enabled else None
-        if not entry or entry.get("type") != ptype or not (self.dir / entry["file"]).is_file():
+        # keys from an older parser may miss proxies the current one finds -> load the list again
+        if (not entry or entry.get("type") != ptype or entry.get("parser") != parsing.PARSER_VERSION
+                or not (self.dir / entry["file"]).is_file()):
             return {}
         headers = {}
         if entry.get("etag"):
@@ -91,7 +94,8 @@ class FetchCache:
         tmp = self.dir / (name + ".tmp")
         tmp.write_bytes(gzip.compress(keys.encode("utf-8"), compresslevel=5))
         tmp.replace(self.dir / name)
-        self.entries[url] = {"etag": etag, "modified": modified, "type": ptype, "file": name, "used": time.time()}
+        self.entries[url] = {"etag": etag, "modified": modified, "type": ptype, "file": name, "used": time.time(),
+                             "parser": parsing.PARSER_VERSION}
 
     def save(self, now: Optional[float] = None) -> None:
         index = self.dir / "index.json"

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/proxyscraper/pages.py

@@ -100,6 +100,15 @@ footer { color: var(--muted); font-size: 14px; display: grid; gap: 6px; }
 """
 
 
+def _list_query(path: str) -> str:
+    """The same filter on the full list: the website reads its filters from the address."""
+    kind = path.strip("/")
+    if kind.startswith("country/"):
+        return f"?country={kind.split('/')[1].upper()}"
+    return {"http": "?types=http", "socks4": "?types=socks4", "socks5": "?types=socks5", "https": "?https=1",
+            "elite": "?elite=1"}.get(kind, "")
+
+
 def _render(path: str, title: str, what: str, rows: List[dict], updated: datetime, nav: str) -> str:
     depth = path.count("/")
     root = "../" * depth
@@ -157,7 +166,7 @@ def _render(path: str, title: str, what: str, rows: List[dict], updated: datetim
     <p class="lede">{escape(lede)}</p>
     <div class="actions">
       <a class="btn primary" href="proxies.txt" download>Download {total:,} as .txt</a>
-      <a class="btn" href="{root}#list">Filter the full list</a>
+      <a class="btn" href="{root}{_list_query(path)}#list">Filter the full list</a>
       <a class="btn" href="{REPO_URL}">Check them from your network</a>
     </div>
   </main>

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/proxyscraper/parsing.py

@@ -43,6 +43,11 @@ RawCandidate = Tuple[str, bytes, bytes, bytes]  # type, ip, port, credentials (b
 _SPACE_SEPARATED_RE = re.compile(rb"\d\.\d{1,3}[ \t]+\d{2,5}(?![\d.])")
 
 
+# Bump whenever parsing finds different proxies in the same bytes: the fetch cache stores parsed keys and
+# reloads every list once when this changes (2: plain lists of "auto" sources are read as http)
+PARSER_VERSION = 2
+
+
 def _needs_full_regex(data: bytes) -> bool:
     """Only space/HTML lists need the slower regex (sample from the start of the file)."""
     sample = data[:8192]
@@ -53,7 +58,9 @@ def extract_candidates(data: bytes, default_type: str) -> Set[RawCandidate]:
     """Finds proxies in text, HTML tables and JSON.
 
     Lines with a type:// prefix keep their own type; everything else gets the type of the source.
-    For sources of type "auto" only lines with a prefix count.
+    For sources of type "auto" the plain lines only count when they are the majority: a generic proxies.txt of
+    plain ip:port lines is read as HTTP, the most common kind (the checks sort out the rest). In a list of mostly
+    prefixed lines, the odd plain one has an unknown type and is left out.
     """
     out: Set[RawCandidate] = set()
     with_scheme = set()
@@ -63,14 +70,18 @@ def extract_candidates(data: bytes, default_type: str) -> Set[RawCandidate]:
             if ptype:
                 out.add((ptype, ip, port, auth))
                 with_scheme.add((ip, port))
-    if default_type == "auto":
-        return out
+    auto = default_type == "auto"
+    if auto:
+        default_type = "http"
     regex = PROXY_RE if _needs_full_regex(data) else PLAIN_RE
     pairs = set(regex.findall(data))
     if b'"ip"' in data:
         pairs.update(JSON_IP_PORT_RE.findall(data))
         pairs.update((ip, port) for port, ip in JSON_PORT_IP_RE.findall(data))
-    out.update((default_type, ip, port, b"") for ip, port in pairs - with_scheme)
+    plain = pairs - with_scheme
+    if auto and len(plain) <= len(out):
+        return out
+    out.update((default_type, ip, port, b"") for ip, port in plain)
     return out
 
 

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/proxyscraper/pipeline.py

@@ -65,6 +65,7 @@ async def collect_sources(opts: RunOptions, quality: srcs.SourceStats) -> Source
             found = await srcs.discover_github(
                 http_get, token, max_repos,
                 on_progress=lambda msg: status.update(f"[bold {ACCENT}]GitHub-Discovery:[/] {msg}"),
+                known=sources,  # curated ones needn't be tried; found ones are, so they stay fresh
             )
         if found:
             srcs.save_discovered(found)

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/proxyscraper/site/index.html

@@ -562,7 +562,7 @@
       <select class="field" id="f-site" aria-label="Gets through to" title="Checked every hour: most free proxies get a captcha or a 403 on these sites">
         <option value="">Any site</option><option value="google">Gets through to Google</option><option value="reddit">Gets through to Reddit</option><option value="amazon">Gets through to Amazon</option></select>
       <label class="range">Up to <input type="range" id="f-lat" min="200" max="8000" step="100" value="8000" aria-label="Maximum latency"><output id="f-lat-v" for="f-lat">any</output></label>
-      <input class="field" type="search" id="f-q" placeholder="Search IP, port, country or provider    /" aria-label="Search" aria-keyshortcuts="/">
+      <input class="field" type="search" id="f-q" placeholder="Search IP, country or provider    /" aria-label="Search" aria-keyshortcuts="/">
     </div>
     <div class="table-panel">
       <div class="table-scroll" data-lenis-prevent-horizontal>
@@ -587,6 +587,7 @@
       <span class="count" id="count" aria-live="polite"></span>
       <div class="actions">
         <button class="btn small" id="more" type="button" hidden>Show more</button>
+        <button class="btn small" id="share" type="button" title="The filters are in the address, so the link opens this exact view">Share view</button>
         <button class="btn small" id="copy-all" type="button">Copy filtered</button>
         <button class="btn small primary" id="download" type="button">Download filtered</button>
       </div>
@@ -755,6 +756,11 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
   const fmt = (n) => Number(n).toLocaleString("en-US");
   const esc = (s) => String(s ?? "").replace(/[&<>"']/g, c => ({"&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;"}[c]));
   const flag = (cc) => /^[A-Z]{2}$/.test(cc || "") ? String.fromCodePoint(...[...cc].map(c => 0x1F1E6 + c.charCodeAt(0) - 65)) : "";
+  // "DE" -> "Germany" (the browser knows the names; without Intl.DisplayNames the code stays)
+  const countryName = (() => {
+    try { const names = new Intl.DisplayNames(["en"], { type: "region" }); return (cc) => { try { return names.of(cc) || cc; } catch (e) { return cc; } }; }
+    catch (e) { return (cc) => cc; }
+  })();
   const pct = (n, of) => Math.round(100 * n / Math.max(of, 1));
   const short = (org) => { const s = String(org || "").replace(/,? (Inc|LLC|Ltd|GmbH|Co|Corporation|Limited|S\.?A|B\.?V)\.?$/i, ""); return s.length > 28 ? s.slice(0, 27) + "…" : s; };
   const upFor = (n) => { const h = (n || 1) * RUN_HOURS; return h < 48 ? `${h} h` : `${Math.round(h / 24)} days`; };
@@ -912,7 +918,7 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
     const rows = state.rows.filter(r => state.types.has(r.ptype) && (!state.https || r.https) && (!state.elite || r.anonymity === "elite")
       && (!state.noDc || !r.hosting) && (!state.noBl || !r.blocklisted) && (!state.stable || (r.streak || 0) >= STABLE) && (!state.country || r.country === state.country) && (!state.site || (r.sites || {})[state.site] === true)
       && (state.maxLat >= 8000 || r.latency <= state.maxLat)  // all the way right = no limit
-      && (!q || `${r.ptype} ${r.proxy} ${r.country} ${r.exit_ip} ${r.org || ""} AS${r.asn || ""}`.toLowerCase().includes(q)));
+      && (!q || `${r.ptype} ${r.proxy} ${r.country} ${r.country ? countryName(r.country) : ""} ${r.exit_ip} ${r.org || ""} AS${r.asn || ""}`.toLowerCase().includes(q)));
     const k = state.sort, d = state.dir;
     return rows.sort((a, b) => (a[k] > b[k] ? 1 : a[k] < b[k] ? -1 : 0) * d || a.latency - b.latency);
   }
@@ -921,7 +927,7 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
     $("#rows").innerHTML = rows.slice(0, state.shown).map(r => `<tr>
       <td><span class="t-type" title="${esc(r.ptype)}"><span class="dot ${esc(r.ptype)}"></span><span class="name">${esc(r.ptype)}</span></span></td>
       <td><button class="addr-btn" type="button" data-url="${esc(r.url)}" aria-label="Copy ${esc(r.url)}">${esc(r.proxy)}<svg aria-hidden="true"><use href="#i-copy"/></svg></button></td>
-      <td>${flag(r.country)} ${esc(r.country || "–")}</td>
+      <td title="${esc(r.country ? countryName(r.country) : "")}">${flag(r.country)} ${esc(r.country || "–")}</td>
       <td>${latency(r.latency)}</td>
       <td class="hide-sm">${r.https ? "Yes" : '<span class="quiet">No</span>'}</td>
       <td class="hide-sm">${share(r)}</td>
@@ -931,9 +937,56 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
       || '<tr><td colspan="9" class="empty">No proxy matches these filters. Try a higher latency limit or fewer filters.</td></tr>';
     $("#count").textContent = rows.length ? `Showing ${fmt(Math.min(state.shown, rows.length))} of ${fmt(rows.length)} matching proxies` : "";
     $("#more").hidden = rows.length <= state.shown;
+    saveView();
     $$("th[data-key]").forEach(th => th.dataset.key === state.sort ? th.setAttribute("aria-sort", state.dir > 0 ? "ascending" : "descending") : th.removeAttribute("aria-sort"));
   }
   const refresh = () => { state.shown = PAGE; render(); };
+
+  // The filters live in the address bar, so a view can be shared or bookmarked
+  const ALL_TYPES = ["http", "socks4", "socks5"], FLAGS = { https: "https", elite: "elite", noDc: "nodc", noBl: "nobl", stable: "stable" };
+  function viewUrl() {
+    const q = new URLSearchParams();
+    if (state.types.size !== ALL_TYPES.length) q.set("types", [...state.types].join(","));
+    Object.entries(FLAGS).forEach(([key, name]) => { if (state[key]) q.set(name, "1"); });
+    if (state.country) q.set("country", state.country);
+    if (state.site) q.set("site", state.site);
+    if (state.maxLat < 8000) q.set("latency", state.maxLat);
+    if (state.q.trim()) q.set("q", state.q.trim());
+    if (state.sort !== "latency" || state.dir !== 1) q.set("sort", (state.dir < 0 ? "-" : "") + state.sort);
+    return location.pathname + (q.toString() ? `?${q}` : "") + location.hash;
+  }
+  // debounced: browsers limit replaceState (Safari throws after ~100 calls in 30 s, e.g. while dragging the slider)
+  let saveTimer = 0;
+  function saveView() {
+    clearTimeout(saveTimer);
+    saveTimer = setTimeout(() => {
+      const url = viewUrl();
+      if (url !== location.pathname + location.search + location.hash) {
+        try { history.replaceState(history.state, "", url); } catch (e) {}
+      }
+    }, 400);
+  }
+  function loadView() {
+    const q = new URLSearchParams(location.search);
+    if (q.has("types")) state.types = new Set(q.get("types").split(",").filter(t => ALL_TYPES.includes(t)));
+    Object.entries(FLAGS).forEach(([key, name]) => { state[key] = q.get(name) === "1"; });
+    state.country = /^[A-Z]{2}$/.test(q.get("country") || "") ? q.get("country") : "";
+    state.site = ["google", "reddit", "amazon"].includes(q.get("site")) ? q.get("site") : "";
+    const lat = parseInt(q.get("latency"), 10);
+    state.maxLat = lat >= 200 && lat < 8000 ? Math.round(lat / 100) * 100 : 8000;
+    state.q = (q.get("q") || "").slice(0, 100);
+    const sort = (q.get("sort") || "").replace(/^-/, "");
+    if ($(`th[data-key="${CSS.escape(sort)}"]`)) { state.sort = sort; state.dir = q.get("sort").startsWith("-") ? -1 : 1; }
+    // show it on the controls
+    $$(".chip[data-type]").forEach(b => b.setAttribute("aria-pressed", state.types.has(b.dataset.type)));
+    Object.keys(FLAGS).forEach((key) => { const b = $({ https: "#f-https", elite: "#f-elite", noDc: "#f-dc", noBl: "#f-bl", stable: "#f-stable" }[key]); b.setAttribute("aria-pressed", state[key]); });
+    $("#f-site").value = state.site;
+    $("#f-lat").value = state.maxLat;
+    $("#f-lat-v").textContent = state.maxLat >= 8000 ? "any" : `${fmt(state.maxLat)} ms`;
+    $("#f-q").value = state.q;
+  }
+  loadView();
+  $("#share").onclick = () => copy(new URL(viewUrl(), location.href).href, "Link to this view copied");
   // chips, the country and sorting let the rows settle in; typing and the slider stay instant
   const morph = (fn) => {
     fn();
@@ -1139,11 +1192,13 @@ proxy-scraper --want 50 --https-only</pre><button class="icon-btn" type="button"
     const byCountry = {};
     rows.forEach(r => { if (/^[A-Z]{2}$/.test(r.country || "")) byCountry[r.country] = (byCountry[r.country] || 0) + 1; });
     const countries = Object.entries(byCountry).sort((a, b) => b[1] - a[1]);
-    bars($("#countries"), countries.slice(0, 7), (cc) => `${flag(cc)} ${cc}`);
+    bars($("#countries"), countries.slice(0, 7), (cc) => `${flag(cc)} ${countryName(cc)}`);
     const buckets = [[300, "< 0.3 s"], [700, "< 0.7 s"], [1500, "< 1.5 s"], [3000, "< 3 s"], [Infinity, "3 s +"]];
     bars($("#latency"), buckets.map(([max, label], i) => [label, rows.filter(r => r.latency < max && (i === 0 || r.latency >= buckets[i - 1][0])).length]), (l) => l);
     const sel = $("#f-country");
-    countries.forEach(([cc, n]) => sel.add(new Option(`${flag(cc)} ${cc} (${n})`, cc)));
+    countries.forEach(([cc, n]) => sel.add(new Option(`${flag(cc)} ${countryName(cc)} (${n})`, cc)));
+    sel.value = state.country;  // from a shared link
+    if (sel.value !== state.country) state.country = "";  // a country that has no proxies this hour
     render();
   });
 })();

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/proxyscraper/sources.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 import asyncio
 import hashlib
+import ipaddress
 import json
 import os
 import re
@@ -19,13 +20,13 @@ import subprocess
 import time
 import warnings
 from collections import Counter
-from dataclasses import asdict, dataclass
+from dataclasses import asdict, dataclass, fields
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
-from typing import Awaitable, Callable, Dict, Iterable, List, Optional, Tuple
-from urllib.parse import quote, urlencode
+from typing import Awaitable, Callable, Dict, Iterable, List, Optional, Set, Tuple
+from urllib.parse import quote, urlencode, urlsplit
 
-from .parsing import PROXY_TYPES, TYPE_ALIASES
+from .parsing import PROXY_TYPES, TYPE_ALIASES, extract_candidates, validate_candidates
 from .paths import DATA_DIR, PACKAGE_DIR, atomic_write
 
 SOURCES_FILE = PACKAGE_DIR / "sources.json"
@@ -262,6 +263,19 @@ _FILE_GENERIC_RE = re.compile(r"(^|[/_\-.])(proxy|proxies|all)[^/]*\.txt$")
 # repos that collect VPN configs: their addresses aren't HTTP/SOCKS proxies, only dead candidates
 _REPO_REJECT_RE = re.compile(r"v2ray|vpn|xray|clash|sing-?box|nekobox|vless|vmess|subscri|config", re.I)
 MAX_FILES_PER_REPO = 12
+# the source lists of other proxy scrapers (sources.json, config.toml, urls.txt …) are full of list URLs
+# (not in hidden folders: .github/ISSUE_TEMPLATE/config.yml or .vscode/settings.json would take the slots)
+_CONFIG_PATH_RE = re.compile(
+    r"^(?!(?:.*/)?\.)(?:.*/)?(sources?|urls?|providers?|proxy[_-]?sources?|sites|config|settings)[^/]*"
+    r"\.(txt|json|ya?ml|toml|py|js|ts)$", re.I)
+_URL_RE = re.compile(rb"https?://[^\s\"'<>)\]}`,]+")
+_LISTISH_RE = re.compile(r"proxy|proxies|socks|\.txt$", re.I)
+MAX_CONFIGS_PER_REPO = 3
+MAX_MINED_CHECKS = 600      # URLs from those files fetched once per discovery to see whether they hold proxies
+MIN_MINED_PROXIES = 20      # ... and kept only with at least this many
+MINE_TIMEOUT = 10.0         # per URL
+MINE_DEADLINE = 120.0       # for all of them: slow hosts from someone else's list mustn't stall the run
+_LOCAL_HOST_RE = re.compile(r"(^|\.)(localhost|local|internal|lan|home|intranet)$", re.I)
 MAX_REPOS_PER_OWNER = 3  # against spam accounts with dozens of identical clone repos
 
 
@@ -290,14 +304,63 @@ def github_token() -> Optional[str]:
     return res.stdout.strip() or None
 
 
+def mined_urls(data: bytes) -> List[str]:
+    """URLs in another scraper's source file that could be proxy lists (not templates, not VPN configs)."""
+    urls = []
+    for raw in _URL_RE.findall(data):
+        url = raw.decode("utf-8", "replace").rstrip(".;:")
+        if "{" in url or "}" in url or "$" in url or _REPO_REJECT_RE.search(url):
+            continue
+        parts = urlsplit(url)
+        if not _public_host(parts):  # someone else's file must not point us at localhost or the local network
+            continue
+        path = parts.path.lower()
+        if not _LISTISH_RE.search(url) or _PATH_REJECT_RE.search(path) or path.endswith((".md", ".html", ".png")):
+            continue
+        url = normalize_url(url) or url
+        if url not in urls:
+            urls.append(url)
+    return urls
+
+
+def _public_host(parts) -> bool:
+    try:
+        port = parts.port
+    except ValueError:
+        return False
+    host = (parts.hostname or "").rstrip(".")
+    if parts.scheme not in ("http", "https") or port not in (None, 80, 443) or not host or _LOCAL_HOST_RE.search(host):
+        return False
+    try:
+        return ipaddress.ip_address(host).is_global
+    except ValueError:
+        return "." in host  # a name – single labels only resolve on a local network
+
+
+def _mined_type(url: str) -> str:
+    """The type a URL names (in the path, or an API's query string like ?protocol=socks4), else "auto"."""
+    parts = urlsplit(url)
+    m = _PATH_TYPE_RE.search(f"{parts.path}?{parts.query}".lower())
+    return TYPE_ALIASES[m[1]] if m else "auto"
+
+
+def _readable_type(url: str, data: bytes) -> Optional[str]:
+    """The type to keep a mined list under (the one the scrape will read it with) – None if it doesn't hold
+    enough proxies that way."""
+    ptype = _mined_type(url)
+    return ptype if len(validate_candidates(extract_candidates(data, ptype))) >= MIN_MINED_PROXIES else None
+
+
 async def discover_github(
     get: Getter,
     token: Optional[str],
     max_repos: int,
     days: int = 3,
     on_progress: Optional[Callable[[str], None]] = None,
+    known: Iterable[str] = (),
 ) -> SourceMap:
-    """Looks for actively maintained proxy list repos and their list files."""
+    """Looks for actively maintained proxy list repos and their list files – and for the source lists of other
+    scrapers, whose URLs are kept if they really hold proxies. `known`: sources that needn't be tried again."""
     headers = {"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}
     if token:
         headers["Authorization"] = f"Bearer {token}"
@@ -366,17 +429,62 @@ async def discover_github(
                 if ptype:
                     files.append((entry["path"].count("/"), entry["path"], ptype))
         files.sort()  # shallow paths first – those are usually the complete lists
+        configs = sorted((e["path"].count("/"), e["path"]) for e in tree
+                         if e.get("type") == "blob" and 100 <= e.get("size", 0) <= 300_000
+                         and e["path"].count("/") <= 3 and _CONFIG_PATH_RE.search(e["path"]))
+        configs_found.extend(f"{GH_RAW}/{name}/{branch}/{quote(path)}" for _d, path in configs[:MAX_CONFIGS_PER_REPO])
         return {
             f"{GH_RAW}/{name}/{branch}/{quote(path)}": ptype
             for _depth, path, ptype in files[:MAX_FILES_PER_REPO]
         }
 
+    configs_found: List[str] = []
     found: SourceMap = {}
     for res in await asyncio.gather(*(scan(n, b) for n, b in chosen)):
         found.update(res)
+    found.update(await _mine(get, configs_found, set(known) | set(found), on_progress))
     return found
 
 
+async def _mine(get: Getter, configs: List[str], known: Set[str],
+                on_progress: Optional[Callable[[str], None]] = None) -> SourceMap:
+    """Read other scrapers' source files, then fetch each new URL once: only real proxy lists are kept."""
+    sem = asyncio.Semaphore(32)
+
+    async def fetch(url: str) -> bytes:
+        async with sem:
+            try:
+                return await get(url, timeout=MINE_TIMEOUT)
+            except Exception:
+                return b""
+
+    candidates: List[str] = []
+    for data in await asyncio.gather(*(fetch(u) for u in configs)):
+        for url in mined_urls(data):
+            if url not in known and url not in candidates:
+                candidates.append(url)
+    candidates = candidates[:MAX_MINED_CHECKS]
+    checked = 0
+
+    async def check(url: str) -> Tuple[str, Optional[str]]:
+        nonlocal checked
+        data = await fetch(url)
+        ptype = await asyncio.to_thread(_readable_type, url, data) if data else None
+        checked += 1
+        if on_progress and checked % 50 == 0:
+            on_progress(f"{checked}/{len(candidates)} lists from other scrapers checked")
+        return url, ptype
+
+    if not candidates:
+        return {}
+    tasks = [asyncio.ensure_future(check(u)) for u in candidates]
+    done, pending = await asyncio.wait(tasks, timeout=MINE_DEADLINE)
+    for task in pending:  # whatever hasn't answered by now gets another chance at the next discovery
+        task.cancel()
+    await asyncio.gather(*pending, return_exceptions=True)
+    return {url: ptype for url, ptype in (t.result() for t in done) if ptype}
+
+
 # --------------------------------------------------------------------------- #
 # learning the quality of each source
 # --------------------------------------------------------------------------- #
@@ -416,7 +524,10 @@ class SourceStats:
         if path.exists():
             try:
                 raw = json.loads(path.read_text(encoding="utf-8"))
-                self.records = {url: SourceRecord(**rec) for url, rec in raw.items()}
+                # fields from another version (newer or older) are left out instead of losing everything
+                known = {f.name for f in fields(SourceRecord)}
+                self.records = {url: SourceRecord(**{k: v for k, v in rec.items() if k in known})
+                                for url, rec in raw.items() if isinstance(rec, dict)}
             except (OSError, ValueError, TypeError) as e:
                 # broken statistics are no reason to abort – we just learn again
                 warnings.warn(f"{path.name} unreadable ({e}), starting without source statistics", stacklevel=2)

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/tests/test_fetchcache.py

@@ -153,6 +153,19 @@ def test_changed_source_type_invalidates_the_entry(tmp_path):
     assert cache.conditional_headers("http://x/list", "socks5") == {}
 
 
+def test_a_newer_parser_reloads_lists_once(tmp_path, monkeypatch):
+    # the cache holds parsed keys: after a parser change, a 304 would keep serving what the old parser found
+    from proxyscraper import fetchcache, parsing
+    cache = FetchCache(tmp_path / "cache")
+    cache.store("http://x/list", {b"etag": b'"a"'}, "http 1.1.1.1:80", "http")
+    cache.save()
+    monkeypatch.setattr(parsing, "PARSER_VERSION", parsing.PARSER_VERSION + 1)
+    again = fetchcache.FetchCache(tmp_path / "cache")
+    assert again.conditional_headers("http://x/list", "http") == {}
+    again.store("http://x/list", {b"etag": b'"a"'}, "http 1.1.1.1:80", "http")
+    assert again.conditional_headers("http://x/list", "http") == {"If-None-Match": '"a"'}
+
+
 @pytest.mark.parametrize("method, headers, body, allowed", [
     ("GET", None, None, True),                                  # public list
     ("GET", {"If-None-Match": '"a"'}, None, True),              # conditional fetch for the cache

proxy_scraper_cli-1.8.1/tests/test_mine_sources.py

@@ -0,0 +1,102 @@
+"""Discovery also reads the source lists of other proxy scrapers – and keeps only URLs that really hold proxies."""
+
+import asyncio
+import json
+
+from proxyscraper import sources as srcs
+
+LIST = "\n".join(f"8.8.{i // 250}.{i % 250 + 1}:8080" for i in range(30)).encode()
+CONFIG = json.dumps({"sources": [
+    "https://raw.githubusercontent.com/someone/lists/main/socks5.txt",      # a real list -> kept, socks5
+    "https://example.org/api/proxies?format=text",                         # a real list, type unknown -> auto
+    "https://www.some-proxy-site.com/free-proxy-list/",                    # an HTML page -> dropped
+    "https://example.org/api/{protocol}.txt",                               # a template -> never fetched
+    "https://raw.githubusercontent.com/someone/v2ray-configs/main/all.txt",  # VPN configs -> never fetched
+    "https://known.example/http.txt",                                       # already a source -> not fetched again
+]}).encode()
+
+
+def fake_get(fetched):
+    async def get(url, timeout=0, headers=None):
+        fetched.append(url)
+        if "search/repositories" in url:
+            return json.dumps({"items": [{"full_name": "dev/scraper", "default_branch": "main",
+                                          "stargazers_count": 5}]}).encode()
+        if "/git/trees/" in url:
+            return json.dumps({"tree": [
+                {"type": "blob", "path": "config/sources.json", "size": 800},
+                {"type": "blob", "path": "main.py", "size": 800},
+            ]}).encode()
+        if url.endswith("config/sources.json"):
+            return CONFIG
+        if "some-proxy-site" in url:
+            return b"<html><body>no proxies in plain text here</body></html>"
+        return LIST
+    return get
+
+
+def test_urls_from_scraper_configs_are_checked_before_they_count():
+    fetched = []
+    found = asyncio.run(srcs.discover_github(fake_get(fetched), token=None, max_repos=10,
+                                             known={"https://known.example/http.txt"}))
+    assert found == {
+        "https://raw.githubusercontent.com/someone/lists/main/socks5.txt": "socks5",
+        "https://example.org/api/proxies?format=text": "auto",   # plain ip:port lines: auto reads them as http
+    }
+    assert not any("{protocol}" in u or "v2ray" in u or "known.example" in u for u in fetched)
+    # only files that look like source lists are read
+    assert "https://raw.githubusercontent.com/dev/scraper/main/main.py" not in fetched
+
+
+def test_mined_urls_are_capped():
+    many = json.dumps([f"https://example.org/list{i}.txt" for i in range(srcs.MAX_MINED_CHECKS + 50)]).encode()
+    fetched = []
+    base = fake_get(fetched)
+
+    async def get(url, timeout=0, headers=None):
+        if url.endswith("config/sources.json"):
+            fetched.append(url)
+            return many
+        return await base(url, timeout, headers)
+
+    found = asyncio.run(srcs.discover_github(get, token=None, max_repos=10))
+    assert len(found) == srcs.MAX_MINED_CHECKS
+
+
+def test_the_type_can_sit_in_the_query_string():
+    assert srcs._mined_type("https://api.example.com/v2/?request=get&protocol=socks4") == "socks4"
+
+
+def test_scheme_lists_stay_auto():
+    data = "\n".join(f"socks5://8.8.{i // 250}.{i % 250 + 1}:1080" for i in range(30)).encode()
+    assert srcs._readable_type("https://example.org/mixed", data) == "auto"
+    assert srcs._readable_type("https://example.org/mixed", b"nothing") is None
+
+
+def test_urls_to_local_or_private_hosts_are_never_fetched():
+    config = json.dumps(["http://127.0.0.1/proxy.txt", "http://localhost/proxies.txt",
+                         "http://169.254.169.254/latest/proxy", "http://10.0.0.5/socks5.txt",
+                         "http://printer.local/proxy.txt", "https://example.org:8443/proxy.txt",
+                         "https://example.org/proxy.txt"]).encode()
+    assert srcs.mined_urls(config) == ["https://example.org/proxy.txt"]
+
+
+def test_repo_meta_files_dont_take_the_config_slots():
+    assert not srcs._CONFIG_PATH_RE.search(".github/ISSUE_TEMPLATE/config.yml")
+    assert not srcs._CONFIG_PATH_RE.search(".vscode/settings.json")
+    assert srcs._CONFIG_PATH_RE.search("src/sources.py")
+
+
+def test_slow_hosts_cant_stall_discovery(monkeypatch):
+    monkeypatch.setattr(srcs, "MINE_DEADLINE", 0.2)
+    config = json.dumps([f"https://slow{i}.example/proxy.txt" for i in range(5)]
+                        + ["https://fast.example/proxy.txt"]).encode()
+
+    async def get(url, timeout=0, headers=None):
+        if "slow" in url:
+            await asyncio.sleep(30)
+        return config if url == "cfg" else LIST
+
+    async def go():
+        return await asyncio.wait_for(srcs._mine(get, ["cfg"], set()), 5)
+    assert asyncio.run(go()) == {"https://fast.example/proxy.txt": "auto"}

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/tests/test_pages.py

@@ -50,3 +50,11 @@ def test_pages_link_to_each_other(tmp_path):
     assert '<a href="../socks5/" aria-current="page">SOCKS5 (1)</a>' in socks5
     assert '<a href="../country/de/">Germany (3)</a>' in socks5
     assert "Japan" not in socks5  # empty countries aren't linked
+
+
+def test_filter_button_opens_the_list_with_the_same_filter(tmp_path):
+    pages.write_pages(ROWS, tmp_path, NOW)
+    expected = {"socks5": "?types=socks5", "https": "?https=1", "elite": "?elite=1", "country/de": "?country=DE"}
+    for path, query in expected.items():
+        html = (tmp_path / path / "index.html").read_text(encoding="utf-8")
+        assert f'href="{"../" * (path.count("/") + 1)}{query}#list">Filter the full list' in html

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/tests/test_parsing.py

@@ -37,6 +37,17 @@ def test_auto_source_only_uses_prefixed_lines():
     assert parse(b"socks4://5.6.7.8:1080\n1.2.3.4:80\n", "auto") == ["socks4 5.6.7.8:1080"]
 
 
+def test_auto_source_mostly_plain_reads_the_plain_lines_as_http():
+    data = b"socks5://9.9.9.9:1080\n" + b"".join(f"1.2.3.{i}:80\n".encode() for i in range(1, 6))
+    got = parse(data, "auto")
+    assert "socks5 9.9.9.9:1080" in got and "http 1.2.3.1:80" in got and len(got) == 6
+
+
+def test_auto_source_without_any_prefix_is_read_as_http():
+    # a generic proxies.txt with plain ip:port lines: before, it gave nothing and got paused as "unreachable"
+    assert parse(b"1.2.3.4:80\n5.6.7.8:3128\n", "auto") == ["http 1.2.3.4:80", "http 5.6.7.8:3128"]
+
+
 def test_private_and_invalid_addresses_are_dropped():
     data = b"10.0.0.1:80\n192.168.1.1:8080\n300.1.1.1:80\n1.1.1.1:70000\n001.002.003.004:0080\n"
     assert parse(data) == ["http 1.2.3.4:80"]

{proxy_scraper_cli-1.8.0 → proxy_scraper_cli-1.8.1}/tests/test_proxy_sources.py

@@ -133,3 +133,17 @@ def test_curated_sources_file_is_valid():
     assert len(sources) > 100
     assert set(sources.values()) <= set(srcs.SOURCE_TYPES)
     assert meta and all("url" in m for m in meta)
+
+
+def test_stats_with_fields_from_another_version_are_still_read(tmp_path):
+    path = tmp_path / "s.json"
+    path.write_text(json.dumps({"u": {"first_seen": 1.0, "count": 5, "some_future_field": "x"}}))
+    st = srcs.SourceStats(path)
+    assert st.get("u").count == 5
+
+
+def test_broken_entries_in_the_stats_file_dont_crash(tmp_path):
+    path = tmp_path / "s.json"
+    path.write_text(json.dumps({"u": None, "v": [1], "w": {"count": 3}}))
+    st = srcs.SourceStats(path)
+    assert st.get("w").count == 3 and "u" not in st.records