proxy-scraper-cli 1.7.1__py3-none-any.whl

proxyscraper/asndb.py

@@ -0,0 +1,181 @@
+"""Providers (ASN) of the exit IPs offline – with the free ASN database from DB-IP (DB-IP Lite, CC BY 4.0).
+
+Like geodb.py: load once a month (≈ 7 MB gzip, ~400,000 IPv4 ranges), turn it into arrays,
+look up by binary search. Plus a guess whether the provider is a datacenter/hoster – proxies
+that exit from datacenters get blocked by many sites sooner than those on home or mobile
+connections.
+
+The guess is a heuristic over the provider name (cloud, hosting, known hosters), not a
+certainty. Measured on 316 working proxies, ~45 % exited from datacenters this way.
+
+IP Geolocation by DB-IP: https://db-ip.com
+"""
+
+from __future__ import annotations
+
+import csv
+import gzip
+import io
+import json
+import re
+import socket
+import struct
+from array import array
+from bisect import bisect_right
+from dataclasses import dataclass
+from datetime import date
+from pathlib import Path
+from typing import List, Optional
+
+from .geodb import _months
+from .netio import http_get
+from .paths import DATA_DIR
+
+DB_FILE = DATA_DIR / "geo" / "dbip-asn-ipv4.bin"
+URL = "https://download.db-ip.com/free/dbip-asn-lite-{month}.csv.gz"
+MAGIC = b"PSASN1"
+
+HOSTING_RE = re.compile(
+    r"host|cloud|data ?cent|server|\bvps\b|colo|amazon|\baws\b|google|microsoft|azure|"
+    r"digitalocean|ovh|hetzner|linode|akamai|vultr|choopa|contabo|leaseweb|m247|alibaba|tencent|oracle|"
+    r"scaleway|ionos|godaddy|fastly|cloudflare|datacamp|cdn77|psychz|colocrossing|quadranet|hostwinds|"
+    r"kamatera|upcloud|netcup|strato|hosteurope|constant company|g-core|gcore|stark industries|aeza|"
+    r"pq hosting|serverius|worldstream|zenlayer|ucloud|huawei|baidu|kingsoft|performive|nocix|datasource|"
+    r"frantech|buyvm|ramnode|namecheap|dreamhost|packet|equinix|dataforest|servinga|hydra communications",
+    re.IGNORECASE,
+)
+
+
+def is_hosting(org: str) -> bool:
+    return bool(HOSTING_RE.search(org or ""))
+
+
+def _ip_to_int(ip: str) -> int:
+    return struct.unpack("!I", socket.inet_aton(ip))[0]
+
+
+@dataclass(frozen=True)
+class Provider:
+    asn: int
+    org: str
+    hosting: bool
+
+
+class AsnDB:
+    def __init__(self, starts: array, ends: array, asns: array, org_index: array, orgs: List[str], month: str):
+        self.starts, self.ends, self.asns, self.org_index = starts, ends, asns, org_index
+        self.orgs = orgs
+        self.hosting = [is_hosting(o) for o in orgs]
+        self.month = month
+
+    def __len__(self) -> int:
+        return len(self.starts)
+
+    def lookup(self, ip: str) -> Optional[Provider]:
+        try:
+            n = _ip_to_int(ip)
+        except OSError:
+            return None
+        i = bisect_right(self.starts, n) - 1
+        if i < 0 or n > self.ends[i]:
+            return None
+        k = self.org_index[i]
+        return Provider(self.asns[i], self.orgs[k], self.hosting[k])
+
+    @classmethod
+    def from_csv(cls, text: str, month: str) -> "AsnDB":
+        """DB-IP CSV ("start,end,asn,\\"provider\\"") -> IPv4 only. Provider names are stored just once."""
+        starts, ends, asns, org_index = array("I"), array("I"), array("I"), array("I")
+        orgs: List[str] = []
+        seen = {}
+        for row in csv.reader(io.StringIO(text)):
+            if len(row) < 4 or ":" in row[0] or not row[2].isdigit():
+                continue
+            try:
+                s, e = _ip_to_int(row[0]), _ip_to_int(row[1])
+            except OSError:
+                continue
+            org = row[3].strip()
+            k = seen.get(org)
+            if k is None:
+                k = seen[org] = len(orgs)
+                orgs.append(org)
+            starts.append(s)
+            ends.append(e)
+            asns.append(int(row[2]))
+            org_index.append(k)
+        return cls(starts, ends, asns, org_index, orgs, month)
+
+    def save(self, path: Path = DB_FILE) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        table = json.dumps(self.orgs, ensure_ascii=False).encode("utf-8")
+        tmp = path.with_suffix(".tmp")
+        with tmp.open("wb") as fh:
+            fh.write(MAGIC + self.month.encode("ascii").ljust(7) + struct.pack("!II", len(self.starts), len(table)))
+            for arr in (self.starts, self.ends, self.asns, self.org_index):
+                arr.tofile(fh)
+            fh.write(table)
+        tmp.replace(path)
+
+    @classmethod
+    def load(cls, path: Path = DB_FILE) -> Optional["AsnDB"]:
+        try:
+            size = path.stat().st_size
+            with path.open("rb") as fh:
+                if fh.read(len(MAGIC)) != MAGIC:
+                    return None
+                month = fh.read(7).decode("ascii").strip()
+                count, table_len = struct.unpack("!II", fh.read(8))
+                if size != len(MAGIC) + 7 + 8 + count * 16 + table_len:
+                    return None  # truncated or broken – don't blindly allocate memory
+                arrays = []
+                for _ in range(4):
+                    arr = array("I")
+                    arr.fromfile(fh, count)
+                    arrays.append(arr)
+                orgs = json.loads(fh.read(table_len).decode("utf-8"))
+        except (OSError, EOFError, ValueError, struct.error, UnicodeDecodeError):
+            return None
+        if not isinstance(orgs, list) or not all(isinstance(o, str) for o in orgs) \
+                or any(k >= len(orgs) for k in arrays[3]):
+            return None
+        return cls(*arrays, orgs, month)
+
+
+def is_current(db: Optional[AsnDB], today: Optional[date] = None) -> bool:
+    return db is not None and db.month == (today or date.today()).strftime("%Y-%m")
+
+
+async def load_asn_db(path: Path = DB_FILE, today: Optional[date] = None, fetch=http_get) -> Optional[AsnDB]:
+    """Database from data/ – new from DB-IP once a month. May download, so call it in the background."""
+    today = today or date.today()
+    current = AsnDB.load(path)
+    if is_current(current, today):
+        return current
+    for month in _months(today):
+        if current and current.month >= month:
+            return current
+        try:
+            data = await fetch(URL.format(month=month), timeout=30)
+            db = AsnDB.from_csv(gzip.decompress(data).decode("utf-8", "replace"), month)
+        except Exception:  # unreachable or broken – try the next month or keep the old file
+            continue
+        if len(db) > 1000:
+            db.save(path)
+            return db
+    return current
+
+
+class ProviderLookup:
+    """Holds the database (which may only be loaded later) and fills in the provider of hits."""
+
+    def __init__(self, db: Optional[AsnDB] = None):
+        self.db = db
+
+    def annotate(self, r) -> None:
+        provider = self.db.lookup(r.exit_ip) if self.db else None
+        if provider:
+            r.asn, r.org, r.hosting = provider.asn, provider.org, provider.hosting
+
+    def __bool__(self) -> bool:
+        return True

proxyscraper/blocklist.py

@@ -0,0 +1,92 @@
+"""Is the exit IP on a spam blocklist? Sites that use such lists answer those proxies with captchas or not at all.
+
+One DNS lookup per exit IP against SpamCop (bl.spamcop.net), cached for the run. Measured on the live
+list: 29 % of exit IPs were listed there. DroneBL lists 59 % (it is essentially a list of open proxies),
+too many to be useful as a filter, so it isn't used.
+
+Some blocklist operators refuse queries coming from large public resolvers and answer every query then.
+Before the run, the documented test address 127.0.0.2 is looked up: only if it comes back as listed and
+a normal address doesn't, the answers are trusted. Otherwise every result stays unknown (None).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import socket
+from typing import Awaitable, Callable, Dict, Optional
+
+ZONE = "bl.spamcop.net"
+CONCURRENT_LOOKUPS = 50
+LOOKUP_TIMEOUT = 5.0
+NOT_LISTED = "NXDOMAIN"  # the name doesn't exist: the only answer that means "not on the list"
+LISTED_PREFIX = "127.0.0."  # a listed IP resolves to 127.0.0.x; 127.255.255.x means "query refused"
+
+Resolve = Callable[[str], Awaitable[Optional[str]]]
+
+
+_NX_ERRORS = {getattr(socket, n) for n in ("EAI_NONAME", "EAI_NODATA") if hasattr(socket, n)}
+
+
+async def system_resolve(name: str) -> Optional[str]:
+    """First IPv4 address for name, NOT_LISTED if the name doesn't exist, None for any other error
+    (timeout, SERVFAIL, no network) – those must not be mistaken for "not listed"."""
+    try:
+        infos = await asyncio.get_running_loop().getaddrinfo(name, None, family=socket.AF_INET)
+    except socket.gaierror as e:
+        return NOT_LISTED if e.errno in _NX_ERRORS else None
+    except OSError:
+        return None
+    return infos[0][4][0] if infos else None
+
+
+def query_name(ip: str, zone: str = ZONE) -> Optional[str]:
+    parts = ip.split(".")
+    if len(parts) != 4 or not all(p.isdigit() and 0 <= int(p) <= 255 for p in parts):
+        return None
+    return ".".join(reversed(parts)) + "." + zone
+
+
+class Blocklist:
+    def __init__(self, resolve: Resolve = system_resolve, zone: str = ZONE):
+        self.resolve = resolve
+        self.zone = zone
+        self.usable: Optional[bool] = None  # None until probed
+        self.listed = 0
+        self._cache: Dict[str, Optional[bool]] = {}
+        self._slots: Optional[asyncio.Semaphore] = None
+
+    async def probe(self) -> bool:
+        """Does the resolver get real answers? 127.0.0.2 must be listed, 127.0.0.1 must not."""
+        test = await self._ask(query_name("127.0.0.2", self.zone))
+        clean = await self._ask(query_name("127.0.0.1", self.zone))
+        self.usable = bool(test and test.startswith(LISTED_PREFIX)) and clean == NOT_LISTED
+        return self.usable
+
+    async def _ask(self, name: str) -> Optional[str]:
+        try:
+            return await asyncio.wait_for(self.resolve(name), LOOKUP_TIMEOUT)
+        except asyncio.TimeoutError:
+            return None
+
+    async def lookup(self, ip: str) -> Optional[bool]:
+        """True = listed, False = not listed, None = unknown (not probed, refused or not an IPv4)."""
+        if not self.usable:
+            return None
+        if ip in self._cache:
+            return self._cache[ip]
+        name = query_name(ip, self.zone)
+        if name is None:
+            return None
+        if self._slots is None:  # created here: before Python 3.10 a semaphore is bound to the event loop
+            self._slots = asyncio.Semaphore(CONCURRENT_LOOKUPS)
+        async with self._slots:
+            answer = await self._ask(name)
+        if answer == NOT_LISTED:
+            result: Optional[bool] = False
+        elif answer and answer.startswith(LISTED_PREFIX):
+            result = True
+        else:
+            result = None  # an error, a timeout, or 127.255.255.x (the operator refused to answer)
+        self._cache[ip] = result
+        self.listed += bool(result)
+        return result