scrapy-rotating-proxy-middleware 0.1.0__py3-none-any.whl

scrapy_rotating_proxy/__init__.py

@@ -0,0 +1,4 @@
+from .middleware import RotatingProxyMiddleware
+
+__all__ = ["RotatingProxyMiddleware"]
+__version__ = "0.1.0"

scrapy_rotating_proxy/middleware.py

@@ -0,0 +1,123 @@
+"""Drop-in Scrapy downloader middleware for rotating proxies with ban detection.
+
+Works with any HTTP/HTTPS/SOCKS proxy — a static list, or a single rotating
+gateway (residential providers rotate the exit IP for you on every connection).
+On an anti-bot block (403/429/Cloudflare/DataDome) it transparently rotates the
+proxy and retries, so your spider keeps flowing instead of dying on bans.
+"""
+import logging
+import random
+from urllib.parse import urlparse, urlunparse
+
+from scrapy.exceptions import NotConfigured
+from w3lib.http import basic_auth_header
+
+logger = logging.getLogger(__name__)
+
+# Default response signals that mean "this exit IP got blocked, try another".
+DEFAULT_BAN_CODES = {403, 407, 429, 503}
+DEFAULT_BAN_MARKERS = (
+    b"cf-chl",                 # Cloudflare challenge
+    b"Just a moment",          # Cloudflare interstitial
+    b"Attention Required",     # Cloudflare block
+    b"Access denied",
+    b"captcha-delivery",       # DataDome
+    b"datadome",
+    b"px-captcha",             # PerimeterX
+)
+
+
+class RotatingProxyMiddleware:
+    """Assign a proxy per request and retry on bans by rotating to a fresh one.
+
+    Settings:
+        ROTATING_PROXY_LIST        list of proxy URLs (http://user:pass@host:port)
+        ROTATING_PROXY_GATEWAY     single rotating-gateway URL (alternative to LIST)
+        ROTATING_PROXY_BAN_CODES   status codes treated as bans (default 403/407/429/503)
+        ROTATING_PROXY_MAX_RETRIES retries per request before giving up (default 5)
+    """
+
+    def __init__(self, proxies, gateway, ban_codes, max_retries):
+        self.proxies = proxies or []
+        self.gateway = gateway
+        self.ban_codes = ban_codes
+        self.max_retries = max_retries
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        s = crawler.settings
+        proxies = s.getlist("ROTATING_PROXY_LIST")
+        gateway = s.get("ROTATING_PROXY_GATEWAY")
+        if not proxies and not gateway:
+            raise NotConfigured(
+                "Set ROTATING_PROXY_LIST or ROTATING_PROXY_GATEWAY to use "
+                "RotatingProxyMiddleware."
+            )
+        ban_codes = set(s.getlist("ROTATING_PROXY_BAN_CODES")) or DEFAULT_BAN_CODES
+        ban_codes = {int(c) for c in ban_codes}
+        max_retries = s.getint("ROTATING_PROXY_MAX_RETRIES", 5)
+        return cls(proxies, gateway, ban_codes, max_retries)
+
+    def _pick(self):
+        """Return the next proxy URL: random from the list, or the gateway."""
+        if self.proxies:
+            return random.choice(self.proxies)
+        return self.gateway
+
+    @staticmethod
+    def _apply(request, proxy_url):
+        """Set the proxy on the request, moving any inline credentials into the
+        Proxy-Authorization header (Scrapy does not read user:pass from the URL)."""
+        parsed = urlparse(proxy_url)
+        if parsed.username:
+            creds = basic_auth_header(parsed.username, parsed.password or "")
+            request.headers[b"Proxy-Authorization"] = creds
+            netloc = parsed.hostname + (f":{parsed.port}" if parsed.port else "")
+            proxy_url = urlunparse((parsed.scheme, netloc, "", "", "", ""))
+        request.meta["proxy"] = proxy_url
+
+    def process_request(self, request, spider):
+        if "proxy" in request.meta and not request.meta.get("_rotating"):
+            return  # respect a proxy the spider set deliberately
+        self._apply(request, self._pick())
+
+    def _is_ban(self, response):
+        if response.status in self.ban_codes:
+            return True
+        body = response.body[:4096].lower()
+        return any(m.lower() in body for m in DEFAULT_BAN_MARKERS)
+
+    def process_response(self, request, response, spider):
+        if not self._is_ban(response):
+            return response
+        retries = request.meta.get("_proxy_retries", 0)
+        if retries >= self.max_retries:
+            logger.warning(
+                "Gave up on %s after %d proxy rotations (last status %d)",
+                request.url, retries, response.status,
+            )
+            return response
+        logger.debug(
+            "Ban detected (%d) on %s — rotating proxy, retry %d/%d",
+            response.status, request.url, retries + 1, self.max_retries,
+        )
+        new = request.copy()
+        new.meta["_proxy_retries"] = retries + 1
+        new.meta["_rotating"] = True
+        new.meta.pop("proxy", None)
+        self._apply(new, self._pick())
+        new.dont_filter = True
+        return new
+
+    def process_exception(self, request, exception, spider):
+        # Connection errors to a dead proxy: rotate and retry, same budget.
+        retries = request.meta.get("_proxy_retries", 0)
+        if retries >= self.max_retries:
+            return None
+        new = request.copy()
+        new.meta["_proxy_retries"] = retries + 1
+        new.meta["_rotating"] = True
+        new.meta.pop("proxy", None)
+        self._apply(new, self._pick())
+        new.dont_filter = True
+        return new

scrapy_rotating_proxy_middleware-0.1.0.dist-info/METADATA

@@ -0,0 +1,104 @@
+Metadata-Version: 2.4
+Name: scrapy-rotating-proxy-middleware
+Version: 0.1.0
+Summary: Scrapy downloader middleware that rotates proxies and retries on Cloudflare/DataDome/PerimeterX bans.
+Author-email: JiBao Proxy <support@jibaoproxy.com>
+License: MIT
+Project-URL: Homepage, https://jibaoproxy.com
+Project-URL: Source, https://github.com/jibaoproxyofficial-pixel/scrapy-rotating-proxy-middleware
+Keywords: scrapy,proxy,rotating-proxy,web-scraping,cloudflare,datadome,anti-bot,residential-proxy
+Classifier: Framework :: Scrapy
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: Scrapy>=2.0
+Requires-Dist: w3lib
+Dynamic: license-file
+
+# scrapy-rotating-proxy-middleware
+
+[![PyPI version](https://img.shields.io/pypi/v/scrapy-rotating-proxy-middleware.svg)](https://pypi.org/project/scrapy-rotating-proxy-middleware/)
+[![Python versions](https://img.shields.io/pypi/pyversions/scrapy-rotating-proxy-middleware.svg)](https://pypi.org/project/scrapy-rotating-proxy-middleware/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+A drop-in [Scrapy](https://scrapy.org) downloader middleware that **rotates proxies and retries on bans** — `403`, `429`, Cloudflare "Just a moment", DataDome, and PerimeterX challenges. Point it at a static proxy list or a single rotating gateway and your spider stops dying on blocks.
+
+```bash
+pip install scrapy-rotating-proxy-middleware
+```
+
+## Why
+
+Scrapy's built-in `HttpProxyMiddleware` assigns **one** proxy and never reacts when that exit IP gets blocked. In practice most anti-bot blocks aren't about your spider logic — they're about the IP and its [TLS fingerprint](https://jibaoproxy.com/blog/ja3-tls-fingerprint-detection-explained.html) being scored before your request reaches the page. This middleware:
+
+- assigns a proxy per request (random from a list, or a rotating gateway),
+- **detects bans** by status code *and* response-body signature (Cloudflare / DataDome / PerimeterX),
+- transparently **rotates to a fresh proxy and retries**, with a per-request retry budget,
+- moves inline `user:pass` credentials into the `Proxy-Authorization` header automatically.
+
+## Setup
+
+Enable it in `settings.py` and disable Scrapy's default proxy middleware:
+
+```python
+DOWNLOADER_MIDDLEWARES = {
+    "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": None,
+    "scrapy_rotating_proxy.middleware.RotatingProxyMiddleware": 610,
+}
+```
+
+### Option A — a rotating residential gateway (recommended)
+
+A residential gateway gives you a **new exit IP on every connection** from a single URL, so you don't manage a list at all:
+
+```python
+# settings.py
+ROTATING_PROXY_GATEWAY = "http://USERNAME:PASSWORD@us.jibaoproxy.com:913"
+```
+
+### Option B — a static proxy list
+
+```python
+ROTATING_PROXY_LIST = [
+    "http://USERNAME:PASSWORD@proxy-a.example.com:8000",
+    "http://USERNAME:PASSWORD@proxy-b.example.com:8000",
+    "socks5://USERNAME:PASSWORD@proxy-c.example.com:1080",
+]
+```
+
+That's it — run your spider as usual.
+
+## Configuration
+
+| Setting | Default | Description |
+| --- | --- | --- |
+| `ROTATING_PROXY_GATEWAY` | – | Single rotating-gateway URL. |
+| `ROTATING_PROXY_LIST` | – | List of proxy URLs (used if no gateway). |
+| `ROTATING_PROXY_BAN_CODES` | `403, 407, 429, 503` | Status codes treated as bans. |
+| `ROTATING_PROXY_MAX_RETRIES` | `5` | Proxy rotations per request before giving up. |
+
+Set a proxy on a single request explicitly and the middleware leaves it alone:
+
+```python
+yield scrapy.Request(url, meta={"proxy": "http://USERNAME:PASSWORD@host:port"})
+```
+
+## Ban detection
+
+A response counts as a ban when its status is in `ROTATING_PROXY_BAN_CODES`, **or** the first 4 KB of the body matches a known anti-bot signature (`cf-chl`, `Just a moment`, `Attention Required`, `captcha-delivery`/DataDome, `px-captcha`/PerimeterX). On a ban the request is re-scheduled with a fresh proxy and `dont_filter=True`, up to the retry budget.
+
+If you keep hitting bans after rotation, the exit IPs themselves are the problem — datacenter ranges get scored as bot traffic at the ASN level. Residential exits with clean ASN reputation are what actually pass. We build [JiBao Proxy](https://jibaoproxy.com) for exactly this: 72M+ residential IPs across 200+ countries, sticky sessions, and SOCKS5/HTTP gateways. The middleware works with any provider, though.
+
+## Related
+
+- [Scrapy proxy middleware: the complete guide](https://jibaoproxy.com/blog/scrapy-proxy-middleware-guide.html)
+- [Why your JA3/TLS fingerprint gets you blocked](https://jibaoproxy.com/blog/ja3-tls-fingerprint-detection-explained.html)
+- [Bypassing DataDome & PerimeterX in 2026](https://jibaoproxy.com/blog/datadome-perimeterx-bypass-2026.html)
+
+## License
+
+MIT

scrapy_rotating_proxy_middleware-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+scrapy_rotating_proxy/__init__.py,sha256=o3vPM0cwNozHr_F9LlQCy8O-iDrD5FSO6c0NgqZ45NQ,109
+scrapy_rotating_proxy/middleware.py,sha256=cZKGY2FyptswODmEcyNHyz4TPaFMYWGU78lhAfVus9w,4971
+scrapy_rotating_proxy_middleware-0.1.0.dist-info/licenses/LICENSE,sha256=EqlXESyqjpkAYUf6imqo1JrW9eYaOurSRNBh4LO7bQE,1068
+scrapy_rotating_proxy_middleware-0.1.0.dist-info/METADATA,sha256=TjyOjlwibr_Q_EVn05Z10XT3gBAEE_RaG0tCxmYOHP0,4984
+scrapy_rotating_proxy_middleware-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+scrapy_rotating_proxy_middleware-0.1.0.dist-info/top_level.txt,sha256=E4N3CweZpjiY77HqEG76gQN1LX5JeMB3nO42Evhc_-k,22
+scrapy_rotating_proxy_middleware-0.1.0.dist-info/RECORD,,

scrapy_rotating_proxy_middleware-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

scrapy_rotating_proxy_middleware-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 JiBao Proxy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scrapy_rotating_proxy_middleware-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+scrapy_rotating_proxy