unblock-requests 0.0.1a2__tar.gz

unblock_requests-0.0.1a2/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: unblock_requests
+Version: 0.0.1a2
+Summary: A requests.Session subclass that bypasses Cloudflare via curl_cffi, FlareSolverr, or the Wayback Machine
+Requires-Python: >=3.9
+Requires-Dist: requests>=2.28
+Provides-Extra: stealth
+Requires-Dist: curl-cffi; extra == "stealth"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"

unblock_requests-0.0.1a2/README.md

@@ -0,0 +1,71 @@
+# unblock_requests
+
+A **drop-in `requests.Session` subclass** that gets your request through
+Cloudflare. It's the anti-bot counterpart to
+[`anon_requests`](https://github.com/TigreGotico/anon_requests) (which handles
+IP anonymity via proxy/Tor rotation): `unblock_requests` handles *bot detection*
+— TLS fingerprinting and JS challenges — and degrades gracefully to the archive.
+
+Because it subclasses `requests.Session` and only overrides `request()`, every
+`.get()/.post()/...` keeps working and anything typed against
+`requests.Session` accepts it unchanged.
+
+```python
+from unblock_requests import CloudflareSession        # alias: Session
+
+s = CloudflareSession(flaresolverr_url="http://192.168.1.116:8191")
+html = s.get("https://www.progarchives.com/artist.asp?id=1").text   # solved live
+import requests; assert isinstance(s, requests.Session)              # True
+```
+
+## Transports
+
+Pick with the `mode=` kwarg, or the `<PREFIX>_TRANSPORT` env var (default prefix
+`UNBLOCK_REQUESTS`). Explicit kwargs always win over the environment.
+
+| Mode | What it does |
+|---|---|
+| `curl_cffi` *(default)* | Chrome TLS impersonation (install the `stealth` extra). Clears the bot check on most networks. |
+| `requests` | Plain `requests`, no impersonation. |
+| `flaresolverr` | Proxy through a [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) headless browser that solves the JS challenge — **live** data. Selected automatically when `flaresolverr_url` is set. |
+| `wayback` | Read the latest Internet Archive snapshot — stale, but needs no infrastructure. |
+
+```python
+CloudflareSession(flaresolverr_url="http://host:8191")   # solve live
+CloudflareSession(mode="wayback")                        # force the archive
+CloudflareSession(flaresolverr_url="http://host:8191", wayback_fallback=True)  # live, archive on failure
+```
+
+## Composing with anon_requests
+
+`unblock_requests` (anti-bot) and `anon_requests` (IP rotation) are orthogonal
+and stack: a modernized `anon_requests` wraps an inner `requests.Session` built
+by a `session_factory`, so you can inject a `CloudflareSession` and get rotation
+**and** challenge-solving together. A rotated proxy flows through every mode —
+including into FlareSolverr (via its `proxy` field):
+
+```python
+from anon_requests import RotatingProxySession           # once modernized
+from unblock_requests import CloudflareSession
+
+session = RotatingProxySession(
+    session_factory=lambda: CloudflareSession(flaresolverr_url="http://host:8191"),
+)
+session.get(url)   # rotates IP + solves Cloudflare
+```
+
+## Install
+
+```bash
+pip install unblock_requests
+pip install unblock_requests[stealth]   # adds curl_cffi (recommended)
+```
+
+## Notes / limits
+
+- In the `wayback`/`flaresolverr` modes the response is **synthesized** from the
+  fetched HTML (real `requests.Response`, but `stream=`/adapters/connection
+  pooling don't apply). `requests`/`curl_cffi` modes are native.
+- Challenge detection is heuristic (`is_challenge()`), used to trigger the
+  optional Wayback fallback on blocked GETs.
+- `wayback_html(url)` and `is_challenge(text)` are exposed for direct use.

unblock_requests-0.0.1a2/pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "unblock_requests"
+dynamic = ["version"]
+description = "A requests.Session subclass that bypasses Cloudflare via curl_cffi, FlareSolverr, or the Wayback Machine"
+requires-python = ">=3.9"
+dependencies = ["requests>=2.28"]
+
+[project.optional-dependencies]
+stealth = ["curl-cffi"]
+test = ["pytest"]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["unblock_requests*"]
+
+[tool.setuptools.dynamic]
+version = {attr = "unblock_requests.version.__version__"}

unblock_requests-0.0.1a2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

unblock_requests-0.0.1a2/tests/test_session.py

@@ -0,0 +1,81 @@
+"""Offline tests for CloudflareSession (no network)."""
+import requests
+
+from unblock_requests import CloudflareSession, is_challenge, wayback_raw_url
+from unblock_requests.session import _full_url, _make_response, _url_variants, _flaresolverr_extract
+
+
+def test_is_a_requests_session():
+    s = CloudflareSession()
+    assert isinstance(s, requests.Session)
+    assert "User-Agent" in s.headers          # inherited Session machinery
+
+
+def test_mode_resolution(monkeypatch):
+    for k in ("UNBLOCK_REQUESTS_TRANSPORT", "UNBLOCK_REQUESTS_FLARESOLVERR_URL", "UNBLOCK_REQUESTS_WAYBACK_FALLBACK"):
+        monkeypatch.delenv(k, raising=False)
+    assert CloudflareSession()._resolved_mode() == "curl_cffi"
+    assert CloudflareSession(mode="wayback")._resolved_mode() == "wayback"
+    assert CloudflareSession(flaresolverr_url="http://x:8191")._resolved_mode() == "flaresolverr"
+
+
+def test_kwarg_beats_env(monkeypatch):
+    monkeypatch.setenv("UNBLOCK_REQUESTS_TRANSPORT", "requests")
+    assert CloudflareSession(mode="wayback")._resolved_mode() == "wayback"
+    assert CloudflareSession()._resolved_mode() == "requests"
+
+
+def test_env_prefix(monkeypatch):
+    monkeypatch.setenv("PYPROG_FLARESOLVERR_URL", "http://box:8191")
+    s = CloudflareSession(env_prefix="PYPROG")
+    assert s._fs_url() == "http://box:8191"
+    assert s._resolved_mode() == "flaresolverr"
+
+
+def test_bad_mode():
+    import pytest
+    with pytest.raises(ValueError):
+        CloudflareSession(mode="nonsense")
+
+
+def test_make_response_is_real_response():
+    r = _make_response("https://x/y", content=b'{"a": 1}',
+                       headers={"Content-Type": "application/json"})
+    assert isinstance(r, requests.Response)
+    assert r.status_code == 200
+    assert r.text == '{"a": 1}'
+    assert r.json() == {"a": 1}
+    assert r.content == b'{"a": 1}'
+    r.raise_for_status()      # 200 → no raise
+
+
+def test_make_response_raise_for_status():
+    import pytest
+    r = _make_response("https://x", content=b"nope", status=503, reason="boom")
+    with pytest.raises(requests.HTTPError):
+        r.raise_for_status()
+
+
+def test_helpers():
+    assert is_challenge("<title>Just a moment...</title>")
+    assert not is_challenge("<html><body>real page</body></html>")
+    assert wayback_raw_url("http://web.archive.org/web/20250101/https://x/y") == \
+        "http://web.archive.org/web/20250101id_/https://x/y"
+    assert _full_url("https://x/p", {"a": "b"}) == "https://x/p?a=b"
+    variants = list(_url_variants("https://www.x.com/p"))
+    assert "http://www.x.com/p" in variants and "www.x.com/p" in variants
+
+
+def test_flaresolverr_extract():
+    import pytest
+    assert _flaresolverr_extract({"status": "ok", "solution": {"response": "<h1>hi</h1>"}}) == "<h1>hi</h1>"
+    with pytest.raises(RuntimeError):
+        _flaresolverr_extract({"status": "error", "message": "x"})
+
+
+def test_proxy_url_selection():
+    s = CloudflareSession()
+    assert s._proxy_url({"proxies": {"https": "socks5://1.2.3.4:1080"}}) == "socks5://1.2.3.4:1080"
+    s.proxies = {"http": "http://5.6.7.8:3128"}
+    assert s._proxy_url({}) == "http://5.6.7.8:3128"
+    assert CloudflareSession()._proxy_url({}) is None

unblock_requests-0.0.1a2/unblock_requests/__init__.py

@@ -0,0 +1,47 @@
+"""unblock_requests — a ``requests.Session`` that shrugs off Cloudflare.
+
+``CloudflareSession`` is a **drop-in subclass of** :class:`requests.Session`.
+Because every ``.get()/.post()/.head()/...`` call funnels through
+``Session.request()``, overriding that one method lets us transparently route
+through:
+
+- ``curl_cffi`` Chrome TLS impersonation (the default when installed),
+- a **FlareSolverr** proxy that solves the JS challenge in a real browser
+  (live data),
+- the **Wayback Machine** (stale but dependency-free),
+
+with an optional live→archive fallback — while always returning genuine
+:class:`requests.Response` objects. Any code typed against ``requests.Session``
+keeps working unchanged::
+
+    from unblock_requests import CloudflareSession
+
+    s = CloudflareSession(flaresolverr_url="http://192.168.1.116:8191")
+    s.get("https://www.progarchives.com/artist.asp?id=1").text   # solved live
+    s.headers["Referer"] = "..."                                  # inherited
+    assert isinstance(s, requests.Session)                        # True
+
+Modes (``mode=`` kwarg, or the ``<PREFIX>_TRANSPORT`` env var):
+``"requests"`` / ``"curl_cffi"`` / ``"wayback"`` / ``"flaresolverr"``.
+Explicit kwargs always win over the environment. Setting ``flaresolverr_url``
+(kwarg or ``<PREFIX>_FLARESOLVERR_URL``) selects FlareSolverr automatically.
+"""
+from unblock_requests.session import (
+    CloudflareSession,
+    is_challenge,
+    wayback_html,
+    wayback_raw_url,
+)
+from unblock_requests.version import __version__
+
+# Convenience alias — read as "a Session, but cloudflare-aware".
+Session = CloudflareSession
+
+__all__ = [
+    "CloudflareSession",
+    "Session",
+    "is_challenge",
+    "wayback_html",
+    "wayback_raw_url",
+    "__version__",
+]

unblock_requests-0.0.1a2/unblock_requests/session.py

@@ -0,0 +1,259 @@
+"""The ``CloudflareSession`` drop-in and its transport helpers."""
+from __future__ import annotations
+
+import os
+import re
+from typing import Any, Optional
+
+import requests
+from requests.models import PreparedRequest, Response
+from requests.structures import CaseInsensitiveDict
+
+_WAYBACK_AVAILABLE_API = "http://archive.org/wayback/available"
+_VALID_MODES = {"requests", "curl_cffi", "wayback", "flaresolverr"}
+
+_DEFAULT_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
+    ),
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+}
+
+
+# ---------------------------------------------------------------------------
+# stateless helpers
+# ---------------------------------------------------------------------------
+
+def is_challenge(text: str) -> bool:
+    """Heuristically detect a Cloudflare interstitial in a response body."""
+    head = (text or "")[:1500].lower()
+    return ("just a moment" in head or "cf-mitigated" in head
+            or "challenge-platform" in head or "cf_chl_opt" in head)
+
+
+def _truthy(value: Optional[str]) -> bool:
+    return (value or "").strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _full_url(url: str, params: Any) -> str:
+    """Bake query *params* into *url* using requests' own encoder."""
+    if not params:
+        return url
+    pr = PreparedRequest()
+    pr.prepare_url(url, params)
+    return pr.url
+
+
+def wayback_raw_url(snapshot_url: str) -> str:
+    """``.../web/<ts>/<original>`` → ``.../web/<ts>id_/<original>`` (raw bytes,
+    no Wayback toolbar or link rewriting)."""
+    return re.sub(r"(/web/\d+)/", r"\1id_/", snapshot_url, count=1)
+
+
+def _url_variants(url: str):
+    seen = set()
+    bare = re.sub(r"^https?://", "", url)
+    for candidate in (url, "http://" + bare, "https://" + bare, bare):
+        if candidate not in seen:
+            seen.add(candidate)
+            yield candidate
+
+
+def _make_response(url: str, *, content: bytes, status: int = 200,
+                   headers: Optional[dict] = None, reason: str = "OK",
+                   request: Optional[PreparedRequest] = None) -> Response:
+    """Build a genuine :class:`requests.Response` from raw bytes."""
+    r = Response()
+    r.status_code = status
+    r._content = content
+    r.url = url
+    r.reason = reason
+    r.encoding = "utf-8"
+    r.headers = CaseInsensitiveDict(headers or {"Content-Type": "text/html; charset=utf-8"})
+    if request is not None:
+        r.request = request
+    return r
+
+
+def wayback_html(url: str, *, timeout: float = 30.0,
+                 headers: Optional[dict] = None) -> Optional[str]:
+    """Return the latest Wayback Machine snapshot of *url* as raw HTML, or
+    ``None`` if the archive has no usable capture. archive.org is not
+    Cloudflare-gated, so plain ``requests`` is used."""
+    headers = headers or _DEFAULT_HEADERS
+    snap = None
+    for candidate in _url_variants(url):
+        try:
+            meta = requests.get(_WAYBACK_AVAILABLE_API, params={"url": candidate},
+                                headers=headers, timeout=timeout).json()
+        except Exception:
+            continue
+        snap = (meta.get("archived_snapshots", {}) or {}).get("closest")
+        if snap and snap.get("available") and snap.get("url"):
+            break
+        snap = None
+    if not snap:
+        return None
+    try:
+        r = requests.get(wayback_raw_url(snap["url"]), headers=headers, timeout=timeout)
+        r.raise_for_status()
+    except Exception:
+        return None
+    return r.text
+
+
+def _flaresolverr_extract(data: dict) -> str:
+    if data.get("status") != "ok":
+        raise RuntimeError(f"FlareSolverr error: {data.get('message') or data.get('status')}")
+    return (data.get("solution") or {}).get("response", "")
+
+
+# ---------------------------------------------------------------------------
+# the Session subclass
+# ---------------------------------------------------------------------------
+
+class CloudflareSession(requests.Session):
+    """A :class:`requests.Session` that transparently bypasses Cloudflare.
+
+    Args:
+        mode:                    ``"requests"`` / ``"curl_cffi"`` / ``"wayback"``
+                                 / ``"flaresolverr"``. ``None`` → resolve from
+                                 the environment, then auto (FlareSolverr if a
+                                 URL is configured, else curl_cffi).
+        flaresolverr_url:        FlareSolverr base URL (e.g.
+                                 ``"http://192.168.1.116:8191"``).
+        flaresolverr_timeout_ms: per-request solve budget (default 60000).
+        wayback_fallback:        fall back to the Wayback Machine when a live
+                                 GET is blocked. ``None`` → read the env flag.
+        impersonate:             curl_cffi browser target (default ``"chrome"``).
+        env_prefix:              namespace for the env fallbacks (default
+                                 ``"UNBLOCK_REQUESTS"``): ``<PREFIX>_TRANSPORT``,
+                                 ``<PREFIX>_FLARESOLVERR_URL``,
+                                 ``<PREFIX>_FLARESOLVERR_TIMEOUT``,
+                                 ``<PREFIX>_WAYBACK_FALLBACK``.
+    """
+
+    def __init__(self, *, mode: Optional[str] = None,
+                 flaresolverr_url: Optional[str] = None,
+                 flaresolverr_timeout_ms: Optional[int] = None,
+                 wayback_fallback: Optional[bool] = None,
+                 impersonate: str = "chrome",
+                 env_prefix: str = "UNBLOCK_REQUESTS") -> None:
+        super().__init__()
+        if mode is not None and mode.lower() not in _VALID_MODES:
+            raise ValueError(f"mode must be one of {sorted(_VALID_MODES)} or None, got {mode!r}")
+        self.cf_mode = mode.lower() if mode else None
+        self.flaresolverr_url = flaresolverr_url
+        self.flaresolverr_timeout_ms = flaresolverr_timeout_ms
+        self.wayback_fallback = wayback_fallback
+        self.impersonate = impersonate
+        self.env_prefix = env_prefix
+        self.headers.update(_DEFAULT_HEADERS)
+        self._curl: Any = None
+
+    # -- config resolution (explicit kwarg > env > default) ----------------
+
+    def _env(self, suffix: str) -> str:
+        return os.environ.get(f"{self.env_prefix}_{suffix}", "").strip()
+
+    def _fs_url(self) -> str:
+        return self.flaresolverr_url or self._env("FLARESOLVERR_URL")
+
+    def _fs_timeout(self) -> int:
+        if self.flaresolverr_timeout_ms is not None:
+            return self.flaresolverr_timeout_ms
+        return int(self._env("FLARESOLVERR_TIMEOUT") or "60000")
+
+    def _resolved_mode(self) -> str:
+        if self.cf_mode:
+            return self.cf_mode
+        env = self._env("TRANSPORT").lower()
+        if env:
+            return env
+        if self._fs_url():
+            return "flaresolverr"
+        return "curl_cffi"
+
+    def _do_wayback_fallback(self) -> bool:
+        if self.wayback_fallback is not None:
+            return self.wayback_fallback
+        return _truthy(self._env("WAYBACK_FALLBACK"))
+
+    # -- per-mode fetchers (all return requests.Response) ------------------
+
+    def _curl_session(self):
+        if self._curl is None:
+            from curl_cffi import requests as cffi  # type: ignore[import]
+            self._curl = cffi.Session(impersonate=self.impersonate)
+            self._curl.headers.update(self.headers)
+        return self._curl
+
+    def _via_curl(self, method: str, url: str, **kwargs) -> Response:
+        allowed = {"params", "data", "json", "headers", "cookies", "timeout",
+                   "allow_redirects", "proxies"}
+        cr = self._curl_session().request(
+            method, url, **{k: v for k, v in kwargs.items() if k in allowed})
+        return _make_response(
+            getattr(cr, "url", url), content=cr.content, status=cr.status_code,
+            headers=dict(cr.headers), reason=getattr(cr, "reason", "") or "OK")
+
+    def _via_flaresolverr(self, url: str, proxy: Optional[str] = None) -> Response:
+        endpoint = (self._fs_url() or "http://localhost:8191").rstrip("/")
+        timeout_ms = self._fs_timeout()
+        payload = {"cmd": "request.get", "url": url, "maxTimeout": timeout_ms}
+        if proxy:
+            # FlareSolverr drives the headless browser through this proxy — so a
+            # rotated proxy (e.g. from anon_requests) reaches the solved request.
+            payload["proxy"] = {"url": proxy}
+        resp = requests.post(f"{endpoint}/v1", json=payload, timeout=timeout_ms / 1000 + 30)
+        resp.raise_for_status()
+        html = _flaresolverr_extract(resp.json())
+        return _make_response(url, content=html.encode("utf-8"))
+
+    def _proxy_url(self, kwargs: dict) -> Optional[str]:
+        """A single proxy URL from per-request ``proxies=`` or ``self.proxies``."""
+        proxies = kwargs.get("proxies") or self.proxies or {}
+        return proxies.get("https") or proxies.get("http") or next(iter(proxies.values()), None)
+
+    def _via_wayback(self, url: str) -> Optional[Response]:
+        html = wayback_html(url, headers=dict(self.headers))
+        if html is None:
+            return None
+        return _make_response(url, content=html.encode("utf-8"))
+
+    # -- the one override that makes the whole Session cloudflare-aware ----
+
+    def request(self, method: str, url: str, **kwargs) -> Response:  # type: ignore[override]
+        mode = self._resolved_mode()
+        full = _full_url(url, kwargs.get("params"))
+        is_get = method.upper() == "GET"
+
+        if mode == "wayback":
+            resp = self._via_wayback(full)
+            if resp is None:
+                raise RuntimeError(f"no Wayback Machine snapshot available for {full}")
+            return resp
+
+        try:
+            if mode == "flaresolverr" and is_get:
+                resp = self._via_flaresolverr(full, proxy=self._proxy_url(kwargs))
+            elif mode == "curl_cffi":
+                try:
+                    resp = self._via_curl(method, url, **kwargs)
+                except ImportError:
+                    resp = super().request(method, url, **kwargs)  # curl_cffi absent
+            else:
+                resp = super().request(method, url, **kwargs)
+            # treat a served challenge as a failure so the fallback can fire
+            if is_get and (resp.status_code in (403, 503) or is_challenge(resp.text)) \
+                    and is_challenge(resp.text):
+                raise RuntimeError("Cloudflare challenge served")
+            return resp
+        except Exception:
+            if is_get and self._do_wayback_fallback():
+                resp = self._via_wayback(full)
+                if resp is not None:
+                    return resp
+            raise

unblock_requests-0.0.1a2/unblock_requests/version.py

@@ -0,0 +1,8 @@
+# START_VERSION_BLOCK
+VERSION_MAJOR = 0
+VERSION_MINOR = 0
+VERSION_BUILD = 1
+VERSION_ALPHA = 2
+# END_VERSION_BLOCK
+
+__version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + (f"a{VERSION_ALPHA}" if VERSION_ALPHA else "")

unblock_requests-0.0.1a2/unblock_requests.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: unblock_requests
+Version: 0.0.1a2
+Summary: A requests.Session subclass that bypasses Cloudflare via curl_cffi, FlareSolverr, or the Wayback Machine
+Requires-Python: >=3.9
+Requires-Dist: requests>=2.28
+Provides-Extra: stealth
+Requires-Dist: curl-cffi; extra == "stealth"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"

unblock_requests-0.0.1a2/unblock_requests.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+README.md
+pyproject.toml
+tests/test_session.py
+unblock_requests/__init__.py
+unblock_requests/session.py
+unblock_requests/version.py
+unblock_requests.egg-info/PKG-INFO
+unblock_requests.egg-info/SOURCES.txt
+unblock_requests.egg-info/dependency_links.txt
+unblock_requests.egg-info/requires.txt
+unblock_requests.egg-info/top_level.txt

unblock_requests-0.0.1a2/unblock_requests.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

unblock_requests-0.0.1a2/unblock_requests.egg-info/requires.txt

@@ -0,0 +1,7 @@
+requests>=2.28
+
+[stealth]
+curl-cffi
+
+[test]
+pytest

unblock_requests-0.0.1a2/unblock_requests.egg-info/top_level.txt

@@ -0,0 +1 @@
+unblock_requests