anti-cf 1.0.1__py3-none-any.whl

anti_cf/__init__.py

@@ -0,0 +1,5 @@
+from ._persistent_session import session
+
+__all__ = [
+    "session",
+]

anti_cf/_constants.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Final
+
+CACHE_PATH = Path.home() / ".cache/anti_cf"
+FLARESOLVERR_PROXY: Final[str] = "http://localhost:8191/"
+CACHE_PATH.mkdir(exist_ok=True, parents=True)
+DEFAULT_TIMEOUT: int = 600

anti_cf/_flaresolverr.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import subprocess
+import time
+
+import requests
+from logprise import logger
+
+from ._constants import FLARESOLVERR_PROXY
+
+
+def get_flaresolverr_settings() -> dict | None:
+    """Check if FlareSolverr API is reachable."""
+    try:
+        resp = requests.get(FLARESOLVERR_PROXY, timeout=0.1)
+        resp.raise_for_status()
+        return resp.json()
+    except:  # noqa: E722
+        return None
+
+
+def start_flaresolverr_docker() -> subprocess.Popen | None:
+    """Start the FlareSolverr docker container."""
+    try:
+        logger.info("Starting FlareSolverr docker container...")
+        process = subprocess.Popen(
+            ["docker", "run", "--rm", "-p", "8191:8191", "ghcr.io/svaningelgem/flaresolverr:latest"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        # Wait for container to be ready
+        for loop in range(10):  # Try for 10 seconds
+            if loop > 0:
+                time.sleep(1)
+
+            if get_flaresolverr_settings() is not None:
+                logger.info("FlareSolverr is ready")
+                return process
+
+        logger.error("FlareSolverr container started but API not responding")
+        return process
+    except Exception as e:
+        logger.error(f"Failed to start FlareSolverr docker: {e}")
+        return None
+
+
+def ensure_flaresolverr_running() -> subprocess.Popen | None:
+    """Ensure FlareSolverr is running, start if needed."""
+    if get_flaresolverr_settings() is not None:
+        logger.info("FlareSolverr API is already running")
+        return None
+
+    return start_flaresolverr_docker()

anti_cf/_persistent_session.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+import pickle
+import tempfile
+from typing import TYPE_CHECKING, ClassVar
+
+import fake_useragent
+from logprise import logger
+
+from ._constants import CACHE_PATH, DEFAULT_TIMEOUT, FLARESOLVERR_PROXY
+from ._flaresolverr import ensure_flaresolverr_running, get_flaresolverr_settings
+
+try:
+    from requests_cache import CachedSession as Session
+
+    _HAS_CACHE = True
+    logger.info("Using CachedSession for persistent session")
+except ImportError:
+    from requests import HTTPError, Session
+
+    _HAS_CACHE = False
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from requests import Response
+
+
+class PersistentSession(Session):
+    _COOKIES_FILE: ClassVar[Path] = CACHE_PATH / "cookies.pkl"
+    _USER_AGENT_FILE: ClassVar[Path] = CACHE_PATH / "user_agent.txt"
+
+    def __init__(self) -> None:
+        if _HAS_CACHE:
+            super().__init__(
+                CACHE_PATH / "url_cache.sqlite",
+                backend="sqlite",
+                cache_control=False,
+                expire_after=2 * 3600,
+                headers={
+                    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                    "Accept-Language": "en-US,en;q=0.5",
+                },
+            )
+        else:
+            super().__init__()
+
+        self._load_cookies()
+        ensure_flaresolverr_running()
+        self.set_user_agent()
+
+    def _get_user_agent(self) -> str:
+        flaresolverr_settings = get_flaresolverr_settings()
+        if flaresolverr_settings is not None:
+            return flaresolverr_settings["userAgent"]
+
+        if self._USER_AGENT_FILE.exists():
+            return self._USER_AGENT_FILE.read_text(encoding="utf8").strip()
+
+        return fake_useragent.UserAgent(os="windows", platforms="pc", browsers="chrome").random
+
+    def set_user_agent(self, user_agent: str | None = None) -> None:
+        if user_agent is None:
+            user_agent = self._get_user_agent()
+
+        self.headers["User-Agent"] = user_agent
+        self._USER_AGENT_FILE.write_text(user_agent, encoding="utf8")
+
+    def _load_cookies(self) -> None:
+        """Load cookies from file if it exists."""
+        if self._COOKIES_FILE.exists():
+            try:
+                with self._COOKIES_FILE.open("rb") as fp:
+                    self.cookies.update(pickle.load(fp))
+            except Exception as e:
+                logger.error(f"Failed to load cookies from {self._COOKIES_FILE}: {e}")
+                self._COOKIES_FILE.unlink()
+
+    def save_cookies(self) -> None:
+        """Save current cookies to file."""
+        self._COOKIES_FILE.write_bytes(pickle.dumps(self.cookies, protocol=4))
+
+    def request(self, *args: object, **kwargs: object) -> Response:
+        """Override request method to save cookies after each request."""
+        response = super().request(*args, **kwargs)
+        self.save_cookies()
+        return response
+
+    def get(self, url: str | bytes, *, try_with_cloudflare: bool = False, _cloudflare_counter: int = 0, **kwargs: object) -> Response | None:
+        if not try_with_cloudflare or "cf_clearance" in self.cookies:
+            try:
+                resp = super().get(url, **kwargs)
+                resp.raise_for_status()
+                return resp
+            except HTTPError as e:
+                if b"just a moment" not in e.response.content.lower():
+                    logger.error("No cloudflare trigger in response?")
+                    with tempfile.NamedTemporaryFile(delete=False) as f:
+                        f.write(e.response.content)
+                        logger.error(f"No cloudflare trigger in response? [exception: {e}] [content: {f.name}]")
+                    logger.exception(e)
+                    return None
+
+                if try_with_cloudflare:
+                    logger.warning("Cloudflare cookie expired")
+                else:
+                    logger.warning("Cloudflare detected, but `try_with_cloudflare` wasn't set to True!")
+
+        try:
+            self._get_url_via_flaresolverr(url)
+            # After the url is retrieved from the flaresolverr proxy, it's not necessarily the one we want
+            #  --> So we'll re-request it here:
+            return super().get(url, **kwargs)
+        except Exception:
+            logger.error("FlareSolverr didn't solve it :(")
+            raise
+
+    def _get_url_via_flaresolverr(self, url: str) -> dict:
+        headers = {"Content-Type": "application/json"}
+        data = {
+            "cmd": "request.get",
+            "url": url,
+            "maxTimeout": DEFAULT_TIMEOUT * 1_000,
+        }
+        response = self.post(FLARESOLVERR_PROXY + "v1", headers=headers, json=data, timeout=DEFAULT_TIMEOUT)
+        response.raise_for_status()
+
+        dta = response.json()
+        for cookie in dta["solution"]["cookies"]:
+            self.cookies.set(
+                name=cookie["name"],  # required
+                value=cookie["value"],  # required
+                version=cookie.get("version", 0),
+                port=cookie.get("port", None),
+                domain=cookie.get("domain", ""),
+                path=cookie.get("path", "/"),
+                secure=cookie.get("secure", False),
+                expires=cookie.get("expires", None),
+                discard=cookie.get("discard", True),
+                comment=cookie.get("comment", None),
+                comment_url=cookie.get("comment_url", None),
+                rest=cookie.get("rest", {"HttpOnly": None}),
+                rfc2109=cookie.get("rfc2109", False),
+            )
+        self.save_cookies()
+
+        return dta
+
+
+session = PersistentSession()

anti_cf-1.0.1.dist-info/METADATA

@@ -0,0 +1,123 @@
+Metadata-Version: 2.3
+Name: anti_cf
+Version: 1.0.1
+Summary: Anti-CloudFlare package
+License: MIT
+Author: Steven Van Ingelgem
+Author-email: steven@vaningelgem.be
+Requires-Python: >=3.11,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: fake-useragent
+Requires-Dist: logprise
+Requires-Dist: requests
+Description-Content-Type: text/markdown
+
+# anti-cf
+
+A Python library for handling Cloudflare-protected websites using FlareSolverr.
+
+## Overview
+
+`anti-cf` provides a persistent session wrapper for handling websites protected by Cloudflare's anti-bot measures. It automatically manages cookies, user agents, and integrates with FlareSolverr to bypass Cloudflare challenges.
+
+## Features
+
+- Persistent cookie storage
+- Automatic FlareSolverr management (including Docker startup)
+- Optional request caching via `requests-cache`
+- Random user agent generation
+- Transparent handling of Cloudflare challenges
+
+## Installation
+
+```bash
+pip install anti-cf
+```
+
+## Usage
+
+### Basic Usage
+
+```python
+from anti_cf import session
+
+# The library will automatically check if FlareSolverr is running
+# and start it if needed using Docker
+
+# For Cloudflare-protected sites
+response = session.get("https://cloudflare-protected-site.com", try_with_cloudflare=True)
+
+# For regular requests
+response = session.get("https://example.com")
+```
+
+### Advanced Usage
+
+```python
+from anti_cf import session
+
+# Set a custom user agent
+session.set_user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
+
+# Post requests work as normal
+response = session.post("https://example.com/api", json={"key": "value"})
+
+# All cookies are automatically saved between requests
+```
+
+### Error Handling
+
+```python
+from anti_cf import session
+from requests import HTTPError
+
+try:
+    response = session.get("https://cloudflare-protected-site.com", try_with_cloudflare=True)
+    response.raise_for_status()
+except HTTPError as e:
+    print(f"HTTP error occurred: {e}")
+```
+
+## Dependencies
+
+- Python 3.11+
+- FlareSolverr
+- Docker (optional, for automatic FlareSolverr startup)
+- `requests` or `requests-cache` (optional for caching)
+- `fake-useragent`
+- `logprise`
+
+## Configuration
+
+The library uses the following default settings:
+- Cache directory: `~/.cache/anti_cf/`
+- FlareSolverr API: `http://localhost:8191/`
+- Default timeout: 600 seconds
+- Cache expiry: 2 hours (when using `requests-cache`)
+
+## How It Works
+
+1. When making a request to a Cloudflare-protected site:
+    - First attempts a normal request
+    - If Cloudflare challenge detected, sends the request through FlareSolverr
+    - Stores the resulting cookies for future requests
+
+2. On startup:
+    - Checks if FlareSolverr API is reachable
+    - If not available, automatically starts the Docker container
+
+## Docker
+
+By default, `anti-cf` will attempt to start the FlareSolverr Docker container:
+
+```
+ghcr.io/svaningelgem/flaresolverr:latest
+```
+
+## License
+
+Copyright © Steven Van Ingelgem <steven@vaningelgem.be>

anti_cf-1.0.1.dist-info/RECORD

@@ -0,0 +1,7 @@
+anti_cf/__init__.py,sha256=ZFXXzfWeiBqKC56vdyod1yXj5_Kg4NXJeTejs2WoEEU,71
+anti_cf/_constants.py,sha256=xFIXkeBVkuHsTQKQU4p2leJkySHdJ_6xApgIoVrBWHA,262
+anti_cf/_flaresolverr.py,sha256=vOuut3fH1CstumQrNyIhm7B58ybLCSPUaK5NDTiQwj8,1633
+anti_cf/_persistent_session.py,sha256=exxEpwqfRXonezFh3BEu4JclZ1lTGShM4l0fSlOIeeI,5634
+anti_cf-1.0.1.dist-info/METADATA,sha256=9exeaEIAuWMlnuLceXcoo8A9dftwHKDh6NLZ5UIWZgk,3187
+anti_cf-1.0.1.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+anti_cf-1.0.1.dist-info/RECORD,,

anti_cf-1.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.1.2
+Root-Is-Purelib: true
+Tag: py3-none-any