stealthfetch 0.2.0__py3-none-any.whl

stealthfetch/__init__.py

@@ -0,0 +1,32 @@
+"""StealthFetch — URL in, LLM-ready markdown out."""
+
+from importlib.metadata import version
+
+from stealthfetch._core import (
+    FetchResult,
+    afetch_markdown,
+    afetch_result,
+    fetch_markdown,
+    fetch_result,
+)
+from stealthfetch._errors import (
+    BrowserNotAvailable,
+    ExtractionError,
+    FetchError,
+    StealthFetchError,
+)
+
+__version__ = version("stealthfetch")
+
+__all__ = [
+    "BrowserNotAvailable",
+    "ExtractionError",
+    "FetchError",
+    "FetchResult",
+    "StealthFetchError",
+    "__version__",
+    "afetch_markdown",
+    "afetch_result",
+    "fetch_markdown",
+    "fetch_result",
+]

stealthfetch/_browsers/__init__.py

@@ -0,0 +1,75 @@
+"""Browser backend dispatcher."""
+
+from __future__ import annotations
+
+from stealthfetch._compat import get_default_backend, require_browser
+
+
+def fetch_browser(
+    url: str,
+    *,
+    backend: str = "auto",
+    timeout: int = 30,
+    proxy: dict[str, str] | None = None,
+    headers: dict[str, str] | None = None,
+) -> str:
+    """Fetch a URL using a stealth browser (sync).
+
+    Args:
+        url: The URL to fetch.
+        backend: "auto", "camoufox", or "patchright".
+        timeout: Timeout in seconds.
+        proxy: Proxy config dict with "server", optional "username"/"password".
+        headers: Additional HTTP headers to send with the request.
+
+    Returns:
+        Rendered HTML string.
+    """
+    name = _resolve_backend(backend)
+    if name == "camoufox":
+        from stealthfetch._browsers._camoufox import fetch as _cfetch
+
+        return _cfetch(url, timeout=timeout, proxy=proxy, headers=headers)
+    else:
+        from stealthfetch._browsers._patchright import fetch as _pfetch
+
+        return _pfetch(url, timeout=timeout, proxy=proxy, headers=headers)
+
+
+async def afetch_browser(
+    url: str,
+    *,
+    backend: str = "auto",
+    timeout: int = 30,
+    proxy: dict[str, str] | None = None,
+    headers: dict[str, str] | None = None,
+) -> str:
+    """Fetch a URL using a stealth browser (async).
+
+    Args:
+        url: The URL to fetch.
+        backend: "auto", "camoufox", or "patchright".
+        timeout: Timeout in seconds.
+        proxy: Proxy config dict with "server", optional "username"/"password".
+        headers: Additional HTTP headers to send with the request.
+
+    Returns:
+        Rendered HTML string.
+    """
+    name = _resolve_backend(backend)
+    if name == "camoufox":
+        from stealthfetch._browsers._camoufox import afetch as _cafetch
+
+        return await _cafetch(url, timeout=timeout, proxy=proxy, headers=headers)
+    else:
+        from stealthfetch._browsers._patchright import afetch as _pafetch
+
+        return await _pafetch(url, timeout=timeout, proxy=proxy, headers=headers)
+
+
+def _resolve_backend(backend: str) -> str:
+    """Resolve 'auto' to a concrete backend name and validate availability."""
+    if backend == "auto":
+        return get_default_backend()
+    require_browser(backend)
+    return backend

stealthfetch/_browsers/_camoufox.py

@@ -0,0 +1,73 @@
+"""Camoufox browser backend — sync and async."""
+
+from __future__ import annotations
+
+import logging
+
+from stealthfetch._browsers._constants import BODY_READY_JS, BODY_READY_TIMEOUT, build_proxy
+
+logger = logging.getLogger(__name__)
+
+
+def fetch(
+    url: str,
+    *,
+    timeout: int = 30,
+    proxy: dict[str, str] | None = None,
+    headers: dict[str, str] | None = None,
+) -> str:
+    """Fetch a URL with Camoufox stealth browser (sync)."""
+    from camoufox.sync_api import Camoufox
+
+    logger.debug("Camoufox sync fetch: %s", url)
+    camoufox_proxy = build_proxy(proxy)
+
+    with Camoufox(  # type: ignore[no-untyped-call]
+        headless=True,
+        proxy=camoufox_proxy,
+        geoip=bool(proxy),
+        block_images=True,
+        block_webrtc=True,
+    ) as browser:
+        page = browser.new_page()
+        page.set_default_timeout(timeout * 1000)
+        if headers:
+            page.set_extra_http_headers(headers)
+        page.goto(url, wait_until="domcontentloaded")
+        try:
+            page.wait_for_function(BODY_READY_JS, timeout=BODY_READY_TIMEOUT)
+        except Exception:
+            logger.debug("Body readiness check timed out, continuing with current content")
+        return str(page.content())
+
+
+async def afetch(
+    url: str,
+    *,
+    timeout: int = 30,
+    proxy: dict[str, str] | None = None,
+    headers: dict[str, str] | None = None,
+) -> str:
+    """Fetch a URL with Camoufox stealth browser (async)."""
+    from camoufox.async_api import AsyncCamoufox
+
+    logger.debug("Camoufox async fetch: %s", url)
+    camoufox_proxy = build_proxy(proxy)
+
+    async with AsyncCamoufox(  # type: ignore[no-untyped-call]
+        headless=True,
+        proxy=camoufox_proxy,
+        geoip=bool(proxy),
+        block_images=True,
+        block_webrtc=True,
+    ) as browser:
+        page = await browser.new_page()
+        page.set_default_timeout(timeout * 1000)
+        if headers:
+            await page.set_extra_http_headers(headers)
+        await page.goto(url, wait_until="domcontentloaded")
+        try:
+            await page.wait_for_function(BODY_READY_JS, timeout=BODY_READY_TIMEOUT)
+        except Exception:
+            logger.debug("Body readiness check timed out, continuing with current content")
+        return str(await page.content())

stealthfetch/_browsers/_constants.py

@@ -0,0 +1,16 @@
+"""Shared constants and utilities for browser backends."""
+
+BODY_READY_JS = "document.body && document.body.innerText.trim().length > 100"
+BODY_READY_TIMEOUT = 10_000  # ms
+
+
+def build_proxy(proxy: dict[str, str] | None) -> dict[str, str] | None:
+    """Convert a stealthfetch proxy dict to a Playwright-compatible proxy dict."""
+    if not proxy:
+        return None
+    result: dict[str, str] = {"server": proxy["server"]}
+    if "username" in proxy:
+        result["username"] = proxy["username"]
+    if "password" in proxy:
+        result["password"] = proxy["password"]
+    return result

stealthfetch/_browsers/_patchright.py

@@ -0,0 +1,67 @@
+"""Patchright browser backend — sync and async."""
+
+from __future__ import annotations
+
+import logging
+
+from stealthfetch._browsers._constants import BODY_READY_JS, BODY_READY_TIMEOUT, build_proxy
+
+logger = logging.getLogger(__name__)
+
+
+def fetch(
+    url: str,
+    *,
+    timeout: int = 30,
+    proxy: dict[str, str] | None = None,
+    headers: dict[str, str] | None = None,
+) -> str:
+    """Fetch a URL with Patchright stealth browser (sync)."""
+    from patchright.sync_api import sync_playwright
+
+    logger.debug("Patchright sync fetch: %s", url)
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True, proxy=build_proxy(proxy))  # type: ignore[arg-type]
+        try:
+            page = browser.new_page()
+            page.set_default_timeout(timeout * 1000)
+            if headers:
+                page.set_extra_http_headers(headers)
+            page.goto(url, wait_until="domcontentloaded")
+            try:
+                page.wait_for_function(BODY_READY_JS, timeout=BODY_READY_TIMEOUT)
+            except Exception:
+                logger.debug("Body readiness check timed out, continuing with current content")
+            return str(page.content())
+        finally:
+            browser.close()
+
+
+async def afetch(
+    url: str,
+    *,
+    timeout: int = 30,
+    proxy: dict[str, str] | None = None,
+    headers: dict[str, str] | None = None,
+) -> str:
+    """Fetch a URL with Patchright stealth browser (async)."""
+    from patchright.async_api import async_playwright
+
+    logger.debug("Patchright async fetch: %s", url)
+
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True, proxy=build_proxy(proxy))  # type: ignore[arg-type]
+        try:
+            page = await browser.new_page()
+            page.set_default_timeout(timeout * 1000)
+            if headers:
+                await page.set_extra_http_headers(headers)
+            await page.goto(url, wait_until="domcontentloaded")
+            try:
+                await page.wait_for_function(BODY_READY_JS, timeout=BODY_READY_TIMEOUT)
+            except Exception:
+                logger.debug("Body readiness check timed out, continuing with current content")
+            return str(await page.content())
+        finally:
+            await browser.close()

stealthfetch/_compat.py

@@ -0,0 +1,50 @@
+"""Lazy imports and feature detection for optional browser backends."""
+
+from __future__ import annotations
+
+from stealthfetch._errors import BrowserNotAvailable
+
+# Not using @functools.cache — these check on every call so that
+# installing a browser backend mid-process (e.g., in a long-running
+# MCP server) is detected without restart.
+
+
+def has_camoufox() -> bool:
+    """Check if camoufox is importable."""
+    try:
+        import camoufox.sync_api  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
+def has_patchright() -> bool:
+    """Check if patchright is importable."""
+    try:
+        import patchright.sync_api  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
+def require_browser(backend: str) -> None:
+    """Raise BrowserNotAvailable if the requested backend is missing."""
+    if backend == "camoufox" and not has_camoufox():
+        raise BrowserNotAvailable("camoufox")
+    if backend == "patchright" and not has_patchright():
+        raise BrowserNotAvailable("patchright")
+
+
+def get_default_backend() -> str:
+    """Return the best available browser backend name.
+
+    Raises:
+        BrowserNotAvailable: If no browser backend is installed.
+    """
+    if has_camoufox():
+        return "camoufox"
+    if has_patchright():
+        return "patchright"
+    raise BrowserNotAvailable("browser")