dataapp_crawler 0.1.0__py3-none-any.whl

dataapp_crawler/providers/browser.py

@@ -0,0 +1,292 @@
+"""Optional Playwright browser providers; importing core never imports Playwright."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+from ..errors import ConfigurationError
+from ..models import Identity
+from .proxy import ProxyHandle
+
+Closer = Callable[[], Awaitable[None]]
+
+
+class BrowserProvider(Protocol):
+    async def open(
+        self, *, identity: Identity | None = None, proxy: ProxyHandle | None = None
+    ) -> BrowserHandle: ...
+
+
+async def _maybe_await(value: Any) -> Any:
+    return await value if inspect.isawaitable(value) else value
+
+
+async def _best_effort_close(*closers: Callable[[], Any]) -> None:
+    for closer in closers:
+        try:
+            await asyncio.wait_for(_maybe_await(closer()), timeout=5)
+        except BaseException:
+            pass
+
+
+async def _start_playwright(factory: Callable[[], Any] | None) -> Any:
+    if factory is None:
+        try:
+            from playwright.async_api import async_playwright
+        except ImportError as error:
+            raise ConfigurationError(
+                "Install dataapp_crawler[browser] to use browser providers"
+            ) from error
+        factory = async_playwright
+    manager = factory()
+    start = getattr(manager, "start", None)
+    if not callable(start):
+        return manager
+    try:
+        return await _maybe_await(start())
+    except BaseException:
+        stop = getattr(manager, "stop", None)
+        if callable(stop):
+            await _best_effort_close(stop)
+        raise
+
+
+async def _origin_headers(context: Any, identity: Identity | None) -> None:
+    if identity is None or not identity.headers:
+        return
+
+    async def add_headers(route: Any, request: Any) -> None:
+        headers = identity.headers_for(request.url)
+        if not headers:
+            await route.continue_()
+            return
+        # Follow redirects in Chromium so each hop is routed and independently scoped.
+        response = await route.fetch(headers={**request.headers, **headers}, max_redirects=0)
+        await route.fulfill(response=response)
+
+    await context.route("**/*", add_headers)
+
+
+async def _apply_identity(context: Any, identity: Identity | None) -> None:
+    if identity is None:
+        return
+    if identity.cookies:
+        await context.add_cookies(_browser_cookies(identity.cookies))
+    await _origin_headers(context, identity)
+
+
+def _browser_cookies(cookies: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    return [{**cookie, "path": cookie.get("path") or "/"} for cookie in cookies]
+
+
+def _has_identity_state(identity: Identity | None) -> bool:
+    return identity is not None and bool(
+        identity.cookies or identity.headers or identity.storage_state is not None
+    )
+
+
+@dataclass
+class BrowserHandle:
+    provider: str
+    capabilities: frozenset[str]
+    owned: bool
+    context: Any = field(repr=False)
+    _closers: list[Closer] = field(default_factory=list, repr=False)
+    _pages: list[Any] = field(default_factory=list, repr=False)
+    _closed: bool = field(default=False, init=False, repr=False)
+    _cleanup_timeout: float = field(default=5, repr=False)
+
+    async def new_page(self) -> Any:
+        if self._closed:
+            raise RuntimeError("Browser handle is closed")
+        page = await self.context.new_page()
+        self._pages.append(page)
+        return page
+
+    async def close_page(self, page: Any) -> None:
+        """Close one acquired Page and stop retaining it after successful release."""
+        try:
+            await asyncio.wait_for(page.close(), timeout=5)
+        except Exception:
+            raise ConfigurationError("Browser page cleanup failed") from None
+        if page in self._pages:
+            self._pages.remove(page)
+
+    async def close(self) -> None:
+        if self._closed:
+            return
+        self._closed = True
+        first_error: BaseException | None = None
+        for page in list(reversed(self._pages)):
+            try:
+                await self.close_page(page)
+            except BaseException as error:
+                first_error = first_error or error
+        for closer in self._closers:
+            try:
+                await asyncio.wait_for(closer(), timeout=self._cleanup_timeout)
+            except BaseException as error:
+                first_error = first_error or error
+        if first_error is not None:
+            if not isinstance(first_error, Exception):
+                raise first_error
+            raise ConfigurationError("Browser resource cleanup failed") from None
+
+
+class CDPBrowser:
+    """Borrow a CDP browser while owning only this Playwright client and its pages."""
+
+    def __init__(
+        self,
+        endpoint_url: str,
+        *,
+        headers: dict[str, str] | None = None,
+        timeout: float = 30,
+        playwright_factory: Callable[[], Any] | None = None,
+    ) -> None:
+        self._endpoint_url = endpoint_url
+        self._headers = dict(headers or {})
+        self._timeout = timeout
+        self._playwright_factory = playwright_factory
+
+    def __repr__(self) -> str:
+        return "CDPBrowser(endpoint_url=<redacted>)"
+
+    async def open(
+        self, *, identity: Identity | None = None, proxy: ProxyHandle | None = None
+    ) -> BrowserHandle:
+        if _has_identity_state(identity) or proxy is not None:
+            raise ConfigurationError(
+                "Borrowed CDP browsers cannot safely apply identity or proxy configuration"
+            )
+        playwright = await _start_playwright(self._playwright_factory)
+        try:
+            browser = await playwright.chromium.connect_over_cdp(
+                self._endpoint_url,
+                headers=self._headers or None,
+                timeout=self._timeout * 1000,
+            )
+            if not browser.contexts:
+                raise ConfigurationError("CDP browser exposed no browser context")
+        except BaseException as error:
+            await _best_effort_close(playwright.stop)
+            if not isinstance(error, Exception):
+                raise
+            if isinstance(error, ConfigurationError):
+                raise
+            raise ConfigurationError(f"CDP connection failed ({type(error).__name__})") from None
+
+        async def stop_client() -> None:
+            # Stopping this Playwright client disconnects without stopping the remote browser.
+            await playwright.stop()
+
+        return BrowserHandle(
+            provider="cdp",
+            capabilities=frozenset({"page", "cdp", "borrowed"}),
+            owned=False,
+            context=browser.contexts[0],
+            _closers=[stop_client],
+        )
+
+
+class PlaywrightBrowser:
+    """Own a local Playwright browser and isolated context."""
+
+    def __init__(
+        self,
+        *,
+        browser: str = "chromium",
+        headless: bool = True,
+        executable_path: str | None = None,
+        launch_options: dict[str, Any] | None = None,
+        context_options: dict[str, Any] | None = None,
+        playwright_factory: Callable[[], Any] | None = None,
+    ) -> None:
+        self._browser = browser
+        self._headless = headless
+        self._executable_path = executable_path
+        self._launch_options = dict(launch_options or {})
+        self._context_options = dict(context_options or {})
+        self._playwright_factory = playwright_factory
+
+    async def _launch(self) -> tuple[Any, Any]:
+        """Launch one native browser process for this provider's configuration."""
+        playwright = await _start_playwright(self._playwright_factory)
+        try:
+            browser_type = getattr(playwright, self._browser)
+        except AttributeError as error:
+            await _best_effort_close(playwright.stop)
+            raise ConfigurationError(f"Unknown Playwright browser: {self._browser}") from error
+        launch = {"headless": self._headless, **self._launch_options}
+        if self._executable_path:
+            launch["executable_path"] = self._executable_path
+        try:
+            browser = await browser_type.launch(**launch)
+            return playwright, browser
+        except BaseException as error:
+            await _best_effort_close(playwright.stop)
+            if not isinstance(error, Exception):
+                raise
+            raise ConfigurationError(
+                f"Playwright browser startup failed ({type(error).__name__})"
+            ) from None
+
+    async def _open_context(
+        self,
+        browser: Any,
+        *,
+        identity: Identity | None = None,
+        proxy: ProxyHandle | None = None,
+    ) -> Any:
+        """Create one isolated context in an already launched native browser."""
+        try:
+            context_options = dict(self._context_options)
+            if proxy is not None:
+                context_options["proxy"] = proxy.as_playwright()
+            if identity is not None and identity.storage_state is not None:
+                context_options["storage_state"] = identity.storage_state
+            if identity is not None and identity.headers:
+                # Context routing does not see requests claimed by a Service Worker.
+                context_options["service_workers"] = "block"
+            context = await browser.new_context(**context_options)
+            await _apply_identity(context, identity)
+        except BaseException as error:
+            if "context" in locals():
+                await _best_effort_close(context.close)
+            if not isinstance(error, Exception):
+                raise
+            raise ConfigurationError(
+                f"Playwright browser context startup failed ({type(error).__name__})"
+            ) from None
+        return context
+
+    async def open(
+        self, *, identity: Identity | None = None, proxy: ProxyHandle | None = None
+    ) -> BrowserHandle:
+        playwright, browser = await self._launch()
+        try:
+            context = await self._open_context(browser, identity=identity, proxy=proxy)
+        except BaseException:
+            await _best_effort_close(browser.close, playwright.stop)
+            raise
+
+        async def close_context() -> None:
+            await context.close()
+
+        async def close_browser() -> None:
+            await browser.close()
+
+        async def stop_playwright() -> None:
+            await playwright.stop()
+
+        return BrowserHandle(
+            provider="playwright",
+            capabilities=frozenset({"page", "cookies", "origin_headers", "proxy", "storage_state"}),
+            owned=True,
+            context=context,
+            _closers=[close_context, close_browser, stop_playwright],
+        )

dataapp_crawler/providers/camoufox.py

@@ -0,0 +1,184 @@
+"""Optional Camoufox provider with one coherent identity per owned context."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Callable, Mapping
+from pathlib import Path
+from typing import Any, Literal
+
+from ..errors import ConfigurationError
+from ..models import Identity
+from .browser import BrowserHandle, _apply_identity, _best_effort_close
+from .proxy import ProxyHandle
+
+CamoufoxFactory = Callable[..., Any]
+ContextFactory = Callable[..., Any]
+CamoufoxHeadless = bool | Literal["virtual"]
+
+_IDENTITY_CONTEXT_OPTIONS = frozenset(
+    {
+        "device_scale_factor",
+        "extra_http_headers",
+        "ff_version",
+        "geolocation",
+        "has_touch",
+        "http_credentials",
+        "is_mobile",
+        "locale",
+        "permissions",
+        "preset",
+        "proxy",
+        "os",
+        "screen",
+        "storage_state",
+        "timezone_id",
+        "user_agent",
+        "viewport",
+        "webrtc_ip",
+    }
+)
+
+
+class CamoufoxBrowser:
+    """Own one Camoufox browser and one independently fingerprinted context.
+
+    A provider instance is safe to use as one Crawler or SessionPool slot
+    generation.  Camoufox draws the context identity during :meth:`open` and the
+    returned handle keeps that identity, proxy and Cookie state together until
+    it is closed.
+    """
+
+    def __init__(
+        self,
+        *,
+        headless: CamoufoxHeadless = True,
+        os: Literal["windows", "macos", "linux"] | None = None,
+        humanize: bool | float = False,
+        executable_path: str | Path | None = None,
+        context_options: Mapping[str, Any] | None = None,
+        camoufox_factory: CamoufoxFactory | None = None,
+        context_factory: ContextFactory | None = None,
+    ) -> None:
+        if not (
+            type(headless) is bool or isinstance(headless, str) and headless == "virtual"
+        ):
+            raise ConfigurationError("Camoufox headless must be true, false or 'virtual'")
+        if os not in {None, "windows", "macos", "linux"}:
+            raise ConfigurationError("Unknown Camoufox operating-system identity")
+        if not isinstance(humanize, bool) and (
+            not isinstance(humanize, (int, float))
+            or not math.isfinite(humanize)
+            or not 0 < humanize <= 5
+        ):
+            raise ConfigurationError("Camoufox humanize must be boolean or 0 < seconds <= 5")
+        if (camoufox_factory is None) != (context_factory is None):
+            raise ConfigurationError("Camoufox test factories must be configured together")
+        options = dict(context_options or {})
+        reserved = sorted(_IDENTITY_CONTEXT_OPTIONS.intersection(options))
+        if reserved:
+            raise ConfigurationError(
+                "Camoufox context identity options are managed by the Provider: "
+                + ", ".join(reserved)
+            )
+        self._headless = headless
+        self._os = os
+        self._humanize = humanize
+        self._executable_path = (
+            str(Path(executable_path).expanduser()) if executable_path is not None else None
+        )
+        self._context_options = options
+        self._camoufox_factory = camoufox_factory
+        self._context_factory = context_factory
+
+    def __repr__(self) -> str:
+        return f"CamoufoxBrowser(headless={self._headless!r}, os={self._os!r})"
+
+    @staticmethod
+    def _optional_api() -> tuple[CamoufoxFactory, ContextFactory]:
+        try:
+            from camoufox.async_api import AsyncCamoufox, AsyncNewContext
+        except ImportError as error:
+            raise ConfigurationError(
+                "Install dataapp_crawler[fingerprint] in its isolated environment "
+                "and fetch the paired Camoufox browser build"
+            ) from error
+        return AsyncCamoufox, AsyncNewContext
+
+    async def open(
+        self, *, identity: Identity | None = None, proxy: ProxyHandle | None = None
+    ) -> BrowserHandle:
+        camoufox_factory = self._camoufox_factory
+        context_factory = self._context_factory
+        if camoufox_factory is None or context_factory is None:
+            camoufox_factory, context_factory = self._optional_api()
+
+        launch_options: dict[str, Any] = {
+            "headless": self._headless,
+            "humanize": self._humanize,
+        }
+        if self._os is not None:
+            # Browser-level OS behavior and the context fingerprint must agree.
+            launch_options["os"] = self._os
+        if self._executable_path is not None:
+            launch_options["executable_path"] = self._executable_path
+
+        manager = camoufox_factory(**launch_options)
+        browser: Any = None
+        try:
+            browser = await manager.__aenter__()
+            context_options = dict(self._context_options)
+            if self._os is not None:
+                context_options["os"] = self._os
+            if proxy is not None:
+                # AsyncNewContext derives WebRTC IP and timezone from this exit.
+                context_options["proxy"] = proxy.as_playwright()
+            if identity is not None and identity.storage_state is not None:
+                context_options["storage_state"] = identity.storage_state
+            if identity is not None and identity.headers:
+                # Origin routing cannot scope requests claimed by a Service Worker.
+                context_options["service_workers"] = "block"
+            context = await context_factory(browser, **context_options)
+            await _apply_identity(context, identity)
+        except BaseException as error:
+            if "context" in locals():
+                await _best_effort_close(context.close)
+            if browser is not None:
+                failure = error
+                error_type = type(error)
+                traceback = error.__traceback__
+
+                async def close_failed_manager() -> None:
+                    await manager.__aexit__(error_type, failure, traceback)
+
+                await _best_effort_close(close_failed_manager)
+            if not isinstance(error, Exception):
+                raise
+            if isinstance(error, ConfigurationError):
+                raise
+            raise ConfigurationError(
+                f"Camoufox browser startup failed ({type(error).__name__})"
+            ) from None
+
+        async def close_context() -> None:
+            await context.close()
+
+        async def close_manager() -> None:
+            await manager.__aexit__(None, None, None)
+
+        return BrowserHandle(
+            provider="camoufox",
+            capabilities=frozenset(
+                {
+                    "page",
+                    "cookies",
+                    "origin_headers",
+                    "proxy",
+                    "storage_state",
+                    "fingerprint",
+                }
+            ),
+            owned=True,
+            context=context,
+            _closers=[close_context, close_manager],
+        )

dataapp_crawler/providers/dataos.py

@@ -0,0 +1,174 @@
+"""DataOS-managed browser and proxy providers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from ..errors import ConfigurationError
+from .proxy import ProxyHandle
+
+_SAFE_RESOURCE_ERROR_CODES = frozenset({
+    "configuration-error",
+    "insufficient-developer-resource-balance",
+    "managed-billing-conflict",
+    "managed-billing-not-found",
+    "managed-billing-request-rejected",
+    "managed-billing-unavailable",
+    "provider-connection-unavailable",
+    "provider-invalid-response",
+    "provider-outcome-unknown",
+    "provider-rejected",
+    "proxy-group-unavailable",
+    "proxy-group-network-conflict",
+    "proxy-location-unavailable",
+    "proxy-location-unsupported",
+    "resource-outcome-unavailable",
+    "resource-proxy-node-unavailable",
+    "resource-proxy-unavailable",
+    "resource-request-in-progress",
+    "resource-request-reserved",
+    "resource-revocation-pending",
+    "transport-error",
+})
+
+
+class _ManagedProxyConfigurationError(ConfigurationError):
+    def __init__(self, code: str, status_code: int | None) -> None:
+        self.code = code
+        self.status_code = status_code
+        detail = f"code={code}" + (
+            f", status={status_code}" if status_code is not None else ""
+        )
+        super().__init__(f"DataOS proxy acquisition failed ({detail})")
+
+
+def _configuration_error(error: Exception) -> ConfigurationError:
+    raw_code = getattr(error, "code", None)
+    code = (
+        raw_code
+        if isinstance(raw_code, str) and raw_code in _SAFE_RESOURCE_ERROR_CODES
+        else "resource-service-error"
+    )
+    raw_status = getattr(error, "status_code", None)
+    status = (
+        raw_status
+        if (
+            isinstance(raw_status, int)
+            and not isinstance(raw_status, bool)
+            and 400 <= raw_status <= 599
+        )
+        else None
+    )
+    # Keep only the protocol's bounded diagnostic labels. Never retain the
+    # upstream message, URL, response body, or supplier credentials.
+    return _ManagedProxyConfigurationError(code, status)
+
+
+class _ProxyResource(Protocol):
+    lease_id: str
+    proxy_url: str
+
+    async def renew(self) -> None: ...
+
+    async def close(self) -> None: ...
+
+
+class ResourceClient(Protocol):
+    async def proxy(self, **kwargs: Any) -> _ProxyResource: ...
+
+
+@dataclass(frozen=True)
+class DataOSProxy:
+    """Acquire a DataOS-managed proxy lease for one crawler generation.
+
+    The first service version cannot authoritatively replace the supplier
+    session or exit IP, so Crawler exit rotation is deliberately unsupported.
+    """
+
+    resources: ResourceClient
+    country: str | None = None
+    expire_time: str | None = None
+    exit_policy: str = "renewable"
+    max_rotations: int = 0
+    network_preference: str | None = None
+    target_hint: str | None = None
+    state: str | None = None
+    city: str | None = None
+    proxy_group: str | None = None
+
+    def __post_init__(self) -> None:
+        if self.network_preference not in {None, "auto", "domestic", "overseas"}:
+            raise ConfigurationError(
+                "network_preference must be auto, domestic or overseas"
+            )
+        if self.proxy_group not in {
+            None,
+            "auto",
+            "domestic_basic",
+            "overseas_basic",
+            "overseas_enhanced",
+        }:
+            raise ConfigurationError(
+                "proxy_group must be auto, domestic_basic, overseas_basic, "
+                "or overseas_enhanced"
+            )
+        group_network = self._group_network()
+        if (
+            group_network is not None
+            and self.network_preference not in {None, "auto", group_network}
+        ):
+            raise ConfigurationError("proxy_group conflicts with network_preference")
+        if self.max_rotations != 0:
+            raise ConfigurationError(
+                "DataOSProxy does not support exit rotation; max_rotations must be 0"
+            )
+
+    def __repr__(self) -> str:
+        return (
+            "DataOSProxy("
+            f"country={self.country!r}, network_preference={self.network_preference!r}, "
+            f"proxy_group={self.proxy_group!r}, exit_policy={self.exit_policy!r})"
+        )
+
+    def _group_network(self) -> str | None:
+        if self.proxy_group == "domestic_basic":
+            return "domestic"
+        if self.proxy_group in {"overseas_basic", "overseas_enhanced"}:
+            return "overseas"
+        return None
+
+    async def acquire(self) -> ProxyHandle:
+        try:
+            options: dict[str, Any] = {
+                "country": self.country,
+                "expire_time": self.expire_time,
+                "exit_policy": self.exit_policy,
+            }
+            group_network = self._group_network()
+            selected_network = group_network or self.network_preference
+            if selected_network not in {None, "auto"}:
+                options["network_preference"] = selected_network
+            if self.proxy_group is not None:
+                options["proxy_group"] = self.proxy_group
+            if self.target_hint is not None:
+                options["target_hint"] = self.target_hint
+            if self.state is not None:
+                options["state"] = self.state
+            if self.city is not None:
+                options["city"] = self.city
+            resource = await self.resources.proxy(**options)
+        except Exception as error:
+            raise _configuration_error(error) from None
+
+        async def release() -> str:
+            await resource.close()
+            return "released"
+
+        return ProxyHandle(
+            provider="dataos",
+            _url=resource.proxy_url,
+            tunnel_id=resource.lease_id,
+            max_rotations=0,
+            _release=release,
+        )