dataapp-crawler-octoparse-browser 0.1.0__py3-none-any.whl

dataapp_crawler_octoparse/__init__.py

@@ -0,0 +1,13 @@
+"""Octoparse browser provider extension for DataApp Crawler."""
+
+from .browserenv import BrowserEnvRelay
+from .captcha import CaptchaType, ManagedCaptchaPlugin, PreparedCaptchaExtension
+from .relay import RelayBrowser
+
+__all__ = [
+    "BrowserEnvRelay",
+    "CaptchaType",
+    "ManagedCaptchaPlugin",
+    "PreparedCaptchaExtension",
+    "RelayBrowser",
+]

dataapp_crawler_octoparse/browserenv.py

@@ -0,0 +1,439 @@
+"""Connect to BrowserEnv's browser or start its managed-plugin mode."""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import ipaddress
+import json
+import math
+import os
+import tempfile
+import threading
+from collections.abc import Callable, Sequence
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any, Protocol
+from urllib.parse import unquote, urlsplit
+
+import httpx
+from dataapp_crawler.errors import ConfigurationError
+from dataapp_crawler.models import Identity
+from dataapp_crawler.providers.browser import (
+    BrowserHandle,
+    _best_effort_close,
+    _maybe_await,
+    _start_playwright,
+)
+from dataapp_crawler.providers.proxy import ProxyHandle
+
+from .captcha import ManagedCaptchaPlugin, PreparedCaptchaExtension
+
+
+class _WafTask(Protocol):
+    @property
+    def indices(self) -> tuple[int, ...] | None: ...
+
+
+class _WafResources(Protocol):
+    async def solve_captcha(
+        self, *, type: str, target: str, images: Sequence[str],
+        idempotency_key: str, timeout: float,
+    ) -> _WafTask: ...
+
+
+class BrowserEnvRelay:
+    """Borrow BrowserEnv's exclusive Relay browser for one Crawler.
+
+    In the default mode, BrowserEnv's ``octoparse-relay-task`` owns the browser
+    process and Relay extension. With ``captcha=...``, this provider obtains a
+    run-scoped plugin grant and asks BrowserEnv's launcher to start the browser
+    only after the current Data App Run exists. A proxy is installed
+    process-wide in BrowserEnv's local GOST chain; BrowserEnv may still route
+    image, audio, and video requests directly.
+    Pass ``aws_waf_resources=app.resources()`` to use the current Run's
+    managed image solver with the Relay extension.
+    """
+
+    _state_lock = threading.Lock()
+    _active = False
+
+    def __init__(
+        self,
+        *,
+        connect_timeout: float = 45,
+        playwright_factory: Callable[[], Any] | None = None,
+        connection_factory: Callable[..., Any] | None = None,
+        http_transport: httpx.AsyncBaseTransport | None = None,
+        captcha: ManagedCaptchaPlugin | None = None,
+        aws_waf_resources: _WafResources | None = None,
+        launcher_path: str | Path = "/usr/local/bin/octoparse-relay-task",
+        launcher_factory: Callable[..., Any] | None = None,
+    ) -> None:
+        if not math.isfinite(connect_timeout) or connect_timeout <= 0:
+            raise ConfigurationError("connect_timeout must be finite and positive")
+        self._connect_timeout = connect_timeout
+        self._playwright_factory = playwright_factory
+        self._connection_factory = connection_factory
+        self._http_transport = http_transport
+        self._captcha = captcha
+        self._aws_waf_resources = aws_waf_resources
+        self._launcher_path = Path(launcher_path)
+        self._launcher_factory = launcher_factory
+
+    def __repr__(self) -> str:
+        return "BrowserEnvRelay()"
+
+    async def _solve_aws_waf(
+        self, target: str, images: list[str], request_id: str
+    ) -> Sequence[int]:
+        resources = self._aws_waf_resources
+        assert resources is not None
+        task = await resources.solve_captcha(
+            type="aws_waf_image",
+            target=target,
+            images=images,
+            idempotency_key=request_id,
+            timeout=45,
+        )
+        if task.indices is None:
+            raise ConfigurationError("Managed AWS WAF solver returned no image selection")
+        return task.indices
+
+    @classmethod
+    def _claim(cls) -> None:
+        with cls._state_lock:
+            if cls._active:
+                raise ConfigurationError(
+                    "BrowserEnv exposes one shared browser; close its current handle "
+                    "before opening another"
+                )
+            cls._active = True
+
+    @classmethod
+    async def _release_claim(cls) -> None:
+        with cls._state_lock:
+            cls._active = False
+
+    async def open(
+        self, *, identity: Identity | None = None, proxy: ProxyHandle | None = None
+    ) -> BrowserHandle:
+        if identity is not None and (
+            identity.cookies or identity.headers or identity.storage_state is not None
+        ):
+            raise ConfigurationError(
+                "BrowserEnv's shared browser cannot safely apply per-Crawler identity state"
+            )
+        if proxy is not None:
+            # Validate before claiming the process-wide browser. A failed GOST
+            # request may still have applied, so later transport failures must
+            # always attempt to restore direct routing.
+            _chain_payload(proxy.http_url())
+        self._claim()
+        playwright = None
+        connection = None
+        proxy_configured = proxy is not None
+        prepared_captcha: PreparedCaptchaExtension | None = None
+        launcher = None
+        launch_runtime: tempfile.TemporaryDirectory[str] | None = None
+        relay_environment: dict[str, str] = {}
+        try:
+            await self._configure_proxy(proxy)
+            if self._captcha is not None:
+                prepared_captcha = await self._captcha.prepare()
+                prepared_captcha.require_active()
+                launcher, launch_runtime, relay_environment = (
+                    await self._start_managed_browser(prepared_captcha)
+                )
+            async with asyncio.timeout(self._connect_timeout):
+                playwright = await _start_playwright(self._playwright_factory)
+                connection_factory = self._connection_factory
+                if connection_factory is None:
+                    try:
+                        from octoparse.playwright_relay import connect_async  # type: ignore[import-untyped]
+                    except ImportError as error:
+                        raise ConfigurationError(
+                            "Install dataapp-crawler-octoparse-browser to use BrowserEnvRelay"
+                        ) from error
+                    connection_factory = connect_async
+                connection_options: dict[str, Any] = {"timeout": self._connect_timeout}
+                if self._aws_waf_resources is not None:
+                    connection_options["aws_waf_solver"] = self._solve_aws_waf
+                with _temporary_environment(relay_environment):
+                    connection = await _maybe_await(
+                        connection_factory(playwright.chromium, **connection_options)
+                    )
+            browser = connection.browser
+            if not browser.contexts:
+                raise ConfigurationError("BrowserEnv Relay exposed no browser context")
+            context = browser.contexts[0]
+            if prepared_captcha is not None:
+                prepared_captcha.require_active()
+        except BaseException as error:
+            closers = []
+            if playwright is not None:
+                closers.append(playwright.stop)
+            if connection is not None and getattr(connection, "server", None) is not None:
+                closers.append(connection.close)
+            if launcher is not None:
+                closers.append(lambda: _stop_launcher(launcher))
+            if prepared_captcha is not None:
+                closers.append(prepared_captcha.close)
+            await _best_effort_close(*closers)
+            if launch_runtime is not None:
+                try:
+                    launch_runtime.cleanup()
+                except Exception:
+                    pass
+            if proxy_configured:
+                try:
+                    await self._configure_proxy(None)
+                except BaseException:
+                    pass
+            await self._release_claim()
+            if not isinstance(error, Exception):
+                raise
+            if isinstance(error, ConfigurationError):
+                raise
+            raise ConfigurationError(
+                f"BrowserEnv Relay connection failed ({type(error).__name__})"
+            ) from None
+
+        async def stop_playwright() -> None:
+            await _maybe_await(playwright.stop())
+
+        async def close_owned_relay() -> None:
+            await _maybe_await(connection.close())
+
+        async def restore_direct() -> None:
+            await self._configure_proxy(None)
+
+        async def stop_launcher() -> None:
+            if launcher is not None:
+                await _stop_launcher(launcher)
+
+        async def cleanup_captcha() -> None:
+            if prepared_captcha is not None:
+                await prepared_captcha.close()
+
+        async def cleanup_launch_runtime() -> None:
+            if launch_runtime is not None:
+                launch_runtime.cleanup()
+
+        closers = [stop_playwright]
+        if getattr(connection, "server", None) is not None:
+            closers.append(close_owned_relay)
+        if launcher is not None:
+            closers.extend([stop_launcher, cleanup_captcha, cleanup_launch_runtime])
+        closers.extend([restore_direct, self._release_claim])
+        capabilities = {"page", "relay", "process_proxy"}
+        if launcher is None:
+            capabilities.add("borrowed")
+        else:
+            capabilities.add("captcha_plugin")
+        return BrowserHandle(
+            provider="browserenv-relay",
+            capabilities=frozenset(capabilities),
+            owned=launcher is not None,
+            context=context,
+            _closers=closers,
+        )
+
+    async def _start_managed_browser(
+        self, prepared: PreparedCaptchaExtension
+    ) -> tuple[Any, tempfile.TemporaryDirectory[str], dict[str, str]]:
+        launcher_path = self._launcher_path
+        if not launcher_path.is_absolute():
+            raise ConfigurationError("BrowserEnv launcher path must be absolute")
+        runtime = tempfile.TemporaryDirectory(prefix="dataos-browserenv-")
+        ready_file = Path(runtime.name) / "ready"
+        factory = self._launcher_factory or asyncio.create_subprocess_exec
+        environment = dict(os.environ)
+        environment.pop("CAPMONSTER_API_KEY", None)
+        environment.pop("CAPMONSTER_CAPTCHA_TYPES", None)
+        try:
+            process = await _maybe_await(
+                factory(
+                    str(launcher_path),
+                    "--browser-only",
+                    "--captcha-extension",
+                    str(prepared.extension_path),
+                    "--ready-file",
+                    str(ready_file),
+                    stdout=asyncio.subprocess.DEVNULL,
+                    env=environment,
+                )
+            )
+            async with asyncio.timeout(self._connect_timeout):
+                while not ready_file.is_file():
+                    if process.returncode is not None:
+                        raise ConfigurationError(
+                            "BrowserEnv launcher exited before browser readiness"
+                        )
+                    await asyncio.sleep(0.05)
+                relay_environment = _read_launcher_ready(ready_file)
+                if not relay_environment:
+                    raise ConfigurationError("BrowserEnv launcher returned invalid readiness")
+                prepared.require_active()
+            return process, runtime, relay_environment
+        except BaseException:
+            if "process" in locals():
+                await _best_effort_close(lambda: _stop_launcher(process))
+            runtime.cleanup()
+            raise
+
+    async def _configure_proxy(self, proxy: ProxyHandle | None) -> None:
+        api_url = _proxy_api_setting(
+            "URL",
+            "http://127.0.0.1:18080/api/config/chains/chain-0",
+        ).strip()
+        endpoint = urlsplit(api_url)
+        if (
+            endpoint.scheme != "http"
+            or endpoint.hostname is None
+            or endpoint.username is not None
+            or endpoint.password is not None
+            or endpoint.query
+            or endpoint.fragment
+            or not _loopback(endpoint.hostname)
+        ):
+            raise ConfigurationError(
+                "BrowserEnv proxy software API must be an HTTP loopback endpoint"
+            )
+        payload = _chain_payload(proxy.http_url() if proxy is not None else None)
+        username = _proxy_api_setting("USERNAME", "admin")
+        password = _proxy_api_setting("PASSWORD", "")
+        authorization = base64.b64encode(f"{username}:{password}".encode()).decode()
+        try:
+            async with httpx.AsyncClient(
+                transport=self._http_transport,
+                timeout=self._connect_timeout,
+                trust_env=False,
+            ) as client:
+                response = await client.put(
+                    api_url,
+                    json=payload,
+                    headers={"Authorization": f"Basic {authorization}"},
+                )
+                response.raise_for_status()
+        except (httpx.HTTPError, ValueError):
+            raise ConfigurationError("BrowserEnv proxy configuration failed") from None
+
+
+def _loopback(hostname: str) -> bool:
+    if hostname.lower().rstrip(".") == "localhost":
+        return True
+    try:
+        return ipaddress.ip_address(hostname).is_loopback
+    except ValueError:
+        return False
+
+
+async def _stop_launcher(process: Any) -> None:
+    if process.returncode is not None:
+        await _maybe_await(process.wait())
+        return
+    process.terminate()
+    try:
+        async with asyncio.timeout(4):
+            await _maybe_await(process.wait())
+    except TimeoutError:
+        process.kill()
+        await _maybe_await(process.wait())
+
+
+def _read_launcher_ready(path: Path) -> dict[str, str]:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeError, json.JSONDecodeError):
+        return {}
+    if payload.get("ready") is not True:
+        return {}
+    browser = payload.get("browser")
+    chrome = payload.get("chrome")
+    profile = payload.get("profile")
+    if browser not in {"chromium", "google-stable", "cloakbrowser"}:
+        return {}
+    if not all(
+        isinstance(value, str)
+        and value.startswith("/")
+        and "\x00" not in value
+        and "\n" not in value
+        for value in (chrome, profile)
+    ):
+        return {}
+    return {
+        "BROWSER": "/usr/local/bin/octoparse-relay-open-url %s",
+        "OCTOPARSE_RELAY_BROWSER": browser,
+        "OCTOPARSE_RELAY_CHROME_BIN": chrome,
+        "OCTOPARSE_RELAY_PROFILE_DIR": profile,
+    }
+
+
+@contextmanager
+def _temporary_environment(values: dict[str, str]):
+    previous = {name: os.environ.get(name) for name in values}
+    os.environ.update(values)
+    try:
+        yield
+    finally:
+        for name, value in previous.items():
+            if value is None:
+                os.environ.pop(name, None)
+            else:
+                os.environ[name] = value
+
+
+def _proxy_api_setting(name: str, default: str) -> str:
+    return os.environ.get(
+        f"DATAAPP_BROWSER_PROXY_API_{name}",
+        os.environ.get(f"OCTOPUS_PROXY_SOFTWARE_API_{name}", default),
+    )
+
+
+def _chain_payload(proxy_url: str | None) -> dict[str, Any]:
+    keepalive = {
+        "keepalive": True,
+        "keepalive.idle": "60s",
+        "keepalive.interval": "15s",
+        "keepalive.count": 4,
+    }
+    if proxy_url is None:
+        node: dict[str, Any] = {
+            "name": "node-0",
+            "addr": ":0",
+            "connector": {"type": "direct"},
+            "dialer": {"type": "direct", "metadata": keepalive},
+        }
+    else:
+        parts = urlsplit(proxy_url)
+        if (
+            parts.scheme not in {"http", "https"}
+            or parts.hostname is None
+            or parts.path not in {"", "/"}
+            or parts.query
+            or parts.fragment
+        ):
+            raise ConfigurationError("BrowserEnv supports only HTTP and HTTPS upstream proxies")
+        if parts.password is not None and parts.username is None:
+            raise ConfigurationError("BrowserEnv proxy password requires a username")
+        host = f"[{parts.hostname}]" if ":" in parts.hostname else parts.hostname
+        tls = parts.scheme == "https"
+        dialer: dict[str, Any] = {"type": "tcp", "metadata": keepalive}
+        if tls:
+            dialer["type"] = "tls"
+            dialer["tls"] = {"secure": True, "serverName": parts.hostname}
+        port = parts.port if parts.port is not None else (443 if tls else 80)
+        node = {
+            "name": "node-0",
+            "addr": f"{host}:{port}",
+            "connector": {
+                "type": "http",
+                "auth": {
+                    "username": unquote(parts.username or ""),
+                    "password": unquote(parts.password or ""),
+                },
+            },
+            "dialer": dialer,
+        }
+    return {"name": "chain-0", "hops": [{"name": "hop-0", "nodes": [node]}]}