cmdop 0.1.22__py3-none-any.whl → 0.1.24__py3-none-any.whl

cmdop/helpers/network_analyzer.py

@@ -0,0 +1,349 @@
+"""Network analyzer for discovering API endpoints and creating request snapshots."""
+
+from __future__ import annotations
+
+import time
+from typing import TYPE_CHECKING, Any
+
+from pydantic import BaseModel, Field
+from urllib.parse import urlparse, parse_qs
+
+if TYPE_CHECKING:
+    from cmdop.services.browser.session import BrowserSession
+    from cmdop.services.browser.models import NetworkExchange
+
+
+class RequestSnapshot(BaseModel):
+    """Complete snapshot of an API request for reproduction."""
+
+    # Request info
+    url: str
+    method: str = "GET"
+    headers: dict[str, str] = Field(default_factory=dict)
+    body: str = ""
+
+    # Response info
+    status: int | None = None
+    content_type: str = ""
+    size: int = 0
+
+    # Parsed URL parts
+    base_url: str = ""
+    path: str = ""
+    query_params: dict[str, list[str]] = Field(default_factory=dict)
+
+    # Data analysis
+    data_key: str | None = None
+    item_count: int | None = None
+    item_fields: list[str] = Field(default_factory=list)
+    sample_response: Any = None
+
+    # Session data
+    cookies: dict[str, str] = Field(default_factory=dict)
+
+    def to_curl(self) -> str:
+        """Generate curl command to reproduce request."""
+        parts = [f"curl -X {self.method}"]
+
+        # Add headers
+        for key, value in self.headers.items():
+            if key.lower() not in ("host", "content-length"):
+                parts.append(f"-H '{key}: {value}'")
+
+        # Add cookies if not in headers
+        if self.cookies and "cookie" not in [k.lower() for k in self.headers]:
+            cookie_str = "; ".join(f"{k}={v}" for k, v in self.cookies.items())
+            parts.append(f"-H 'Cookie: {cookie_str}'")
+
+        # Add body
+        if self.body:
+            parts.append(f"-d '{self.body}'")
+
+        # Add URL
+        parts.append(f"'{self.url}'")
+
+        return " \\\n  ".join(parts)
+
+    def to_httpx(self) -> str:
+        """Generate httpx Python code to reproduce request."""
+        lines = ["import httpx", ""]
+
+        # Headers
+        if self.headers:
+            lines.append("headers = {")
+            for key, value in self.headers.items():
+                if key.lower() not in ("host", "content-length"):
+                    lines.append(f'    "{key}": "{value}",')
+            lines.append("}")
+        else:
+            lines.append("headers = {}")
+
+        # Cookies
+        if self.cookies:
+            lines.append("")
+            lines.append("cookies = {")
+            for key, value in self.cookies.items():
+                lines.append(f'    "{key}": "{value}",')
+            lines.append("}")
+        else:
+            lines.append("cookies = {}")
+
+        # Request
+        lines.append("")
+        if self.method == "GET":
+            lines.append(f'response = httpx.get("{self.url}", headers=headers, cookies=cookies)')
+        elif self.method == "POST":
+            if self.body:
+                lines.append(f'data = {repr(self.body)}')
+                lines.append(f'response = httpx.post("{self.url}", headers=headers, cookies=cookies, content=data)')
+            else:
+                lines.append(f'response = httpx.post("{self.url}", headers=headers, cookies=cookies)')
+        else:
+            lines.append(f'response = httpx.request("{self.method}", "{self.url}", headers=headers, cookies=cookies)')
+
+        lines.append("print(response.json())")
+
+        return "\n".join(lines)
+
+
+class NetworkSnapshot(BaseModel):
+    """Complete snapshot of network activity for a site."""
+
+    url: str
+    timestamp: str = ""
+
+    # Session data
+    cookies: dict[str, str] = Field(default_factory=dict)
+    local_storage: dict[str, str] = Field(default_factory=dict)
+
+    # Captured requests
+    api_requests: list[RequestSnapshot] = Field(default_factory=list)
+    json_requests: list[RequestSnapshot] = Field(default_factory=list)
+    other_requests: list[dict] = Field(default_factory=list)
+
+    # Stats
+    total_requests: int = 0
+    total_bytes: int = 0
+
+    def to_json(self, indent: int = 2) -> str:
+        """Convert to JSON string."""
+        return self.model_dump_json(indent=indent)
+
+    def best_api(self) -> RequestSnapshot | None:
+        """Get the best data API (most items)."""
+        if not self.api_requests:
+            return None
+        return max(self.api_requests, key=lambda r: r.item_count or 0)
+
+
+class NetworkAnalyzer:
+    """Analyze network requests to discover API endpoints.
+
+    Creates complete request snapshots including cookies, headers, and
+    all data needed to reproduce API calls.
+
+    Usage:
+        from cmdop import CMDOPClient
+        from cmdop.helpers import NetworkAnalyzer
+
+        client = CMDOPClient.local()
+        with client.browser.create_session(headless=False) as b:
+            analyzer = NetworkAnalyzer(b)
+
+            # Interactive mode - user clicks pagination
+            snapshot = analyzer.capture("https://example.com/cars", wait_seconds=30)
+
+            # Get best API endpoint
+            if snapshot.api_requests:
+                best = snapshot.best_api()
+                print(f"API: {best.url}")
+                print(f"Curl: {best.to_curl()}")
+    """
+
+    # Common keys that contain data arrays
+    DATA_KEYS = [
+        "data", "items", "results", "list", "records",
+        "cars", "vehicles", "products", "listings", "entries",
+        "rows", "content", "objects", "elements", "collection",
+    ]
+
+    def __init__(self, session: "BrowserSession"):
+        """Initialize with browser session."""
+        self._session = session
+
+    def capture(
+        self,
+        url: str,
+        wait_seconds: int = 30,
+        url_pattern: str = "",
+        clear_initial: bool = True,
+        same_origin: bool = True,
+        min_size: int = 100,
+        max_size: int = 500_000,
+        countdown_message: str = "Click pagination!",
+    ) -> NetworkSnapshot:
+        """Capture network requests while user interacts with page.
+
+        Args:
+            url: Page URL to open
+            wait_seconds: Time to wait for user interactions
+            url_pattern: Optional regex filter for API URLs
+            clear_initial: Clear page load requests before capture
+            same_origin: Only capture requests to same domain (default True)
+            min_size: Min response size in bytes (filter tracking pixels)
+            max_size: Max response size in bytes (filter images/assets)
+            countdown_message: Message to show in countdown toast
+
+        Returns:
+            NetworkSnapshot with all captured requests and session data
+        """
+        from cmdop.services.browser.models import WaitUntil
+        from datetime import datetime
+
+        b = self._session
+        snapshot = NetworkSnapshot(
+            url=url,
+            timestamp=datetime.now().isoformat(),
+        )
+
+        # Extract base domain for filtering
+        base_domain = self._extract_base_domain(url)
+
+        # Enable network capture
+        b.network.enable(max_exchanges=500, max_response_size=5_000_000)
+
+        try:
+            print(f"Opening {url}...")
+            b.navigate(url, timeout_ms=90000, wait_until=WaitUntil.LOAD)
+
+            # Wait for page to be interactive
+            try:
+                b.wait_for("body", timeout_ms=10000)
+            except Exception:
+                pass
+            time.sleep(2)
+
+            if clear_initial:
+                b.network.clear()
+
+            # Show countdown while user interacts
+            b.visual.countdown(wait_seconds, countdown_message)
+
+            # Get cookies
+            try:
+                cookies = b.get_cookies()
+                snapshot.cookies = {c.name: c.value for c in cookies}
+            except Exception:
+                pass
+
+            # Get stats
+            stats = b.network.stats()
+            snapshot.total_requests = stats.total_captured
+            snapshot.total_bytes = stats.total_bytes
+
+            print(f"\nCaptured {stats.total_captured} requests ({stats.total_bytes} bytes)")
+
+            # Get XHR/Fetch calls
+            api_calls = b.network.filter(
+                url_pattern=url_pattern,
+                resource_types=["xhr", "fetch"],
+            )
+
+            # Filter by domain
+            if same_origin:
+                api_calls = [
+                    call for call in api_calls
+                    if base_domain in urlparse(call.request.url).netloc
+                ]
+
+            # Filter by response size (ignore tracking pixels and heavy assets)
+            api_calls = [
+                call for call in api_calls
+                if call.response and min_size <= call.response.size <= max_size
+            ]
+
+            print(f"Found {len(api_calls)} XHR/Fetch requests (domain: {base_domain}, {min_size}-{max_size} bytes)")
+
+            # Analyze calls
+            for call in api_calls:
+                req = self._create_snapshot(call, snapshot.cookies)
+                if req:
+                    if req.data_key or (req.item_count and req.item_count > 0):
+                        snapshot.api_requests.append(req)
+                    elif req.content_type and "json" in req.content_type:
+                        snapshot.json_requests.append(req)
+                    else:
+                        snapshot.other_requests.append({
+                            "url": call.request.url,
+                            "method": call.request.method,
+                            "status": call.response.status if call.response else None,
+                        })
+
+        finally:
+            b.network.disable()
+
+        return snapshot
+
+    def _extract_base_domain(self, url: str) -> str:
+        """Extract base domain from URL, handling country-code TLDs."""
+        parsed = urlparse(url)
+        host = parsed.netloc.replace("www.", "")
+        parts = host.split(".")
+
+        # Country-code second-level domains
+        cc_slds = {"co", "com", "net", "org", "ac", "go", "ne", "or"}
+
+        if len(parts) >= 3 and parts[-2] in cc_slds:
+            return ".".join(parts[-3:])  # bobaedream.co.kr
+        elif len(parts) >= 2:
+            return ".".join(parts[-2:])  # kcar.com
+        return host
+
+    def _create_snapshot(
+        self,
+        exchange: "NetworkExchange",
+        session_cookies: dict[str, str],
+    ) -> RequestSnapshot | None:
+        """Create request snapshot from network exchange."""
+        if not exchange.response:
+            return None
+
+        parsed = urlparse(exchange.request.url)
+
+        snapshot = RequestSnapshot(
+            url=exchange.request.url,
+            method=exchange.request.method,
+            headers=dict(exchange.request.headers),
+            body=exchange.request.body.decode("utf-8", errors="ignore") if exchange.request.body else "",
+            status=exchange.response.status,
+            content_type=exchange.response.content_type or "",
+            size=exchange.response.size,
+            base_url=f"{parsed.scheme}://{parsed.netloc}",
+            path=parsed.path,
+            query_params=parse_qs(parsed.query),
+            cookies=session_cookies,
+        )
+
+        # Parse JSON response
+        if "json" in snapshot.content_type.lower():
+            try:
+                data = exchange.json_body()
+                snapshot.sample_response = data
+
+                if isinstance(data, list):
+                    snapshot.item_count = len(data)
+                    if data and isinstance(data[0], dict):
+                        snapshot.item_fields = list(data[0].keys())
+                elif isinstance(data, dict):
+                    for key in self.DATA_KEYS:
+                        if key in data and isinstance(data[key], list):
+                            snapshot.data_key = key
+                            snapshot.item_count = len(data[key])
+                            if data[key] and isinstance(data[key][0], dict):
+                                snapshot.item_fields = list(data[key][0].keys())
+                            break
+            except Exception:
+                pass
+
+        return snapshot
+

cmdop/services/browser/capabilities/__init__.py

@@ -5,6 +5,8 @@ from .input import InputCapability
 from .timing import TimingCapability
 from .dom import DOMCapability
 from .fetch import FetchCapability
+from .network import NetworkCapability
+from .visual import VisualCapability
 
 __all__ = [
     "ScrollCapability",
@@ -12,4 +14,6 @@ __all__ = [
     "TimingCapability",
     "DOMCapability",
     "FetchCapability",
+    "NetworkCapability",
+    "VisualCapability",
 ]

cmdop/services/browser/capabilities/fetch.py

@@ -35,8 +35,7 @@ class FetchCapability(BaseCapability):
         if not urls:
             return {}
         js = build_fetch_all_js(urls, headers, credentials)
-        # fetch_all returns via execute_js (async wrapper)
-        wrapped = build_async_js(js.replace("return ", ""))
+        wrapped = build_async_js(js)
         result = parse_json_result(self._js(wrapped))
         return result if isinstance(result, dict) else {}
 

cmdop/services/browser/capabilities/network.py

@@ -0,0 +1,245 @@
+"""Network capture capability (v2.19.0)."""
+
+from __future__ import annotations
+from typing import Any
+
+from cmdop.services.browser.models import (
+    NetworkExchange,
+    NetworkRequest,
+    NetworkResponse,
+    NetworkTiming,
+    NetworkStats,
+    NetworkFilter,
+)
+
+from ._base import BaseCapability
+
+
+class NetworkCapability(BaseCapability):
+    """Network capture operations.
+
+    Captures HTTP requests/responses made by the browser.
+    Useful for:
+    - Intercepting API responses
+    - Debugging network issues
+    - Extracting data from XHR/Fetch calls
+
+    Usage:
+        # Enable capture
+        session.network.enable()
+
+        # Navigate and trigger requests
+        session.navigate("https://example.com")
+
+        # Get all captured exchanges
+        exchanges = session.network.get_all()
+
+        # Get last API response
+        api = session.network.last("/api/data")
+        data = api.json_body()
+
+        # Filter by criteria
+        xhr = session.network.filter(
+            url_pattern="/api/",
+            methods=["POST"],
+            status_codes=[200],
+        )
+
+        # Disable capture
+        session.network.disable()
+    """
+
+    def enable(self, max_exchanges: int = 1000, max_response_size: int = 10_000_000) -> None:
+        """Enable network capture.
+
+        Args:
+            max_exchanges: Max exchanges to keep in memory (FIFO eviction)
+            max_response_size: Max response body size in bytes
+        """
+        self._call("network_enable", max_exchanges, max_response_size)
+
+    def disable(self) -> None:
+        """Disable network capture."""
+        self._call("network_disable")
+
+    def get_all(self, limit: int = 0) -> list[NetworkExchange]:
+        """Get all captured exchanges.
+
+        Args:
+            limit: Max results (0 = unlimited)
+        """
+        return self.filter(limit=limit)
+
+    def filter(
+        self,
+        url_pattern: str = "",
+        methods: list[str] | None = None,
+        status_codes: list[int] | None = None,
+        resource_types: list[str] | None = None,
+        limit: int = 0,
+    ) -> list[NetworkExchange]:
+        """Get exchanges matching filter criteria.
+
+        Args:
+            url_pattern: Regex pattern for URL matching
+            methods: HTTP methods (GET, POST, etc.)
+            status_codes: HTTP status codes (200, 404, etc.)
+            resource_types: xhr, fetch, document, script, image, etc.
+            limit: Max results (0 = unlimited)
+        """
+        data = self._call(
+            "network_get_exchanges",
+            url_pattern,
+            methods or [],
+            status_codes or [],
+            resource_types or [],
+            limit,
+        )
+        return [self._parse_exchange(e) for e in data.get("exchanges", [])]
+
+    def get(self, exchange_id: str) -> NetworkExchange | None:
+        """Get specific exchange by ID."""
+        data = self._call("network_get_exchange", exchange_id)
+        exchange = data.get("exchange")
+        if exchange:
+            return self._parse_exchange(exchange)
+        return None
+
+    def last(self, url_pattern: str = "") -> NetworkExchange | None:
+        """Get most recent exchange matching URL pattern.
+
+        Args:
+            url_pattern: Regex pattern for URL (empty = any)
+        """
+        data = self._call("network_get_last", url_pattern)
+        exchange = data.get("exchange")
+        if exchange:
+            return self._parse_exchange(exchange)
+        return None
+
+    def clear(self) -> None:
+        """Clear all captured exchanges."""
+        self._call("network_clear")
+
+    def stats(self) -> NetworkStats:
+        """Get capture statistics."""
+        data = self._call("network_stats")
+        return NetworkStats(
+            enabled=data.get("enabled", False),
+            total_captured=data.get("total_captured", 0),
+            total_errors=data.get("total_errors", 0),
+            total_bytes=data.get("total_bytes", 0),
+            average_duration_ms=data.get("average_duration_ms", 0),
+        )
+
+    def export_har(
+        self,
+        url_pattern: str = "",
+        methods: list[str] | None = None,
+        status_codes: list[int] | None = None,
+        resource_types: list[str] | None = None,
+    ) -> bytes:
+        """Export captured exchanges to HAR format.
+
+        Args:
+            url_pattern: Regex pattern for URL matching
+            methods: HTTP methods filter
+            status_codes: HTTP status codes filter
+            resource_types: Resource types filter
+
+        Returns:
+            HAR JSON as bytes
+        """
+        data = self._call(
+            "network_export_har",
+            url_pattern,
+            methods or [],
+            status_codes or [],
+            resource_types or [],
+        )
+        return data.get("har_data", b"")
+
+    # === Convenience Methods ===
+
+    def api_calls(self, url_pattern: str = "/api/") -> list[NetworkExchange]:
+        """Get XHR/Fetch API calls matching pattern."""
+        return self.filter(
+            url_pattern=url_pattern,
+            resource_types=["xhr", "fetch"],
+        )
+
+    def last_json(self, url_pattern: str = "") -> Any:
+        """Get JSON body from most recent matching response."""
+        exchange = self.last(url_pattern)
+        if exchange:
+            return exchange.json_body()
+        return None
+
+    def wait_for(self, url_pattern: str, timeout_ms: int = 30000) -> NetworkExchange | None:
+        """Wait for a matching request to be captured.
+
+        Args:
+            url_pattern: Regex pattern for URL
+            timeout_ms: Timeout in milliseconds
+
+        Returns:
+            Matching exchange or None if timeout
+        """
+        import time
+        start = time.time()
+        timeout_sec = timeout_ms / 1000
+
+        while time.time() - start < timeout_sec:
+            exchange = self.last(url_pattern)
+            if exchange:
+                return exchange
+            time.sleep(0.1)
+
+        return None
+
+    # === Internal ===
+
+    def _parse_exchange(self, data: dict[str, Any]) -> NetworkExchange:
+        """Parse exchange from dict."""
+        request_data = data.get("request", {})
+        response_data = data.get("response")
+        timing_data = data.get("timing", {})
+
+        request = NetworkRequest(
+            url=request_data.get("url", ""),
+            method=request_data.get("method", "GET"),
+            headers=request_data.get("headers", {}),
+            body=request_data.get("body", b""),
+            content_type=request_data.get("content_type", ""),
+            resource_type=request_data.get("resource_type", ""),
+        )
+
+        response = None
+        if response_data:
+            response = NetworkResponse(
+                status=response_data.get("status", 0),
+                status_text=response_data.get("status_text", ""),
+                headers=response_data.get("headers", {}),
+                body=response_data.get("body", b""),
+                content_type=response_data.get("content_type", ""),
+                size=response_data.get("size", 0),
+                from_cache=response_data.get("from_cache", False),
+            )
+
+        timing = NetworkTiming(
+            started_at_ms=timing_data.get("started_at_ms", 0),
+            ended_at_ms=timing_data.get("ended_at_ms", 0),
+            duration_ms=timing_data.get("duration_ms", 0),
+            wait_time_ms=timing_data.get("wait_time_ms", 0),
+            receive_time_ms=timing_data.get("receive_time_ms", 0),
+        )
+
+        return NetworkExchange(
+            id=data.get("id", ""),
+            request=request,
+            response=response,
+            timing=timing,
+            error=data.get("error", ""),
+            frame_id=data.get("frame_id", ""),
+            initiator=data.get("initiator", ""),
+        )