cmdop 0.1.21__py3-none-any.whl → 0.1.23__py3-none-any.whl

cmdop/services/browser/capabilities/fetch.py

@@ -0,0 +1,45 @@
+"""Fetch capability."""
+
+from typing import Any
+
+from cmdop.services.browser.js import (
+    build_fetch_js,
+    build_fetch_all_js,
+    build_async_js,
+    parse_json_result,
+)
+
+from ._base import BaseCapability
+
+
+class FetchCapability(BaseCapability):
+    """HTTP fetch operations from browser context.
+
+    Usage:
+        data = session.fetch.json("/api/data")
+        results = session.fetch.all({"a": "/api/a", "b": "/api/b"})
+    """
+
+    def json(self, url: str) -> dict | list | None:
+        """Fetch JSON from URL."""
+        js = build_fetch_js(url)
+        return parse_json_result(self._js(js))
+
+    def all(
+        self,
+        urls: dict[str, str],
+        headers: dict[str, str] | None = None,
+        credentials: bool = False,
+    ) -> dict[str, Any]:
+        """Fetch multiple URLs in parallel. Returns {id: {data, error}}."""
+        if not urls:
+            return {}
+        js = build_fetch_all_js(urls, headers, credentials)
+        wrapped = build_async_js(js)
+        result = parse_json_result(self._js(wrapped))
+        return result if isinstance(result, dict) else {}
+
+    def execute(self, code: str) -> dict | list | str | None:
+        """Execute async JS code (can use await, fetch, etc)."""
+        js = build_async_js(code)
+        return parse_json_result(self._js(js))

cmdop/services/browser/capabilities/input.py

@@ -0,0 +1,49 @@
+"""Input capability."""
+
+from cmdop.services.browser.js import (
+    build_click_js,
+    build_press_key_js,
+    build_click_all_by_text_js,
+    build_hover_js,
+)
+
+from ._base import BaseCapability
+from ._helpers import to_dict
+
+
+class InputCapability(BaseCapability):
+    """Input operations: clicks, keyboard, hover.
+
+    Usage:
+        session.input.click_js(".button")
+        session.input.key("Escape")
+        session.input.click_all("See more")
+    """
+
+    def click_js(self, selector: str, scroll_into_view: bool = True) -> bool:
+        """Click using JavaScript. More reliable than native click."""
+        js = build_click_js(selector, scroll_into_view)
+        return to_dict(self._js(js)).get("success", False)
+
+    def key(self, key: str, selector: str | None = None) -> bool:
+        """Press keyboard key. Keys: Escape, Enter, Tab, ArrowDown, etc."""
+        js = build_press_key_js(key, selector)
+        return to_dict(self._js(js)).get("success", False)
+
+    def click_all(self, text: str, role: str = "button") -> int:
+        """Click all elements containing text. Returns count clicked."""
+        js = build_click_all_by_text_js(text, role)
+        return to_dict(self._js(js)).get("clicked", 0)
+
+    def hover_js(self, selector: str) -> bool:
+        """Hover using JavaScript."""
+        js = build_hover_js(selector)
+        return to_dict(self._js(js)).get("success", False)
+
+    def hover(self, selector: str, timeout_ms: int = 5000) -> None:
+        """Hover using native browser API."""
+        self._call("hover", selector, timeout_ms)
+
+    def mouse_move(self, x: int, y: int, steps: int = 10) -> None:
+        """Move mouse to coordinates with human-like movement."""
+        self._call("mouse_move", x, y, steps)

cmdop/services/browser/capabilities/network.py

@@ -0,0 +1,245 @@
+"""Network capture capability (v2.19.0)."""
+
+from __future__ import annotations
+from typing import Any
+
+from cmdop.services.browser.models import (
+    NetworkExchange,
+    NetworkRequest,
+    NetworkResponse,
+    NetworkTiming,
+    NetworkStats,
+    NetworkFilter,
+)
+
+from ._base import BaseCapability
+
+
+class NetworkCapability(BaseCapability):
+    """Network capture operations.
+
+    Captures HTTP requests/responses made by the browser.
+    Useful for:
+    - Intercepting API responses
+    - Debugging network issues
+    - Extracting data from XHR/Fetch calls
+
+    Usage:
+        # Enable capture
+        session.network.enable()
+
+        # Navigate and trigger requests
+        session.navigate("https://example.com")
+
+        # Get all captured exchanges
+        exchanges = session.network.get_all()
+
+        # Get last API response
+        api = session.network.last("/api/data")
+        data = api.json_body()
+
+        # Filter by criteria
+        xhr = session.network.filter(
+            url_pattern="/api/",
+            methods=["POST"],
+            status_codes=[200],
+        )
+
+        # Disable capture
+        session.network.disable()
+    """
+
+    def enable(self, max_exchanges: int = 1000, max_response_size: int = 10_000_000) -> None:
+        """Enable network capture.
+
+        Args:
+            max_exchanges: Max exchanges to keep in memory (FIFO eviction)
+            max_response_size: Max response body size in bytes
+        """
+        self._call("network_enable", max_exchanges, max_response_size)
+
+    def disable(self) -> None:
+        """Disable network capture."""
+        self._call("network_disable")
+
+    def get_all(self, limit: int = 0) -> list[NetworkExchange]:
+        """Get all captured exchanges.
+
+        Args:
+            limit: Max results (0 = unlimited)
+        """
+        return self.filter(limit=limit)
+
+    def filter(
+        self,
+        url_pattern: str = "",
+        methods: list[str] | None = None,
+        status_codes: list[int] | None = None,
+        resource_types: list[str] | None = None,
+        limit: int = 0,
+    ) -> list[NetworkExchange]:
+        """Get exchanges matching filter criteria.
+
+        Args:
+            url_pattern: Regex pattern for URL matching
+            methods: HTTP methods (GET, POST, etc.)
+            status_codes: HTTP status codes (200, 404, etc.)
+            resource_types: xhr, fetch, document, script, image, etc.
+            limit: Max results (0 = unlimited)
+        """
+        data = self._call(
+            "network_get_exchanges",
+            url_pattern,
+            methods or [],
+            status_codes or [],
+            resource_types or [],
+            limit,
+        )
+        return [self._parse_exchange(e) for e in data.get("exchanges", [])]
+
+    def get(self, exchange_id: str) -> NetworkExchange | None:
+        """Get specific exchange by ID."""
+        data = self._call("network_get_exchange", exchange_id)
+        exchange = data.get("exchange")
+        if exchange:
+            return self._parse_exchange(exchange)
+        return None
+
+    def last(self, url_pattern: str = "") -> NetworkExchange | None:
+        """Get most recent exchange matching URL pattern.
+
+        Args:
+            url_pattern: Regex pattern for URL (empty = any)
+        """
+        data = self._call("network_get_last", url_pattern)
+        exchange = data.get("exchange")
+        if exchange:
+            return self._parse_exchange(exchange)
+        return None
+
+    def clear(self) -> None:
+        """Clear all captured exchanges."""
+        self._call("network_clear")
+
+    def stats(self) -> NetworkStats:
+        """Get capture statistics."""
+        data = self._call("network_stats")
+        return NetworkStats(
+            enabled=data.get("enabled", False),
+            total_captured=data.get("total_captured", 0),
+            total_errors=data.get("total_errors", 0),
+            total_bytes=data.get("total_bytes", 0),
+            average_duration_ms=data.get("average_duration_ms", 0),
+        )
+
+    def export_har(
+        self,
+        url_pattern: str = "",
+        methods: list[str] | None = None,
+        status_codes: list[int] | None = None,
+        resource_types: list[str] | None = None,
+    ) -> bytes:
+        """Export captured exchanges to HAR format.
+
+        Args:
+            url_pattern: Regex pattern for URL matching
+            methods: HTTP methods filter
+            status_codes: HTTP status codes filter
+            resource_types: Resource types filter
+
+        Returns:
+            HAR JSON as bytes
+        """
+        data = self._call(
+            "network_export_har",
+            url_pattern,
+            methods or [],
+            status_codes or [],
+            resource_types or [],
+        )
+        return data.get("har_data", b"")
+
+    # === Convenience Methods ===
+
+    def api_calls(self, url_pattern: str = "/api/") -> list[NetworkExchange]:
+        """Get XHR/Fetch API calls matching pattern."""
+        return self.filter(
+            url_pattern=url_pattern,
+            resource_types=["xhr", "fetch"],
+        )
+
+    def last_json(self, url_pattern: str = "") -> Any:
+        """Get JSON body from most recent matching response."""
+        exchange = self.last(url_pattern)
+        if exchange:
+            return exchange.json_body()
+        return None
+
+    def wait_for(self, url_pattern: str, timeout_ms: int = 30000) -> NetworkExchange | None:
+        """Wait for a matching request to be captured.
+
+        Args:
+            url_pattern: Regex pattern for URL
+            timeout_ms: Timeout in milliseconds
+
+        Returns:
+            Matching exchange or None if timeout
+        """
+        import time
+        start = time.time()
+        timeout_sec = timeout_ms / 1000
+
+        while time.time() - start < timeout_sec:
+            exchange = self.last(url_pattern)
+            if exchange:
+                return exchange
+            time.sleep(0.1)
+
+        return None
+
+    # === Internal ===
+
+    def _parse_exchange(self, data: dict[str, Any]) -> NetworkExchange:
+        """Parse exchange from dict."""
+        request_data = data.get("request", {})
+        response_data = data.get("response")
+        timing_data = data.get("timing", {})
+
+        request = NetworkRequest(
+            url=request_data.get("url", ""),
+            method=request_data.get("method", "GET"),
+            headers=request_data.get("headers", {}),
+            body=request_data.get("body", b""),
+            content_type=request_data.get("content_type", ""),
+            resource_type=request_data.get("resource_type", ""),
+        )
+
+        response = None
+        if response_data:
+            response = NetworkResponse(
+                status=response_data.get("status", 0),
+                status_text=response_data.get("status_text", ""),
+                headers=response_data.get("headers", {}),
+                body=response_data.get("body", b""),
+                content_type=response_data.get("content_type", ""),
+                size=response_data.get("size", 0),
+                from_cache=response_data.get("from_cache", False),
+            )
+
+        timing = NetworkTiming(
+            started_at_ms=timing_data.get("started_at_ms", 0),
+            ended_at_ms=timing_data.get("ended_at_ms", 0),
+            duration_ms=timing_data.get("duration_ms", 0),
+            wait_time_ms=timing_data.get("wait_time_ms", 0),
+            receive_time_ms=timing_data.get("receive_time_ms", 0),
+        )
+
+        return NetworkExchange(
+            id=data.get("id", ""),
+            request=request,
+            response=response,
+            timing=timing,
+            error=data.get("error", ""),
+            frame_id=data.get("frame_id", ""),
+            initiator=data.get("initiator", ""),
+        )

cmdop/services/browser/capabilities/scroll.py

@@ -0,0 +1,147 @@
+"""Scroll capability."""
+
+import time
+from typing import Any, Callable
+
+from cmdop.services.browser.js import (
+    build_scroll_js,
+    build_scroll_to_bottom_js,
+    build_get_scroll_info_js,
+    build_infinite_scroll_js,
+)
+from cmdop.services.browser.models import ScrollResult, ScrollInfo, InfiniteScrollResult
+
+from ._base import BaseCapability
+from ._helpers import to_dict
+
+
+class ScrollCapability(BaseCapability):
+    """Scroll operations.
+
+    Usage:
+        session.scroll.js("down", 500)
+        session.scroll.to_bottom()
+        info = session.scroll.info()
+    """
+
+    def js(
+        self,
+        direction: str = "down",
+        amount: int = 500,
+        selector: str | None = None,
+        smooth: bool = True,
+        human_like: bool = False,
+        container: str | None = None,
+    ) -> ScrollResult:
+        """Scroll using JavaScript. Use when native scroll doesn't work."""
+        js = build_scroll_js(direction, amount, selector, smooth, human_like, container)
+        data = to_dict(self._js(js))
+        return ScrollResult(
+            success=data.get("success", False),
+            scroll_y=int(data.get("scrollY", 0)),
+            scrolled_by=int(data.get("scrolledBy", 0)),
+            at_bottom=data.get("atBottom", False),
+            error=data.get("error"),
+        )
+
+    def to_bottom(self) -> ScrollResult:
+        """Scroll to page bottom."""
+        data = to_dict(self._js(build_scroll_to_bottom_js()))
+        return ScrollResult(
+            success=data.get("success", False),
+            scroll_y=int(data.get("scrollY", 0)),
+            scrolled_by=int(data.get("scrolledBy", 0)),
+            at_bottom=True,
+        )
+
+    def to_element(self, selector: str) -> ScrollResult:
+        """Scroll element into view."""
+        return self.js(selector=selector)
+
+    def info(self) -> ScrollInfo:
+        """Get scroll position and page dimensions."""
+        data = to_dict(self._js(build_get_scroll_info_js()))
+        return ScrollInfo(
+            scroll_x=int(data.get("scrollX", 0)),
+            scroll_y=int(data.get("scrollY", 0)),
+            page_height=int(data.get("pageHeight", 0)),
+            page_width=int(data.get("pageWidth", 0)),
+            viewport_height=int(data.get("viewportHeight", 0)),
+            viewport_width=int(data.get("viewportWidth", 0)),
+            at_bottom=data.get("atBottom", False),
+            at_top=data.get("atTop", True),
+        )
+
+    def native(
+        self,
+        direction: str = "down",
+        amount: int = 500,
+        selector: str | None = None,
+        smooth: bool = True,
+    ) -> ScrollResult:
+        """Scroll using native browser API."""
+        data = self._call("scroll", direction, amount, selector, smooth)
+        return ScrollResult(
+            success=True,
+            scroll_y=data.get("scroll_y", 0),
+            scrolled_by=data.get("scrolled_by", 0),
+            at_bottom=data.get("at_bottom", False),
+        )
+
+    def collect(
+        self,
+        seen_keys: set[str],
+        key_selector: str = "a[href]",
+        key_attr: str = "href",
+        container_selector: str = "body",
+    ) -> InfiniteScrollResult:
+        """Extract new keys for infinite scroll patterns. Updates seen_keys in-place."""
+        js = build_infinite_scroll_js(list(seen_keys), key_selector, key_attr, container_selector)
+        data = to_dict(self._js(js))
+        new_keys = data.get("new_keys", [])
+        seen_keys.update(new_keys)
+        return InfiniteScrollResult(
+            new_keys=new_keys,
+            at_bottom=data.get("at_bottom", False),
+            total_seen=data.get("total_seen", len(seen_keys)),
+            error=data.get("error"),
+        )
+
+    def infinite(
+        self,
+        extract_fn: Callable[[], list[Any]],
+        limit: int = 100,
+        max_scrolls: int = 50,
+        max_no_new: int = 3,
+        scroll_amount: int = 800,
+        delay: float = 1.0,
+    ) -> list[Any]:
+        """Smart infinite scroll with extraction.
+
+        Args:
+            extract_fn: Returns new items each call (dedup is caller's job)
+            limit: Stop after this many items
+            max_scrolls: Max scroll attempts
+            max_no_new: Stop after N scrolls with no new items
+            scroll_amount: Pixels per scroll
+            delay: Seconds between scrolls
+        """
+        items: list[Any] = []
+        no_new = 0
+
+        for _ in range(max_scrolls):
+            new = extract_fn()
+            if new:
+                items.extend(new)
+                no_new = 0
+                if len(items) >= limit:
+                    break
+            else:
+                no_new += 1
+                if no_new >= max_no_new:
+                    break
+
+            self.js("down", scroll_amount)
+            time.sleep(delay)
+
+        return items[:limit]

cmdop/services/browser/capabilities/timing.py

@@ -0,0 +1,66 @@
+"""Timing capability."""
+
+import random
+import time
+import threading
+from typing import Callable, TypeVar
+
+from ._base import BaseCapability
+
+T = TypeVar("T")
+
+
+class TimingCapability(BaseCapability):
+    """Timing operations: wait, delays, timeouts.
+
+    Usage:
+        session.timing.wait(1000)
+        session.timing.random(0.5, 2.0)
+        result, ok = session.timing.timeout(lambda: slow_fn(), 30)
+    """
+
+    def wait(self, ms: int, jitter: float = 0.1) -> None:
+        """Wait milliseconds with jitter (±10% by default)."""
+        actual = (ms / 1000) * (1 + random.uniform(-jitter, jitter))
+        time.sleep(actual)
+
+    def seconds(self, sec: float, jitter: float = 0.1) -> None:
+        """Wait seconds with jitter."""
+        self.wait(int(sec * 1000), jitter)
+
+    def random(self, min_sec: float = 0.5, max_sec: float = 1.5) -> None:
+        """Wait random time between min and max seconds."""
+        time.sleep(min_sec + random.random() * (max_sec - min_sec))
+
+    def timeout(
+        self,
+        fn: Callable[[], T],
+        seconds: float = 60.0,
+        on_timeout: Callable[[], None] | None = None,
+    ) -> tuple[T | None, bool]:
+        """Run function with timeout. Returns (result, success)."""
+        result: list[T | None] = [None]
+        error: list[Exception | None] = [None]
+        done = threading.Event()
+
+        def run():
+            try:
+                result[0] = fn()
+            except Exception as e:
+                error[0] = e
+            finally:
+                done.set()
+
+        threading.Thread(target=run, daemon=True).start()
+
+        if done.wait(timeout=seconds):
+            if error[0]:
+                raise error[0]
+            return result[0], True
+
+        if on_timeout:
+            try:
+                on_timeout()
+            except Exception:
+                pass
+        return None, False

cmdop/services/browser/js/__init__.py

@@ -7,29 +7,30 @@ This module provides JavaScript code generators for common browser operations:
 - Interaction: Hover, select, modals
 """
 
-from cmdop.services.browser.js.core import (
+from .core import (
     parse_json_result,
     build_async_js,
 )
 
-from cmdop.services.browser.js.fetch import (
+from .fetch import (
     build_fetch_js,
     build_fetch_all_js,
 )
 
-from cmdop.services.browser.js.scroll import (
+from .scroll import (
     build_scroll_js,
     build_scroll_to_bottom_js,
     build_infinite_scroll_js,
     build_get_scroll_info_js,
 )
 
-from cmdop.services.browser.js.interaction import (
+from .interaction import (
     build_hover_js,
     build_select_js,
     build_close_modal_js,
     build_click_all_by_text_js,
     build_press_key_js,
+    build_click_js,
 )
 
 __all__ = [
@@ -50,4 +51,5 @@ __all__ = [
     "build_close_modal_js",
     "build_click_all_by_text_js",
     "build_press_key_js",
+    "build_click_js",
 ]

cmdop/services/browser/js/interaction.py

@@ -178,3 +178,37 @@ def build_click_all_by_text_js(text: str, role: str = "button") -> str:
         return JSON.stringify({{ clicked: clicked }});
     }})()
     """
+
+
+def build_click_js(selector: str, scroll_into_view: bool = True) -> str:
+    """
+    Build JS to click element via JavaScript (more reliable than CDP click).
+
+    This is useful when native CDP click hangs or doesn't work properly.
+    Uses document.querySelector to find element and calls .click() directly.
+
+    Args:
+        selector: CSS selector for the element to click
+        scroll_into_view: If True, scroll element into view before clicking (default: True)
+
+    Returns:
+        JS code that returns { success: true/false, error?: string }
+    """
+    scroll_code = 'el.scrollIntoView({block: "center", behavior: "instant"});' if scroll_into_view else ''
+    selector_escaped = json.dumps(selector)
+
+    return f"""
+    (function() {{
+        const el = document.querySelector({selector_escaped});
+        if (!el) {{
+            return JSON.stringify({{ success: false, error: 'Element not found' }});
+        }}
+        try {{
+            {scroll_code}
+            el.click();
+            return JSON.stringify({{ success: true }});
+        }} catch (e) {{
+            return JSON.stringify({{ success: false, error: e.message }});
+        }}
+    }})()
+    """