cmdop 0.1.21__py3-none-any.whl → 0.1.22__py3-none-any.whl

cmdop/__init__.py

@@ -124,7 +124,7 @@ from cmdop.logging import (
     get_log_dir,
 )
 
-__version__ = "0.1.21"
+__version__ = "0.1.22"
 
 __all__ = [
     # Version

cmdop/client.py

@@ -439,15 +439,9 @@ class AsyncCMDOPClient:
     @property
     def browser(self) -> AsyncBrowserService:
         """
-        Async browser service for direct programmatic control.
+        Async browser service (stub - not implemented).
 
-        Provides: create_session, navigate, click, type, extract, etc.
-
-        Example:
-            >>> async with client.browser.create_session() as session:
-            ...     await session.navigate("https://google.com")
-            ...     await session.type("input[name='q']", "Python")
-            ...     results = await session.extract(".result-title")
+        Use sync CMDOPClient.browser instead.
         """
         if self._browser is None:
             self._browser = AsyncBrowserService(self._transport)

cmdop/services/browser/__init__.py

@@ -1,46 +1,59 @@
-"""Browser automation service.
-
-Structure:
-    browser/
-        models.py       - Data models (BrowserCookie, BrowserState)
-        js.py           - JavaScript code builders
-        base/           - Base classes (BaseSession, BaseServiceMixin)
-        sync/           - Sync implementation (BrowserSession, BrowserService)
-        aio/            - Async implementation (AsyncBrowserSession, AsyncBrowserService)
+"""Browser SDK with capability-based API.
 
 Usage:
-    # Sync
-    with client.browser.create_session() as session:
+    from cmdop.services.browser import BrowserSession
+
+    with service.create_session() as session:
         session.navigate("https://example.com")
-        data = session.fetch_all(urls, headers={"Accept": "application/json"})
 
-    # Async
-    async with client.browser.create_session() as session:
-        await session.navigate("https://example.com")
-        data = await session.fetch_all(urls, headers={"Accept": "application/json"})
-"""
+        # Capabilities
+        session.scroll.js("down", 500)
+        session.scroll.to_bottom()
 
-from cmdop.services.browser.models import BrowserCookie, BrowserState, PageInfo, ScrollResult, raise_browser_error
+        session.input.click_js(".button")
+        session.input.key("Escape")
 
-# Sync
-from cmdop.services.browser.sync.session import BrowserSession
-from cmdop.services.browser.sync.service import BrowserService
+        session.timing.wait(1000)
 
-# Async
-from cmdop.services.browser.aio.session import AsyncBrowserSession
-from cmdop.services.browser.aio.service import AsyncBrowserService
+        soup = session.dom.soup()
+        data = session.fetch.json("/api/data")
+"""
+
+from .session import BrowserSession
+from .service.sync import BrowserService
+from .service.aio import AsyncBrowserService
+from .models import (
+    BrowserCookie,
+    BrowserState,
+    PageInfo,
+    ScrollResult,
+    ScrollInfo,
+    InfiniteScrollResult,
+)
+from .capabilities import (
+    ScrollCapability,
+    InputCapability,
+    TimingCapability,
+    DOMCapability,
+    FetchCapability,
+)
 
 __all__ = [
+    # Session & Service
+    "BrowserSession",
+    "BrowserService",
+    "AsyncBrowserService",
     # Models
     "BrowserCookie",
     "BrowserState",
     "PageInfo",
     "ScrollResult",
-    "raise_browser_error",
-    # Sync
-    "BrowserSession",
-    "BrowserService",
-    # Async
-    "AsyncBrowserSession",
-    "AsyncBrowserService",
+    "ScrollInfo",
+    "InfiniteScrollResult",
+    # Capabilities
+    "ScrollCapability",
+    "InputCapability",
+    "TimingCapability",
+    "DOMCapability",
+    "FetchCapability",
 ]

cmdop/services/browser/capabilities/__init__.py

@@ -0,0 +1,15 @@
+"""Browser capabilities."""
+
+from .scroll import ScrollCapability
+from .input import InputCapability
+from .timing import TimingCapability
+from .dom import DOMCapability
+from .fetch import FetchCapability
+
+__all__ = [
+    "ScrollCapability",
+    "InputCapability",
+    "TimingCapability",
+    "DOMCapability",
+    "FetchCapability",
+]

cmdop/services/browser/capabilities/_base.py

@@ -0,0 +1,28 @@
+"""Base capability class."""
+
+from __future__ import annotations
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from ..session import BrowserSession
+
+
+class BaseCapability:
+    """Base class for all capabilities.
+
+    Capabilities group related browser operations and delegate
+    actual execution to the session.
+    """
+
+    __slots__ = ("_s",)
+
+    def __init__(self, session: "BrowserSession") -> None:
+        self._s = session
+
+    def _js(self, script: str) -> str:
+        """Execute JS via session."""
+        return self._s.execute_script(script)
+
+    def _call(self, method: str, *args: Any, **kwargs: Any) -> Any:
+        """Call service method via session."""
+        return self._s._call_service(method, *args, **kwargs)

cmdop/services/browser/capabilities/_helpers.py

@@ -0,0 +1,16 @@
+"""Shared parsing helpers for capabilities."""
+
+from typing import Any
+from cmdop.services.browser.js import parse_json_result
+
+
+def to_dict(raw: str) -> dict[str, Any]:
+    """Parse JS result to dict, default empty."""
+    result = parse_json_result(raw)
+    return result if isinstance(result, dict) else {}
+
+
+def to_list(raw: str) -> list[Any]:
+    """Parse JS result to list, default empty."""
+    result = parse_json_result(raw)
+    return result if isinstance(result, list) else []

cmdop/services/browser/capabilities/dom.py

@@ -0,0 +1,76 @@
+"""DOM capability."""
+
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+from cmdop.services.browser.js import build_select_js, build_close_modal_js
+from cmdop.services.browser.parsing import parse_html, SoupWrapper
+
+from ._base import BaseCapability
+from ._helpers import to_dict
+
+if TYPE_CHECKING:
+    from bs4 import BeautifulSoup
+
+
+class DOMCapability(BaseCapability):
+    """DOM operations: extraction, forms, modals.
+
+    Usage:
+        html = session.dom.html()
+        soup = session.dom.soup()
+        session.dom.select("#country", value="US")
+    """
+
+    def html(self, selector: str | None = None) -> str:
+        """Get page HTML."""
+        return self._call("get_html", selector)
+
+    def text(self, selector: str | None = None) -> str:
+        """Get page text content."""
+        return self._call("get_text", selector)
+
+    def soup(self, selector: str | None = None) -> SoupWrapper:
+        """Get HTML as SoupWrapper with chainable API."""
+        return SoupWrapper(html=self.html(selector))
+
+    def parse(self, html: str | None = None, selector: str | None = None) -> "BeautifulSoup":
+        """Parse HTML with BeautifulSoup."""
+        if html is None:
+            html = self.html(selector)
+        return parse_html(html)
+
+    def select(
+        self,
+        selector: str,
+        value: str | None = None,
+        text: str | None = None,
+    ) -> dict:
+        """Select dropdown option by value or text."""
+        js = build_select_js(selector, value, text)
+        return to_dict(self._js(js))
+
+    def close_modal(self, selectors: list[str] | None = None) -> bool:
+        """Try to close modal/dialog."""
+        js = build_close_modal_js(selectors)
+        return to_dict(self._js(js)).get("success", False)
+
+    def extract(
+        self, selector: str, attr: str | None = None, limit: int = 100
+    ) -> list[str]:
+        """Extract text or attribute from elements."""
+        return self._call("extract", selector, attr, limit)
+
+    def extract_regex(
+        self, pattern: str, from_html: bool = False, limit: int = 100
+    ) -> list[str]:
+        """Extract matches using regex pattern."""
+        return self._call("extract_regex", pattern, from_html, limit)
+
+    def validate_selectors(self, item: str, fields: dict[str, str]) -> dict:
+        """Validate CSS selectors on page."""
+        return self._call("validate_selectors", item, fields)
+
+    def extract_data(self, item: str, fields_json: str, limit: int = 100) -> dict:
+        """Extract structured data from page."""
+        return self._call("extract_data", item, fields_json, limit)

cmdop/services/browser/capabilities/fetch.py

@@ -0,0 +1,46 @@
+"""Fetch capability."""
+
+from typing import Any
+
+from cmdop.services.browser.js import (
+    build_fetch_js,
+    build_fetch_all_js,
+    build_async_js,
+    parse_json_result,
+)
+
+from ._base import BaseCapability
+
+
+class FetchCapability(BaseCapability):
+    """HTTP fetch operations from browser context.
+
+    Usage:
+        data = session.fetch.json("/api/data")
+        results = session.fetch.all({"a": "/api/a", "b": "/api/b"})
+    """
+
+    def json(self, url: str) -> dict | list | None:
+        """Fetch JSON from URL."""
+        js = build_fetch_js(url)
+        return parse_json_result(self._js(js))
+
+    def all(
+        self,
+        urls: dict[str, str],
+        headers: dict[str, str] | None = None,
+        credentials: bool = False,
+    ) -> dict[str, Any]:
+        """Fetch multiple URLs in parallel. Returns {id: {data, error}}."""
+        if not urls:
+            return {}
+        js = build_fetch_all_js(urls, headers, credentials)
+        # fetch_all returns via execute_js (async wrapper)
+        wrapped = build_async_js(js.replace("return ", ""))
+        result = parse_json_result(self._js(wrapped))
+        return result if isinstance(result, dict) else {}
+
+    def execute(self, code: str) -> dict | list | str | None:
+        """Execute async JS code (can use await, fetch, etc)."""
+        js = build_async_js(code)
+        return parse_json_result(self._js(js))

cmdop/services/browser/capabilities/input.py

@@ -0,0 +1,49 @@
+"""Input capability."""
+
+from cmdop.services.browser.js import (
+    build_click_js,
+    build_press_key_js,
+    build_click_all_by_text_js,
+    build_hover_js,
+)
+
+from ._base import BaseCapability
+from ._helpers import to_dict
+
+
+class InputCapability(BaseCapability):
+    """Input operations: clicks, keyboard, hover.
+
+    Usage:
+        session.input.click_js(".button")
+        session.input.key("Escape")
+        session.input.click_all("See more")
+    """
+
+    def click_js(self, selector: str, scroll_into_view: bool = True) -> bool:
+        """Click using JavaScript. More reliable than native click."""
+        js = build_click_js(selector, scroll_into_view)
+        return to_dict(self._js(js)).get("success", False)
+
+    def key(self, key: str, selector: str | None = None) -> bool:
+        """Press keyboard key. Keys: Escape, Enter, Tab, ArrowDown, etc."""
+        js = build_press_key_js(key, selector)
+        return to_dict(self._js(js)).get("success", False)
+
+    def click_all(self, text: str, role: str = "button") -> int:
+        """Click all elements containing text. Returns count clicked."""
+        js = build_click_all_by_text_js(text, role)
+        return to_dict(self._js(js)).get("clicked", 0)
+
+    def hover_js(self, selector: str) -> bool:
+        """Hover using JavaScript."""
+        js = build_hover_js(selector)
+        return to_dict(self._js(js)).get("success", False)
+
+    def hover(self, selector: str, timeout_ms: int = 5000) -> None:
+        """Hover using native browser API."""
+        self._call("hover", selector, timeout_ms)
+
+    def mouse_move(self, x: int, y: int, steps: int = 10) -> None:
+        """Move mouse to coordinates with human-like movement."""
+        self._call("mouse_move", x, y, steps)

cmdop/services/browser/capabilities/scroll.py

@@ -0,0 +1,147 @@
+"""Scroll capability."""
+
+import time
+from typing import Any, Callable
+
+from cmdop.services.browser.js import (
+    build_scroll_js,
+    build_scroll_to_bottom_js,
+    build_get_scroll_info_js,
+    build_infinite_scroll_js,
+)
+from cmdop.services.browser.models import ScrollResult, ScrollInfo, InfiniteScrollResult
+
+from ._base import BaseCapability
+from ._helpers import to_dict
+
+
+class ScrollCapability(BaseCapability):
+    """Scroll operations.
+
+    Usage:
+        session.scroll.js("down", 500)
+        session.scroll.to_bottom()
+        info = session.scroll.info()
+    """
+
+    def js(
+        self,
+        direction: str = "down",
+        amount: int = 500,
+        selector: str | None = None,
+        smooth: bool = True,
+        human_like: bool = False,
+        container: str | None = None,
+    ) -> ScrollResult:
+        """Scroll using JavaScript. Use when native scroll doesn't work."""
+        js = build_scroll_js(direction, amount, selector, smooth, human_like, container)
+        data = to_dict(self._js(js))
+        return ScrollResult(
+            success=data.get("success", False),
+            scroll_y=int(data.get("scrollY", 0)),
+            scrolled_by=int(data.get("scrolledBy", 0)),
+            at_bottom=data.get("atBottom", False),
+            error=data.get("error"),
+        )
+
+    def to_bottom(self) -> ScrollResult:
+        """Scroll to page bottom."""
+        data = to_dict(self._js(build_scroll_to_bottom_js()))
+        return ScrollResult(
+            success=data.get("success", False),
+            scroll_y=int(data.get("scrollY", 0)),
+            scrolled_by=int(data.get("scrolledBy", 0)),
+            at_bottom=True,
+        )
+
+    def to_element(self, selector: str) -> ScrollResult:
+        """Scroll element into view."""
+        return self.js(selector=selector)
+
+    def info(self) -> ScrollInfo:
+        """Get scroll position and page dimensions."""
+        data = to_dict(self._js(build_get_scroll_info_js()))
+        return ScrollInfo(
+            scroll_x=int(data.get("scrollX", 0)),
+            scroll_y=int(data.get("scrollY", 0)),
+            page_height=int(data.get("pageHeight", 0)),
+            page_width=int(data.get("pageWidth", 0)),
+            viewport_height=int(data.get("viewportHeight", 0)),
+            viewport_width=int(data.get("viewportWidth", 0)),
+            at_bottom=data.get("atBottom", False),
+            at_top=data.get("atTop", True),
+        )
+
+    def native(
+        self,
+        direction: str = "down",
+        amount: int = 500,
+        selector: str | None = None,
+        smooth: bool = True,
+    ) -> ScrollResult:
+        """Scroll using native browser API."""
+        data = self._call("scroll", direction, amount, selector, smooth)
+        return ScrollResult(
+            success=True,
+            scroll_y=data.get("scroll_y", 0),
+            scrolled_by=data.get("scrolled_by", 0),
+            at_bottom=data.get("at_bottom", False),
+        )
+
+    def collect(
+        self,
+        seen_keys: set[str],
+        key_selector: str = "a[href]",
+        key_attr: str = "href",
+        container_selector: str = "body",
+    ) -> InfiniteScrollResult:
+        """Extract new keys for infinite scroll patterns. Updates seen_keys in-place."""
+        js = build_infinite_scroll_js(list(seen_keys), key_selector, key_attr, container_selector)
+        data = to_dict(self._js(js))
+        new_keys = data.get("new_keys", [])
+        seen_keys.update(new_keys)
+        return InfiniteScrollResult(
+            new_keys=new_keys,
+            at_bottom=data.get("at_bottom", False),
+            total_seen=data.get("total_seen", len(seen_keys)),
+            error=data.get("error"),
+        )
+
+    def infinite(
+        self,
+        extract_fn: Callable[[], list[Any]],
+        limit: int = 100,
+        max_scrolls: int = 50,
+        max_no_new: int = 3,
+        scroll_amount: int = 800,
+        delay: float = 1.0,
+    ) -> list[Any]:
+        """Smart infinite scroll with extraction.
+
+        Args:
+            extract_fn: Returns new items each call (dedup is caller's job)
+            limit: Stop after this many items
+            max_scrolls: Max scroll attempts
+            max_no_new: Stop after N scrolls with no new items
+            scroll_amount: Pixels per scroll
+            delay: Seconds between scrolls
+        """
+        items: list[Any] = []
+        no_new = 0
+
+        for _ in range(max_scrolls):
+            new = extract_fn()
+            if new:
+                items.extend(new)
+                no_new = 0
+                if len(items) >= limit:
+                    break
+            else:
+                no_new += 1
+                if no_new >= max_no_new:
+                    break
+
+            self.js("down", scroll_amount)
+            time.sleep(delay)
+
+        return items[:limit]

cmdop/services/browser/capabilities/timing.py

@@ -0,0 +1,66 @@
+"""Timing capability."""
+
+import random
+import time
+import threading
+from typing import Callable, TypeVar
+
+from ._base import BaseCapability
+
+T = TypeVar("T")
+
+
+class TimingCapability(BaseCapability):
+    """Timing operations: wait, delays, timeouts.
+
+    Usage:
+        session.timing.wait(1000)
+        session.timing.random(0.5, 2.0)
+        result, ok = session.timing.timeout(lambda: slow_fn(), 30)
+    """
+
+    def wait(self, ms: int, jitter: float = 0.1) -> None:
+        """Wait milliseconds with jitter (±10% by default)."""
+        actual = (ms / 1000) * (1 + random.uniform(-jitter, jitter))
+        time.sleep(actual)
+
+    def seconds(self, sec: float, jitter: float = 0.1) -> None:
+        """Wait seconds with jitter."""
+        self.wait(int(sec * 1000), jitter)
+
+    def random(self, min_sec: float = 0.5, max_sec: float = 1.5) -> None:
+        """Wait random time between min and max seconds."""
+        time.sleep(min_sec + random.random() * (max_sec - min_sec))
+
+    def timeout(
+        self,
+        fn: Callable[[], T],
+        seconds: float = 60.0,
+        on_timeout: Callable[[], None] | None = None,
+    ) -> tuple[T | None, bool]:
+        """Run function with timeout. Returns (result, success)."""
+        result: list[T | None] = [None]
+        error: list[Exception | None] = [None]
+        done = threading.Event()
+
+        def run():
+            try:
+                result[0] = fn()
+            except Exception as e:
+                error[0] = e
+            finally:
+                done.set()
+
+        threading.Thread(target=run, daemon=True).start()
+
+        if done.wait(timeout=seconds):
+            if error[0]:
+                raise error[0]
+            return result[0], True
+
+        if on_timeout:
+            try:
+                on_timeout()
+            except Exception:
+                pass
+        return None, False

cmdop/services/browser/js/__init__.py

@@ -7,29 +7,30 @@ This module provides JavaScript code generators for common browser operations:
 - Interaction: Hover, select, modals
 """
 
-from cmdop.services.browser.js.core import (
+from .core import (
     parse_json_result,
     build_async_js,
 )
 
-from cmdop.services.browser.js.fetch import (
+from .fetch import (
     build_fetch_js,
     build_fetch_all_js,
 )
 
-from cmdop.services.browser.js.scroll import (
+from .scroll import (
     build_scroll_js,
     build_scroll_to_bottom_js,
     build_infinite_scroll_js,
     build_get_scroll_info_js,
 )
 
-from cmdop.services.browser.js.interaction import (
+from .interaction import (
     build_hover_js,
     build_select_js,
     build_close_modal_js,
     build_click_all_by_text_js,
     build_press_key_js,
+    build_click_js,
 )
 
 __all__ = [
@@ -50,4 +51,5 @@ __all__ = [
     "build_close_modal_js",
     "build_click_all_by_text_js",
     "build_press_key_js",
+    "build_click_js",
 ]

cmdop/services/browser/js/interaction.py

@@ -178,3 +178,37 @@ def build_click_all_by_text_js(text: str, role: str = "button") -> str:
         return JSON.stringify({{ clicked: clicked }});
     }})()
     """
+
+
+def build_click_js(selector: str, scroll_into_view: bool = True) -> str:
+    """
+    Build JS to click element via JavaScript (more reliable than CDP click).
+
+    This is useful when native CDP click hangs or doesn't work properly.
+    Uses document.querySelector to find element and calls .click() directly.
+
+    Args:
+        selector: CSS selector for the element to click
+        scroll_into_view: If True, scroll element into view before clicking (default: True)
+
+    Returns:
+        JS code that returns { success: true/false, error?: string }
+    """
+    scroll_code = 'el.scrollIntoView({block: "center", behavior: "instant"});' if scroll_into_view else ''
+    selector_escaped = json.dumps(selector)
+
+    return f"""
+    (function() {{
+        const el = document.querySelector({selector_escaped});
+        if (!el) {{
+            return JSON.stringify({{ success: false, error: 'Element not found' }});
+        }}
+        try {{
+            {scroll_code}
+            el.click();
+            return JSON.stringify({{ success: true }});
+        }} catch (e) {{
+            return JSON.stringify({{ success: false, error: e.message }});
+        }}
+    }})()
+    """

cmdop/services/browser/service/__init__.py

@@ -0,0 +1,5 @@
+"""Browser service layer (gRPC)."""
+
+from .sync import BrowserService
+
+__all__ = ["BrowserService"]