cmdop 0.1.17__tar.gz → 0.1.18__tar.gz

{cmdop-0.1.17 → cmdop-0.1.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cmdop
-Version: 0.1.17
+Version: 0.1.18
 Summary: Python SDK for CMDOP agent interaction
 Project-URL: Homepage, https://cmdop.com
 Project-URL: Documentation, https://cmdop.com
@@ -148,56 +148,63 @@ health: Health = result.output  # Typed!
 ```python
 with client.browser.create_session() as b:
     b.navigate("https://shop.com/products")
+    b.close_modal()  # Close popups
 
-    # DOM extraction
-    products = b.extract_data(".product-card", '{"name": "h2", "price": ".price"}', limit=100)
+    # BeautifulSoup parsing
+    soup = b.soup()  # SoupWrapper with chainable API
+    for item in soup.select(".product"):
+        title = item.select_one("h2").text()
+        price = item.attr("data-price")
 
-    # Get HTML for BeautifulSoup parsing
-    html = b.get_html("[role='feed']")
-    soup = b.parse_html(html)  # Returns BeautifulSoup object
-
-    # Scroll & extract pattern
+    # Human-like scrolling with random delays
     for _ in range(10):
-        html = b.get_html(".listings")
-        # ... parse with soup ...
-        b.scroll("down", 800)
-        b.wait_seconds(1.0)
+        soup = b.soup(".listings")
+        # ... parse ...
+        b.scroll("down", 700, human_like=True)  # Natural micro-scrolls
+        b.wait_random(0.8, 1.5)  # Random delay
+
+    # Scroll inside container (Facebook, Twitter feeds)
+    b.scroll("down", 800, container="[role='feed']")
+
+    # Click all "See more" buttons
+    b.click_all_by_text("See more")
 
     # JS fetch (bypass CORS, inherit cookies)
     data = b.fetch_json("https://api.site.com/v1/items")
-
-    # Parallel fetch
-    results = b.fetch_all({
-        "users": "https://api.site.com/v1/users",
-        "orders": "https://api.site.com/v1/orders",
-    }, credentials=True)
 ```
 
 | Method | Description |
 |--------|-------------|
 | `navigate(url)` | Go to URL |
 | `click(selector)` | Click element |
+| `click_all_by_text(text, role)` | Click all matching elements |
 | `type(selector, text)` | Type text |
 | `wait_for(selector, ms)` | Wait for element |
 | `wait_seconds(n)` | Sleep |
+| `wait_random(min, max)` | Random sleep |
 | `extract(selector, attr)` | Get text/attr |
 | `get_html(selector)` | Get HTML |
-| `get_text(selector)` | Get text |
+| `soup(selector)` | → SoupWrapper |
 | `parse_html(html)` | → BeautifulSoup |
-| `extract_data(item, fields, limit)` | Bulk extract |
 | `fetch_json(url)` | JS fetch → dict |
-| `fetch_all(urls, credentials)` | Parallel fetch |
+| `fetch_all(urls)` | Parallel fetch |
 | `execute_js(code)` | Run async JS |
 | `screenshot()` | PNG bytes |
-| `scroll(dir, amount)` | Scroll page |
+| `scroll(dir, amount, ...)` | Scroll page/container |
 | `scroll_to(selector)` | Scroll to element |
 | `get_scroll_info()` | Position + page size |
-| `infinite_scroll(fn, limit)` | Smart scroll loop |
 | `hover(selector)` | Hover |
 | `select(selector, value)` | Dropdown select |
 | `close_modal()` | Close dialogs |
 | `get/set_cookies()` | Cookie management |
 
+**scroll() parameters:**
+- `direction`: "up", "down", "left", "right"
+- `amount`: pixels to scroll
+- `smooth`: animate scroll (default True)
+- `human_like`: random micro-scrolls + variation
+- `container`: CSS selector for scroll container
+
 ## SDKBaseModel
 
 Auto-cleaning Pydantic model for scraped data:

{cmdop-0.1.17 → cmdop-0.1.18}/README.md

@@ -107,56 +107,63 @@ health: Health = result.output  # Typed!
 ```python
 with client.browser.create_session() as b:
     b.navigate("https://shop.com/products")
+    b.close_modal()  # Close popups
 
-    # DOM extraction
-    products = b.extract_data(".product-card", '{"name": "h2", "price": ".price"}', limit=100)
+    # BeautifulSoup parsing
+    soup = b.soup()  # SoupWrapper with chainable API
+    for item in soup.select(".product"):
+        title = item.select_one("h2").text()
+        price = item.attr("data-price")
 
-    # Get HTML for BeautifulSoup parsing
-    html = b.get_html("[role='feed']")
-    soup = b.parse_html(html)  # Returns BeautifulSoup object
-
-    # Scroll & extract pattern
+    # Human-like scrolling with random delays
     for _ in range(10):
-        html = b.get_html(".listings")
-        # ... parse with soup ...
-        b.scroll("down", 800)
-        b.wait_seconds(1.0)
+        soup = b.soup(".listings")
+        # ... parse ...
+        b.scroll("down", 700, human_like=True)  # Natural micro-scrolls
+        b.wait_random(0.8, 1.5)  # Random delay
+
+    # Scroll inside container (Facebook, Twitter feeds)
+    b.scroll("down", 800, container="[role='feed']")
+
+    # Click all "See more" buttons
+    b.click_all_by_text("See more")
 
     # JS fetch (bypass CORS, inherit cookies)
     data = b.fetch_json("https://api.site.com/v1/items")
-
-    # Parallel fetch
-    results = b.fetch_all({
-        "users": "https://api.site.com/v1/users",
-        "orders": "https://api.site.com/v1/orders",
-    }, credentials=True)
 ```
 
 | Method | Description |
 |--------|-------------|
 | `navigate(url)` | Go to URL |
 | `click(selector)` | Click element |
+| `click_all_by_text(text, role)` | Click all matching elements |
 | `type(selector, text)` | Type text |
 | `wait_for(selector, ms)` | Wait for element |
 | `wait_seconds(n)` | Sleep |
+| `wait_random(min, max)` | Random sleep |
 | `extract(selector, attr)` | Get text/attr |
 | `get_html(selector)` | Get HTML |
-| `get_text(selector)` | Get text |
+| `soup(selector)` | → SoupWrapper |
 | `parse_html(html)` | → BeautifulSoup |
-| `extract_data(item, fields, limit)` | Bulk extract |
 | `fetch_json(url)` | JS fetch → dict |
-| `fetch_all(urls, credentials)` | Parallel fetch |
+| `fetch_all(urls)` | Parallel fetch |
 | `execute_js(code)` | Run async JS |
 | `screenshot()` | PNG bytes |
-| `scroll(dir, amount)` | Scroll page |
+| `scroll(dir, amount, ...)` | Scroll page/container |
 | `scroll_to(selector)` | Scroll to element |
 | `get_scroll_info()` | Position + page size |
-| `infinite_scroll(fn, limit)` | Smart scroll loop |
 | `hover(selector)` | Hover |
 | `select(selector, value)` | Dropdown select |
 | `close_modal()` | Close dialogs |
 | `get/set_cookies()` | Cookie management |
 
+**scroll() parameters:**
+- `direction`: "up", "down", "left", "right"
+- `amount`: pixels to scroll
+- `smooth`: animate scroll (default True)
+- `human_like`: random micro-scrolls + variation
+- `container`: CSS selector for scroll container
+
 ## SDKBaseModel
 
 Auto-cleaning Pydantic model for scraped data:

{cmdop-0.1.17 → cmdop-0.1.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cmdop"
-version = "0.1.17"
+version = "0.1.18"
 description = "Python SDK for CMDOP agent interaction"
 readme = "README.md"
 license = { text = "MIT" }

{cmdop-0.1.17 → cmdop-0.1.18}/src/cmdop/__init__.py

@@ -124,7 +124,7 @@ from cmdop.logging import (
     get_log_dir,
 )
 
-__version__ = "0.1.17"
+__version__ = "0.1.18"
 
 __all__ = [
     # Version

{cmdop-0.1.17 → cmdop-0.1.18}/src/cmdop/services/browser/aio/session.py

@@ -4,9 +4,17 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
+import asyncio
+
 from cmdop.services.browser.base.session import BaseSession
-from cmdop.services.browser.models import BrowserCookie, BrowserState
+from cmdop.services.browser.models import (
+    BrowserCookie,
+    BrowserState,
+    ScrollInfo,
+    ScrollResult,
+)
 from cmdop.services.browser.js import parse_json_result
+from cmdop.services.browser.parsing import parse_html as _parse_html, SoupWrapper
 
 if TYPE_CHECKING:
     from cmdop.services.browser.aio.service import AsyncBrowserService
@@ -158,6 +166,112 @@ class AsyncBrowserSession(BaseSession):
             self._session_id, item, fields_json, limit
         )
 
+    # === HTML Parsing ===
+
+    async def parse_html(self, html: str | None = None, selector: str | None = None) -> "BeautifulSoup":
+        """Parse HTML with BeautifulSoup."""
+        if html is None:
+            html = await self.get_html(selector)
+        return _parse_html(html)
+
+    async def soup(self, selector: str | None = None) -> SoupWrapper:
+        """Get page HTML as SoupWrapper."""
+        html = await self.get_html(selector)
+        return SoupWrapper(html=html)
+
+    # === Scroll & Navigation ===
+
+    async def scroll(
+        self,
+        direction: str = "down",
+        amount: int = 500,
+        selector: str | None = None,
+        smooth: bool = True,
+        human_like: bool = False,
+        container: str | None = None,
+    ) -> ScrollResult:
+        """Scroll the page or container."""
+        js = self._build_scroll(direction, amount, selector, smooth, human_like, container)
+        result = await self.execute_script(js)
+        data = parse_json_result(result) or {}
+        return ScrollResult(
+            success=data.get("success", False),
+            scroll_y=int(data.get("scrollY", 0)),
+            scrolled_by=int(data.get("scrolledBy", 0)),
+            at_bottom=data.get("atBottom", False),
+            error=data.get("error"),
+        )
+
+    async def scroll_to(self, selector: str) -> ScrollResult:
+        """Scroll element into view."""
+        return await self.scroll(selector=selector)
+
+    async def scroll_to_bottom(self) -> ScrollResult:
+        """Scroll to page bottom."""
+        js = self._build_scroll_to_bottom()
+        result = await self.execute_script(js)
+        data = parse_json_result(result) or {}
+        return ScrollResult(
+            success=data.get("success", False),
+            scroll_y=int(data.get("scrollY", 0)),
+            scrolled_by=int(data.get("scrolledBy", 0)),
+            at_bottom=True,
+        )
+
+    async def get_scroll_info(self) -> ScrollInfo:
+        """Get current scroll position and page dimensions."""
+        js = self._build_get_scroll_info()
+        result = await self.execute_script(js)
+        data = parse_json_result(result) or {}
+        return ScrollInfo(
+            scroll_x=int(data.get("scrollX", 0)),
+            scroll_y=int(data.get("scrollY", 0)),
+            page_height=int(data.get("pageHeight", 0)),
+            page_width=int(data.get("pageWidth", 0)),
+            viewport_height=int(data.get("viewportHeight", 0)),
+            viewport_width=int(data.get("viewportWidth", 0)),
+            at_bottom=data.get("atBottom", False),
+            at_top=data.get("atTop", True),
+        )
+
+    # === UI Interaction Helpers ===
+
+    async def hover(self, selector: str) -> bool:
+        """Hover over element."""
+        js = self._build_hover(selector)
+        result = await self.execute_script(js)
+        data = parse_json_result(result) or {}
+        return data.get("success", False)
+
+    async def select(self, selector: str, value: str | None = None, text: str | None = None) -> dict:
+        """Select option from dropdown."""
+        js = self._build_select(selector, value, text)
+        result = await self.execute_script(js)
+        return parse_json_result(result) or {}
+
+    async def close_modal(self, selectors: list[str] | None = None) -> bool:
+        """Try to close modal/dialog."""
+        js = self._build_close_modal(selectors)
+        result = await self.execute_script(js)
+        data = parse_json_result(result) or {}
+        return data.get("success", False)
+
+    async def wait_seconds(self, seconds: float) -> None:
+        """Wait for specified seconds."""
+        await asyncio.sleep(seconds)
+
+    async def wait_random(self, min_sec: float = 0.5, max_sec: float = 1.5) -> None:
+        """Wait for random time between min and max seconds."""
+        import random
+        await asyncio.sleep(min_sec + random.random() * (max_sec - min_sec))
+
+    async def click_all_by_text(self, text: str, role: str = "button") -> int:
+        """Click all elements containing specific text."""
+        js = self._build_click_all_by_text(text, role)
+        result = await self.execute_script(js)
+        data = parse_json_result(result) or {}
+        return data.get("clicked", 0)
+
     # === Context Manager ===
 
     async def close(self) -> None:

{cmdop-0.1.17 → cmdop-0.1.18}/src/cmdop/services/browser/base/session.py

@@ -10,6 +10,13 @@ from cmdop.services.browser.js import (
     build_async_js,
     build_fetch_js,
     build_fetch_all_js,
+    build_scroll_js,
+    build_scroll_to_bottom_js,
+    build_get_scroll_info_js,
+    build_hover_js,
+    build_select_js,
+    build_close_modal_js,
+    build_click_all_by_text_js,
     parse_json_result,
 )
 
@@ -70,3 +77,43 @@ class BaseSession(ABC):
     def _parse_fetch_all(self, result: Any) -> dict[str, Any]:
         """Ensure fetch_all returns dict."""
         return result if isinstance(result, dict) else {}
+
+    # === Scroll helpers ===
+
+    def _build_scroll(
+        self,
+        direction: str,
+        amount: int,
+        selector: str | None,
+        smooth: bool,
+        human_like: bool,
+        container: str | None = None,
+    ) -> str:
+        """Build scroll JS."""
+        return build_scroll_js(direction, amount, selector, smooth, human_like, container)
+
+    def _build_scroll_to_bottom(self) -> str:
+        """Build scroll to bottom JS."""
+        return build_scroll_to_bottom_js()
+
+    def _build_get_scroll_info(self) -> str:
+        """Build get scroll info JS."""
+        return build_get_scroll_info_js()
+
+    # === Interaction helpers ===
+
+    def _build_hover(self, selector: str) -> str:
+        """Build hover JS."""
+        return build_hover_js(selector)
+
+    def _build_select(self, selector: str, value: str | None, text: str | None) -> str:
+        """Build select JS."""
+        return build_select_js(selector, value, text)
+
+    def _build_close_modal(self, selectors: list[str] | None) -> str:
+        """Build close modal JS."""
+        return build_close_modal_js(selectors)
+
+    def _build_click_all_by_text(self, text: str, role: str) -> str:
+        """Build click all by text JS."""
+        return build_click_all_by_text_js(text, role)

{cmdop-0.1.17 → cmdop-0.1.18}/src/cmdop/services/browser/js/__init__.py

@@ -28,6 +28,7 @@ from cmdop.services.browser.js.interaction import (
     build_hover_js,
     build_select_js,
     build_close_modal_js,
+    build_click_all_by_text_js,
 )
 
 __all__ = [
@@ -46,4 +47,5 @@ __all__ = [
     "build_hover_js",
     "build_select_js",
     "build_close_modal_js",
+    "build_click_all_by_text_js",
 ]

{cmdop-0.1.17 → cmdop-0.1.18}/src/cmdop/services/browser/js/interaction.py

@@ -107,3 +107,31 @@ def build_close_modal_js(
         return JSON.stringify({{ success: false, error: 'No modal close button found' }});
     }})()
     """
+
+
+def build_click_all_by_text_js(text: str, role: str = "button") -> str:
+    """
+    Build JS to click all elements containing specific text.
+
+    Args:
+        text: Text to match (case-insensitive)
+        role: Element role to filter (default: "button")
+
+    Returns:
+        JS code that returns { clicked: number }
+    """
+    return f"""
+    (function() {{
+        const elements = document.querySelectorAll('[role="{role}"]');
+        let clicked = 0;
+        const targetText = "{text}".toLowerCase();
+        elements.forEach(el => {{
+            const elText = el.textContent.trim().toLowerCase();
+            if (elText === targetText || elText.includes(targetText)) {{
+                el.click();
+                clicked++;
+            }}
+        }});
+        return JSON.stringify({{ clicked: clicked }});
+    }})()
+    """

cmdop-0.1.18/src/cmdop/services/browser/js/scroll.py

@@ -0,0 +1,224 @@
+"""Scroll JavaScript builders for browser automation."""
+
+from __future__ import annotations
+
+import json
+
+
+def build_scroll_js(
+    direction: str = "down",
+    amount: int = 500,
+    selector: str | None = None,
+    smooth: bool = True,
+    human_like: bool = False,
+    container: str | None = None,
+) -> str:
+    """
+    Build JS for scrolling.
+
+    Args:
+        direction: "up", "down", "left", "right"
+        amount: Pixels to scroll (ignored if selector provided)
+        selector: CSS selector to scroll element into view
+        smooth: Use smooth scroll animation (default True)
+        human_like: Add random variations for natural scrolling
+        container: CSS selector for scroll container (default: window)
+    """
+    if selector:
+        behavior = "smooth" if smooth else "instant"
+        return f"""
+        (async function() {{
+            const el = document.querySelector("{selector}");
+            if (el) {{
+                el.scrollIntoView({{ behavior: '{behavior}', block: 'center' }});
+                await new Promise(r => setTimeout(r, {300 if smooth else 50}));
+                return JSON.stringify({{ success: true, scrollY: window.scrollY }});
+            }}
+            return JSON.stringify({{ success: false, error: 'Element not found' }});
+        }})()
+        """
+
+    behavior = "smooth" if smooth else "instant"
+
+    # Container scroll target
+    container_js = f'document.querySelector("{container}")' if container else "null"
+
+    if human_like:
+        # Human-like scroll with random variation and micro-scrolls
+        return f"""
+        (async function() {{
+            const container = {container_js};
+            const scrollTarget = container || window;
+            const getScrollY = () => container ? container.scrollTop : window.scrollY;
+            const getMaxScroll = () => container
+                ? container.scrollHeight - container.clientHeight
+                : document.body.scrollHeight - window.innerHeight;
+
+            const before = getScrollY();
+            const baseAmount = {amount};
+            const direction = "{direction}";
+
+            // Random variation ±15%
+            const variation = 0.85 + Math.random() * 0.3;
+            const actualAmount = Math.round(baseAmount * variation);
+
+            // Split into 2-4 micro-scrolls
+            const steps = 2 + Math.floor(Math.random() * 3);
+            const stepAmount = actualAmount / steps;
+
+            for (let i = 0; i < steps; i++) {{
+                const stepVariation = 0.8 + Math.random() * 0.4;
+                const thisStep = Math.round(stepAmount * stepVariation);
+
+                let x = 0, y = 0;
+                if (direction === "down") y = thisStep;
+                else if (direction === "up") y = -thisStep;
+                else if (direction === "right") x = thisStep;
+                else if (direction === "left") x = -thisStep;
+
+                if (container) {{
+                    container.scrollBy({{ left: x, top: y, behavior: '{behavior}' }});
+                }} else {{
+                    window.scrollBy({{ left: x, top: y, behavior: '{behavior}' }});
+                }}
+
+                // Random micro-pause 50-150ms between steps
+                if (i < steps - 1) {{
+                    await new Promise(r => setTimeout(r, 50 + Math.random() * 100));
+                }}
+            }}
+
+            // Wait for smooth scroll to finish
+            await new Promise(r => setTimeout(r, {250 if smooth else 50}));
+
+            const after = getScrollY();
+            return JSON.stringify({{
+                success: true,
+                scrollY: after,
+                scrolledBy: after - before,
+                atBottom: after >= getMaxScroll() - 50
+            }});
+        }})()
+        """
+    else:
+        # Standard scroll
+        scroll_x = 0
+        scroll_y = 0
+        if direction == "down":
+            scroll_y = amount
+        elif direction == "up":
+            scroll_y = -amount
+        elif direction == "right":
+            scroll_x = amount
+        elif direction == "left":
+            scroll_x = -amount
+
+        return f"""
+        (async function() {{
+            const container = {container_js};
+            const getScrollY = () => container ? container.scrollTop : window.scrollY;
+            const getMaxScroll = () => container
+                ? container.scrollHeight - container.clientHeight
+                : document.body.scrollHeight - window.innerHeight;
+
+            const before = getScrollY();
+
+            if (container) {{
+                container.scrollBy({{ left: {scroll_x}, top: {scroll_y}, behavior: '{behavior}' }});
+            }} else {{
+                window.scrollBy({{ left: {scroll_x}, top: {scroll_y}, behavior: '{behavior}' }});
+            }}
+
+            // Wait for scroll to complete
+            await new Promise(r => setTimeout(r, {200 if smooth else 30}));
+
+            const after = getScrollY();
+            return JSON.stringify({{
+                success: true,
+                scrollY: after,
+                scrolledBy: after - before,
+                atBottom: after >= getMaxScroll() - 50
+            }});
+        }})()
+        """
+
+
+def build_scroll_to_bottom_js() -> str:
+    """Build JS to scroll to page bottom."""
+    return """
+    (function() {
+        const before = window.scrollY;
+        window.scrollTo(0, document.body.scrollHeight);
+        return JSON.stringify({
+            success: true,
+            scrollY: window.scrollY,
+            scrolledBy: window.scrollY - before,
+            atBottom: true
+        });
+    })()
+    """
+
+
+def build_infinite_scroll_js(
+    seen_keys: list[str],
+    key_selector: str = "a[href]",
+    key_attr: str = "href",
+    container_selector: str = "body",
+) -> str:
+    """
+    Build JS for infinite scroll with deduplication.
+
+    Args:
+        seen_keys: List of already seen keys (e.g., URLs)
+        key_selector: CSS selector for key elements
+        key_attr: Attribute to use as key (default: href)
+        container_selector: Container to find elements in
+
+    Returns:
+        JS that returns {new_keys: [...], at_bottom: bool}
+    """
+    seen_json = json.dumps(seen_keys)
+    return f"""
+    (function() {{
+        const seen = new Set({seen_json});
+        const container = document.querySelector("{container_selector}");
+        if (!container) return JSON.stringify({{ new_keys: [], at_bottom: true, error: 'Container not found' }});
+
+        const elements = container.querySelectorAll("{key_selector}");
+        const newKeys = [];
+
+        elements.forEach(el => {{
+            const key = el.getAttribute("{key_attr}") || el.textContent.trim();
+            if (key && !seen.has(key)) {{
+                seen.add(key);
+                newKeys.push(key);
+            }}
+        }});
+
+        const atBottom = (window.innerHeight + window.scrollY) >= document.body.scrollHeight - 100;
+
+        return JSON.stringify({{
+            new_keys: newKeys,
+            at_bottom: atBottom,
+            total_seen: seen.size
+        }});
+    }})()
+    """
+
+
+def build_get_scroll_info_js() -> str:
+    """Build JS to get current scroll position and page dimensions."""
+    return """
+    (function() {
+        return JSON.stringify({
+            scrollX: window.scrollX,
+            scrollY: window.scrollY,
+            pageHeight: document.body.scrollHeight,
+            pageWidth: document.body.scrollWidth,
+            viewportHeight: window.innerHeight,
+            viewportWidth: window.innerWidth,
+            atBottom: (window.innerHeight + window.scrollY) >= document.body.scrollHeight - 10,
+            atTop: window.scrollY <= 10
+        });
+    })()
+    """