cmdop 0.1.16__py3-none-any.whl → 0.1.17__py3-none-any.whl

cmdop/__init__.py

@@ -124,7 +124,7 @@ from cmdop.logging import (
     get_log_dir,
 )
 
-__version__ = "0.1.16"
+__version__ = "0.1.17"
 
 __all__ = [
     # Version

cmdop/services/browser/js/__init__.py

@@ -0,0 +1,49 @@
+"""JavaScript builders for browser automation.
+
+This module provides JavaScript code generators for common browser operations:
+- Core: JSON parsing, async wrappers
+- Fetch: HTTP requests from browser context
+- Scroll: Page scrolling and infinite scroll
+- Interaction: Hover, select, modals
+"""
+
+from cmdop.services.browser.js.core import (
+    parse_json_result,
+    build_async_js,
+)
+
+from cmdop.services.browser.js.fetch import (
+    build_fetch_js,
+    build_fetch_all_js,
+)
+
+from cmdop.services.browser.js.scroll import (
+    build_scroll_js,
+    build_scroll_to_bottom_js,
+    build_infinite_scroll_js,
+    build_get_scroll_info_js,
+)
+
+from cmdop.services.browser.js.interaction import (
+    build_hover_js,
+    build_select_js,
+    build_close_modal_js,
+)
+
+__all__ = [
+    # Core
+    "parse_json_result",
+    "build_async_js",
+    # Fetch
+    "build_fetch_js",
+    "build_fetch_all_js",
+    # Scroll
+    "build_scroll_js",
+    "build_scroll_to_bottom_js",
+    "build_infinite_scroll_js",
+    "build_get_scroll_info_js",
+    # Interaction
+    "build_hover_js",
+    "build_select_js",
+    "build_close_modal_js",
+]

cmdop/services/browser/js/core.py

@@ -0,0 +1,38 @@
+"""Core JavaScript utilities for browser automation."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def parse_json_result(result: str) -> dict | list | None:
+    """Parse JSON result from JS execution."""
+    if result:
+        try:
+            data = json.loads(result)
+            if isinstance(data, dict) and "__error" in data:
+                return None
+            return data
+        except json.JSONDecodeError:
+            return None
+    return None
+
+
+def build_async_js(code: str) -> str:
+    """
+    Wrap JS code in async IIFE with error handling.
+
+    The code should return a value (will be JSON.stringify'd).
+    Use for async operations like fetch, Promise.all, etc.
+    """
+    return f"""
+    (async function() {{
+        try {{
+            const result = await (async () => {{ {code} }})();
+            return JSON.stringify(result);
+        }} catch(e) {{
+            return JSON.stringify({{__error: e.message}});
+        }}
+    }})()
+    """

cmdop/services/browser/{js.py → js/fetch.py}

@@ -1,28 +1,8 @@
-"""JavaScript code builders for browser automation."""
+"""Fetch JavaScript builders for browser automation."""
 
 from __future__ import annotations
 
 import json
-from typing import Any
-
-
-def build_async_js(code: str) -> str:
-    """
-    Wrap JS code in async IIFE with error handling.
-
-    The code should return a value (will be JSON.stringify'd).
-    Use for async operations like fetch, Promise.all, etc.
-    """
-    return f"""
-    (async function() {{
-        try {{
-            const result = await (async () => {{ {code} }})();
-            return JSON.stringify(result);
-        }} catch(e) {{
-            return JSON.stringify({{__error: e.message}});
-        }}
-    }})()
-    """
 
 
 def build_fetch_js(url: str) -> str:
@@ -94,16 +74,3 @@ def build_fetch_all_js(
 
     return results;
     """
-
-
-def parse_json_result(result: str) -> dict | list | None:
-    """Parse JSON result from JS execution."""
-    if result:
-        try:
-            data = json.loads(result)
-            if isinstance(data, dict) and "__error" in data:
-                return None
-            return data
-        except json.JSONDecodeError:
-            return None
-    return None

cmdop/services/browser/js/interaction.py

@@ -0,0 +1,109 @@
+"""Interaction JavaScript builders for browser automation."""
+
+from __future__ import annotations
+
+import json
+
+
+def build_hover_js(selector: str) -> str:
+    """Build JS to hover over element."""
+    return f"""
+    (function() {{
+        const el = document.querySelector("{selector}");
+        if (!el) return JSON.stringify({{ success: false, error: 'Element not found' }});
+
+        const event = new MouseEvent('mouseover', {{
+            bubbles: true,
+            cancelable: true,
+            view: window
+        }});
+        el.dispatchEvent(event);
+
+        return JSON.stringify({{ success: true }});
+    }})()
+    """
+
+
+def build_select_js(selector: str, value: str | None = None, text: str | None = None) -> str:
+    """
+    Build JS to select option from dropdown.
+
+    Args:
+        selector: CSS selector for <select> element
+        value: Option value to select
+        text: Option text to select (if value not provided)
+    """
+    if value is not None:
+        select_code = f'select.value = "{value}";'
+    elif text is not None:
+        select_code = f'''
+        const option = Array.from(select.options).find(o => o.text === "{text}");
+        if (option) select.value = option.value;
+        '''
+    else:
+        return 'JSON.stringify({ success: false, error: "Need value or text" })'
+
+    return f"""
+    (function() {{
+        const select = document.querySelector("{selector}");
+        if (!select) return JSON.stringify({{ success: false, error: 'Select not found' }});
+
+        {select_code}
+
+        // Trigger change event
+        select.dispatchEvent(new Event('change', {{ bubbles: true }}));
+
+        return JSON.stringify({{
+            success: true,
+            selected_value: select.value,
+            selected_text: select.options[select.selectedIndex]?.text
+        }});
+    }})()
+    """
+
+
+def build_close_modal_js(
+    selectors: list[str] | None = None,
+) -> str:
+    """
+    Build JS to close modal dialogs.
+
+    Args:
+        selectors: Custom selectors to try. Default tries common patterns.
+    """
+    default_selectors = [
+        '[aria-label="Close"]',
+        '[aria-label="Dismiss"]',
+        'button[class*="close"]',
+        'button[class*="dismiss"]',
+        '[data-testid="close"]',
+        '.modal-close',
+        '.dialog-close',
+        'button:has(svg[class*="close"])',
+        'div[role="dialog"] button:first-child',
+    ]
+    all_selectors = selectors or default_selectors
+    selectors_json = json.dumps(all_selectors)
+
+    return f"""
+    (function() {{
+        const selectors = {selectors_json};
+
+        for (const sel of selectors) {{
+            try {{
+                const el = document.querySelector(sel);
+                if (el && el.offsetParent !== null) {{
+                    el.click();
+                    return JSON.stringify({{ success: true, selector: sel }});
+                }}
+            }} catch(e) {{
+                // Skip invalid selectors
+            }}
+        }}
+
+        // Try pressing Escape
+        document.dispatchEvent(new KeyboardEvent('keydown', {{ key: 'Escape', code: 'Escape' }}));
+
+        return JSON.stringify({{ success: false, error: 'No modal close button found' }});
+    }})()
+    """

cmdop/services/browser/js/scroll.py

@@ -0,0 +1,133 @@
+"""Scroll JavaScript builders for browser automation."""
+
+from __future__ import annotations
+
+import json
+
+
+def build_scroll_js(
+    direction: str = "down",
+    amount: int = 500,
+    selector: str | None = None,
+) -> str:
+    """
+    Build JS for scrolling.
+
+    Args:
+        direction: "up", "down", "left", "right"
+        amount: Pixels to scroll (ignored if selector provided)
+        selector: CSS selector to scroll into view
+    """
+    if selector:
+        return f"""
+        (function() {{
+            const el = document.querySelector("{selector}");
+            if (el) {{
+                el.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
+                return JSON.stringify({{ success: true }});
+            }}
+            return JSON.stringify({{ success: false, error: 'Element not found' }});
+        }})()
+        """
+
+    scroll_map = {
+        "down": f"window.scrollBy(0, {amount})",
+        "up": f"window.scrollBy(0, -{amount})",
+        "right": f"window.scrollBy({amount}, 0)",
+        "left": f"window.scrollBy(-{amount}, 0)",
+    }
+    scroll_code = scroll_map.get(direction, scroll_map["down"])
+
+    return f"""
+    (function() {{
+        const before = window.scrollY;
+        {scroll_code};
+        return JSON.stringify({{
+            success: true,
+            scrollY: window.scrollY,
+            scrolledBy: window.scrollY - before,
+            atBottom: (window.innerHeight + window.scrollY) >= document.body.scrollHeight
+        }});
+    }})()
+    """
+
+
+def build_scroll_to_bottom_js() -> str:
+    """Build JS to scroll to page bottom."""
+    return """
+    (function() {
+        const before = window.scrollY;
+        window.scrollTo(0, document.body.scrollHeight);
+        return JSON.stringify({
+            success: true,
+            scrollY: window.scrollY,
+            scrolledBy: window.scrollY - before,
+            atBottom: true
+        });
+    })()
+    """
+
+
+def build_infinite_scroll_js(
+    seen_keys: list[str],
+    key_selector: str = "a[href]",
+    key_attr: str = "href",
+    container_selector: str = "body",
+) -> str:
+    """
+    Build JS for infinite scroll with deduplication.
+
+    Args:
+        seen_keys: List of already seen keys (e.g., URLs)
+        key_selector: CSS selector for key elements
+        key_attr: Attribute to use as key (default: href)
+        container_selector: Container to find elements in
+
+    Returns:
+        JS that returns {new_keys: [...], at_bottom: bool}
+    """
+    seen_json = json.dumps(seen_keys)
+    return f"""
+    (function() {{
+        const seen = new Set({seen_json});
+        const container = document.querySelector("{container_selector}");
+        if (!container) return JSON.stringify({{ new_keys: [], at_bottom: true, error: 'Container not found' }});
+
+        const elements = container.querySelectorAll("{key_selector}");
+        const newKeys = [];
+
+        elements.forEach(el => {{
+            const key = el.getAttribute("{key_attr}") || el.textContent.trim();
+            if (key && !seen.has(key)) {{
+                seen.add(key);
+                newKeys.push(key);
+            }}
+        }});
+
+        const atBottom = (window.innerHeight + window.scrollY) >= document.body.scrollHeight - 100;
+
+        return JSON.stringify({{
+            new_keys: newKeys,
+            at_bottom: atBottom,
+            total_seen: seen.size
+        }});
+    }})()
+    """
+
+
+def build_get_scroll_info_js() -> str:
+    """Build JS to get current scroll position and page dimensions."""
+    return """
+    (function() {
+        return JSON.stringify({
+            scrollX: window.scrollX,
+            scrollY: window.scrollY,
+            pageHeight: document.body.scrollHeight,
+            pageWidth: document.body.scrollWidth,
+            viewportHeight: window.innerHeight,
+            viewportWidth: window.innerWidth,
+            atBottom: (window.innerHeight + window.scrollY) >= document.body.scrollHeight - 10,
+            atTop: window.scrollY <= 10
+        });
+    })()
+    """

cmdop/services/browser/models.py

@@ -48,3 +48,35 @@ class BrowserState(BaseModel):
 
     url: str
     title: str
+
+
+class ScrollInfo(BaseModel):
+    """Scroll position and page dimensions."""
+
+    scroll_x: int = 0
+    scroll_y: int = 0
+    page_height: int = 0
+    page_width: int = 0
+    viewport_height: int = 0
+    viewport_width: int = 0
+    at_bottom: bool = False
+    at_top: bool = True
+
+
+class ScrollResult(BaseModel):
+    """Result of scroll operation."""
+
+    success: bool = True
+    scroll_y: int = 0
+    scrolled_by: int = 0
+    at_bottom: bool = False
+    error: str | None = None
+
+
+class InfiniteScrollResult(BaseModel):
+    """Result of infinite scroll extraction."""
+
+    new_keys: list[str] = []
+    at_bottom: bool = False
+    total_seen: int = 0
+    error: str | None = None

cmdop/services/browser/parsing.py

@@ -0,0 +1,104 @@
+"""HTML parsing utilities with BeautifulSoup integration."""
+
+from __future__ import annotations
+
+from typing import Any, Iterator
+
+from bs4 import BeautifulSoup, Tag
+
+
+def parse_html(html: str, parser: str = "html.parser") -> BeautifulSoup:
+    """Parse HTML string into BeautifulSoup object."""
+    return BeautifulSoup(html, parser)
+
+
+class SoupWrapper:
+    """Wrapper around BeautifulSoup with convenience methods."""
+
+    def __init__(self, html: str | None = None, soup: BeautifulSoup | Tag | None = None):
+        if soup is not None:
+            self._soup: BeautifulSoup | Tag = soup
+        elif html is not None:
+            self._soup = BeautifulSoup(html, "html.parser")
+        else:
+            raise ValueError("Either html or soup must be provided")
+
+    @property
+    def soup(self) -> BeautifulSoup | Tag:
+        """Get underlying BeautifulSoup/Tag object."""
+        return self._soup
+
+    def select(self, selector: str) -> list[SoupWrapper]:
+        """Select all matching elements."""
+        return [SoupWrapper(soup=el) for el in self._soup.select(selector)]
+
+    def select_one(self, selector: str) -> SoupWrapper | None:
+        """Select first matching element."""
+        el = self._soup.select_one(selector)
+        return SoupWrapper(soup=el) if el else None
+
+    def find(self, name: str, **kwargs: Any) -> SoupWrapper | None:
+        """Find first element by tag name."""
+        el = self._soup.find(name, **kwargs)
+        if el and isinstance(el, Tag):
+            return SoupWrapper(soup=el)
+        return None
+
+    def find_all(self, name: str | None = None, **kwargs: Any) -> list[SoupWrapper]:
+        """Find all elements by tag name."""
+        elements = self._soup.find_all(name, **kwargs) if name else self._soup.find_all(**kwargs)
+        return [SoupWrapper(soup=el) for el in elements if isinstance(el, Tag)]
+
+    def children(self) -> list[SoupWrapper]:
+        """Get direct child elements (not recursive)."""
+        return [SoupWrapper(soup=el) for el in self._soup.children if isinstance(el, Tag)]
+
+    def text(self, strip: bool = True, separator: str = "") -> str:
+        """Get text content."""
+        return self._soup.get_text(strip=strip, separator=separator)
+
+    def attr(self, name: str, default: str = "") -> str:
+        """Get attribute value."""
+        val = self._soup.get(name, default) if hasattr(self._soup, 'get') else default
+        return str(val) if val else default
+
+    def attrs(self) -> dict[str, Any]:
+        """Get all attributes."""
+        return dict(self._soup.attrs) if hasattr(self._soup, 'attrs') else {}
+
+    def html(self) -> str:
+        """Get HTML."""
+        return str(self._soup)
+
+    # Convenience methods
+
+    def texts(self, selector: str, strip: bool = True) -> list[str]:
+        """Get text from all matching elements."""
+        return [el.text(strip=strip) for el in self.select(selector)]
+
+    def attrs_list(self, selector: str, attr_name: str) -> list[str]:
+        """Get attribute from all matching elements."""
+        return [el.attr(attr_name) for el in self.select(selector) if el.attr(attr_name)]
+
+    def links(self, selector: str = "a[href]") -> list[str]:
+        """Get all href values."""
+        return self.attrs_list(selector, "href")
+
+    def images(self, selector: str = "img[src]") -> list[str]:
+        """Get all image src values."""
+        return self.attrs_list(selector, "src")
+
+    def __bool__(self) -> bool:
+        return self._soup is not None
+
+    def __iter__(self) -> Iterator[SoupWrapper]:
+        for child in self.children():
+            yield child
+
+    def __repr__(self) -> str:
+        return f"<SoupWrapper({getattr(self._soup, 'name', 'soup')})>"
+
+
+def soup(html: str) -> SoupWrapper:
+    """Create SoupWrapper from HTML string."""
+    return SoupWrapper(html=html)