code-puppy 0.0.169__py3-none-any.whl → 0.0.366__py3-none-any.whl

code_puppy/tools/browser/browser_screenshot.py

@@ -0,0 +1,179 @@
+"""Screenshot tool for browser automation.
+
+Captures screenshots and returns them via ToolReturn with BinaryContent
+so multimodal models can directly see and analyze - no separate VQA agent needed.
+"""
+
+import time
+from datetime import datetime
+from pathlib import Path
+from tempfile import gettempdir, mkdtemp
+from typing import Any, Dict, Optional, Union
+
+from pydantic_ai import BinaryContent, RunContext, ToolReturn
+
+from code_puppy.messaging import emit_error, emit_info, emit_success
+from code_puppy.tools.common import generate_group_id
+
+from .browser_manager import get_session_browser_manager
+
+_TEMP_SCREENSHOT_ROOT = Path(
+    mkdtemp(prefix="code_puppy_screenshots_", dir=gettempdir())
+)
+
+
+def _build_screenshot_path(timestamp: str) -> Path:
+    """Return the target path for a screenshot."""
+    filename = f"screenshot_{timestamp}.png"
+    return _TEMP_SCREENSHOT_ROOT / filename
+
+
+async def _capture_screenshot(
+    page,
+    full_page: bool = False,
+    element_selector: Optional[str] = None,
+    save_screenshot: bool = True,
+    group_id: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Internal screenshot capture function."""
+    try:
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+
+        # Take screenshot
+        if element_selector:
+            element = await page.locator(element_selector).first
+            if not await element.is_visible():
+                return {
+                    "success": False,
+                    "error": f"Element '{element_selector}' is not visible",
+                }
+            screenshot_bytes = await element.screenshot()
+        else:
+            screenshot_bytes = await page.screenshot(full_page=full_page)
+
+        result: Dict[str, Any] = {
+            "success": True,
+            "screenshot_bytes": screenshot_bytes,
+            "timestamp": timestamp,
+        }
+
+        if save_screenshot:
+            screenshot_path = _build_screenshot_path(timestamp)
+            screenshot_path.parent.mkdir(parents=True, exist_ok=True)
+            with open(screenshot_path, "wb") as f:
+                f.write(screenshot_bytes)
+            result["screenshot_path"] = str(screenshot_path)
+
+            if group_id:
+                emit_success(
+                    f"Screenshot saved: {screenshot_path}", message_group=group_id
+                )
+
+        return result
+
+    except Exception as e:
+        return {"success": False, "error": str(e)}
+
+
+async def take_screenshot(
+    full_page: bool = False,
+    element_selector: Optional[str] = None,
+    save_screenshot: bool = True,
+) -> Union[ToolReturn, Dict[str, Any]]:
+    """Take a screenshot of the browser page.
+
+    Returns a ToolReturn with BinaryContent so multimodal models can
+    directly see and analyze the screenshot.
+
+    Args:
+        full_page: Whether to capture full page or just viewport.
+        element_selector: Optional selector to screenshot specific element.
+        save_screenshot: Whether to save the screenshot to disk.
+
+    Returns:
+        ToolReturn containing:
+            - return_value: Success message with screenshot path
+            - content: List with description and BinaryContent image
+            - metadata: Screenshot details (path, target, timestamp)
+        Or Dict with error info if failed.
+    """
+    target = element_selector or ("full_page" if full_page else "viewport")
+    group_id = generate_group_id("browser_screenshot", target)
+    emit_info(f"BROWSER SCREENSHOT 📷 target={target}", message_group=group_id)
+
+    try:
+        browser_manager = get_session_browser_manager()
+        page = await browser_manager.get_current_page()
+
+        if not page:
+            error_msg = "No active browser page. Navigate to a webpage first."
+            emit_error(error_msg, message_group=group_id)
+            return {"success": False, "error": error_msg}
+
+        result = await _capture_screenshot(
+            page,
+            full_page=full_page,
+            element_selector=element_selector,
+            save_screenshot=save_screenshot,
+            group_id=group_id,
+        )
+
+        if not result["success"]:
+            emit_error(result.get("error", "Screenshot failed"), message_group=group_id)
+            return {"success": False, "error": result.get("error")}
+
+        screenshot_path = result.get("screenshot_path", "(not saved)")
+
+        # Return as ToolReturn with BinaryContent so the model can SEE the image!
+        return ToolReturn(
+            return_value=f"Screenshot captured successfully. Saved to: {screenshot_path}",
+            content=[
+                f"Here's the browser screenshot ({target}):",
+                BinaryContent(
+                    data=result["screenshot_bytes"],
+                    media_type="image/png",
+                ),
+                "Please analyze what you see and describe any relevant details.",
+            ],
+            metadata={
+                "success": True,
+                "screenshot_path": screenshot_path,
+                "target": target,
+                "full_page": full_page,
+                "element_selector": element_selector,
+                "timestamp": time.time(),
+            },
+        )
+
+    except Exception as e:
+        error_msg = f"Screenshot failed: {str(e)}"
+        emit_error(error_msg, message_group=group_id)
+        return {"success": False, "error": error_msg}
+
+
+def register_take_screenshot_and_analyze(agent):
+    """Register the screenshot tool."""
+
+    @agent.tool
+    async def browser_screenshot_analyze(
+        context: RunContext,
+        full_page: bool = False,
+        element_selector: Optional[str] = None,
+    ) -> Union[ToolReturn, Dict[str, Any]]:
+        """
+        Take a screenshot of the browser page.
+
+        Returns the screenshot via ToolReturn with BinaryContent that you can
+        see directly. Use this to see what's displayed in the browser.
+
+        Args:
+            full_page: Capture full page (True) or just viewport (False).
+            element_selector: Optional CSS selector to screenshot specific element.
+
+        Returns:
+            ToolReturn with the screenshot image you can analyze, or error dict.
+        """
+        return await take_screenshot(
+            full_page=full_page,
+            element_selector=element_selector,
+        )

code_puppy/tools/browser/browser_scripts.py

@@ -0,0 +1,462 @@
+"""JavaScript execution and advanced page manipulation tools."""
+
+from typing import Any, Dict, Optional
+
+from pydantic_ai import RunContext
+
+from code_puppy.messaging import emit_error, emit_info, emit_success
+from code_puppy.tools.common import generate_group_id
+
+from .browser_manager import get_session_browser_manager
+
+
+async def execute_javascript(
+    script: str,
+    timeout: int = 30000,
+) -> Dict[str, Any]:
+    """Execute JavaScript code in the browser context."""
+    group_id = generate_group_id("browser_execute_js", script[:100])
+    emit_info(
+        f"BROWSER EXECUTE JS 📜 script='{script[:100]}{'...' if len(script) > 100 else ''}'",
+        message_group=group_id,
+    )
+    try:
+        browser_manager = get_session_browser_manager()
+        page = await browser_manager.get_current_page()
+
+        if not page:
+            return {"success": False, "error": "No active browser page available"}
+
+        # Execute JavaScript
+        # Note: page.evaluate() does NOT accept a timeout parameter
+        # The timeout arg to this function is kept for API compatibility but unused
+        result = await page.evaluate(script)
+
+        emit_success("JavaScript executed successfully", message_group=group_id)
+
+        return {"success": True, "script": script, "result": result}
+
+    except Exception as e:
+        emit_error(f"JavaScript execution failed: {str(e)}", message_group=group_id)
+        return {"success": False, "error": str(e), "script": script}
+
+
+async def scroll_page(
+    direction: str = "down",
+    amount: int = 3,
+    element_selector: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Scroll the page or a specific element."""
+    target = element_selector or "page"
+    group_id = generate_group_id("browser_scroll", f"{direction}_{amount}_{target}")
+    emit_info(
+        f"BROWSER SCROLL 📋 direction={direction} amount={amount} target='{target}'",
+        message_group=group_id,
+    )
+    try:
+        browser_manager = get_session_browser_manager()
+        page = await browser_manager.get_current_page()
+
+        if not page:
+            return {"success": False, "error": "No active browser page available"}
+
+        if element_selector:
+            # Scroll specific element
+            element = page.locator(element_selector).first
+            await element.scroll_into_view_if_needed()
+
+            # Get element's current scroll position and dimensions
+            scroll_info = await element.evaluate("""
+                el => {
+                    const rect = el.getBoundingClientRect();
+                    return {
+                        scrollTop: el.scrollTop,
+                        scrollLeft: el.scrollLeft,
+                        scrollHeight: el.scrollHeight,
+                        scrollWidth: el.scrollWidth,
+                        clientHeight: el.clientHeight,
+                        clientWidth: el.clientWidth
+                    };
+                }
+            """)
+
+            # Calculate scroll amount based on element size
+            scroll_amount = scroll_info["clientHeight"] * amount / 3
+
+            if direction.lower() == "down":
+                await element.evaluate(f"el => el.scrollTop += {scroll_amount}")
+            elif direction.lower() == "up":
+                await element.evaluate(f"el => el.scrollTop -= {scroll_amount}")
+            elif direction.lower() == "left":
+                await element.evaluate(f"el => el.scrollLeft -= {scroll_amount}")
+            elif direction.lower() == "right":
+                await element.evaluate(f"el => el.scrollLeft += {scroll_amount}")
+
+            target = f"element '{element_selector}'"
+
+        else:
+            # Scroll page
+            viewport_height = await page.evaluate("() => window.innerHeight")
+            scroll_amount = viewport_height * amount / 3
+
+            if direction.lower() == "down":
+                await page.evaluate(f"window.scrollBy(0, {scroll_amount})")
+            elif direction.lower() == "up":
+                await page.evaluate(f"window.scrollBy(0, -{scroll_amount})")
+            elif direction.lower() == "left":
+                await page.evaluate(f"window.scrollBy(-{scroll_amount}, 0)")
+            elif direction.lower() == "right":
+                await page.evaluate(f"window.scrollBy({scroll_amount}, 0)")
+
+            target = "page"
+
+        # Get current scroll position
+        scroll_pos = await page.evaluate("""
+            () => ({
+                x: window.pageXOffset,
+                y: window.pageYOffset
+            })
+        """)
+
+        emit_success(f"Scrolled {target} {direction}", message_group=group_id)
+
+        return {
+            "success": True,
+            "direction": direction,
+            "amount": amount,
+            "target": target,
+            "scroll_position": scroll_pos,
+        }
+
+    except Exception as e:
+        return {
+            "success": False,
+            "error": str(e),
+            "direction": direction,
+            "element_selector": element_selector,
+        }
+
+
+async def scroll_to_element(
+    selector: str,
+    timeout: int = 10000,
+) -> Dict[str, Any]:
+    """Scroll to bring an element into view."""
+    group_id = generate_group_id("browser_scroll_to_element", selector[:100])
+    emit_info(
+        f"BROWSER SCROLL TO ELEMENT 🎯 selector='{selector}'",
+        message_group=group_id,
+    )
+    try:
+        browser_manager = get_session_browser_manager()
+        page = await browser_manager.get_current_page()
+
+        if not page:
+            return {"success": False, "error": "No active browser page available"}
+
+        element = page.locator(selector).first
+        await element.wait_for(state="attached", timeout=timeout)
+        await element.scroll_into_view_if_needed()
+
+        # Check if element is now visible
+        is_visible = await element.is_visible()
+
+        emit_success(f"Scrolled to element: {selector}", message_group=group_id)
+
+        return {"success": True, "selector": selector, "visible": is_visible}
+
+    except Exception as e:
+        return {"success": False, "error": str(e), "selector": selector}
+
+
+async def set_viewport_size(
+    width: int,
+    height: int,
+) -> Dict[str, Any]:
+    """Set the viewport size."""
+    group_id = generate_group_id("browser_set_viewport", f"{width}x{height}")
+    emit_info(
+        f"BROWSER SET VIEWPORT 🖥️ size={width}x{height}",
+        message_group=group_id,
+    )
+    try:
+        browser_manager = get_session_browser_manager()
+        page = await browser_manager.get_current_page()
+
+        if not page:
+            return {"success": False, "error": "No active browser page available"}
+
+        await page.set_viewport_size({"width": width, "height": height})
+
+        emit_success(
+            f"Set viewport size to {width}x{height}",
+            message_group=group_id,
+        )
+
+        return {"success": True, "width": width, "height": height}
+
+    except Exception as e:
+        return {"success": False, "error": str(e), "width": width, "height": height}
+
+
+async def wait_for_element(
+    selector: str,
+    state: str = "visible",
+    timeout: int = 30000,
+) -> Dict[str, Any]:
+    """Wait for an element to reach a specific state."""
+    group_id = generate_group_id("browser_wait_for_element", f"{selector[:50]}_{state}")
+    emit_info(
+        f"BROWSER WAIT FOR ELEMENT ⏱️ selector='{selector}' state={state} timeout={timeout}ms",
+        message_group=group_id,
+    )
+    try:
+        browser_manager = get_session_browser_manager()
+        page = await browser_manager.get_current_page()
+
+        if not page:
+            return {"success": False, "error": "No active browser page available"}
+
+        element = page.locator(selector).first
+        await element.wait_for(state=state, timeout=timeout)
+
+        emit_success(f"Element {selector} is now {state}", message_group=group_id)
+
+        return {"success": True, "selector": selector, "state": state}
+
+    except Exception as e:
+        return {"success": False, "error": str(e), "selector": selector, "state": state}
+
+
+async def highlight_element(
+    selector: str,
+    color: str = "red",
+    timeout: int = 10000,
+) -> Dict[str, Any]:
+    """Highlight an element with a colored border."""
+    group_id = generate_group_id(
+        "browser_highlight_element", f"{selector[:50]}_{color}"
+    )
+    emit_info(
+        f"BROWSER HIGHLIGHT ELEMENT 🔦 selector='{selector}' color={color}",
+        message_group=group_id,
+    )
+    try:
+        browser_manager = get_session_browser_manager()
+        page = await browser_manager.get_current_page()
+
+        if not page:
+            return {"success": False, "error": "No active browser page available"}
+
+        element = page.locator(selector).first
+        await element.wait_for(state="visible", timeout=timeout)
+
+        # Add highlight style
+        highlight_script = f"""
+            el => {{
+                el.style.outline = '3px solid {color}';
+                el.style.outlineOffset = '2px';
+                el.style.backgroundColor = '{color}20';  // 20% opacity
+                el.setAttribute('data-highlighted', 'true');
+            }}
+        """
+
+        await element.evaluate(highlight_script)
+
+        emit_success(f"Highlighted element: {selector}", message_group=group_id)
+
+        return {"success": True, "selector": selector, "color": color}
+
+    except Exception as e:
+        return {"success": False, "error": str(e), "selector": selector}
+
+
+async def clear_highlights() -> Dict[str, Any]:
+    """Clear all element highlights."""
+    group_id = generate_group_id("browser_clear_highlights")
+    emit_info(
+        "BROWSER CLEAR HIGHLIGHTS 🧹",
+        message_group=group_id,
+    )
+    try:
+        browser_manager = get_session_browser_manager()
+        page = await browser_manager.get_current_page()
+
+        if not page:
+            return {"success": False, "error": "No active browser page available"}
+
+        # Remove all highlights
+        clear_script = """
+            () => {
+                const highlighted = document.querySelectorAll('[data-highlighted="true"]');
+                highlighted.forEach(el => {
+                    el.style.outline = '';
+                    el.style.outlineOffset = '';
+                    el.style.backgroundColor = '';
+                    el.removeAttribute('data-highlighted');
+                });
+                return highlighted.length;
+            }
+        """
+
+        count = await page.evaluate(clear_script)
+
+        emit_success(f"Cleared {count} highlights", message_group=group_id)
+
+        return {"success": True, "cleared_count": count}
+
+    except Exception as e:
+        return {"success": False, "error": str(e)}
+
+
+# Tool registration functions
+def register_execute_javascript(agent):
+    """Register the JavaScript execution tool."""
+
+    @agent.tool
+    async def browser_execute_js(
+        context: RunContext,
+        script: str,
+        timeout: int = 30000,
+    ) -> Dict[str, Any]:
+        """
+        Execute JavaScript code in the browser context.
+
+        Args:
+            script: JavaScript code to execute
+            timeout: Timeout in milliseconds
+
+        Returns:
+            Dict with execution results
+        """
+        return await execute_javascript(script, timeout)
+
+
+def register_scroll_page(agent):
+    """Register the scroll page tool."""
+
+    @agent.tool
+    async def browser_scroll(
+        context: RunContext,
+        direction: str = "down",
+        amount: int = 3,
+        element_selector: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """
+        Scroll the page or a specific element.
+
+        Args:
+            direction: Scroll direction (up, down, left, right)
+            amount: Scroll amount multiplier (1-10)
+            element_selector: Optional selector to scroll specific element
+
+        Returns:
+            Dict with scroll results
+        """
+        return await scroll_page(direction, amount, element_selector)
+
+
+def register_scroll_to_element(agent):
+    """Register the scroll to element tool."""
+
+    @agent.tool
+    async def browser_scroll_to_element(
+        context: RunContext,
+        selector: str,
+        timeout: int = 10000,
+    ) -> Dict[str, Any]:
+        """
+        Scroll to bring an element into view.
+
+        Args:
+            selector: CSS or XPath selector for the element
+            timeout: Timeout in milliseconds
+
+        Returns:
+            Dict with scroll results
+        """
+        return await scroll_to_element(selector, timeout)
+
+
+def register_set_viewport_size(agent):
+    """Register the viewport size tool."""
+
+    @agent.tool
+    async def browser_set_viewport(
+        context: RunContext,
+        width: int,
+        height: int,
+    ) -> Dict[str, Any]:
+        """
+        Set the browser viewport size.
+
+        Args:
+            width: Viewport width in pixels
+            height: Viewport height in pixels
+
+        Returns:
+            Dict with viewport size results
+        """
+        return await set_viewport_size(width, height)
+
+
+def register_wait_for_element(agent):
+    """Register the wait for element tool."""
+
+    @agent.tool
+    async def browser_wait_for_element(
+        context: RunContext,
+        selector: str,
+        state: str = "visible",
+        timeout: int = 30000,
+    ) -> Dict[str, Any]:
+        """
+        Wait for an element to reach a specific state.
+
+        Args:
+            selector: CSS or XPath selector for the element
+            state: State to wait for (visible, hidden, attached, detached)
+            timeout: Timeout in milliseconds
+
+        Returns:
+            Dict with wait results
+        """
+        return await wait_for_element(selector, state, timeout)
+
+
+def register_browser_highlight_element(agent):
+    """Register the element highlighting tool."""
+
+    @agent.tool
+    async def browser_highlight_element(
+        context: RunContext,
+        selector: str,
+        color: str = "red",
+        timeout: int = 10000,
+    ) -> Dict[str, Any]:
+        """
+        Highlight an element with a colored border for visual identification.
+
+        Args:
+            selector: CSS or XPath selector for the element
+            color: Highlight color (red, blue, green, yellow, etc.)
+            timeout: Timeout in milliseconds
+
+        Returns:
+            Dict with highlight results
+        """
+        return await highlight_element(selector, color, timeout)
+
+
+def register_browser_clear_highlights(agent):
+    """Register the clear highlights tool."""
+
+    @agent.tool
+    async def browser_clear_highlights(context: RunContext) -> Dict[str, Any]:
+        """
+        Clear all element highlights from the page.
+
+        Returns:
+            Dict with clear results
+        """
+        return await clear_highlights()