browsercontrol 0.1.0__py3-none-any.whl

browsercontrol/tools/interaction.py

@@ -0,0 +1,204 @@
+"""Interaction tools for browser control."""
+
+import logging
+from fastmcp import FastMCP
+from fastmcp.utilities.types import Image
+
+from browsercontrol.browser import browser, get_element_map
+
+logger = logging.getLogger(__name__)
+
+
+async def _get_screenshot_with_summary() -> tuple[Image, str]:
+    """Helper to get annotated screenshot with element summary."""
+    screenshot_bytes, elem_map = await browser.screenshot_with_som()
+    image = Image(data=screenshot_bytes, format="png")
+    
+    summary_lines = [f"Found {len(elem_map)} interactive elements:"]
+    for eid, elem in list(elem_map.items())[:30]:
+        desc = elem["text"][:40] if elem["text"] else elem["tag"]
+        summary_lines.append(f"  [{eid}] {elem['tag']} - {desc}")
+    
+    if len(elem_map) > 30:
+        summary_lines.append(f"  ... and {len(elem_map) - 30} more")
+    
+    return image, "\n".join(summary_lines)
+
+
+def register_interaction_tools(mcp: FastMCP) -> None:
+    """Register interaction tools with the MCP server."""
+    
+    @mcp.tool()
+    async def click(element_id: int) -> tuple[str, Image]:
+        """
+        Click on an element by its ID number shown in the screenshot.
+        
+        Args:
+            element_id: The number label shown on the element in the screenshot
+        """
+        try:
+            await browser.ensure_started()
+            elem_map = get_element_map()
+            
+            if element_id not in elem_map:
+                image, summary = await _get_screenshot_with_summary()
+                return f"Error: Element {element_id} not found. Valid IDs: {list(elem_map.keys())[:20]}\n\n{summary}", image
+            
+            elem = elem_map[element_id]
+            logger.info(f"Clicking element {element_id}: {elem['tag']} - {elem.get('text', '')[:30]}")
+            await browser.page.mouse.click(elem["centerX"], elem["centerY"])
+            await browser.page.wait_for_timeout(500)
+            
+            image, summary = await _get_screenshot_with_summary()
+            return f"Clicked element {element_id} ({elem['tag']}: {elem['text'][:30] if elem['text'] else 'no text'})\n\n{summary}", image
+            
+        except Exception as e:
+            logger.error(f"Click failed: {e}")
+            try:
+                image, summary = await _get_screenshot_with_summary()
+                return f"Error clicking element {element_id}: {e}\n\n{summary}", image
+            except Exception:
+                raise RuntimeError(f"Click failed: {e}")
+    
+    @mcp.tool()
+    async def click_at(x: int, y: int) -> tuple[str, Image]:
+        """
+        Click at specific x,y coordinates.
+        
+        Args:
+            x: X coordinate
+            y: Y coordinate
+        """
+        try:
+            await browser.ensure_started()
+            logger.info(f"Clicking at ({x}, {y})")
+            await browser.page.mouse.click(x, y)
+            await browser.page.wait_for_timeout(500)
+            image, summary = await _get_screenshot_with_summary()
+            return f"Clicked at ({x}, {y})\n\n{summary}", image
+        except Exception as e:
+            logger.error(f"Click at coordinates failed: {e}")
+            raise RuntimeError(f"Click at ({x}, {y}) failed: {e}")
+    
+    @mcp.tool()
+    async def type_text(element_id: int, text: str) -> tuple[str, Image]:
+        """
+        Type text into an input element by its ID number.
+        
+        Args:
+            element_id: The number label shown on the element
+            text: Text to type
+        """
+        try:
+            await browser.ensure_started()
+            elem_map = get_element_map()
+            
+            if element_id not in elem_map:
+                image, summary = await _get_screenshot_with_summary()
+                return f"Error: Element {element_id} not found.\n\n{summary}", image
+            
+            elem = elem_map[element_id]
+            logger.info(f"Typing into element {element_id}")
+            await browser.page.mouse.click(elem["centerX"], elem["centerY"])
+            await browser.page.keyboard.press("Control+a")
+            await browser.page.keyboard.type(text)
+            
+            image, summary = await _get_screenshot_with_summary()
+            return f"Typed '{text}' into element {element_id}\n\n{summary}", image
+            
+        except Exception as e:
+            logger.error(f"Type text failed: {e}")
+            raise RuntimeError(f"Type text failed: {e}")
+    
+    @mcp.tool()
+    async def press_key(key: str) -> tuple[str, Image]:
+        """
+        Press a keyboard key.
+        
+        Args:
+            key: Key to press (e.g., "Enter", "Tab", "Escape", "ArrowDown", "Backspace")
+        """
+        try:
+            await browser.ensure_started()
+            logger.info(f"Pressing key: {key}")
+            await browser.page.keyboard.press(key)
+            await browser.page.wait_for_timeout(300)
+            image, summary = await _get_screenshot_with_summary()
+            return f"Pressed key '{key}'\n\n{summary}", image
+        except Exception as e:
+            logger.error(f"Press key failed: {e}")
+            raise RuntimeError(f"Press key '{key}' failed: {e}")
+    
+    @mcp.tool()
+    async def hover(element_id: int) -> tuple[str, Image]:
+        """
+        Hover over an element by its ID number.
+        
+        Args:
+            element_id: The number label shown on the element
+        """
+        try:
+            await browser.ensure_started()
+            elem_map = get_element_map()
+            
+            if element_id not in elem_map:
+                image, summary = await _get_screenshot_with_summary()
+                return f"Error: Element {element_id} not found.\n\n{summary}", image
+            
+            elem = elem_map[element_id]
+            logger.info(f"Hovering over element {element_id}")
+            await browser.page.mouse.move(elem["centerX"], elem["centerY"])
+            await browser.page.wait_for_timeout(300)
+            
+            image, summary = await _get_screenshot_with_summary()
+            return f"Hovering over element {element_id}\n\n{summary}", image
+            
+        except Exception as e:
+            logger.error(f"Hover failed: {e}")
+            raise RuntimeError(f"Hover failed: {e}")
+    
+    @mcp.tool()
+    async def scroll_to_element(element_id: int) -> tuple[str, Image]:
+        """
+        Scroll to bring an element into view.
+        
+        Args:
+            element_id: The number label shown on the element
+        """
+        try:
+            await browser.ensure_started()
+            elem_map = get_element_map()
+            
+            if element_id not in elem_map:
+                image, summary = await _get_screenshot_with_summary()
+                return f"Error: Element {element_id} not found.\n\n{summary}", image
+            
+            elem = elem_map[element_id]
+            await browser.page.evaluate(f"window.scrollTo(0, {elem['y'] - 100})")
+            await browser.page.wait_for_timeout(300)
+            
+            image, summary = await _get_screenshot_with_summary()
+            return f"Scrolled to element {element_id}\n\n{summary}", image
+            
+        except Exception as e:
+            logger.error(f"Scroll to element failed: {e}")
+            raise RuntimeError(f"Scroll to element failed: {e}")
+    
+    @mcp.tool()
+    async def wait(seconds: float = 1.0) -> tuple[str, Image]:
+        """
+        Wait for a specified time (useful for pages with animations or loading).
+        
+        Args:
+            seconds: Time to wait in seconds (default: 1.0)
+        """
+        try:
+            await browser.ensure_started()
+            await browser.page.wait_for_timeout(int(seconds * 1000))
+            image, summary = await _get_screenshot_with_summary()
+            return f"Waited {seconds}s\n\n{summary}", image
+        except Exception as e:
+            logger.error(f"Wait failed: {e}")
+            raise RuntimeError(f"Wait failed: {e}")
+    
+    logger.debug("Registered interaction tools")

browsercontrol/tools/navigation.py

@@ -0,0 +1,163 @@
+"""Navigation tools for browser control."""
+
+import logging
+from fastmcp import FastMCP
+from fastmcp.utilities.types import Image
+
+from browsercontrol.browser import browser
+from browsercontrol.config import config
+
+logger = logging.getLogger(__name__)
+
+
+async def _get_screenshot_with_summary() -> tuple[Image, str]:
+    """Helper to get annotated screenshot with element summary."""
+    screenshot_bytes, elem_map = await browser.screenshot_with_som()
+    image = Image(data=screenshot_bytes, format="png")
+    
+    summary_lines = [f"Found {len(elem_map)} interactive elements:"]
+    for eid, elem in list(elem_map.items())[:30]:
+        desc = elem["text"][:40] if elem["text"] else elem["tag"]
+        summary_lines.append(f"  [{eid}] {elem['tag']} - {desc}")
+    
+    if len(elem_map) > 30:
+        summary_lines.append(f"  ... and {len(elem_map) - 30} more")
+    
+    return image, "\n".join(summary_lines)
+
+
+def register_navigation_tools(mcp: FastMCP) -> None:
+    """Register navigation tools with the MCP server."""
+    
+    @mcp.tool()
+    async def navigate_to(url: str) -> tuple[str, Image]:
+        """
+        Navigate to a URL. Returns an annotated screenshot with numbered interactive elements.
+        
+        Args:
+            url: The URL to navigate to
+            
+        Returns:
+            Element summary and annotated screenshot
+        """
+        try:
+            await browser.ensure_started()
+            logger.info(f"Navigating to: {url}")
+            
+            try:
+                await browser.page.goto(url, wait_until="domcontentloaded", timeout=config.timeout_ms)
+            except Exception as e:
+                # Handle localhost vs 127.0.0.1 resolution issues
+                if "ERR_CONNECTION_REFUSED" in str(e) and "localhost" in url:
+                    fallback_url = url.replace("localhost", "127.0.0.1")
+                    logger.info(f"Navigation to localhost failed, retrying with: {fallback_url}")
+                    await browser.page.goto(fallback_url, wait_until="domcontentloaded", timeout=config.timeout_ms)
+                    url = fallback_url  # Update for success message
+                else:
+                    raise e
+
+            await browser.page.wait_for_timeout(500)
+            image, summary = await _get_screenshot_with_summary()
+            return f"Navigated to {url}\n\n{summary}", image
+        except Exception as e:
+            logger.error(f"Navigation failed: {e}")
+            try:
+                image, summary = await _get_screenshot_with_summary()
+                return f"Error navigating to {url}: {e}\n\n{summary}", image
+            except Exception:
+                raise RuntimeError(f"Navigation failed: {e}")
+    
+    @mcp.tool()
+    async def go_back() -> tuple[str, Image]:
+        """Navigate back to the previous page."""
+        try:
+            await browser.ensure_started()
+            await browser.page.go_back(timeout=config.timeout_ms)
+            await browser.page.wait_for_timeout(500)
+            image, summary = await _get_screenshot_with_summary()
+            return f"Navigated back\n\n{summary}", image
+        except Exception as e:
+            logger.error(f"Go back failed: {e}")
+            image, summary = await _get_screenshot_with_summary()
+            return f"Error going back: {e}\n\n{summary}", image
+    
+    @mcp.tool()
+    async def go_forward() -> tuple[str, Image]:
+        """Navigate forward to the next page."""
+        try:
+            await browser.ensure_started()
+            await browser.page.go_forward(timeout=config.timeout_ms)
+            await browser.page.wait_for_timeout(500)
+            image, summary = await _get_screenshot_with_summary()
+            return f"Navigated forward\n\n{summary}", image
+        except Exception as e:
+            logger.error(f"Go forward failed: {e}")
+            image, summary = await _get_screenshot_with_summary()
+            return f"Error going forward: {e}\n\n{summary}", image
+    
+    @mcp.tool()
+    async def refresh_page() -> tuple[str, Image]:
+        """Refresh the current page."""
+        try:
+            await browser.ensure_started()
+            await browser.page.reload(timeout=config.timeout_ms)
+            await browser.page.wait_for_timeout(500)
+            image, summary = await _get_screenshot_with_summary()
+            return f"Page refreshed\n\n{summary}", image
+        except Exception as e:
+            logger.error(f"Refresh failed: {e}")
+            image, summary = await _get_screenshot_with_summary()
+            return f"Error refreshing: {e}\n\n{summary}", image
+    
+    @mcp.tool()
+    async def scroll(
+        direction: str = "down",
+        amount: str = "medium"
+    ) -> tuple[str, Image]:
+        """
+        Scroll the page.
+        
+        Args:
+            direction: "up", "down", "left", or "right"
+            amount: "small" (100px), "medium" (400px), "large" (800px), 
+                    "page" (full viewport), "top", "bottom", or pixels like "500"
+        """
+        try:
+            await browser.ensure_started()
+            
+            amount_map = {"small": 100, "medium": 400, "large": 800, "page": 720}
+            
+            if amount == "top":
+                await browser.page.evaluate("window.scrollTo(0, 0)")
+                image, summary = await _get_screenshot_with_summary()
+                return f"Scrolled to top\n\n{summary}", image
+            
+            if amount == "bottom":
+                await browser.page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+                image, summary = await _get_screenshot_with_summary()
+                return f"Scrolled to bottom\n\n{summary}", image
+            
+            pixels = amount_map.get(amount)
+            if pixels is None:
+                try:
+                    pixels = int(amount)
+                except ValueError:
+                    pixels = 400
+            
+            if direction == "up":
+                await browser.page.evaluate(f"window.scrollBy(0, -{pixels})")
+            elif direction == "down":
+                await browser.page.evaluate(f"window.scrollBy(0, {pixels})")
+            elif direction == "left":
+                await browser.page.evaluate(f"window.scrollBy(-{pixels}, 0)")
+            elif direction == "right":
+                await browser.page.evaluate(f"window.scrollBy({pixels}, 0)")
+            
+            image, summary = await _get_screenshot_with_summary()
+            return f"Scrolled {direction} by {pixels}px\n\n{summary}", image
+            
+        except Exception as e:
+            logger.error(f"Scroll failed: {e}")
+            raise RuntimeError(f"Scroll failed: {e}")
+    
+    logger.debug("Registered navigation tools")

browsercontrol/tools/recording.py

@@ -0,0 +1,221 @@
+"""Session recording tools for browser control."""
+
+import logging
+import os
+from pathlib import Path
+from datetime import datetime
+
+from fastmcp import FastMCP
+from fastmcp.utilities.types import Image
+
+from browsercontrol.browser import browser
+from browsercontrol.config import config
+
+logger = logging.getLogger(__name__)
+
+# Recording state
+_recording_path: Path | None = None
+_recording_active: bool = False
+
+
+def register_recording_tools(mcp: FastMCP) -> None:
+    """Register session recording tools with the MCP server."""
+    
+    @mcp.tool()
+    async def start_recording(name: str = "") -> tuple[str, Image]:
+        """
+        Start recording the browser session as a video.
+        The video will be saved when stop_recording is called.
+        
+        Args:
+            name: Optional name for the recording (default: timestamp)
+            
+        Returns:
+            Status message and screenshot
+        """
+        global _recording_path, _recording_active
+        
+        try:
+            await browser.ensure_started()
+            
+            if _recording_active:
+                screenshot_bytes, elem_map = await browser.screenshot_with_som()
+                image = Image(data=screenshot_bytes, format="png")
+                return "Recording already in progress. Call stop_recording() first.", image
+            
+            # Create recordings directory
+            recordings_dir = config.user_data_dir.parent / "recordings"
+            recordings_dir.mkdir(parents=True, exist_ok=True)
+            
+            # Generate filename
+            if not name:
+                name = datetime.now().strftime("%Y%m%d_%H%M%S")
+            
+            _recording_path = recordings_dir / f"{name}.webm"
+            
+            # Start video recording via CDP
+            cdp = await browser.page.context.new_cdp_session(browser.page)
+            await cdp.send("Page.startScreencast", {
+                "format": "png",
+                "quality": 80,
+                "everyNthFrame": 2
+            })
+            
+            _recording_active = True
+            logger.info(f"Started recording: {_recording_path}")
+            
+            screenshot_bytes, elem_map = await browser.screenshot_with_som()
+            image = Image(data=screenshot_bytes, format="png")
+            return f"🔴 Recording started: {_recording_path.name}\n\nCall stop_recording() when done.", image
+            
+        except Exception as e:
+            logger.error(f"Start recording failed: {e}")
+            # Fallback: use Playwright's built-in tracing
+            try:
+                await browser.page.context.tracing.start(screenshots=True, snapshots=True)
+                _recording_active = True
+                
+                screenshot_bytes, elem_map = await browser.screenshot_with_som()
+                image = Image(data=screenshot_bytes, format="png")
+                return f"🔴 Recording started (trace mode)\n\nCall stop_recording() when done.", image
+            except Exception as e2:
+                raise RuntimeError(f"Failed to start recording: {e2}")
+    
+    @mcp.tool()
+    async def stop_recording() -> tuple[str, Image]:
+        """
+        Stop recording and save the session.
+        
+        Returns:
+            Path to saved recording and screenshot
+        """
+        global _recording_path, _recording_active
+        
+        try:
+            await browser.ensure_started()
+            
+            if not _recording_active:
+                screenshot_bytes, elem_map = await browser.screenshot_with_som()
+                image = Image(data=screenshot_bytes, format="png")
+                return "No recording in progress. Call start_recording() first.", image
+            
+            # Stop tracing and save
+            recordings_dir = config.user_data_dir.parent / "recordings"
+            recordings_dir.mkdir(parents=True, exist_ok=True)
+            
+            if _recording_path is None:
+                _recording_path = recordings_dir / f"recording_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip"
+            
+            trace_path = _recording_path.with_suffix(".zip")
+            
+            try:
+                await browser.page.context.tracing.stop(path=str(trace_path))
+                logger.info(f"Recording saved: {trace_path}")
+                result_path = trace_path
+            except Exception:
+                # If tracing wasn't active, just note it
+                result_path = _recording_path
+            
+            _recording_active = False
+            _recording_path = None
+            
+            screenshot_bytes, elem_map = await browser.screenshot_with_som()
+            image = Image(data=screenshot_bytes, format="png")
+            return f"⏹️ Recording saved: {result_path}\n\nView with: npx playwright show-trace {result_path}", image
+            
+        except Exception as e:
+            _recording_active = False
+            logger.error(f"Stop recording failed: {e}")
+            raise RuntimeError(f"Failed to stop recording: {e}")
+    
+    @mcp.tool()
+    async def take_snapshot(name: str = "") -> tuple[str, Image]:
+        """
+        Take a named snapshot (screenshot + HTML) for later reference.
+        
+        Args:
+            name: Optional name for the snapshot (default: timestamp)
+            
+        Returns:
+            Path to saved snapshot and screenshot
+        """
+        try:
+            await browser.ensure_started()
+            
+            # Create snapshots directory
+            snapshots_dir = config.user_data_dir.parent / "snapshots"
+            snapshots_dir.mkdir(parents=True, exist_ok=True)
+            
+            # Generate filename
+            if not name:
+                name = datetime.now().strftime("%Y%m%d_%H%M%S")
+            
+            # Save screenshot
+            screenshot_path = snapshots_dir / f"{name}.png"
+            await browser.page.screenshot(path=str(screenshot_path))
+            
+            # Save HTML
+            html_path = snapshots_dir / f"{name}.html"
+            html_content = await browser.page.content()
+            html_path.write_text(html_content)
+            
+            # Save URL
+            url_path = snapshots_dir / f"{name}.url"
+            url_path.write_text(browser.page.url)
+            
+            logger.info(f"Snapshot saved: {screenshot_path}")
+            
+            screenshot_bytes, elem_map = await browser.screenshot_with_som()
+            image = Image(data=screenshot_bytes, format="png")
+            return f"📸 Snapshot saved:\n  - {screenshot_path.name}\n  - {html_path.name}\n  - {url_path.name}", image
+            
+        except Exception as e:
+            logger.error(f"Take snapshot failed: {e}")
+            raise RuntimeError(f"Failed to take snapshot: {e}")
+    
+    @mcp.tool()
+    async def list_recordings() -> tuple[str, Image]:
+        """
+        List all saved recordings and snapshots.
+        
+        Returns:
+            List of recordings and screenshot
+        """
+        try:
+            await browser.ensure_started()
+            
+            base_dir = config.user_data_dir.parent
+            recordings_dir = base_dir / "recordings"
+            snapshots_dir = base_dir / "snapshots"
+            
+            lines = ["📁 Saved Sessions:\n"]
+            
+            # List recordings
+            if recordings_dir.exists():
+                recordings = list(recordings_dir.glob("*"))
+                if recordings:
+                    lines.append("Recordings:")
+                    for r in sorted(recordings)[-10:]:  # Last 10
+                        size = r.stat().st_size // 1024
+                        lines.append(f"  📹 {r.name} ({size}KB)")
+            
+            # List snapshots
+            if snapshots_dir.exists():
+                snapshots = list(snapshots_dir.glob("*.png"))
+                if snapshots:
+                    lines.append("\nSnapshots:")
+                    for s in sorted(snapshots)[-10:]:  # Last 10
+                        lines.append(f"  📸 {s.stem}")
+            
+            if len(lines) == 1:
+                lines.append("No recordings or snapshots found.")
+            
+            screenshot_bytes, elem_map = await browser.screenshot_with_som()
+            image = Image(data=screenshot_bytes, format="png")
+            return "\n".join(lines), image
+            
+        except Exception as e:
+            logger.error(f"List recordings failed: {e}")
+            raise RuntimeError(f"Failed to list recordings: {e}")
+    
+    logger.debug("Registered recording tools")