meshagent-computers 0.6.7__py3-none-any.whl

meshagent/computers/__init__.py

@@ -0,0 +1,21 @@
+from .computer import Computer
+from .browserbase import BrowserbaseBrowser
+from .local_playwright import LocalPlaywrightComputer
+from .docker import DockerComputer
+from .scrapybara import ScrapybaraBrowser, ScrapybaraUbuntu
+from .operator import Operator
+from .agent import ComputerAgent
+from .version import __version__
+
+
+__all__ = [
+    Computer,
+    BrowserbaseBrowser,
+    LocalPlaywrightComputer,
+    DockerComputer,
+    ScrapybaraBrowser,
+    ScrapybaraUbuntu,
+    Operator,
+    ComputerAgent,
+    __version__,
+]

meshagent/computers/agent.py

@@ -0,0 +1,229 @@
+from meshagent.agents import LLMAdapter
+from meshagent.tools import Tool, Toolkit, ToolContext
+from meshagent.computers import Computer, Operator, BrowserbaseBrowser
+from meshagent.agents.chat import ChatBot, ChatThreadContext
+from meshagent.api import RemoteParticipant
+from meshagent.openai.tools.responses_adapter import OpenAIResponsesTool
+
+from typing import Optional, Type, Callable
+import base64
+import logging
+
+logger = logging.getLogger("computer")
+logger.setLevel(logging.WARN)
+
+
+class ComputerToolkit(Toolkit):
+    def __init__(
+        self, name: str, computer: Computer, operator: Operator, tools: list[Tool]
+    ):
+        super().__init__(name=name, tools=tools)
+        self.computer = computer
+        self.operator = operator
+        self.started = False
+        self._starting = None
+
+    async def ensure_started(self):
+        self.started = False
+
+        if not self.started:
+            self.started = True
+            await self.computer.__aenter__()
+
+
+def make_computer_toolkit(
+    *,
+    operator_cls: Type[Operator],
+    computer_cls: Type[Computer],
+    render_screen: Callable[[bytes], None],
+):
+    operator = operator_cls()
+    computer = computer_cls()
+
+    class ComputerTool(OpenAIResponsesTool):
+        def __init__(
+            self,
+            *,
+            operator: Operator,
+            computer: Computer,
+            title="computer_call",
+            description="handle computer calls from computer use preview",
+            rules=[],
+            thumbnail_url=None,
+        ):
+            super().__init__(
+                name="computer_call",
+                # TODO: give a correct schema
+                title=title,
+                description=description,
+                rules=rules,
+                thumbnail_url=thumbnail_url,
+            )
+            self.computer = computer
+
+        def get_open_ai_tool_definitions(self) -> list[dict]:
+            return [
+                {
+                    "type": "computer_use_preview",
+                    "display_width": self.computer.dimensions[0],
+                    "display_height": self.computer.dimensions[1],
+                    "environment": self.computer.environment,
+                }
+            ]
+
+        def get_open_ai_output_handlers(self):
+            return {"computer_call": self.handle_computer_call}
+
+        async def handle_computer_call(self, context: ToolContext, **arguments):
+            outputs = await operator.play(computer=self.computer, item=arguments)
+            for output in outputs:
+                if output["type"] == "computer_call_output":
+                    if output["output"] is not None:
+                        if output["output"]["type"] == "input_image":
+                            b64: str = output["output"]["image_url"]
+                            image_data_b64 = b64.split(",", 1)
+
+                            image_bytes = base64.b64decode(image_data_b64[1])
+                            render_screen(image_bytes)
+
+            nonlocal computer_toolkit
+            if len(computer_toolkit.tools) == 1:
+                # HACK: after looking at the page, add the other tools,
+                # if we add these first then the computer-use-preview mode fails if it calls them before using the computer
+                computer_toolkit.tools.extend(
+                    [
+                        ScreenshotTool(computer=computer),
+                        GotoURL(computer=computer),
+                    ]
+                )
+            return outputs[0]
+
+    class ScreenshotTool(Tool):
+        def __init__(self, computer: Computer):
+            self.computer = computer
+
+            super().__init__(
+                name="screenshot",
+                # TODO: give a correct schema
+                input_schema={
+                    "additionalProperties": False,
+                    "type": "object",
+                    "required": ["full_page", "save_path"],
+                    "properties": {
+                        "full_page": {"type": "boolean"},
+                        "save_path": {
+                            "type": "string",
+                            "description": "a file path to save the screenshot to (should end with .png)",
+                        },
+                    },
+                },
+                description="take a screenshot of the current page",
+            )
+
+        async def execute(self, context: ToolContext, save_path: str, full_page: bool):
+            screenshot_bytes = await self.computer.screenshot_bytes(full_page=full_page)
+            handle = await context.room.storage.open(path=save_path, overwrite=True)
+            await context.room.storage.write(handle=handle, data=screenshot_bytes)
+            await context.room.storage.close(handle=handle)
+
+            return f"saved screenshot to {save_path}"
+
+    class GotoURL(Tool):
+        def __init__(self, computer: Computer):
+            self.computer = computer
+
+            super().__init__(
+                name="goto",
+                description="goes to a specific URL. Make sure it starts with http:// or https://",
+                # TODO: give a correct schema
+                input_schema={
+                    "additionalProperties": False,
+                    "type": "object",
+                    "required": ["url"],
+                    "properties": {
+                        "url": {
+                            "type": "string",
+                            "description": "Fully qualified URL to navigate to.",
+                        }
+                    },
+                },
+            )
+
+        async def execute(self, context: ToolContext, url: str):
+            if not url.startswith("https://") and not url.startswith("http://"):
+                url = "https://" + url
+
+            await self.computer.goto(url)
+
+            render_screen(await self.computer.screenshot_bytes(full_page=False))
+
+    computer_tool = ComputerTool(computer=computer, operator=operator)
+
+    computer_toolkit = ComputerToolkit(
+        name="meshagent.openai.computer",
+        computer=computer,
+        operator=operator,
+        tools=[computer_tool],
+    )
+
+    return computer_toolkit
+
+
+class ComputerAgent(ChatBot):
+    def __init__(
+        self,
+        *,
+        name,
+        title=None,
+        description=None,
+        requires=None,
+        labels=None,
+        computer_cls: Type[Computer] = BrowserbaseBrowser,
+        operator_cls: Type[Operator] = Operator,
+        rules: Optional[list[str]] = None,
+        llm_adapter: Optional[LLMAdapter] = None,
+        toolkits: list[Toolkit] = None,
+    ):
+        if rules is None:
+            rules = [
+                "if asked to go to a URL, you MUST use the goto function to go to the url if it is available",
+                "after going directly to a URL, the screen will change so you should take a look at it to know what to do next",
+            ]
+        super().__init__(
+            name=name,
+            title=title,
+            description=description,
+            requires=requires,
+            labels=labels,
+            llm_adapter=llm_adapter,
+            toolkits=toolkits,
+            rules=rules,
+        )
+        self.computer_cls = computer_cls
+        self.operator_cls = operator_cls
+
+    async def get_thread_toolkits(
+        self, *, thread_context: ChatThreadContext, participant: RemoteParticipant
+    ):
+        toolkits = await super().get_thread_toolkits(
+            thread_context=thread_context, participant=participant
+        )
+
+        def render_screen(image_bytes: bytes):
+            for participant in thread_context.participants:
+                self.room.messaging.send_message_nowait(
+                    to=participant,
+                    type="computer_screen",
+                    message={},
+                    attachment=image_bytes,
+                )
+
+        computer_toolkit = make_computer_toolkit(
+            operator_cls=self.operator_cls,
+            computer_cls=self.computer_cls,
+            render_screen=render_screen,
+        )
+
+        await computer_toolkit.ensure_started()
+
+        return [computer_toolkit, *toolkits]

meshagent/computers/base_playwright.py

@@ -0,0 +1,173 @@
+import time
+import base64
+from typing import List, Dict, Literal
+from playwright.async_api import async_playwright, Browser, Page, Route, Request
+from meshagent.computers.utils import check_blocklisted_url
+
+# Optional: key mapping if your model uses "CUA" style keys
+CUA_KEY_TO_PLAYWRIGHT_KEY = {
+    "/": "Divide",
+    "\\": "Backslash",
+    "alt": "Alt",
+    "arrowdown": "ArrowDown",
+    "arrowleft": "ArrowLeft",
+    "arrowright": "ArrowRight",
+    "arrowup": "ArrowUp",
+    "backspace": "Backspace",
+    "capslock": "CapsLock",
+    "cmd": "Meta",
+    "ctrl": "Control",
+    "delete": "Delete",
+    "end": "End",
+    "enter": "Enter",
+    "esc": "Escape",
+    "home": "Home",
+    "insert": "Insert",
+    "option": "Alt",
+    "pagedown": "PageDown",
+    "pageup": "PageUp",
+    "shift": "Shift",
+    "space": " ",
+    "super": "Meta",
+    "tab": "Tab",
+    "win": "Meta",
+}
+
+
+class BasePlaywrightComputer:
+    """
+    Abstract base for Playwright-based computers:
+
+      - Subclasses override `_get_browser_and_page()` to do local or remote connection,
+        returning (Browser, Page).
+      - This base class handles context creation (`__enter__`/`__exit__`),
+        plus standard "Computer" actions like click, scroll, etc.
+      - We also have extra browser actions: `goto(url)` and `back()`.
+    """
+
+    environment: Literal["browser"] = "browser"
+    dimensions = (1024, 768)
+
+    def __init__(self):
+        self._playwright = None
+        self._browser: Browser | None = None
+        self._page: Page | None = None
+
+    async def __aenter__(self):
+        # Start Playwright and call the subclass hook for getting browser/page
+        self._context = async_playwright()
+        self._playwright = await self._context.__aenter__()
+        self._browser, self._page = await self._get_browser_and_page()
+
+        # Set up network interception to flag URLs matching domains in BLOCKED_DOMAINS
+        async def handle_route(route: Route, request: Request):
+            url = request.url
+            if check_blocklisted_url(url):
+                print(f"Flagging blocked domain: {url}")
+                await route.abort()
+            else:
+                await route.continue_()
+
+        await self._page.route("**/*", handle_route)
+
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self._browser:
+            await self._browser.close()
+        if self._playwright:
+            await self._context.__aexit__(exc_type, exc_val, exc_tb)
+
+    async def ensure_page(self):
+        # After a timeout, we might loose our browser
+        if self._page is None or not self._browser.is_connected:
+            self._browser, self._page = await self._get_browser_and_page()
+
+    # --- Common "Computer" actions ---
+
+    async def screenshot_bytes(self, full_page: bool = False) -> bytes:
+        await self.ensure_page()
+        png_bytes = await self._page.screenshot(full_page=full_page)
+        return png_bytes
+
+    async def screenshot(self, full_page: bool = False) -> str:
+        await self.ensure_page()
+        png_bytes = await self.screenshot_bytes(full_page=full_page)
+        return base64.b64encode(png_bytes).decode("utf-8")
+
+    async def click(self, x: int, y: int, button: str = "left") -> None:
+        await self.ensure_page()
+        match button:
+            case "back":
+                await self.back()
+            case "forward":
+                await self.forward()
+            case "wheel":
+                await self._page.mouse.wheel(x, y)
+            case _:
+                button_mapping = {"left": "left", "right": "right"}
+                button_type = button_mapping.get(button, "left")
+                await self._page.mouse.click(x, y, button=button_type)
+
+    async def double_click(self, x: int, y: int) -> None:
+        await self.ensure_page()
+        await self._page.mouse.dblclick(x, y)
+
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        await self.ensure_page()
+        await self._page.mouse.move(x, y)
+        await self._page.evaluate(f"window.scrollBy({scroll_x}, {scroll_y})")
+
+    async def type(self, text: str) -> None:
+        await self.ensure_page()
+        await self._page.keyboard.type(text)
+
+    async def wait(self, ms: int = 1000) -> None:
+        await self.ensure_page()
+        time.sleep(ms / 1000)
+
+    async def move(self, x: int, y: int) -> None:
+        await self.ensure_page()
+        await self._page.mouse.move(x, y)
+
+    async def keypress(self, keys: List[str]) -> None:
+        await self.ensure_page()
+        for key in keys:
+            mapped_key = CUA_KEY_TO_PLAYWRIGHT_KEY.get(key.lower(), key)
+            await self._page.keyboard.press(mapped_key)
+
+    async def drag(self, path: List[Dict[str, int]]) -> None:
+        await self.ensure_page()
+        if not path:
+            return
+
+        await self._page.mouse.move(path[0]["x"], path[0]["y"])
+        await self._page.mouse.down()
+        for point in path[1:]:
+            await self._page.mouse.move(point["x"], point["y"])
+        await self._page.mouse.up()
+
+    async def get_current_url(self) -> str:
+        await self.ensure_page()
+        return self._page.url
+
+    # --- Extra browser-oriented actions ---
+    async def goto(self, url: str) -> None:
+        await self.ensure_page()
+        try:
+            return await self._page.goto(url)
+        except Exception as e:
+            print(f"Error navigating to {url}: {e}")
+
+    async def back(self) -> None:
+        await self.ensure_page()
+        return await self._page.go_back()
+
+    async def forward(self) -> None:
+        await self.ensure_page()
+        return await self._page.go_forward()
+
+    # --- Subclass hook ---
+    async def _get_browser_and_page(self) -> tuple[Browser, Page]:
+        """Subclasses must implement, returning (Browser, Page)."""
+        raise NotImplementedError

meshagent/computers/browserbase.py

@@ -0,0 +1,196 @@
+import os
+from typing import Tuple
+from playwright.async_api import Browser, Page, Error as PlaywrightError
+from .base_playwright import BasePlaywrightComputer
+from browserbase import AsyncBrowserbase
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+class BrowserbaseBrowser(BasePlaywrightComputer):
+    """
+    Browserbase is a headless browser platform that offers a remote browser API. You can use it to control thousands of browsers from anywhere.
+    You can find more information about Browserbase at https://www.browserbase.com/computer-use or view our OpenAI CUA Quickstart at https://docs.browserbase.com/integrations/openai-cua/introduction.
+
+    IMPORTANT: This Browserbase computer requires the use of the `goto` tool defined in playwright_with_custom_functions.py.
+    Make sure to include this tool in your configuration when using the Browserbase computer.
+    """
+
+    def __init__(
+        self,
+        width: int = 1024,
+        height: int = 768,
+        region: str = "us-west-2",
+        proxy: bool = False,
+        virtual_mouse: bool = True,
+        ad_blocker: bool = False,
+    ):
+        """
+        Initialize the Browserbase instance. Additional configuration options for features such as persistent cookies, ad blockers, file downloads and more can be found in the Browserbase API documentation: https://docs.browserbase.com/reference/api/create-a-session
+
+        Args:
+            width (int): The width of the browser viewport. Default is 1024.
+            height (int): The height of the browser viewport. Default is 768.
+            region (str): The region for the Browserbase session. Default is "us-west-2". Pick a region close to you for better performance. https://docs.browserbase.com/guides/multi-region
+            proxy (bool): Whether to use a proxy for the session. Default is False. Turn on proxies if you're browsing is frequently interrupted. https://docs.browserbase.com/features/proxies
+            virtual_mouse (bool): Whether to enable the virtual mouse cursor. Default is True.
+            ad_blocker (bool): Whether to enable the built-in ad blocker. Default is False.
+        """
+        super().__init__()
+        self.bb = AsyncBrowserbase(api_key=os.getenv("BROWSERBASE_API_KEY"))
+        self.project_id = os.getenv("BROWSERBASE_PROJECT_ID")
+        self.session = None
+        self.dimensions = (width, height)
+        self.region = region
+        self.proxy = proxy
+        self.virtual_mouse = virtual_mouse
+        self.ad_blocker = ad_blocker
+
+    async def _get_browser_and_page(self) -> Tuple[Browser, Page]:
+        """
+        Create a Browserbase session and connect to it.
+
+        Returns:
+            Tuple[Browser, Page]: A tuple containing the connected browser and page objects.
+        """
+        # Create a session on Browserbase with specified parameters
+        width, height = self.dimensions
+        session_params = {
+            "project_id": self.project_id,
+            "browser_settings": {
+                "viewport": {"width": width, "height": height},
+                "blockAds": self.ad_blocker,
+            },
+            "region": self.region,
+            "proxies": self.proxy,
+        }
+        self.session = await self.bb.sessions.create(**session_params)
+
+        # Print the live session URL
+        print(
+            f"Watch and control this browser live at https://www.browserbase.com/sessions/{self.session.id}"
+        )
+
+        # Connect to the remote session
+        browser = await self._playwright.chromium.connect_over_cdp(
+            self.session.connect_url, timeout=60000
+        )
+        context = browser.contexts[0]
+
+        # Add event listeners for page creation and closure
+        context.on("page", self._handle_new_page)
+
+        # Only add the init script if virtual_mouse is True
+        if self.virtual_mouse:
+            await context.add_init_script("""
+            // Only run in the top frame
+            if (window.self === window.top) {
+                function initCursor() {
+                    const CURSOR_ID = '__cursor__';
+
+                    // Check if cursor element already exists
+                    if (document.getElementById(CURSOR_ID)) return;
+
+                    const cursor = document.createElement('div');
+                    cursor.id = CURSOR_ID;
+                    Object.assign(cursor.style, {
+                        position: 'fixed',
+                        top: '0px',
+                        left: '0px',
+                        width: '20px',
+                        height: '20px',
+                        backgroundImage: 'url("data:image/svg+xml;utf8,<svg xmlns=\\'http://www.w3.org/2000/svg\\' viewBox=\\'0 0 24 24\\' fill=\\'black\\' stroke=\\'white\\' stroke-width=\\'1\\' stroke-linejoin=\\'round\\' stroke-linecap=\\'round\\'><polygon points=\\'2,2 2,22 8,16 14,22 17,19 11,13 20,13\\'/></svg>")',
+                        backgroundSize: 'cover',
+                        pointerEvents: 'none',
+                        zIndex: '99999',
+                        transform: 'translate(-2px, -2px)',
+                    });
+
+                    document.body.appendChild(cursor);
+
+                    document.addEventListener("mousemove", (e) => {
+                        cursor.style.top = e.clientY + "px";
+                        cursor.style.left = e.clientX + "px";
+                    });
+                }
+
+                // Use requestAnimationFrame for early execution
+                requestAnimationFrame(function checkBody() {
+                    if (document.body) {
+                        initCursor();
+                    } else {
+                        requestAnimationFrame(checkBody);
+                    }
+                });
+            }
+            """)
+
+        page = context.pages[0]
+        page.on("close", self._handle_page_close)
+
+        await page.goto("https://google.com")
+
+        return browser, page
+
+    async def _handle_new_page(self, page: Page):
+        """Handle the creation of a new page."""
+        print("New page created")
+        self._page = page
+        page.on("close", self._handle_page_close)
+
+    async def _handle_page_close(self, page: Page):
+        """Handle the closure of a page."""
+        print("Page closed")
+        if self._page == page:
+            if self._browser.contexts[0].pages:
+                self._page = self._browser.contexts[0].pages[-1]
+            else:
+                print("Warning: All pages have been closed.")
+                self._page = None
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """
+        Clean up resources when exiting the context manager.
+
+        Args:
+            exc_type: The type of the exception that caused the context to be exited.
+            exc_val: The exception instance that caused the context to be exited.
+            exc_tb: A traceback object encapsulating the call stack at the point where the exception occurred.
+        """
+        if self._page:
+            await self._page.close()
+        if self._browser:
+            await self._browser.close()
+        if self._playwright:
+            await self._playwright.stop()
+
+        if self.session:
+            print(
+                f"Session completed. View replay at https://browserbase.com/sessions/{self.session.id}"
+            )
+
+    async def screenshot(self) -> str:
+        await self.ensure_page()
+
+        """
+        Capture a screenshot of the current viewport using CDP.
+
+        Returns:
+            str: A base64 encoded string of the screenshot.
+        """
+        try:
+            # Get CDP session from the page
+            cdp_session = await self._page.context.new_cdp_session(self._page)
+
+            # Capture screenshot using CDP
+            result = await cdp_session.send(
+                "Page.captureScreenshot", {"format": "png", "fromSurface": True}
+            )
+
+            return result["data"]
+        except PlaywrightError as error:
+            print(
+                f"CDP screenshot failed, falling back to standard screenshot: {error}"
+            )
+            return await super().screenshot()

meshagent/computers/computer.py

@@ -0,0 +1,36 @@
+from typing import Protocol, List, Literal, Dict
+
+
+class Computer(Protocol):
+    """Defines the 'shape' (methods/properties) our loop expects."""
+
+    @property
+    def environment(self) -> Literal["windows", "mac", "linux", "browser"]: ...
+    @property
+    def dimensions(self) -> tuple[int, int]: ...
+
+    async def screenshot(self) -> str: ...
+
+    async def click(self, x: int, y: int, button: str = "left") -> None: ...
+
+    async def double_click(self, x: int, y: int) -> None: ...
+
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None: ...
+
+    async def type(self, text: str) -> None: ...
+
+    async def wait(self, ms: int = 1000) -> None: ...
+
+    async def move(self, x: int, y: int) -> None: ...
+
+    async def keypress(self, keys: List[str]) -> None: ...
+
+    async def drag(self, path: List[Dict[str, int]]) -> None: ...
+
+    async def get_current_url() -> str: ...
+
+    async def __aenter__(self) -> "Computer":
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb) -> "Computer":
+        return self