meshagent-computers 0.0.7__py3-none-any.whl

meshagent/computers/__init__.py

@@ -0,0 +1,7 @@
+from .computer import Computer
+from .browserbase import BrowserbaseBrowser
+from .local_playwright import LocalPlaywrightComputer
+from .docker import DockerComputer
+from .scrapybara import ScrapybaraBrowser, ScrapybaraUbuntu
+from .operator import Operator
+from .agent import ComputerAgent

meshagent/computers/agent.py

@@ -0,0 +1,225 @@
+from meshagent.openai import OpenAIResponsesAdapter
+from meshagent.agents import LLMAdapter, AgentChatContext
+from meshagent.tools import Tool, Toolkit, ToolContext
+from meshagent.agents.prompt import PromptAgent
+from meshagent.computers import Computer, Operator
+from meshagent.agents.chat import ChatBot, ChatThreadContext
+from meshagent.api import RemoteParticipant, FileResponse
+from meshagent.api.messaging import RawOutputs
+
+from typing import Optional
+import base64
+import json
+import logging
+
+logging.basicConfig()
+logger = logging.getLogger("computer")
+logger.setLevel(logging.INFO)
+
+class ComputerAgent[ComputerType:Computer, OperatorType:Operator](ChatBot):
+    def __init__(self, *, name, 
+            title=None,
+            description=None,
+            requires=None,
+            labels = None,
+            computer_cls: ComputerType,
+            operator_cls: OperatorType,
+            rules: Optional[list[str]] = None,
+            llm_adapter: Optional[LLMAdapter] = None,
+            toolkits: list[Toolkit] = None
+        ):
+
+        if rules == None:
+            rules=[
+                "if asked to go to a URL, you MUST use the goto function to go to the url if it is available",
+                "after going directly to a URL, the screen will change so you should take a look at it to know what to do next"
+            ]
+        super().__init__(
+            name=name,
+            title=title,
+            description=description,
+            requires=requires,
+            labels=labels,
+            llm_adapter=llm_adapter,
+            toolkits=toolkits,
+            rules=rules
+        )
+        self.computer_cls = computer_cls
+        self.operator_cls = operator_cls
+
+
+    async def init_thread_context(self, *, thread_context: ChatThreadContext):
+        
+        operator : Operator = self.operator_cls()
+        computer : Computer = self.computer_cls()
+        started = False
+
+        class ComputerTool(Tool):
+            def __init__(self, *, operator: Operator, computer: Computer, title = "computer_call", description = "handle computer calls from computer use preview", rules = [], thumbnail_url = None, defs = None):
+                super().__init__(
+                    name="computer_call",
+                    # TODO: give a correct schema
+                    input_schema={
+                        "additionalProperties" : False,
+                        "type" : "object",
+                        "required" : [],
+                        "properties" : {}
+                    },
+                    title=title,
+                    description=description,
+                    rules=rules,
+                    thumbnail_url=thumbnail_url,
+                    defs=defs,
+                   
+                )
+                self.computer = computer
+
+
+            @property
+            def options(self):
+                return {
+                    "type": "computer-preview",
+                    "display_width": self.computer.dimensions[0],
+                    "display_height": self.computer.dimensions[1],
+                    "environment": self.computer.environment,
+                }
+
+            async def execute(self,  context: ToolContext, *, arguments):
+                
+                nonlocal started
+                if started == False:
+                    await self.computer.__aenter__()
+                    started = True
+
+                for participant in thread_context.participants:
+                    await context.room.messaging.send_message(
+                        to=participant,
+                        type="computer_use",
+                        message={
+                            "arguments" : arguments
+                        }
+                    )
+
+                outputs = await operator.play(computer=self.computer, item=arguments)
+                for output in outputs:
+                      if output["type"] == "computer_call_output":
+                          if output["output"] != None:
+                              if output["output"]["type"] == "input_image":
+                                  
+                                b64 : str = output["output"]["image_url"]
+                                image_data_b64 = b64.split(",", 1)
+                                
+                                image_bytes = base64.b64decode(image_data_b64[1])
+
+                                for participant in thread_context.participants:
+                                    context.room.messaging.send_message_nowait(
+                                        to=participant,
+                                        type="computer_screen",
+                                        message={
+                                        },
+                                        attachment=image_bytes
+                                    )
+                
+                nonlocal computer_toolkit
+                if len(computer_toolkit.tools) == 1:
+                    # HACK: after looking at the page, add the other tools,
+                    # if we add these first then the computer-use-preview mode fails if it calls them before using the computer
+                    computer_toolkit.tools.extend([
+                        ScreenshotTool(computer=computer),
+                        GotoURL(computer=computer),
+                    ])
+                return RawOutputs(outputs=outputs)
+            
+        class ScreenshotTool(Tool):
+            def __init__(self, computer: Computer):
+                self.computer = computer
+
+                super().__init__(
+                    name="screenshot",
+                    # TODO: give a correct schema
+                    input_schema={
+                        "additionalProperties" : False,
+                        "type" : "object",
+                        "required" : ["full_page","save_path"],
+                        "properties" : {
+                            "full_page" : {
+                                "type" : "boolean"
+                            },
+                            "save_path" : {
+                                "type" : "string",
+                                "description" : "a file path to save the screenshot to (should end with .png)"
+                            }
+                        }
+                    },
+                    description="take a screenshot of the current page",               
+                )
+
+            
+            async def execute(self, context: ToolContext, save_path: str, full_page: bool):
+                nonlocal started
+                if started == False:
+                    await self.computer.__aenter__()
+                    started = True
+
+                screenshot_bytes = await self.computer.screenshot_bytes(full_page=full_page)
+                handle = await context.room.storage.open(path=save_path, overwrite=True)
+                await context.room.storage.write(handle=handle, data=screenshot_bytes)
+                await context.room.storage.close(handle=handle)
+
+                return f"saved screenshot to {save_path}"
+            
+        class GotoURL(Tool):
+            def __init__(self, computer: Computer):
+                self.computer = computer
+
+                super().__init__(
+                    name="goto",
+                    description="goes to a specific URL. Make sure it starts with http:// or https://",
+                    # TODO: give a correct schema
+                    input_schema={
+                        "additionalProperties" : False,
+                        "type" : "object",
+                        "required" : ["url"],
+                        "properties" : {
+                            "url" : {
+                                "type" : "string",
+                                "description": "Fully qualified URL to navigate to.",
+                            }
+                        }
+                    },
+                )
+
+            
+            async def execute(self, context: ToolContext, url: str):
+                nonlocal started
+                if started == False:
+                    await self.computer.__aenter__()
+                    started = True
+
+                if url.startswith("https://") == False and url.startswith("http://") == False:
+                    url = "https://"+url
+
+                await self.computer.goto(url)
+
+                # send an updated screen out
+                for participant in thread_context.participants:
+                    context.room.messaging.send_message_nowait(
+                        to=participant,
+                        type="computer_screen",
+                        message={
+                        },
+                        attachment = await self.computer.screenshot_bytes(full_page=False)
+                    )
+        
+        computer_tool = ComputerTool(computer=computer, operator=operator)
+        
+        computer_toolkit = Toolkit(name="meshagent.openai.computer", tools=[
+            computer_tool
+        ])
+
+        thread_context.toolkits = [
+            computer_toolkit,
+            *thread_context.toolkits
+        ]
+
+

meshagent/computers/base_playwright.py

@@ -0,0 +1,179 @@
+import time
+import base64
+from typing import List, Dict, Literal
+from playwright.async_api import async_playwright, Browser, Page, Route, Request
+from meshagent.computers.utils import check_blocklisted_url
+
+# Optional: key mapping if your model uses "CUA" style keys
+CUA_KEY_TO_PLAYWRIGHT_KEY = {
+    "/": "Divide",
+    "\\": "Backslash",
+    "alt": "Alt",
+    "arrowdown": "ArrowDown",
+    "arrowleft": "ArrowLeft",
+    "arrowright": "ArrowRight",
+    "arrowup": "ArrowUp",
+    "backspace": "Backspace",
+    "capslock": "CapsLock",
+    "cmd": "Meta",
+    "ctrl": "Control",
+    "delete": "Delete",
+    "end": "End",
+    "enter": "Enter",
+    "esc": "Escape",
+    "home": "Home",
+    "insert": "Insert",
+    "option": "Alt",
+    "pagedown": "PageDown",
+    "pageup": "PageUp",
+    "shift": "Shift",
+    "space": " ",
+    "super": "Meta",
+    "tab": "Tab",
+    "win": "Meta",
+}
+
+
+class BasePlaywrightComputer:
+    """
+    Abstract base for Playwright-based computers:
+
+      - Subclasses override `_get_browser_and_page()` to do local or remote connection,
+        returning (Browser, Page).
+      - This base class handles context creation (`__enter__`/`__exit__`),
+        plus standard "Computer" actions like click, scroll, etc.
+      - We also have extra browser actions: `goto(url)` and `back()`.
+    """
+
+    environment: Literal["browser"] = "browser"
+    dimensions = (1024, 768)
+
+    def __init__(self):
+        self._playwright = None
+        self._browser: Browser | None = None
+        self._page: Page | None = None
+
+    async def __aenter__(self):
+        # Start Playwright and call the subclass hook for getting browser/page
+        self._context = async_playwright()
+        self._playwright = await self._context.__aenter__()
+        self._browser, self._page = await self._get_browser_and_page()
+
+        # Set up network interception to flag URLs matching domains in BLOCKED_DOMAINS
+        async def handle_route(route: Route, request: Request):
+
+            url = request.url
+            if check_blocklisted_url(url):
+                print(f"Flagging blocked domain: {url}")
+                await route.abort()
+            else:
+                await route.continue_()
+
+        await self._page.route("**/*", handle_route)
+
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self._browser:
+            await self._browser.close()
+        if self._playwright:
+            await self._context.__aexit__(exc_type, exc_val, exc_tb)
+
+    def get_current_url(self) -> str:
+        if self._page == None:
+            return "about:blank"
+        
+    async def ensure_page(self):
+        # After a timeout, we might loose our browser
+        if self._page == None or self._browser.is_connected == False:
+            self._browser, self._page = await self._get_browser_and_page()
+
+    # --- Common "Computer" actions ---
+
+    
+    async def screenshot_bytes(self, full_page: bool = False) -> bytes:
+        await self.ensure_page()
+        png_bytes = await self._page.screenshot(full_page=full_page)
+        return png_bytes
+
+    async def screenshot(self, full_page: bool = False) -> str:
+        await self.ensure_page()
+        png_bytes = await self.screenshot_bytes(full_page=full_page)
+        return base64.b64encode(png_bytes).decode("utf-8")
+
+    async def click(self, x: int, y: int, button: str = "left") -> None:
+        await self.ensure_page()
+        match button:
+            case "back":
+                await self.back()
+            case "forward":
+                await self.forward()
+            case "wheel":
+                await self._page.mouse.wheel(x, y)
+            case _:
+                button_mapping = {"left": "left", "right": "right"}
+                button_type = button_mapping.get(button, "left")
+                await self._page.mouse.click(x, y, button=button_type)
+
+    async def double_click(self, x: int, y: int) -> None:
+        await self.ensure_page()
+        await self._page.mouse.dblclick(x, y)
+
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        await self.ensure_page()
+        await self._page.mouse.move(x, y)
+        await self._page.evaluate(f"window.scrollBy({scroll_x}, {scroll_y})")
+
+    async def type(self, text: str) -> None:
+        await self.ensure_page()
+        await self._page.keyboard.type(text)
+
+    async def wait(self, ms: int = 1000) -> None:
+        await self.ensure_page()
+        time.sleep(ms / 1000)
+
+    async def move(self, x: int, y: int) -> None:
+        await self.ensure_page()
+        await self._page.mouse.move(x, y)
+
+    async def keypress(self, keys: List[str]) -> None:
+        await self.ensure_page()
+        for key in keys:
+            mapped_key = CUA_KEY_TO_PLAYWRIGHT_KEY.get(key.lower(), key)
+            await self._page.keyboard.press(mapped_key)
+
+    async def drag(self, path: List[Dict[str, int]]) -> None:
+        await self.ensure_page()
+        if not path:
+            return
+        
+        await self._page.mouse.move(path[0]["x"], path[0]["y"])
+        await self._page.mouse.down()
+        for point in path[1:]:
+            await self._page.mouse.move(point["x"], point["y"])
+        await self._page.mouse.up()
+
+    async def get_current_url(self) -> str:
+        await self.ensure_page()
+        return self._page.url
+
+    # --- Extra browser-oriented actions ---
+    async def goto(self, url: str) -> None:
+        await self.ensure_page()
+        try:
+            return await self._page.goto(url)
+        except Exception as e:
+            print(f"Error navigating to {url}: {e}")
+
+    async def back(self) -> None:
+        await self.ensure_page()
+        return await self._page.go_back()
+
+    async def forward(self) -> None:
+        await self.ensure_page()
+        return await self._page.go_forward()
+
+    # --- Subclass hook ---
+    async def _get_browser_and_page(self) -> tuple[Browser, Page]:
+        """Subclasses must implement, returning (Browser, Page)."""
+        raise NotImplementedError

meshagent/computers/browserbase.py

@@ -0,0 +1,197 @@
+import os
+from typing import Tuple, Dict, List, Union, Optional
+from playwright.async_api import Browser, Page, BrowserContext, Error as PlaywrightError
+from .base_playwright import BasePlaywrightComputer
+from browserbase import AsyncBrowserbase
+from dotenv import load_dotenv
+import base64
+
+load_dotenv()
+
+
+class BrowserbaseBrowser(BasePlaywrightComputer):
+    """
+    Browserbase is a headless browser platform that offers a remote browser API. You can use it to control thousands of browsers from anywhere.
+    You can find more information about Browserbase at https://www.browserbase.com/computer-use or view our OpenAI CUA Quickstart at https://docs.browserbase.com/integrations/openai-cua/introduction.
+
+    IMPORTANT: This Browserbase computer requires the use of the `goto` tool defined in playwright_with_custom_functions.py.
+    Make sure to include this tool in your configuration when using the Browserbase computer.
+    """
+
+    def __init__(
+        self,
+        width: int = 1024,
+        height: int = 768,
+        region: str = "us-west-2",
+        proxy: bool = False,
+        virtual_mouse: bool = True,
+        ad_blocker: bool = False,
+    ):
+        """
+        Initialize the Browserbase instance. Additional configuration options for features such as persistent cookies, ad blockers, file downloads and more can be found in the Browserbase API documentation: https://docs.browserbase.com/reference/api/create-a-session
+
+        Args:
+            width (int): The width of the browser viewport. Default is 1024.
+            height (int): The height of the browser viewport. Default is 768.
+            region (str): The region for the Browserbase session. Default is "us-west-2". Pick a region close to you for better performance. https://docs.browserbase.com/guides/multi-region
+            proxy (bool): Whether to use a proxy for the session. Default is False. Turn on proxies if you're browsing is frequently interrupted. https://docs.browserbase.com/features/proxies
+            virtual_mouse (bool): Whether to enable the virtual mouse cursor. Default is True.
+            ad_blocker (bool): Whether to enable the built-in ad blocker. Default is False.
+        """
+        super().__init__()
+        self.bb = AsyncBrowserbase(api_key=os.getenv("BROWSERBASE_API_KEY"))
+        self.project_id = os.getenv("BROWSERBASE_PROJECT_ID")
+        self.session = None
+        self.dimensions = (width, height)
+        self.region = region
+        self.proxy = proxy
+        self.virtual_mouse = virtual_mouse
+        self.ad_blocker = ad_blocker
+
+    async def _get_browser_and_page(self) -> Tuple[Browser, Page]:
+        """
+        Create a Browserbase session and connect to it.
+
+        Returns:
+            Tuple[Browser, Page]: A tuple containing the connected browser and page objects.
+        """
+        # Create a session on Browserbase with specified parameters
+        width, height = self.dimensions
+        session_params = {
+            "project_id": self.project_id,
+            "browser_settings": {
+                "viewport": {"width": width, "height": height},
+                "blockAds": self.ad_blocker,
+            },
+            "region": self.region,
+            "proxies": self.proxy,
+        }
+        self.session = await self.bb.sessions.create(**session_params)
+
+        # Print the live session URL
+        print(
+            f"Watch and control this browser live at https://www.browserbase.com/sessions/{self.session.id}"
+        )
+
+        # Connect to the remote session
+        browser = await self._playwright.chromium.connect_over_cdp(
+            self.session.connect_url,
+            timeout=60000
+        )
+        context = browser.contexts[0]
+
+        # Add event listeners for page creation and closure
+        context.on("page", self._handle_new_page)
+
+        # Only add the init script if virtual_mouse is True
+        if self.virtual_mouse:
+            await context.add_init_script("""
+            // Only run in the top frame
+            if (window.self === window.top) {
+                function initCursor() {
+                    const CURSOR_ID = '__cursor__';
+
+                    // Check if cursor element already exists
+                    if (document.getElementById(CURSOR_ID)) return;
+
+                    const cursor = document.createElement('div');
+                    cursor.id = CURSOR_ID;
+                    Object.assign(cursor.style, {
+                        position: 'fixed',
+                        top: '0px',
+                        left: '0px',
+                        width: '20px',
+                        height: '20px',
+                        backgroundImage: 'url("data:image/svg+xml;utf8,<svg xmlns=\\'http://www.w3.org/2000/svg\\' viewBox=\\'0 0 24 24\\' fill=\\'black\\' stroke=\\'white\\' stroke-width=\\'1\\' stroke-linejoin=\\'round\\' stroke-linecap=\\'round\\'><polygon points=\\'2,2 2,22 8,16 14,22 17,19 11,13 20,13\\'/></svg>")',
+                        backgroundSize: 'cover',
+                        pointerEvents: 'none',
+                        zIndex: '99999',
+                        transform: 'translate(-2px, -2px)',
+                    });
+
+                    document.body.appendChild(cursor);
+
+                    document.addEventListener("mousemove", (e) => {
+                        cursor.style.top = e.clientY + "px";
+                        cursor.style.left = e.clientX + "px";
+                    });
+                }
+
+                // Use requestAnimationFrame for early execution
+                requestAnimationFrame(function checkBody() {
+                    if (document.body) {
+                        initCursor();
+                    } else {
+                        requestAnimationFrame(checkBody);
+                    }
+                });
+            }
+            """)
+
+        page = context.pages[0]
+        page.on("close", self._handle_page_close)
+
+        await page.goto("https://google.com")
+
+        return browser, page
+
+    async def _handle_new_page(self, page: Page):
+        """Handle the creation of a new page."""
+        print("New page created")
+        self._page = page
+        page.on("close", self._handle_page_close)
+
+    async def _handle_page_close(self, page: Page):
+        """Handle the closure of a page."""
+        print("Page closed")
+        if self._page == page:
+            if self._browser.contexts[0].pages:
+                self._page = self._browser.contexts[0].pages[-1]
+            else:
+                print("Warning: All pages have been closed.")
+                self._page = None
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """
+        Clean up resources when exiting the context manager.
+
+        Args:
+            exc_type: The type of the exception that caused the context to be exited.
+            exc_val: The exception instance that caused the context to be exited.
+            exc_tb: A traceback object encapsulating the call stack at the point where the exception occurred.
+        """
+        if self._page:
+            await self._page.close()
+        if self._browser:
+            await self._browser.close()
+        if self._playwright:
+            await self._playwright.stop()
+
+        if self.session:
+            print(
+                f"Session completed. View replay at https://browserbase.com/sessions/{self.session.id}"
+            )
+
+    async def screenshot(self) -> str:
+        await self.ensure_page()
+
+        """
+        Capture a screenshot of the current viewport using CDP.
+
+        Returns:
+            str: A base64 encoded string of the screenshot.
+        """
+        try:
+            # Get CDP session from the page
+            cdp_session = await self._page.context.new_cdp_session(self._page)
+
+            # Capture screenshot using CDP
+            result = await cdp_session.send("Page.captureScreenshot", {
+                "format": "png",
+                "fromSurface": True
+            })
+
+            return result['data']
+        except PlaywrightError as error:
+            print(f"CDP screenshot failed, falling back to standard screenshot: {error}")
+            return await super().screenshot()

meshagent/computers/computer.py

@@ -0,0 +1,36 @@
+from typing import Protocol, List, Literal, Dict
+ 
+
+class Computer(Protocol):
+    """Defines the 'shape' (methods/properties) our loop expects."""
+
+    @property
+    def environment(self) -> Literal["windows", "mac", "linux", "browser"]: ...
+    @property
+    def dimensions(self) -> tuple[int, int]: ...
+
+    async def screenshot(self) -> str: ...
+
+    async def click(self, x: int, y: int, button: str = "left") -> None: ...
+
+    async def double_click(self, x: int, y: int) -> None: ...
+
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None: ...
+
+    async def type(self, text: str) -> None: ...
+
+    async def wait(self, ms: int = 1000) -> None: ...
+
+    async def move(self, x: int, y: int) -> None: ...
+
+    async def keypress(self, keys: List[str]) -> None: ...
+
+    async def drag(self, path: List[Dict[str, int]]) -> None: ...
+
+    async def get_current_url() -> str: ...
+    
+    async def __aenter__(self) -> 'Computer':
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb) -> 'Computer':
+        return self