meshagent-computers 0.0.7__py3-none-any.whl

meshagent/computers/docker.py

@@ -0,0 +1,179 @@
+import subprocess
+import time
+import shlex
+import asyncio
+
+
+async def _async_check_output(*args, **kwargs):
+    proc = await asyncio.create_subprocess_exec(
+        *args,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+        **kwargs
+    )
+    stdout, stderr = await proc.communicate()
+    if proc.returncode != 0:
+        raise subprocess.CalledProcessError(proc.returncode, args, output=stdout, stderr=stderr)
+    return stdout
+
+class DockerComputer:
+    environment = "linux"
+    dimensions = (1280, 720)  # Default fallback; will be updated in __enter__.
+
+    def __init__(
+        self,
+        container_name="cua-sample-app",
+        image="ghcr.io/openai/openai-cua-sample-app:latest",
+        display=":99",
+        port_mapping="5900:5900",
+    ):
+        self.container_name = container_name
+        self.image = image
+        self.display = display
+        self.port_mapping = port_mapping
+
+    async def __aenter__(self):
+        # Check if the container is running
+        result = subprocess.run(
+            ["docker", "ps", "-q", "-f", f"name={self.container_name}"],
+            capture_output=True,
+            text=True,
+        )
+
+        if not result.stdout.strip():
+            raise RuntimeError(
+                f"Container {self.container_name} is not running. Build and run with:\n"
+                f"docker build -t {self.container_name} .\n"
+                f"docker run --rm -it --name {self.container_name} "
+                f"-p {self.port_mapping} -e DISPLAY={self.display} {self.container_name}"
+            )
+
+        # Fetch display geometry
+        geometry = await self._exec(
+            f"DISPLAY={self.display} xdotool getdisplaygeometry"
+        ).strip()
+        if geometry:
+            w, h = geometry.split()
+            self.dimensions = (int(w), int(h))
+        # print("Starting Docker container...")
+        # # Run the container detached, removing it automatically when it stops
+        # subprocess.check_call(
+        #     [
+        #         "docker",
+        #         "run",
+        #         "-d",
+        #         "--rm",
+        #         "--name",
+        #         self.container_name,
+        #         "-p",
+        #         self.port_mapping,
+        #         self.image,
+        #     ]
+        # )
+        # # Give the container a moment to start
+        # time.sleep(3)
+        # print("Entering DockerComputer context")
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        # print("Stopping Docker container...")
+        # subprocess.check_call(["docker", "stop", self.container_name])
+        # print("Exiting DockerComputer context")
+        pass
+
+    async def _exec(self, cmd: str) -> str:
+        """
+        Run 'cmd' in the container.
+        We wrap cmd in double quotes and escape any double quotes inside it,
+        so spaces or quotes don't break the shell call.
+        """
+        # Escape any existing double quotes in cmd
+        safe_cmd = cmd.replace('"', '\\"')
+
+        # Then wrap the entire cmd in double quotes for `sh -c`
+        docker_cmd = f'docker exec {self.container_name} sh -c "{safe_cmd}"'
+
+        return (await _async_check_output(docker_cmd, shell=True)).decode(
+            "utf-8", errors="ignore"
+        )
+
+    async def screenshot(self) -> str:
+        """
+        Takes a screenshot with ImageMagick (import), returning base64-encoded PNG.
+        Requires 'import'.
+        """
+        # cmd = (
+        #     f"export DISPLAY={self.display} && "
+        #     "import -window root /tmp/screenshot.png && "
+        #     "base64 /tmp/screenshot.png"
+        # )
+        cmd = (
+            f"export DISPLAY={self.display} && "
+            "import -window root png:- | base64 -w 0"
+        )
+
+        return await self._exec(cmd)
+
+    async def click(self, x: int, y: int, button: str = "left") -> None:
+        button_map = {"left": 1, "middle": 2, "right": 3}
+        b = button_map.get(button, 1)
+        await self._exec(f"DISPLAY={self.display} xdotool mousemove {x} {y} click {b}")
+
+    async def double_click(self, x: int, y: int) -> None:
+        await self._exec(
+            f"DISPLAY={self.display} xdotool mousemove {x} {y} click --repeat 2 1"
+        )
+
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        """
+        For simple vertical scrolling: xdotool click 4 (scroll up) or 5 (scroll down).
+        """
+        await self._exec(f"DISPLAY={self.display} xdotool mousemove {x} {y}")
+        clicks = abs(scroll_y)
+        button = 4 if scroll_y < 0 else 5
+        for _ in range(clicks):
+            await self._exec(f"DISPLAY={self.display} xdotool click {button}")
+
+    async def type(self, text: str) -> None:
+        """
+        Type the given text via xdotool, preserving spaces and quotes.
+        """
+        # Escape single quotes in the user text: ' -> '\'\''
+        safe_text = text.replace("'", "'\\''")
+        # Then wrap everything in single quotes for xdotool
+        cmd = f"DISPLAY={self.display} xdotool type -- '{safe_text}'"
+        await self._exec(cmd)
+
+    async def wait(self, ms: int = 1000) -> None:
+        time.sleep(ms / 1000)
+
+    async def move(self, x: int, y: int) -> None:
+        await self._exec(f"DISPLAY={self.display} xdotool mousemove {x} {y}")
+
+    async def keypress(self, keys: list[str]) -> None:
+        mapping = {
+            "ENTER": "Return",
+            "LEFT": "Left",
+            "RIGHT": "Right",
+            "UP": "Up",
+            "DOWN": "Down",
+            "ESC": "Escape",
+            "SPACE": "space",
+            "BACKSPACE": "BackSpace",
+            "TAB": "Tab",
+        }
+        mapped_keys = [mapping.get(key, key) for key in keys]
+        combo = "+".join(mapped_keys)
+        await self._exec(f"DISPLAY={self.display} xdotool key {combo}")
+
+    async def drag(self, path: list[dict[str, int]]) -> None:
+        if not path:
+            return
+        start_x = path[0]["x"]
+        start_y = path[0]["y"]
+        self._exec(
+            f"DISPLAY={self.display} xdotool mousemove {start_x} {start_y} mousedown 1"
+        )
+        for point in path[1:]:
+            await self._exec(f"DISPLAY={self.display} xdotool mousemove {point['x']} {point['y']}")
+        await self._exec(f"DISPLAY={self.display} xdotool mouseup 1")

meshagent/computers/local_playwright.py

@@ -0,0 +1,24 @@
+from playwright.async_api import Browser, Page
+from .base_playwright import BasePlaywrightComputer
+
+
+class LocalPlaywrightComputer(BasePlaywrightComputer):
+    """Launches a local Chromium instance using Playwright."""
+
+    def __init__(self, headless: bool = False):
+        super().__init__()
+        self.headless = headless
+
+    async def _get_browser_and_page(self) -> tuple[Browser, Page]:
+        width, height = self.dimensions
+        launch_args = [f"--window-size={width},{height}", "--disable-extensions", "--disable-file-system"]
+        browser = await self._playwright.chromium.launch(
+            chromium_sandbox=True,
+            headless=self.headless,
+            args=launch_args,
+            env={}
+        )
+        page = await browser.new_page()
+        await page.set_viewport_size({"width": width, "height": height})
+        await page.goto("https://google.com")
+        return browser, page

meshagent/computers/operator.py

@@ -0,0 +1,78 @@
+from .computer import Computer
+from .utils import check_blocklisted_url
+import json
+
+class Operator:
+    def __init__(self):
+        self.print_steps = False
+        self.show_images = False
+
+    async def acknowledge_safety_check_callback(self, data: dict): 
+        return True
+
+    async def show_image(self, base_64: str):
+        pass
+
+    async def play(self, *, computer: Computer, item: dict) -> list:
+        """Handle each item; may cause a computer action + screenshot."""
+        if item["type"] == "message":
+            if self.print_steps:
+                print(item["content"][0]["text"])
+
+        if item["type"] == "function_call":
+            name, args = item["name"], json.loads(item["arguments"])
+            if self.print_steps:
+                print(f"{name}({args})")
+
+            if hasattr(computer, name):  # if function exists on computer, call it
+                method = getattr(computer, name)
+                await method(**args)
+            return [
+                {
+                    "type": "function_call_output",
+                    "call_id": item["call_id"],
+                    "output": "success",  # hard-coded output for demo
+                }
+            ]
+
+        if item["type"] == "computer_call":
+            action = item["action"]
+            action_type = action["type"]
+            action_args = {k: v for k, v in action.items() if k != "type"}
+            if self.print_steps:
+                print(f"{action_type}({action_args})")
+
+            method = getattr(computer, action_type)
+            await method(**action_args)
+
+            screenshot_base64 = await computer.screenshot()
+            if self.show_images:
+                self.show_image(screenshot_base64)
+
+            # if user doesn't ack all safety checks exit with error
+            pending_checks = item.get("pending_safety_checks", [])
+            for check in pending_checks:
+                message = check["message"]
+                if not await self.acknowledge_safety_check_callback(message):
+                    raise ValueError(
+                        f"Safety check failed: {message}. Cannot continue with unacknowledged safety checks."
+                    )
+
+            call_output = {
+                "type": "computer_call_output",
+                "call_id": item["call_id"],
+                "acknowledged_safety_checks": pending_checks,
+                "output": {
+                    "type": "input_image",
+                    "image_url": f"data:image/png;base64,{screenshot_base64}",
+                },
+            }
+
+            # additional URL safety checks for browser environments
+            if computer.environment == "browser":
+                current_url = await computer.get_current_url()
+                check_blocklisted_url(current_url)
+                call_output["output"]["current_url"] = current_url
+
+            return [call_output]
+        return []

meshagent/computers/scrapybara.py

@@ -0,0 +1,212 @@
+import os
+import time
+from dotenv import load_dotenv
+from scrapybara import AsyncScrapybara
+from playwright.async_api import async_playwright, Browser, Page
+from meshagent.computers.utils import BLOCKED_DOMAINS
+
+load_dotenv()
+
+CUA_KEY_TO_SCRAPYBARA_KEY = {
+    "/": "slash",
+    "\\": "backslash",
+    "arrowdown": "Down",
+    "arrowleft": "Left",
+    "arrowright": "Right",
+    "arrowup": "Up",
+    "backspace": "BackSpace",
+    "capslock": "Caps_Lock",
+    "cmd": "Meta_L",
+    "delete": "Delete",
+    "end": "End",
+    "enter": "Return",
+    "esc": "Escape",
+    "home": "Home",
+    "insert": "Insert",
+    "option": "Alt_L",
+    "pagedown": "Page_Down",
+    "pageup": "Page_Up",
+    "tab": "Tab",
+    "win": "Meta_L",
+}
+
+
+class ScrapybaraBrowser:
+    """
+    Scrapybara provides virtual desktops and browsers in the cloud. https://scrapybara.com
+    You can try OpenAI CUA for free at https://computer.new or read our CUA Quickstart at https://computer.new/cua.
+    """
+
+    def __init__(self):
+        self.client = AsyncScrapybara(api_key=os.getenv("SCRAPYBARA_API_KEY"))
+        self.environment = "browser"
+        self.dimensions = (1024, 768)
+        self._playwright = None
+        self._browser: Browser | None = None
+        self._page: Page | None = None
+
+    async def __aenter__(self):
+        print("Starting scrapybara browser")
+        blocked_domains = [
+            domain.replace("https://", "").replace("www.", "")
+            for domain in BLOCKED_DOMAINS
+        ]
+        self.instance = await self.client.start_browser(blocked_domains=blocked_domains)
+        print("Scrapybara browser started ₍ᐢ•(ܫ)•ᐢ₎")
+        print(
+            f"You can view and interact with the stream at {self.instance.get_stream_url().stream_url}"
+        )
+        self._playwright_context = async_playwright()
+        self._playwright = await self._playwright_context.__aenter__()
+        self._browser = await self._playwright.chromium.connect_over_cdp(
+            (await self.instance.get_cdp_url()).cdp_url
+        )
+        self._page = self._browser.contexts[0].pages[0]
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self._playwright_context.__aexit__(exc_type, exc_val, exc_tb)
+
+        print("Stopping scrapybara browser")
+        await self.instance.stop()
+        print("Scrapybara browser stopped ₍ᐢ-(ｪ)-ᐢ₎")
+
+    async def goto(self, url: str) -> None:
+        await self._page.goto(url)
+
+    async def get_current_url(self) -> str:
+        return (await self.instance.get_current_url()).current_url
+
+    async def screenshot(self) -> str:
+        return (await self.instance.screenshot()).base_64_image
+
+    async def click(self, x: int, y: int, button: str = "left") -> None:
+        button = "middle" if button == "wheel" else button
+        await self.instance.computer(
+            action="click_mouse",
+            click_type="click",
+            button=button,
+            coordinates=[x, y],
+            num_clicks=1,
+        )
+
+    async def double_click(self, x: int, y: int) -> None:
+        await self.instance.computer(
+            action="click_mouse",
+            click_type="click",
+            button="left",
+            coordinates=[x, y],
+            num_clicks=2,
+        )
+
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        await self.instance.computer(
+            action="scroll",
+            coordinates=[x, y],
+            delta_x=scroll_x // 20,
+            delta_y=scroll_y // 20,
+        )
+
+    async def type(self, text: str) -> None:
+        await self.instance.computer(action="type_text", text=text)
+
+    async def wait(self, ms: int = 1000) -> None:
+        time.sleep(ms / 1000)
+        # Scrapybara also has `self.instance.computer(action="wait", duration=ms / 1000)`
+
+    async def move(self, x: int, y: int) -> None:
+        await self.instance.computer(action="move_mouse", coordinates=[x, y])
+
+    async def keypress(self, keys: list[str]) -> None:
+        mapped_keys = [
+            CUA_KEY_TO_SCRAPYBARA_KEY.get(key.lower(), key.lower()) for key in keys
+        ]
+        await self.instance.computer(action="press_key", keys=mapped_keys)
+
+    async def drag(self, path: list[dict[str, int]]) -> None:
+        if not path:
+            return
+        path = [[point["x"], point["y"]] for point in path]
+        await self.instance.computer(action="drag_mouse", path=path)
+
+
+class ScrapybaraUbuntu:
+    """
+    Scrapybara provides virtual desktops and browsers in the cloud.
+    You can try OpenAI CUA for free at https://computer.new or read our CUA Quickstart at https://computer.new/cua.
+    """
+
+    def __init__(self):
+        self.client = AsyncScrapybara(api_key=os.getenv("SCRAPYBARA_API_KEY"))
+        self.environment = "linux"  # "windows", "mac", "linux", or "browser"
+        self.dimensions = (1024, 768)
+
+    async def __aenter__(self):
+        print("Starting Scrapybara Ubuntu instance")
+        blocked_domains = [
+            domain.replace("https://", "").replace("www.", "")
+            for domain in BLOCKED_DOMAINS
+        ]
+        self.instance = await self.client.start_ubuntu(blocked_domains=blocked_domains)
+        print("Scrapybara Ubuntu instance started ₍ᐢ•(ܫ)•ᐢ₎")
+        print(
+            f"You can view and interact with the stream at {self.instance.get_stream_url().stream_url}"
+        )
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        print("Stopping Scrapybara Ubuntu instance")
+        (await self.instance).stop()
+        print("Scrapybara Ubuntu instance stopped ₍ᐢ-(ｪ)-ᐢ₎")
+
+    async def screenshot(self) -> str:
+        return (await self.instance.screenshot()).base_64_image
+
+    async def click(self, x: int, y: int, button: str = "left") -> None:
+        button = "middle" if button == "wheel" else button
+        await self.instance.computer(
+            action="click_mouse",
+            click_type="click",
+            button=button,
+            coordinates=[x, y],
+            num_clicks=1,
+        )
+
+    async def double_click(self, x: int, y: int) -> None:
+        await self.instance.computer(
+            action="click_mouse",
+            click_type="click",
+            button="left",
+            coordinates=[x, y],
+            num_clicks=2,
+        )
+
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        await self.instance.computer(
+            action="scroll",
+            coordinates=[x, y],
+            delta_x=scroll_x // 20,
+            delta_y=scroll_y // 20,
+        )
+
+    async def type(self, text: str) -> None:
+        await self.instance.computer(action="type_text", text=text)
+
+    async def wait(self, ms: int = 1000) -> None:
+        time.sleep(ms / 1000)
+        # Scrapybara also has `self.instance.computer(action="wait", duration=ms / 1000)`
+
+    async def move(self, x: int, y: int) -> None:
+        await self.instance.computer(action="move_mouse", coordinates=[x, y])
+
+    async def keypress(self, keys: list[str]) -> None:
+        mapped_keys = [
+            CUA_KEY_TO_SCRAPYBARA_KEY.get(key.lower(), key.lower()) for key in keys
+        ]
+        await self.instance.computer(action="press_key", keys=mapped_keys)
+
+    async def drag(self, path: list[dict[str, int]]) -> None:
+        if not path:
+            return
+        path = [[point["x"], point["y"]] for point in path]
+        await self.instance.computer(action="drag_mouse", path=path)

meshagent/computers/utils.py

@@ -0,0 +1,78 @@
+import os
+import requests
+from dotenv import load_dotenv
+import json
+import base64
+from PIL import Image
+from io import BytesIO
+import io
+from urllib.parse import urlparse
+
+load_dotenv(override=True)
+
+BLOCKED_DOMAINS = [
+    "maliciousbook.com",
+    "evilvideos.com",
+    "darkwebforum.com",
+    "shadytok.com",
+    "suspiciouspins.com",
+    "ilanbigio.com",
+]
+
+
+def pp(obj):
+    print(json.dumps(obj, indent=4))
+
+
+def show_image(base_64_image):
+    image_data = base64.b64decode(base_64_image)
+    image = Image.open(BytesIO(image_data))
+    image.show()
+
+
+def calculate_image_dimensions(base_64_image):
+    image_data = base64.b64decode(base_64_image)
+    image = Image.open(io.BytesIO(image_data))
+    return image.size
+
+
+def sanitize_message(msg: dict) -> dict:
+    """Return a copy of the message with image_url omitted for computer_call_output messages."""
+    if msg.get("type") == "computer_call_output":
+        output = msg.get("output", {})
+        if isinstance(output, dict):
+            sanitized = msg.copy()
+            sanitized["output"] = {**output, "image_url": "[omitted]"}
+            return sanitized
+    return msg
+
+
+def create_response(**kwargs):
+    url = "https://api.openai.com/v1/responses"
+    headers = {
+        "Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}",
+        "Content-Type": "application/json",
+        # TODO: remove for launch
+        "Openai-beta": "responses=v1",
+    }
+
+    openai_org = os.getenv("OPENAI_ORG")
+    if openai_org:
+        headers["Openai-Organization"] = openai_org
+
+    response = requests.post(url, headers=headers, json=kwargs)
+
+    if response.status_code != 200:
+        print(f"Error: {response.status_code} {response.text}")
+
+    return response.json()
+
+
+def check_blocklisted_url(url: str) -> None:
+    """Raise ValueError if the given URL (including subdomains) is in the blocklist."""
+    hostname = urlparse(url).hostname or ""
+    if any(
+        hostname == blocked or hostname.endswith(f".{blocked}")
+        for blocked in BLOCKED_DOMAINS
+    ):
+        raise ValueError(f"Blocked URL: {url}")

meshagent_computers-0.0.7.dist-info/METADATA

@@ -0,0 +1,28 @@
+Metadata-Version: 2.4
+Name: meshagent-computers
+Version: 0.0.7
+Summary: Computer Building Blocks for Meshagent
+Home-page: 
+License: Apache License 2.0
+Project-URL: Documentation, https://meshagent.com
+Project-URL: Website, https://meshagent.com
+Project-URL: Source, https://github.com/meshagent
+Requires-Python: >=3.9.0
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pytest>=8.3.4
+Requires-Dist: pytest-asyncio>=0.24.0
+Requires-Dist: openai>=1.66.2
+Requires-Dist: meshagent-api>=0.0.7
+Requires-Dist: meshagent-agents>=0.0.7
+Requires-Dist: meshagent-tools>=0.0.7
+Requires-Dist: playwright~=1.50.0
+Requires-Dist: browserbase~=1.2.0
+Requires-Dist: scrapybara~=2.4.1
+Dynamic: description-content-type
+Dynamic: license
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

meshagent_computers-0.0.7.dist-info/RECORD

@@ -0,0 +1,15 @@
+meshagent/computers/__init__.py,sha256=4aVM46U1kVLeN8rurNulc6_24a-rHSMIY0v0rc1ydOI,288
+meshagent/computers/agent.py,sha256=w9Yg9I2cfx-fNo9Y0TOcndxcoyOTz2wxRoijxGt9dB4,9089
+meshagent/computers/base_playwright.py,sha256=NhQoOzWmekBmobQ2kvN7w4I3hZeyMIuMKI2eHbG1_mk,6017
+meshagent/computers/browserbase.py,sha256=ZT-ZDndkBD-MHD3UjxXpwpx7-_VCkkanjW1rIJr7-98,8161
+meshagent/computers/computer.py,sha256=465GKDV3cJgzp1SB5B0xzLoftMKU0AxAuBVrZ9drBBc,1072
+meshagent/computers/docker.py,sha256=BGkrlspIYlg3WViaqctyRUye-fONfF40AvLXOQuXPQ8,6331
+meshagent/computers/local_playwright.py,sha256=W7VhxIYEXn2BO3wZZVsc-hH-CU7NTssxFGO7pwz_n1I,919
+meshagent/computers/operator.py,sha256=JTyGvoPnKOqUDaZv58p_2n5Q6m6dm8V214VMIhVEu08,2913
+meshagent/computers/scrapybara.py,sha256=IoeV02QcBDkpW05A1NH-lc4jnBEmZ4SPsRo3nnXC3UE,7502
+meshagent/computers/utils.py,sha256=t9G9dzfJ5a0DSyrW1KNdzlhLJW_S6eGUcS39wnZyHU8,2081
+meshagent_computers-0.0.7.dist-info/licenses/LICENSE,sha256=eTt0SPW-sVNdkZe9PS_S8WfCIyLjRXRl7sUBWdlteFg,10254
+meshagent_computers-0.0.7.dist-info/METADATA,sha256=eYIzYXp2Ia_WFN1mjBrF7u9iPk08P-MZLifVB56PqRM,852
+meshagent_computers-0.0.7.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+meshagent_computers-0.0.7.dist-info/top_level.txt,sha256=GlcXnHtRP6m7zlG3Df04M35OsHtNXy_DY09oFwWrH74,10
+meshagent_computers-0.0.7.dist-info/RECORD,,

meshagent_computers-0.0.7.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (77.0.3)
+Root-Is-Purelib: true
+Tag: py3-none-any
+