cua 0.1.0__tar.gz

cua-0.1.0/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.3
+Name: cua
+Version: 0.1.0
+Summary: CUA (Computer Use) interface for host-VM communication on Apple Silicon
+License: MIT
+Keywords: macos,virtualization,vm,apple-silicon
+Author: TryCua
+Author-email: f@trycua.com
+Requires-Python: >=3.10,<4.0
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: paramiko (>=3.4.0,<4.0.0)
+Requires-Dist: pylume (>=0.1.8,<0.2.0)
+Requires-Dist: websockets (>=12.0,<13.0)
+Project-URL: Homepage, https://github.com/trycua/computer
+Project-URL: Repository, https://github.com/trycua/computer
+Description-Content-Type: text/markdown
+
+

cua-0.1.0/computer/__init__.py

@@ -0,0 +1,33 @@
+"""
+Computer SDK - A Python SDK for facilitating the development of Computer-use AI agents (CUA).
+
+Example:
+    >>> from computer import Computer, Image, Display
+    >>> computer = Computer(
+    ...     image=Image(
+    ...         image="macos-sequoia-xcode",
+    ...         tag="latest",
+    ...         name="macos-sequoia-xcode-py"
+    ...     ),
+    ...     display=Display(
+    ...         width=1920,
+    ...         height=1080
+    ...     ),
+    ...     memory="16GB",
+    ...     cpu="4",
+    ... )
+    >>> computer.run(start_vnc_client=True)
+    >>> computer.type("Hello, world!")
+    >>> computer.press_key("Enter")
+    >>> computer.stop()
+"""
+
+from .models import Image, Display
+from .computer import Computer
+
+__version__ = "0.1.0"
+__all__ = [
+    "Computer",
+    "Image",
+    "Display",
+]

cua-0.1.0/computer/computer.py

@@ -0,0 +1,185 @@
+from typing import Optional, List, Literal
+from pylume import PyLume, VMRunOpts, SharedDirectory
+import asyncio
+from .models import Computer as ComputerConfig, Image, Display
+from .interface.factory import InterfaceFactory
+from pylume import ImageRef, VMUpdateOpts, PyLume
+import time
+
+OSType = Literal['macos', 'linux']
+
+class Computer:
+    def __init__(
+        self,
+        display: Display,
+        memory: str,
+        cpu: str,
+        os: OSType = 'macos',
+        name: str = '',
+        image: str = "macos-sequoia-xcode:latest",
+        shared_directories: Optional[List[SharedDirectory]] = None,
+        use_host_computer_server: bool = False,
+    ):
+        """Initialize a new Computer instance.
+        
+        Args:
+            image: The VM image name
+            tag: The VM image tag
+            name: The VM name
+            display: The display configuration
+            memory: The VM memory allocation
+            cpu: The VM CPU allocation
+            os: The operating system type ('macos' or 'linux')
+            shared_directories: Optional list of directories to share with the VM
+            use_host_computer_server: If True, target localhost instead of starting a VM
+        """
+        if not use_host_computer_server:
+            if ':' not in image or len(image.split(':')) != 2:
+                raise ValueError("Image must be in the format <image_name>:<tag>")
+            
+            if not name:
+                # Normalize the name to be used for the VM
+                name = image.replace(':', '_')
+            
+            self.config = ComputerConfig(
+                image=image.split(':')[0],
+                tag=image.split(':')[1],
+                name=name,
+                display=display,
+                memory=memory,
+                cpu=cpu,
+            )
+
+        self._interface = None
+        self.os = os
+        self.shared_directories = shared_directories or []
+        self._pylume_context = None
+        self.use_host_computer_server = use_host_computer_server
+        
+    async def __aenter__(self):
+        """Enter the async context manager."""
+        if not self.use_host_computer_server:
+            self.config.pylume = PyLume(debug=True, port=3000, use_existing_server=False)
+            self._pylume_context = await self.config.pylume.__aenter__()
+        return self
+        
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Exit the async context manager."""
+        if self._pylume_context:
+            await self.config.pylume.__aexit__(exc_type, exc_val, exc_tb)
+            
+    @property
+    def ip(self) -> str:
+        """Get the IP address of the VM or localhost if using host computer server."""
+        if self.use_host_computer_server:
+            return "127.0.0.1"
+        return self.config.ip
+        
+    async def wait_vm_ready(self, timeout: int = 60, interval: float = 1.0) -> None:
+        """Wait for the VM to be ready with required properties (ip_address, vnc_address).
+        
+        Args:
+            timeout: Maximum time to wait in seconds
+            interval: Time between attempts in seconds
+            
+        Raises:
+            TimeoutError: If the VM is not ready within the timeout period
+        """
+        if self.use_host_computer_server:
+            return None
+            
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            try:
+                vm = await self.config.pylume.get_vm(self.config.name)
+                if vm.ip_address and vm.vnc_url:
+                    return vm
+            except Exception:
+                pass
+            await asyncio.sleep(interval)
+                
+        raise TimeoutError(f"VM {self.config.name} not ready after {timeout} seconds")
+
+    async def run(self, no_display: bool = False):
+        """Run the VM and initialize the computer interface."""
+        if not self._pylume_context and not self.use_host_computer_server:
+            raise RuntimeError("Computer must be used as an async context manager when not using host computer server")
+        
+        ip_address = "127.0.0.1"
+
+        if not self.use_host_computer_server:
+            # Try to get the VM, if it doesn't exist, create it and pull the image
+            try:
+                vm = await self.config.pylume.get_vm(self.config.name)
+            except Exception:
+                image_ref = ImageRef(
+                    image=self.config.image,
+                    tag=self.config.tag,
+                    registry="ghcr.io",
+                    organization="trycua"
+                )
+                await self.config.pylume.pull_image(image_ref, name=self.config.name)
+
+            # Run with shared directory
+            run_opts = VMRunOpts(
+                no_display=False,
+                shared_directories=self.shared_directories
+            )
+            await self.config.pylume.run_vm(self.config.name, run_opts)
+            
+            # Wait for VM to be ready with required properties
+            vm = await self.wait_vm_ready()
+            ip_address = vm.ip_address
+        
+        # Initialize the interface using the factory with the specified OS
+        self._interface = InterfaceFactory.create_interface_for_os(
+            os=self.os,
+            ip_address=ip_address
+        )
+
+        # wait for the websocket interface to be ready
+        await self._interface.wait_for_ready()
+
+        # Create an event to keep the VM running in background if needed
+        if not self.use_host_computer_server:
+            self._stop_event = asyncio.Event()
+            self._keep_alive_task = asyncio.create_task(self._stop_event.wait())
+
+    async def stop(self):
+        """Stop the VM and close the interface."""
+        if hasattr(self, '_stop_event'):
+            self._stop_event.set()
+            if hasattr(self, '_keep_alive_task'):
+                await self._keep_alive_task
+        if self._interface:
+            self._interface.close()
+        await self.config.pylume.stop_vm(self.config.image.name)
+    
+    async def update(self, cpu: Optional[int] = None, memory: Optional[str] = None):
+        """Update VM settings."""
+        update_opts = VMUpdateOpts(
+            cpu=cpu or int(self.config.cpu),
+            memory=memory or self.config.memory
+        )
+        await self.config.pylume.update_vm(self.config.image.name, update_opts)
+        
+    @property
+    def interface(self):
+        """Get the computer interface for interacting with the VM.
+        
+        Returns:
+            BaseComputerInterface: The interface for controlling the VM
+            
+        Raises:
+            RuntimeError: If the interface is not initialized (run() not called)
+        """
+        if self._interface is None:
+            raise RuntimeError("Computer interface not initialized. Call run() first.")
+        return self._interface
+
+    def __getattr__(self, name: str):
+        """Delegate all other method calls to the interface.
+        
+        This is kept for backward compatibility, prefer using computer.interface directly.
+        """
+        return getattr(self.interface, name)

cua-0.1.0/computer/interface/__init__.py

@@ -0,0 +1,15 @@
+"""
+Interface package for Computer SDK.
+"""
+
+from .factory import InterfaceFactory
+from .base import BaseComputerInterface
+from .linux import LinuxComputerInterface
+from .macos import MacOSComputerInterface
+
+__all__ = [
+    "InterfaceFactory",
+    "BaseComputerInterface",
+    "LinuxComputerInterface",
+    "MacOSComputerInterface",
+] 

cua-0.1.0/computer/interface/base.py

@@ -0,0 +1,150 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Dict, Any, Tuple
+
+class BaseComputerInterface(ABC):
+    """Abstract base class for OS-specific computer interfaces.
+    
+    This class defines the contract that all OS-specific implementations must follow.
+    
+    Categories:
+    - Connection Management: Methods for managing connections to the remote system
+    - Mouse Actions: Methods for controlling the mouse
+    - Keyboard Actions: Methods for keyboard input
+    - Scrolling Actions: Methods for scrolling
+    - Screen Actions: Methods for screen interaction and information
+    - Clipboard Actions: Methods for clipboard operations
+    - File System Actions: Methods for file system operations
+    - Accessibility Actions: Methods for accessibility features
+    """
+
+    def __init__(self, ip_address: str, username: str = "lume", password: str = "lume"):
+        self.ip_address = ip_address
+        self.username = username
+        self.password = password
+        
+    @abstractmethod
+    async def wait_for_ready(self, timeout: int = 60, interval: float = 1.0) -> None:
+        """Wait for the interface to be ready."""
+        pass
+        
+    @abstractmethod
+    async def close(self) -> None:
+        """Close all connections."""
+        pass
+
+    # Mouse Actions
+    @abstractmethod
+    async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Perform a left click."""
+        pass
+
+    @abstractmethod
+    async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Perform a right click."""
+        pass
+
+    @abstractmethod
+    async def double_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Perform a double click."""
+        pass
+
+    @abstractmethod
+    async def move_cursor(self, x: int, y: int) -> None:
+        """Move the cursor to specified position."""
+        pass
+
+    @abstractmethod
+    async def drag_to(self, x: int, y: int, button: str = "left", duration: float = 0.5) -> None:
+        """Drag from current position to specified coordinates.
+        
+        Args:
+            x: The x coordinate to drag to
+            y: The y coordinate to drag to
+            button: The mouse button to use ('left', 'middle', 'right')
+            duration: How long the drag should take in seconds
+        """
+        pass
+
+    # Keyboard Actions
+    @abstractmethod
+    async def type(self, text: str) -> None:
+        """Type the specified text."""
+        pass
+
+    @abstractmethod
+    async def press_key(self, key: str) -> None:
+        """Press a single key."""
+        pass
+
+    @abstractmethod
+    async def hotkey(self, *keys: str) -> None:
+        """Press multiple keys simultaneously."""
+        pass
+
+    # Scrolling Actions
+    @abstractmethod
+    async def scroll_down(self, clicks: int = 1) -> None:
+        """Scroll down."""
+        pass
+
+    @abstractmethod
+    async def scroll_up(self, clicks: int = 1) -> None:
+        """Scroll up."""
+        pass
+
+    # Screen Actions
+    @abstractmethod
+    async def screenshot(self) -> bytes:
+        """Take a screenshot."""
+        pass
+
+    @abstractmethod
+    async def get_screen_size(self) -> Dict[str, int]:
+        """Get screen dimensions."""
+        pass
+
+    @abstractmethod
+    async def get_cursor_position(self) -> Dict[str, int]:
+        """Get current cursor position."""
+        pass
+
+    # Clipboard Actions
+    @abstractmethod
+    async def copy_to_clipboard(self) -> str:
+        """Get clipboard content."""
+        pass
+
+    @abstractmethod
+    async def set_clipboard(self, text: str) -> None:
+        """Set clipboard content."""
+        pass
+
+    # File System Actions
+    @abstractmethod
+    async def file_exists(self, path: str) -> bool:
+        """Check if file exists."""
+        pass
+
+    @abstractmethod
+    async def directory_exists(self, path: str) -> bool:
+        """Check if directory exists."""
+        pass
+
+    @abstractmethod
+    async def run_command(self, command: str) -> Tuple[str, str]:
+        """Run shell command."""
+        pass
+
+    # Accessibility Actions
+    @abstractmethod
+    async def get_accessibility_tree(self) -> Dict:
+        """Get accessibility tree."""
+        pass
+
+    @abstractmethod
+    async def find_element_by_criteria(self, 
+                                     role: Optional[str] = None, 
+                                     title: Optional[str] = None,
+                                     value: Optional[str] = None) -> Optional[Dict]:
+        """Find element by accessibility criteria."""
+        pass 

cua-0.1.0/computer/interface/factory.py

@@ -0,0 +1,33 @@
+import platform
+from typing import Type, Literal
+from .base import BaseComputerInterface
+from .macos import MacOSComputerInterface
+from .linux import LinuxComputerInterface
+
+OSType = Literal['macos', 'linux']
+
+class InterfaceFactory:
+    """Factory for creating OS-specific computer interfaces."""
+    
+    @staticmethod
+    def create_interface_for_os(os: OSType, ip_address: str, username: str = "lume", password: str = "lume") -> BaseComputerInterface:
+        """Create and return appropriate interface for the specified OS.
+        
+        Args:
+            os: The operating system type ('macos' or 'linux')
+            ip_address: The IP address of the remote computer
+            username: The username for connection
+            password: The password for connection
+            
+        Returns:
+            BaseComputerInterface: The appropriate interface implementation for the OS
+            
+        Raises:
+            NotImplementedError: If the OS is not supported
+        """
+        if os == 'macos':
+            return MacOSComputerInterface(ip_address, username, password)
+        elif os == 'linux':
+            return LinuxComputerInterface(ip_address, username, password)
+        else:
+            raise NotImplementedError(f"OS '{os}' is not supported") 

cua-0.1.0/computer/interface/linux.py

@@ -0,0 +1,190 @@
+import asyncio
+import websockets
+import json
+import logging
+import socket
+import time
+from typing import Optional, Dict, Any, Tuple
+from .base import BaseComputerInterface
+
+logger = logging.getLogger(__name__)
+
+class LinuxComputerInterface(BaseComputerInterface):
+    """Linux-specific implementation of the computer interface."""
+
+    def __init__(self, ip_address: str, username: str = "lume", password: str = "lume"):
+        super().__init__(ip_address, username, password)
+        self.ws_uri = f"ws://{ip_address}:8000/ws"
+        self._ws = None
+        self._reconnect_task = None
+        self._closed = False
+
+    async def _keep_alive(self):
+        """Keep the WebSocket connection alive with automatic reconnection."""
+        while not self._closed:
+            try:
+                if self._ws is None or self._ws.closed:
+                    self._ws = await websockets.connect(
+                        self.ws_uri,
+                        max_size=1024 * 1024 * 10,  # 10MB limit
+                        max_queue=32
+                    )
+                    logger.info("WebSocket connection established")
+                
+                await asyncio.sleep(30)
+                if self._ws and not self._ws.closed:
+                    try:
+                        pong = await self._ws.ping()
+                        await asyncio.wait_for(pong, timeout=10)
+                    except:
+                        await self._ws.close()
+                        self._ws = None
+                        
+            except Exception as e:
+                logger.error(f"WebSocket connection error: {e}")
+                if self._ws:
+                    await self._ws.close()
+                self._ws = None
+                await asyncio.sleep(5)
+
+    async def _ensure_connection(self):
+        """Ensure WebSocket connection is established."""
+        if self._reconnect_task is None:
+            self._reconnect_task = asyncio.create_task(self._keep_alive())
+        
+        retry_count = 0
+        while self._ws is None or self._ws.closed:
+            if retry_count >= 10:
+                raise ConnectionError("Failed to establish WebSocket connection")
+            await asyncio.sleep(0.5)
+            retry_count += 1
+
+    async def _send_command(self, command: str, params: Optional[Dict] = None) -> Dict[str, Any]:
+        """Send command through WebSocket."""
+        await self._ensure_connection()
+        message = {
+            "command": command,
+            "params": params or {}
+        }
+        await self._ws.send(json.dumps(message))
+        response = await self._ws.recv()
+        return json.loads(response)
+
+    async def wait_for_ready(self, timeout: int = 60, interval: float = 1.0):
+        """Wait for WebSocket connection to become available."""
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            try:
+                if self._reconnect_task is None:
+                    self._reconnect_task = asyncio.create_task(self._keep_alive())
+                await self._ensure_connection()
+                return
+            except Exception:
+                await asyncio.sleep(interval)
+                continue
+        raise TimeoutError(f"Could not connect to {self.ip_address} after {timeout} seconds")
+
+    def close(self):
+        """Close WebSocket connection."""
+        self._closed = True
+        if self._reconnect_task:
+            self._reconnect_task.cancel()
+        if self._ws:
+            asyncio.create_task(self._ws.close())
+            self._ws = None
+
+    # Mouse Actions
+    async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        await self._send_command("left_click", {"x": x, "y": y})
+
+    async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        await self._send_command("right_click", {"x": x, "y": y})
+
+    async def double_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        await self._send_command("double_click", {"x": x, "y": y})
+
+    async def move_cursor(self, x: int, y: int) -> None:
+        await self._send_command("move_cursor", {"x": x, "y": y})
+
+    async def drag_to(self, x: int, y: int, button: str = "left", duration: float = 0.5) -> None:
+        await self._send_command("drag_to", {"x": x, "y": y, "button": button, "duration": duration})
+
+    # Keyboard Actions
+    async def type(self, text: str) -> None:
+        await self._send_command("type", {"text": text})
+
+    async def press_key(self, key: str) -> None:
+        await self._send_command("press_key", {"key": key})
+
+    async def hotkey(self, *keys: str) -> None:
+        await self._send_command("hotkey", {"keys": list(keys)})
+
+    # Scrolling Actions
+    async def scroll_down(self, clicks: int = 1) -> None:
+        await self._send_command("scroll", {"direction": "down", "clicks": clicks})
+
+    async def scroll_up(self, clicks: int = 1) -> None:
+        await self._send_command("scroll", {"direction": "up", "clicks": clicks})
+
+    # Screen Actions
+    async def screenshot(self) -> bytes:
+        result = await self._send_command("screenshot")
+        if result.get('image_data'):
+            import base64
+            return base64.b64decode(result['image_data'])
+        raise RuntimeError("Failed to take screenshot")
+
+    async def get_screen_size(self) -> Dict[str, int]:
+        result = await self._send_command("get_screen_size")
+        if result['success'] and result['size']:
+            return result['size']
+        raise RuntimeError("Failed to get screen size")
+
+    async def get_cursor_position(self) -> Dict[str, int]:
+        result = await self._send_command("get_cursor_position")
+        if result['success'] and result['position']:
+            return result['position']
+        raise RuntimeError("Failed to get cursor position")
+
+    # Clipboard Actions
+    async def copy_to_clipboard(self) -> str:
+        result = await self._send_command("copy_to_clipboard")
+        if result['success'] and result['content']:
+            return result['content']
+        raise RuntimeError("Failed to get clipboard content")
+
+    async def set_clipboard(self, text: str) -> None:
+        await self._send_command("set_clipboard", {"text": text})
+
+    # File System Actions
+    async def file_exists(self, path: str) -> bool:
+        result = await self._send_command("file_exists", {"path": path})
+        return result.get('exists', False)
+
+    async def directory_exists(self, path: str) -> bool:
+        result = await self._send_command("directory_exists", {"path": path})
+        return result.get('exists', False)
+
+    async def run_command(self, command: str) -> Tuple[str, str]:
+        result = await self._send_command("run_command", {"command": command})
+        if not result.get('success', False):
+            raise RuntimeError(result.get('error', 'Failed to run command'))
+        return result.get('stdout', ''), result.get('stderr', '')
+
+    # Accessibility Actions
+    async def get_accessibility_tree(self) -> Dict:
+        result = await self._send_command("get_accessibility_tree")
+        if 'error' in result:
+            raise RuntimeError(f"Failed to get accessibility tree: {result['error']}")
+        return result
+
+    async def find_element_by_criteria(self, 
+                                     role: Optional[str] = None, 
+                                     title: Optional[str] = None,
+                                     value: Optional[str] = None) -> Optional[Dict]:
+        result = await self._send_command("find_element", {
+            "role": role,
+            "title": title,
+            "value": value
+        })
+        return result.get('element') 

cua-0.1.0/computer/interface/macos.py

@@ -0,0 +1,209 @@
+import asyncio
+import websockets
+import json
+import logging
+import socket
+import time
+from typing import Optional, Dict, Any, Tuple, List, TypedDict
+from .base import BaseComputerInterface
+import tempfile
+import subprocess
+import os
+
+logger = logging.getLogger(__name__)
+
+class AccessibilityWindow(TypedDict):
+    app_name: str
+    pid: int
+    frontmost: bool
+    has_windows: bool
+    windows: List[Dict[str, Any]]
+
+class AccessibilityTree(TypedDict):
+    success: bool
+    frontmost_application: str
+    windows: List[AccessibilityWindow]
+
+class MacOSComputerInterface(BaseComputerInterface):
+    """macOS-specific implementation of the computer interface."""
+
+    def __init__(self, ip_address: str, username: str = "lume", password: str = "lume"):
+        super().__init__(ip_address, username, password)
+        self.ws_uri = f"ws://{ip_address}:8000/ws"
+        self._ws = None
+        self._reconnect_task = None
+        self._closed = False
+
+    async def _keep_alive(self):
+        """Keep the WebSocket connection alive with automatic reconnection."""
+        while not self._closed:
+            try:
+                if self._ws is None or self._ws.closed:
+                    self._ws = await websockets.connect(
+                        self.ws_uri,
+                        max_size=1024 * 1024 * 10,  # 10MB limit
+                        max_queue=32
+                    )
+                    logger.info("WebSocket connection established")
+                
+                await asyncio.sleep(30)
+                if self._ws and not self._ws.closed:
+                    try:
+                        pong = await self._ws.ping()
+                        await asyncio.wait_for(pong, timeout=10)
+                    except:
+                        await self._ws.close()
+                        self._ws = None
+                        
+            except Exception as e:
+                logger.error(f"WebSocket connection error: {e}")
+                if self._ws:
+                    await self._ws.close()
+                self._ws = None
+                await asyncio.sleep(5)
+
+    async def _ensure_connection(self):
+        """Ensure WebSocket connection is established."""
+        if self._reconnect_task is None:
+            self._reconnect_task = asyncio.create_task(self._keep_alive())
+        
+        retry_count = 0
+        while self._ws is None or self._ws.closed:
+            if retry_count >= 10:
+                raise ConnectionError("Failed to establish WebSocket connection")
+            await asyncio.sleep(0.5)
+            retry_count += 1
+
+    async def _send_command(self, command: str, params: Optional[Dict] = None) -> Dict[str, Any]:
+        """Send command through WebSocket."""
+        await self._ensure_connection()
+        message = {
+            "command": command,
+            "params": params or {}
+        }
+        await self._ws.send(json.dumps(message))
+        response = await self._ws.recv()
+        return json.loads(response)
+
+    async def wait_for_ready(self, timeout: int = 60, interval: float = 1.0):
+        """Wait for WebSocket connection to become available."""
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            try:
+                if self._reconnect_task is None:
+                    self._reconnect_task = asyncio.create_task(self._keep_alive())
+                await self._ensure_connection()
+                return
+            except Exception:
+                await asyncio.sleep(interval)
+                continue
+        raise TimeoutError(f"Could not connect to {self.ip_address} after {timeout} seconds")
+
+    def close(self):
+        """Close WebSocket connection."""
+        self._closed = True
+        if self._reconnect_task:
+            self._reconnect_task.cancel()
+        if self._ws:
+            asyncio.create_task(self._ws.close())
+            self._ws = None
+
+    # Mouse Actions
+    async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        await self._send_command("left_click", {"x": x, "y": y})
+
+    async def right_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        await self._send_command("right_click", {"x": x, "y": y})
+
+    async def double_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        await self._send_command("double_click", {"x": x, "y": y})
+
+    async def move_cursor(self, x: int, y: int) -> None:
+        await self._send_command("move_cursor", {"x": x, "y": y})
+
+    async def drag_to(self, x: int, y: int, button: str = "left", duration: float = 0.5) -> None:
+        await self._send_command("drag_to", {"x": x, "y": y, "button": button, "duration": duration})
+
+    # Keyboard Actions
+    async def type(self, text: str) -> None:
+        await self._send_command("type", {"text": text})
+
+    async def press_key(self, key: str) -> None:
+        await self._send_command("press_key", {"key": key})
+
+    async def hotkey(self, *keys: str) -> None:
+        await self._send_command("hotkey", {"keys": list(keys)})
+
+    # Scrolling Actions
+    async def scroll_down(self, clicks: int = 1) -> None:
+        await self._send_command("scroll", {"direction": "down", "clicks": clicks})
+
+    async def scroll_up(self, clicks: int = 1) -> None:
+        await self._send_command("scroll", {"direction": "up", "clicks": clicks})
+
+    # Screen Actions
+    async def screenshot(self) -> bytes:
+        result = await self._send_command("screenshot")
+        if result.get('image_data'):
+            import base64
+            return base64.b64decode(result['image_data'])
+        raise RuntimeError("Failed to take screenshot")
+
+    async def get_screen_size(self) -> Dict[str, int]:
+        result = await self._send_command("get_screen_size")
+        if result['success'] and result['size']:
+            return result['size']
+        raise RuntimeError("Failed to get screen size")
+
+    async def get_cursor_position(self) -> Dict[str, int]:
+        result = await self._send_command("get_cursor_position")
+        if result['success'] and result['position']:
+            return result['position']
+        raise RuntimeError("Failed to get cursor position")
+
+    # Clipboard Actions
+    async def copy_to_clipboard(self) -> str:
+        result = await self._send_command("copy_to_clipboard")
+        if result['success'] and result['content']:
+            return result['content']
+        raise RuntimeError("Failed to get clipboard content")
+
+    async def set_clipboard(self, text: str) -> None:
+        await self._send_command("set_clipboard", {"text": text})
+
+    # File System Actions
+    async def file_exists(self, path: str) -> bool:
+        result = await self._send_command("file_exists", {"path": path})
+        return result.get('exists', False)
+
+    async def directory_exists(self, path: str) -> bool:
+        result = await self._send_command("directory_exists", {"path": path})
+        return result.get('exists', False)
+
+    async def run_command(self, command: str) -> Tuple[str, str]:
+        result = await self._send_command("run_command", {"command": command})
+        if not result.get('success', False):
+            raise RuntimeError(result.get('error', 'Failed to run command'))
+        return result.get('stdout', ''), result.get('stderr', '')
+
+    # Accessibility Actions
+    async def get_accessibility_tree(self) -> AccessibilityTree:
+        """Get accessibility tree."""
+        result = await self._send_command("get_accessibility_tree")
+        if not result.get('success', False):
+            raise RuntimeError(result.get('error', 'Failed to get accessibility tree'))
+        return result
+
+    async def find_element_by_criteria(self, 
+                                     role: Optional[str] = None, 
+                                     title: Optional[str] = None,
+                                     value: Optional[str] = None) -> Optional[Dict]:
+        """Find element by accessibility criteria."""
+        result = await self._send_command("find_element", {
+            "role": role,
+            "title": title,
+            "value": value
+        })
+        if not result.get('success', False):
+            raise RuntimeError(result.get('error', 'Failed to find element'))
+        return result.get('element') 

cua-0.1.0/computer/models.py

@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+from typing import Optional
+from pylume import PyLume
+
+@dataclass
+class Display:
+    width: int
+    height: int
+
+@dataclass
+class Image:
+    image: str
+    tag: str
+    name: str
+
+@dataclass
+class Computer:
+    image: str
+    tag: str
+    name: str
+    display: Display
+    memory: str
+    cpu: str
+    pylume: Optional[PyLume] = None
+    
+    @property
+    def ip(self) -> str:
+        """Get the IP address of the VM."""
+        vm = self.pylume.get_vm(self.image.name)
+        return vm.ip 

cua-0.1.0/pyproject.toml

@@ -0,0 +1,35 @@
+[tool.poetry]
+name = "cua"
+version = "0.1.0"
+description = "CUA (Computer Use) interface for host-VM communication on Apple Silicon"
+authors = ["TryCua <f@trycua.com>"]
+readme = "README.md"
+license = "MIT"
+homepage = "https://github.com/trycua/computer"
+repository = "https://github.com/trycua/computer"
+keywords = ["macos", "virtualization", "vm", "apple-silicon"]
+packages = [
+    { include = "computer" }
+]
+classifiers = [
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: MacOS :: MacOS X",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+paramiko = "^3.4.0"
+pylume = "^0.1.8"
+websockets = "^12.0"
+
+[tool.poetry.group.dev.dependencies]
+ipykernel = "^6.29.5"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"