cua-computer 0.2.6__py3-none-any.whl → 0.2.8__py3-none-any.whl

computer/computer.py

@@ -21,6 +21,20 @@ OSType = Literal["macos", "linux", "windows"]
 class Computer:
     """Computer is the main class for interacting with the computer."""
 
+    def create_desktop_from_apps(self, apps):
+        """
+        Create a virtual desktop from a list of app names, returning a DioramaComputer
+        that proxies Diorama.Interface but uses diorama_cmds via the computer interface.
+
+        Args:
+            apps (list[str]): List of application names to include in the desktop.
+        Returns:
+            DioramaComputer: A proxy object with the Diorama interface, but using diorama_cmds.
+        """
+        assert "app-use" in self.experiments, "App Usage is an experimental feature. Enable it by passing experiments=['app-use'] to Computer()"
+        from .diorama_computer import DioramaComputer
+        return DioramaComputer(self, apps)
+
     def __init__(
         self,
         display: Union[Display, Dict[str, int], str] = "1024x768",
@@ -39,7 +53,8 @@ class Computer:
         host: str = os.environ.get("PYLUME_HOST", "localhost"),
         storage: Optional[str] = None,
         ephemeral: bool = False,
-        api_key: Optional[str] = None
+        api_key: Optional[str] = None,
+        experiments: Optional[List[str]] = None
     ):
         """Initialize a new Computer instance.
 
@@ -65,6 +80,8 @@ class Computer:
             host: Host to use for VM provider connections (e.g. "localhost", "host.docker.internal")
             storage: Optional path for persistent VM storage (Lumier provider)
             ephemeral: Whether to use ephemeral storage
+            api_key: Optional API key for cloud providers
+            experiments: Optional list of experimental features to enable (e.g. ["app-use"])
         """
 
         self.logger = Logger("cua.computer", verbosity)
@@ -80,6 +97,10 @@ class Computer:
         self.ephemeral = ephemeral
         
         self.api_key = api_key
+        self.experiments = experiments or []
+        
+        if "app-use" in self.experiments:
+            assert self.os_type == "macos", "App use experiment is only supported on macOS"
 
         # The default is currently to use non-ephemeral storage
         if storage and ephemeral and storage != "ephemeral":

computer/diorama_computer.py

@@ -0,0 +1,93 @@
+import asyncio
+
+class DioramaComputer:
+    """
+    A Computer-compatible proxy for Diorama that sends commands over the ComputerInterface.
+    """
+    def __init__(self, computer, apps):
+        self.computer = computer
+        self.apps = apps
+        self.interface = DioramaComputerInterface(computer, apps)
+        self._initialized = False
+
+    async def __aenter__(self):
+        self._initialized = True
+        return self
+
+    async def run(self):
+        if not self._initialized:
+            await self.__aenter__()
+        return self
+
+class DioramaComputerInterface:
+    """
+    Diorama Interface proxy that sends diorama_cmds via the Computer's interface.
+    """
+    def __init__(self, computer, apps):
+        self.computer = computer
+        self.apps = apps
+        self._scene_size = None
+
+    async def _send_cmd(self, action, arguments=None):
+        arguments = arguments or {}
+        arguments = {"app_list": self.apps, **arguments}
+        # Use the computer's interface (must be initialized)
+        iface = getattr(self.computer, "_interface", None)
+        if iface is None:
+            raise RuntimeError("Computer interface not initialized. Call run() first.")
+        result = await iface.diorama_cmd(action, arguments)
+        if not result.get("success"):
+            raise RuntimeError(f"Diorama command failed: {result.get('error')}")
+        return result.get("result")
+
+    async def screenshot(self, as_bytes=True):
+        from PIL import Image
+        import base64
+        result = await self._send_cmd("screenshot")
+        # assume result is a b64 string of an image
+        img_bytes = base64.b64decode(result)
+        import io
+        img = Image.open(io.BytesIO(img_bytes))
+        self._scene_size = img.size
+        return img_bytes if as_bytes else img
+
+    async def get_screen_size(self):
+        if not self._scene_size:
+            await self.screenshot(as_bytes=False)
+        return {"width": self._scene_size[0], "height": self._scene_size[1]}
+
+    async def move_cursor(self, x, y):
+        await self._send_cmd("move_cursor", {"x": x, "y": y})
+
+    async def left_click(self, x=None, y=None):
+        await self._send_cmd("left_click", {"x": x, "y": y})
+
+    async def right_click(self, x=None, y=None):
+        await self._send_cmd("right_click", {"x": x, "y": y})
+
+    async def double_click(self, x=None, y=None):
+        await self._send_cmd("double_click", {"x": x, "y": y})
+
+    async def scroll_up(self, clicks=1):
+        await self._send_cmd("scroll_up", {"clicks": clicks})
+
+    async def scroll_down(self, clicks=1):
+        await self._send_cmd("scroll_down", {"clicks": clicks})
+
+    async def drag_to(self, x, y, duration=0.5):
+        await self._send_cmd("drag_to", {"x": x, "y": y, "duration": duration})
+
+    async def get_cursor_position(self):
+        return await self._send_cmd("get_cursor_position")
+
+    async def type_text(self, text):
+        await self._send_cmd("type_text", {"text": text})
+
+    async def press_key(self, key):
+        await self._send_cmd("press_key", {"key": key})
+
+    async def hotkey(self, *keys):
+        await self._send_cmd("hotkey", {"keys": list(keys)})
+
+    async def to_screen_coordinates(self, x, y):
+        return await self._send_cmd("to_screen_coordinates", {"x": x, "y": y})

computer/interface/base.py

@@ -177,7 +177,7 @@ class BaseComputerInterface(ABC):
     async def get_accessibility_tree(self) -> Dict:
         """Get the accessibility tree of the current screen."""
         pass
-
+    
     @abstractmethod
     async def to_screen_coordinates(self, x: float, y: float) -> tuple[float, float]:
         """Convert screenshot coordinates to screen coordinates.

computer/interface/macos.py

@@ -346,6 +346,10 @@ class MacOSComputerInterface(BaseComputerInterface):
             asyncio.create_task(self._ws.close())
             self._ws = None
 
+    async def diorama_cmd(self, action: str, arguments: Optional[dict] = None) -> dict:
+        """Send a diorama command to the server (macOS only)."""
+        return await self._send_command("diorama_cmd", {"action": action, "arguments": arguments or {}})
+
     # Mouse Actions
     async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
         await self._send_command("left_click", {"x": x, "y": y})
@@ -568,7 +572,7 @@ class MacOSComputerInterface(BaseComputerInterface):
         if not result.get("success", False):
             raise RuntimeError(result.get("error", "Failed to get accessibility tree"))
         return result
-
+    
     async def get_active_window_bounds(self) -> Dict[str, int]:
         """Get the bounds of the currently active window."""
         result = await self._send_command("get_active_window_bounds")

computer/providers/base.py

@@ -1,11 +1,11 @@
 """Base provider interface for VM backends."""
 
 import abc
-from enum import Enum
+from enum import StrEnum
 from typing import Dict, List, Optional, Any, AsyncContextManager
 
 
-class VMProviderType(str, Enum):
+class VMProviderType(StrEnum):
     """Enum of supported VM provider types."""
     LUME = "lume"
     LUMIER = "lumier"

{cua_computer-0.2.6.dist-info → cua_computer-0.2.8.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: cua-computer
-Version: 0.2.6
+Version: 0.2.8
 Summary: Computer-Use Interface (CUI) framework powering Cua
 Author-Email: TryCua <gh@trycua.com>
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Requires-Dist: pillow>=10.0.0
 Requires-Dist: websocket-client>=1.8.0
 Requires-Dist: websockets>=12.0

{cua_computer-0.2.6.dist-info → cua_computer-0.2.8.dist-info}/RECORD

@@ -1,15 +1,16 @@
 computer/__init__.py,sha256=QOxNrrJAuLRnsUC2zIFgRfzVSuDSXiYHlEF-9vkhV0o,1241
-computer/computer.py,sha256=Rc32XFZdKr7XZKO0zhbEom-REvYYPPlvmvjDbw5gP9k,32218
+computer/computer.py,sha256=FtHpNcVjQKNclLlQo5Idt2tBYnuf0417nzVjDMGTRxk,33306
+computer/diorama_computer.py,sha256=tOzTCTyARD38Eeoc8gjF1rF3eo0kYCkV-70bsxsKAZE,3297
 computer/interface/__init__.py,sha256=xQvYjq5PMn9ZJOmRR5mWtONTl_0HVd8ACvW6AQnzDdw,262
-computer/interface/base.py,sha256=CD9WpDp-6qP-ID5MjhXA8qpYs0XhJ4TPkR917l2FFSo,6021
+computer/interface/base.py,sha256=wmLBiX7rB8cG2Q4fmchdKpjralktzicuYhAh6fDIeqw,6025
 computer/interface/factory.py,sha256=RjAZAB_jFuS8JierYjLbapRX6RqFE0qE3BiIyP5UDOE,1441
 computer/interface/linux.py,sha256=CT1N0QA52TNKBbFG2LXdN6yAGWWJ12_2hTMEI8yNoM4,26865
-computer/interface/macos.py,sha256=_8R_IroxbcVmh1WagrjDQOitaT6tVkCHVzGgA_lwTrM,27077
+computer/interface/macos.py,sha256=SZh3CB_Co9y5xPVfPoo1yIXFqAyDoRSx5nEXNN1od1I,27340
 computer/interface/models.py,sha256=RZKVUdwKrKUoFqwlx2Dk8Egkmq_AInlIu_d0xg7SZzw,3238
 computer/logger.py,sha256=UVvnmZGOWVF9TCsixEbeQnDZ3wBPAJ2anW3Zp-MoJ8Y,2896
 computer/models.py,sha256=iFNM1QfZArD8uf66XJXb2EDIREsfrxqqA5_liLBMfrE,1188
 computer/providers/__init__.py,sha256=hS9lLxmmHa1u82XJJ_xuqSKipClsYUEPx-8OK9ogtVg,194
-computer/providers/base.py,sha256=pUrM5aVBSUQS2TbfoiyMkNz20dWN7-F_aEjSf8EiJRc,3623
+computer/providers/base.py,sha256=J_9r6pJsvGAFDRl56jog_atN7e8uzrvlCQEdRRqye_U,3624
 computer/providers/cloud/__init__.py,sha256=SDAcfhI2BlmVBrBZOHxQd3i1bJZjMIfl7QgmqjXa4z8,144
 computer/providers/cloud/provider.py,sha256=gpBl_ZVbwk-0FhYycne-69KslnrAoDSZcyzetpLfiKE,2864
 computer/providers/factory.py,sha256=9qVdt-fIovSNOokGMZ_2B1VPCLSZeDky4edcXyelZy4,4616
@@ -23,7 +24,7 @@ computer/ui/__init__.py,sha256=pmo05ek9qiB_x7DPeE6Vf_8RsIOqTD0w1dBLMHfoOnY,45
 computer/ui/gradio/__init__.py,sha256=5_KimixM48-X74FCsLw7LbSt39MQfUMEL8-M9amK3Cw,117
 computer/ui/gradio/app.py,sha256=o31nphBcb6zM5OKPuODTjuOzSJ3lt61kQHpUeMBBs70,65077
 computer/utils.py,sha256=zY50NXB7r51GNLQ6l7lhG_qv0_ufpQ8n0-SDhCei8m4,2838
-cua_computer-0.2.6.dist-info/METADATA,sha256=2eZab1PZTMnKx2lIrYSOZGCI5gT596kOmp5Hu1xb4Tg,5844
-cua_computer-0.2.6.dist-info/WHEEL,sha256=tSfRZzRHthuv7vxpI4aehrdN9scLjk-dCJkPLzkHxGg,90
-cua_computer-0.2.6.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
-cua_computer-0.2.6.dist-info/RECORD,,
+cua_computer-0.2.8.dist-info/METADATA,sha256=DU7TQmx6VicwaYyE6faehHUy6oLweVKJ1nGBOctOSGY,5844
+cua_computer-0.2.8.dist-info/WHEEL,sha256=tSfRZzRHthuv7vxpI4aehrdN9scLjk-dCJkPLzkHxGg,90
+cua_computer-0.2.8.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
+cua_computer-0.2.8.dist-info/RECORD,,