cua-computer 0.2.7__py3-none-any.whl → 0.2.9__py3-none-any.whl

computer/computer.py

@@ -11,6 +11,7 @@ import json
 import logging
 from .telemetry import record_computer_initialization
 import os
+from . import helpers
 
 # Import provider related modules
 from .providers.base import VMProviderType
@@ -21,6 +22,20 @@ OSType = Literal["macos", "linux", "windows"]
 class Computer:
     """Computer is the main class for interacting with the computer."""
 
+    def create_desktop_from_apps(self, apps):
+        """
+        Create a virtual desktop from a list of app names, returning a DioramaComputer
+        that proxies Diorama.Interface but uses diorama_cmds via the computer interface.
+
+        Args:
+            apps (list[str]): List of application names to include in the desktop.
+        Returns:
+            DioramaComputer: A proxy object with the Diorama interface, but using diorama_cmds.
+        """
+        assert "app-use" in self.experiments, "App Usage is an experimental feature. Enable it by passing experiments=['app-use'] to Computer()"
+        from .diorama_computer import DioramaComputer
+        return DioramaComputer(self, apps)
+
     def __init__(
         self,
         display: Union[Display, Dict[str, int], str] = "1024x768",
@@ -39,7 +54,8 @@ class Computer:
         host: str = os.environ.get("PYLUME_HOST", "localhost"),
         storage: Optional[str] = None,
         ephemeral: bool = False,
-        api_key: Optional[str] = None
+        api_key: Optional[str] = None,
+        experiments: Optional[List[str]] = None
     ):
         """Initialize a new Computer instance.
 
@@ -65,6 +81,8 @@ class Computer:
             host: Host to use for VM provider connections (e.g. "localhost", "host.docker.internal")
             storage: Optional path for persistent VM storage (Lumier provider)
             ephemeral: Whether to use ephemeral storage
+            api_key: Optional API key for cloud providers
+            experiments: Optional list of experimental features to enable (e.g. ["app-use"])
         """
 
         self.logger = Logger("cua.computer", verbosity)
@@ -80,6 +98,10 @@ class Computer:
         self.ephemeral = ephemeral
         
         self.api_key = api_key
+        self.experiments = experiments or []
+        
+        if "app-use" in self.experiments:
+            assert self.os_type == "macos", "App use experiment is only supported on macOS"
 
         # The default is currently to use non-ephemeral storage
         if storage and ephemeral and storage != "ephemeral":
@@ -439,6 +461,10 @@ class Computer:
 
             # Set the initialization flag and clear the initializing flag
             self._initialized = True
+            
+            # Set this instance as the default computer for remote decorators
+            helpers.set_default_computer(self)
+            
             self.logger.info("Computer successfully initialized")
         except Exception as e:
             raise
@@ -701,3 +727,177 @@ class Computer:
             tuple[float, float]: (x, y) coordinates in screenshot space
         """
         return await self.interface.to_screenshot_coordinates(x, y)
+
+
+    # Add virtual environment management functions to computer interface
+    async def venv_install(self, venv_name: str, requirements: list[str]) -> tuple[str, str]:
+        """Install packages in a virtual environment.
+        
+        Args:
+            venv_name: Name of the virtual environment
+            requirements: List of package requirements to install
+            
+        Returns:
+            Tuple of (stdout, stderr) from the installation command
+        """
+        requirements = requirements or []
+
+        # Create virtual environment if it doesn't exist
+        venv_path = f"~/.venvs/{venv_name}"
+        create_cmd = f"mkdir -p ~/.venvs && python3 -m venv {venv_path}"
+        
+        # Check if venv exists, if not create it
+        check_cmd = f"test -d {venv_path} || ({create_cmd})"
+        _, _ = await self.interface.run_command(check_cmd)
+        
+        # Install packages
+        requirements_str = " ".join(requirements)
+        install_cmd = f". {venv_path}/bin/activate && pip install {requirements_str}"
+        return await self.interface.run_command(install_cmd)
+    
+    async def venv_cmd(self, venv_name: str, command: str) -> tuple[str, str]:
+        """Execute a shell command in a virtual environment.
+        
+        Args:
+            venv_name: Name of the virtual environment
+            command: Shell command to execute in the virtual environment
+            
+        Returns:
+            Tuple of (stdout, stderr) from the command execution
+        """
+        venv_path = f"~/.venvs/{venv_name}"
+        
+        # Check if virtual environment exists
+        check_cmd = f"test -d {venv_path}"
+        stdout, stderr = await self.interface.run_command(check_cmd)
+        
+        if stderr or "test:" in stdout:  # venv doesn't exist
+            return "", f"Virtual environment '{venv_name}' does not exist. Create it first using venv_install."
+        
+        # Activate virtual environment and run command
+        full_command = f". {venv_path}/bin/activate && {command}"
+        return await self.interface.run_command(full_command)
+    
+    async def venv_exec(self, venv_name: str, python_func, *args, **kwargs):
+        """Execute Python function in a virtual environment using source code extraction.
+        
+        Args:
+            venv_name: Name of the virtual environment
+            python_func: A callable function to execute
+            *args: Positional arguments to pass to the function
+            **kwargs: Keyword arguments to pass to the function
+            
+        Returns:
+            The result of the function execution, or raises any exception that occurred
+        """
+        import base64
+        import inspect
+        import json
+        import textwrap
+        
+        try:
+            # Get function source code using inspect.getsource
+            source = inspect.getsource(python_func)
+            # Remove common leading whitespace (dedent)
+            func_source = textwrap.dedent(source).strip()
+            
+            # Remove decorators
+            while func_source.lstrip().startswith("@"):
+                func_source = func_source.split("\n", 1)[1].strip()
+            
+            # Get function name for execution
+            func_name = python_func.__name__
+            
+            # Serialize args and kwargs as JSON (safer than dill for cross-version compatibility)
+            args_json = json.dumps(args, default=str)
+            kwargs_json = json.dumps(kwargs, default=str)
+            
+        except OSError as e:
+            raise Exception(f"Cannot retrieve source code for function {python_func.__name__}: {e}")
+        except Exception as e:
+            raise Exception(f"Failed to reconstruct function source: {e}")
+        
+        # Create Python code that will define and execute the function
+        python_code = f'''
+import json
+import traceback
+
+try:
+    # Define the function from source
+{textwrap.indent(func_source, "    ")}
+    
+    # Deserialize args and kwargs from JSON
+    args_json = """{args_json}"""
+    kwargs_json = """{kwargs_json}"""
+    args = json.loads(args_json)
+    kwargs = json.loads(kwargs_json)
+    
+    # Execute the function
+    result = {func_name}(*args, **kwargs)
+
+    # Create success output payload
+    output_payload = {{
+        "success": True,
+        "result": result,
+        "error": None
+    }}
+    
+except Exception as e:
+    # Create error output payload
+    output_payload = {{
+        "success": False,
+        "result": None,
+        "error": {{
+            "type": type(e).__name__,
+            "message": str(e),
+            "traceback": traceback.format_exc()
+        }}
+    }}
+
+# Serialize the output payload as JSON
+import json
+output_json = json.dumps(output_payload, default=str)
+
+# Print the JSON output with markers
+print(f"<<<VENV_EXEC_START>>>{{output_json}}<<<VENV_EXEC_END>>>")
+'''
+        
+        # Encode the Python code in base64 to avoid shell escaping issues
+        encoded_code = base64.b64encode(python_code.encode('utf-8')).decode('ascii')
+        
+        # Execute the Python code in the virtual environment
+        python_command = f"python -c \"import base64; exec(base64.b64decode('{encoded_code}').decode('utf-8'))\""
+        stdout, stderr = await self.venv_cmd(venv_name, python_command)
+        
+        # Parse the output to extract the payload
+        start_marker = "<<<VENV_EXEC_START>>>"
+        end_marker = "<<<VENV_EXEC_END>>>"
+
+        # Print original stdout
+        print(stdout[:stdout.find(start_marker)])
+        
+        if start_marker in stdout and end_marker in stdout:
+            start_idx = stdout.find(start_marker) + len(start_marker)
+            end_idx = stdout.find(end_marker)
+            
+            if start_idx < end_idx:
+                output_json = stdout[start_idx:end_idx]
+
+                try:
+                    # Decode and deserialize the output payload from JSON
+                    output_payload = json.loads(output_json)
+                except Exception as e:
+                    raise Exception(f"Failed to decode output payload: {e}")
+                
+                if output_payload["success"]:
+                    return output_payload["result"]
+                else:
+                    # Recreate and raise the original exception
+                    error_info = output_payload["error"]
+                    error_class = eval(error_info["type"])
+                    raise error_class(error_info["message"])
+            else:
+                raise Exception("Invalid output format: markers found but no content between them")
+        else:
+            # Fallback: return stdout/stderr if no payload markers found
+            raise Exception(f"No output payload found. stdout: {stdout}, stderr: {stderr}")

computer/diorama_computer.py

@@ -0,0 +1,104 @@
+import asyncio
+from .interface.models import KeyType, Key
+
+class DioramaComputer:
+    """
+    A Computer-compatible proxy for Diorama that sends commands over the ComputerInterface.
+    """
+    def __init__(self, computer, apps):
+        self.computer = computer
+        self.apps = apps
+        self.interface = DioramaComputerInterface(computer, apps)
+        self._initialized = False
+
+    async def __aenter__(self):
+        self._initialized = True
+        return self
+
+    async def run(self):
+        if not self._initialized:
+            await self.__aenter__()
+        return self
+
+class DioramaComputerInterface:
+    """
+    Diorama Interface proxy that sends diorama_cmds via the Computer's interface.
+    """
+    def __init__(self, computer, apps):
+        self.computer = computer
+        self.apps = apps
+        self._scene_size = None
+
+    async def _send_cmd(self, action, arguments=None):
+        arguments = arguments or {}
+        arguments = {"app_list": self.apps, **arguments}
+        # Use the computer's interface (must be initialized)
+        iface = getattr(self.computer, "_interface", None)
+        if iface is None:
+            raise RuntimeError("Computer interface not initialized. Call run() first.")
+        result = await iface.diorama_cmd(action, arguments)
+        if not result.get("success"):
+            raise RuntimeError(f"Diorama command failed: {result.get('error')}\n{result.get('trace')}")
+        return result.get("result")
+
+    async def screenshot(self, as_bytes=True):
+        from PIL import Image
+        import base64
+        result = await self._send_cmd("screenshot")
+        # assume result is a b64 string of an image
+        img_bytes = base64.b64decode(result)
+        import io
+        img = Image.open(io.BytesIO(img_bytes))
+        self._scene_size = img.size
+        return img_bytes if as_bytes else img
+
+    async def get_screen_size(self):
+        if not self._scene_size:
+            await self.screenshot(as_bytes=False)
+        return {"width": self._scene_size[0], "height": self._scene_size[1]}
+
+    async def move_cursor(self, x, y):
+        await self._send_cmd("move_cursor", {"x": x, "y": y})
+
+    async def left_click(self, x=None, y=None):
+        await self._send_cmd("left_click", {"x": x, "y": y})
+
+    async def right_click(self, x=None, y=None):
+        await self._send_cmd("right_click", {"x": x, "y": y})
+
+    async def double_click(self, x=None, y=None):
+        await self._send_cmd("double_click", {"x": x, "y": y})
+
+    async def scroll_up(self, clicks=1):
+        await self._send_cmd("scroll_up", {"clicks": clicks})
+
+    async def scroll_down(self, clicks=1):
+        await self._send_cmd("scroll_down", {"clicks": clicks})
+
+    async def drag_to(self, x, y, duration=0.5):
+        await self._send_cmd("drag_to", {"x": x, "y": y, "duration": duration})
+
+    async def get_cursor_position(self):
+        return await self._send_cmd("get_cursor_position")
+
+    async def type_text(self, text):
+        await self._send_cmd("type_text", {"text": text})
+
+    async def press_key(self, key):
+        await self._send_cmd("press_key", {"key": key})
+
+    async def hotkey(self, *keys):
+        actual_keys = []
+        for key in keys:
+            if isinstance(key, Key):
+                actual_keys.append(key.value)
+            elif isinstance(key, str):
+                # Try to convert to enum if it matches a known key
+                key_or_enum = Key.from_string(key)
+                actual_keys.append(key_or_enum.value if isinstance(key_or_enum, Key) else key_or_enum)
+            else:
+                raise ValueError(f"Invalid key type: {type(key)}. Must be Key enum or string.")
+        await self._send_cmd("hotkey", {"keys": actual_keys})
+
+    async def to_screen_coordinates(self, x, y):
+        return await self._send_cmd("to_screen_coordinates", {"x": x, "y": y})

computer/helpers.py

@@ -0,0 +1,49 @@
+"""
+Helper functions and decorators for the Computer module.
+"""
+import asyncio
+from functools import wraps
+from typing import Any, Callable, Optional, TypeVar, cast
+
+# Global reference to the default computer instance
+_default_computer = None
+
+def set_default_computer(computer):
+    """
+    Set the default computer instance to be used by the remote decorator.
+    
+    Args:
+        computer: The computer instance to use as default
+    """
+    global _default_computer
+    _default_computer = computer
+
+
+def sandboxed(venv_name: str = "default", computer: str = "default", max_retries: int = 3):
+    """
+    Decorator that wraps a function to be executed remotely via computer.venv_exec
+    
+    Args:
+        venv_name: Name of the virtual environment to execute in
+        computer: The computer instance to use, or "default" to use the globally set default
+        max_retries: Maximum number of retries for the remote execution
+    """
+    def decorator(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            # Determine which computer instance to use
+            comp = computer if computer != "default" else _default_computer
+            
+            if comp is None:
+                raise RuntimeError("No computer instance available. Either specify a computer instance or call set_default_computer() first.")
+            
+            for i in range(max_retries):
+                try:
+                    return await comp.venv_exec(venv_name, func, *args, **kwargs)
+                except Exception as e:
+                    print(f"Attempt {i+1} failed: {e}")
+                    await asyncio.sleep(1)
+                    if i == max_retries - 1:
+                        raise e
+        return wrapper
+    return decorator

computer/interface/base.py

@@ -177,7 +177,7 @@ class BaseComputerInterface(ABC):
     async def get_accessibility_tree(self) -> Dict:
         """Get the accessibility tree of the current screen."""
         pass
-
+    
     @abstractmethod
     async def to_screen_coordinates(self, x: float, y: float) -> tuple[float, float]:
         """Convert screenshot coordinates to screen coordinates.

computer/interface/linux.py

@@ -27,6 +27,7 @@ class LinuxComputerInterface(BaseComputerInterface):
         self._max_reconnect_delay = 30  # Maximum delay between reconnection attempts
         self._log_connection_attempts = True  # Flag to control connection attempt logging
         self._authenticated = False  # Track authentication status
+        self._command_lock = asyncio.Lock()  # Lock to ensure only one command at a time
 
         # Set logger name for Linux interface
         self.logger = Logger("cua.interface.linux", LogLevel.NORMAL)
@@ -193,58 +194,62 @@ class LinuxComputerInterface(BaseComputerInterface):
         retry_count = 0
         last_error = None
 
-        while retry_count < max_retries:
-            try:
-                await self._ensure_connection()
-                if not self._ws:
-                    raise ConnectionError("WebSocket connection is not established")
-
-                # Handle authentication if needed
-                if self.api_key and self.vm_name and not self._authenticated:
-                    self.logger.info("Performing authentication handshake...")
-                    auth_message = {
-                        "command": "authenticate",
-                        "params": {
-                            "api_key": self.api_key,
-                            "container_name": self.vm_name
+        # Acquire lock to ensure only one command is processed at a time
+        async with self._command_lock:
+            self.logger.debug(f"Acquired lock for command: {command}")
+            while retry_count < max_retries:
+                try:
+                    await self._ensure_connection()
+                    if not self._ws:
+                        raise ConnectionError("WebSocket connection is not established")
+
+                    # Handle authentication if needed
+                    if self.api_key and self.vm_name and not self._authenticated:
+                        self.logger.info("Performing authentication handshake...")
+                        auth_message = {
+                            "command": "authenticate",
+                            "params": {
+                                "api_key": self.api_key,
+                                "container_name": self.vm_name
+                            }
                         }
-                    }
-                    await self._ws.send(json.dumps(auth_message))
-                    
-                    # Wait for authentication response
-                    auth_response = await asyncio.wait_for(self._ws.recv(), timeout=10)
-                    auth_result = json.loads(auth_response)
-                    
-                    if not auth_result.get("success"):
-                        error_msg = auth_result.get("error", "Authentication failed")
-                        self.logger.error(f"Authentication failed: {error_msg}")
-                        self._authenticated = False
-                        raise ConnectionError(f"Authentication failed: {error_msg}")
-                    
-                    self.logger.info("Authentication successful")
-                    self._authenticated = True
-
-                message = {"command": command, "params": params or {}}
-                await self._ws.send(json.dumps(message))
-                response = await asyncio.wait_for(self._ws.recv(), timeout=30)
-                return json.loads(response)
-            except Exception as e:
-                last_error = e
-                retry_count += 1
-                if retry_count < max_retries:
-                    # Only log at debug level for intermediate retries
-                    self.logger.debug(
-                        f"Command '{command}' failed (attempt {retry_count}/{max_retries}): {e}"
-                    )
-                    await asyncio.sleep(1)
-                    continue
-                else:
-                    # Only log at error level for the final failure
-                    self.logger.error(
-                        f"Failed to send command '{command}' after {max_retries} retries"
-                    )
-                    self.logger.debug(f"Command failure details: {e}")
-                raise last_error if last_error else RuntimeError("Failed to send command")
+                        await self._ws.send(json.dumps(auth_message))
+                        
+                        # Wait for authentication response
+                        auth_response = await asyncio.wait_for(self._ws.recv(), timeout=10)
+                        auth_result = json.loads(auth_response)
+                        
+                        if not auth_result.get("success"):
+                            error_msg = auth_result.get("error", "Authentication failed")
+                            self.logger.error(f"Authentication failed: {error_msg}")
+                            self._authenticated = False
+                            raise ConnectionError(f"Authentication failed: {error_msg}")
+                        
+                        self.logger.info("Authentication successful")
+                        self._authenticated = True
+
+                    message = {"command": command, "params": params or {}}
+                    await self._ws.send(json.dumps(message))
+                    response = await asyncio.wait_for(self._ws.recv(), timeout=30)
+                    self.logger.debug(f"Completed command: {command}")
+                    return json.loads(response)
+                except Exception as e:
+                    last_error = e
+                    retry_count += 1
+                    if retry_count < max_retries:
+                        # Only log at debug level for intermediate retries
+                        self.logger.debug(
+                            f"Command '{command}' failed (attempt {retry_count}/{max_retries}): {e}"
+                        )
+                        await asyncio.sleep(1)
+                        continue
+                    else:
+                        # Only log at error level for the final failure
+                        self.logger.error(
+                            f"Failed to send command '{command}' after {max_retries} retries"
+                        )
+                        self.logger.debug(f"Command failure details: {e}")
+                        raise last_error if last_error else RuntimeError("Failed to send command")
 
     async def wait_for_ready(self, timeout: int = 60, interval: float = 1.0):
         """Wait for WebSocket connection to become available."""

computer/interface/macos.py

@@ -26,6 +26,7 @@ class MacOSComputerInterface(BaseComputerInterface):
         self._reconnect_delay = 1  # Start with 1 second delay
         self._max_reconnect_delay = 30  # Maximum delay between reconnection attempts
         self._log_connection_attempts = True  # Flag to control connection attempt logging
+        self._command_lock = asyncio.Lock()  # Lock to ensure only one command at a time
 
         # Set logger name for macOS interface
         self.logger = Logger("cua.interface.macos", LogLevel.NORMAL)
@@ -219,35 +220,39 @@ class MacOSComputerInterface(BaseComputerInterface):
         retry_count = 0
         last_error = None
 
-        while retry_count < max_retries:
-            try:
-                await self._ensure_connection()
-                if not self._ws:
-                    raise ConnectionError("WebSocket connection is not established")
-
-                message = {"command": command, "params": params or {}}
-                await self._ws.send(json.dumps(message))
-                response = await asyncio.wait_for(self._ws.recv(), timeout=30)
-                return json.loads(response)
-            except Exception as e:
-                last_error = e
-                retry_count += 1
-                if retry_count < max_retries:
-                    # Only log at debug level for intermediate retries
-                    self.logger.debug(
-                        f"Command '{command}' failed (attempt {retry_count}/{max_retries}): {e}"
-                    )
-                    await asyncio.sleep(1)
-                    continue
-                else:
-                    # Only log at error level for the final failure
-                    self.logger.error(
-                        f"Failed to send command '{command}' after {max_retries} retries"
-                    )
-                    self.logger.debug(f"Command failure details: {e}")
-                    raise
+        # Acquire lock to ensure only one command is processed at a time
+        async with self._command_lock:
+            self.logger.debug(f"Acquired lock for command: {command}")
+            while retry_count < max_retries:
+                try:
+                    await self._ensure_connection()
+                    if not self._ws:
+                        raise ConnectionError("WebSocket connection is not established")
+
+                    message = {"command": command, "params": params or {}}
+                    await self._ws.send(json.dumps(message))
+                    response = await asyncio.wait_for(self._ws.recv(), timeout=30)
+                    self.logger.debug(f"Completed command: {command}")
+                    return json.loads(response)
+                except Exception as e:
+                    last_error = e
+                    retry_count += 1
+                    if retry_count < max_retries:
+                        # Only log at debug level for intermediate retries
+                        self.logger.debug(
+                            f"Command '{command}' failed (attempt {retry_count}/{max_retries}): {e}"
+                        )
+                        await asyncio.sleep(1)
+                        continue
+                    else:
+                        # Only log at error level for the final failure
+                        self.logger.error(
+                            f"Failed to send command '{command}' after {max_retries} retries"
+                        )
+                        self.logger.debug(f"Command failure details: {e}")
+                        raise
 
-        raise last_error if last_error else RuntimeError("Failed to send command")
+            raise last_error if last_error else RuntimeError("Failed to send command")
 
     async def wait_for_ready(self, timeout: int = 60, interval: float = 1.0):
         """Wait for WebSocket connection to become available."""
@@ -346,6 +351,10 @@ class MacOSComputerInterface(BaseComputerInterface):
             asyncio.create_task(self._ws.close())
             self._ws = None
 
+    async def diorama_cmd(self, action: str, arguments: Optional[dict] = None) -> dict:
+        """Send a diorama command to the server (macOS only)."""
+        return await self._send_command("diorama_cmd", {"action": action, "arguments": arguments or {}})
+
     # Mouse Actions
     async def left_click(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
         await self._send_command("left_click", {"x": x, "y": y})
@@ -568,7 +577,7 @@ class MacOSComputerInterface(BaseComputerInterface):
         if not result.get("success", False):
             raise RuntimeError(result.get("error", "Failed to get accessibility tree"))
         return result
-
+    
     async def get_active_window_bounds(self) -> Dict[str, int]:
         """Get the bounds of the currently active window."""
         result = await self._send_command("get_active_window_bounds")

computer/ui/gradio/app.py

@@ -463,7 +463,7 @@ async def execute(name, action, arguments):
         elif action == "left_click":
             if "x" in arguments and "y" in arguments:
                 await computer.interface.move_cursor(arguments["x"], arguments["y"])
-            await computer.interface.left_click()
+            await computer.interface.left_click(arguments["x"], arguments["y"])
             await asyncio.sleep(0.5)
         elif action == "right_click":
             if "x" in arguments and "y" in arguments:
@@ -528,43 +528,75 @@ async def execute(name, action, arguments):
     
     return results
 
-async def handle_init_computer(os_choice: str):
-    """Initialize the computer instance and tools for macOS or Ubuntu"""
+async def handle_init_computer(os_choice: str, app_list=None, provider="lume"):
+    """Initialize the computer instance and tools for macOS or Ubuntu
+    
+    Args:
+        os_choice: The OS to use ("macOS" or "Ubuntu")
+        app_list: Optional list of apps to focus on using the app-use experiment
+        provider: The provider to use ("lume" or "self")
+    """
     global computer, tool_call_logs, tools
-
+    
+    # Check if we should enable app-use experiment
+    use_app_experiment = app_list and len(app_list) > 0
+    experiments = ["app-use"] if use_app_experiment else None
+    
+    # Determine if we should use host computer server
+    use_host_computer_server = provider == "self"
+    
     if os_choice == "Ubuntu":
-        computer = Computer(
-            image="ubuntu-noble-vanilla:latest",
-            os_type="linux",
-            provider_type=VMProviderType.LUME,
-            display="1024x768",
-            memory="8GB",
-            cpu="4"
-        )
         os_type_str = "linux"
         image_str = "ubuntu-noble-vanilla:latest"
+    else:
+        os_type_str = "macos"
+        image_str = "macos-sequoia-cua:latest"
+    
+    # Create computer instance with appropriate configuration
+    if use_host_computer_server:
+        computer = Computer(
+            os_type=os_type_str,
+            use_host_computer_server=True,
+            experiments=experiments
+        )
     else:
         computer = Computer(
-            image="macos-sequoia-cua:latest",
-            os_type="macos",
+            image=image_str,
+            os_type=os_type_str,
             provider_type=VMProviderType.LUME,
             display="1024x768",
             memory="8GB",
-            cpu="4"
+            cpu="4",
+            experiments=experiments
         )
-        os_type_str = "macos"
-        image_str = "macos-sequoia-cua:latest"
 
     await computer.run()
+    
+    # If app list is provided, create desktop from apps
+    if use_app_experiment:
+        computer = computer.create_desktop_from_apps(app_list)
 
     # Log computer initialization as a tool call
-    result = await execute("computer", "initialize", {
+    init_params = {
         "os": os_type_str,
-        "image": image_str,
-        "display": "1024x768",
-        "memory": "8GB",
-        "cpu": "4"
-    })
+        "provider": provider
+    }
+    
+    # Add VM-specific parameters if not using host computer server
+    if not use_host_computer_server:
+        init_params.update({
+            "image": image_str,
+            "display": "1024x768",
+            "memory": "8GB",
+            "cpu": "4"
+        })
+    
+    # Add app list to the log if provided
+    if use_app_experiment:
+        init_params["apps"] = app_list
+        init_params["experiments"] = ["app-use"]
+    
+    result = await execute("computer", "initialize", init_params)
 
     return result["screenshot"], json.dumps(tool_call_logs, indent=2)
 
@@ -1029,12 +1061,31 @@ def create_gradio_ui():
                     setup_status = gr.Textbox(label="Setup Status", value="")
                 
                 with gr.Group():
-                    os_choice = gr.Radio(
-                        label="OS",
-                        choices=["macOS", "Ubuntu"],
-                        value="macOS",
-                        interactive=False # disable until the ubuntu image is ready
-                    )
+                    with gr.Accordion("Computer Configuration", open=False):
+                        with gr.Row():
+                            os_choice = gr.Radio(
+                                label="OS",
+                                choices=["macOS", "Ubuntu"],
+                                value="macOS",
+                                interactive=False # disable until the ubuntu image is ready
+                            )
+                            
+                            # Provider selection radio
+                            provider_choice = gr.Radio(
+                                label="Provider",
+                                choices=["lume", "self"],
+                                value="lume",
+                                info="'lume' uses a VM, 'self' uses the host computer server"
+                            )
+                        
+                        # App filtering dropdown for app-use experiment
+                        app_filter = gr.Dropdown(
+                            label="Filter by apps (App-Use)",
+                            multiselect=True,
+                            allow_custom_value=True,
+                            info="When apps are selected, the computer will focus on those apps using the app-use experiment"
+                        )
+                    
                     start_btn = gr.Button("Initialize Computer")
                 
                 with gr.Group():
@@ -1199,7 +1250,7 @@ def create_gradio_ui():
         )
                 
         img.select(handle_click, inputs=[img, click_type], outputs=[img, action_log])
-        start_btn.click(handle_init_computer, inputs=[os_choice], outputs=[img, action_log])
+        start_btn.click(handle_init_computer, inputs=[os_choice, app_filter, provider_choice], outputs=[img, action_log])
         wait_btn.click(handle_wait, outputs=[img, action_log])
         
         # DONE and FAIL buttons just do a placeholder action

{cua_computer-0.2.7.dist-info → cua_computer-0.2.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cua-computer
-Version: 0.2.7
+Version: 0.2.9
 Summary: Computer-Use Interface (CUI) framework powering Cua
 Author-Email: TryCua <gh@trycua.com>
 Requires-Python: >=3.11

{cua_computer-0.2.7.dist-info → cua_computer-0.2.9.dist-info}/RECORD

@@ -1,10 +1,12 @@
 computer/__init__.py,sha256=QOxNrrJAuLRnsUC2zIFgRfzVSuDSXiYHlEF-9vkhV0o,1241
-computer/computer.py,sha256=Rc32XFZdKr7XZKO0zhbEom-REvYYPPlvmvjDbw5gP9k,32218
+computer/computer.py,sha256=zmx_jsQdWy8nsbeb1u9rptJRv5ChR0l0JeyQJcrhiSc,40246
+computer/diorama_computer.py,sha256=jOP7_eXxxU6SMIoE25ni0YXPK0E7p5sZeLKmkYLh6G8,3871
+computer/helpers.py,sha256=0ob9d9ynVGi0JRxhHCgXTuHPHFpa8AVKldn6k0hvxOo,1766
 computer/interface/__init__.py,sha256=xQvYjq5PMn9ZJOmRR5mWtONTl_0HVd8ACvW6AQnzDdw,262
-computer/interface/base.py,sha256=CD9WpDp-6qP-ID5MjhXA8qpYs0XhJ4TPkR917l2FFSo,6021
+computer/interface/base.py,sha256=wmLBiX7rB8cG2Q4fmchdKpjralktzicuYhAh6fDIeqw,6025
 computer/interface/factory.py,sha256=RjAZAB_jFuS8JierYjLbapRX6RqFE0qE3BiIyP5UDOE,1441
-computer/interface/linux.py,sha256=CT1N0QA52TNKBbFG2LXdN6yAGWWJ12_2hTMEI8yNoM4,26865
-computer/interface/macos.py,sha256=_8R_IroxbcVmh1WagrjDQOitaT6tVkCHVzGgA_lwTrM,27077
+computer/interface/linux.py,sha256=WA-jpjNHId3blaT__ftd_X7qhKi_50vwy97-jq2yd6g,27412
+computer/interface/macos.py,sha256=B-siwSKNOwNNBVvUQh9TO0nWIaMRvvCUbIQ3oziIF5A,27791
 computer/interface/models.py,sha256=RZKVUdwKrKUoFqwlx2Dk8Egkmq_AInlIu_d0xg7SZzw,3238
 computer/logger.py,sha256=UVvnmZGOWVF9TCsixEbeQnDZ3wBPAJ2anW3Zp-MoJ8Y,2896
 computer/models.py,sha256=iFNM1QfZArD8uf66XJXb2EDIREsfrxqqA5_liLBMfrE,1188
@@ -21,9 +23,9 @@ computer/providers/lumier/provider.py,sha256=CXwAKwJfR9ALFGM5u7UIZ-YrFwPvew_01wT
 computer/telemetry.py,sha256=FvNFpxgeRuCMdNpREuSL7bOMZy9gSzY4J0rLeNDw0CU,3746
 computer/ui/__init__.py,sha256=pmo05ek9qiB_x7DPeE6Vf_8RsIOqTD0w1dBLMHfoOnY,45
 computer/ui/gradio/__init__.py,sha256=5_KimixM48-X74FCsLw7LbSt39MQfUMEL8-M9amK3Cw,117
-computer/ui/gradio/app.py,sha256=o31nphBcb6zM5OKPuODTjuOzSJ3lt61kQHpUeMBBs70,65077
+computer/ui/gradio/app.py,sha256=pLMoMpxyKsGhg9wlsiqyKiRujd-lzubs0nGWAtkleL0,67316
 computer/utils.py,sha256=zY50NXB7r51GNLQ6l7lhG_qv0_ufpQ8n0-SDhCei8m4,2838
-cua_computer-0.2.7.dist-info/METADATA,sha256=NM_bxqQbNmvZR4bHfHHu0tXCc9GEF6kNgVEyjR6XYc8,5844
-cua_computer-0.2.7.dist-info/WHEEL,sha256=tSfRZzRHthuv7vxpI4aehrdN9scLjk-dCJkPLzkHxGg,90
-cua_computer-0.2.7.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
-cua_computer-0.2.7.dist-info/RECORD,,
+cua_computer-0.2.9.dist-info/METADATA,sha256=wbPfNftTjYS0NSsy77kUlDdle8a3c8TBQ07ncdq65zM,5844
+cua_computer-0.2.9.dist-info/WHEEL,sha256=tSfRZzRHthuv7vxpI4aehrdN9scLjk-dCJkPLzkHxGg,90
+cua_computer-0.2.9.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
+cua_computer-0.2.9.dist-info/RECORD,,