cua-agent 0.4.12__py3-none-any.whl → 0.4.13__py3-none-any.whl

agent/computers/base.py

@@ -0,0 +1,70 @@
+"""
+Base computer interface protocol for agent interactions.
+"""
+
+from typing import Protocol, Literal, List, Dict, Any, Union, Optional, runtime_checkable
+
+
+@runtime_checkable
+class AsyncComputerHandler(Protocol):
+    """Protocol defining the interface for computer interactions."""
+    
+    # ==== Computer-Use-Preview Action Space ==== 
+
+    async def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]:
+        """Get the current environment type."""
+        ...
+    
+    async def get_dimensions(self) -> tuple[int, int]:
+        """Get screen dimensions as (width, height)."""
+        ...
+    
+    async def screenshot(self) -> str:
+        """Take a screenshot and return as base64 string."""
+        ...
+    
+    async def click(self, x: int, y: int, button: str = "left") -> None:
+        """Click at coordinates with specified button."""
+        ...
+    
+    async def double_click(self, x: int, y: int) -> None:
+        """Double click at coordinates."""
+        ...
+    
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        """Scroll at coordinates with specified scroll amounts."""
+        ...
+    
+    async def type(self, text: str) -> None:
+        """Type text."""
+        ...
+    
+    async def wait(self, ms: int = 1000) -> None:
+        """Wait for specified milliseconds."""
+        ...
+    
+    async def move(self, x: int, y: int) -> None:
+        """Move cursor to coordinates."""
+        ...
+    
+    async def keypress(self, keys: Union[List[str], str]) -> None:
+        """Press key combination."""
+        ...
+    
+    async def drag(self, path: List[Dict[str, int]]) -> None:
+        """Drag along specified path."""
+        ...
+    
+    async def get_current_url(self) -> str:
+        """Get current URL (for browser environments)."""
+        ...
+    
+    # ==== Anthropic Action Space ==== 
+
+    async def left_mouse_down(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Left mouse down at coordinates."""
+        ...
+    
+    async def left_mouse_up(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Left mouse up at coordinates."""
+        ...

agent/{computer_handler.py → computers/cua.py}

@@ -4,35 +4,44 @@ Computer handler implementation for OpenAI computer-use-preview protocol.
 
 import base64
 from typing import Dict, List, Any, Literal, Union, Optional
-from .types import Computer
+from .base import AsyncComputerHandler
+from computer import Computer
 
-
-class OpenAIComputerHandler:
+class cuaComputerHandler(AsyncComputerHandler):
     """Computer handler that implements the Computer protocol using the computer interface."""
     
-    def __init__(self, computer_interface):
+    def __init__(self, cua_computer: Computer):
         """Initialize with a computer interface (from tool schema)."""
-        self.interface = computer_interface
+        self.cua_computer = cua_computer
+        self.interface = None
+
+    async def _initialize(self):
+        if hasattr(self.cua_computer, '_initialized') and not self.cua_computer._initialized:
+            await self.cua_computer.run()
+        self.interface = self.cua_computer.interface
     
     # ==== Computer-Use-Preview Action Space ==== 
 
     async def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]:
         """Get the current environment type."""
-        # For now, return a default - this could be enhanced to detect actual environment
-        return "windows"
+        # TODO: detect actual environment
+        return "linux"
 
     async def get_dimensions(self) -> tuple[int, int]:
         """Get screen dimensions as (width, height)."""
+        assert self.interface is not None
         screen_size = await self.interface.get_screen_size()
         return screen_size["width"], screen_size["height"]
     
     async def screenshot(self) -> str:
         """Take a screenshot and return as base64 string."""
+        assert self.interface is not None
         screenshot_bytes = await self.interface.screenshot()
         return base64.b64encode(screenshot_bytes).decode('utf-8')
     
     async def click(self, x: int, y: int, button: str = "left") -> None:
         """Click at coordinates with specified button."""
+        assert self.interface is not None
         if button == "left":
             await self.interface.left_click(x, y)
         elif button == "right":
@@ -43,28 +52,34 @@ class OpenAIComputerHandler:
     
     async def double_click(self, x: int, y: int) -> None:
         """Double click at coordinates."""
+        assert self.interface is not None
         await self.interface.double_click(x, y)
     
     async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
         """Scroll at coordinates with specified scroll amounts."""
+        assert self.interface is not None
         await self.interface.move_cursor(x, y)
         await self.interface.scroll(scroll_x, scroll_y)
     
     async def type(self, text: str) -> None:
         """Type text."""
+        assert self.interface is not None
         await self.interface.type_text(text)
     
     async def wait(self, ms: int = 1000) -> None:
         """Wait for specified milliseconds."""
+        assert self.interface is not None
         import asyncio
         await asyncio.sleep(ms / 1000.0)
     
     async def move(self, x: int, y: int) -> None:
         """Move cursor to coordinates."""
+        assert self.interface is not None
         await self.interface.move_cursor(x, y)
     
     async def keypress(self, keys: Union[List[str], str]) -> None:
         """Press key combination."""
+        assert self.interface is not None
         if isinstance(keys, str):
             keys = keys.replace("-", "+").split("+")
         if len(keys) == 1:
@@ -75,6 +90,7 @@ class OpenAIComputerHandler:
     
     async def drag(self, path: List[Dict[str, int]]) -> None:
         """Drag along specified path."""
+        assert self.interface is not None
         if not path:
             return
         
@@ -99,23 +115,10 @@ class OpenAIComputerHandler:
     # ==== Anthropic Computer Action Space ==== 
     async def left_mouse_down(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
         """Left mouse down at coordinates."""
+        assert self.interface is not None
         await self.interface.mouse_down(x, y, button="left")
     
     async def left_mouse_up(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
         """Left mouse up at coordinates."""
-        await self.interface.mouse_up(x, y, button="left")
-
-def acknowledge_safety_check_callback(message: str, allow_always: bool = False) -> bool:
-    """Safety check callback for user acknowledgment."""
-    if allow_always:
-        return True
-    response = input(
-        f"Safety Check Warning: {message}\nDo you want to acknowledge and proceed? (y/n): "
-    ).lower()
-    return response.strip() == "y"
-
-
-def check_blocklisted_url(url: str) -> None:
-    """Check if URL is blocklisted (placeholder implementation)."""
-    # This would contain actual URL checking logic
-    pass
+        assert self.interface is not None
+        await self.interface.mouse_up(x, y, button="left")

agent/computers/custom.py

@@ -0,0 +1,209 @@
+"""
+Custom computer handler implementation that accepts a dictionary of functions.
+"""
+
+import base64
+from typing import Dict, List, Any, Literal, Union, Optional, Callable
+from PIL import Image
+import io
+from .base import AsyncComputerHandler
+
+
+class CustomComputerHandler(AsyncComputerHandler):
+    """Computer handler that implements the Computer protocol using a dictionary of custom functions."""
+    
+    def __init__(self, functions: Dict[str, Callable]):
+        """
+        Initialize with a dictionary of functions.
+        
+        Args:
+            functions: Dictionary where keys are method names and values are callable functions.
+                      Only 'screenshot' is required, all others are optional.
+        
+        Raises:
+            ValueError: If required 'screenshot' function is not provided.
+        """
+        if 'screenshot' not in functions:
+            raise ValueError("'screenshot' function is required in functions dictionary")
+        
+        self.functions = functions
+        self._last_screenshot_size: Optional[tuple[int, int]] = None
+    
+    async def _call_function(self, func, *args, **kwargs):
+        """
+        Call a function, handling both async and sync functions.
+        
+        Args:
+            func: The function to call
+            *args: Positional arguments to pass to the function
+            **kwargs: Keyword arguments to pass to the function
+            
+        Returns:
+            The result of the function call
+        """
+        import asyncio
+        import inspect
+        
+        if callable(func):
+            if inspect.iscoroutinefunction(func):
+                return await func(*args, **kwargs)
+            else:
+                return func(*args, **kwargs)
+        else:
+            return func
+    
+    async def _get_value(self, attribute: str):
+        """
+        Get value for an attribute, checking both 'get_{attribute}' and '{attribute}' keys.
+        
+        Args:
+            attribute: The attribute name to look for
+            
+        Returns:
+            The value from the functions dict, called if callable, returned directly if not
+        """
+        # Check for 'get_{attribute}' first
+        get_key = f"get_{attribute}"
+        if get_key in self.functions:
+            return await self._call_function(self.functions[get_key])
+        
+        # Check for '{attribute}' 
+        if attribute in self.functions:
+            return await self._call_function(self.functions[attribute])
+        
+        return None
+    
+    def _to_b64_str(self, img: Union[bytes, Image.Image, str]) -> str:
+        """
+        Convert image to base64 string.
+        
+        Args:
+            img: Image as bytes, PIL Image, or base64 string
+            
+        Returns:
+            str: Base64 encoded image string
+        """
+        if isinstance(img, str):
+            # Already a base64 string
+            return img
+        elif isinstance(img, bytes):
+            # Raw bytes
+            return base64.b64encode(img).decode('utf-8')
+        elif isinstance(img, Image.Image):
+            # PIL Image
+            buffer = io.BytesIO()
+            img.save(buffer, format='PNG')
+            return base64.b64encode(buffer.getvalue()).decode('utf-8')
+        else:
+            raise ValueError(f"Unsupported image type: {type(img)}")
+    
+    # ==== Computer-Use-Preview Action Space ==== 
+
+    async def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]:
+        """Get the current environment type."""
+        result = await self._get_value('environment')
+        if result is None:
+            return "linux"
+        assert result in ["windows", "mac", "linux", "browser"]
+        return result # type: ignore
+
+    async def get_dimensions(self) -> tuple[int, int]:
+        """Get screen dimensions as (width, height)."""
+        result = await self._get_value('dimensions')
+        if result is not None:
+            return result # type: ignore
+        
+        # Fallback: use last screenshot size if available
+        if not self._last_screenshot_size:
+            await self.screenshot()
+        assert self._last_screenshot_size is not None, "Failed to get screenshot size"
+        
+        return self._last_screenshot_size
+    
+    async def screenshot(self) -> str:
+        """Take a screenshot and return as base64 string."""
+        result = await self._call_function(self.functions['screenshot'])
+        b64_str = self._to_b64_str(result) # type: ignore
+        
+        # Try to extract dimensions for fallback use
+        try:
+            if isinstance(result, Image.Image):
+                self._last_screenshot_size = result.size
+            elif isinstance(result, bytes):
+                # Try to decode bytes to get dimensions
+                img = Image.open(io.BytesIO(result))
+                self._last_screenshot_size = img.size
+        except Exception:
+            # If we can't get dimensions, that's okay
+            pass
+        
+        return b64_str
+    
+    async def click(self, x: int, y: int, button: str = "left") -> None:
+        """Click at coordinates with specified button."""
+        if 'click' in self.functions:
+            await self._call_function(self.functions['click'], x, y, button)
+        # No-op if not implemented
+    
+    async def double_click(self, x: int, y: int) -> None:
+        """Double click at coordinates."""
+        if 'double_click' in self.functions:
+            await self._call_function(self.functions['double_click'], x, y)
+        # No-op if not implemented
+    
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        """Scroll at coordinates with specified scroll amounts."""
+        if 'scroll' in self.functions:
+            await self._call_function(self.functions['scroll'], x, y, scroll_x, scroll_y)
+        # No-op if not implemented
+    
+    async def type(self, text: str) -> None:
+        """Type text."""
+        if 'type' in self.functions:
+            await self._call_function(self.functions['type'], text)
+        # No-op if not implemented
+    
+    async def wait(self, ms: int = 1000) -> None:
+        """Wait for specified milliseconds."""
+        if 'wait' in self.functions:
+            await self._call_function(self.functions['wait'], ms)
+        else:
+            # Default implementation
+            import asyncio
+            await asyncio.sleep(ms / 1000.0)
+    
+    async def move(self, x: int, y: int) -> None:
+        """Move cursor to coordinates."""
+        if 'move' in self.functions:
+            await self._call_function(self.functions['move'], x, y)
+        # No-op if not implemented
+    
+    async def keypress(self, keys: Union[List[str], str]) -> None:
+        """Press key combination."""
+        if 'keypress' in self.functions:
+            await self._call_function(self.functions['keypress'], keys)
+        # No-op if not implemented
+    
+    async def drag(self, path: List[Dict[str, int]]) -> None:
+        """Drag along specified path."""
+        if 'drag' in self.functions:
+            await self._call_function(self.functions['drag'], path)
+        # No-op if not implemented
+    
+    async def get_current_url(self) -> str:
+        """Get current URL (for browser environments)."""
+        if 'get_current_url' in self.functions:
+            return await self._get_value('current_url') # type: ignore
+        return ""  # Default fallback
+    
+    async def left_mouse_down(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Left mouse down at coordinates."""
+        if 'left_mouse_down' in self.functions:
+            await self._call_function(self.functions['left_mouse_down'], x, y)
+        # No-op if not implemented
+    
+    async def left_mouse_up(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Left mouse up at coordinates."""
+        if 'left_mouse_up' in self.functions:
+            await self._call_function(self.functions['left_mouse_up'], x, y)
+        # No-op if not implemented

agent/human_tool/__init__.py

@@ -0,0 +1,29 @@
+"""
+Human-in-the-Loop Completion Tool
+
+This package provides a human-in-the-loop completion system that allows
+AI agents to request human assistance for complex decisions or responses.
+
+Components:
+- server.py: FastAPI server with completion queue management
+- ui.py: Gradio UI for human interaction
+- __main__.py: Combined server and UI application
+
+Usage:
+    # Run the server and UI
+    python -m agent.human_tool
+    
+    # Or run components separately
+    python -m agent.human_tool.server  # API server only
+    python -m agent.human_tool.ui      # UI only
+"""
+
+from .server import CompletionQueue, completion_queue
+from .ui import HumanCompletionUI, create_ui
+
+__all__ = [
+    "CompletionQueue",
+    "completion_queue", 
+    "HumanCompletionUI",
+    "create_ui"
+]

agent/human_tool/__main__.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""
+Human-in-the-Loop Completion Server and UI
+
+This module combines the FastAPI server for handling completion requests
+with a Gradio UI for human interaction.
+"""
+
+import gradio as gr
+from fastapi import FastAPI
+from .server import app as fastapi_app
+from .ui import create_ui
+
+# Create the Gradio demo
+gradio_demo = create_ui()
+
+# Mount Gradio on FastAPI
+CUSTOM_PATH = "/gradio"
+app = gr.mount_gradio_app(fastapi_app, gradio_demo, path=CUSTOM_PATH)
+
+# Add a redirect from root to Gradio UI
+@fastapi_app.get("/")
+async def redirect_to_ui():
+    """Redirect root to Gradio UI."""
+    return {
+        "message": "Human Completion Server is running",
+        "ui_url": "/gradio",
+        "api_docs": "/docs"
+    }
+
+if __name__ == "__main__":
+    import uvicorn
+    print("🚀 Starting Human-in-the-Loop Completion Server...")
+    print("📊 API Server: http://localhost:8002")
+    print("🎨 Gradio UI: http://localhost:8002/gradio")
+    print("📚 API Docs: http://localhost:8002/docs")
+    
+    uvicorn.run(app, host="0.0.0.0", port=8002)