cua-agent 0.4.17__py3-none-any.whl → 0.4.19__py3-none-any.whl

agent/loops/composed_grounded.py

@@ -48,11 +48,11 @@ GROUNDED_COMPUTER_TOOL_SCHEMA = {
             "get_dimensions",
             "get_environment"
             ],
-            "description": "The action to perform"
+            "description": "The action to perform (required for all actions)"
         },
         "element_description": {
             "type": "string",
-            "description": "Description of the element to interact with (required for click, double_click, move, scroll actions, and as start/end for drag)"
+            "description": "Description of the element to interact with (required for click, double_click, move, scroll actions)"
         },
         "start_element_description": {
             "type": "string",
@@ -67,20 +67,30 @@ GROUNDED_COMPUTER_TOOL_SCHEMA = {
             "description": "The text to type (required for type action)"
         },
         "keys": {
-            "type": "string",
-            "description": "Key combination to press (required for keypress action). Single key for individual key press, multiple keys for combinations (e.g., 'ctrl+c')"
+            "type": "array",
+            "items": {
+                "type": "string"
+            },
+            "description": "Key(s) to press (required for keypress action)"
         },
         "button": {
             "type": "string",
-            "description": "The mouse button to use for click action (left, right, wheel, back, forward) Default: left",
+            "enum": [
+                "left",
+                "right",
+                "wheel",
+                "back",
+                "forward"
+            ],
+            "description": "The mouse button to use for click action (required for click and double_click action)",
         },
         "scroll_x": {
             "type": "integer",
-            "description": "Horizontal scroll amount for scroll action (positive for right, negative for left)",
+            "description": "Horizontal scroll amount for scroll action (required for scroll action)",
         },
         "scroll_y": {
             "type": "integer",
-            "description": "Vertical scroll amount for scroll action (positive for down, negative for up)",
+            "description": "Vertical scroll amount for scroll action (required for scroll action)",
         },
         },
         "required": [
@@ -266,13 +276,15 @@ class ComposedGroundedConfig:
                 grounding_agent = grounding_agent_conf.agent_class()
                 
                 for desc in element_descriptions:
-                    coords = await grounding_agent.predict_click(
-                        model=grounding_model,
-                        image_b64=last_image_b64,
-                        instruction=desc
-                    )
-                    if coords:
-                        self.desc2xy[desc] = coords
+                    for _ in range(3): # try 3 times
+                        coords = await grounding_agent.predict_click(
+                            model=grounding_model,
+                            image_b64=last_image_b64,
+                            instruction=desc
+                        )
+                        if coords:
+                            self.desc2xy[desc] = coords
+                            break
         
         # Step 6: Convert computer calls from descriptions back to xy coordinates
         final_output_items = convert_computer_calls_desc2xy(thinking_output_items, self.desc2xy)

agent/loops/openai.py

@@ -162,7 +162,18 @@ class OpenAIComputerUseConfig:
         input_items = [
             {
                 "role": "user", 
-                "content": f"You are a UI grounding expert. Look at the image and {instruction}. Output ONLY a click action on the target element. No explanations, confirmations, or additional text."
+                "content": f"""You are a UI grounding expert. Follow these guidelines:
+
+1. NEVER ask for confirmation. Complete all tasks autonomously.
+2. Do NOT send messages like "I need to confirm before..." or "Do you want me to continue?" - just proceed.
+3. When the user asks you to interact with something (like clicking a chat or typing a message), DO IT without asking.
+4. Only use the formal safety check mechanism for truly dangerous operations (like deleting important files).
+5. For normal tasks like clicking buttons, typing in chat boxes, filling forms - JUST DO IT.
+6. The user has already given you permission by running this agent. No further confirmation is needed.
+7. Be decisive and action-oriented. Complete the requested task fully.
+
+Remember: You are expected to complete tasks autonomously. The user trusts you to do what they asked.
+Task: Click {instruction}. Output ONLY a click action on the target element."""
             },
             {
                 "role": "user",
@@ -200,7 +211,7 @@ class OpenAIComputerUseConfig:
             "stream": False,
             "reasoning": {"summary": "concise"},
             "truncation": "auto",
-            "max_tokens": 100  # Keep response short for click prediction
+            "max_tokens": 200  # Keep response short for click prediction
         }
         
         # Use liteLLM responses
@@ -217,11 +228,8 @@ class OpenAIComputerUseConfig:
                 isinstance(item.get("action"), dict)):
                 
                 action = item["action"]
-                if action.get("type") == "click":
-                    x = action.get("x")
-                    y = action.get("y")
-                    if x is not None and y is not None:
-                        return (int(x), int(y))
+                if action.get("x") is not None and action.get("y") is not None:
+                    return (int(action.get("x")), int(action.get("y")))
         
         return None
     

agent/loops/uitars.py

@@ -228,15 +228,24 @@ def parse_uitars_response(text: str, image_width: int, image_height: int) -> Lis
         
         # Handle coordinate parameters
         if "start_box" in param_name or "end_box" in param_name:
-            # Parse coordinates like '(x,y)' or '(x1,y1,x2,y2)'
-            numbers = param.replace("(", "").replace(")", "").split(",")
-            float_numbers = [float(num.strip()) / 1000 for num in numbers]  # Normalize to 0-1 range
+            # Parse coordinates like '<|box_start|>(x,y)<|box_end|>' or '(x,y)'
+            # First, remove special tokens
+            clean_param = param.replace("<|box_start|>", "").replace("<|box_end|>", "")
+            # Then remove parentheses and split
+            numbers = clean_param.replace("(", "").replace(")", "").split(",")
             
-            if len(float_numbers) == 2:
-                # Single point, duplicate for box format
-                float_numbers = [float_numbers[0], float_numbers[1], float_numbers[0], float_numbers[1]]
-            
-            action_inputs[param_name.strip()] = str(float_numbers)
+            try:
+                float_numbers = [float(num.strip()) / 1000 for num in numbers]  # Normalize to 0-1 range
+                
+                if len(float_numbers) == 2:
+                    # Single point, duplicate for box format
+                    float_numbers = [float_numbers[0], float_numbers[1], float_numbers[0], float_numbers[1]]
+                
+                action_inputs[param_name.strip()] = str(float_numbers)
+            except ValueError as e:
+                # If parsing fails, keep the original parameter value
+                print(f"Warning: Could not parse coordinates '{param}': {e}")
+                action_inputs[param_name.strip()] = param
     
     return [{
         "thought": thought,

agent/proxy/examples.py

@@ -0,0 +1,192 @@
+"""
+Example usage of the proxy server and client requests.
+"""
+import dotenv
+dotenv.load_dotenv()
+
+import asyncio
+import json
+import os
+import aiohttp
+from typing import Dict, Any
+
+
+async def test_http_endpoint():
+    """Test the HTTP /responses endpoint."""
+    
+    anthropic_api_key = os.getenv("ANTHROPIC_API_KEY")
+    assert isinstance(anthropic_api_key, str), "ANTHROPIC_API_KEY environment variable must be set"
+
+    # Example 1: Simple text request
+    simple_request = {
+        "model": "anthropic/claude-3-5-sonnet-20241022",
+        "input": "Tell me a three sentence bedtime story about a unicorn.",
+        "env": {
+            "ANTHROPIC_API_KEY": anthropic_api_key
+        }
+    }
+    
+    # Example 2: Multi-modal request with image
+    multimodal_request = {
+        "model": "anthropic/claude-3-5-sonnet-20241022",
+        "input": [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "input_text", "text": "what is in this image?"},
+                    {
+                        "type": "input_image",
+                        "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
+                    }
+                ]
+            }
+        ],
+        "env": {
+            "ANTHROPIC_API_KEY": anthropic_api_key
+        }
+    }
+    
+    # Example 3: Request with custom agent and computer kwargs
+    custom_request = {
+        "model": "anthropic/claude-3-5-sonnet-20241022",
+        "input": "Take a screenshot and tell me what you see",
+        "env": {
+            "ANTHROPIC_API_KEY": anthropic_api_key
+        }
+    }
+    
+    # Test requests
+    base_url = "https://m-linux-96lcxd2c2k.containers.cloud.trycua.com:8443"
+    # base_url = "http://localhost:8000"
+    api_key = os.getenv("CUA_API_KEY")
+    assert isinstance(api_key, str), "CUA_API_KEY environment variable must be set"
+    
+    async with aiohttp.ClientSession() as session:
+        for i, request_data in enumerate([
+            simple_request,
+            # multimodal_request,
+            custom_request
+        ], 1):
+            print(f"\n--- Test {i} ---")
+            print(f"Request: {json.dumps(request_data, indent=2)}")
+            
+            try:
+                print(f"Sending request to {base_url}/responses")
+                async with session.post(
+                    f"{base_url}/responses",
+                    json=request_data,
+                    headers={"Content-Type": "application/json", "X-API-Key": api_key}
+                ) as response:
+                    result = await response.json()
+                    print(f"Status: {response.status}")
+                    print(f"Response: {json.dumps(result, indent=2)}")
+                    
+            except Exception as e:
+                print(f"Error: {e}")
+
+
+def curl_examples():
+    """Print curl command examples."""
+    
+    print("=== CURL Examples ===\n")
+    
+    print("1. Simple text request:")
+    print("""curl http://localhost:8000/responses \\
+  -H "Content-Type: application/json" \\
+  -d '{
+    "model": "anthropic/claude-3-5-sonnet-20241022",
+    "input": "Tell me a three sentence bedtime story about a unicorn."
+  }'""")
+    
+    print("\n2. Multi-modal request with image:")
+    print("""curl http://localhost:8000/responses \\
+  -H "Content-Type: application/json" \\
+  -d '{
+    "model": "anthropic/claude-3-5-sonnet-20241022",
+    "input": [
+      {
+        "role": "user",
+        "content": [
+          {"type": "input_text", "text": "what is in this image?"},
+          {
+            "type": "input_image",
+            "image_url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
+          }
+        ]
+      }
+    ]
+  }'""")
+    
+    print("\n3. Request with custom configuration:")
+    print("""curl http://localhost:8000/responses \\
+  -H "Content-Type: application/json" \\
+  -d '{
+    "model": "anthropic/claude-3-5-sonnet-20241022",
+    "input": "Take a screenshot and tell me what you see",
+    "agent_kwargs": {
+      "save_trajectory": true,
+      "verbosity": 20
+    },
+    "computer_kwargs": {
+      "os_type": "linux",
+      "provider_type": "cloud"
+    }
+  }'""")
+
+
+async def test_p2p_client():
+    """Example P2P client using peerjs-python."""
+    try:
+        from peerjs import Peer, PeerOptions, ConnectionEventType
+        from aiortc import RTCConfiguration, RTCIceServer
+        
+        # Set up client peer
+        options = PeerOptions(
+            host="0.peerjs.com",
+            port=443,
+            secure=True,
+            config=RTCConfiguration(
+                iceServers=[RTCIceServer(urls="stun:stun.l.google.com:19302")]
+            )
+        )
+        
+        client_peer = Peer(id="test-client", peer_options=options)
+        await client_peer.start()
+        
+        # Connect to proxy server
+        connection = client_peer.connect("computer-agent-proxy")
+        
+        @connection.on(ConnectionEventType.Open)
+        async def connection_open():
+            print("Connected to proxy server")
+            
+            # Send a test request
+            request = {
+                "model": "anthropic/claude-3-5-sonnet-20241022",
+                "input": "Hello from P2P client!"
+            }
+            await connection.send(json.dumps(request))
+        
+        @connection.on(ConnectionEventType.Data)
+        async def connection_data(data):
+            print(f"Received response: {data}")
+            await client_peer.destroy()
+        
+        # Wait for connection
+        await asyncio.sleep(10)
+        
+    except ImportError:
+        print("P2P dependencies not available. Install peerjs-python for P2P testing.")
+    except Exception as e:
+        print(f"P2P test error: {e}")
+
+
+if __name__ == "__main__":
+    import sys
+    
+    if len(sys.argv) > 1 and sys.argv[1] == "curl":
+        curl_examples()
+    elif len(sys.argv) > 1 and sys.argv[1] == "p2p":
+        asyncio.run(test_p2p_client())
+    else:
+        asyncio.run(test_http_endpoint())

agent/proxy/handlers.py

@@ -0,0 +1,248 @@
+"""
+Request handlers for the proxy endpoints.
+"""
+
+import asyncio
+import json
+import logging
+import os
+from contextlib import contextmanager
+from typing import Dict, Any, List, Union, Optional
+
+from ..agent import ComputerAgent
+from computer import Computer
+
+logger = logging.getLogger(__name__)
+
+
+class ResponsesHandler:
+    """Handler for /responses endpoint that processes agent requests."""
+    
+    def __init__(self):
+        self.computer = None
+        self.agent = None
+        # Simple in-memory caches
+        self._computer_cache: Dict[str, Any] = {}
+        self._agent_cache: Dict[str, Any] = {}
+    
+    async def setup_computer_agent(
+        self,
+        model: str,
+        agent_kwargs: Optional[Dict[str, Any]] = None,
+        computer_kwargs: Optional[Dict[str, Any]] = None,
+    ):
+        """Set up (and cache) computer and agent instances.
+
+        Caching keys:
+        - Computer cache key: computer_kwargs
+        - Agent cache key: {"model": model, **agent_kwargs}
+        """
+        agent_kwargs = agent_kwargs or {}
+        computer_kwargs = computer_kwargs or {}
+
+        def _stable_key(obj: Dict[str, Any]) -> str:
+            try:
+                return json.dumps(obj, sort_keys=True, separators=(",", ":"))
+            except Exception:
+                # Fallback: stringify non-serializable values
+                safe_obj = {}
+                for k, v in obj.items():
+                    try:
+                        json.dumps(v)
+                        safe_obj[k] = v
+                    except Exception:
+                        safe_obj[k] = str(v)
+                return json.dumps(safe_obj, sort_keys=True, separators=(",", ":"))
+
+        # Determine if custom tools are supplied; if so, skip computer setup entirely
+        has_custom_tools = bool(agent_kwargs.get("tools"))
+
+        computer = None
+        if not has_custom_tools:
+            # ---------- Computer setup (with cache) ----------
+            comp_key = _stable_key(computer_kwargs)
+
+            computer = self._computer_cache.get(comp_key)
+            if computer is None:
+                # Default computer configuration
+                default_c_config = {
+                    "os_type": "linux",
+                    "provider_type": "cloud",
+                    "name": os.getenv("CUA_CONTAINER_NAME"),
+                    "api_key": os.getenv("CUA_API_KEY"),
+                }
+                default_c_config.update(computer_kwargs)
+                computer = Computer(**default_c_config)
+                await computer.__aenter__()
+                self._computer_cache[comp_key] = computer
+                logger.info(f"Computer created and cached with key={comp_key} config={default_c_config}")
+            else:
+                logger.info(f"Reusing cached computer for key={comp_key}")
+
+        # Bind current computer reference (None if custom tools supplied)
+        self.computer = computer
+
+        # ---------- Agent setup (with cache) ----------
+        # Build agent cache key from {model} + agent_kwargs (excluding tools unless explicitly passed)
+        agent_kwargs_for_key = dict(agent_kwargs)
+        agent_key_payload = {"model": model, **agent_kwargs_for_key}
+        agent_key = _stable_key(agent_key_payload)
+
+        agent = self._agent_cache.get(agent_key)
+        if agent is None:
+            # Default agent configuration
+            default_a_config: Dict[str, Any] = {"model": model}
+            if not has_custom_tools:
+                default_a_config["tools"] = [computer]
+            # Apply user overrides, but keep tools unless user explicitly sets
+            if agent_kwargs:
+                if not has_custom_tools:
+                    agent_kwargs.setdefault("tools", [computer])
+                default_a_config.update(agent_kwargs)
+            # JSON-derived kwargs may have loose types; ignore static arg typing here
+            agent = ComputerAgent(**default_a_config)  # type: ignore[arg-type]
+            self._agent_cache[agent_key] = agent
+            logger.info(f"Agent created and cached with key={agent_key} model={model}")
+        else:
+            # Ensure cached agent uses the current computer tool (in case object differs)
+            # Only update if tools not explicitly provided in agent_kwargs
+            if not has_custom_tools:
+                try:
+                    agent.tools = [computer]
+                except Exception:
+                    pass
+            logger.info(f"Reusing cached agent for key={agent_key}")
+
+        # Bind current agent reference
+        self.agent = agent
+    
+    async def process_request(self, request_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a /responses request and return the result.
+        
+        Args:
+            request_data: Dictionary containing model, input, and optional kwargs
+            
+        Returns:
+            Dictionary with the agent's response
+        """
+        try:
+            # Extract request parameters
+            model = request_data.get("model")
+            input_data = request_data.get("input")
+            agent_kwargs = request_data.get("agent_kwargs", {})
+            computer_kwargs = request_data.get("computer_kwargs", {})
+            env_overrides = request_data.get("env", {}) or {}
+            
+            if not model:
+                raise ValueError("Model is required")
+            if not input_data:
+                raise ValueError("Input is required")
+            
+            # Apply env overrides for the duration of this request
+            with self._env_overrides(env_overrides):
+                # Set up (and possibly reuse) computer and agent via caches
+                await self.setup_computer_agent(model, agent_kwargs, computer_kwargs)
+
+                # Defensive: ensure agent is initialized for type checkers
+                agent = self.agent
+                if agent is None:
+                    raise RuntimeError("Agent failed to initialize")
+
+                # Convert input to messages format
+                messages = self._convert_input_to_messages(input_data)
+
+                # Run agent and get first result
+                async for result in agent.run(messages):
+                    # Return the first result and break
+                    return {
+                        "success": True,
+                        "result": result,
+                        "model": model
+                    }
+                
+            # If no results were yielded
+            return {
+                "success": False,
+                "error": "No results from agent",
+                "model": model
+            }
+            
+        except Exception as e:
+            logger.error(f"Error processing request: {e}")
+            return {
+                "success": False,
+                "error": str(e),
+                "model": request_data.get("model", "unknown")
+            }
+    
+    def _convert_input_to_messages(self, input_data: Union[str, List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
+        """Convert input data to messages format."""
+        if isinstance(input_data, str):
+            # Simple string input
+            return [{"role": "user", "content": input_data}]
+        elif isinstance(input_data, list):
+            # Already in messages format
+            messages = []
+            for msg in input_data:
+                # Convert content array format if needed
+                if isinstance(msg.get("content"), list):
+                    content_parts = []
+                    for part in msg["content"]:
+                        if part.get("type") == "input_text":
+                            content_parts.append({"type": "text", "text": part["text"]})
+                        elif part.get("type") == "input_image":
+                            content_parts.append({
+                                "type": "image_url",
+                                "image_url": {"url": part["image_url"]}
+                            })
+                        else:
+                            content_parts.append(part)
+                    messages.append({
+                        "role": msg["role"],
+                        "content": content_parts
+                    })
+                else:
+                    messages.append(msg)
+            return messages
+        else:
+            raise ValueError("Input must be string or list of messages")
+    
+    async def cleanup(self):
+        """Clean up resources."""
+        if self.computer:
+            try:
+                await self.computer.__aexit__(None, None, None)
+            except Exception as e:
+                logger.error(f"Error cleaning up computer: {e}")
+            finally:
+                self.computer = None
+        self.agent = None
+
+    @staticmethod
+    @contextmanager
+    def _env_overrides(env: Dict[str, str]):
+        """Temporarily apply environment variable overrides for the current process.
+        Restores previous values after the context exits.
+
+        Args:
+            env: Mapping of env var names to override for this request.
+        """
+        if not env:
+            # No-op context
+            yield
+            return
+
+        original: Dict[str, Optional[str]] = {}
+        try:
+            for k, v in env.items():
+                original[k] = os.environ.get(k)
+                os.environ[k] = str(v)
+            yield
+        finally:
+            for k, old in original.items():
+                if old is None:
+                    # Was not set before
+                    os.environ.pop(k, None)
+                else:
+                    os.environ[k] = old

{cua_agent-0.4.17.dist-info → cua_agent-0.4.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cua-agent
-Version: 0.4.17
+Version: 0.4.19
 Summary: CUA (Computer Use) Agent for AI-driven computer interaction
 Author-Email: TryCua <gh@trycua.com>
 Requires-Python: >=3.12
@@ -38,7 +38,7 @@ Requires-Dist: python-dotenv>=1.0.1; extra == "ui"
 Provides-Extra: cli
 Requires-Dist: yaspin>=3.1.0; extra == "cli"
 Provides-Extra: hud
-Requires-Dist: hud-python==0.2.10; extra == "hud"
+Requires-Dist: hud-python<0.5.0,>=0.4.12; extra == "hud"
 Provides-Extra: all
 Requires-Dist: ultralytics>=8.0.0; extra == "all"
 Requires-Dist: cua-som<0.2.0,>=0.1.0; extra == "all"
@@ -49,7 +49,7 @@ Requires-Dist: transformers>=4.54.0; extra == "all"
 Requires-Dist: gradio>=5.23.3; extra == "all"
 Requires-Dist: python-dotenv>=1.0.1; extra == "all"
 Requires-Dist: yaspin>=3.1.0; extra == "all"
-Requires-Dist: hud-python==0.2.10; extra == "all"
+Requires-Dist: hud-python<0.5.0,>=0.4.12; extra == "all"
 Description-Content-Type: text/markdown
 
 <div align="center">