cua-agent 0.4.12__py3-none-any.whl → 0.4.14__py3-none-any.whl

agent/adapters/__init__.py

@@ -3,7 +3,9 @@ Adapters package for agent - Custom LLM adapters for LiteLLM
 """
 
 from .huggingfacelocal_adapter import HuggingFaceLocalAdapter
+from .human_adapter import HumanAdapter
 
 __all__ = [
     "HuggingFaceLocalAdapter",
+    "HumanAdapter",
 ]

agent/adapters/huggingfacelocal_adapter.py

@@ -1,5 +1,7 @@
 import asyncio
+import functools
 import warnings
+from concurrent.futures import ThreadPoolExecutor
 from typing import Iterator, AsyncIterator, Dict, List, Any, Optional
 from litellm.types.utils import GenericStreamingChunk, ModelResponse
 from litellm.llms.custom_llm import CustomLLM
@@ -28,6 +30,7 @@ class HuggingFaceLocalAdapter(CustomLLM):
         self.device = device
         self.models = {}  # Cache for loaded models
         self.processors = {}  # Cache for loaded processors
+        self._executor = ThreadPoolExecutor(max_workers=1)  # Single thread pool
         
     def _load_model_and_processor(self, model_name: str):
         """Load model and processor if not already cached.
@@ -51,7 +54,8 @@ class HuggingFaceLocalAdapter(CustomLLM):
             processor = AutoProcessor.from_pretrained(
                 model_name,
                 min_pixels=3136,
-                max_pixels=4096 * 2160
+                max_pixels=4096 * 2160,
+                device_map=self.device
             )
             
             # Cache them
@@ -185,7 +189,11 @@ class HuggingFaceLocalAdapter(CustomLLM):
             ModelResponse with generated text
         """
         # Run _generate in thread pool to avoid blocking
-        generated_text = await asyncio.to_thread(self._generate, **kwargs)
+        loop = asyncio.get_event_loop()
+        generated_text = await loop.run_in_executor(
+            self._executor, 
+            functools.partial(self._generate, **kwargs)
+        )
         
         return await acompletion(
             model=f"huggingface-local/{kwargs['model']}",
@@ -218,7 +226,11 @@ class HuggingFaceLocalAdapter(CustomLLM):
             AsyncIterator of GenericStreamingChunk
         """
         # Run _generate in thread pool to avoid blocking
-        generated_text = await asyncio.to_thread(self._generate, **kwargs)
+        loop = asyncio.get_event_loop()
+        generated_text = await loop.run_in_executor(
+            self._executor, 
+            functools.partial(self._generate, **kwargs)
+        )
         
         generic_streaming_chunk: GenericStreamingChunk = {
             "finish_reason": "stop",

agent/adapters/human_adapter.py

@@ -0,0 +1,348 @@
+import os
+import asyncio
+import requests
+from typing import List, Dict, Any, Iterator, AsyncIterator
+from litellm.types.utils import GenericStreamingChunk, ModelResponse
+from litellm.llms.custom_llm import CustomLLM
+from litellm import completion, acompletion
+
+
+class HumanAdapter(CustomLLM):
+    """Human Adapter for human-in-the-loop completions.
+    
+    This adapter sends completion requests to a human completion server
+    where humans can review and respond to AI requests.
+    """
+    
+    def __init__(self, base_url: str | None = None, timeout: float = 300.0, **kwargs):
+        """Initialize the human adapter.
+        
+        Args:
+            base_url: Base URL for the human completion server.
+                     Defaults to HUMAN_BASE_URL environment variable or http://localhost:8002
+            timeout: Timeout in seconds for waiting for human response
+            **kwargs: Additional arguments
+        """
+        super().__init__()
+        self.base_url = base_url or os.getenv('HUMAN_BASE_URL', 'http://localhost:8002')
+        self.timeout = timeout
+        
+        # Ensure base_url doesn't end with slash
+        self.base_url = self.base_url.rstrip('/')
+    
+    def _queue_completion(self, messages: List[Dict[str, Any]], model: str) -> str:
+        """Queue a completion request and return the call ID.
+        
+        Args:
+            messages: Messages in OpenAI format
+            model: Model name
+            
+        Returns:
+            Call ID for tracking the request
+            
+        Raises:
+            Exception: If queueing fails
+        """
+        try:
+            response = requests.post(
+                f"{self.base_url}/queue",
+                json={"messages": messages, "model": model},
+                timeout=10
+            )
+            response.raise_for_status()
+            return response.json()["id"]
+        except requests.RequestException as e:
+            raise Exception(f"Failed to queue completion request: {e}")
+    
+    def _wait_for_completion(self, call_id: str) -> Dict[str, Any]:
+        """Wait for human to complete the call.
+        
+        Args:
+            call_id: ID of the queued completion call
+            
+        Returns:
+            Dict containing response and/or tool_calls
+            
+        Raises:
+            TimeoutError: If timeout is exceeded
+            Exception: If completion fails
+        """
+        import time
+        
+        start_time = time.time()
+        
+        while True:
+            try:
+                # Check status
+                status_response = requests.get(f"{self.base_url}/status/{call_id}")
+                status_response.raise_for_status()
+                status_data = status_response.json()
+                
+                if status_data["status"] == "completed":
+                    result = {}
+                    if "response" in status_data and status_data["response"]:
+                        result["response"] = status_data["response"]
+                    if "tool_calls" in status_data and status_data["tool_calls"]:
+                        result["tool_calls"] = status_data["tool_calls"]
+                    return result
+                elif status_data["status"] == "failed":
+                    error_msg = status_data.get("error", "Unknown error")
+                    raise Exception(f"Completion failed: {error_msg}")
+                
+                # Check timeout
+                if time.time() - start_time > self.timeout:
+                    raise TimeoutError(f"Timeout waiting for human response after {self.timeout} seconds")
+                
+                # Wait before checking again
+                time.sleep(1.0)
+                
+            except requests.RequestException as e:
+                if time.time() - start_time > self.timeout:
+                    raise TimeoutError(f"Timeout waiting for human response: {e}")
+                # Continue trying if we haven't timed out
+                time.sleep(1.0)
+    
+    async def _async_wait_for_completion(self, call_id: str) -> Dict[str, Any]:
+        """Async version of wait_for_completion.
+        
+        Args:
+            call_id: ID of the queued completion call
+            
+        Returns:
+            Dict containing response and/or tool_calls
+            
+        Raises:
+            TimeoutError: If timeout is exceeded
+            Exception: If completion fails
+        """
+        import aiohttp
+        import time
+        
+        start_time = time.time()
+        
+        async with aiohttp.ClientSession() as session:
+            while True:
+                try:
+                    # Check status
+                    async with session.get(f"{self.base_url}/status/{call_id}") as response:
+                        response.raise_for_status()
+                        status_data = await response.json()
+                    
+                    if status_data["status"] == "completed":
+                        result = {}
+                        if "response" in status_data and status_data["response"]:
+                            result["response"] = status_data["response"]
+                        if "tool_calls" in status_data and status_data["tool_calls"]:
+                            result["tool_calls"] = status_data["tool_calls"]
+                        return result
+                    elif status_data["status"] == "failed":
+                        error_msg = status_data.get("error", "Unknown error")
+                        raise Exception(f"Completion failed: {error_msg}")
+                    
+                    # Check timeout
+                    if time.time() - start_time > self.timeout:
+                        raise TimeoutError(f"Timeout waiting for human response after {self.timeout} seconds")
+                    
+                    # Wait before checking again
+                    await asyncio.sleep(1.0)
+                    
+                except Exception as e:
+                    if time.time() - start_time > self.timeout:
+                        raise TimeoutError(f"Timeout waiting for human response: {e}")
+                    # Continue trying if we haven't timed out
+                    await asyncio.sleep(1.0)
+    
+    def _generate_response(self, messages: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+        """Generate a human response for the given messages.
+        
+        Args:
+            messages: Messages in OpenAI format
+            model: Model name
+            
+        Returns:
+            Dict containing response and/or tool_calls
+        """
+        # Queue the completion request
+        call_id = self._queue_completion(messages, model)
+        
+        # Wait for human response
+        response = self._wait_for_completion(call_id)
+        
+        return response
+    
+    async def _async_generate_response(self, messages: List[Dict[str, Any]], model: str) -> Dict[str, Any]:
+        """Async version of _generate_response.
+        
+        Args:
+            messages: Messages in OpenAI format
+            model: Model name
+            
+        Returns:
+            Dict containing response and/or tool_calls
+        """
+        # Queue the completion request (sync operation)
+        call_id = self._queue_completion(messages, model)
+        
+        # Wait for human response (async)
+        response = await self._async_wait_for_completion(call_id)
+        
+        return response
+    
+    def completion(self, *args, **kwargs) -> ModelResponse:
+        """Synchronous completion method.
+        
+        Returns:
+            ModelResponse with human-generated text or tool calls
+        """
+        messages = kwargs.get('messages', [])
+        model = kwargs.get('model', 'human')
+        
+        # Generate human response
+        human_response_data = self._generate_response(messages, model)
+        
+        # Create ModelResponse with proper structure
+        from litellm.types.utils import ModelResponse, Choices, Message
+        import uuid
+        import time
+        
+        # Create message content based on response type
+        if "tool_calls" in human_response_data and human_response_data["tool_calls"]:
+            # Tool calls response
+            message = Message(
+                role="assistant",
+                content=human_response_data.get("response", ""),
+                tool_calls=human_response_data["tool_calls"]
+            )
+        else:
+            # Text response
+            message = Message(
+                role="assistant",
+                content=human_response_data.get("response", "")
+            )
+        
+        choice = Choices(
+            finish_reason="stop",
+            index=0,
+            message=message
+        )
+        
+        result = ModelResponse(
+            id=f"human-{uuid.uuid4()}",
+            choices=[choice],
+            created=int(time.time()),
+            model=f"human/{model}",
+            object="chat.completion"
+        )
+        
+        return result
+    
+    async def acompletion(self, *args, **kwargs) -> ModelResponse:
+        """Asynchronous completion method.
+        
+        Returns:
+            ModelResponse with human-generated text or tool calls
+        """
+        messages = kwargs.get('messages', [])
+        model = kwargs.get('model', 'human')
+        
+        # Generate human response
+        human_response_data = await self._async_generate_response(messages, model)
+        
+        # Create ModelResponse with proper structure
+        from litellm.types.utils import ModelResponse, Choices, Message
+        import uuid
+        import time
+        
+        # Create message content based on response type
+        if "tool_calls" in human_response_data and human_response_data["tool_calls"]:
+            # Tool calls response
+            message = Message(
+                role="assistant",
+                content=human_response_data.get("response", ""),
+                tool_calls=human_response_data["tool_calls"]
+            )
+        else:
+            # Text response
+            message = Message(
+                role="assistant",
+                content=human_response_data.get("response", "")
+            )
+        
+        choice = Choices(
+            finish_reason="stop",
+            index=0,
+            message=message
+        )
+        
+        result = ModelResponse(
+            id=f"human-{uuid.uuid4()}",
+            choices=[choice],
+            created=int(time.time()),
+            model=f"human/{model}",
+            object="chat.completion"
+        )
+        
+        return result
+    
+    def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]:
+        """Synchronous streaming method.
+        
+        Yields:
+            Streaming chunks with human-generated text or tool calls
+        """
+        messages = kwargs.get('messages', [])
+        model = kwargs.get('model', 'human')
+        
+        # Generate human response
+        human_response_data = self._generate_response(messages, model)
+        
+        import time
+        
+        # Handle tool calls vs text response
+        if "tool_calls" in human_response_data and human_response_data["tool_calls"]:
+            # Stream tool calls as a single chunk
+            generic_chunk: GenericStreamingChunk = {
+                "finish_reason": "tool_calls",
+                "index": 0,
+                "is_finished": True,
+                "text": human_response_data.get("response", ""),
+                "tool_use": human_response_data["tool_calls"],
+                "usage": {"completion_tokens": 1, "prompt_tokens": 0, "total_tokens": 1},
+            }
+            yield generic_chunk
+        else:
+            # Stream text response
+            response_text = human_response_data.get("response", "")
+            generic_chunk: GenericStreamingChunk = {
+                "finish_reason": "stop",
+                "index": 0,
+                "is_finished": True,
+                "text": response_text,
+                "tool_use": None,
+                "usage": {"completion_tokens": len(response_text.split()), "prompt_tokens": 0, "total_tokens": len(response_text.split())},
+            }
+            yield generic_chunk
+    
+    async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]:
+        """Asynchronous streaming method.
+        
+        Yields:
+            Streaming chunks with human-generated text or tool calls
+        """
+        messages = kwargs.get('messages', [])
+        model = kwargs.get('model', 'human')
+        
+        # Generate human response
+        human_response = await self._async_generate_response(messages, model)
+        
+        # Return as single streaming chunk
+        generic_streaming_chunk: GenericStreamingChunk = {
+            "finish_reason": "stop",
+            "index": 0,
+            "is_finished": True,
+            "text": human_response,
+            "tool_use": None,
+            "usage": {"completion_tokens": len(human_response.split()), "prompt_tokens": 0, "total_tokens": len(human_response.split())},
+        }
+        
+        yield generic_streaming_chunk

agent/agent.py

@@ -7,14 +7,16 @@ from typing import Dict, List, Any, Optional, AsyncGenerator, Union, cast, Calla
 
 from litellm.responses.utils import Usage
 
-from .types import Messages, Computer, AgentCapability
+from .types import Messages, AgentCapability
 from .decorators import find_agent_config
-from .computer_handler import OpenAIComputerHandler, acknowledge_safety_check_callback, check_blocklisted_url
 import json
 import litellm
 import litellm.utils
 import inspect
-from .adapters import HuggingFaceLocalAdapter
+from .adapters import (
+    HuggingFaceLocalAdapter,
+    HumanAdapter,
+)
 from .callbacks import (
     ImageRetentionCallback, 
     LoggingCallback, 
@@ -22,9 +24,14 @@ from .callbacks import (
     BudgetManagerCallback,
     TelemetryCallback,
 )
+from .computers import (
+    AsyncComputerHandler,
+    is_agent_computer,
+    make_computer_handler
+)
 
 def get_json(obj: Any, max_depth: int = 10) -> Any:
-    def custom_serializer(o: Any, depth: int = 0, seen: Set[int] = None) -> Any:
+    def custom_serializer(o: Any, depth: int = 0, seen: Optional[Set[int]] = None) -> Any:
         if seen is None:
             seen = set()
         
@@ -211,8 +218,10 @@ class ComputerAgent:
         hf_adapter = HuggingFaceLocalAdapter(
             device="auto"
         )
+        human_adapter = HumanAdapter()
         litellm.custom_provider_map = [
-            {"provider": "huggingface-local", "custom_handler": hf_adapter}
+            {"provider": "huggingface-local", "custom_handler": hf_adapter},
+            {"provider": "human", "custom_handler": human_adapter}
         ]
         litellm.suppress_debug_info = True
 
@@ -236,10 +245,6 @@ class ComputerAgent:
     async def _initialize_computers(self):
         """Initialize computer objects"""
         if not self.tool_schemas:
-            for tool in self.tools:
-                if hasattr(tool, '_initialized') and not tool._initialized:
-                    await tool.run()
-                
             # Process tools and create tool schemas
             self.tool_schemas = self._process_tools()
             
@@ -247,7 +252,7 @@ class ComputerAgent:
             computer_handler = None
             for schema in self.tool_schemas:
                 if schema["type"] == "computer":
-                    computer_handler = OpenAIComputerHandler(schema["computer"].interface)
+                    computer_handler = await make_computer_handler(schema["computer"])
                     break
             self.computer_handler = computer_handler
     
@@ -263,7 +268,7 @@ class ComputerAgent:
         
         for tool in self.tools:
             # Check if it's a computer object (has interface attribute)
-            if hasattr(tool, 'interface'):
+            if is_agent_computer(tool):
                 # This is a computer tool - will be handled by agent loop
                 schemas.append({
                     "type": "computer",
@@ -398,7 +403,7 @@ class ComputerAgent:
     # AGENT OUTPUT PROCESSING
     # ============================================================================
     
-    async def _handle_item(self, item: Any, computer: Optional[Computer] = None, ignore_call_ids: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+    async def _handle_item(self, item: Any, computer: Optional[AsyncComputerHandler] = None, ignore_call_ids: Optional[List[str]] = None) -> List[Dict[str, Any]]:
         """Handle each item; may cause a computer action + screenshot."""
         if ignore_call_ids and item.get("call_id") and item.get("call_id") in ignore_call_ids:
             return []
@@ -450,10 +455,12 @@ class ComputerAgent:
             acknowledged_checks = []
             for check in pending_checks:
                 check_message = check.get("message", str(check))
-                if acknowledge_safety_check_callback(check_message, allow_always=True): # TODO: implement a callback for safety checks
-                    acknowledged_checks.append(check)
-                else:
-                    raise ValueError(f"Safety check failed: {check_message}")
+                acknowledged_checks.append(check)
+                # TODO: implement a callback for safety checks
+                # if acknowledge_safety_check_callback(check_message, allow_always=True):
+                #     acknowledged_checks.append(check)
+                # else:
+                #     raise ValueError(f"Safety check failed: {check_message}")
             
             # Create call output
             call_output = {
@@ -466,11 +473,12 @@ class ComputerAgent:
                 },
             }
             
-            # Additional URL safety checks for browser environments
-            if await computer.get_environment() == "browser":
-                current_url = await computer.get_current_url()
-                call_output["output"]["current_url"] = current_url
-                check_blocklisted_url(current_url)
+            # # Additional URL safety checks for browser environments
+            # if await computer.get_environment() == "browser":
+            #     current_url = await computer.get_current_url()
+            #     call_output["output"]["current_url"] = current_url
+            #     # TODO: implement a callback for URL safety checks
+            #     # check_blocklisted_url(current_url)
             
             result = [call_output]
             await self._on_computer_call_end(item, result)

agent/callbacks/trajectory_saver.py

@@ -51,12 +51,14 @@ class TrajectorySaverCallback(AsyncCallbackHandler):
     within the trajectory gets its own folder with screenshots and responses.
     """
     
-    def __init__(self, trajectory_dir: str):
+    def __init__(self, trajectory_dir: str, reset_on_run: bool = True):
         """
         Initialize trajectory saver.
         
         Args:
             trajectory_dir: Base directory to save trajectories
+            reset_on_run: If True, reset trajectory_id/turn/artifact on each run.
+                         If False, continue using existing trajectory_id if set.
         """
         self.trajectory_dir = Path(trajectory_dir)
         self.trajectory_id: Optional[str] = None
@@ -64,6 +66,7 @@ class TrajectorySaverCallback(AsyncCallbackHandler):
         self.current_artifact: int = 0
         self.model: Optional[str] = None
         self.total_usage: Dict[str, Any] = {}
+        self.reset_on_run = reset_on_run
         
         # Ensure trajectory directory exists
         self.trajectory_dir.mkdir(parents=True, exist_ok=True)
@@ -113,32 +116,38 @@ class TrajectorySaverCallback(AsyncCallbackHandler):
     async def on_run_start(self, kwargs: Dict[str, Any], old_items: List[Dict[str, Any]]) -> None:
         """Initialize trajectory tracking for a new run."""
         model = kwargs.get("model", "unknown")
-        model_name_short = model.split("+")[-1].split("/")[-1].lower()[:16]
-        if "+" in model:
-            model_name_short = model.split("+")[0].lower()[:4] + "_" + model_name_short
-
-        # id format: yyyy-mm-dd_model_hhmmss_uuid[:4]
-        now = datetime.now()
-        self.trajectory_id = f"{now.strftime('%Y-%m-%d')}_{model_name_short}_{now.strftime('%H%M%S')}_{str(uuid.uuid4())[:4]}"
-        self.current_turn = 0
-        self.current_artifact = 0
-        self.model = model
-        self.total_usage = {}
-        
-        # Create trajectory directory
-        trajectory_path = self.trajectory_dir / self.trajectory_id
-        trajectory_path.mkdir(parents=True, exist_ok=True)
-        
-        # Save trajectory metadata
-        metadata = {
-            "trajectory_id": self.trajectory_id,
-            "created_at": str(uuid.uuid1().time),
-            "status": "running",
-            "kwargs": kwargs,
-        }
         
-        with open(trajectory_path / "metadata.json", "w") as f:
-            json.dump(metadata, f, indent=2)
+        # Only reset trajectory state if reset_on_run is True or no trajectory exists
+        if self.reset_on_run or not self.trajectory_id:
+            model_name_short = model.split("+")[-1].split("/")[-1].lower()[:16]
+            if "+" in model:
+                model_name_short = model.split("+")[0].lower()[:4] + "_" + model_name_short
+
+            # id format: yyyy-mm-dd_model_hhmmss_uuid[:4]
+            now = datetime.now()
+            self.trajectory_id = f"{now.strftime('%Y-%m-%d')}_{model_name_short}_{now.strftime('%H%M%S')}_{str(uuid.uuid4())[:4]}"
+            self.current_turn = 0
+            self.current_artifact = 0
+            self.model = model
+            self.total_usage = {}
+            
+            # Create trajectory directory
+            trajectory_path = self.trajectory_dir / self.trajectory_id
+            trajectory_path.mkdir(parents=True, exist_ok=True)
+            
+            # Save trajectory metadata
+            metadata = {
+                "trajectory_id": self.trajectory_id,
+                "created_at": str(uuid.uuid1().time),
+                "status": "running",
+                "kwargs": kwargs,
+            }
+            
+            with open(trajectory_path / "metadata.json", "w") as f:
+                json.dump(metadata, f, indent=2)
+        else:
+            # Continue with existing trajectory - just update model if needed
+            self.model = model
 
     @override
     async def on_run_end(self, kwargs: Dict[str, Any], old_items: List[Dict[str, Any]], new_items: List[Dict[str, Any]]) -> None:

agent/cli.py

@@ -94,7 +94,7 @@ def print_action(action_type: str, details: Dict[str, Any], total_cost: float):
     # Format action details
     args_str = ""
     if action_type == "click" and "x" in details and "y" in details:
-        args_str = f"_{details['button']}({details['x']}, {details['y']})"
+        args_str = f"_{details.get('button', 'left')}({details['x']}, {details['y']})"
     elif action_type == "type" and "text" in details:
         text = details["text"]
         if len(text) > 50:

agent/computers/__init__.py

@@ -0,0 +1,41 @@
+"""
+Computer handler factory and interface definitions.
+
+This module provides a factory function to create computer handlers from different
+computer interface types, supporting both the ComputerHandler protocol and the
+Computer library interface.
+"""
+
+from .base import AsyncComputerHandler
+from .cua import cuaComputerHandler
+from .custom import CustomComputerHandler
+from computer import Computer as cuaComputer
+
+def is_agent_computer(computer):
+    """Check if the given computer is a ComputerHandler or CUA Computer."""
+    return isinstance(computer, AsyncComputerHandler) or \
+        isinstance(computer, cuaComputer) or \
+        (isinstance(computer, dict)) #and "screenshot" in computer)
+
+async def make_computer_handler(computer):
+    """
+    Create a computer handler from a computer interface.
+    
+    Args:
+        computer: Either a ComputerHandler instance, Computer instance, or dict of functions
+        
+    Returns:
+        ComputerHandler: A computer handler instance
+        
+    Raises:
+        ValueError: If the computer type is not supported
+    """
+    if isinstance(computer, AsyncComputerHandler):
+        return computer
+    if isinstance(computer, cuaComputer):
+        computer_handler = cuaComputerHandler(computer)
+        await computer_handler._initialize()
+        return computer_handler
+    if isinstance(computer, dict):
+        return CustomComputerHandler(computer)
+    raise ValueError(f"Unsupported computer type: {type(computer)}")