cua-agent 0.4.0b3__py3-none-any.whl → 0.4.1__py3-none-any.whl

agent/__init__.py

@@ -2,6 +2,9 @@
 agent - Decorator-based Computer Use Agent with liteLLM integration
 """
 
+import logging
+import sys
+
 from .decorators import agent_loop
 from .agent import ComputerAgent
 from .types import Messages, AgentResponse
@@ -16,4 +19,46 @@ __all__ = [
     "AgentResponse"
 ]
 
-__version__ = "0.4.0b3"
+__version__ = "0.4.0"
+
+logger = logging.getLogger(__name__)
+
+# Initialize telemetry when the package is imported
+try:
+    # Import from core telemetry for basic functions
+    from core.telemetry import (
+        is_telemetry_enabled,
+        flush,
+        record_event,
+    )
+
+    # Import set_dimension from our own telemetry module
+    from .telemetry import set_dimension
+
+    # Check if telemetry is enabled
+    if is_telemetry_enabled():
+        logger.info("Telemetry is enabled")
+
+        # Record package initialization
+        record_event(
+            "module_init",
+            {
+                "module": "agent",
+                "version": __version__,
+                "python_version": sys.version,
+            },
+        )
+
+        # Set the package version as a dimension
+        set_dimension("agent_version", __version__)
+
+        # Flush events to ensure they're sent
+        flush()
+    else:
+        logger.info("Telemetry is disabled")
+except ImportError as e:
+    # Telemetry not available
+    logger.warning(f"Telemetry not available: {e}")
+except Exception as e:
+    # Other issues with telemetry
+    logger.warning(f"Error initializing telemetry: {e}")

agent/agent.py

@@ -6,6 +6,7 @@ import asyncio
 from typing import Dict, List, Any, Optional, AsyncGenerator, Union, cast, Callable, Set
 
 from litellm.responses.utils import Usage
+
 from .types import Messages, Computer
 from .decorators import find_agent_loop
 from .computer_handler import OpenAIComputerHandler, acknowledge_safety_check_callback, check_blocklisted_url
@@ -14,7 +15,13 @@ import litellm
 import litellm.utils
 import inspect
 from .adapters import HuggingFaceLocalAdapter
-from .callbacks import ImageRetentionCallback, LoggingCallback, TrajectorySaverCallback, BudgetManagerCallback
+from .callbacks import (
+    ImageRetentionCallback, 
+    LoggingCallback, 
+    TrajectorySaverCallback, 
+    BudgetManagerCallback,
+    TelemetryCallback,
+)
 
 def get_json(obj: Any, max_depth: int = 10) -> Any:
     def custom_serializer(o: Any, depth: int = 0, seen: Set[int] = None) -> Any:
@@ -129,6 +136,7 @@ class ComputerAgent:
         screenshot_delay: Optional[float | int] = 0.5,
         use_prompt_caching: Optional[bool] = False,
         max_trajectory_budget: Optional[float | dict] = None,
+        telemetry_enabled: Optional[bool] = True,
         **kwargs
     ):
         """
@@ -146,6 +154,7 @@ class ComputerAgent:
             screenshot_delay: Delay before screenshots in seconds
             use_prompt_caching: If set, use prompt caching to avoid reprocessing the same prompt. Intended for use with anthropic providers.
             max_trajectory_budget: If set, adds BudgetManagerCallback to track usage costs and stop when budget is exceeded
+            telemetry_enabled: If set, adds TelemetryCallback to track anonymized usage data. Enabled by default.
             **kwargs: Additional arguments passed to the agent loop
         """
         self.model = model
@@ -158,10 +167,18 @@ class ComputerAgent:
         self.max_retries = max_retries
         self.screenshot_delay = screenshot_delay
         self.use_prompt_caching = use_prompt_caching
+        self.telemetry_enabled = telemetry_enabled
         self.kwargs = kwargs
 
         # == Add built-in callbacks ==
 
+        # Add telemetry callback if telemetry_enabled is set
+        if self.telemetry_enabled:
+            if isinstance(self.telemetry_enabled, bool):
+                self.callbacks.append(TelemetryCallback(self))
+            else:
+                self.callbacks.append(TelemetryCallback(self, **self.telemetry_enabled))
+
         # Add logging callback if verbosity is set
         if self.verbosity is not None:
             self.callbacks.append(LoggingCallback(level=self.verbosity))

agent/callbacks/__init__.py

@@ -7,6 +7,7 @@ from .image_retention import ImageRetentionCallback
 from .logging import LoggingCallback
 from .trajectory_saver import TrajectorySaverCallback
 from .budget_manager import BudgetManagerCallback
+from .telemetry import TelemetryCallback
 
 __all__ = [
     "AsyncCallbackHandler",
@@ -14,4 +15,5 @@ __all__ = [
     "LoggingCallback",
     "TrajectorySaverCallback",
     "BudgetManagerCallback",
+    "TelemetryCallback",
 ]

agent/callbacks/budget_manager.py

@@ -1,44 +1,44 @@
-from typing import Dict, List, Any
-from .base import AsyncCallbackHandler
-
-class BudgetExceededError(Exception):
-    """Exception raised when budget is exceeded."""
-    pass
-
-class BudgetManagerCallback(AsyncCallbackHandler):
-    """Budget manager callback that tracks usage costs and can stop execution when budget is exceeded."""
-    
-    def __init__(self, max_budget: float, reset_after_each_run: bool = True, raise_error: bool = False):
-        """
-        Initialize BudgetManagerCallback.
-        
-        Args:
-            max_budget: Maximum budget allowed
-            reset_after_each_run: Whether to reset budget after each run
-            raise_error: Whether to raise an error when budget is exceeded
-        """
-        self.max_budget = max_budget
-        self.reset_after_each_run = reset_after_each_run
-        self.raise_error = raise_error
-        self.total_cost = 0.0
-    
-    async def on_run_start(self, kwargs: Dict[str, Any], old_items: List[Dict[str, Any]]) -> None:
-        """Reset budget if configured to do so."""
-        if self.reset_after_each_run:
-            self.total_cost = 0.0
-    
-    async def on_usage(self, usage: Dict[str, Any]) -> None:
-        """Track usage costs."""
-        if "response_cost" in usage:
-            self.total_cost += usage["response_cost"]
-    
-    async def on_run_continue(self, kwargs: Dict[str, Any], old_items: List[Dict[str, Any]], new_items: List[Dict[str, Any]]) -> bool:
-        """Check if budget allows continuation."""
-        if self.total_cost >= self.max_budget:
-            if self.raise_error:
-                raise BudgetExceededError(f"Budget exceeded: ${self.total_cost} >= ${self.max_budget}")
-            else:
-                print(f"Budget exceeded: ${self.total_cost} >= ${self.max_budget}")
-            return False
-        return True
+from typing import Dict, List, Any
+from .base import AsyncCallbackHandler
+
+class BudgetExceededError(Exception):
+    """Exception raised when budget is exceeded."""
+    pass
+
+class BudgetManagerCallback(AsyncCallbackHandler):
+    """Budget manager callback that tracks usage costs and can stop execution when budget is exceeded."""
+    
+    def __init__(self, max_budget: float, reset_after_each_run: bool = True, raise_error: bool = False):
+        """
+        Initialize BudgetManagerCallback.
+        
+        Args:
+            max_budget: Maximum budget allowed
+            reset_after_each_run: Whether to reset budget after each run
+            raise_error: Whether to raise an error when budget is exceeded
+        """
+        self.max_budget = max_budget
+        self.reset_after_each_run = reset_after_each_run
+        self.raise_error = raise_error
+        self.total_cost = 0.0
+    
+    async def on_run_start(self, kwargs: Dict[str, Any], old_items: List[Dict[str, Any]]) -> None:
+        """Reset budget if configured to do so."""
+        if self.reset_after_each_run:
+            self.total_cost = 0.0
+    
+    async def on_usage(self, usage: Dict[str, Any]) -> None:
+        """Track usage costs."""
+        if "response_cost" in usage:
+            self.total_cost += usage["response_cost"]
+    
+    async def on_run_continue(self, kwargs: Dict[str, Any], old_items: List[Dict[str, Any]], new_items: List[Dict[str, Any]]) -> bool:
+        """Check if budget allows continuation."""
+        if self.total_cost >= self.max_budget:
+            if self.raise_error:
+                raise BudgetExceededError(f"Budget exceeded: ${self.total_cost} >= ${self.max_budget}")
+            else:
+                print(f"Budget exceeded: ${self.total_cost} >= ${self.max_budget}")
+            return False
+        return True
     

agent/callbacks/telemetry.py

@@ -0,0 +1,210 @@
+"""
+Telemetry callback handler for Computer-Use Agent (cua-agent)
+"""
+
+import time
+import uuid
+from typing import List, Dict, Any, Optional, Union
+
+from .base import AsyncCallbackHandler
+from ..telemetry import (
+    record_event,
+    is_telemetry_enabled,
+    set_dimension,
+    SYSTEM_INFO,
+)
+
+
+class TelemetryCallback(AsyncCallbackHandler):
+    """
+    Telemetry callback handler for Computer-Use Agent (cua-agent)
+    
+    Tracks agent usage, performance metrics, and optionally trajectory data.
+    """
+    
+    def __init__(
+        self, 
+        agent, 
+        log_trajectory: bool = False
+    ):
+        """
+        Initialize telemetry callback.
+        
+        Args:
+            agent: The ComputerAgent instance
+            log_trajectory: Whether to log full trajectory items (opt-in)
+        """
+        self.agent = agent
+        self.log_trajectory = log_trajectory
+        
+        # Generate session/run IDs
+        self.session_id = str(uuid.uuid4())
+        self.run_id = None
+        
+        # Track timing and metrics
+        self.run_start_time = None
+        self.step_count = 0
+        self.step_start_time = None
+        self.total_usage = {
+            "prompt_tokens": 0,
+            "completion_tokens": 0,
+            "total_tokens": 0,
+            "response_cost": 0.0
+        }
+        
+        # Record agent initialization
+        if is_telemetry_enabled():
+            self._record_agent_initialization()
+    
+    def _record_agent_initialization(self) -> None:
+        """Record agent type/model and session initialization."""
+        agent_info = {
+            "session_id": self.session_id,
+            "agent_type": self.agent.agent_loop.__name__,
+            "model": getattr(self.agent, 'model', 'unknown'),
+            **SYSTEM_INFO
+        }
+        
+        # Set session-level dimensions
+        set_dimension("session_id", self.session_id)
+        set_dimension("agent_type", agent_info["agent_type"])
+        set_dimension("model", agent_info["model"])
+        
+        record_event("agent_session_start", agent_info)
+    
+    async def on_run_start(self, kwargs: Dict[str, Any], old_items: List[Dict[str, Any]]) -> None:
+        """Called at the start of an agent run loop."""
+        if not is_telemetry_enabled():
+            return
+            
+        self.run_id = str(uuid.uuid4())
+        self.run_start_time = time.time()
+        self.step_count = 0
+        
+        # Calculate input context size
+        input_context_size = self._calculate_context_size(old_items)
+        
+        run_data = {
+            "session_id": self.session_id,
+            "run_id": self.run_id,
+            "start_time": self.run_start_time,
+            "input_context_size": input_context_size,
+            "num_existing_messages": len(old_items)
+        }
+        
+        # Log trajectory if opted in
+        if self.log_trajectory:
+            trajectory = self._extract_trajectory(old_items)
+            if trajectory:
+                run_data["uploaded_trajectory"] = trajectory
+        
+        set_dimension("run_id", self.run_id)
+        record_event("agent_run_start", run_data)
+    
+    async def on_run_end(self, kwargs: Dict[str, Any], old_items: List[Dict[str, Any]], new_items: List[Dict[str, Any]]) -> None:
+        """Called at the end of an agent run loop."""
+        if not is_telemetry_enabled() or not self.run_start_time:
+            return
+            
+        run_duration = time.time() - self.run_start_time
+        
+        run_data = {
+            "session_id": self.session_id,
+            "run_id": self.run_id,
+            "end_time": time.time(),
+            "duration_seconds": run_duration,
+            "num_steps": self.step_count,
+            "total_usage": self.total_usage.copy()
+        }
+        
+        # Log trajectory if opted in
+        if self.log_trajectory:
+            trajectory = self._extract_trajectory(new_items)
+            if trajectory:
+                run_data["uploaded_trajectory"] = trajectory
+        
+        record_event("agent_run_end", run_data)
+    
+    async def on_usage(self, usage: Dict[str, Any]) -> None:
+        """Called when usage information is received."""
+        if not is_telemetry_enabled():
+            return
+            
+        # Accumulate usage stats
+        self.total_usage["prompt_tokens"] += usage.get("prompt_tokens", 0)
+        self.total_usage["completion_tokens"] += usage.get("completion_tokens", 0) 
+        self.total_usage["total_tokens"] += usage.get("total_tokens", 0)
+        self.total_usage["response_cost"] += usage.get("response_cost", 0.0)
+        
+        # Record individual usage event
+        usage_data = {
+            "session_id": self.session_id,
+            "run_id": self.run_id,
+            "step": self.step_count,
+            **usage
+        }
+        
+        record_event("agent_usage", usage_data)
+    
+    async def on_responses(self, kwargs: Dict[str, Any], responses: Dict[str, Any]) -> None:
+        """Called when responses are received."""
+        if not is_telemetry_enabled():
+            return
+            
+        self.step_count += 1
+        step_duration = None
+        
+        if self.step_start_time:
+            step_duration = time.time() - self.step_start_time
+        
+        self.step_start_time = time.time()
+        
+        step_data = {
+            "session_id": self.session_id,
+            "run_id": self.run_id,
+            "step": self.step_count,
+            "timestamp": self.step_start_time
+        }
+        
+        if step_duration is not None:
+            step_data["duration_seconds"] = step_duration
+        
+        record_event("agent_step", step_data)
+    
+    def _calculate_context_size(self, items: List[Dict[str, Any]]) -> int:
+        """Calculate approximate context size in tokens/characters."""
+        total_size = 0
+        
+        for item in items:
+            if item.get("type") == "message" and "content" in item:
+                content = item["content"]
+                if isinstance(content, str):
+                    total_size += len(content)
+                elif isinstance(content, list):
+                    for part in content:
+                        if isinstance(part, dict) and "text" in part:
+                            total_size += len(part["text"])
+            elif "content" in item and isinstance(item["content"], str):
+                total_size += len(item["content"])
+                
+        return total_size
+    
+    def _extract_trajectory(self, items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Extract trajectory items that should be logged."""
+        trajectory = []
+        
+        for item in items:
+            # Include user messages, assistant messages, reasoning, computer calls, and computer outputs
+            if (
+                item.get("role") == "user" or  # User inputs
+                (item.get("type") == "message" and item.get("role") == "assistant") or  # Model outputs
+                item.get("type") == "reasoning" or  # Reasoning traces
+                item.get("type") == "computer_call" or  # Computer actions
+                item.get("type") == "computer_call_output"  # Computer outputs
+            ):
+                # Create a copy of the item with timestamp
+                trajectory_item = item.copy()
+                trajectory_item["logged_at"] = time.time()
+                trajectory.append(trajectory_item)
+        
+        return trajectory

agent/cli.py

@@ -10,14 +10,21 @@ Examples:
     python -m agent.cli omniparser+anthropic/claude-3-5-sonnet-20241022
 """
 
-import asyncio
-import argparse
-import os
-import sys
-import json
-from typing import List, Dict, Any
-import dotenv
-from yaspin import yaspin
+try:
+    import asyncio
+    import argparse
+    import os
+    import sys
+    import json
+    from typing import List, Dict, Any
+    import dotenv
+    from yaspin import yaspin
+except ImportError:
+    if __name__ == "__main__":
+        raise ImportError(
+            "CLI dependencies not found. "
+            "Please install with: pip install \"cua-agent[cli]\""
+        )
 
 # Load environment variables
 dotenv.load_dotenv()

agent/responses.py

@@ -1,207 +1,207 @@
-"""
-Functions for making various Responses API items from different types of responses.
-Based on the OpenAI spec for Responses API items.
-"""
-
-import base64
-import json
-import uuid
-from typing import List, Dict, Any, Literal, Union, Optional
-
-from openai.types.responses.response_computer_tool_call_param import (
-    ResponseComputerToolCallParam, 
-    ActionClick,
-    ActionDoubleClick,
-    ActionDrag,
-    ActionDragPath,
-    ActionKeypress,
-    ActionMove,
-    ActionScreenshot,
-    ActionScroll,
-    ActionType as ActionTypeAction,
-    ActionWait,
-    PendingSafetyCheck
-)
-
-from openai.types.responses.response_function_tool_call_param import ResponseFunctionToolCallParam
-from openai.types.responses.response_output_text_param import ResponseOutputTextParam
-from openai.types.responses.response_reasoning_item_param import ResponseReasoningItemParam, Summary
-from openai.types.responses.response_output_message_param import ResponseOutputMessageParam
-from openai.types.responses.easy_input_message_param import EasyInputMessageParam
-from openai.types.responses.response_input_image_param import ResponseInputImageParam
-
-def random_id():
-    return str(uuid.uuid4())
-
-# User message items
-def make_input_image_item(image_data: Union[str, bytes]) -> EasyInputMessageParam:
-    return EasyInputMessageParam(
-        content=[
-            ResponseInputImageParam(
-                type="input_image",
-                image_url=f"data:image/png;base64,{base64.b64encode(image_data).decode('utf-8') if isinstance(image_data, bytes) else image_data}"
-            )
-        ],
-        role="user",
-        type="message"
-    )
-
-# Text items
-def make_reasoning_item(reasoning: str) -> ResponseReasoningItemParam:
-    return ResponseReasoningItemParam(
-        id=random_id(),
-        summary=[
-            Summary(text=reasoning, type="summary_text")
-        ],
-        type="reasoning"
-    )
-
-def make_output_text_item(content: str) -> ResponseOutputMessageParam:
-    return ResponseOutputMessageParam(
-        id=random_id(),
-        content=[
-            ResponseOutputTextParam(
-                text=content,
-                type="output_text",
-                annotations=[]
-            )
-        ],
-        role="assistant",
-        status="completed",
-        type="message"
-    )
-
-# Function call items
-def make_function_call_item(function_name: str, arguments: Dict[str, Any], call_id: Optional[str] = None) -> ResponseFunctionToolCallParam:
-    return ResponseFunctionToolCallParam(
-        id=random_id(),
-        call_id=call_id if call_id else random_id(),
-        name=function_name,
-        arguments=json.dumps(arguments),
-        status="completed",
-        type="function_call"
-    )
-
-# Computer tool call items
-def make_click_item(x: int, y: int, button: Literal["left", "right", "wheel", "back", "forward"] = "left", call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
-    return ResponseComputerToolCallParam(
-        id=random_id(),
-        call_id=call_id if call_id else random_id(),
-        action=ActionClick(
-            button=button,
-            type="click",
-            x=x,
-            y=y
-        ),
-        pending_safety_checks=[],
-        status="completed",
-        type="computer_call"
-    )
-
-def make_double_click_item(x: int, y: int, call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
-    return ResponseComputerToolCallParam(
-        id=random_id(),
-        call_id=call_id if call_id else random_id(),
-        action=ActionDoubleClick(
-            type="double_click",
-            x=x,
-            y=y
-        ),
-        pending_safety_checks=[],
-        status="completed",
-        type="computer_call"
-    )
-
-def make_drag_item(path: List[Dict[str, int]], call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
-    drag_path = [ActionDragPath(x=point["x"], y=point["y"]) for point in path]
-    return ResponseComputerToolCallParam(
-        id=random_id(),
-        call_id=call_id if call_id else random_id(),
-        action=ActionDrag(
-            path=drag_path,
-            type="drag"
-        ),
-        pending_safety_checks=[],
-        status="completed",
-        type="computer_call"
-    )
-
-def make_keypress_item(keys: List[str], call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
-    return ResponseComputerToolCallParam(
-        id=random_id(),
-        call_id=call_id if call_id else random_id(),
-        action=ActionKeypress(
-            keys=keys,
-            type="keypress"
-        ),
-        pending_safety_checks=[],
-        status="completed",
-        type="computer_call"
-    )
-
-def make_move_item(x: int, y: int, call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
-    return ResponseComputerToolCallParam(
-        id=random_id(),
-        call_id=call_id if call_id else random_id(),
-        action=ActionMove(
-            type="move",
-            x=x,
-            y=y
-        ),
-        pending_safety_checks=[],
-        status="completed",
-        type="computer_call"
-    )
-
-def make_screenshot_item(call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
-    return ResponseComputerToolCallParam(
-        id=random_id(),
-        call_id=call_id if call_id else random_id(),
-        action=ActionScreenshot(
-            type="screenshot"
-        ),
-        pending_safety_checks=[],
-        status="completed",
-        type="computer_call"
-    )
-
-def make_scroll_item(x: int, y: int, scroll_x: int, scroll_y: int, call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
-    return ResponseComputerToolCallParam(
-        id=random_id(),
-        call_id=call_id if call_id else random_id(),
-        action=ActionScroll(
-            scroll_x=scroll_x,
-            scroll_y=scroll_y,
-            type="scroll",
-            x=x,
-            y=y
-        ),
-        pending_safety_checks=[],
-        status="completed",
-        type="computer_call"
-    )
-
-def make_type_item(text: str, call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
-    return ResponseComputerToolCallParam(
-        id=random_id(),
-        call_id=call_id if call_id else random_id(),
-        action=ActionTypeAction(
-            text=text,
-            type="type"
-        ),
-        pending_safety_checks=[],
-        status="completed",
-        type="computer_call"
-    )
-
-def make_wait_item(call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
-    return ResponseComputerToolCallParam(
-        id=random_id(),
-        call_id=call_id if call_id else random_id(),
-        action=ActionWait(
-            type="wait"
-        ),
-        pending_safety_checks=[],
-        status="completed",
-        type="computer_call"
-    )
+"""
+Functions for making various Responses API items from different types of responses.
+Based on the OpenAI spec for Responses API items.
+"""
+
+import base64
+import json
+import uuid
+from typing import List, Dict, Any, Literal, Union, Optional
+
+from openai.types.responses.response_computer_tool_call_param import (
+    ResponseComputerToolCallParam, 
+    ActionClick,
+    ActionDoubleClick,
+    ActionDrag,
+    ActionDragPath,
+    ActionKeypress,
+    ActionMove,
+    ActionScreenshot,
+    ActionScroll,
+    ActionType as ActionTypeAction,
+    ActionWait,
+    PendingSafetyCheck
+)
+
+from openai.types.responses.response_function_tool_call_param import ResponseFunctionToolCallParam
+from openai.types.responses.response_output_text_param import ResponseOutputTextParam
+from openai.types.responses.response_reasoning_item_param import ResponseReasoningItemParam, Summary
+from openai.types.responses.response_output_message_param import ResponseOutputMessageParam
+from openai.types.responses.easy_input_message_param import EasyInputMessageParam
+from openai.types.responses.response_input_image_param import ResponseInputImageParam
+
+def random_id():
+    return str(uuid.uuid4())
+
+# User message items
+def make_input_image_item(image_data: Union[str, bytes]) -> EasyInputMessageParam:
+    return EasyInputMessageParam(
+        content=[
+            ResponseInputImageParam(
+                type="input_image",
+                image_url=f"data:image/png;base64,{base64.b64encode(image_data).decode('utf-8') if isinstance(image_data, bytes) else image_data}"
+            )
+        ],
+        role="user",
+        type="message"
+    )
+
+# Text items
+def make_reasoning_item(reasoning: str) -> ResponseReasoningItemParam:
+    return ResponseReasoningItemParam(
+        id=random_id(),
+        summary=[
+            Summary(text=reasoning, type="summary_text")
+        ],
+        type="reasoning"
+    )
+
+def make_output_text_item(content: str) -> ResponseOutputMessageParam:
+    return ResponseOutputMessageParam(
+        id=random_id(),
+        content=[
+            ResponseOutputTextParam(
+                text=content,
+                type="output_text",
+                annotations=[]
+            )
+        ],
+        role="assistant",
+        status="completed",
+        type="message"
+    )
+
+# Function call items
+def make_function_call_item(function_name: str, arguments: Dict[str, Any], call_id: Optional[str] = None) -> ResponseFunctionToolCallParam:
+    return ResponseFunctionToolCallParam(
+        id=random_id(),
+        call_id=call_id if call_id else random_id(),
+        name=function_name,
+        arguments=json.dumps(arguments),
+        status="completed",
+        type="function_call"
+    )
+
+# Computer tool call items
+def make_click_item(x: int, y: int, button: Literal["left", "right", "wheel", "back", "forward"] = "left", call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
+    return ResponseComputerToolCallParam(
+        id=random_id(),
+        call_id=call_id if call_id else random_id(),
+        action=ActionClick(
+            button=button,
+            type="click",
+            x=x,
+            y=y
+        ),
+        pending_safety_checks=[],
+        status="completed",
+        type="computer_call"
+    )
+
+def make_double_click_item(x: int, y: int, call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
+    return ResponseComputerToolCallParam(
+        id=random_id(),
+        call_id=call_id if call_id else random_id(),
+        action=ActionDoubleClick(
+            type="double_click",
+            x=x,
+            y=y
+        ),
+        pending_safety_checks=[],
+        status="completed",
+        type="computer_call"
+    )
+
+def make_drag_item(path: List[Dict[str, int]], call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
+    drag_path = [ActionDragPath(x=point["x"], y=point["y"]) for point in path]
+    return ResponseComputerToolCallParam(
+        id=random_id(),
+        call_id=call_id if call_id else random_id(),
+        action=ActionDrag(
+            path=drag_path,
+            type="drag"
+        ),
+        pending_safety_checks=[],
+        status="completed",
+        type="computer_call"
+    )
+
+def make_keypress_item(keys: List[str], call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
+    return ResponseComputerToolCallParam(
+        id=random_id(),
+        call_id=call_id if call_id else random_id(),
+        action=ActionKeypress(
+            keys=keys,
+            type="keypress"
+        ),
+        pending_safety_checks=[],
+        status="completed",
+        type="computer_call"
+    )
+
+def make_move_item(x: int, y: int, call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
+    return ResponseComputerToolCallParam(
+        id=random_id(),
+        call_id=call_id if call_id else random_id(),
+        action=ActionMove(
+            type="move",
+            x=x,
+            y=y
+        ),
+        pending_safety_checks=[],
+        status="completed",
+        type="computer_call"
+    )
+
+def make_screenshot_item(call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
+    return ResponseComputerToolCallParam(
+        id=random_id(),
+        call_id=call_id if call_id else random_id(),
+        action=ActionScreenshot(
+            type="screenshot"
+        ),
+        pending_safety_checks=[],
+        status="completed",
+        type="computer_call"
+    )
+
+def make_scroll_item(x: int, y: int, scroll_x: int, scroll_y: int, call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
+    return ResponseComputerToolCallParam(
+        id=random_id(),
+        call_id=call_id if call_id else random_id(),
+        action=ActionScroll(
+            scroll_x=scroll_x,
+            scroll_y=scroll_y,
+            type="scroll",
+            x=x,
+            y=y
+        ),
+        pending_safety_checks=[],
+        status="completed",
+        type="computer_call"
+    )
+
+def make_type_item(text: str, call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
+    return ResponseComputerToolCallParam(
+        id=random_id(),
+        call_id=call_id if call_id else random_id(),
+        action=ActionTypeAction(
+            text=text,
+            type="type"
+        ),
+        pending_safety_checks=[],
+        status="completed",
+        type="computer_call"
+    )
+
+def make_wait_item(call_id: Optional[str] = None) -> ResponseComputerToolCallParam:
+    return ResponseComputerToolCallParam(
+        id=random_id(),
+        call_id=call_id if call_id else random_id(),
+        action=ActionWait(
+            type="wait"
+        ),
+        pending_safety_checks=[],
+        status="completed",
+        type="computer_call"
+    )

agent/telemetry.py

@@ -0,0 +1,142 @@
+"""Agent telemetry for tracking anonymous usage and feature usage."""
+
+import logging
+import os
+import platform
+import sys
+from typing import Dict, Any, Callable
+
+# Import the core telemetry module
+TELEMETRY_AVAILABLE = False
+
+
+# Local fallbacks in case core telemetry isn't available
+def _noop(*args: Any, **kwargs: Any) -> None:
+    """No-op function for when telemetry is not available."""
+    pass
+
+
+# Define default functions with unique names to avoid shadowing
+_default_record_event = _noop
+_default_increment_counter = _noop
+_default_set_dimension = _noop
+_default_get_telemetry_client = lambda: None
+_default_flush = _noop
+_default_is_telemetry_enabled = lambda: False
+_default_is_telemetry_globally_disabled = lambda: True
+
+# Set the actual functions to the defaults initially
+record_event = _default_record_event
+increment_counter = _default_increment_counter
+set_dimension = _default_set_dimension
+get_telemetry_client = _default_get_telemetry_client
+flush = _default_flush
+is_telemetry_enabled = _default_is_telemetry_enabled
+is_telemetry_globally_disabled = _default_is_telemetry_globally_disabled
+
+logger = logging.getLogger("agent.telemetry")
+
+try:
+    # Import from core telemetry
+    from core.telemetry import (
+        record_event as core_record_event,
+        increment as core_increment,
+        get_telemetry_client as core_get_telemetry_client,
+        flush as core_flush,
+        is_telemetry_enabled as core_is_telemetry_enabled,
+        is_telemetry_globally_disabled as core_is_telemetry_globally_disabled,
+    )
+
+    # Override the default functions with actual implementations
+    record_event = core_record_event
+    get_telemetry_client = core_get_telemetry_client
+    flush = core_flush
+    is_telemetry_enabled = core_is_telemetry_enabled
+    is_telemetry_globally_disabled = core_is_telemetry_globally_disabled
+
+    def increment_counter(counter_name: str, value: int = 1) -> None:
+        """Wrapper for increment to maintain backward compatibility."""
+        if is_telemetry_enabled():
+            core_increment(counter_name, value)
+
+    def set_dimension(name: str, value: Any) -> None:
+        """Set a dimension that will be attached to all events."""
+        logger.debug(f"Setting dimension {name}={value}")
+
+    TELEMETRY_AVAILABLE = True
+    logger.info("Successfully imported telemetry")
+except ImportError as e:
+    logger.warning(f"Could not import telemetry: {e}")
+    logger.debug("Telemetry not available, using no-op functions")
+
+# Get system info once to use in telemetry
+SYSTEM_INFO = {
+    "os": platform.system().lower(),
+    "os_version": platform.release(),
+    "python_version": platform.python_version(),
+}
+
+
+def enable_telemetry() -> bool:
+    """Enable telemetry if available.
+
+    Returns:
+        bool: True if telemetry was successfully enabled, False otherwise
+    """
+    global TELEMETRY_AVAILABLE, record_event, increment_counter, get_telemetry_client, flush, is_telemetry_enabled, is_telemetry_globally_disabled
+
+    # Check if globally disabled using core function
+    if TELEMETRY_AVAILABLE and is_telemetry_globally_disabled():
+        logger.info("Telemetry is globally disabled via environment variable - cannot enable")
+        return False
+
+    # Already enabled
+    if TELEMETRY_AVAILABLE:
+        return True
+
+    # Try to import and enable
+    try:
+        from core.telemetry import (
+            record_event,
+            increment,
+            get_telemetry_client,
+            flush,
+            is_telemetry_globally_disabled,
+        )
+
+        # Check again after import
+        if is_telemetry_globally_disabled():
+            logger.info("Telemetry is globally disabled via environment variable - cannot enable")
+            return False
+
+        TELEMETRY_AVAILABLE = True
+        logger.info("Telemetry successfully enabled")
+        return True
+    except ImportError as e:
+        logger.warning(f"Could not enable telemetry: {e}")
+        return False
+
+
+def is_telemetry_enabled() -> bool:
+    """Check if telemetry is enabled.
+
+    Returns:
+        bool: True if telemetry is enabled, False otherwise
+    """
+    # Use the core function if available, otherwise use our local flag
+    if TELEMETRY_AVAILABLE:
+        from core.telemetry import is_telemetry_enabled as core_is_enabled
+
+        return core_is_enabled()
+    return False
+
+
+def record_agent_initialization() -> None:
+    """Record when an agent instance is initialized."""
+    if TELEMETRY_AVAILABLE and is_telemetry_enabled():
+        record_event("agent_initialized", SYSTEM_INFO)
+
+        # Set dimensions that will be attached to all events
+        set_dimension("os", SYSTEM_INFO["os"])
+        set_dimension("os_version", SYSTEM_INFO["os_version"])
+        set_dimension("python_version", SYSTEM_INFO["python_version"])

agent/ui/__init__.py

@@ -2,6 +2,6 @@
 UI components for agent
 """
 
-from .gradio import test_cua, create_gradio_ui
+from .gradio import launch_ui, create_gradio_ui
 
-__all__ = ["test_cua", "create_gradio_ui"]
+__all__ = ["launch_ui", "create_gradio_ui"]

agent/ui/__main__.py

@@ -0,0 +1,4 @@
+from .gradio import launch_ui
+
+if __name__ == "__main__":
+    launch_ui()

agent/ui/gradio/__init__.py

@@ -2,7 +2,7 @@
 Gradio UI for agent
 """
 
-from .app import test_cua
+from .app import launch_ui
 from .ui_components import create_gradio_ui
 
-__all__ = ["test_cua", "create_gradio_ui"]
+__all__ = ["launch_ui", "create_gradio_ui"]

agent/ui/gradio/app.py

@@ -72,26 +72,26 @@ def save_settings(settings: Dict[str, Any]):
         print(f"Warning: Could not save settings to {SETTINGS_FILE}: {e}")
 
 
-# Custom Screenshot Handler for Gradio chat
-class GradioChatScreenshotHandler:
-    """Custom handler that adds screenshots to the Gradio chatbot."""
+# # Custom Screenshot Handler for Gradio chat
+# class GradioChatScreenshotHandler:
+#     """Custom handler that adds screenshots to the Gradio chatbot."""
 
-    def __init__(self, chatbot_history: List[gr.ChatMessage]):
-        self.chatbot_history = chatbot_history
-        print("GradioChatScreenshotHandler initialized")
+#     def __init__(self, chatbot_history: List[gr.ChatMessage]):
+#         self.chatbot_history = chatbot_history
+#         print("GradioChatScreenshotHandler initialized")
 
-    async def on_screenshot(self, screenshot_base64: str, action_type: str = "") -> None:
-        """Add screenshot to chatbot when a screenshot is taken."""
-        image_markdown = f"![Screenshot after {action_type}](data:image/png;base64,{screenshot_base64})"
+#     async def on_screenshot(self, screenshot_base64: str, action_type: str = "") -> None:
+#         """Add screenshot to chatbot when a screenshot is taken."""
+#         image_markdown = f"![Screenshot after {action_type}](data:image/png;base64,{screenshot_base64})"
         
-        if self.chatbot_history is not None:
-            self.chatbot_history.append(
-                gr.ChatMessage(
-                    role="assistant",
-                    content=image_markdown,
-                    metadata={"title": f"🖥️ Screenshot - {action_type}", "status": "done"},
-                )
-            )
+#         if self.chatbot_history is not None:
+#             self.chatbot_history.append(
+#                 gr.ChatMessage(
+#                     role="assistant",
+#                     content=image_markdown,
+#                     metadata={"title": f"🖥️ Screenshot - {action_type}", "status": "done"},
+#                 )
+#             )
 
 
 # Detect platform capabilities
@@ -236,7 +236,7 @@ def create_agent(
     return global_agent
 
 
-def test_cua():
+def launch_ui():
     """Standalone function to launch the Gradio app."""
     from agent.ui.gradio.ui_components import create_gradio_ui
     print(f"Starting Gradio app for CUA Agent...")
@@ -245,4 +245,4 @@ def test_cua():
 
 
 if __name__ == "__main__":
-    test_cua()
+    launch_ui()

agent/ui/gradio/ui_components.py

@@ -14,9 +14,12 @@ from gradio.components.chatbot import MetadataDict
 
 from .app import (
     load_settings, save_settings, create_agent, get_model_string, 
-    get_ollama_models, GradioChatScreenshotHandler, global_agent, global_computer
+    get_ollama_models, global_agent, global_computer
 )
 
+# Global messages array to maintain conversation history
+global_messages = []
+
 
 def create_gradio_ui() -> gr.Blocks:
     """Create a Gradio UI for the Computer-Use Agent."""
@@ -571,15 +574,17 @@ if __name__ == "__main__":
                             yield history
                             return
 
-                        # Create message list for agent
-                        messages = [{"role": "user", "content": last_user_message}]
-
+                        # Add user message to global history
+                        global global_messages
+                        global_messages.append({"role": "user", "content": last_user_message})
+                        
                         # Stream responses from the agent
-                        async for result in global_agent.run(messages):
-                            print(f"DEBUG - Agent response ------- START")
-                            from pprint import pprint
-                            pprint(result)
-                            print(f"DEBUG - Agent response ------- END")
+                        async for result in global_agent.run(global_messages):
+                            global_messages += result.get("output", [])
+                            # print(f"DEBUG - Agent response ------- START")
+                            # from pprint import pprint
+                            # pprint(result)
+                            # print(f"DEBUG - Agent response ------- END")
                             
                             # Process the result output
                             for item in result.get("output", []):
@@ -619,6 +624,14 @@ if __name__ == "__main__":
                                         content=f"📤 Function output:\n```\n{output}\n```",
                                         metadata={"title": "Function Output"}
                                     ))
+                                elif item.get("type") == "computer_call_output":
+                                    output = item.get("output", {}).get("image_url", "")
+                                    image_markdown = f"![Computer output]({output})"
+                                    history.append(gr.ChatMessage(
+                                        role="assistant",
+                                        content=image_markdown,
+                                        metadata={"title": "🖥️ Computer Output"}
+                                    ))
                             
                             yield history
                             
@@ -661,7 +674,12 @@ if __name__ == "__main__":
                 )
 
                 # Clear button functionality
-                clear.click(lambda: None, None, chatbot_history, queue=False)
+                def clear_chat():
+                    global global_messages
+                    global_messages.clear()
+                    return None
+                
+                clear.click(clear_chat, None, chatbot_history, queue=False)
                 
                 # Connect cancel button
                 cancel_button.click(

{cua_agent-0.4.0b3.dist-info → cua_agent-0.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cua-agent
-Version: 0.4.0b3
+Version: 0.4.1
 Summary: CUA (Computer Use) Agent for AI-driven computer interaction
 Author-Email: TryCua <gh@trycua.com>
 Requires-Python: >=3.11

cua_agent-0.4.1.dist-info/RECORD

@@ -0,0 +1,33 @@
+agent/__init__.py,sha256=PfRgVa_aJQL9fK0D1g2r__Kdg3627EigNS31_M8Ivkk,1539
+agent/__main__.py,sha256=lBUe8Niqa5XoCjwFfXyX7GtnUwjjZXC1-j4V9mvUYSc,538
+agent/adapters/__init__.py,sha256=szM2HMten2WkcqXeRnan__-sXjpyS4eyvIW0LXSfj4U,178
+agent/adapters/huggingfacelocal_adapter.py,sha256=dnzzxYCvFiuDdNzsb_1uM-boWv1eS__dWMve_fAnlUc,8038
+agent/agent.py,sha256=Vn7ygehx19It5FarZJ2NwVwNTOtNYtD21x8LEBhlWcE,24609
+agent/callbacks/__init__.py,sha256=yxxBXUqpXQ-jRi_ixJMtmQPxoNRy5Vz1PUBzNNa1Dwg,538
+agent/callbacks/base.py,sha256=UnnnYlh6XCm6HKZZsAPaT_Eyo9LUYLyjyNwF-QRm6Ns,4691
+agent/callbacks/budget_manager.py,sha256=RyKM-7iXQcDotYvrw3eURzeEHEXvQjID-NobtvQWE7k,1832
+agent/callbacks/image_retention.py,sha256=tiuRT5ke9xXTb2eP8Gz-2ITyAMY29LURUH6AbjX3RP8,6165
+agent/callbacks/logging.py,sha256=OOxU97EzrxlnUAtiEnvy9FB7SwCUK90-rdpDFA2Ae4E,10921
+agent/callbacks/pii_anonymization.py,sha256=UKAqNacHG3z92_6uocVzOIl8gJoqyofldCoCmB4UVIE,10268
+agent/callbacks/telemetry.py,sha256=sYsE_-tnZkt1ydIRbp_GfCETlz7QG9DNbawq6hM4Bqw,7445
+agent/callbacks/trajectory_saver.py,sha256=POE8aPT-MBzfW873wr6C7iiVUHtp483KwvLPxC1S3EY,11626
+agent/cli.py,sha256=WZFyhmTbFnA7QgZmqKO5tGoWsKeO12-GVlBab314o9Q,10002
+agent/computer_handler.py,sha256=2gfFBeDk9Vd54x9mOqnswMo8BdjUduLo5I0RbBPLovY,3964
+agent/decorators.py,sha256=bCmcCjP31WEjWg1D91OE2jo7AZTfGa9cNgCnYUvjiyw,2832
+agent/loops/__init__.py,sha256=_qpP_--3ePdFkTZP8qmUEFlBsy6m4h8fj0gGLDKA7zw,217
+agent/loops/anthropic.py,sha256=w5s_zvkXdcHt0DgBMYjDQGDMBXK4bPu-SyeIMhA1Rrs,32243
+agent/loops/omniparser.py,sha256=m3bDNQ0Igc_HHVoAbjVNj599uRoC9Eap3DCALg6RZ54,11422
+agent/loops/openai.py,sha256=ArTqadeJY8F9N8ZLKfswlzgHV_54HbWJgLd4l6ele9w,3010
+agent/loops/uitars.py,sha256=L0NYxKoIiMfIHbyomnaiK3ZGLmLv3QMx9nX57GruAk0,26323
+agent/responses.py,sha256=ztSMEz8q4ykQAXF21UyQxkSZ6GeoDMydT5HZqKBPAXg,6660
+agent/telemetry.py,sha256=87ZTyBaT0wEPQn4v76II3g0V3GERuIVbypoX-Ug6FKQ,4786
+agent/types.py,sha256=GiLxIcF7s1XIh_WaY7tjdQPFpdTXb5MWVe_ZUPA0gkY,2364
+agent/ui/__init__.py,sha256=DTZpK85QXscXK2nM9HtpAhVBF13yAamUrtwrQSuV-kM,126
+agent/ui/__main__.py,sha256=vudWXYvGM0aNT5aZ94HPtGW8YXOZ4cLXepHyhUM_k1g,73
+agent/ui/gradio/__init__.py,sha256=yv4Mrfo-Sj2U5sVn_UJHAuwYCezo-5O4ItR2C9jzNko,145
+agent/ui/gradio/app.py,sha256=X7he4jzyFqWJDP1y_M8yfZvfdy6GHNuclLn4k9iIwAw,8824
+agent/ui/gradio/ui_components.py,sha256=WxFE-4wvdEgj7FPLNXUrs118sXJ9vN3kLkZxtto-weo,34474
+cua_agent-0.4.1.dist-info/METADATA,sha256=Yf2tVl9529nOxprqpjmvqTtqPOnYWmDtDjuo6UuFddg,12060
+cua_agent-0.4.1.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90
+cua_agent-0.4.1.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
+cua_agent-0.4.1.dist-info/RECORD,,

cua_agent-0.4.0b3.dist-info/RECORD

@@ -1,30 +0,0 @@
-agent/__init__.py,sha256=aRumS6zu5IoVToS-fasvIJ7msjE-gufNsMUY-LPAf0Q,356
-agent/__main__.py,sha256=lBUe8Niqa5XoCjwFfXyX7GtnUwjjZXC1-j4V9mvUYSc,538
-agent/adapters/__init__.py,sha256=szM2HMten2WkcqXeRnan__-sXjpyS4eyvIW0LXSfj4U,178
-agent/adapters/huggingfacelocal_adapter.py,sha256=dnzzxYCvFiuDdNzsb_1uM-boWv1eS__dWMve_fAnlUc,8038
-agent/agent.py,sha256=tXVnqzwC721UUN57OWEatfoZXFAzaDfrZ0G-EuVK3Ug,24022
-agent/callbacks/__init__.py,sha256=SO9NKTrmk4sZ7ZwvWFhTtK9co1FgwubUe3bwkIXYwn0,472
-agent/callbacks/base.py,sha256=UnnnYlh6XCm6HKZZsAPaT_Eyo9LUYLyjyNwF-QRm6Ns,4691
-agent/callbacks/budget_manager.py,sha256=XNrL1z3pxAZkywy6f8kfbRjDWq2ZvKcpnP2ElC8LBJs,1875
-agent/callbacks/image_retention.py,sha256=tiuRT5ke9xXTb2eP8Gz-2ITyAMY29LURUH6AbjX3RP8,6165
-agent/callbacks/logging.py,sha256=OOxU97EzrxlnUAtiEnvy9FB7SwCUK90-rdpDFA2Ae4E,10921
-agent/callbacks/pii_anonymization.py,sha256=UKAqNacHG3z92_6uocVzOIl8gJoqyofldCoCmB4UVIE,10268
-agent/callbacks/trajectory_saver.py,sha256=POE8aPT-MBzfW873wr6C7iiVUHtp483KwvLPxC1S3EY,11626
-agent/cli.py,sha256=QqIgDYjBkXf_0-bxugafxQr-Otp4avoA1SI5ZbtFhBc,9768
-agent/computer_handler.py,sha256=2gfFBeDk9Vd54x9mOqnswMo8BdjUduLo5I0RbBPLovY,3964
-agent/decorators.py,sha256=bCmcCjP31WEjWg1D91OE2jo7AZTfGa9cNgCnYUvjiyw,2832
-agent/loops/__init__.py,sha256=_qpP_--3ePdFkTZP8qmUEFlBsy6m4h8fj0gGLDKA7zw,217
-agent/loops/anthropic.py,sha256=w5s_zvkXdcHt0DgBMYjDQGDMBXK4bPu-SyeIMhA1Rrs,32243
-agent/loops/omniparser.py,sha256=m3bDNQ0Igc_HHVoAbjVNj599uRoC9Eap3DCALg6RZ54,11422
-agent/loops/openai.py,sha256=ArTqadeJY8F9N8ZLKfswlzgHV_54HbWJgLd4l6ele9w,3010
-agent/loops/uitars.py,sha256=L0NYxKoIiMfIHbyomnaiK3ZGLmLv3QMx9nX57GruAk0,26323
-agent/responses.py,sha256=8vnCqi21x3gzo59yDmcsA_E_-CyYxIuCwgfQF0PIomg,6867
-agent/types.py,sha256=GiLxIcF7s1XIh_WaY7tjdQPFpdTXb5MWVe_ZUPA0gkY,2364
-agent/ui/__init__.py,sha256=o7NIpZGFiCTUhgj6eB7gWRRpVhrUAsrOdWnPgbfDgVM,124
-agent/ui/gradio/__init__.py,sha256=j3dc1i14daVIDEo-3K1w_v5EyDsHeB3TdgKkuteDivM,143
-agent/ui/gradio/app.py,sha256=9i4YzLZRkvPrYKnrl6eDx1mR2QaSSsWgG-YQVlhedX8,8788
-agent/ui/gradio/ui_components.py,sha256=IZsYOwn3dwQN8E5-3cFVlLhGlu77I35YhdPY8QxB0Uk,33513
-cua_agent-0.4.0b3.dist-info/METADATA,sha256=muJBA7zaMJGzGOC2t8drwiOCRsvNO1BXw8pUvQFaj6w,12062
-cua_agent-0.4.0b3.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90
-cua_agent-0.4.0b3.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
-cua_agent-0.4.0b3.dist-info/RECORD,,