cua-agent 0.3.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

agent/providers/uitars/prompts.py

@@ -1,63 +0,0 @@
-"""Prompts for UI-TARS agent."""
-
-MAC_SPECIFIC_NOTES = """
-(You are operating on macOS, use 'cmd' instead of 'ctrl' for most shortcuts e.g., hotkey(key='cmd c') for copy, hotkey(key='cmd v') for paste, hotkey(key='cmd t') for new tab).)
-"""
-
-SYSTEM_PROMPT = "You are a helpful assistant."
-
-COMPUTER_USE = """You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
-
-## Output Format
-```
-Thought: ...
-Action: ...
-```
-
-## Action Space
-
-click(start_box='<|box_start|>(x1,y1)<|box_end|>')
-left_double(start_box='<|box_start|>(x1,y1)<|box_end|>')
-right_single(start_box='<|box_start|>(x1,y1)<|box_end|>')
-drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
-hotkey(key='')
-type(content='xxx') # Use escape characters \\', \\\", and \\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\n at the end of content. 
-scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left')
-wait() #Sleep for 5s and take a screenshot to check for any changes.
-finished(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format.
-
-
-## Note
-- Use {language} in `Thought` part.
-- Write a small plan and finally summarize your next action (with its target element) in one sentence in `Thought` part.
-
-## User Instruction
-{instruction}
-"""
-
-MOBILE_USE = """You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 
-## Output Format
-```
-Thought: ...
-Action: ...
-```
-## Action Space
-
-click(start_box='<|box_start|>(x1,y1)<|box_end|>')
-long_press(start_box='<|box_start|>(x1,y1)<|box_end|>')
-type(content='') #If you want to submit your input, use "\\n" at the end of `content`.
-scroll(start_box='<|box_start|>(x1,y1)<|box_end|>', direction='down or up or right or left')
-open_app(app_name=\'\')
-drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
-press_home()
-press_back()
-finished(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format.
-
-
-## Note
-- Use {language} in `Thought` part.
-- Write a small plan and finally summarize your next action (with its target element) in one sentence in `Thought` part.
-
-## User Instruction
-{instruction}
-"""

agent/providers/uitars/tools/__init__.py

@@ -1 +0,0 @@
-"""UI-TARS tools package."""

agent/providers/uitars/tools/computer.py

@@ -1,283 +0,0 @@
-"""Computer tool for UI-TARS."""
-
-import asyncio
-import base64
-import logging
-import re
-from typing import Any, Dict, List, Optional, Literal, Union
-
-from computer import Computer
-from ....core.tools.base import ToolResult, ToolFailure
-from ....core.tools.computer import BaseComputerTool
-
-logger = logging.getLogger(__name__)
-
-
-class ComputerTool(BaseComputerTool):
-    """
-    A tool that allows the UI-TARS agent to interact with the screen, keyboard, and mouse.
-    """
-
-    name: str = "computer"
-    width: Optional[int] = None
-    height: Optional[int] = None
-    computer: Computer
-    
-    def __init__(self, computer: Computer):
-        """Initialize the computer tool.
-
-        Args:
-            computer: Computer instance
-        """
-        super().__init__(computer)
-        self.computer = computer
-        self.width = None
-        self.height = None
-        self.logger = logging.getLogger(__name__)
-        
-    def to_params(self) -> Dict[str, Any]:
-        """Convert tool to API parameters.
-
-        Returns:
-            Dictionary with tool parameters
-        """
-        if self.width is None or self.height is None:
-            raise RuntimeError(
-                "Screen dimensions not initialized. Call initialize_dimensions() first."
-            )
-        return {
-            "type": "computer",
-            "display_width": self.width,
-            "display_height": self.height,
-        }
-
-    async def initialize_dimensions(self) -> None:
-        """Initialize screen dimensions from the computer interface."""
-        try:
-            display_size = await self.computer.interface.get_screen_size()
-            self.width = display_size["width"]
-            self.height = display_size["height"]
-            self.logger.info(f"Initialized screen dimensions to {self.width}x{self.height}")
-        except Exception as e:
-            # Fall back to defaults if we can't get accurate dimensions
-            self.width = 1024
-            self.height = 768
-            self.logger.warning(
-                f"Failed to get screen dimensions, using defaults: {self.width}x{self.height}. Error: {e}"
-            )
-
-    async def __call__(
-        self,
-        *,
-        action: str,
-        **kwargs,
-    ) -> ToolResult:
-        """Execute a computer action.
-        
-        Args:
-            action: The action to perform (based on UI-TARS action space)
-            **kwargs: Additional parameters for the action
-            
-        Returns:
-            ToolResult containing action output and possibly a base64 image
-        """
-        try:
-            # Ensure dimensions are initialized
-            if self.width is None or self.height is None:
-                await self.initialize_dimensions()
-                if self.width is None or self.height is None:
-                    return ToolFailure(error="Failed to initialize screen dimensions")
-            
-            # Handle actions defined in UI-TARS action space (from prompts.py)
-            # Handle standard click (left click)
-            if action == "click":
-                if "x" in kwargs and "y" in kwargs:
-                    x, y = kwargs["x"], kwargs["y"]
-                    await self.computer.interface.left_click(x, y)
-                    
-                    # Wait briefly for UI to update
-                    await asyncio.sleep(0.5)
-                    
-                    # Take screenshot after action
-                    screenshot = await self.computer.interface.screenshot()
-                    base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
-                    
-                    return ToolResult(
-                        output=f"Clicked at ({x}, {y})",
-                        base64_image=base64_screenshot,
-                    )
-                else:
-                    return ToolFailure(error="Missing coordinates for click action")
-            
-            # Handle double click
-            elif action == "left_double":
-                if "x" in kwargs and "y" in kwargs:
-                    x, y = kwargs["x"], kwargs["y"]
-                    await self.computer.interface.double_click(x, y)
-                    
-                    # Wait briefly for UI to update
-                    await asyncio.sleep(0.5)
-                    
-                    # Take screenshot after action
-                    screenshot = await self.computer.interface.screenshot()
-                    base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
-                    
-                    return ToolResult(
-                        output=f"Double-clicked at ({x}, {y})",
-                        base64_image=base64_screenshot,
-                    )
-                else:
-                    return ToolFailure(error="Missing coordinates for left_double action")
-            
-            # Handle right click
-            elif action == "right_single":
-                if "x" in kwargs and "y" in kwargs:
-                    x, y = kwargs["x"], kwargs["y"]
-                    await self.computer.interface.right_click(x, y)
-                    
-                    # Wait briefly for UI to update
-                    await asyncio.sleep(0.5)
-                    
-                    # Take screenshot after action
-                    screenshot = await self.computer.interface.screenshot()
-                    base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
-                    
-                    return ToolResult(
-                        output=f"Right-clicked at ({x}, {y})",
-                        base64_image=base64_screenshot,
-                    )
-                else:
-                    return ToolFailure(error="Missing coordinates for right_single action")
-            
-            # Handle typing text
-            elif action == "type_text":
-                if "text" in kwargs:
-                    text = kwargs["text"]
-                    await self.computer.interface.type_text(text)
-                    
-                    # Wait for UI to update
-                    await asyncio.sleep(0.3)
-                    
-                    # Take screenshot after action
-                    screenshot = await self.computer.interface.screenshot()
-                    base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
-                    
-                    return ToolResult(
-                        output=f"Typed: {text}",
-                        base64_image=base64_screenshot,
-                    )
-                else:
-                    return ToolFailure(error="Missing text for type action")
-            
-            # Handle hotkey
-            elif action == "hotkey":
-                if "keys" in kwargs:
-                    keys = kwargs["keys"]
-                    
-                    if len(keys) > 1:
-                        await self.computer.interface.hotkey(*keys)
-                    else:
-                        # Single key press
-                        await self.computer.interface.press_key(keys[0])
-                        
-                    # Wait for UI to update
-                    await asyncio.sleep(0.3)
-                    
-                    # Take screenshot after action
-                    screenshot = await self.computer.interface.screenshot()
-                    base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
-                    
-                    return ToolResult(
-                        output=f"Pressed hotkey: {', '.join(keys)}",
-                        base64_image=base64_screenshot,
-                    )
-                else:
-                    return ToolFailure(error="Missing keys for hotkey action")
-            
-            # Handle drag action
-            elif action == "drag":
-                if all(k in kwargs for k in ["start_x", "start_y", "end_x", "end_y"]):
-                    start_x, start_y = kwargs["start_x"], kwargs["start_y"]
-                    end_x, end_y = kwargs["end_x"], kwargs["end_y"]
-                    
-                    # Perform drag
-                    await self.computer.interface.move_cursor(start_x, start_y)
-                    await self.computer.interface.drag_to(end_x, end_y)
-                    
-                    # Wait for UI to update
-                    await asyncio.sleep(0.5)
-                    
-                    # Take screenshot after action
-                    screenshot = await self.computer.interface.screenshot()
-                    base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
-                    
-                    return ToolResult(
-                        output=f"Dragged from ({start_x}, {start_y}) to ({end_x}, {end_y})",
-                        base64_image=base64_screenshot,
-                    )
-                else:
-                    return ToolFailure(error="Missing coordinates for drag action")
-            
-            # Handle scroll action
-            elif action == "scroll":
-                if all(k in kwargs for k in ["x", "y", "direction"]):
-                    x, y = kwargs["x"], kwargs["y"]
-                    direction = kwargs["direction"]
-                    
-                    # Move cursor to position
-                    await self.computer.interface.move_cursor(x, y)
-                    
-                    # Scroll based on direction
-                    if direction == "down":
-                        await self.computer.interface.scroll_down(5)
-                    elif direction == "up":
-                        await self.computer.interface.scroll_up(5)
-                    elif direction == "right":
-                        pass # await self.computer.interface.scroll_right(5)
-                    elif direction == "left":
-                        pass # await self.computer.interface.scroll_left(5)
-                    else:
-                        return ToolFailure(error=f"Invalid scroll direction: {direction}")
-                    
-                    # Wait for UI to update
-                    await asyncio.sleep(0.5)
-                    
-                    # Take screenshot after action
-                    screenshot = await self.computer.interface.screenshot()
-                    base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
-                    
-                    return ToolResult(
-                        output=f"Scrolled {direction} at ({x}, {y})",
-                        base64_image=base64_screenshot,
-                    )
-                else:
-                    return ToolFailure(error="Missing parameters for scroll action")
-            
-            # Handle wait action
-            elif action == "wait":
-                # Sleep for 5 seconds as specified in the action space
-                await asyncio.sleep(5)
-                
-                # Take screenshot after waiting
-                screenshot = await self.computer.interface.screenshot()
-                base64_screenshot = base64.b64encode(screenshot).decode("utf-8")
-                
-                return ToolResult(
-                    output="Waited for 5 seconds",
-                    base64_image=base64_screenshot,
-                )
-            
-            # Handle finished action (task completion)
-            elif action == "finished":
-                content = kwargs.get("content", "Task completed")
-                return ToolResult(
-                    output=f"Task finished: {content}",
-                )
-            
-                return await self._handle_scroll(action)
-            else:
-                return ToolFailure(error=f"Unsupported action: {action}")
-                
-        except Exception as e:
-            self.logger.error(f"Error in ComputerTool.__call__: {str(e)}")
-            return ToolFailure(error=f"Failed to execute {action}: {str(e)}")

agent/providers/uitars/tools/manager.py

@@ -1,60 +0,0 @@
-"""Tool manager for the UI-TARS provider."""
-
-import logging
-from typing import Any, Dict, List, Optional
-
-from computer import Computer
-from ....core.tools import BaseToolManager
-from ....core.tools.collection import ToolCollection
-from .computer import ComputerTool
-
-logger = logging.getLogger(__name__)
-
-
-class ToolManager(BaseToolManager):
-    """Manages UI-TARS provider tool initialization and execution."""
-
-    def __init__(self, computer: Computer):
-        """Initialize the tool manager.
-
-        Args:
-            computer: Computer instance for computer-related tools
-        """
-        super().__init__(computer)
-        # Initialize UI-TARS-specific tools
-        self.computer_tool = ComputerTool(self.computer)
-        self._initialized = False
-
-    def _initialize_tools(self) -> ToolCollection:
-        """Initialize all available tools."""
-        return ToolCollection(self.computer_tool)
-
-    async def _initialize_tools_specific(self) -> None:
-        """Initialize UI-TARS provider-specific tool requirements."""
-        await self.computer_tool.initialize_dimensions()
-
-    def get_tool_params(self) -> List[Dict[str, Any]]:
-        """Get tool parameters for API calls.
-
-        Returns:
-            List of tool parameters for the current provider's API
-        """
-        if self.tools is None:
-            raise RuntimeError("Tools not initialized. Call initialize() first.")
-
-        return self.tools.to_params()
-
-    async def execute_tool(self, name: str, tool_input: dict[str, Any]) -> Any:
-        """Execute a tool with the given input.
-
-        Args:
-            name: Name of the tool to execute
-            tool_input: Input parameters for the tool
-
-        Returns:
-            Result of the tool execution
-        """
-        if self.tools is None:
-            raise RuntimeError("Tools not initialized. Call initialize() first.")
-
-        return await self.tools.run(name=name, tool_input=tool_input)

agent/providers/uitars/utils.py

@@ -1,264 +0,0 @@
-"""Utility functions for the UI-TARS provider."""
-
-import logging
-import base64
-import re
-from typing import Any, Dict, List, Optional, Union, Tuple
-from datetime import datetime
-
-logger = logging.getLogger(__name__)
-
-from ...core.types import AgentResponse
-
-async def to_agent_response_format(
-    response: Dict[str, Any],
-    messages: List[Dict[str, Any]],
-    model: Optional[str] = None,
-) -> AgentResponse:
-    """Convert raw UI-TARS response to agent response format.
-    
-    Args:
-        response: Raw UI-TARS response
-        messages: List of messages in standard format
-        model: Optional model name
-    
-    Returns:
-        AgentResponse: Standardized agent response format
-    """
-    # Create unique IDs for this response
-    response_id = f"resp_{datetime.now().strftime('%Y%m%d%H%M%S')}_{id(response)}"
-    reasoning_id = f"rs_{response_id}"
-    action_id = f"cu_{response_id}"
-    call_id = f"call_{response_id}"
-
-    # Parse actions from the raw response
-    content = response["choices"][0]["message"]["content"]
-    actions = parse_actions(content)
-    
-    # Extract thought content if available
-    reasoning_text = ""
-    if "Thought:" in content:
-        thought_match = re.search(r"Thought: (.*?)(?=\s*Action:|$)", content, re.DOTALL)
-        if thought_match:
-            reasoning_text = thought_match.group(1).strip()
-    
-    # Create output items
-    output_items = []
-    if reasoning_text:
-        output_items.append({
-            "type": "reasoning",
-            "id": reasoning_id,
-            "text": reasoning_text
-        })
-    if actions:
-        for i, action in enumerate(actions):
-            action_name, tool_args = parse_action_parameters(action)
-            if action_name == "finished":
-                output_items.append({
-                    "type": "message",
-                    "role": "assistant",
-                    "content": [{
-                        "type": "output_text",
-                        "text": tool_args["content"]
-                    }],
-                    "id": f"action_{i}_{action_id}",
-                    "status": "completed"
-                })
-            else:
-                if tool_args.get("action") == action_name:
-                    del tool_args["action"]
-                output_items.append({
-                    "type": "computer_call",
-                    "id": f"{action}_{i}_{action_id}",
-                    "call_id": f"call_{i}_{action_id}",
-                    "action": { "type": action_name, **tool_args },
-                    "pending_safety_checks": [],
-                    "status": "completed"
-                })
-    
-    # Create agent response
-    agent_response = AgentResponse(
-        id=response_id,
-        object="response",
-        created_at=int(datetime.now().timestamp()),
-        status="completed",
-        error=None,
-        incomplete_details=None,
-        instructions=None,
-        max_output_tokens=None,
-        model=model or response["model"],
-        output=output_items,
-        parallel_tool_calls=True,
-        previous_response_id=None,
-        reasoning={"effort": "medium"},
-        store=True,
-        temperature=0.0,
-        top_p=0.7,
-        text={"format": {"type": "text"}},
-        tool_choice="auto",
-        tools=[
-            {
-                "type": "computer_use_preview",
-                "display_height": 768,
-                "display_width": 1024,
-                "environment": "mac",
-            }
-        ],
-        truncation="auto",
-        usage=response.get("usage", {}),
-        user=None,
-        metadata={},
-        response=response
-    )
-    return agent_response
-
-
-def add_box_token(input_string: str) -> str:
-    """Add box tokens to the coordinates in the model response.
-    
-    Args:
-        input_string: Raw model response
-        
-    Returns:
-        String with box tokens added
-    """
-    if "Action: " not in input_string or "start_box=" not in input_string:
-        return input_string
-        
-    suffix = input_string.split("Action: ")[0] + "Action: "
-    actions = input_string.split("Action: ")[1:]
-    processed_actions = []
-    
-    for action in actions:
-        action = action.strip()
-        coordinates = re.findall(r"(start_box|end_box)='\((\d+),\s*(\d+)\)'", action)
-        
-        updated_action = action
-        for coord_type, x, y in coordinates:
-            updated_action = updated_action.replace(
-                f"{coord_type}='({x},{y})'", 
-                f"{coord_type}='<|box_start|>({x},{y})<|box_end|>'"
-            )
-        processed_actions.append(updated_action)
-    
-    return suffix + "\n\n".join(processed_actions)
-
-
-def parse_actions(response: str) -> List[str]:
-    """Parse actions from UI-TARS model response.
-
-    Args:
-        response: The raw model response text
-        
-    Returns:
-        List of parsed actions
-    """
-    actions = []
-    # Extract the Action part from the response
-    if "Action:" in response:
-        action_text = response.split("Action:")[-1].strip()
-        # Clean up and format action
-        if action_text:
-            # Handle multiple actions separated by newlines
-            action_parts = action_text.split("\n\n")
-            for part in action_parts:
-                if part.strip():
-                    actions.append(part.strip())
-    
-    return actions
-
-
-def parse_action_parameters(action: str) -> Tuple[str, Dict[str, Any]]:
-    """Parse parameters from an action string.
-    
-    Args:
-        action: The action string to parse
-        
-    Returns:
-        Tuple of (action_name, action_parameters)
-    """
-    # Handle "finished" action
-    if action.startswith("finished"):
-        # Parse content if it exists
-        content_match = re.search(r"content='([^']*)'", action)
-        if content_match:
-            content = content_match.group(1)
-            return "finished", {"content": content}
-        else:
-            return "finished", {}
-    
-    # Parse action parameters
-    action_match = re.match(r'(\w+)\((.*)\)', action)
-    if not action_match:
-        logger.warning(f"Could not parse action: {action}")
-        return "", {}
-        
-    action_name = action_match.group(1)
-    action_params_str = action_match.group(2)
-    
-    tool_args = {"action": action_name}
-    
-    # Extract coordinate values from the action
-    if "start_box" in action_params_str:
-        # Extract all box coordinates
-        box_pattern = r"(start_box|end_box)='(?:<\|box_start\|>)?\((\d+),\s*(\d+)\)(?:<\|box_end\|>)?'"
-        box_matches = re.findall(box_pattern, action_params_str)
-        
-        # Handle click-type actions
-        if action_name in ["click", "left_double", "right_single"]:
-            # Get coordinates from start_box
-            for box_type, x, y in box_matches:
-                if box_type == "start_box":
-                    tool_args["x"] = int(x)
-                    tool_args["y"] = int(y)
-                    break
-        
-        # Handle drag action
-        elif action_name == "drag":
-            start_x, start_y = None, None
-            end_x, end_y = None, None
-            
-            for box_type, x, y in box_matches:
-                if box_type == "start_box":
-                    start_x, start_y = int(x), int(y)
-                elif box_type == "end_box":
-                    end_x, end_y = int(x), int(y)
-            
-            if not None in [start_x, start_y, end_x, end_y]:
-                tool_args["start_x"] = start_x
-                tool_args["start_y"] = start_y
-                tool_args["end_x"] = end_x
-                tool_args["end_y"] = end_y
-            
-        # Handle scroll action
-        elif action_name == "scroll":
-            # Get coordinates from start_box
-            for box_type, x, y in box_matches:
-                if box_type == "start_box":
-                    tool_args["x"] = int(x)
-                    tool_args["y"] = int(y)
-                    break
-            
-            # Extract direction
-            direction_match = re.search(r"direction='([^']+)'", action_params_str)
-            if direction_match:
-                tool_args["direction"] = direction_match.group(1)
-    
-    # Handle typing text
-    elif action_name == "type":
-        # Extract text content
-        content_match = re.search(r"content='([^']*)'", action_params_str)
-        if content_match:
-            # Unescape escaped characters
-            text = content_match.group(1).replace("\\'", "'").replace('\\"', '"').replace("\\n", "\n")
-            tool_args = {"action": "type_text", "text": text}
-    
-    # Handle hotkey
-    elif action_name == "hotkey":
-        # Extract key combination
-        key_match = re.search(r"key='([^']*)'", action_params_str)
-        if key_match:
-            keys = key_match.group(1).split()
-            tool_args = {"action": "hotkey", "keys": keys}
-    
-    return action_name, tool_args