cua-agent 0.3.2__py3-none-any.whl → 0.4.0b1__py3-none-any.whl

agent/providers/omni/parser.py

@@ -1,307 +0,0 @@
-"""Parser implementation for the Omni provider."""
-
-import logging
-from typing import Any, Dict, List, Optional, Tuple
-import base64
-import torch
-
-# Import from the SOM package
-from som import OmniParser as OmniDetectParser
-from som.models import ParseResult, ParserMetadata
-
-logger = logging.getLogger(__name__)
-
-
-class OmniParser:
-    """Parser for handling responses from multiple providers."""
-
-    # Class-level shared OmniDetectParser instance
-    _shared_parser = None
-
-    def __init__(self, force_device: Optional[str] = None):
-        """Initialize the OmniParser.
-
-        Args:
-            force_device: Optional device to force for detection (cpu/cuda/mps)
-        """
-        self.response_buffer = []
-
-        # Use shared parser if available, otherwise create a new one
-        if OmniParser._shared_parser is None:
-            logger.info("Initializing shared OmniDetectParser...")
-
-            # Determine the best device to use
-            device = force_device
-            if not device:
-                if torch.cuda.is_available():
-                    device = "cuda"
-                elif (
-                    hasattr(torch, "backends")
-                    and hasattr(torch.backends, "mps")
-                    and torch.backends.mps.is_available()
-                ):
-                    device = "mps"
-                else:
-                    device = "cpu"
-
-            logger.info(f"Using device: {device} for OmniDetectParser")
-            self.detect_parser = OmniDetectParser(force_device=device)
-
-            # Preload the detection model to avoid repeated loading
-            try:
-                # Access the detector to trigger model loading
-                detector = self.detect_parser.detector
-                if detector.model is None:
-                    logger.info("Preloading detection model...")
-                    detector.load_model()
-                    logger.info("Detection model preloaded successfully")
-            except Exception as e:
-                logger.error(f"Error preloading detection model: {str(e)}")
-
-            # Store as shared instance
-            OmniParser._shared_parser = self.detect_parser
-        else:
-            logger.info("Using existing shared OmniDetectParser")
-            self.detect_parser = OmniParser._shared_parser
-
-    async def parse_screen(self, computer: Any) -> ParseResult:
-        """Parse a screenshot and extract screen information.
-
-        Args:
-            computer: Computer instance
-
-        Returns:
-            ParseResult with screen elements and image data
-        """
-        try:
-            # Get screenshot from computer
-            logger.info("Taking screenshot...")
-            screenshot = await computer.interface.screenshot()
-
-            # Log screenshot info
-            logger.info(f"Screenshot type: {type(screenshot)}")
-            logger.info(f"Screenshot is bytes: {isinstance(screenshot, bytes)}")
-            logger.info(f"Screenshot is str: {isinstance(screenshot, str)}")
-            logger.info(f"Screenshot length: {len(screenshot) if screenshot else 0}")
-
-            # If screenshot is a string (likely base64), convert it to bytes
-            if isinstance(screenshot, str):
-                try:
-                    screenshot = base64.b64decode(screenshot)
-                    logger.info("Successfully converted base64 string to bytes")
-                    logger.info(f"Decoded bytes length: {len(screenshot)}")
-                except Exception as e:
-                    logger.error(f"Error decoding base64: {str(e)}")
-                    logger.error(f"First 100 chars of screenshot string: {screenshot[:100]}")
-
-            # Pass screenshot to OmniDetectParser
-            logger.info("Passing screenshot to OmniDetectParser...")
-            parse_result = self.detect_parser.parse(
-                screenshot_data=screenshot, box_threshold=0.3, iou_threshold=0.1, use_ocr=True
-            )
-            logger.info("Screenshot parsed successfully")
-            logger.info(f"Parse result has {len(parse_result.elements)} elements")
-
-            # Log element IDs for debugging
-            for i, elem in enumerate(parse_result.elements):
-                logger.info(
-                    f"Element {i+1} (ID: {elem.id}): {elem.type} with confidence {elem.confidence:.3f}"
-                )
-
-            return parse_result
-
-        except Exception as e:
-            logger.error(f"Error parsing screen: {str(e)}")
-            import traceback
-
-            logger.error(traceback.format_exc())
-
-            # Create a minimal valid result for error cases
-            return ParseResult(
-                elements=[],
-                screen_info=None,
-                annotated_image_base64="",
-                parsed_content_list=[{"error": str(e)}],
-                metadata=ParserMetadata(
-                    image_size=(0, 0),
-                    num_icons=0,
-                    num_text=0,
-                    device="cpu",
-                    ocr_enabled=False,
-                    latency=0.0,
-                ),
-            )
-
-    def parse_tool_call(self, response: Dict[str, Any]) -> Optional[Dict[str, Any]]:
-        """Parse a tool call from the response.
-
-        Args:
-            response: Response from the provider
-
-        Returns:
-            Parsed tool call or None if no tool call found
-        """
-        try:
-            # Handle Anthropic format
-            if "tool_calls" in response:
-                tool_call = response["tool_calls"][0]
-                return {
-                    "name": tool_call["function"]["name"],
-                    "arguments": tool_call["function"]["arguments"],
-                }
-
-            # Handle OpenAI format
-            if "function_call" in response:
-                return {
-                    "name": response["function_call"]["name"],
-                    "arguments": response["function_call"]["arguments"],
-                }
-
-            # Handle Groq format (OpenAI-compatible)
-            if "choices" in response and response["choices"]:
-                choice = response["choices"][0]
-                if "function_call" in choice:
-                    return {
-                        "name": choice["function_call"]["name"],
-                        "arguments": choice["function_call"]["arguments"],
-                    }
-
-            return None
-
-        except Exception as e:
-            logger.error(f"Error parsing tool call: {str(e)}")
-            return None
-
-    def parse_response(self, response: Dict[str, Any]) -> Tuple[str, Dict[str, Any]]:
-        """Parse a response from any provider.
-
-        Args:
-            response: Response from the provider
-
-        Returns:
-            Tuple of (content, metadata)
-        """
-        try:
-            content = ""
-            metadata = {}
-
-            # Handle Anthropic format
-            if "content" in response and isinstance(response["content"], list):
-                for item in response["content"]:
-                    if item["type"] == "text":
-                        content += item["text"]
-
-            # Handle OpenAI format
-            elif "choices" in response and response["choices"]:
-                content = response["choices"][0]["message"]["content"]
-
-            # Handle direct content
-            elif isinstance(response.get("content"), str):
-                content = response["content"]
-
-            # Extract metadata if present
-            if "metadata" in response:
-                metadata = response["metadata"]
-
-            return content, metadata
-
-        except Exception as e:
-            logger.error(f"Error parsing response: {str(e)}")
-            return str(e), {"error": True}
-
-    def format_for_provider(
-        self, messages: List[Dict[str, Any]], provider: str
-    ) -> List[Dict[str, Any]]:
-        """Format messages for a specific provider.
-
-        Args:
-            messages: List of messages to format
-            provider: Provider to format for
-
-        Returns:
-            Formatted messages
-        """
-        try:
-            formatted = []
-
-            for msg in messages:
-                formatted_msg = {"role": msg["role"]}
-
-                # Handle content formatting
-                if isinstance(msg["content"], list):
-                    # For providers that support multimodal
-                    if provider in ["anthropic", "openai"]:
-                        formatted_msg["content"] = msg["content"]
-                    else:
-                        # Extract text only for other providers
-                        text_content = next(
-                            (item["text"] for item in msg["content"] if item["type"] == "text"), ""
-                        )
-                        formatted_msg["content"] = text_content
-                else:
-                    formatted_msg["content"] = msg["content"]
-
-                formatted.append(formatted_msg)
-
-            return formatted
-
-        except Exception as e:
-            logger.error(f"Error formatting messages: {str(e)}")
-            return messages  # Return original messages on error
-
-    async def calculate_click_coordinates(
-        self, box_id: int, parsed_screen: ParseResult
-    ) -> Tuple[int, int]:
-        """Calculate click coordinates based on box ID.
-
-        Args:
-            box_id: The ID of the box to click
-            parsed_screen: The parsed screen information
-
-        Returns:
-            Tuple of (x, y) coordinates
-
-        Raises:
-            ValueError: If box_id is invalid or missing from parsed screen
-        """
-        # First try to use structured elements data
-        logger.info(f"Elements count: {len(parsed_screen.elements)}")
-
-        # Try to find element with matching ID
-        for element in parsed_screen.elements:
-            if element.id == box_id:
-                logger.info(f"Found element with ID {box_id}: {element}")
-                bbox = element.bbox
-
-                # Get screen dimensions from the metadata if available, or fallback
-                width = parsed_screen.metadata.width if parsed_screen.metadata else 1920
-                height = parsed_screen.metadata.height if parsed_screen.metadata else 1080
-                logger.info(f"Screen dimensions: width={width}, height={height}")
-
-                # Create a dictionary from the element's bbox for calculate_element_center
-                bbox_dict = {"x1": bbox.x1, "y1": bbox.y1, "x2": bbox.x2, "y2": bbox.y2}
-                from ...core.visualization import calculate_element_center
-
-                center_x, center_y = calculate_element_center(bbox_dict, width, height)
-                logger.info(f"Calculated center: ({center_x}, {center_y})")
-
-                # Validate coordinates - if they're (0,0) or unreasonably small,
-                # use a default position in the center of the screen
-                if center_x == 0 and center_y == 0:
-                    logger.warning("Got (0,0) coordinates, using fallback position")
-                    center_x = width // 2
-                    center_y = height // 2
-                    logger.info(f"Using fallback center: ({center_x}, {center_y})")
-
-                return center_x, center_y
-
-        # If we couldn't find the box, use center of screen
-        logger.error(
-            f"Box ID {box_id} not found in structured elements (count={len(parsed_screen.elements)})"
-        )
-
-        # Use center of screen as fallback
-        width = parsed_screen.metadata.width if parsed_screen.metadata else 1920
-        height = parsed_screen.metadata.height if parsed_screen.metadata else 1080
-        logger.warning(f"Using fallback position in center of screen ({width//2}, {height//2})")
-        return width // 2, height // 2

agent/providers/omni/prompts.py

@@ -1,64 +0,0 @@
-"""Prompts for the Omni agent."""
-
-SYSTEM_PROMPT = """
-You are using a macOS device.
-You are able to use a mouse and keyboard to interact with the computer based on the given task and screenshot.
-
-You may be given some history plan and actions, this is the response from the previous loop.
-You should carefully consider your plan base on the task, screenshot, and history actions.
-
-Your available "Next Action" only include:
-- type_text: types a string of text.
-- left_click: move mouse to box id and left clicks.
-- right_click: move mouse to box id and right clicks.
-- double_click: move mouse to box id and double clicks.
-- move_cursor: move mouse to box id.
-- scroll_up: scrolls the screen up to view previous content.
-- scroll_down: scrolls the screen down, when the desired button is not visible, or you need to see more content. 
-- hotkey: press a sequence of keys.
-- wait: waits for 1 second for the device to load or respond.
-
-Based on the visual information from the screenshot image and the detected bounding boxes, please determine the next action, the Box ID you should operate on (if action is one of 'type', 'hover', 'scroll_up', 'scroll_down', 'wait', there should be no Box ID field), and the value (if the action is 'type') in order to complete the task.
-
-Output format:
-{
-    "Explanation": str, # describe what is in the current screen, taking into account the history, then describe your step-by-step thoughts on how to achieve the task, choose one action from available actions at a time.
-    "Action": "action_type, action description" | "None" # one action at a time, describe it in short and precisely. 
-    "Box ID": n,
-    "Value": "xxx" # only provide value field if the action is type, else don't include value key
-}
-
-One Example:
-{  
-    "Explanation": "The current screen shows google result of amazon, in previous action I have searched amazon on google. Then I need to click on the first search results to go to amazon.com.",
-    "Action": "left_click",
-    "Box ID": 4
-}
-
-Another Example:
-{
-    "Explanation": "The current screen shows the front page of amazon. There is no previous action. Therefore I need to type "Apple watch" in the search bar.",
-    "Action": "type_text",
-    "Box ID": 2,
-    "Value": "Apple watch"
-}
-
-Another Example:
-{
-    "Explanation": "I am starting a Spotlight search to find the Safari browser.",
-    "Action": "hotkey",
-    "Value": "command+space"
-}
-
-IMPORTANT NOTES:
-1. You should only give a single action at a time.
-2. The Box ID is the id of the element you should operate on, it is a number. Its background color corresponds to the color of the bounding box of the element.
-3. You should give an analysis to the current screen, and reflect on what has been done by looking at the history, then describe your step-by-step thoughts on how to achieve the task.
-4. Attach the next action prediction in the "Action" field.
-5. For starting applications, always use the "hotkey" action with command+space for starting a Spotlight search.
-6. When the task is completed, don't complete additional actions. You should say "Action": "None" in the json field.
-7. The tasks involve buying multiple products or navigating through multiple pages. You should break it into subgoals and complete each subgoal one by one in the order of the instructions.
-8. Avoid choosing the same action/elements multiple times in a row, if it happens, reflect to yourself, what may have gone wrong, and predict a different action.
-9. Reflect whether the element is clickable or not, for example reflect if it is an hyperlink or a button or a normal text.
-10. If you are prompted with login information page or captcha page, or you think it need user's permission to do the next action, you should say "Action": "None" in the json field.
-"""

agent/providers/omni/tools/__init__.py

@@ -1,30 +0,0 @@
-"""Omni provider tools - compatible with multiple LLM providers."""
-
-from ....core.tools import BaseTool, ToolResult, ToolError, ToolFailure, CLIResult
-from .base import BaseOmniTool
-from .computer import ComputerTool
-from .bash import BashTool
-from .manager import ToolManager
-
-# Re-export the tools with Omni-specific names for backward compatibility
-OmniToolResult = ToolResult
-OmniToolError = ToolError
-OmniToolFailure = ToolFailure
-OmniCLIResult = CLIResult
-
-# We'll export specific tools once implemented
-__all__ = [
-    "BaseTool",
-    "BaseOmniTool",
-    "ToolResult",
-    "ToolError",
-    "ToolFailure",
-    "CLIResult",
-    "OmniToolResult",
-    "OmniToolError",
-    "OmniToolFailure",
-    "OmniCLIResult",
-    "ComputerTool",
-    "BashTool",
-    "ToolManager",
-]

agent/providers/omni/tools/base.py

@@ -1,29 +0,0 @@
-"""Omni-specific tool base classes."""
-
-from abc import ABCMeta, abstractmethod
-from typing import Any, Dict
-
-from ....core.tools.base import BaseTool
-
-
-class BaseOmniTool(BaseTool, metaclass=ABCMeta):
-    """Abstract base class for Omni provider tools."""
-
-    def __init__(self):
-        """Initialize the base Omni tool."""
-        # No specific initialization needed yet, but included for future extensibility
-        pass
-
-    @abstractmethod
-    async def __call__(self, **kwargs) -> Any:
-        """Executes the tool with the given arguments."""
-        ...
-
-    @abstractmethod
-    def to_params(self) -> Dict[str, Any]:
-        """Convert tool to Omni provider-specific API parameters.
-
-        Returns:
-            Dictionary with tool parameters for the specific API
-        """
-        raise NotImplementedError

agent/providers/omni/tools/bash.py

@@ -1,74 +0,0 @@
-"""Bash tool for Omni provider."""
-
-import logging
-from typing import Any, Dict
-
-from computer import Computer
-from ....core.tools import ToolResult, ToolError
-from .base import BaseOmniTool
-
-logger = logging.getLogger(__name__)
-
-
-class BashTool(BaseOmniTool):
-    """Tool for executing bash commands."""
-
-    name = "bash"
-    description = "Execute bash commands on the system"
-
-    def __init__(self, computer: Computer):
-        """Initialize the bash tool.
-
-        Args:
-            computer: Computer instance
-        """
-        super().__init__()
-        self.computer = computer
-
-    def to_params(self) -> Dict[str, Any]:
-        """Convert tool to API parameters.
-
-        Returns:
-            Dictionary with tool parameters
-        """
-        return {
-            "type": "function",
-            "function": {
-                "name": self.name,
-                "description": self.description,
-                "parameters": {
-                    "type": "object",
-                    "properties": {
-                        "command": {
-                            "type": "string",
-                            "description": "The bash command to execute",
-                        },
-                    },
-                    "required": ["command"],
-                },
-            },
-        }
-
-    async def __call__(self, **kwargs) -> ToolResult:
-        """Execute bash command.
-
-        Args:
-            **kwargs: Command parameters
-
-        Returns:
-            Tool execution result
-        """
-        try:
-            command = kwargs.get("command", "")
-            if not command:
-                return ToolResult(error="No command specified")
-
-            # The true implementation would use the actual method to run terminal commands
-            # Since we're getting linter errors, we'll just implement a placeholder that will
-            # be replaced with the correct implementation when this tool is fully integrated
-            logger.info(f"Would execute command: {command}")
-            return ToolResult(output=f"Command executed (placeholder): {command}")
-
-        except Exception as e:
-            logger.error(f"Error in bash tool: {str(e)}")
-            return ToolResult(error=f"Error: {str(e)}")

agent/providers/omni/tools/computer.py

@@ -1,179 +0,0 @@
-"""Computer tool for Omni provider."""
-
-import logging
-from typing import Any, Dict
-import json
-
-from computer import Computer
-from ....core.tools import ToolResult, ToolError
-from .base import BaseOmniTool
-from ..parser import ParseResult
-
-logger = logging.getLogger(__name__)
-
-
-class ComputerTool(BaseOmniTool):
-    """Tool for interacting with the computer UI."""
-
-    name = "computer"
-    description = "Interact with the computer's graphical user interface"
-
-    def __init__(self, computer: Computer):
-        """Initialize the computer tool.
-
-        Args:
-            computer: Computer instance
-        """
-        super().__init__()
-        self.computer = computer
-        # Default to standard screen dimensions (will be set more accurately during initialization)
-        self.screen_dimensions = {"width": 1440, "height": 900}
-
-    async def initialize_dimensions(self) -> None:
-        """Initialize screen dimensions."""
-        # For now, we'll use default values
-        # In the future, we can implement proper screen dimension detection
-        logger.info(f"Using default screen dimensions: {self.screen_dimensions}")
-
-    def to_params(self) -> Dict[str, Any]:
-        """Convert tool to API parameters.
-
-        Returns:
-            Dictionary with tool parameters
-        """
-        return {
-            "type": "function",
-            "function": {
-                "name": self.name,
-                "description": self.description,
-                "parameters": {
-                    "type": "object",
-                    "properties": {
-                        "action": {
-                            "type": "string",
-                            "enum": [
-                                "left_click",
-                                "right_click",
-                                "double_click",
-                                "move_cursor",
-                                "drag_to",
-                                "type_text",
-                                "press_key",
-                                "hotkey",
-                                "scroll_up",
-                                "scroll_down",
-                            ],
-                            "description": "The action to perform",
-                        },
-                        "x": {
-                            "type": "number",
-                            "description": "X coordinate for click or cursor movement",
-                        },
-                        "y": {
-                            "type": "number",
-                            "description": "Y coordinate for click or cursor movement",
-                        },
-                        "box_id": {
-                            "type": "integer",
-                            "description": "ID of the UI element to interact with",
-                        },
-                        "text": {
-                            "type": "string",
-                            "description": "Text to type",
-                        },
-                        "key": {
-                            "type": "string",
-                            "description": "Key to press",
-                        },
-                        "keys": {
-                            "type": "array",
-                            "items": {"type": "string"},
-                            "description": "Keys to press as hotkey combination",
-                        },
-                        "amount": {
-                            "type": "integer",
-                            "description": "Amount to scroll",
-                        },
-                        "duration": {
-                            "type": "number",
-                            "description": "Duration for drag operations",
-                        },
-                    },
-                    "required": ["action"],
-                },
-            },
-        }
-
-    async def __call__(self, **kwargs) -> ToolResult:
-        """Execute computer action.
-
-        Args:
-            **kwargs: Action parameters
-
-        Returns:
-            Tool execution result
-        """
-        try:
-            action = kwargs.get("action", "").lower()
-            if not action:
-                return ToolResult(error="No action specified")
-
-            # Execute the action on the computer
-            method = getattr(self.computer.interface, action, None)
-            if not method:
-                return ToolResult(error=f"Unsupported action: {action}")
-
-            # Prepare arguments based on action type
-            args = {}
-            if action in ["left_click", "right_click", "double_click", "move_cursor"]:
-                x = kwargs.get("x")
-                y = kwargs.get("y")
-                if x is None or y is None:
-                    box_id = kwargs.get("box_id")
-                    if box_id is None:
-                        return ToolResult(error="Box ID or coordinates required")
-                    # Get coordinates from box_id implementation would be here
-                    # For now, return error
-                    return ToolResult(error="Box ID-based clicking not implemented yet")
-                args["x"] = x
-                args["y"] = y
-            elif action == "drag_to":
-                x = kwargs.get("x")
-                y = kwargs.get("y")
-                if x is None or y is None:
-                    return ToolResult(error="Coordinates required for drag_to")
-                args.update(
-                    {
-                        "x": x,
-                        "y": y,
-                        "button": kwargs.get("button", "left"),
-                        "duration": float(kwargs.get("duration", 0.5)),
-                    }
-                )
-            elif action == "type_text":
-                text = kwargs.get("text")
-                if not text:
-                    return ToolResult(error="Text required for type_text")
-                args["text"] = text
-            elif action == "press_key":
-                key = kwargs.get("key")
-                if not key:
-                    return ToolResult(error="Key required for press_key")
-                args["key"] = key
-            elif action == "hotkey":
-                keys = kwargs.get("keys")
-                if not keys:
-                    return ToolResult(error="Keys required for hotkey")
-                # Call with positional arguments instead of kwargs
-                await method(*keys)
-                return ToolResult(output=f"Hotkey executed: {'+'.join(keys)}")
-            elif action in ["scroll_down", "scroll_up"]:
-                args["clicks"] = int(kwargs.get("amount", 1))
-
-            # Execute action with prepared arguments
-            await method(**args)
-            return ToolResult(output=f"Action {action} executed successfully")
-
-        except Exception as e:
-            logger.error(f"Error executing computer action: {str(e)}")
-            return ToolResult(error=f"Error: {str(e)}")