cua-agent 0.4.22__py3-none-any.whl → 0.7.16__py3-none-any.whl

agent/loops/fara/helpers.py

@@ -0,0 +1,357 @@
+# Source: https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/llm/fncall_prompts/nous_fncall_prompt.py
+
+import copy
+import json
+import os
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+
+from .schema import ContentItem, Message
+
+FN_CALL_TEMPLATE_QWEN = """# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>"""
+
+FN_CALL_TEMPLATE = """You are a web automation agent that performs actions on websites to fulfill user requests by calling various tools.
+* You should stop execution at Critical Points. A Critical Point would be encountered in tasks like 'Checkout', 'Book', 'Purchase', 'Call', 'Email', 'Order', etc where a binding transaction/agreement would require the user's permission/personal or sensitive information (name, email, credit card, address, payment information, resume, etc) in order to complete a transaction (purchase, reservation, sign-up etc), or to communicate in a way that a human would be expected to do (call, email, apply to a job, etc).
+* Solve the task as far as you can up until a Critical Point:
+    - For example, if the task is to "call a restaurant to make a reservation", you should not actually make the call but should navigate to the restaurant's page and find the phone number.
+    - Similarly, if the task is to "order new size 12 running shoes" you should not actually place the order but should instead search for the right shoes that meet the criteria and add them to the cart.
+    - Some tasks, like answering questions, may not encounter a Critical Point at all.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>"""
+
+
+SPECIAL_CODE_MODE = os.getenv("SPECIAL_CODE_MODE", "false").lower() == "true"
+CODE_TOOL_PATTERN = "code_interpreter"
+FN_CALL_TEMPLATE_WITH_CI = """# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>
+For code parameters, use placeholders first, and then put the code within <code></code> XML tags, such as:
+<tool_call>
+{{"name": <function-name>, "arguments": {{"code": ""}}}}
+<code>
+Here is the code.
+</code>
+</tool_call>"""
+
+
+class NousFnCallPrompt:
+    def __init__(self, template_name: str = "default"):
+        """Initialize NousFnCallPrompt with a specific template.
+
+        Args:
+            template_name: Name of the template to use. Options:
+                          "default", "qwen", "with_ci"
+        """
+        self.template_name = template_name
+        self.template_map = {
+            "default": FN_CALL_TEMPLATE,
+            "qwen": FN_CALL_TEMPLATE_QWEN,
+            "with_ci": FN_CALL_TEMPLATE_WITH_CI,
+        }
+
+        if template_name not in self.template_map:
+            raise ValueError(
+                f"Unknown template_name: {template_name}. "
+                f"Available options: {list(self.template_map.keys())}"
+            )
+
+    def preprocess_fncall_messages(
+        self,
+        messages: List[Message],
+        functions: List[dict],
+        lang: Literal["en", "zh"],
+        parallel_function_calls: bool = True,
+        function_choice: Union[Literal["auto"], str] = "auto",
+    ) -> List[Message]:
+        del lang  # ignored
+        del parallel_function_calls  # ignored
+        if function_choice != "auto":
+            raise NotImplementedError
+
+        ori_messages = messages
+
+        # Change function_call responses to plaintext responses:
+        messages = []
+        for msg in copy.deepcopy(ori_messages):
+            role, content, reasoning_content = (
+                msg.role,
+                msg.content,
+                msg.reasoning_content,
+            )
+            if role in ("system", "user"):
+                messages.append(msg)
+            elif role == "assistant":
+                content = content or []
+                fn_call = msg.function_call
+                if fn_call:
+                    if (not SPECIAL_CODE_MODE) or (CODE_TOOL_PATTERN not in fn_call.name):
+                        fc = {
+                            "name": fn_call.name,
+                            "arguments": json.loads(fn_call.arguments),
+                        }
+                        fc = json.dumps(fc, ensure_ascii=False)
+                        fc = f"<tool_call>\n{fc}\n</tool_call>"
+                    else:
+                        para = json.loads(fn_call.arguments)
+                        code = para["code"]
+                        para["code"] = ""
+                        fc = {"name": fn_call.name, "arguments": para}
+                        fc = json.dumps(fc, ensure_ascii=False)
+                        fc = f"<tool_call>\n{fc}\n<code>\n{code}\n</code>\n</tool_call>"
+
+                    content.append(ContentItem(text=fc))
+                if messages[-1].role == "assistant":
+                    messages[-1].content.append(ContentItem(text="\n"))
+                    messages[-1].content.extend(content)
+                else:
+                    # TODO: Assuming there will only be one continuous reasoning_content here
+                    messages.append(
+                        Message(
+                            role=role,
+                            content=content,
+                            reasoning_content=reasoning_content,
+                        )
+                    )
+            elif role == "function":
+                assert isinstance(content, list)
+                assert len(content) == 1
+                assert content[0].text
+                fc = f"<tool_response>\n{content[0].text}\n</tool_response>"
+                content = [ContentItem(text=fc)]
+                if messages[-1].role == "user":
+                    messages[-1].content.append(ContentItem(text="\n"))
+                    messages[-1].content.extend(content)
+                else:
+                    messages.append(Message(role="user", content=content))
+            else:
+                raise TypeError
+
+        tool_descs = [{"type": "function", "function": f} for f in functions]
+        tool_names = [
+            function.get("name_for_model", function.get("name", "")) for function in functions
+        ]
+        tool_descs = "\n".join([json.dumps(f, ensure_ascii=False) for f in tool_descs])
+
+        # Select template based on configuration
+        if SPECIAL_CODE_MODE and any([CODE_TOOL_PATTERN in x for x in tool_names]):
+            selected_template = FN_CALL_TEMPLATE_WITH_CI
+        else:
+            selected_template = self.template_map[self.template_name]
+
+        tool_system = selected_template.format(tool_descs=tool_descs)
+        if messages[0].role == "system":
+            messages[0].content.append(ContentItem(text="\n\n" + tool_system))
+        else:
+            messages = [Message(role="system", content=[ContentItem(text=tool_system)])] + messages
+        return messages
+
+
+# Mainly for removing incomplete special tokens when streaming the output
+# This assumes that '<tool_call>\n{"name": "' is the special token for the NousFnCallPrompt
+def remove_incomplete_special_tokens(text: str) -> str:
+    if text in '<tool_call>\n{"name": "':
+        text = ""
+    return text
+
+
+def extract_fn(text: str):
+    fn_name, fn_args = "", ""
+    fn_name_s = '"name": "'
+    fn_name_e = '", "'
+    fn_args_s = '"arguments": '
+    i = text.find(fn_name_s)
+    k = text.find(fn_args_s)
+    if i > 0:
+        _text = text[i + len(fn_name_s) :]
+        j = _text.find(fn_name_e)
+        if j > -1:
+            fn_name = _text[:j]
+    if k > 0:
+        fn_args = text[k + len(fn_args_s) :]
+
+    if len(fn_args) > 5:
+        fn_args = fn_args[:-5]
+    else:
+        fn_args = ""
+    return fn_name, fn_args
+
+
+def build_nous_system(functions: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+    """Use original FARA NousFnCallPrompt to generate a system message embedding tool schema."""
+    from .schema import ContentItem as NousContentItem
+    from .schema import Message as NousMessage
+
+    msgs = NousFnCallPrompt().preprocess_fncall_messages(
+        messages=[
+            NousMessage(
+                role="system", content=[NousContentItem(text="You are a helpful assistant.")]
+            )
+        ],
+        functions=functions,
+        lang="en",
+    )
+    sys = msgs[0].model_dump()
+    # Convert structured content to OpenAI-style content list
+    content = [{"type": "text", "text": c["text"]} for c in sys.get("content", [])]
+    return {"role": "system", "content": content}
+
+
+def parse_tool_call_from_text(text: str) -> Optional[Dict[str, Any]]:
+    """Extract JSON object within <tool_call>...</tool_call> from model text.
+
+    Accepts both </tool_call> and <tool_call> as closing tags for robustness.
+    Handles nested braces in JSON objects.
+    """
+    # Find the opening tag
+    start_idx = text.find("<tool_call>")
+    if start_idx == -1:
+        return None
+
+    # Find the start of JSON (first '{' after opening tag)
+    json_start = text.find("{", start_idx)
+    if json_start == -1:
+        return None
+
+    # Extract JSON by counting braces
+    brace_count = 0
+    json_end = json_start
+    for i in range(json_start, len(text)):
+        if text[i] == "{":
+            brace_count += 1
+        elif text[i] == "}":
+            brace_count -= 1
+            if brace_count == 0:
+                json_end = i + 1
+                break
+
+    if brace_count != 0:
+        return None
+
+    json_str = text[json_start:json_end]
+    try:
+        return json.loads(json_str)
+    except Exception:
+        return None
+
+
+async def unnormalize_coordinate(args: Dict[str, Any], dims: Tuple[int, int]) -> Dict[str, Any]:
+    """Coordinates appear in 0..1000 space, scale to actual screen size using dims if provided."""
+    coord = args.get("coordinate")
+    if not coord or not isinstance(coord, (list, tuple)) or len(coord) < 2:
+        return args
+    x, y = float(coord[0]), float(coord[1])
+    width, height = float(dims[0]), float(dims[1])
+    x_abs = max(0.0, min(width, (x / 1000.0) * width))
+    y_abs = max(0.0, min(height, (y / 1000.0) * height))
+    args = {**args, "coordinate": [round(x_abs), round(y_abs)]}
+    return args
+
+
+def convert_qwen_tool_args_to_computer_action(args: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """
+    Convert Qwen computer tool arguments to the Computer Calls action schema.
+
+    Qwen (example):
+        {"action": "left_click", "coordinate": [114, 68]}
+
+    Target (example):
+        {"action": "left_click", "x": 114, "y": 68}
+
+    Other mappings:
+    - right_click, middle_click, double_click (triple_click -> double_click)
+    - mouse_move -> { action: "move", x, y }
+    - key -> { action: "keypress", keys: [...] }
+    - type -> { action: "type", text }
+    - scroll/hscroll -> { action: "scroll", scroll_x, scroll_y, x, y }
+    - wait -> { action: "wait" }
+    - terminate/answer are not direct UI actions; return None for now
+    """
+    if not isinstance(args, dict):
+        return None
+
+    action = args.get("action")
+    if not isinstance(action, str):
+        return None
+
+    # Coordinates helper
+    coord = args.get("coordinate")
+    x = y = None
+    if isinstance(coord, (list, tuple)) and len(coord) >= 2:
+        try:
+            x = int(round(float(coord[0])))
+            y = int(round(float(coord[1])))
+        except Exception:
+            x = y = None
+
+    # Map actions
+    a = action.lower()
+    if a in {"left_click", "right_click", "middle_click", "double_click"}:
+        if x is None or y is None:
+            return None
+        return {"action": a, "x": x, "y": y}
+    if a == "triple_click":
+        # Approximate as double_click
+        if x is None or y is None:
+            return None
+        return {"action": "double_click", "x": x, "y": y}
+    if a == "mouse_move":
+        if x is None or y is None:
+            return None
+        return {"action": "move", "x": x, "y": y}
+    if a == "key":
+        keys = args.get("keys")
+        if isinstance(keys, list) and all(isinstance(k, str) for k in keys):
+            return {"action": "keypress", "keys": keys}
+        return None
+    if a == "type":
+        text = args.get("text")
+        if isinstance(text, str):
+            return {"action": "type", "text": text}
+        return None
+    if a in {"scroll", "hscroll"}:
+        pixels = args.get("pixels") or 0
+        try:
+            pixels_val = int(round(float(pixels)))
+        except Exception:
+            pixels_val = 0
+        scroll_x = pixels_val if a == "hscroll" else 0
+        scroll_y = pixels_val if a == "scroll" else 0
+        # Include cursor position if available (optional)
+        out: Dict[str, Any] = {"action": "scroll", "scroll_x": scroll_x, "scroll_y": scroll_y}
+        if x is not None and y is not None:
+            out.update({"x": x, "y": y})
+        return out
+    if a == "wait":
+        return {"action": "wait"}
+
+    # Non-UI or terminal actions: terminate/answer -> not mapped here
+    return None

agent/loops/fara/schema.py

@@ -0,0 +1,143 @@
+# Source: https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/llm/schema.py
+
+from typing import List, Literal, Optional, Tuple, Union
+
+from pydantic import BaseModel, field_validator, model_validator
+
+
+class BaseModelCompatibleDict(BaseModel):
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def model_dump(self, **kwargs):
+        if "exclude_none" not in kwargs:
+            kwargs["exclude_none"] = True
+        return super().model_dump(**kwargs)
+
+    def model_dump_json(self, **kwargs):
+        if "exclude_none" not in kwargs:
+            kwargs["exclude_none"] = True
+        return super().model_dump_json(**kwargs)
+
+    def get(self, key, default=None):
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            return default
+
+    def __str__(self):
+        return f"{self.model_dump()}"
+
+
+class FunctionCall(BaseModelCompatibleDict):
+    name: str
+    arguments: str
+
+    def __init__(self, name: str, arguments: str):
+        super().__init__(name=name, arguments=arguments)
+
+    def __repr__(self):
+        return f"FunctionCall({self.model_dump()})"
+
+
+class ContentItem(BaseModelCompatibleDict):
+    text: Optional[str] = None
+    image: Optional[str] = None
+    file: Optional[str] = None
+    audio: Optional[Union[str, dict]] = None
+    video: Optional[Union[str, list]] = None
+
+    def __init__(
+        self,
+        text: Optional[str] = None,
+        image: Optional[str] = None,
+        file: Optional[str] = None,
+        audio: Optional[Union[str, dict]] = None,
+        video: Optional[Union[str, list]] = None,
+    ):
+        super().__init__(text=text, image=image, file=file, audio=audio, video=video)
+
+    @model_validator(mode="after")
+    def check_exclusivity(self):
+        provided_fields = 0
+        if self.text is not None:
+            provided_fields += 1
+        if self.image:
+            provided_fields += 1
+        if self.file:
+            provided_fields += 1
+        if self.audio:
+            provided_fields += 1
+        if self.video:
+            provided_fields += 1
+
+        if provided_fields != 1:
+            raise ValueError(
+                "Exactly one of 'text', 'image', 'file', 'audio', or 'video' must be provided."
+            )
+        return self
+
+    def __repr__(self):
+        return f"ContentItem({self.model_dump()})"
+
+    def get_type_and_value(
+        self,
+    ) -> Tuple[Literal["text", "image", "file", "audio", "video"], str]:
+        ((t, v),) = self.model_dump().items()
+        assert t in ("text", "image", "file", "audio", "video")
+        return t, v
+
+    @property
+    def type(self) -> Literal["text", "image", "file", "audio", "video"]:
+        t, _ = self.get_type_and_value()
+        return t
+
+    @property
+    def value(self) -> str:
+        _, v = self.get_type_and_value()
+        return v
+
+
+class Message(BaseModelCompatibleDict):
+    role: str
+    content: Union[str, List[ContentItem]]
+    reasoning_content: Optional[Union[str, List[ContentItem]]] = None
+    name: Optional[str] = None
+    function_call: Optional[FunctionCall] = None
+    extra: Optional[dict] = None
+
+    def __init__(
+        self,
+        role: str,
+        content: Union[str, List[ContentItem]],
+        reasoning_content: Optional[Union[str, List[ContentItem]]] = None,
+        name: Optional[str] = None,
+        function_call: Optional[FunctionCall] = None,
+        extra: Optional[dict] = None,
+        **kwargs,
+    ):
+        if content is None:
+            content = ""
+        if reasoning_content is None:
+            reasoning_content = ""
+        super().__init__(
+            role=role,
+            content=content,
+            reasoning_content=reasoning_content,
+            name=name,
+            function_call=function_call,
+            extra=extra,
+        )
+
+    def __repr__(self):
+        return f"Message({self.model_dump()})"
+
+    @field_validator("role")
+    def role_checker(cls, value: str) -> str:
+        values = ["system", "user", "assistant", "function"]
+        if value not in values:
+            raise ValueError(f'{value} must be one of {",".join(values)}')
+        return value

agent/loops/gelato.py

@@ -0,0 +1,183 @@
+"""
+Gelato agent loop implementation for click prediction using litellm.acompletion
+Model: https://huggingface.co/mlfoundations/Gelato-30B-A3B
+Code: https://github.com/mlfoundations/Gelato/tree/main
+"""
+
+import base64
+import math
+import re
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from PIL import Image
+
+from ..decorators import register_agent
+from ..loops.base import AsyncAgentConfig
+from ..types import AgentCapability
+
+SYSTEM_PROMPT = """
+You are an expert UI element locator. Given a GUI image and a user's element description, provide the coordinates of the specified element as a single (x,y) point. For elements with area, return the center point.
+
+Output the coordinate pair exactly:
+(x,y)
+"""
+
+
+def extract_coordinates(raw_string):
+    """
+    Extract the coordinates from the raw string.
+    Args:
+        raw_string: str (e.g. "(100, 200)")
+    Returns:
+        x: float (e.g. 100.0)
+        y: float (e.g. 200.0)
+    """
+    try:
+        matches = re.findall(r"\((-?\d*\.?\d+),\s*(-?\d*\.?\d+)\)", raw_string)
+        return [tuple(map(int, match)) for match in matches][0]
+    except:
+        return 0, 0
+
+
+def smart_resize(
+    height: int,
+    width: int,
+    factor: int = 28,
+    min_pixels: int = 3136,
+    max_pixels: int = 8847360,
+) -> Tuple[int, int]:
+    """Smart resize function similar to qwen_vl_utils."""
+    # Calculate the total pixels
+    total_pixels = height * width
+
+    # If already within bounds, return original dimensions
+    if min_pixels <= total_pixels <= max_pixels:
+        # Round to nearest factor
+        new_height = (height // factor) * factor
+        new_width = (width // factor) * factor
+        return new_height, new_width
+
+    # Calculate scaling factor
+    if total_pixels > max_pixels:
+        scale = (max_pixels / total_pixels) ** 0.5
+    else:
+        scale = (min_pixels / total_pixels) ** 0.5
+
+    # Apply scaling
+    new_height = int(height * scale)
+    new_width = int(width * scale)
+
+    # Round to nearest factor
+    new_height = (new_height // factor) * factor
+    new_width = (new_width // factor) * factor
+
+    # Ensure minimum size
+    new_height = max(new_height, factor)
+    new_width = max(new_width, factor)
+
+    return new_height, new_width
+
+
+@register_agent(models=r".*Gelato.*")
+class GelatoConfig(AsyncAgentConfig):
+    """Gelato agent configuration implementing AsyncAgentConfig protocol for click prediction."""
+
+    def __init__(self):
+        self.current_model = None
+        self.last_screenshot_b64 = None
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        raise NotImplementedError()
+
+    async def predict_click(
+        self, model: str, image_b64: str, instruction: str, **kwargs
+    ) -> Optional[Tuple[float, float]]:
+        """
+        Predict click coordinates using UI-Ins model via litellm.acompletion.
+
+        Args:
+            model: The UI-Ins model name
+            image_b64: Base64 encoded image
+            instruction: Instruction for where to click
+
+        Returns:
+            Tuple of (x, y) coordinates or None if prediction fails
+        """
+        # Decode base64 image
+        image_data = base64.b64decode(image_b64)
+        image = Image.open(BytesIO(image_data))
+        width, height = image.width, image.height
+
+        # Smart resize the image (similar to qwen_vl_utils)
+        resized_height, resized_width = smart_resize(
+            height,
+            width,
+            factor=28,  # Default factor for Qwen models
+            min_pixels=3136,
+            max_pixels=4096 * 2160,
+        )
+        resized_image = image.resize((resized_width, resized_height))
+        scale_x, scale_y = width / resized_width, height / resized_height
+
+        # Convert resized image back to base64
+        buffered = BytesIO()
+        resized_image.save(buffered, format="PNG")
+        resized_image_b64 = base64.b64encode(buffered.getvalue()).decode()
+
+        # Prepare system and user messages
+        system_message = {
+            "role": "system",
+            "content": [{"type": "text", "text": SYSTEM_PROMPT.strip()}],
+        }
+
+        user_message = {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/png;base64,{resized_image_b64}"},
+                },
+                {"type": "text", "text": instruction},
+            ],
+        }
+
+        # Prepare API call kwargs
+        api_kwargs = {
+            "model": model,
+            "messages": [system_message, user_message],
+            "max_tokens": 2056,
+            "temperature": 0.0,
+            **kwargs,
+        }
+
+        # Use liteLLM acompletion
+        response = await litellm.acompletion(**api_kwargs)
+
+        # Extract response text
+        output_text = response.choices[0].message.content  # type: ignore
+
+        # Extract and rescale coordinates
+        pred_x, pred_y = extract_coordinates(output_text)  # type: ignore
+        pred_x *= scale_x
+        pred_y *= scale_y
+
+        return (math.floor(pred_x), math.floor(pred_y))
+
+    def get_capabilities(self) -> List[AgentCapability]:
+        """Return the capabilities supported by this agent."""
+        return ["click"]