cua-agent 0.4.31__py3-none-any.whl → 0.4.32__py3-none-any.whl

agent/loops/opencua.py

@@ -0,0 +1,142 @@
+"""
+OpenCUA agent loop implementation for click prediction using litellm.acompletion
+Based on OpenCUA model for GUI grounding tasks.
+"""
+
+import asyncio
+import json
+import re
+import base64
+from typing import Dict, List, Any, AsyncGenerator, Union, Optional, Tuple
+from io import BytesIO
+import uuid
+from PIL import Image
+import litellm
+import math
+
+from .composed_grounded import ComposedGroundedConfig
+from ..decorators import register_agent
+from ..types import Messages, AgentResponse, Tools, AgentCapability
+from ..loops.base import AsyncAgentConfig
+
+def extract_coordinates_from_pyautogui(text: str) -> Optional[Tuple[int, int]]:
+    """Extract coordinates from pyautogui.click(x=..., y=...) format."""
+    try:
+        # Look for pyautogui.click(x=1443, y=343) pattern
+        pattern = r"pyautogui\.click\(x=(\d+),\s*y=(\d+)\)"
+        match = re.search(pattern, text)
+        if match:
+            x, y = int(match.group(1)), int(match.group(2))
+            return (x, y)
+        return None
+    except Exception:
+        return None
+
+@register_agent(models=r"(?i).*OpenCUA.*")
+class OpenCUAConfig(ComposedGroundedConfig):
+    """OpenCUA agent configuration implementing AsyncAgentConfig protocol for click prediction."""
+    
+    def __init__(self):
+        super().__init__()
+        self.current_model = None
+        self.last_screenshot_b64 = None
+
+    async def predict_step(
+        self,
+        messages: List[Dict[str, Any]],
+        model: str,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        max_retries: Optional[int] = None,
+        stream: bool = False,
+        computer_handler=None,
+        _on_api_start=None,
+        _on_api_end=None,
+        _on_usage=None,
+        _on_screenshot=None,
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Fallback to a self-composed model"""
+        return await super().predict_step(
+            messages=messages,
+            model=f"{model}+{model}",
+            tools=tools,
+            max_retries=max_retries,
+            stream=stream,
+            computer_handler=computer_handler,
+            _on_api_start=_on_api_start,
+            _on_api_end=_on_api_end,
+            _on_usage=_on_usage,
+            _on_screenshot=_on_screenshot,
+            **kwargs
+        )
+
+    async def predict_click(
+        self,
+        model: str,
+        image_b64: str,
+        instruction: str,
+        **kwargs
+    ) -> Optional[Tuple[int, int]]:
+        """
+        Predict click coordinates using OpenCUA model via litellm.acompletion.
+        
+        Args:
+            model: The OpenCUA model name
+            image_b64: Base64 encoded image
+            instruction: Instruction for where to click
+            
+        Returns:
+            Tuple of (x, y) coordinates or None if prediction fails
+        """
+        # Prepare system message
+        system_prompt = (
+            "You are a GUI agent. You are given a task and a screenshot of the screen. "
+            "You need to perform a series of pyautogui actions to complete the task."
+        )
+        
+        system_message = {
+            "role": "system",
+            "content": system_prompt
+        }
+        
+        # Prepare user message with image and instruction
+        user_message = {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/png;base64,{image_b64}"
+                    }
+                },
+                {
+                    "type": "text",
+                    "text": f"Click on {instruction}"
+                }
+            ]
+        }
+        
+        # Prepare API call kwargs
+        api_kwargs = {
+            "model": model,
+            "messages": [system_message, user_message],
+            "max_new_tokens": 2056,
+            "temperature": 0,
+            **kwargs
+        }
+        
+        # Use liteLLM acompletion
+        response = await litellm.acompletion(**api_kwargs)
+        
+        # Extract response text
+        output_text = response.choices[0].message.content
+        # print(output_text)
+        
+        # Extract coordinates from pyautogui format
+        coordinates = extract_coordinates_from_pyautogui(output_text)
+        
+        return coordinates
+    
+    def get_capabilities(self) -> List[AgentCapability]:
+        """Return the capabilities supported by this agent."""
+        return ["click"]

agent/loops/uitars.py

@@ -780,7 +780,7 @@ class UITARSConfig:
             api_kwargs = {
                 "model": model,
                 "messages": litellm_messages,
-                "max_tokens": 100,
+                "max_tokens": 2056,
                 "temperature": 0.0,
                 "do_sample": False
             }

{cua_agent-0.4.31.dist-info → cua_agent-0.4.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cua-agent
-Version: 0.4.31
+Version: 0.4.32
 Summary: CUA (Computer Use) Agent for AI-driven computer interaction
 Author-Email: TryCua <gh@trycua.com>
 Requires-Python: >=3.12
@@ -31,22 +31,38 @@ Provides-Extra: glm45v-hf
 Requires-Dist: accelerate; extra == "glm45v-hf"
 Requires-Dist: torch; extra == "glm45v-hf"
 Requires-Dist: transformers-v4.55.0-GLM-4.5V-preview; extra == "glm45v-hf"
+Provides-Extra: opencua-hf
+Requires-Dist: accelerate; extra == "opencua-hf"
+Requires-Dist: torch; extra == "opencua-hf"
+Requires-Dist: transformers==4.53.0; extra == "opencua-hf"
+Requires-Dist: tiktoken>=0.11.0; extra == "opencua-hf"
+Requires-Dist: blobfile>=3.0.0; extra == "opencua-hf"
+Provides-Extra: internvl-hf
+Requires-Dist: accelerate; extra == "internvl-hf"
+Requires-Dist: torch; extra == "internvl-hf"
+Requires-Dist: transformers>=4.55.0; extra == "internvl-hf"
+Requires-Dist: einops; extra == "internvl-hf"
+Requires-Dist: timm; extra == "internvl-hf"
 Provides-Extra: ui
 Requires-Dist: gradio>=5.23.3; extra == "ui"
 Requires-Dist: python-dotenv>=1.0.1; extra == "ui"
 Provides-Extra: cli
 Requires-Dist: yaspin>=3.1.0; extra == "cli"
 Provides-Extra: hud
-Requires-Dist: hud-python==0.4.19; extra == "hud"
+Requires-Dist: hud-python==0.4.26; extra == "hud"
 Provides-Extra: all
 Requires-Dist: mlx-vlm>=0.1.27; sys_platform == "darwin" and extra == "all"
 Requires-Dist: accelerate; extra == "all"
 Requires-Dist: torch; extra == "all"
-Requires-Dist: transformers>=4.54.0; extra == "all"
+Requires-Dist: transformers>=4.55.0; extra == "all"
+Requires-Dist: einops; extra == "all"
+Requires-Dist: timm; extra == "all"
+Requires-Dist: tiktoken>=0.11.0; extra == "all"
+Requires-Dist: blobfile>=3.0.0; extra == "all"
 Requires-Dist: gradio>=5.23.3; extra == "all"
 Requires-Dist: python-dotenv>=1.0.1; extra == "all"
 Requires-Dist: yaspin>=3.1.0; extra == "all"
-Requires-Dist: hud-python==0.4.19; extra == "all"
+Requires-Dist: hud-python==0.4.26; extra == "all"
 Description-Content-Type: text/markdown
 
 <div align="center">

{cua_agent-0.4.31.dist-info → cua_agent-0.4.32.dist-info}/RECORD

@@ -1,10 +1,15 @@
 agent/__init__.py,sha256=MaW-BczJ-lCACPYH39DvFhE7ZWiSo7sBO6pBfyO7Nxc,1269
 agent/__main__.py,sha256=lBUe8Niqa5XoCjwFfXyX7GtnUwjjZXC1-j4V9mvUYSc,538
 agent/adapters/__init__.py,sha256=Q_OxxwXBcBIetQ_DtHS5bwZWXrvCKPX2grCg8R0UKek,301
-agent/adapters/huggingfacelocal_adapter.py,sha256=Uqjtcohhzd33VFh38Ra2y4Uv_lTghMswoqS1t-KKFkw,8480
+agent/adapters/huggingfacelocal_adapter.py,sha256=3ht4jCUP4rpjPxi7vj8xOJNelTgfsUq0YbS44FwVN0c,7089
 agent/adapters/human_adapter.py,sha256=xT4nnfNXb1z-vnGFlLmFEZN7TMcoMBGS40MtR1Zwv4o,13079
 agent/adapters/mlxvlm_adapter.py,sha256=4VhhKDZfLLKL5joL1v4PPFvYw-R8spoDsat3vOAGnpE,14864
-agent/agent.py,sha256=F_nPTRv-FCo3oKaEFCugun-sj6_iD_iw2RQxcaTMVmM,30192
+agent/adapters/models/__init__.py,sha256=23ETHLbn1C3VW4zMzSn1Ql-gsZz-wQCeJX12Lv_0n8M,1412
+agent/adapters/models/generic.py,sha256=hpJt73jQePV80NYRAJTtOaElcrtGcpYK-QnInZIUkk0,2768
+agent/adapters/models/internvl.py,sha256=PbFjU_Fu1JIahlEtR5pTO7RSV2UYJ9CAlQrMXKXJrTA,11385
+agent/adapters/models/opencua.py,sha256=gPoZBMXyjiPEJ7Py2mpRWHiyay8X-y-pOSKw1LC_ihU,3924
+agent/adapters/models/qwen2_5_vl.py,sha256=kc9YrtCB0FFy-oB1EkD_zasxYnbJg1wGiss6i0ilwdo,2809
+agent/agent.py,sha256=NCRK1xRGt_sa1Yh_do2IPRceV024P2p7KR81wwCYlY8,30462
 agent/callbacks/__init__.py,sha256=VqYHFt_wk1mc3hKudMZk2Qakrh-bn2rVKh_4xebF0tI,725
 agent/callbacks/base.py,sha256=UnnnYlh6XCm6HKZZsAPaT_Eyo9LUYLyjyNwF-QRm6Ns,4691
 agent/callbacks/budget_manager.py,sha256=RyKM-7iXQcDotYvrw3eURzeEHEXvQjID-NobtvQWE7k,1832
@@ -14,8 +19,8 @@ agent/callbacks/operator_validator.py,sha256=T5tp62pkShkcdHu2rgREUGdk8fryL_ziJsI
 agent/callbacks/pii_anonymization.py,sha256=NEkUTUjQBi82nqus7kT-1E4RaeQ2hQrY7YCnKndLhP8,3272
 agent/callbacks/prompt_instructions.py,sha256=RUqsJhiNiXqaOM_P2AfyBinWUDdgDku46BExLMUJHn4,1517
 agent/callbacks/telemetry.py,sha256=RbUDhE41mTi8g9hNre0EpltK_NUZkLj8buJLWBzs0Ek,7363
-agent/callbacks/trajectory_saver.py,sha256=rslgg4Ak7JHSNmmJgANRQ5TsUYWGuUJDZ6amureaz_o,15963
-agent/cli.py,sha256=AgaXwywHd3nGQWuqMRj6SbPyFaCPjfo5980Y1ApQOTQ,12413
+agent/callbacks/trajectory_saver.py,sha256=-XNgiKU6T8Qw_i2AZMQuw0HuUe6MHkU89rjn_T386Rw,16128
+agent/cli.py,sha256=HddU18IvvKdyvQu0ru21nAcNc6k7toYuyjgORIzX_qo,16110
 agent/computers/__init__.py,sha256=39ISJsaREaQIZckpzxSuLhuR763wUU3TxUux78EKjAg,1477
 agent/computers/base.py,sha256=hZntX4vgc1ahD3EnFeb9lUjtBmgka1vb27hndPl9tKQ,2187
 agent/computers/cua.py,sha256=xp2A34kT2C1NKqSRo2GB6766gkraM-UtpFjRv8LUTSc,4889
@@ -28,16 +33,19 @@ agent/human_tool/ui.py,sha256=wu9eZorhxCkyPTlBSZjYaVzutoHMlucAz8UGNpAT4bM,30644
 agent/integrations/hud/__init__.py,sha256=xir5BVAlG2cFc7rHSx_Ea_2b1kp2TtFuKJk07jny7qY,5969
 agent/integrations/hud/agent.py,sha256=GBikd9MhjDNKMiMG8J7PE3OMSmvmC_JLZ1p5xr2cZoc,14006
 agent/integrations/hud/proxy.py,sha256=8HUoh7uZ8Z3vkhPXK0dskgePGsP8oCqyYij0mE_E7X8,10902
-agent/loops/__init__.py,sha256=Ef8aj07l3osibwDk-DTo80PrpL4_GdKRTP1ikl_b-BQ,328
-agent/loops/anthropic.py,sha256=q7lr1PjI6VPtlozoweluY2c3hCGqa_2s-whzxa37iKE,70250
+agent/loops/__init__.py,sha256=c6stEkT15smK8ZIf9j2kyOko84uz1YIvHXx0Mbe2wq8,472
+agent/loops/anthropic.py,sha256=ODrMvmTkyzIOLjGq6HbKzzgBu19TE_Xlsi--7vc5T6o,70196
 agent/loops/base.py,sha256=LK7kSTnc2CB88LI7qr2VP7LMq0eS5r2bSEnrxO6IN5U,2345
-agent/loops/composed_grounded.py,sha256=8oJoqaRzKWbI9I4VoFuAoUzQ11_CFnYT-EdPOy-NVEQ,12349
-agent/loops/glm45v.py,sha256=V1f-5vAifbYcY-qTc7fW2KXVRkAfApQI_EjavH3X2ak,35110
-agent/loops/gta1.py,sha256=ha5TaUWqUzTffx_ow1WiBU8i3VNP-6FL5XC66ajPFjg,5829
+agent/loops/composed_grounded.py,sha256=Um_8G0v5DEzF_A9wWIGp_IDPDMvv4IXDTFpEDH92Vto,12367
+agent/loops/glm45v.py,sha256=EKAoh-PWkcCdzBVebjXbdqoDNkXgcmJpIqmTNPiZ8TM,35127
+agent/loops/gta1.py,sha256=uGIcUH5ChzO75eGvoQxuKMBWjX-1J9-xmC7vPetobjU,5831
+agent/loops/holo.py,sha256=peQ0xx4XQDBQ3g2XKRLCgyrU_2PkXe3RaysNBqFyS90,7481
+agent/loops/internvl.py,sha256=iQs6DSoP9JOyUxRAz_HPuv4Hi2Sbv-Jc3022W-oPX5Y,6596
 agent/loops/model_types.csv,sha256=GmFn4x80yoUpQZuQ-GXtJkPVlOLYWZ5u_5A73HRyeNE,112
 agent/loops/omniparser.py,sha256=-db8JUL2Orn47ERIaLbuNShAXn4LeIgYzRWphn_9Dg4,15071
 agent/loops/openai.py,sha256=3UEXdecqGkyknhTgp6zxr_cNCVg5vM-61I6SKMNl6m8,8692
-agent/loops/uitars.py,sha256=QyEWyrhkI5MCksTunntY-5jtETd4pwcQB-DyzmiWezo,32350
+agent/loops/opencua.py,sha256=Chb4UASHDrdcX_fO__Gw2e9ay4Hl6Vq38K5x-IoHyuo,4432
+agent/loops/uitars.py,sha256=mVPt4V-HabX7ZiQnM55BVQt73CuZUjmUAsbm4Tf6TXk,32351
 agent/proxy/examples.py,sha256=GYFJ-sfDsSNZr9n_qpvDx_0rShqoKE5JW0ibbljWfoo,6192
 agent/proxy/handlers.py,sha256=48mMNyZOU3dJQ6oI5r2kDDe29rcU49MConlB0MZeCsU,9602
 agent/responses.py,sha256=_SoN4BkaTxMHMB21EOtDc_aDBIJlfDwsCzszMBnIkH0,30764
@@ -47,7 +55,7 @@ agent/ui/__main__.py,sha256=vudWXYvGM0aNT5aZ94HPtGW8YXOZ4cLXepHyhUM_k1g,73
 agent/ui/gradio/__init__.py,sha256=yv4Mrfo-Sj2U5sVn_UJHAuwYCezo-5O4ItR2C9jzNko,145
 agent/ui/gradio/app.py,sha256=Ol97YEbwREZZQ9_PMjVHlfOcu9BGsawxgAGAm79hT80,9117
 agent/ui/gradio/ui_components.py,sha256=dJUvKDmc1oSejtoR_gU_oWWYwxaOOQyPloSYRGMrUCQ,36068
-cua_agent-0.4.31.dist-info/METADATA,sha256=HywszWj2eDguXdge9eANyyz57gVlY9YZr4yacHHkUhU,5610
-cua_agent-0.4.31.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90
-cua_agent-0.4.31.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
-cua_agent-0.4.31.dist-info/RECORD,,
+cua_agent-0.4.32.dist-info/METADATA,sha256=9DM4yfZ8hH6-JeNvke6WOgzZLEF0i3A8cDeb3aTGpyk,6340
+cua_agent-0.4.32.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90
+cua_agent-0.4.32.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
+cua_agent-0.4.32.dist-info/RECORD,,