cua-agent 0.4.12__py3-none-any.whl → 0.4.13__py3-none-any.whl

agent/integrations/hud/computer_handler.py

@@ -0,0 +1,187 @@
+"""HUD Computer Handler for ComputerAgent integration."""
+
+import base64
+from io import BytesIO
+from typing import Literal, Optional, Any, Dict, Callable
+from PIL import Image
+
+from agent.computers import AsyncComputerHandler
+
+
+class HUDComputerHandler(AsyncComputerHandler):
+    """Computer handler that interfaces with HUD environment."""
+    
+    def __init__(
+        self,
+        environment: Literal["windows", "mac", "linux", "browser"] = "linux",
+        dimensions: tuple[int, int] = (1024, 768),
+        screenshot_callback: Optional[Callable] = None,
+        action_callback: Optional[Callable] = None,
+    ):
+        """
+        Initialize HUD computer handler.
+        
+        Args:
+            environment: The environment type for HUD
+            dimensions: Screen dimensions as (width, height)
+            screenshot_callback: Optional callback to get screenshots from HUD environment
+            action_callback: Optional callback to execute actions in HUD environment
+        """
+        super().__init__()
+        self._environment = environment
+        self._dimensions = dimensions
+        self._screenshot_callback = screenshot_callback
+        self._action_callback = action_callback
+        
+        # Store the last screenshot for reuse
+        self._last_screenshot: Optional[str] = None
+        
+    def set_screenshot_callback(self, callback: Callable) -> None:
+        """Set the screenshot callback."""
+        self._screenshot_callback = callback
+        
+    def set_action_callback(self, callback: Callable) -> None:
+        """Set the action callback."""
+        self._action_callback = callback
+        
+    def update_screenshot(self, screenshot: str) -> None:
+        """Update the stored screenshot (base64 string)."""
+        self._last_screenshot = screenshot
+
+    async def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]:
+        """Get the current environment type."""
+        return self._environment # type: ignore
+    
+    async def get_dimensions(self) -> tuple[int, int]:
+        """Get screen dimensions as (width, height)."""
+        return self._dimensions
+    
+    async def screenshot(self) -> str:
+        """Take a screenshot and return as base64 string."""
+        if self._screenshot_callback:
+            screenshot = await self._screenshot_callback()
+            if isinstance(screenshot, str):
+                self._last_screenshot = screenshot
+                return screenshot
+            elif isinstance(screenshot, Image.Image):
+                # Convert PIL Image to base64
+                buffer = BytesIO()
+                screenshot.save(buffer, format="PNG")
+                screenshot_b64 = base64.b64encode(buffer.getvalue()).decode()
+                self._last_screenshot = screenshot_b64
+                return screenshot_b64
+            elif isinstance(screenshot, bytes):
+                screenshot_b64 = base64.b64encode(screenshot).decode()
+                self._last_screenshot = screenshot_b64
+                return screenshot_b64
+        
+        # Return last screenshot if available, otherwise create a blank one
+        if self._last_screenshot:
+            return self._last_screenshot
+            
+        # Create a blank screenshot as fallback
+        blank_image = Image.new('RGB', self._dimensions, color='white')
+        buffer = BytesIO()
+        blank_image.save(buffer, format="PNG")
+        screenshot_b64 = base64.b64encode(buffer.getvalue()).decode()
+        self._last_screenshot = screenshot_b64
+        return screenshot_b64
+    
+    async def click(self, x: int, y: int, button: str = "left") -> None:
+        """Click at coordinates with specified button."""
+        if self._action_callback:
+            await self._action_callback({
+                "type": "click",
+                "x": x,
+                "y": y,
+                "button": button
+            })
+    
+    async def double_click(self, x: int, y: int) -> None:
+        """Double click at coordinates."""
+        if self._action_callback:
+            await self._action_callback({
+                "type": "double_click",
+                "x": x,
+                "y": y
+            })
+    
+    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
+        """Scroll at coordinates with specified scroll amounts."""
+        if self._action_callback:
+            await self._action_callback({
+                "type": "scroll",
+                "x": x,
+                "y": y,
+                "scroll_x": scroll_x,
+                "scroll_y": scroll_y
+            })
+    
+    async def type(self, text: str) -> None:
+        """Type text."""
+        if self._action_callback:
+            await self._action_callback({
+                "type": "type",
+                "text": text
+            })
+    
+    async def wait(self, ms: int = 1000) -> None:
+        """Wait for specified milliseconds."""
+        if self._action_callback:
+            await self._action_callback({
+                "type": "wait",
+                "ms": ms
+            })
+    
+    async def move(self, x: int, y: int) -> None:
+        """Move cursor to coordinates."""
+        if self._action_callback:
+            await self._action_callback({
+                "type": "move",
+                "x": x,
+                "y": y
+            })
+    
+    async def keypress(self, keys: list[str] | str) -> None:
+        """Press key combination."""
+        if isinstance(keys, str):
+            keys = [keys]
+        if self._action_callback:
+            await self._action_callback({
+                "type": "keypress",
+                "keys": keys
+            })
+    
+    async def drag(self, path: list[dict[str, int]]) -> None:
+        """Drag along a path of points."""
+        if self._action_callback:
+            await self._action_callback({
+                "type": "drag",
+                "path": path
+            })
+
+    async def left_mouse_down(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Left mouse down at coordinates."""
+        if self._action_callback:
+            await self._action_callback({
+                "type": "left_mouse_down",
+                "x": x,
+                "y": y
+            })
+    
+    async def left_mouse_up(self, x: Optional[int] = None, y: Optional[int] = None) -> None:
+        """Left mouse up at coordinates."""
+        if self._action_callback:
+            await self._action_callback({
+                "type": "left_mouse_up",
+                "x": x,
+                "y": y
+            })
+    
+    async def get_current_url(self) -> str:
+        """Get the current URL."""
+        if self._action_callback:
+            return await self._action_callback({
+                "type": "get_current_url"
+            })
+        return ""

agent/types.py

@@ -9,7 +9,7 @@ from litellm import ResponseInputParam, ResponsesAPIResponse, ToolParam
 from collections.abc import Iterable
 
 # Agent input types
-Messages = str | ResponseInputParam
+Messages = str | ResponseInputParam | List[Dict[str, Any]]
 Tools = Optional[Iterable[ToolParam]]
 
 # Agent output types
@@ -27,55 +27,3 @@ class AgentConfigInfo(BaseModel):
     def matches_model(self, model: str) -> bool:
         """Check if this agent config matches the given model"""
         return bool(re.match(self.models_regex, model))
-
-# Computer tool interface
-class Computer(Protocol):
-    """Protocol defining the interface for computer interactions."""
-    
-    async def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]:
-        """Get the current environment type."""
-        ...
-    
-    async def get_dimensions(self) -> tuple[int, int]:
-        """Get screen dimensions as (width, height)."""
-        ...
-    
-    async def screenshot(self) -> str:
-        """Take a screenshot and return as base64 string."""
-        ...
-    
-    async def click(self, x: int, y: int, button: str = "left") -> None:
-        """Click at coordinates with specified button."""
-        ...
-    
-    async def double_click(self, x: int, y: int) -> None:
-        """Double click at coordinates."""
-        ...
-    
-    async def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None:
-        """Scroll at coordinates with specified scroll amounts."""
-        ...
-    
-    async def type(self, text: str) -> None:
-        """Type text."""
-        ...
-    
-    async def wait(self, ms: int = 1000) -> None:
-        """Wait for specified milliseconds."""
-        ...
-    
-    async def move(self, x: int, y: int) -> None:
-        """Move cursor to coordinates."""
-        ...
-    
-    async def keypress(self, keys: List[str]) -> None:
-        """Press key combination."""
-        ...
-    
-    async def drag(self, path: List[Dict[str, int]]) -> None:
-        """Drag along specified path."""
-        ...
-    
-    async def get_current_url(self) -> str:
-        """Get current URL (for browser environments)."""
-        ...

agent/ui/gradio/app.py

@@ -39,6 +39,7 @@ global_agent = None
 global_computer = None
 SETTINGS_FILE = Path(".gradio_settings.json")
 
+logging.basicConfig(level=logging.INFO)
 
 import dotenv
 if dotenv.load_dotenv():

agent/ui/gradio/ui_components.py

@@ -187,7 +187,7 @@ if __name__ == "__main__":
                     """
                     <div style="display: flex; justify-content: center; margin-bottom: 0.5em">
                         <img alt="CUA Logo" style="width: 80px;"
-                             src="https://github.com/trycua/cua/blob/main/img/logo_black.png?raw=true" />
+                             src="https://github.com/trycua/cua/blob/main/img/logo_white.png?raw=true" />
                     </div>
                     """
                 )
@@ -201,22 +201,33 @@ if __name__ == "__main__":
                     )
                     
                 with gr.Accordion("Computer Configuration", open=True):
-                    computer_os = gr.Radio(
-                        choices=["macos", "linux", "windows"],
-                        label="Operating System",
-                        value="macos",
-                        info="Select the operating system for the computer",
-                    )
-                    
                     is_windows = platform.system().lower() == "windows"
                     is_mac = platform.system().lower() == "darwin"
                     
-                    providers = ["cloud", "localhost"]
+                    providers = ["cloud", "localhost", "docker"]
                     if is_mac:
                         providers += ["lume"]
                     if is_windows:
                         providers += ["winsandbox"]
 
+                    # Remove unavailable options
+                    # MacOS is unavailable if Lume is not available
+                    # Windows is unavailable if Winsandbox is not available
+                    # Linux is always available
+                    # This should be removed once we support macOS and Windows on the cloud provider
+                    computer_choices = ["macos", "linux", "windows"]
+                    if not is_mac or "lume" not in providers:
+                        computer_choices.remove("macos")
+                    if not is_windows or "winsandbox" not in providers:
+                        computer_choices.remove("windows")
+
+                    computer_os = gr.Radio(
+                        choices=computer_choices,
+                        label="Operating System",
+                        value=computer_choices[0],
+                        info="Select the operating system for the computer",
+                    )
+                    
                     computer_provider = gr.Radio(
                         choices=providers,
                         label="Provider",

{cua_agent-0.4.12.dist-info → cua_agent-0.4.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cua-agent
-Version: 0.4.12
+Version: 0.4.13
 Summary: CUA (Computer Use) Agent for AI-driven computer interaction
 Author-Email: TryCua <gh@trycua.com>
 Requires-Python: >=3.11
@@ -28,15 +28,17 @@ Provides-Extra: uitars-hf
 Requires-Dist: accelerate; extra == "uitars-hf"
 Requires-Dist: torch; extra == "uitars-hf"
 Requires-Dist: transformers>=4.54.0; extra == "uitars-hf"
+Provides-Extra: glm45v-hf
+Requires-Dist: accelerate; extra == "glm45v-hf"
+Requires-Dist: torch; extra == "glm45v-hf"
+Requires-Dist: transformers-v4.55.0-GLM-4.5V-preview; extra == "glm45v-hf"
 Provides-Extra: ui
 Requires-Dist: gradio>=5.23.3; extra == "ui"
 Requires-Dist: python-dotenv>=1.0.1; extra == "ui"
 Provides-Extra: cli
 Requires-Dist: yaspin>=3.1.0; extra == "cli"
-Provides-Extra: glm45v-hf
-Requires-Dist: accelerate; extra == "glm45v-hf"
-Requires-Dist: torch; extra == "glm45v-hf"
-Requires-Dist: transformers-v4.55.0-GLM-4.5V-preview; extra == "glm45v-hf"
+Provides-Extra: hud
+Requires-Dist: hud-python==0.2.10; extra == "hud"
 Provides-Extra: all
 Requires-Dist: ultralytics>=8.0.0; extra == "all"
 Requires-Dist: cua-som<0.2.0,>=0.1.0; extra == "all"
@@ -47,6 +49,7 @@ Requires-Dist: transformers>=4.54.0; extra == "all"
 Requires-Dist: gradio>=5.23.3; extra == "all"
 Requires-Dist: python-dotenv>=1.0.1; extra == "all"
 Requires-Dist: yaspin>=3.1.0; extra == "all"
+Requires-Dist: hud-python==0.2.10; extra == "all"
 Description-Content-Type: text/markdown
 
 <div align="center">

{cua_agent-0.4.12.dist-info → cua_agent-0.4.13.dist-info}/RECORD

@@ -1,8 +1,9 @@
 agent/__init__.py,sha256=vWbQYgjkzIso7zILSm4OAbNU_vrmN4HyYkfX8vC-Yi0,1547
 agent/__main__.py,sha256=lBUe8Niqa5XoCjwFfXyX7GtnUwjjZXC1-j4V9mvUYSc,538
-agent/adapters/__init__.py,sha256=szM2HMten2WkcqXeRnan__-sXjpyS4eyvIW0LXSfj4U,178
-agent/adapters/huggingfacelocal_adapter.py,sha256=CT3dwJnOWItB5eTqpn5i0Y1Ec6yRjaW7zhA14Ot9gz8,8066
-agent/agent.py,sha256=PP6UvNq_QYBYuEt97Dhono7g3hz1fQlMIapSQRhw59c,27761
+agent/adapters/__init__.py,sha256=lNH6srgIMmZOI7dgicJs3LCk_1MeqLF0lou9n7b23Ts,238
+agent/adapters/huggingfacelocal_adapter.py,sha256=Uqjtcohhzd33VFh38Ra2y4Uv_lTghMswoqS1t-KKFkw,8480
+agent/adapters/human_adapter.py,sha256=xT4nnfNXb1z-vnGFlLmFEZN7TMcoMBGS40MtR1Zwv4o,13079
+agent/agent.py,sha256=mEbmN5G6y8jZ0FrlUnHfJQFE7r_GlXrHxqC93twv54k,27881
 agent/callbacks/__init__.py,sha256=yxxBXUqpXQ-jRi_ixJMtmQPxoNRy5Vz1PUBzNNa1Dwg,538
 agent/callbacks/base.py,sha256=UnnnYlh6XCm6HKZZsAPaT_Eyo9LUYLyjyNwF-QRm6Ns,4691
 agent/callbacks/budget_manager.py,sha256=RyKM-7iXQcDotYvrw3eURzeEHEXvQjID-NobtvQWE7k,1832
@@ -10,10 +11,21 @@ agent/callbacks/image_retention.py,sha256=tiuRT5ke9xXTb2eP8Gz-2ITyAMY29LURUH6Abj
 agent/callbacks/logging.py,sha256=OOxU97EzrxlnUAtiEnvy9FB7SwCUK90-rdpDFA2Ae4E,10921
 agent/callbacks/pii_anonymization.py,sha256=NEkUTUjQBi82nqus7kT-1E4RaeQ2hQrY7YCnKndLhP8,3272
 agent/callbacks/telemetry.py,sha256=PU7pkK7W1v1xjDN-9gA30lGvn4-WhqK3BPHGW3HpTOc,7497
-agent/callbacks/trajectory_saver.py,sha256=POE8aPT-MBzfW873wr6C7iiVUHtp483KwvLPxC1S3EY,11626
-agent/cli.py,sha256=cVMwUzwsLXHK9nA-dR7SFx1WwJdSnaQROLtynjEQhD8,12401
-agent/computer_handler.py,sha256=32w0Nnby3TJOSgOtceVhj46ExGIhAPSbwqIqm0HGi0A,4649
+agent/callbacks/trajectory_saver.py,sha256=VHbiDQzI_XludkWhZIVqIMrsxgwKfFWwVtqaRot_D4U,12231
+agent/cli.py,sha256=AgaXwywHd3nGQWuqMRj6SbPyFaCPjfo5980Y1ApQOTQ,12413
+agent/computers/__init__.py,sha256=39ISJsaREaQIZckpzxSuLhuR763wUU3TxUux78EKjAg,1477
+agent/computers/base.py,sha256=hZntX4vgc1ahD3EnFeb9lUjtBmgka1vb27hndPl9tKQ,2187
+agent/computers/cua.py,sha256=xp2A34kT2C1NKqSRo2GB6766gkraM-UtpFjRv8LUTSc,4889
+agent/computers/custom.py,sha256=I_CHXvczLg43c_QBk8F_WDOlaSOOzK6b-Tkum2OSRIM,8029
 agent/decorators.py,sha256=n8VvMsififWkmuk75Q7HIpo0xAA2yAeQ6J-OOiwbAKc,1836
+agent/human_tool/__init__.py,sha256=3m5_g-Fo_0yX5vi7eg-A92oTqO0N3aY929Ajp78HKsE,771
+agent/human_tool/__main__.py,sha256=VsW2BAghlonOuqZbP_xuCsaec9bemA1I_ibnDcED9D4,1068
+agent/human_tool/server.py,sha256=ceuL5kw_RjgAi8fueLU3nTjyzOLE25Shv1oTJnSHsoQ,7964
+agent/human_tool/ui.py,sha256=2Jk3Bh-Jctya8GUG-qtYbdi-1qDdwOtcAlUeiIqsoIE,26584
+agent/integrations/hud/__init__.py,sha256=1lqeM6vJAekr38l7yteLNa-Hn3R2eXCusT2FAaY8VPE,2943
+agent/integrations/hud/adapter.py,sha256=M7J71q29Ndr4xXIW7Y6H_HIlJmnp-JlKG_4zKZTuyps,4088
+agent/integrations/hud/agent.py,sha256=vXmI7OBez5lokQ9dCcgWeT8N68xfWpsWT3S36MLhdas,17264
+agent/integrations/hud/computer_handler.py,sha256=N5pVKeKW9bJ-oceYrE7IIHbx6ZrQRQnHItTGrytoHRM,6788
 agent/loops/__init__.py,sha256=Ef8aj07l3osibwDk-DTo80PrpL4_GdKRTP1ikl_b-BQ,328
 agent/loops/anthropic.py,sha256=lvDscOaOcESBWZvnjKntQRWJZ4cEaFJhSsmmFc7J1ow,69562
 agent/loops/base.py,sha256=LK7kSTnc2CB88LI7qr2VP7LMq0eS5r2bSEnrxO6IN5U,2345
@@ -26,13 +38,13 @@ agent/loops/openai.py,sha256=8Ad_XufpENmLq1nEnhzF3oswPrPK1EPz-C5NU8UOEs0,8035
 agent/loops/uitars.py,sha256=EDq8AO20lrnwB013uJoWSkkz3TVRU9oG8DQ1VviXltc,31445
 agent/responses.py,sha256=TTJ3wXN_eb0J26GKhO3cVQngOiZ1AgUPIUadozLUQyE,28991
 agent/telemetry.py,sha256=87ZTyBaT0wEPQn4v76II3g0V3GERuIVbypoX-Ug6FKQ,4786
-agent/types.py,sha256=zXev_CV9LvlYRkxzO_EmW1ZT70Z8qeGG3iHbzyYmV30,2425
+agent/types.py,sha256=ZoWY8a3GZtB8V0SnOzoI7DQy4nP_GRubxJKbuLPOc8c,840
 agent/ui/__init__.py,sha256=DTZpK85QXscXK2nM9HtpAhVBF13yAamUrtwrQSuV-kM,126
 agent/ui/__main__.py,sha256=vudWXYvGM0aNT5aZ94HPtGW8YXOZ4cLXepHyhUM_k1g,73
 agent/ui/gradio/__init__.py,sha256=yv4Mrfo-Sj2U5sVn_UJHAuwYCezo-5O4ItR2C9jzNko,145
-agent/ui/gradio/app.py,sha256=9UOPwuwspLrnHGY91zdzuRqkMH4cmwOBH-f-BC0gVC4,9077
-agent/ui/gradio/ui_components.py,sha256=hVMGZxAEq1LBHOqKj-RbDXJsj1j0Qw5dOV0ecWIHxmc,35397
-cua_agent-0.4.12.dist-info/METADATA,sha256=V2vFnsnQ8atsj-8B5aVfBp2-5Hpl2df2uPxflc5vNNo,12496
-cua_agent-0.4.12.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90
-cua_agent-0.4.12.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
-cua_agent-0.4.12.dist-info/RECORD,,
+agent/ui/gradio/app.py,sha256=m2yDd6Tua_lMMZT1zCzOty2meYEy756d8OlFF7lpdeU,9117
+agent/ui/gradio/ui_components.py,sha256=vfsqVo_COsFfw11ouMHClib9fdBf3q52G-qbuo0RyOY,36068
+cua_agent-0.4.13.dist-info/METADATA,sha256=Xr2mN1uCE8Mu5TiYskFjb6DpN-FgmihPTyNyPa665yo,12616
+cua_agent-0.4.13.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90
+cua_agent-0.4.13.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
+cua_agent-0.4.13.dist-info/RECORD,,