cua-computer 0.1.22__tar.gz → 0.1.24__tar.gz

{cua_computer-0.1.22 → cua_computer-0.1.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cua-computer
-Version: 0.1.22
+Version: 0.1.24
 Summary: Computer-Use Interface (CUI) framework powering Cua
 Author-Email: TryCua <gh@trycua.com>
 Requires-Python: >=3.10
@@ -11,6 +11,9 @@ Requires-Dist: websockets>=12.0
 Requires-Dist: aiohttp>=3.9.0
 Requires-Dist: cua-core<0.2.0,>=0.1.0
 Requires-Dist: pydantic>=2.11.1
+Provides-Extra: ui
+Requires-Dist: gradio<6.0.0,>=5.23.3; extra == "ui"
+Requires-Dist: python-dotenv<2.0.0,>=1.0.1; extra == "ui"
 Description-Content-Type: text/markdown
 
 <div align="center">
@@ -78,4 +81,82 @@ The `cua-computer` PyPi package pulls automatically the latest executable versio
 
 Refer to this notebook for a step-by-step guide on how to use the Computer-Use Interface (CUI):
 
-- [Computer-Use Interface (CUI)](../../notebooks/computer_nb.ipynb)
+- [Computer-Use Interface (CUI)](../../notebooks/computer_nb.ipynb)
+
+## Using the Gradio Computer UI
+
+The computer module includes a Gradio UI for creating and sharing demonstration data. We make it easy for people to build community datasets for better computer use models with an upload to Huggingface feature.
+
+```bash
+# Install with UI support
+pip install "cua-computer[ui]"
+```
+
+> **Note:** For precise control of the computer, we recommend using VNC or Screen Sharing instead of the Computer Gradio UI.
+
+### Building and Sharing Demonstrations with Huggingface
+
+Follow these steps to contribute your own demonstrations:
+
+#### 1. Set up Huggingface Access
+
+Set your HF_TOKEN in a .env file or in your environment variables:
+
+```bash
+# In .env file
+HF_TOKEN=your_huggingface_token
+```
+
+#### 2. Launch the Computer UI
+
+```python
+# launch_ui.py
+from computer.ui.gradio.app import create_gradio_ui
+from dotenv import load_dotenv
+load_dotenv('.env')
+
+app = create_gradio_ui()
+app.launch(share=False)
+```
+
+For examples, see [Computer UI Examples](../../examples/computer_ui_examples.py)
+
+#### 3. Record Your Tasks
+
+<details open>
+<summary>View demonstration video</summary>
+<video src="https://github.com/user-attachments/assets/de3c3477-62fe-413c-998d-4063e48de176" controls width="600"></video>
+</details>
+
+Record yourself performing various computer tasks using the UI.
+
+#### 4. Save Your Demonstrations
+
+<details open>
+<summary>View demonstration video</summary>
+<video src="https://github.com/user-attachments/assets/5ad1df37-026a-457f-8b49-922ae805faef" controls width="600"></video>
+</details>
+
+Save each task by picking a descriptive name and adding relevant tags (e.g., "office", "web-browsing", "coding").
+
+#### 5. Record Additional Demonstrations
+
+Repeat steps 3 and 4 until you have a good amount of demonstrations covering different tasks and scenarios.
+
+#### 6. Upload to Huggingface
+
+<details open>
+<summary>View demonstration video</summary>
+<video src="https://github.com/user-attachments/assets/c586d460-3877-4b5f-a736-3248886d2134" controls width="600"></video>
+</details>
+
+Upload your dataset to Huggingface by:
+- Naming it as `{your_username}/{dataset_name}`
+- Choosing public or private visibility
+- Optionally selecting specific tags to upload only tasks with certain tags
+
+#### Examples and Resources
+
+- Example Dataset: [ddupont/test-dataset](https://huggingface.co/datasets/ddupont/test-dataset)
+- Find Community Datasets: 🔍 [Browse CUA Datasets on Huggingface](https://huggingface.co/datasets?other=cua)
+

{cua_computer-0.1.22 → cua_computer-0.1.24}/README.md

@@ -63,4 +63,82 @@ The `cua-computer` PyPi package pulls automatically the latest executable versio
 
 Refer to this notebook for a step-by-step guide on how to use the Computer-Use Interface (CUI):
 
-- [Computer-Use Interface (CUI)](../../notebooks/computer_nb.ipynb)
+- [Computer-Use Interface (CUI)](../../notebooks/computer_nb.ipynb)
+
+## Using the Gradio Computer UI
+
+The computer module includes a Gradio UI for creating and sharing demonstration data. We make it easy for people to build community datasets for better computer use models with an upload to Huggingface feature.
+
+```bash
+# Install with UI support
+pip install "cua-computer[ui]"
+```
+
+> **Note:** For precise control of the computer, we recommend using VNC or Screen Sharing instead of the Computer Gradio UI.
+
+### Building and Sharing Demonstrations with Huggingface
+
+Follow these steps to contribute your own demonstrations:
+
+#### 1. Set up Huggingface Access
+
+Set your HF_TOKEN in a .env file or in your environment variables:
+
+```bash
+# In .env file
+HF_TOKEN=your_huggingface_token
+```
+
+#### 2. Launch the Computer UI
+
+```python
+# launch_ui.py
+from computer.ui.gradio.app import create_gradio_ui
+from dotenv import load_dotenv
+load_dotenv('.env')
+
+app = create_gradio_ui()
+app.launch(share=False)
+```
+
+For examples, see [Computer UI Examples](../../examples/computer_ui_examples.py)
+
+#### 3. Record Your Tasks
+
+<details open>
+<summary>View demonstration video</summary>
+<video src="https://github.com/user-attachments/assets/de3c3477-62fe-413c-998d-4063e48de176" controls width="600"></video>
+</details>
+
+Record yourself performing various computer tasks using the UI.
+
+#### 4. Save Your Demonstrations
+
+<details open>
+<summary>View demonstration video</summary>
+<video src="https://github.com/user-attachments/assets/5ad1df37-026a-457f-8b49-922ae805faef" controls width="600"></video>
+</details>
+
+Save each task by picking a descriptive name and adding relevant tags (e.g., "office", "web-browsing", "coding").
+
+#### 5. Record Additional Demonstrations
+
+Repeat steps 3 and 4 until you have a good amount of demonstrations covering different tasks and scenarios.
+
+#### 6. Upload to Huggingface
+
+<details open>
+<summary>View demonstration video</summary>
+<video src="https://github.com/user-attachments/assets/c586d460-3877-4b5f-a736-3248886d2134" controls width="600"></video>
+</details>
+
+Upload your dataset to Huggingface by:
+- Naming it as `{your_username}/{dataset_name}`
+- Choosing public or private visibility
+- Optionally selecting specific tags to upload only tasks with certain tags
+
+#### Examples and Resources
+
+- Example Dataset: [ddupont/test-dataset](https://huggingface.co/datasets/ddupont/test-dataset)
+- Find Community Datasets: 🔍 [Browse CUA Datasets on Huggingface](https://huggingface.co/datasets?other=cua)
+

{cua_computer-0.1.22 → cua_computer-0.1.24}/computer/computer.py

@@ -227,24 +227,18 @@ class Computer:
                         self.logger.error(f"Failed to initialize PyLume context: {e}")
                         raise RuntimeError(f"Failed to initialize PyLume: {e}")
 
-                # Try to get the VM, if it doesn't exist, create it and pull the image
+                # Try to get the VM, if it doesn't exist, return an error
                 try:
                     vm = await self.config.pylume.get_vm(self.config.name)  # type: ignore[attr-defined]
                     self.logger.verbose(f"Found existing VM: {self.config.name}")
                 except Exception as e:
-                    self.logger.verbose(f"VM not found, pulling image: {e}")
-                    image_ref = ImageRef(
-                        image=self.config.image,
-                        tag=self.config.tag,
-                        registry="ghcr.io",
-                        organization="trycua",
+                    self.logger.error(f"VM not found: {self.config.name}")
+                    self.logger.error(
+                        f"Please pull the VM first with lume pull macos-sequoia-cua-sparse:latest: {e}"
+                    )
+                    raise RuntimeError(
+                        f"VM not found: {self.config.name}. Please pull the VM first."
                     )
-                    self.logger.info(f"Pulling image {self.config.image}:{self.config.tag}...")
-                    try:
-                        await self.config.pylume.pull_image(image_ref, name=self.config.name)  # type: ignore[attr-defined]
-                    except Exception as pull_error:
-                        self.logger.error(f"Failed to pull image: {pull_error}")
-                        raise RuntimeError(f"Failed to pull VM image: {pull_error}")
 
                 # Convert paths to SharedDirectory objects
                 shared_directories = []

{cua_computer-0.1.22 → cua_computer-0.1.24}/computer/interface/macos.py

@@ -377,17 +377,47 @@ class MacOSComputerInterface(BaseComputerInterface):
         """
         await self.press(key)
 
-    async def hotkey(self, *keys: str) -> None:
-        await self._send_command("hotkey", {"keys": list(keys)})
+    async def hotkey(self, *keys: "KeyType") -> None:
+        """Press multiple keys simultaneously.
+
+        Args:
+            *keys: Multiple keys to press simultaneously. Each key can be any of:
+                - A Key enum value (recommended), e.g. Key.COMMAND
+                - A direct key value string, e.g. 'command'
+                - A single character string, e.g. 'a'
+
+        Examples:
+            ```python
+            # Using enums (recommended)
+            await interface.hotkey(Key.COMMAND, Key.C)  # Copy
+            await interface.hotkey(Key.COMMAND, Key.V)  # Paste
+
+            # Using mixed formats
+            await interface.hotkey(Key.COMMAND, 'a')  # Select all
+            ```
+
+        Raises:
+            ValueError: If any key type is invalid or not recognized
+        """
+        actual_keys = []
+        for key in keys:
+            if isinstance(key, Key):
+                actual_keys.append(key.value)
+            elif isinstance(key, str):
+                # Try to convert to enum if it matches a known key
+                key_or_enum = Key.from_string(key)
+                actual_keys.append(key_or_enum.value if isinstance(key_or_enum, Key) else key_or_enum)
+            else:
+                raise ValueError(f"Invalid key type: {type(key)}. Must be Key enum or string.")
+        
+        await self._send_command("hotkey", {"keys": actual_keys})
 
     # Scrolling Actions
     async def scroll_down(self, clicks: int = 1) -> None:
-        for _ in range(clicks):
-            await self.hotkey("pagedown")
-
+        await self._send_command("scroll_down", {"clicks": clicks})
+        
     async def scroll_up(self, clicks: int = 1) -> None:
-        for _ in range(clicks):
-            await self.hotkey("pageup")
+        await self._send_command("scroll_up", {"clicks": clicks})
 
     # Screen Actions
     async def screenshot(

{cua_computer-0.1.22 → cua_computer-0.1.24}/computer/interface/models.py

@@ -7,6 +7,9 @@ NavigationKey = Literal['pagedown', 'pageup', 'home', 'end', 'left', 'right', 'u
 # Special key literals
 SpecialKey = Literal['enter', 'esc', 'tab', 'space', 'backspace', 'del']
 
+# Modifier key literals
+ModifierKey = Literal['ctrl', 'alt', 'shift', 'win', 'command', 'option']
+
 # Function key literals
 FunctionKey = Literal['f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12']
 
@@ -35,6 +38,14 @@ class Key(Enum):
     BACKSPACE = 'backspace'
     DELETE = 'del'
     
+    # Modifier keys
+    ALT = 'alt'
+    CTRL = 'ctrl'
+    SHIFT = 'shift'
+    WIN = 'win'
+    COMMAND = 'command'
+    OPTION = 'option'
+    
     # Function keys
     F1 = 'f1'
     F2 = 'f2'
@@ -73,14 +84,27 @@ class Key(Enum):
             'escape': cls.ESCAPE,
             'esc': cls.ESC,
             'delete': cls.DELETE,
-            'del': cls.DELETE
+            'del': cls.DELETE,
+            # Modifier key mappings
+            'alt': cls.ALT,
+            'ctrl': cls.CTRL,
+            'control': cls.CTRL,
+            'shift': cls.SHIFT,
+            'win': cls.WIN,
+            'windows': cls.WIN,
+            'super': cls.WIN,
+            'command': cls.COMMAND,
+            'cmd': cls.COMMAND,
+            '⌘': cls.COMMAND,
+            'option': cls.OPTION,
+            '⌥': cls.OPTION,
         }
         
         normalized = key.lower().strip()
         return key_mapping.get(normalized, key)
 
 # Combined key type
-KeyType = Union[Key, NavigationKey, SpecialKey, FunctionKey, str]
+KeyType = Union[Key, NavigationKey, SpecialKey, ModifierKey, FunctionKey, str]
 
 class AccessibilityWindow(TypedDict):
     """Information about a window in the accessibility tree."""

cua_computer-0.1.24/computer/ui/__init__.py

@@ -0,0 +1 @@
+"""UI modules for the Computer Interface."""

cua_computer-0.1.24/computer/ui/gradio/__init__.py

@@ -0,0 +1,6 @@
+"""Gradio UI for Computer UI."""
+
+import gradio as gr
+from typing import Optional
+
+from .app import create_gradio_ui