cua-agent 0.1.23__tar.gz → 0.1.25__tar.gz

{cua_agent-0.1.23 → cua_agent-0.1.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cua-agent
-Version: 0.1.23
+Version: 0.1.25
 Summary: CUA (Computer Use) Agent for AI-driven computer interaction
 Author-Email: TryCua <gh@trycua.com>
 Requires-Python: <3.13,>=3.10
@@ -151,12 +151,61 @@ pip install "cua-agent[ui]"
 
 # Create a simple launcher script
 ```python
+# launch_ui.py
 from agent.ui.gradio.app import create_gradio_ui
 
 app = create_gradio_ui()
 app.launch(share=False)
 ```
 
+# Run the launcher
+python launch_ui.py
+```
+
+### Setting up API Keys
+
+For the Gradio UI to show available models, you need to set API keys as environment variables:
+
+```bash
+# For OpenAI models
+export OPENAI_API_KEY=your_openai_key_here
+
+# For Anthropic models
+export ANTHROPIC_API_KEY=your_anthropic_key_here
+
+# Launch with both keys set
+OPENAI_API_KEY=your_key ANTHROPIC_API_KEY=your_key python launch_ui.py
+```
+
+### Using Local Models
+
+You can use local models with the OMNI loop provider by selecting "Custom model..." from the dropdown. The default provider URL is set to `http://localhost:1234/v1` which works with LM Studio. 
+
+If you're using a different local model server:
+- vLLM: `http://localhost:8000/v1`
+- LocalAI: `http://localhost:8080/v1`
+- Ollama with OpenAI compat API: `http://localhost:11434/v1`
+
+To change the URL, modify the `provider_base_url` in your launcher script:
+
+```python
+# In your launcher script
+from agent.ui.gradio.app import create_gradio_ui
+from agent import LLM, LLMProvider
+
+# Create a custom model with a specific URL
+custom_model = LLM(
+    provider=LLMProvider.OAICOMPAT,
+    name="your-model-name",
+    provider_base_url="http://localhost:8000/v1"  # Change to your server URL
+)
+
+app = create_gradio_ui(custom_model=custom_model)
+app.launch()
+```
+
+Without these environment variables, the UI will show "No models available" for the corresponding providers, but you can still use local models with the OMNI loop provider.
+
 The Gradio UI provides:
 - Selection of different agent loops (OpenAI, Anthropic, OMNI)
 - Model selection for each provider
@@ -169,14 +218,8 @@ You can also embed the Gradio UI in your own application:
 # Import directly in your application
 from agent.ui.gradio.app import create_gradio_ui
 
-# Create the UI with advanced features
-demo = create_gradio_ui()
-demo.launch()
-
-# Or for a simpler interface
-from agent.ui.gradio import registry
-demo = registry(name='cua:gpt-4o')
-demo.launch()
+app = create_gradio_ui()
+app.launch()
 ```
 
 ## Agent Loops
@@ -187,7 +230,7 @@ The `cua-agent` package provides three agent loops variations, based on differen
 |:-----------|:-----------------|:------------|:-------------|
 | `AgentLoop.OPENAI` | • `computer_use_preview` | Use OpenAI Operator CUA model | Not Required |
 | `AgentLoop.ANTHROPIC` | • `claude-3-5-sonnet-20240620`<br>• `claude-3-7-sonnet-20250219` | Use Anthropic Computer-Use | Not Required |
-| `AgentLoop.OMNI` | • `claude-3-5-sonnet-20240620`<br>• `claude-3-7-sonnet-20250219`<br>• `gpt-4.5-preview`<br>• `gpt-4o`<br>• `gpt-4`<br>• `phi4`<br>• `phi4-mini`<br>• `gemma3`<br>• `...`<br>• `Any Ollama-compatible model` | Use OmniParser for element pixel-detection (SoM) and any VLMs for UI Grounding and Reasoning | OmniParser |
+| `AgentLoop.OMNI` | • `claude-3-5-sonnet-20240620`<br>• `claude-3-7-sonnet-20250219`<br>• `gpt-4.5-preview`<br>• `gpt-4o`<br>• `gpt-4`<br>• `phi4`<br>• `phi4-mini`<br>• `gemma3`<br>• `...`<br>• `Any Ollama or OpenAI-compatible model` | Use OmniParser for element pixel-detection (SoM) and any VLMs for UI Grounding and Reasoning | OmniParser |
 
 ## AgentResponse
 The `AgentResponse` class represents the structured output returned after each agent turn. It contains the agent's response, reasoning, tool usage, and other metadata. The response format aligns with the new [OpenAI Agent SDK specification](https://platform.openai.com/docs/api-reference/responses) for better consistency across different agent loops.

{cua_agent-0.1.23 → cua_agent-0.1.25}/README.md

@@ -83,12 +83,61 @@ pip install "cua-agent[ui]"
 
 # Create a simple launcher script
 ```python
+# launch_ui.py
 from agent.ui.gradio.app import create_gradio_ui
 
 app = create_gradio_ui()
 app.launch(share=False)
 ```
 
+# Run the launcher
+python launch_ui.py
+```
+
+### Setting up API Keys
+
+For the Gradio UI to show available models, you need to set API keys as environment variables:
+
+```bash
+# For OpenAI models
+export OPENAI_API_KEY=your_openai_key_here
+
+# For Anthropic models
+export ANTHROPIC_API_KEY=your_anthropic_key_here
+
+# Launch with both keys set
+OPENAI_API_KEY=your_key ANTHROPIC_API_KEY=your_key python launch_ui.py
+```
+
+### Using Local Models
+
+You can use local models with the OMNI loop provider by selecting "Custom model..." from the dropdown. The default provider URL is set to `http://localhost:1234/v1` which works with LM Studio. 
+
+If you're using a different local model server:
+- vLLM: `http://localhost:8000/v1`
+- LocalAI: `http://localhost:8080/v1`
+- Ollama with OpenAI compat API: `http://localhost:11434/v1`
+
+To change the URL, modify the `provider_base_url` in your launcher script:
+
+```python
+# In your launcher script
+from agent.ui.gradio.app import create_gradio_ui
+from agent import LLM, LLMProvider
+
+# Create a custom model with a specific URL
+custom_model = LLM(
+    provider=LLMProvider.OAICOMPAT,
+    name="your-model-name",
+    provider_base_url="http://localhost:8000/v1"  # Change to your server URL
+)
+
+app = create_gradio_ui(custom_model=custom_model)
+app.launch()
+```
+
+Without these environment variables, the UI will show "No models available" for the corresponding providers, but you can still use local models with the OMNI loop provider.
+
 The Gradio UI provides:
 - Selection of different agent loops (OpenAI, Anthropic, OMNI)
 - Model selection for each provider
@@ -101,14 +150,8 @@ You can also embed the Gradio UI in your own application:
 # Import directly in your application
 from agent.ui.gradio.app import create_gradio_ui
 
-# Create the UI with advanced features
-demo = create_gradio_ui()
-demo.launch()
-
-# Or for a simpler interface
-from agent.ui.gradio import registry
-demo = registry(name='cua:gpt-4o')
-demo.launch()
+app = create_gradio_ui()
+app.launch()
 ```
 
 ## Agent Loops
@@ -119,7 +162,7 @@ The `cua-agent` package provides three agent loops variations, based on differen
 |:-----------|:-----------------|:------------|:-------------|
 | `AgentLoop.OPENAI` | • `computer_use_preview` | Use OpenAI Operator CUA model | Not Required |
 | `AgentLoop.ANTHROPIC` | • `claude-3-5-sonnet-20240620`<br>• `claude-3-7-sonnet-20250219` | Use Anthropic Computer-Use | Not Required |
-| `AgentLoop.OMNI` | • `claude-3-5-sonnet-20240620`<br>• `claude-3-7-sonnet-20250219`<br>• `gpt-4.5-preview`<br>• `gpt-4o`<br>• `gpt-4`<br>• `phi4`<br>• `phi4-mini`<br>• `gemma3`<br>• `...`<br>• `Any Ollama-compatible model` | Use OmniParser for element pixel-detection (SoM) and any VLMs for UI Grounding and Reasoning | OmniParser |
+| `AgentLoop.OMNI` | • `claude-3-5-sonnet-20240620`<br>• `claude-3-7-sonnet-20250219`<br>• `gpt-4.5-preview`<br>• `gpt-4o`<br>• `gpt-4`<br>• `phi4`<br>• `phi4-mini`<br>• `gemma3`<br>• `...`<br>• `Any Ollama or OpenAI-compatible model` | Use OmniParser for element pixel-detection (SoM) and any VLMs for UI Grounding and Reasoning | OmniParser |
 
 ## AgentResponse
 The `AgentResponse` class represents the structured output returned after each agent turn. It contains the agent's response, reasoning, tool usage, and other metadata. The response format aligns with the new [OpenAI Agent SDK specification](https://platform.openai.com/docs/api-reference/responses) for better consistency across different agent loops.

{cua_agent-0.1.23 → cua_agent-0.1.25}/agent/core/types.py

@@ -54,23 +54,6 @@ LLMModel = LLM
 Model = LLM
 
 
-# Default models for each provider
-PROVIDER_TO_DEFAULT_MODEL: Dict[LLMProvider, str] = {
-    LLMProvider.ANTHROPIC: "claude-3-7-sonnet-20250219",
-    LLMProvider.OPENAI: "gpt-4o",
-    LLMProvider.OLLAMA: "gemma3:4b-it-q4_K_M",
-    LLMProvider.OAICOMPAT: "Qwen2.5-VL-7B-Instruct",
-}
-
-# Environment variable names for each provider
-PROVIDER_TO_ENV_VAR: Dict[LLMProvider, str] = {
-    LLMProvider.ANTHROPIC: "ANTHROPIC_API_KEY",
-    LLMProvider.OPENAI: "OPENAI_API_KEY",
-    LLMProvider.OLLAMA: "none",
-    LLMProvider.OAICOMPAT: "none",
-}
-
-
 class AgentResponse(TypedDict, total=False):
     """Agent response format."""
 

{cua_agent-0.1.23 → cua_agent-0.1.25}/agent/providers/omni/loop.py

@@ -443,6 +443,8 @@ class OmniLoop(BaseLoop):
                     except (json.JSONDecodeError, IndexError):
                         try:
                             # Look for JSON object pattern
+                            import re  # Local import to ensure availability
+
                             json_pattern = r"\{[^}]+\}"
                             json_match = re.search(json_pattern, raw_text)
                             if json_match:
@@ -453,8 +455,20 @@ class OmniLoop(BaseLoop):
                                 logger.error(f"No JSON found in content")
                                 return True, action_screenshot_saved
                         except json.JSONDecodeError as e:
-                            logger.error(f"Failed to parse JSON from text: {str(e)}")
-                            return True, action_screenshot_saved
+                            # Try to sanitize the JSON string and retry
+                            try:
+                                # Remove or replace invalid control characters
+                                import re  # Local import to ensure availability
+
+                                sanitized_text = re.sub(r"[\x00-\x1F\x7F]", "", raw_text)
+                                # Try parsing again with sanitized text
+                                parsed_content = json.loads(sanitized_text)
+                                logger.info(
+                                    "Successfully parsed JSON after sanitizing control characters"
+                                )
+                            except json.JSONDecodeError:
+                                logger.error(f"Failed to parse JSON from text: {str(e)}")
+                                return True, action_screenshot_saved
 
             # Step 4: Process the parsed content if available
             if parsed_content:

{cua_agent-0.1.23 → cua_agent-0.1.25}/agent/ui/gradio/app.py

@@ -271,16 +271,19 @@ def create_agent(
             api_key = os.environ.get("ANTHROPIC_API_KEY", "")
 
     # Create LLM model object with appropriate parameters
-    provider_base_url = "http://localhost:8000/v1" if use_oaicompat else None
+    provider_base_url = "http://localhost:1234/v1" if use_oaicompat else None
 
     if use_oaicompat:
-        # Special handling for OAICOMPAT - use OPENAI provider with custom base URL
-        print(f"DEBUG - Creating OAICOMPAT agent with model: {model_name}")
+        # Special handling for OAICOMPAT - use OAICOMPAT provider with custom base URL
+        print(
+            f"DEBUG - Creating OAICOMPAT agent with model: {model_name}, URL: {provider_base_url}"
+        )
         llm = LLM(
-            provider=provider,  # Already set to OPENAI
+            provider=LLMProvider.OAICOMPAT,  # Set to OAICOMPAT instead of using original provider
             name=model_name,
             provider_base_url=provider_base_url,
         )
+        print(f"DEBUG - LLM provider is now: {llm.provider}, base URL: {llm.provider_base_url}")
         # Note: Don't pass use_oaicompat to the agent, as it doesn't accept this parameter
     elif provider == LLMProvider.OAICOMPAT:
         # This path is unlikely to be taken with our current approach
@@ -461,8 +464,10 @@ def respond(
         # Special handling for OAICOMPAT to bypass provider-specific errors
         # Creates the agent with OPENAI provider but using custom model name and provider base URL
         is_oaicompat = str(provider) == "oaicompat"
-        if is_oaicompat:
-            provider = LLMProvider.OPENAI
+
+        # Don't override the provider for OAICOMPAT - instead pass it through
+        # if is_oaicompat:
+        #    provider = LLMProvider.OPENAI
 
         # Get API key based on provider
         if provider == LLMProvider.OPENAI:

{cua_agent-0.1.23 → cua_agent-0.1.25}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "cua-agent"
-version = "0.1.23"
+version = "0.1.25"
 description = "CUA (Computer Use) Agent for AI-driven computer interaction"
 readme = "README.md"
 authors = [
@@ -105,7 +105,7 @@ target-version = [
 
 [tool.ruff]
 line-length = 100
-target-version = "0.1.23"
+target-version = "0.1.25"
 select = [
     "E",
     "F",
@@ -119,7 +119,7 @@ docstring-code-format = true
 
 [tool.mypy]
 strict = true
-python_version = "0.1.23"
+python_version = "0.1.25"
 ignore_missing_imports = true
 disallow_untyped_defs = true
 check_untyped_defs = true