cua-agent 0.4.17__py3-none-any.whl → 0.4.18__py3-none-any.whl

agent/integrations/hud/__init__.py

@@ -1,77 +1,228 @@
-"""HUD integration for ComputerAgent."""
-
-import logging
-from typing import Any, Optional, Dict
-from hud import run_job as hud_run_job
-
-from .agent import ComputerAgent
-from .adapter import ComputerAgentAdapter
-from .computer_handler import HUDComputerHandler
-
-
-async def run_job(
-    model: str,
-    task_or_taskset: Any,
-    job_name: str,
-    # Job kwargs
-    auto_reply_question: bool = False,
-    adapter_cls: Any = None,
-    adapter_kwargs: Optional[Dict[str, Any]] = None,
-    max_steps_per_task: int = 20,
-    run_parallel: bool = True,
-    job_metadata: Optional[Dict[str, Any]] = None,
-    show_progress: bool = True,
-    max_concurrent_env_creations: Optional[int] = 30,  # Limits gym.make calls
-    max_concurrent_agent_predictions: Optional[int] = None,  # No limit on LLM calls
-    max_concurrent_tasks: Optional[int] = 30,  # Limits overall task concurrency
-    **agent_kwargs: Any
-) -> Any:
-    """
-    Run a job using ComputerAgent with the specified model.
-    
-    Args:
-        model: Model string for ComputerAgent (e.g., "anthropic/claude-3-5-sonnet-20241022")
-        task_or_taskset: Task or TaskSet to run
-        job_name: Name for the job
-        auto_reply_question: Whether to auto-reply to questions
-        adapter_cls: Custom adapter class (defaults to ComputerAgentAdapter)
-        adapter_kwargs: Additional kwargs for the adapter
-        max_steps_per_task: Maximum steps per task
-        run_parallel: Whether to run tasks in parallel
-        job_metadata: Additional metadata for the job
-        show_progress: Whether to show progress
-        max_concurrent_env_creations: Max concurrent environment creations
-        max_concurrent_agent_predictions: Max concurrent agent predictions
-        max_concurrent_tasks: Max concurrent tasks
-        **agent_kwargs: Additional kwargs to pass to ComputerAgent
-    
-    Returns:
-        Job instance from HUD
+"""HUD integration: Generic HuggingFace dataset evaluation runner (CUA proxy).
+
+This module exposes two helpers to evaluate HUD-compatible datasets using
+HUD's OperatorAgent, while proxying model calls through our ComputerAgent via
+`FakeAsyncOpenAI` (see `agent/integrations/hud/agent.py`).
+
+Exports:
+- run_single_task(dataset_name, *, agent_type="cua-proxy", model=None, allowed_tools=None)
+- run_full_dataset(dataset_name, *, agent_type="cua-proxy", model=None, allowed_tools=None, max_concurrent=30, max_steps=50)
+"""
+import time
+from typing import Any, Optional
+
+from PIL import Image
+from datasets import load_dataset, Dataset
+from hud.agents import OperatorAgent
+from hud.datasets import Task, run_dataset
+from hud.tools.computer.settings import computer_settings
+from hud import trace
+
+from agent.agent import ComputerAgent as BaseComputerAgent
+from .proxy import FakeAsyncOpenAI
+
+
+# ---------------------------------------------------------------------------
+# Proxy OperatorAgent
+# ---------------------------------------------------------------------------
+
+
+class ProxyOperatorAgent(OperatorAgent):
+    """OperatorAgent that proxies model calls through our ComputerAgent.
+
+    Accepts the same config keys we pass via hud.run_dataset `agent_config`:
+    - model: str | None
+    - allowed_tools: list[str] | None
+    Additional kwargs are forwarded to OperatorAgent (if any are supported).
     """
-    # combine verbose and verbosity kwargs
-    if "verbose" in agent_kwargs:
-        agent_kwargs["verbosity"] = logging.INFO
-        del agent_kwargs["verbose"]
-    verbose = True if agent_kwargs.get("verbosity", logging.WARNING) > logging.INFO else False
+
+    def __init__(
+        self,
+        *,
+        model: str | None = None,
+        allowed_tools: list[str] | None = None,
+        trajectory_dir: str | None = None,
+        # === ComputerAgent kwargs ===
+        tools: list[Any] | None = None,
+        custom_loop: Any | None = None,
+        only_n_most_recent_images: int | None = None,
+        callbacks: list[Any] | None = None,
+        verbosity: int | None = None,
+        max_retries: int | None = 3,
+        screenshot_delay: float | int = 0.5,
+        use_prompt_caching: bool | None = False,
+        max_trajectory_budget: float | dict | None = None,
+        telemetry_enabled: bool | None = True,
+        **kwargs: Any,
+    ) -> None:
+        model = model or "computer-use-preview"
+        allowed_tools = allowed_tools or ["openai_computer"]
+        
+        computer_shim = {
+            'screenshot': lambda: Image.new('RGB', (computer_settings.OPENAI_COMPUTER_WIDTH, computer_settings.OPENAI_COMPUTER_HEIGHT)),
+            'environment': 'linux',
+            'dimensions': (computer_settings.OPENAI_COMPUTER_WIDTH, computer_settings.OPENAI_COMPUTER_HEIGHT)
+        }
+        # Build tools ensuring the computer_shim is included
+        agent_tools: list[Any] = [computer_shim]
+        if tools:
+            agent_tools.extend(tools)
+
+        computer_agent = BaseComputerAgent(
+            model=model,
+            tools=agent_tools,
+            custom_loop=custom_loop,
+            only_n_most_recent_images=only_n_most_recent_images,
+            callbacks=callbacks,
+            verbosity=verbosity,
+            trajectory_dir=trajectory_dir,
+            max_retries=max_retries,
+            screenshot_delay=screenshot_delay,
+            use_prompt_caching=use_prompt_caching,
+            max_trajectory_budget=max_trajectory_budget,
+            telemetry_enabled=telemetry_enabled,
+        )
+        model_client = FakeAsyncOpenAI(computer_agent)
+
+        super().__init__( 
+            model_client=model_client, # type: ignore[arg-type]
+            model=model,
+            allowed_tools=allowed_tools,
+            **kwargs,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Single-task runner
+# ---------------------------------------------------------------------------
+
+
+async def run_single_task(
+    dataset: str | Dataset | list[dict[str, Any]],
+    *,
+    task_id: int = 0,
+    model: str | None = None,
+    allowed_tools: list[str] | None = None,
+    # === ComputerAgent kwargs ===
+    tools: list[Any] | None = None,
+    custom_loop: Any | None = None,
+    only_n_most_recent_images: int | None = None,
+    callbacks: list[Any] | None = None,
+    verbosity: int | None = None,
+    trajectory_dir: str | None = None,
+    max_retries: int | None = 3,
+    screenshot_delay: float | int = 0.5,
+    use_prompt_caching: bool | None = False,
+    max_trajectory_budget: float | dict | None = None,
+    telemetry_enabled: bool | None = True,
+) -> None:
+    """Load one task from the dataset and execute it with Operator+CUA proxy."""
+
+    # Load dataset and pick a sample
+    if isinstance(dataset, str):
+        dataset = load_dataset(dataset, split="train") # type: ignore[arg-type]
+    elif isinstance(dataset, list):
+        dataset = dataset
+    else:
+        dataset = dataset["train"]
     
-    # run job
-    return await hud_run_job(
-        agent_cls=ComputerAgent,
-        agent_kwargs={"model": model, **agent_kwargs},
-        task_or_taskset=task_or_taskset,
-        job_name=job_name,
-        auto_reply_question=auto_reply_question,
-        adapter_cls=adapter_cls,
-        adapter_kwargs=adapter_kwargs,
-        max_steps_per_task=max_steps_per_task,
-        run_parallel=run_parallel,
-        job_metadata=job_metadata,
-        show_progress=show_progress,
-        verbose=verbose,
-        max_concurrent_env_creations=max_concurrent_env_creations,
-        max_concurrent_agent_predictions=max_concurrent_agent_predictions,
-        max_concurrent_tasks=max_concurrent_tasks
+    sample_task = dataset[task_id]  # type: ignore[index]
+    task_prompt = sample_task.get("prompt", f"Task {sample_task.get('id', 0)}")  # type: ignore[attr-defined]
+
+    with trace(name=task_prompt):
+        task = Task(**sample_task)  # type: ignore[arg-type]
+
+        agent = ProxyOperatorAgent(
+            model=model,
+            allowed_tools=allowed_tools,
+            # === ComputerAgent kwargs passthrough ===
+            tools=tools,
+            custom_loop=custom_loop,
+            only_n_most_recent_images=only_n_most_recent_images,
+            callbacks=callbacks,
+            verbosity=verbosity,
+            trajectory_dir=trajectory_dir,
+            max_retries=max_retries,
+            screenshot_delay=screenshot_delay,
+            use_prompt_caching=use_prompt_caching,
+            max_trajectory_budget=max_trajectory_budget,
+            telemetry_enabled=telemetry_enabled,
+        )
+        print(f"Running: {task_prompt}")
+        result = await agent.run(task, max_steps=10)
+        print(f"✅ Reward: {getattr(result, 'reward')}")
+
+
+# ---------------------------------------------------------------------------
+# Full-dataset runner
+# ---------------------------------------------------------------------------
+
+
+async def run_full_dataset(
+    dataset: str | Dataset | list[dict[str, Any]],
+    *,
+    job_name: Optional[str] = None,
+    model: str | None = None,
+    allowed_tools: list[str] | None = None,
+    max_concurrent: int = 30,
+    max_steps: int = 50,
+    split: str = "train",
+    trajectory_dir: str | None = None,
+    # === ComputerAgent kwargs ===
+    tools: list[Any] | None = None,
+    custom_loop: Any | None = None,
+    only_n_most_recent_images: int | None = 5,
+    callbacks: list[Any] | None = None,
+    verbosity: int | None = None,
+    max_retries: int | None = 3,
+    screenshot_delay: float | int = 0.5,
+    use_prompt_caching: bool | None = False,
+    max_trajectory_budget: float | dict | None = None,
+    telemetry_enabled: bool | None = True,
+) -> list[Any]:
+    """Run evaluation across the entire dataset using hud.datasets.run_dataset."""
+
+    # We pass OperatorAgent as the class and provide a config that injects our
+    # FakeAsyncOpenAI per agent instantiation.
+
+    if isinstance(dataset, str):
+        dataset_name = dataset.split('/')[-1]
+        job_name = job_name or f"Evaluation {dataset_name}"
+        dataset = load_dataset(dataset, split=split) # type: ignore[arg-type]
+    else:
+        dataset_name = "custom"
+        job_name = job_name or f"Evaluation {time.strftime('%H:%M %Y-%m-%d')}"
+
+    # Execute evaluation
+    return await run_dataset(
+        name=job_name,
+        dataset=dataset,
+        agent_class=ProxyOperatorAgent,
+        agent_config={
+            "model": model,
+            "allowed_tools": allowed_tools,
+            "trajectory_dir": trajectory_dir,
+            # === ComputerAgent kwargs passthrough ===
+            "tools": tools,
+            "custom_loop": custom_loop,
+            "only_n_most_recent_images": only_n_most_recent_images,
+            "callbacks": callbacks,
+            "verbosity": verbosity,
+            "max_retries": max_retries,
+            "screenshot_delay": screenshot_delay,
+            "use_prompt_caching": use_prompt_caching,
+            "max_trajectory_budget": max_trajectory_budget,
+            "telemetry_enabled": telemetry_enabled,
+        },
+        max_concurrent=max_concurrent,
+        metadata={"dataset": dataset_name},
+        max_steps=max_steps,
+        auto_respond=True,
     )
 
 
-__all__ = ["ComputerAgent", "ComputerAgentAdapter", "HUDComputerHandler", "run_job"]
+__all__ = [
+    "run_single_task",
+    "run_full_dataset",
+    "ProxyOperatorAgent",
+]

agent/integrations/hud/proxy.py

@@ -0,0 +1,183 @@
+"""HUD ComputerAgent wrapper and Fake AsyncOpenAI client.
+
+Provides FakeAsyncOpenAI that adapts our ComputerAgent to the OpenAI Responses
+interface needed by HUD's OperatorAgent. It implements only `responses.create`
+and returns an OpenAI Response object with `id` and `output` fields, where `output` is a list of
+OpenAI-like response blocks. We intentionally only support a single-step call
+by consuming the first yielded result from `ComputerAgent.run()`.
+"""
+
+import traceback
+import time
+import uuid
+from typing import Any, Dict, List, Optional
+
+from agent.agent import ComputerAgent as BaseComputerAgent
+
+# OpenAI Responses typed models (required)
+from openai.types.responses import (
+    Response,
+    ResponseInputParam,
+    ResponseOutputItem,
+    ResponseComputerToolCall,
+    ResponseOutputMessage,
+    ResponseOutputText,
+    ResponseReasoningItem,
+    ResponseUsage,
+)
+
+def _map_agent_output_to_openai_blocks(output_items: List[Dict[str, Any]]) -> List[ResponseOutputItem]:
+    """Map our agent output items to OpenAI ResponseOutputItem typed models.
+
+    Only a subset is supported: computer_call, assistant message (text), and reasoning.
+    Unknown types are ignored.
+    """
+    blocks: List[ResponseOutputItem] = []
+    for item in output_items or []:
+        t = item.get("type")
+        if t == "computer_call":
+            comp = ResponseComputerToolCall.model_validate({
+                "id": item.get("id") or f"cu_{uuid.uuid4().hex}",
+                "type": "computer_call",
+                "call_id": item["call_id"],
+                "action": item["action"],
+                "pending_safety_checks": item.get("pending_safety_checks", []),
+                "status": "completed",
+            })
+            blocks.append(comp)
+            # we will exit early here as the responses api only supports a single step
+            break
+        elif t == "message" and item.get("role") == "assistant":
+            content_blocks: List[ResponseOutputText] = []
+            for c in item.get("content", []) or []:
+                content_blocks.append(
+                    ResponseOutputText.model_validate({
+                        "type": "output_text",
+                        "text": c["text"],
+                        "annotations": [],
+                    })
+                )
+            if content_blocks:
+                msg = ResponseOutputMessage.model_validate({
+                    "id": item.get("id") or f"msg_{uuid.uuid4()}",
+                    "type": "message",
+                    "role": "assistant",
+                    "status": "completed",
+                    "content": [ct.model_dump() for ct in content_blocks],
+                })
+                blocks.append(msg)
+        elif t == "reasoning":
+            reasoning = ResponseReasoningItem.model_validate({
+                "id": item.get("id") or f"rsn_{uuid.uuid4()}",
+                "type": "reasoning",
+                "summary": item["summary"],
+            })
+            blocks.append(reasoning)
+        # Unhandled types are ignored
+    return blocks
+
+def _to_plain_dict_list(items: Any) -> List[Dict[str, Any]]:
+    out: List[Dict[str, Any]] = []
+    for it in list(items):
+        if hasattr(it, "model_dump"):
+            out.append(it.model_dump())  # type: ignore[attr-defined]
+        elif isinstance(it, dict):
+            out.append(it)
+        else:
+            # Strict: rely on default __dict__ if present
+            out.append(dict(it))  # may raise if not mapping
+    return out
+
+class FakeAsyncOpenAI:
+    """Minimal fake OpenAI client with only `responses.create` implemented.
+
+    It uses a provided `ComputerAgent` instance to produce a single-step
+    response compatible with HUD's OperatorAgent loop.
+    """
+
+    def __init__(self, computer_agent: BaseComputerAgent) -> None:
+        self._agent = computer_agent
+        self.responses = self._Responses(self)
+
+    class _Responses:
+        def __init__(self, parent: "FakeAsyncOpenAI") -> None:
+            # Caches for cross-call context when using previous_response_id
+            self.blocks_cache: Dict[str, ResponseInputParam | ResponseOutputItem] = {}
+            self.context_cache: Dict[str, List[str]] = {}
+            self.agent = parent._agent
+
+        async def create(
+            self,
+            *,
+            model: str,
+            input: ResponseInputParam,
+            tools: Optional[List[Dict[str, Any]]] = None,
+            instructions: Optional[str] = None,
+            previous_response_id: Optional[str] = None,
+            max_retries: int = 5,
+            **_: Any,
+        ) -> Any:
+            for attempt in range(max_retries):
+                # Prepend cached blocks from previous_response_id to input
+                full_input = input
+                if previous_response_id is not None:
+                    prev_block_ids = self.context_cache[previous_response_id]
+                    prev_blocks = [self.blocks_cache[b_id] for b_id in prev_block_ids]
+                    full_input = _to_plain_dict_list(prev_blocks + input)
+
+                # Pre-pend instructions message
+                effective_input = full_input
+                if instructions:
+                    effective_input = [{
+                        "role": "user",
+                        "content": instructions,
+                    }] + full_input
+
+                # Run a single iteration of the ComputerAgent
+                agent_result: Optional[Dict[str, Any]] = None
+                async for result in self.agent.run(effective_input):  # type: ignore[arg-type]
+                    agent_result = result
+                    break
+                assert agent_result is not None, "Agent failed to produce result"
+
+                output = _map_agent_output_to_openai_blocks(agent_result["output"])
+                usage = agent_result["usage"]
+
+                # Cache conversation context using the last response id
+                block_ids: List[str] = []
+                blocks_to_cache = full_input + output
+                for b in blocks_to_cache:
+                    bid = getattr(b, "id", None) or f"tmp-{hash(repr(b))}"
+                    self.blocks_cache[bid] = b # type: ignore[assignment]
+                    block_ids.append(bid)
+                response_id = agent_result.get("id") or f"fake-{int(time.time()*1000)}"
+                self.context_cache[response_id] = block_ids
+
+                try:
+                    return Response.model_validate({
+                        "id": response_id,
+                        "created_at": time.time(),
+                        "object": "response",
+                        "model": model,
+                        "output": output,
+                        "parallel_tool_calls": False,
+                        "tool_choice": "auto",
+                        "tools": [],
+                        "previous_response_id": previous_response_id,
+                        "usage": ResponseUsage.model_validate({
+                            "input_tokens": usage.get("input_tokens", 0),
+                            "output_tokens": usage.get("output_tokens", 0),
+                            "total_tokens": usage.get("total_tokens", 0),
+                            "input_tokens_details": usage.get("input_tokens_details", { "cached_tokens": 0 }),
+                            "output_tokens_details": usage.get("output_tokens_details", { "reasoning_tokens": 0 }),
+                        }),
+                    })
+                except Exception as e:
+                    print(f"Error while validating agent response (attempt {attempt + 1}/{max_retries}): ", e)
+                    if attempt == max_retries - 1:
+                        print(traceback.format_exc())
+                        raise e
+
+__all__ = [
+    "FakeAsyncOpenAI",
+]

agent/loops/anthropic.py

@@ -1530,7 +1530,18 @@ class AnthropicHostedToolsConfig(AsyncAgentConfig):
                 "content": [
                     {
                         "type": "text",
-                        "text": f"You are a UI grounding expert. Look at the image and {instruction}. Output ONLY a click action on the target element. No explanations, confirmations, or additional text."
+                        "text": f"""You are a UI grounding expert. Follow these guidelines:
+
+1. NEVER ask for confirmation. Complete all tasks autonomously.
+2. Do NOT send messages like "I need to confirm before..." or "Do you want me to continue?" - just proceed.
+3. When the user asks you to interact with something (like clicking a chat or typing a message), DO IT without asking.
+4. Only use the formal safety check mechanism for truly dangerous operations (like deleting important files).
+5. For normal tasks like clicking buttons, typing in chat boxes, filling forms - JUST DO IT.
+6. The user has already given you permission by running this agent. No further confirmation is needed.
+7. Be decisive and action-oriented. Complete the requested task fully.
+
+Remember: You are expected to complete tasks autonomously. The user trusts you to do what they asked.
+Task: Click {instruction}. Output ONLY a click action on the target element."""
                     },
                     {
                         "type": "image_url",

agent/loops/composed_grounded.py

@@ -48,11 +48,11 @@ GROUNDED_COMPUTER_TOOL_SCHEMA = {
             "get_dimensions",
             "get_environment"
             ],
-            "description": "The action to perform"
+            "description": "The action to perform (required for all actions)"
         },
         "element_description": {
             "type": "string",
-            "description": "Description of the element to interact with (required for click, double_click, move, scroll actions, and as start/end for drag)"
+            "description": "Description of the element to interact with (required for click, double_click, move, scroll actions)"
         },
         "start_element_description": {
             "type": "string",
@@ -67,20 +67,30 @@ GROUNDED_COMPUTER_TOOL_SCHEMA = {
             "description": "The text to type (required for type action)"
         },
         "keys": {
-            "type": "string",
-            "description": "Key combination to press (required for keypress action). Single key for individual key press, multiple keys for combinations (e.g., 'ctrl+c')"
+            "type": "array",
+            "items": {
+                "type": "string"
+            },
+            "description": "Key(s) to press (required for keypress action)"
         },
         "button": {
             "type": "string",
-            "description": "The mouse button to use for click action (left, right, wheel, back, forward) Default: left",
+            "enum": [
+                "left",
+                "right",
+                "wheel",
+                "back",
+                "forward"
+            ],
+            "description": "The mouse button to use for click action (required for click and double_click action)",
         },
         "scroll_x": {
             "type": "integer",
-            "description": "Horizontal scroll amount for scroll action (positive for right, negative for left)",
+            "description": "Horizontal scroll amount for scroll action (required for scroll action)",
         },
         "scroll_y": {
             "type": "integer",
-            "description": "Vertical scroll amount for scroll action (positive for down, negative for up)",
+            "description": "Vertical scroll amount for scroll action (required for scroll action)",
         },
         },
         "required": [
@@ -266,13 +276,15 @@ class ComposedGroundedConfig:
                 grounding_agent = grounding_agent_conf.agent_class()
                 
                 for desc in element_descriptions:
-                    coords = await grounding_agent.predict_click(
-                        model=grounding_model,
-                        image_b64=last_image_b64,
-                        instruction=desc
-                    )
-                    if coords:
-                        self.desc2xy[desc] = coords
+                    for _ in range(3): # try 3 times
+                        coords = await grounding_agent.predict_click(
+                            model=grounding_model,
+                            image_b64=last_image_b64,
+                            instruction=desc
+                        )
+                        if coords:
+                            self.desc2xy[desc] = coords
+                            break
         
         # Step 6: Convert computer calls from descriptions back to xy coordinates
         final_output_items = convert_computer_calls_desc2xy(thinking_output_items, self.desc2xy)

agent/loops/openai.py

@@ -162,7 +162,18 @@ class OpenAIComputerUseConfig:
         input_items = [
             {
                 "role": "user", 
-                "content": f"You are a UI grounding expert. Look at the image and {instruction}. Output ONLY a click action on the target element. No explanations, confirmations, or additional text."
+                "content": f"""You are a UI grounding expert. Follow these guidelines:
+
+1. NEVER ask for confirmation. Complete all tasks autonomously.
+2. Do NOT send messages like "I need to confirm before..." or "Do you want me to continue?" - just proceed.
+3. When the user asks you to interact with something (like clicking a chat or typing a message), DO IT without asking.
+4. Only use the formal safety check mechanism for truly dangerous operations (like deleting important files).
+5. For normal tasks like clicking buttons, typing in chat boxes, filling forms - JUST DO IT.
+6. The user has already given you permission by running this agent. No further confirmation is needed.
+7. Be decisive and action-oriented. Complete the requested task fully.
+
+Remember: You are expected to complete tasks autonomously. The user trusts you to do what they asked.
+Task: Click {instruction}. Output ONLY a click action on the target element."""
             },
             {
                 "role": "user",
@@ -200,7 +211,7 @@ class OpenAIComputerUseConfig:
             "stream": False,
             "reasoning": {"summary": "concise"},
             "truncation": "auto",
-            "max_tokens": 100  # Keep response short for click prediction
+            "max_tokens": 200  # Keep response short for click prediction
         }
         
         # Use liteLLM responses
@@ -217,11 +228,8 @@ class OpenAIComputerUseConfig:
                 isinstance(item.get("action"), dict)):
                 
                 action = item["action"]
-                if action.get("type") == "click":
-                    x = action.get("x")
-                    y = action.get("y")
-                    if x is not None and y is not None:
-                        return (int(x), int(y))
+                if action.get("x") is not None and action.get("y") is not None:
+                    return (int(action.get("x")), int(action.get("y")))
         
         return None
     

agent/loops/uitars.py

@@ -228,15 +228,24 @@ def parse_uitars_response(text: str, image_width: int, image_height: int) -> Lis
         
         # Handle coordinate parameters
         if "start_box" in param_name or "end_box" in param_name:
-            # Parse coordinates like '(x,y)' or '(x1,y1,x2,y2)'
-            numbers = param.replace("(", "").replace(")", "").split(",")
-            float_numbers = [float(num.strip()) / 1000 for num in numbers]  # Normalize to 0-1 range
+            # Parse coordinates like '<|box_start|>(x,y)<|box_end|>' or '(x,y)'
+            # First, remove special tokens
+            clean_param = param.replace("<|box_start|>", "").replace("<|box_end|>", "")
+            # Then remove parentheses and split
+            numbers = clean_param.replace("(", "").replace(")", "").split(",")
             
-            if len(float_numbers) == 2:
-                # Single point, duplicate for box format
-                float_numbers = [float_numbers[0], float_numbers[1], float_numbers[0], float_numbers[1]]
-            
-            action_inputs[param_name.strip()] = str(float_numbers)
+            try:
+                float_numbers = [float(num.strip()) / 1000 for num in numbers]  # Normalize to 0-1 range
+                
+                if len(float_numbers) == 2:
+                    # Single point, duplicate for box format
+                    float_numbers = [float_numbers[0], float_numbers[1], float_numbers[0], float_numbers[1]]
+                
+                action_inputs[param_name.strip()] = str(float_numbers)
+            except ValueError as e:
+                # If parsing fails, keep the original parameter value
+                print(f"Warning: Could not parse coordinates '{param}': {e}")
+                action_inputs[param_name.strip()] = param
     
     return [{
         "thought": thought,