hud 0.6.17__tar.gz → 0.6.18.dev2__tar.gz

{hud-0.6.17 → hud-0.6.18.dev2}/.gitignore

@@ -59,6 +59,8 @@ hud/rl/checkpoints_test/
 .hud/
 .hud_eval_config
 .hud_eval.toml
+# The template hud init copies into new projects.
+!environments/*/.hud_eval.toml
 
 docs/internal
 

{hud-0.6.17 → hud-0.6.18.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hud
-Version: 0.6.17
+Version: 0.6.18.dev2
 Summary: SDK for the HUD platform.
 Project-URL: Homepage, https://github.com/hud-evals/hud-python
 Project-URL: Bug Tracker, https://github.com/hud-evals/hud-python/issues
@@ -234,17 +234,7 @@ A **capability** is a connection the environment exposes; a **harness** attaches
 
 From the [platform UI](https://hud.ai) you can run batches, compare models on the same taskset, and inspect every trace.
 
-A **project** holds the environments and tasksets a team creates and decides who can see them. Deploying without one uses your team's default project, so nothing here is required to get started. To put an environment somewhere else, pin the directory once and every later `hud deploy` and `hud sync tasks` follows it:
-
-```bash
-hud project list                   # projects you can see
-hud project use browser-evals      # writes projectId to .hud/config.json
-hud project                        # where does a deploy here land?
-```
-
-`hud deploy` and `hud sync tasks` also take `--project <name-or-id>` as a one-command override; it does not change the directory configuration. Use `hud set HUD_DEFAULT_PROJECT=<name-or-id>` for a machine-wide fallback. Precedence is the flag, then the directory's `.hud/config.json`, then `HUD_DEFAULT_PROJECT`, then your team default. An environment or taskset that already exists stays where it is; naming a different project fails rather than moving it.
-
-→ [Projects](https://docs.hud.ai/v6/reference/projects) · [Run & deploy](https://docs.hud.ai/v6/reference/runtime)
+→ [Run & deploy](https://docs.hud.ai/v6/reference/runtime)
 
 ## Train on rewards
 

{hud-0.6.17 → hud-0.6.18.dev2}/README.md

@@ -148,17 +148,7 @@ A **capability** is a connection the environment exposes; a **harness** attaches
 
 From the [platform UI](https://hud.ai) you can run batches, compare models on the same taskset, and inspect every trace.
 
-A **project** holds the environments and tasksets a team creates and decides who can see them. Deploying without one uses your team's default project, so nothing here is required to get started. To put an environment somewhere else, pin the directory once and every later `hud deploy` and `hud sync tasks` follows it:
-
-```bash
-hud project list                   # projects you can see
-hud project use browser-evals      # writes projectId to .hud/config.json
-hud project                        # where does a deploy here land?
-```
-
-`hud deploy` and `hud sync tasks` also take `--project <name-or-id>` as a one-command override; it does not change the directory configuration. Use `hud set HUD_DEFAULT_PROJECT=<name-or-id>` for a machine-wide fallback. Precedence is the flag, then the directory's `.hud/config.json`, then `HUD_DEFAULT_PROJECT`, then your team default. An environment or taskset that already exists stays where it is; naming a different project fails rather than moving it.
-
-→ [Projects](https://docs.hud.ai/v6/reference/projects) · [Run & deploy](https://docs.hud.ai/v6/reference/runtime)
+→ [Run & deploy](https://docs.hud.ai/v6/reference/runtime)
 
 ## Train on rewards
 

{hud-0.6.17 → hud-0.6.18.dev2}/hud/__main__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from hud.cli import main
+from hud.cli.__main__ import main
 
 if __name__ == "__main__":
     main()

{hud-0.6.17 → hud-0.6.18.dev2}/hud/agents/__init__.py

@@ -10,11 +10,7 @@ from typing import TYPE_CHECKING, Any, cast
 from hud.settings import settings
 from hud.types import AgentType
 from hud.utils.exceptions import HudAuthenticationError
-from hud.utils.gateway import (
-    gateway_model_aliases,
-    list_gateway_models,
-    normalize_gateway_model_id,
-)
+from hud.utils.gateway import resolve_gateway_model
 
 if TYPE_CHECKING:
     from typing import TypeAlias
@@ -31,10 +27,11 @@ if TYPE_CHECKING:
 def create_agent(model: str, **kwargs: Any) -> GatewayAgent:
     """Create an agent routed through the HUD gateway.
 
-    Leaves ``model_client`` unset so provider agent constructors build the HUD
-    gateway client locally, while :class:`~hud.eval.runtime.HostedRuntime` can
-    serialize the config and rebuild the client remotely. Explicitly supplied
-    clients remain custom/BYOK and are not serializable.
+    Sets ``gateway=True`` on the config instead of attaching a client, so the
+    provider agent builds the gateway client locally and
+    :class:`~hud.eval.runtime.HostedRuntime` can serialize the config and rebuild
+    it remotely. Explicitly supplied clients remain custom/BYOK and are not
+    serializable.
 
     For direct API access with provider API keys, instantiate the agent classes
     directly.
@@ -49,72 +46,40 @@ def create_agent(model: str, **kwargs: Any) -> GatewayAgent:
     if not settings.api_key:
         raise HudAuthenticationError("HUD_API_KEY is required to create a gateway agent")
 
-    requested_model = model
-    model = normalize_gateway_model_id(model)
-    agent_type = next((candidate for candidate in AgentType if candidate.value == model), None)
-    if agent_type is not None:
-        model_id = model
-    else:
-        try:
-            gateway_models = list_gateway_models()
-        except Exception:
-            gateway_models = []
-        gateway_models = list(gateway_models)
-        for gateway_model in gateway_models:
-            if model in (
-                gateway_model.id,
-                gateway_model.name,
-                gateway_model.model_name,
-            ):
-                agent_str = gateway_model.sdk_agent_type
-                if agent_str == "operator":
-                    raise ValueError(
-                        "Operator agent is no longer supported; use openai with a supported "
-                        "OpenAI computer model."
-                    )
-                if agent_str == "gemini_cua":
-                    raise ValueError(
-                        "Gemini CUA agent is no longer supported; use gemini with a supported "
-                        "Gemini computer-use model."
-                    )
-                if not isinstance(agent_str, str):
-                    raise ValueError(f"Model '{model}' has invalid agent type metadata")
-
-                try:
-                    agent_type = AgentType(agent_str)
-                except ValueError as exc:
-                    raise ValueError(f"Model '{model}' has invalid agent type metadata") from exc
-                model_id = gateway_model.model_name or model
-                break
-        else:
-            import difflib
-
-            known = [c.value for c in AgentType] + [
-                n
-                for gm in gateway_models
-                for n in (gm.id, gm.name, gm.model_name)
-                if isinstance(n, str)
-            ]
-            known.extend(gateway_model_aliases())
-            near = difflib.get_close_matches(requested_model, known, n=3, cutoff=0.5)
-            hint = (
-                f" Did you mean: {', '.join(near)}?"
-                if near
-                else " Run `hud models` to list available models."
-            )
-            source = (
-                "the HUD gateway registry"
-                if gateway_models
-                else "the HUD gateway registry (empty — is HUD_API_KEY set?)"
-            )
-            raise ValueError(f"Model {requested_model!r} not found in {source}.{hint}")
-
+    agent_type, model_id = resolve_agent_model(model)
     kwargs.setdefault("model", model_id)
+    kwargs["gateway"] = True
     # cls/config_cls are matched unions; the pairing is correct by construction.
     config = agent_type.config_cls(**kwargs)
     return agent_type.cls(cast("Any", config))
 
 
+def resolve_agent_model(model: str) -> tuple[AgentType, str]:
+    """Resolve a catalog id/name/slug or an agent type without constructing a client."""
+    agent_type = next((candidate for candidate in AgentType if candidate.value == model), None)
+    if agent_type is not None:
+        return agent_type, model
+    try:
+        entry = resolve_gateway_model(model)
+    except HudAuthenticationError:
+        types = ", ".join(candidate.value for candidate in AgentType)
+        raise ValueError(
+            f"Resolving {model!r} needs the HUD model catalog; set HUD_API_KEY, or pass an "
+            f"agent type ({types}) and choose the model with its config."
+        ) from None
+    if entry.sdk_agent_type in ("operator", "gemini_cua"):
+        replacement = "openai" if entry.sdk_agent_type == "operator" else "gemini"
+        raise ValueError(
+            f"The {entry.sdk_agent_type} agent is no longer supported; use {replacement} with a "
+            "supported computer-use model."
+        )
+    try:
+        agent_type = AgentType(entry.sdk_agent_type)
+    except ValueError as exc:
+        raise ValueError(f"Model {model!r} has invalid agent type metadata") from exc
+    return agent_type, entry.model_name or model
+
+
 _LAZY_EXPORTS = {
     "ClaudeAgent": ("hud.agents.claude", "ClaudeAgent"),
     "ClaudeSDKAgent": ("hud.agents.claude", "ClaudeSDKAgent"),
@@ -134,6 +99,7 @@ __all__ = [
     "OpenAIAgent",
     "OpenAIChatAgent",
     "create_agent",
+    "resolve_agent_model",
 ]
 
 

{hud-0.6.17 → hud-0.6.18.dev2}/hud/agents/claude/agent.py

@@ -38,7 +38,6 @@ from anthropic.types.beta import (
 
 from hud.agents.tool_agent import RunState, ToolAgent
 from hud.agents.types import AgentStep, Citation, ClaudeConfig, Usage
-from hud.settings import settings
 from hud.types import MCPToolCall, MCPToolResult
 from hud.utils import gateway
 
@@ -108,12 +107,11 @@ class ClaudeAgent(ToolAgent[BetaMessageParam, ClaudeConfig]):
     def _resolve_client(self) -> AsyncAnthropic | AsyncAnthropicBedrock:
         if self.config.model_client is not None:
             return cast("AsyncAnthropic | AsyncAnthropicBedrock", self.config.model_client)
-        if settings.api_key:
-            return cast("AsyncAnthropic", gateway.build_gateway_client("anthropic"))
-        if settings.anthropic_api_key:
-            return AsyncAnthropic(api_key=settings.anthropic_api_key)
-        raise ValueError(
-            "No API key found for Claude. Set HUD_API_KEY (gateway) or ANTHROPIC_API_KEY.",
+        return cast(
+            "AsyncAnthropic | AsyncAnthropicBedrock",
+            gateway.build_model_client(
+                "anthropic", model=self.config.model, gateway=self.config.gateway
+            ),
         )
 
     # ─── ToolAgent hooks ──────────────────────────────────────────────

{hud-0.6.17 → hud-0.6.18.dev2}/hud/agents/gemini/agent.py

@@ -4,18 +4,19 @@ from __future__ import annotations
 
 import base64
 import logging
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, cast
 
 import mcp.types as mcp_types
-from google import genai
 from google.genai import types as genai_types
 
 from hud.agents.tool_agent import RunState, ToolAgent
 from hud.agents.types import AgentStep, Citation, GeminiConfig, Usage
-from hud.settings import settings
 from hud.types import MCPToolCall, MCPToolResult
 from hud.utils import gateway
 
+if TYPE_CHECKING:
+    from google import genai
+
 from .settings import gemini_agent_settings
 from .tools import (
     PREDEFINED_COMPUTER_USE_FUNCTIONS,
@@ -54,14 +55,7 @@ class GeminiAgent(ToolAgent[genai_types.Content, GeminiConfig]):
 
         model_client = config.model_client
         if model_client is None:
-            if settings.api_key:
-                model_client = gateway.build_gateway_client("gemini")
-            elif settings.gemini_api_key:
-                model_client = genai.Client(api_key=settings.gemini_api_key)
-            else:
-                raise ValueError(
-                    "No API key for Gemini. Set HUD_API_KEY or GEMINI_API_KEY.",
-                )
+            model_client = gateway.build_model_client("gemini", gateway=config.gateway)
 
         self.gemini_client: genai.Client = cast("genai.Client", model_client)
         self.temperature = config.temperature
@@ -230,6 +224,9 @@ class GeminiAgent(ToolAgent[genai_types.Content, GeminiConfig]):
 
         if candidate.finish_reason is not None:
             result.finish_reason = candidate.finish_reason.name
+            if result.finish_reason == "MALFORMED_FUNCTION_CALL":
+                result.error = "Provider returned a malformed function call"
+                result.stop_reason = "malformed_tool_call"
 
         return result
 

{hud-0.6.17 → hud-0.6.18.dev2}/hud/agents/openai/agent.py

@@ -25,8 +25,8 @@ from openai.types.responses.response_input_param import (
 from openai.types.shared_params.reasoning import Reasoning  # noqa: TC002
 
 from hud.agents.tool_agent import DegenerateTurnError, RunState, ToolAgent
+from hud.agents.tools.base import result_text
 from hud.agents.types import AgentStep, Citation, OpenAIConfig, Usage
-from hud.settings import settings
 from hud.types import MCPToolCall, MCPToolResult
 from hud.utils import gateway
 
@@ -72,14 +72,7 @@ class OpenAIAgent(ToolAgent[ResponseInputItemParam, OpenAIConfig]):
 
         model_client = config.model_client
         if model_client is None:
-            if settings.api_key:
-                model_client = gateway.build_gateway_client("openai")
-            elif settings.openai_api_key:
-                model_client = AsyncOpenAI(api_key=settings.openai_api_key)
-            else:
-                raise ValueError(
-                    "No API key for OpenAI. Set HUD_API_KEY or OPENAI_API_KEY.",
-                )
+            model_client = gateway.build_model_client("openai", gateway=config.gateway)
 
         self.openai_client: AsyncOpenAI = cast("AsyncOpenAI", model_client)
         self._model = config.model
@@ -116,8 +109,10 @@ class OpenAIAgent(ToolAgent[ResponseInputItemParam, OpenAIConfig]):
         if isinstance(tool, OpenAIComputerTool):
             screenshot = last_image_content(result)
             if screenshot is None:
-                logger.warning("Computer tool result missing screenshot for call %s", call.name)
-                return None
+                raise RuntimeError(
+                    f"Computer tool result missing screenshot for call {call.id}: "
+                    f"{result_text(result)}"
+                )
             output = ComputerCallOutput(
                 type="computer_call_output",
                 call_id=call.id,
@@ -142,7 +137,17 @@ class OpenAIAgent(ToolAgent[ResponseInputItemParam, OpenAIConfig]):
                         acknowledged.append(raw_check)
                 if acknowledged:
                     output["acknowledged_safety_checks"] = acknowledged
-            return cast("ResponseInputItemParam", output)
+            if result.isError:
+                return [
+                    output,
+                    self._format_message(
+                        "user",
+                        f"Computer call {call.id} stopped with an error. "
+                        "Remaining actions in this call were not executed.\n"
+                        f"{result_text(result)}",
+                    ),
+                ]
+            return output
 
         if isinstance(tool, OpenAIShellTool):
             structured = (
@@ -150,8 +155,6 @@ class OpenAIAgent(ToolAgent[ResponseInputItemParam, OpenAIConfig]):
             )
             output_list = structured.get("output")
             if not isinstance(output_list, list):
-                from hud.agents.tools.base import result_text
-
                 text = result_text(result)
                 output_list = [shell_output("", text, 1 if result.isError else 0)]
             response: dict[str, Any] = {
@@ -313,7 +316,7 @@ class OpenAIAgent(ToolAgent[ResponseInputItemParam, OpenAIConfig]):
                     elif item.action is not None:
                         arguments = item.action.to_dict()
                     else:
-                        raise ValueError("OpenAI computer_call missing action")
+                        arguments = {"actions": []}
                     call_dict: dict[str, Any] = {
                         "name": "computer",
                         "arguments": arguments,

{hud-0.6.17 → hud-0.6.18.dev2}/hud/agents/openai/tools/computer.py

@@ -38,17 +38,24 @@ def last_image_content(result: MCPToolResult) -> mcp_types.ImageContent | None:
 OPENAI_KEY_ALIASES: dict[str, str] = {
     "return": "Return",
     "escape": "Escape",
+    "esc": "Escape",
     "arrowup": "Up",
+    "up": "Up",
     "arrowdown": "Down",
+    "down": "Down",
     "arrowleft": "Left",
+    "left": "Left",
     "arrowright": "Right",
+    "right": "Right",
     "backspace": "BackSpace",
     "delete": "Delete",
+    "del": "Delete",
     "tab": "Tab",
     "space": "space",
     "control": "Control_L",
     "ctrl": "Control_L",
     "alt": "Alt_L",
+    "option": "Alt_L",
     "shift": "Shift_L",
     "meta": "Super_L",
     "cmd": "Super_L",
@@ -62,18 +69,6 @@ OPENAI_KEY_ALIASES: dict[str, str] = {
     "enter": "Return",
 }
 
-_SCREENSHOT_ACTIONS = {
-    "screenshot",
-    "click",
-    "double_click",
-    "scroll",
-    "type",
-    "move",
-    "keypress",
-    "drag",
-    "wait",
-}
-
 
 class OpenAIComputerTool(RFBTool):
     """Translate OpenAI native computer calls into RFBTool primitives."""
@@ -90,14 +85,16 @@ class OpenAIComputerTool(RFBTool):
 
     async def execute(self, arguments: dict[str, Any]) -> MCPToolResult:
         actions = arguments.get("actions")
-        if isinstance(actions, list):
+        if "actions" in arguments:
+            if not isinstance(actions, list):
+                return await self._error_result("actions must be a list")
             action_list = cast("list[Any]", actions)
             if not action_list:
-                return tool_err("actions list is empty")
+                return await self._error_result("actions list is empty")
             result = MCPToolResult(content=[], isError=False)
             for index, raw_action in enumerate(action_list):
                 if not isinstance(raw_action, dict):
-                    return tool_err("actions must be objects")
+                    return await self._error_result("actions must be objects")
                 action = cast("dict[str, Any]", raw_action)
                 result = await self._execute_one(
                     action,
@@ -108,6 +105,17 @@ class OpenAIComputerTool(RFBTool):
             return result
         return await self._execute_one(arguments, ensure_screenshot=True)
 
+    async def _error_result(self, message: str) -> MCPToolResult:
+        result = tool_err(message)
+        try:
+            result.content.extend((await self.screenshot()).content)
+        except TimeoutError as exc:
+            exc.add_note(message)
+            raise
+        except Exception as exc:
+            return tool_err(f"{message}\nScreenshot capture also failed: {exc}")
+        return result
+
     async def _execute_one(
         self,
         arguments: dict[str, Any],
@@ -116,26 +124,28 @@ class OpenAIComputerTool(RFBTool):
     ) -> MCPToolResult:
         action_type = arguments.get("type")
         if not isinstance(action_type, str):
-            return tool_err("type is required")
+            return await self._error_result("type is required")
 
         if action_type == "response":
             text = arguments.get("text")
             if not isinstance(text, str):
-                return tool_err("text is required for response")
-            return MCPToolResult(
+                return await self._error_result("text is required for response")
+            result = MCPToolResult(
                 content=[mcp_types.TextContent(type="text", text=text)],
             )
+            if ensure_screenshot:
+                result.content.extend((await self.screenshot()).content)
+            return result
 
         try:
             await self._dispatch(action_type, arguments)
+        except TimeoutError:
+            raise
         except Exception as exc:
             logger.exception("OpenAIComputerTool action %s failed", action_type)
-            return tool_err(f"computer action {action_type!r} failed: {exc}")
+            return await self._error_result(f"computer action {action_type!r} failed: {exc}")
 
-        needs_screenshot = (
-            ensure_screenshot and action_type in _SCREENSHOT_ACTIONS and action_type != "screenshot"
-        )
-        if action_type == "screenshot" or needs_screenshot:
+        if action_type == "screenshot" or ensure_screenshot:
             return await self.screenshot()
         return MCPToolResult(content=[], isError=False)
 
@@ -144,13 +154,13 @@ class OpenAIComputerTool(RFBTool):
             return
 
         if action_type == "click":
-            button_raw = args.get("button")
-            if button_raw == "wheel":
-                button = "middle"
-            elif isinstance(button_raw, str):
-                button = button_raw
-            else:
+            button = args.get("button", "left")
+            if button is None:
                 button = "left"
+            elif button == "wheel":
+                button = "middle"
+            if button not in ("left", "middle", "right"):
+                raise ValueError(f"Unsupported mouse button: {button!r}")
             hold = _hold_keys(args.get("keys"))
             await self.click(
                 args.get("x"),
@@ -183,23 +193,28 @@ class OpenAIComputerTool(RFBTool):
 
         elif action_type == "type":
             text = args.get("text")
-            if isinstance(text, str):
-                await self.type_text(text)
+            if not isinstance(text, str):
+                raise ValueError("text is required for type")
+            await self.type_text(text)
 
         elif action_type == "wait":
-            ms = int(args.get("ms") or 1000)
+            duration = args.get("ms")
+            ms = 1000 if duration is None else int(duration)
+            if ms < 0:
+                raise ValueError("wait duration must be non-negative")
             await self.wait(ms)
 
         elif action_type == "move":
             x, y = args.get("x"), args.get("y")
-            if x is not None and y is not None:
-                await self.move(int(x), int(y))
+            if x is None or y is None:
+                raise ValueError("x and y are required for move")
+            await self.move(int(x), int(y))
 
         elif action_type == "keypress":
-            keys = args.get("keys")
-            if isinstance(keys, list):
-                mapped = [_map_key(str(k)) for k in cast("list[Any]", keys)]
-                await self.press_keys(mapped)
+            keys = _hold_keys(args.get("keys"))
+            if not keys:
+                raise ValueError("keypress requires at least one key")
+            await self.press_keys(keys)
 
         elif action_type == "drag":
             path_raw = args.get("path")
@@ -208,7 +223,7 @@ class OpenAIComputerTool(RFBTool):
             points = cast("list[dict[str, Any]]", path_raw)
             if len(points) < 2:
                 raise ValueError("drag requires a path with at least 2 points")
-            path = [(int(p.get("x", 0)), int(p.get("y", 0))) for p in points]
+            path = [(int(p["x"]), int(p["y"])) for p in points]
             hold = _hold_keys(args.get("keys"))
             await self.drag(path, hold_keys=hold)
 
@@ -224,9 +239,11 @@ def _map_key(key: str) -> str:
 
 
 def _hold_keys(keys: Any) -> list[str] | None:
-    if not isinstance(keys, list):
+    if keys is None:
         return None
-    return [_map_key(str(key)) for key in cast("list[Any]", keys)]
+    if not isinstance(keys, list) or not all(isinstance(key, str) and key for key in keys):
+        raise ValueError("keys must be a list of non-empty strings")
+    return [_map_key(key) for key in cast("list[str]", keys)]
 
 
 __all__ = ["OPENAI_COMPUTER_SPEC", "OpenAIComputerTool"]

{hud-0.6.17 → hud-0.6.18.dev2}/hud/agents/openai/tools/tests/test_computer.py

@@ -9,12 +9,16 @@ from types import SimpleNamespace
 from typing import TYPE_CHECKING, Any
 from unittest.mock import AsyncMock, patch
 
+import mcp.types as mcp_types
+import pytest
+
 from hud.agents.openai.tools.computer import (
     OpenAIComputerTool,
     _hold_keys,
     _map_key,
 )
-from hud.agents.tools.base import result_text, tool_ok
+from hud.agents.tools.base import result_text
+from hud.types import MCPToolResult
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
@@ -27,9 +31,11 @@ class RecordingOpenAI(OpenAIComputerTool):
         self.calls: list[tuple[Any, ...]] = []
         self.client = SimpleNamespace(width=200, height=100)
 
-    async def screenshot(self) -> Any:
+    async def screenshot(self) -> MCPToolResult:
         self.calls.append(("screenshot",))
-        return tool_ok("shot")
+        return MCPToolResult(
+            content=[mcp_types.ImageContent(type="image", data="c2hvdA==", mimeType="image/png")],
+        )
 
     async def click(
         self,
@@ -76,8 +82,10 @@ class RecordingOpenAI(OpenAIComputerTool):
 def test_key_mapping() -> None:
     assert _map_key("ctrl") == "Control_L"
     assert _map_key("x") == "x"
+    assert _map_key("ESC") == "Escape"
+    assert _map_key("RIGHT") == "Right"
     assert _hold_keys(["ctrl", "c"]) == ["Control_L", "c"]
-    assert _hold_keys("notalist") is None
+    assert _hold_keys(None) is None
 
 
 def test_to_params() -> None:
@@ -99,19 +107,23 @@ async def test_type_and_keypress() -> None:
     assert ("keys", ("Control_L", "c")) in tool.calls
 
 
-async def test_drag_and_wait() -> None:
+@pytest.mark.parametrize(("ms", "seconds"), [(500, 0.5), (0, 0)])
+async def test_drag_and_wait(ms: int, seconds: float) -> None:
     tool = RecordingOpenAI()
     await tool.execute({"type": "drag", "path": [{"x": 0, "y": 0}, {"x": 5, "y": 5}]})
     with patch("hud.agents.tools.rfb.asyncio.sleep", new_callable=AsyncMock) as sleep:
-        await tool.execute({"type": "wait", "ms": 500})
+        result = await tool.execute({"type": "wait", "ms": ms})
     assert ("drag", ((0, 0), (5, 5))) in tool.calls
-    sleep.assert_awaited_once_with(0.5)
+    assert not result.isError
+    sleep.assert_awaited_once_with(seconds)
 
 
-async def test_response_action_returns_text() -> None:
+async def test_response_action_returns_text_and_screenshot() -> None:
     tool = RecordingOpenAI()
     result = await tool.execute({"type": "response", "text": "all done"})
     assert result_text(result) == "all done"
+    assert any(isinstance(block, mcp_types.ImageContent) for block in result.content)
+    assert tool.calls[-1] == ("screenshot",)
 
 
 async def test_actions_list_runs_each() -> None:
@@ -123,12 +135,40 @@ async def test_actions_list_runs_each() -> None:
     assert ("type", "a") in tool.calls
 
 
-async def test_empty_actions_errors() -> None:
+@pytest.mark.parametrize(
+    "arguments", [{"actions": []}, {"actions": "invalid", "type": "move", "x": 3, "y": 4}]
+)
+async def test_invalid_actions_errors(arguments: dict[str, Any]) -> None:
     tool = RecordingOpenAI()
-    assert (await tool.execute({"actions": []})).isError
+    assert (await tool.execute(arguments)).isError
+    assert tool.calls == [("screenshot",)]
 
 
 async def test_invalid_type_errors() -> None:
     tool = RecordingOpenAI()
     assert (await tool.execute({"type": "frobnicate"})).isError
     assert (await tool.execute({})).isError
+
+
+@pytest.mark.parametrize(
+    "action",
+    [
+        {"type": "type"},
+        {"type": "move", "x": 10},
+        {"type": "keypress", "keys": "ESC"},
+        {"type": "keypress", "keys": []},
+        {"type": "drag", "path": [{"x": 1, "y": 2}, {"x": 3}]},
+        {"type": "click", "button": 1},
+        {"type": "wait", "ms": -1},
+    ],
+)
+async def test_invalid_action_stops_batch_and_returns_screenshot(action: dict[str, Any]) -> None:
+    tool = RecordingOpenAI()
+    result = await tool.execute(
+        {"actions": [{"type": "move", "x": 3, "y": 4}, action, {"type": "type", "text": "later"}]},
+    )
+
+    assert result.isError
+    assert result_text(result)
+    assert any(isinstance(block, mcp_types.ImageContent) for block in result.content)
+    assert tool.calls == [("move", 3, 4), ("screenshot",)]