hud 0.6.16.dev4__tar.gz → 0.6.18.dev1__tar.gz

{hud-0.6.16.dev4 → hud-0.6.18.dev1}/.gitignore

@@ -59,11 +59,11 @@ hud/rl/checkpoints_test/
 .hud/
 .hud_eval_config
 .hud_eval.toml
+# The template hud init copies into new projects.
+!environments/*/.hud_eval.toml
 
 docs/internal
 
-environments/
-
 experiments/
 .memories/
 

{hud-0.6.16.dev4 → hud-0.6.18.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hud
-Version: 0.6.16.dev4
+Version: 0.6.18.dev1
 Summary: SDK for the HUD platform.
 Project-URL: Homepage, https://github.com/hud-evals/hud-python
 Project-URL: Bug Tracker, https://github.com/hud-evals/hud-python/issues
@@ -34,7 +34,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Python: <3.13,>=3.11
-Requires-Dist: anthropic>=0.78.0
+Requires-Dist: anthropic>=0.98.0
 Requires-Dist: asyncssh>=2.23.0
 Requires-Dist: asyncvnc>=1.3.0
 Requires-Dist: fastmcp==3.2.0
@@ -56,7 +56,7 @@ Requires-Dist: websockets>=15.0.1
 Provides-Extra: agent
 Provides-Extra: agents
 Provides-Extra: bedrock
-Requires-Dist: anthropic[bedrock]>=0.78.0; extra == 'bedrock'
+Requires-Dist: anthropic[bedrock]>=0.98.0; extra == 'bedrock'
 Provides-Extra: browseruse
 Requires-Dist: browser-use>=0.11.13; extra == 'browseruse'
 Provides-Extra: daytona
@@ -123,10 +123,12 @@ hud set HUD_API_KEY=your-key-here
 # or: export HUD_API_KEY=your-key-here
 ```
 
-Then scaffold your first environment:
+Then start from the coding environment, or choose another example environment:
 
 ```bash
 hud init my-env
+hud init my-desktop-env --template cua
+hud init my-custom-env --template blank
 ```
 
 ![Agent running on SheetBench](docs/src/images/trace_sheet.gif)
@@ -166,16 +168,6 @@ hud sync tasks my-taskset
 hud eval my-taskset --remote
 ```
 
-For local iteration, the same protocol works against a container on your laptop:
-
-```bash
-docker build -f Dockerfile.hud -t my-env .
-docker run -d --name run1 -p 8765:8765 my-env
-hud task start fix_bug --url tcp://127.0.0.1:8765
-hud task grade fix_bug --url tcp://127.0.0.1:8765 --answer "..."
-docker rm -f run1
-```
-
 → [Run & deploy](https://docs.hud.ai/v6/reference/runtime)
 
 ## Environments & templates

{hud-0.6.16.dev4 → hud-0.6.18.dev1}/README.md

@@ -37,10 +37,12 @@ hud set HUD_API_KEY=your-key-here
 # or: export HUD_API_KEY=your-key-here
 ```
 
-Then scaffold your first environment:
+Then start from the coding environment, or choose another example environment:
 
 ```bash
 hud init my-env
+hud init my-desktop-env --template cua
+hud init my-custom-env --template blank
 ```
 
 ![Agent running on SheetBench](docs/src/images/trace_sheet.gif)
@@ -80,16 +82,6 @@ hud sync tasks my-taskset
 hud eval my-taskset --remote
 ```
 
-For local iteration, the same protocol works against a container on your laptop:
-
-```bash
-docker build -f Dockerfile.hud -t my-env .
-docker run -d --name run1 -p 8765:8765 my-env
-hud task start fix_bug --url tcp://127.0.0.1:8765
-hud task grade fix_bug --url tcp://127.0.0.1:8765 --answer "..."
-docker rm -f run1
-```
-
 → [Run & deploy](https://docs.hud.ai/v6/reference/runtime)
 
 ## Environments & templates

{hud-0.6.16.dev4 → hud-0.6.18.dev1}/hud/__main__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from hud.cli import main
+from hud.cli.__main__ import main
 
 if __name__ == "__main__":
     main()

{hud-0.6.16.dev4 → hud-0.6.18.dev1}/hud/agents/__init__.py

@@ -10,11 +10,7 @@ from typing import TYPE_CHECKING, Any, cast
 from hud.settings import settings
 from hud.types import AgentType
 from hud.utils.exceptions import HudAuthenticationError
-from hud.utils.gateway import (
-    gateway_model_aliases,
-    list_gateway_models,
-    normalize_gateway_model_id,
-)
+from hud.utils.gateway import resolve_gateway_model
 
 if TYPE_CHECKING:
     from typing import TypeAlias
@@ -31,10 +27,11 @@ if TYPE_CHECKING:
 def create_agent(model: str, **kwargs: Any) -> GatewayAgent:
     """Create an agent routed through the HUD gateway.
 
-    Leaves ``model_client`` unset so provider agent constructors build the HUD
-    gateway client locally, while :class:`~hud.eval.runtime.HostedRuntime` can
-    serialize the config and rebuild the client remotely. Explicitly supplied
-    clients remain custom/BYOK and are not serializable.
+    Sets ``gateway=True`` on the config instead of attaching a client, so the
+    provider agent builds the gateway client locally and
+    :class:`~hud.eval.runtime.HostedRuntime` can serialize the config and rebuild
+    it remotely. Explicitly supplied clients remain custom/BYOK and are not
+    serializable.
 
     For direct API access with provider API keys, instantiate the agent classes
     directly.
@@ -49,72 +46,40 @@ def create_agent(model: str, **kwargs: Any) -> GatewayAgent:
     if not settings.api_key:
         raise HudAuthenticationError("HUD_API_KEY is required to create a gateway agent")
 
-    requested_model = model
-    model = normalize_gateway_model_id(model)
-    agent_type = next((candidate for candidate in AgentType if candidate.value == model), None)
-    if agent_type is not None:
-        model_id = model
-    else:
-        try:
-            gateway_models = list_gateway_models()
-        except Exception:
-            gateway_models = []
-        gateway_models = list(gateway_models)
-        for gateway_model in gateway_models:
-            if model in (
-                gateway_model.id,
-                gateway_model.name,
-                gateway_model.model_name,
-            ):
-                agent_str = gateway_model.sdk_agent_type
-                if agent_str == "operator":
-                    raise ValueError(
-                        "Operator agent is no longer supported; use openai with a supported "
-                        "OpenAI computer model."
-                    )
-                if agent_str == "gemini_cua":
-                    raise ValueError(
-                        "Gemini CUA agent is no longer supported; use gemini with a supported "
-                        "Gemini computer-use model."
-                    )
-                if not isinstance(agent_str, str):
-                    raise ValueError(f"Model '{model}' has invalid agent type metadata")
-
-                try:
-                    agent_type = AgentType(agent_str)
-                except ValueError as exc:
-                    raise ValueError(f"Model '{model}' has invalid agent type metadata") from exc
-                model_id = gateway_model.model_name or model
-                break
-        else:
-            import difflib
-
-            known = [c.value for c in AgentType] + [
-                n
-                for gm in gateway_models
-                for n in (gm.id, gm.name, gm.model_name)
-                if isinstance(n, str)
-            ]
-            known.extend(gateway_model_aliases())
-            near = difflib.get_close_matches(requested_model, known, n=3, cutoff=0.5)
-            hint = (
-                f" Did you mean: {', '.join(near)}?"
-                if near
-                else " Run `hud models` to list available models."
-            )
-            source = (
-                "the HUD gateway registry"
-                if gateway_models
-                else "the HUD gateway registry (empty — is HUD_API_KEY set?)"
-            )
-            raise ValueError(f"Model {requested_model!r} not found in {source}.{hint}")
-
+    agent_type, model_id = resolve_agent_model(model)
     kwargs.setdefault("model", model_id)
+    kwargs["gateway"] = True
     # cls/config_cls are matched unions; the pairing is correct by construction.
     config = agent_type.config_cls(**kwargs)
     return agent_type.cls(cast("Any", config))
 
 
+def resolve_agent_model(model: str) -> tuple[AgentType, str]:
+    """Resolve a catalog id/name/slug or an agent type without constructing a client."""
+    agent_type = next((candidate for candidate in AgentType if candidate.value == model), None)
+    if agent_type is not None:
+        return agent_type, model
+    try:
+        entry = resolve_gateway_model(model)
+    except HudAuthenticationError:
+        types = ", ".join(candidate.value for candidate in AgentType)
+        raise ValueError(
+            f"Resolving {model!r} needs the HUD model catalog; set HUD_API_KEY, or pass an "
+            f"agent type ({types}) and choose the model with its config."
+        ) from None
+    if entry.sdk_agent_type in ("operator", "gemini_cua"):
+        replacement = "openai" if entry.sdk_agent_type == "operator" else "gemini"
+        raise ValueError(
+            f"The {entry.sdk_agent_type} agent is no longer supported; use {replacement} with a "
+            "supported computer-use model."
+        )
+    try:
+        agent_type = AgentType(entry.sdk_agent_type)
+    except ValueError as exc:
+        raise ValueError(f"Model {model!r} has invalid agent type metadata") from exc
+    return agent_type, entry.model_name or model
+
+
 _LAZY_EXPORTS = {
     "ClaudeAgent": ("hud.agents.claude", "ClaudeAgent"),
     "ClaudeSDKAgent": ("hud.agents.claude", "ClaudeSDKAgent"),
@@ -134,6 +99,7 @@ __all__ = [
     "OpenAIAgent",
     "OpenAIChatAgent",
     "create_agent",
+    "resolve_agent_model",
 ]
 
 

{hud-0.6.16.dev4 → hud-0.6.18.dev1}/hud/agents/claude/agent.py

@@ -38,7 +38,6 @@ from anthropic.types.beta import (
 
 from hud.agents.tool_agent import RunState, ToolAgent
 from hud.agents.types import AgentStep, Citation, ClaudeConfig, Usage
-from hud.settings import settings
 from hud.types import MCPToolCall, MCPToolResult
 from hud.utils import gateway
 
@@ -108,12 +107,11 @@ class ClaudeAgent(ToolAgent[BetaMessageParam, ClaudeConfig]):
     def _resolve_client(self) -> AsyncAnthropic | AsyncAnthropicBedrock:
         if self.config.model_client is not None:
             return cast("AsyncAnthropic | AsyncAnthropicBedrock", self.config.model_client)
-        if settings.api_key:
-            return cast("AsyncAnthropic", gateway.build_gateway_client("anthropic"))
-        if settings.anthropic_api_key:
-            return AsyncAnthropic(api_key=settings.anthropic_api_key)
-        raise ValueError(
-            "No API key found for Claude. Set HUD_API_KEY (gateway) or ANTHROPIC_API_KEY.",
+        return cast(
+            "AsyncAnthropic | AsyncAnthropicBedrock",
+            gateway.build_model_client(
+                "anthropic", model=self.config.model, gateway=self.config.gateway
+            ),
         )
 
     # ─── ToolAgent hooks ──────────────────────────────────────────────
@@ -403,6 +401,12 @@ class ClaudeAgent(ToolAgent[BetaMessageParam, ClaudeConfig]):
         if thinking_parts:
             result.reasoning = "\n".join(thinking_parts)
         result.finish_reason = response.stop_reason
+        if response.stop_reason == "refusal" and response.stop_details is not None:
+            explanation = response.stop_details.explanation
+            if explanation:
+                result.refusal = explanation
+            elif response.stop_details.category is not None:
+                result.refusal = f"This request was refused ({response.stop_details.category})."
         return result
 
     @staticmethod

{hud-0.6.16.dev4 → hud-0.6.18.dev1}/hud/agents/claude/sdk/agent.py

@@ -10,6 +10,7 @@ Inspired by harbor-framework/harbor's ClaudeCode agent.
 
 from __future__ import annotations
 
+import base64
 import json
 import logging
 import shlex
@@ -20,6 +21,7 @@ from typing import TYPE_CHECKING, Any, cast
 from hud.agents.base import Agent
 from hud.agents.types import AgentStep, ClaudeSDKConfig, Usage
 from hud.settings import settings
+from hud.telemetry.context import get_current_trace_id, get_trace_headers
 from hud.types import Step
 
 if TYPE_CHECKING:
@@ -83,6 +85,7 @@ class ClaudeSDKAgent(Agent):
 
     async def __call__(self, run: Run) -> None:
         mcp_servers: dict[str, dict[str, Any]] = {}
+        trace_id = get_current_trace_id()
         manifest = run.client.manifest
         bindings = manifest.bindings if manifest is not None else []
         families = {c.protocol.split("/", 1)[0] for c in bindings}
@@ -100,8 +103,11 @@ class ClaudeSDKAgent(Agent):
                     token = cap.params.get("auth_token")
                     transport = "http" if cap.params["transport"] == "streamable-http" else "sse"
                     server_config: dict[str, Any] = {"type": transport, "url": cap.url}
+                    headers = {"Trace-Id": trace_id} if trace_id is not None else {}
                     if token:
-                        server_config["headers"] = {"Authorization": f"Bearer {token}"}
+                        headers["Authorization"] = f"Bearer {token}"
+                    if headers:
+                        server_config["headers"] = headers
                     if cap.name in mcp_servers:
                         raise RuntimeError(f"duplicate MCP server name {cap.name!r}")
                     mcp_servers[cap.name] = server_config
@@ -188,6 +194,11 @@ class ClaudeSDKAgent(Agent):
         if settings.api_key:
             env["ANTHROPIC_BASE_URL"] = settings.hud_gateway_url
             env["ANTHROPIC_API_KEY"] = settings.api_key
+            trace_headers = get_trace_headers()
+            if trace_headers:
+                env["ANTHROPIC_CUSTOM_HEADERS"] = "\n".join(
+                    f"{name}: {value}" for name, value in trace_headers.items()
+                )
         elif settings.anthropic_api_key:
             env["ANTHROPIC_API_KEY"] = settings.anthropic_api_key
 
@@ -259,16 +270,18 @@ class ClaudeSDKAgent(Agent):
             # Solution: use `cmd /c claude [args]` (no inline prompt) and feed the
             # prompt via stdin from .hud_prompt.txt. claude --print reads stdin as
             # the initial message when no -- argument is provided.
-            set_parts = [f"set {k}={v}" for k, v in env_vars.items()]
             cmd_args = ["cmd", "/c", "claude"] + base_args[1:]  # noqa: RUF005
             py_args_repr = "[" + ",".join(f"'{a}'" for a in cmd_args) + "]"
+            encoded_env = base64.b64encode(json.dumps(env_vars).encode()).decode()
             python_launcher = (
                 'python -c "'
-                "import subprocess,sys;"
-                f"r=subprocess.run({py_args_repr},stdin=open('.hud_prompt.txt','rb'));"
+                "import base64,json,os,subprocess,sys;"
+                "env=os.environ.copy();"
+                f"env.update(json.loads(base64.b64decode('{encoded_env}')));"
+                f"r=subprocess.run({py_args_repr},stdin=open('.hud_prompt.txt','rb'),env=env);"
                 'sys.exit(r.returncode)"'
             )
-            return " && ".join([*set_parts, python_launcher])
+            return python_launcher
 
         # POSIX path: shell-quote everything and embed prompt inline.
         cli_parts = [shlex.quote(a) for a in base_args]

{hud-0.6.16.dev4 → hud-0.6.18.dev1}/hud/agents/gemini/agent.py

@@ -4,18 +4,19 @@ from __future__ import annotations
 
 import base64
 import logging
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, cast
 
 import mcp.types as mcp_types
-from google import genai
 from google.genai import types as genai_types
 
 from hud.agents.tool_agent import RunState, ToolAgent
 from hud.agents.types import AgentStep, Citation, GeminiConfig, Usage
-from hud.settings import settings
 from hud.types import MCPToolCall, MCPToolResult
 from hud.utils import gateway
 
+if TYPE_CHECKING:
+    from google import genai
+
 from .settings import gemini_agent_settings
 from .tools import (
     PREDEFINED_COMPUTER_USE_FUNCTIONS,
@@ -54,14 +55,7 @@ class GeminiAgent(ToolAgent[genai_types.Content, GeminiConfig]):
 
         model_client = config.model_client
         if model_client is None:
-            if settings.api_key:
-                model_client = gateway.build_gateway_client("gemini")
-            elif settings.gemini_api_key:
-                model_client = genai.Client(api_key=settings.gemini_api_key)
-            else:
-                raise ValueError(
-                    "No API key for Gemini. Set HUD_API_KEY or GEMINI_API_KEY.",
-                )
+            model_client = gateway.build_model_client("gemini", gateway=config.gateway)
 
         self.gemini_client: genai.Client = cast("genai.Client", model_client)
         self.temperature = config.temperature

{hud-0.6.16.dev4 → hud-0.6.18.dev1}/hud/agents/openai/agent.py

@@ -24,9 +24,9 @@ from openai.types.responses.response_input_param import (
 )
 from openai.types.shared_params.reasoning import Reasoning  # noqa: TC002
 
-from hud.agents.tool_agent import RunState, ToolAgent
+from hud.agents.tool_agent import DegenerateTurnError, RunState, ToolAgent
+from hud.agents.tools.base import result_text
 from hud.agents.types import AgentStep, Citation, OpenAIConfig, Usage
-from hud.settings import settings
 from hud.types import MCPToolCall, MCPToolResult
 from hud.utils import gateway
 
@@ -41,6 +41,16 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+class EmptyShellCallError(DegenerateTurnError):
+    """The model emitted a shell_call with an empty ``commands`` array.
+
+    OpenAI stores such a turn but returns 500 for every request that continues
+    from it via ``previous_response_id``. The turn is rejected before its
+    response id is committed, so the run state still points at the last
+    healthy response and the next sample continues from there.
+    """
+
+
 @dataclass
 class OpenAIRunState(RunState[ResponseInputItemParam]):
     last_response_id: str | None = None
@@ -62,14 +72,7 @@ class OpenAIAgent(ToolAgent[ResponseInputItemParam, OpenAIConfig]):
 
         model_client = config.model_client
         if model_client is None:
-            if settings.api_key:
-                model_client = gateway.build_gateway_client("openai")
-            elif settings.openai_api_key:
-                model_client = AsyncOpenAI(api_key=settings.openai_api_key)
-            else:
-                raise ValueError(
-                    "No API key for OpenAI. Set HUD_API_KEY or OPENAI_API_KEY.",
-                )
+            model_client = gateway.build_model_client("openai", gateway=config.gateway)
 
         self.openai_client: AsyncOpenAI = cast("AsyncOpenAI", model_client)
         self._model = config.model
@@ -106,8 +109,10 @@ class OpenAIAgent(ToolAgent[ResponseInputItemParam, OpenAIConfig]):
         if isinstance(tool, OpenAIComputerTool):
             screenshot = last_image_content(result)
             if screenshot is None:
-                logger.warning("Computer tool result missing screenshot for call %s", call.name)
-                return None
+                raise RuntimeError(
+                    f"Computer tool result missing screenshot for call {call.id}: "
+                    f"{result_text(result)}"
+                )
             output = ComputerCallOutput(
                 type="computer_call_output",
                 call_id=call.id,
@@ -132,7 +137,17 @@ class OpenAIAgent(ToolAgent[ResponseInputItemParam, OpenAIConfig]):
                         acknowledged.append(raw_check)
                 if acknowledged:
                     output["acknowledged_safety_checks"] = acknowledged
-            return cast("ResponseInputItemParam", output)
+            if result.isError:
+                return [
+                    output,
+                    self._format_message(
+                        "user",
+                        f"Computer call {call.id} stopped with an error. "
+                        "Remaining actions in this call were not executed.\n"
+                        f"{result_text(result)}",
+                    ),
+                ]
+            return output
 
         if isinstance(tool, OpenAIShellTool):
             structured = (
@@ -140,8 +155,6 @@ class OpenAIAgent(ToolAgent[ResponseInputItemParam, OpenAIConfig]):
             )
             output_list = structured.get("output")
             if not isinstance(output_list, list):
-                from hud.agents.tools.base import result_text
-
                 text = result_text(result)
                 output_list = [shell_output("", text, 1 if result.isError else 0)]
             response: dict[str, Any] = {
@@ -223,10 +236,23 @@ class OpenAIAgent(ToolAgent[ResponseInputItemParam, OpenAIConfig]):
             previous_response_id=(
                 oai_state.last_response_id if oai_state.last_response_id is not None else Omit()
             ),
+            prompt_cache_key=self.config.prompt_cache_key,
             truncation=self.truncation if self.truncation is not None else Omit(),
             include=include_param,
         )
 
+        empty_shell_calls = [
+            item.call_id
+            for item in response.output
+            if item.type == "shell_call" and item.action.to_dict().get("commands") == []
+        ]
+        if empty_shell_calls:
+            raise EmptyShellCallError(
+                f"Model emitted shell_call(s) with an empty commands array "
+                f"({', '.join(empty_shell_calls)}); continuing from response {response.id} "
+                "would fail with OpenAI 500s, so it was not committed to the run state."
+            )
+
         oai_state.last_response_id = response.id
         oai_state.message_cursor = len(messages)
 
@@ -290,7 +316,7 @@ class OpenAIAgent(ToolAgent[ResponseInputItemParam, OpenAIConfig]):
                     elif item.action is not None:
                         arguments = item.action.to_dict()
                     else:
-                        raise ValueError("OpenAI computer_call missing action")
+                        arguments = {"actions": []}
                     call_dict: dict[str, Any] = {
                         "name": "computer",
                         "arguments": arguments,
@@ -335,4 +361,4 @@ class OpenAIAgent(ToolAgent[ResponseInputItemParam, OpenAIConfig]):
         )
 
 
-__all__ = ["OpenAIAgent"]
+__all__ = ["EmptyShellCallError", "OpenAIAgent"]