hud-python 0.4.34__py3-none-any.whl → 0.4.35__py3-none-any.whl

hud/agents/claude.py

@@ -6,7 +6,7 @@ import copy
 import logging
 from typing import TYPE_CHECKING, Any, ClassVar, cast
 
-from anthropic import AsyncAnthropic, BadRequestError
+from anthropic import Anthropic, AsyncAnthropic, BadRequestError
 from anthropic.types.beta import BetaContentBlockParam, BetaImageBlockParam, BetaTextBlockParam
 
 import hud
@@ -54,6 +54,7 @@ class ClaudeAgent(MCPAgent):
         model: str = "claude-sonnet-4-20250514",
         max_tokens: int = 4096,
         use_computer_beta: bool = True,
+        validate_api_key: bool = True,
         **kwargs: Any,
     ) -> None:
         """
@@ -75,6 +76,13 @@ class ClaudeAgent(MCPAgent):
                 raise ValueError("Anthropic API key not found. Set ANTHROPIC_API_KEY.")
             model_client = AsyncAnthropic(api_key=api_key)
 
+        # validate api key if requested
+        if validate_api_key:
+            try:
+                Anthropic(api_key=model_client.api_key).models.list()
+            except Exception as e:
+                raise ValueError(f"Anthropic API key is invalid: {e}") from e
+
         self.anthropic_client = model_client
         self.model = model
         self.max_tokens = max_tokens

hud/agents/openai.py

@@ -6,7 +6,7 @@ import logging
 from typing import Any, ClassVar, Literal
 
 import mcp.types as types
-from openai import AsyncOpenAI
+from openai import AsyncOpenAI, OpenAI
 from openai.types.responses import (
     ResponseComputerToolCall,
     ResponseInputMessageContentListParam,
@@ -45,6 +45,7 @@ class OperatorAgent(MCPAgent):
         model_client: AsyncOpenAI | None = None,
         model: str = "computer-use-preview",
         environment: Literal["windows", "mac", "linux", "browser"] = "linux",
+        validate_api_key: bool = True,
         **kwargs: Any,
     ) -> None:
         """
@@ -76,6 +77,13 @@ class OperatorAgent(MCPAgent):
         self.pending_call_id: str | None = None
         self.pending_safety_checks: list[Any] = []
 
+        # validate api key if requested
+        if validate_api_key:
+            try:
+                OpenAI(api_key=self.openai_client.api_key).models.list()
+            except Exception as e:
+                raise ValueError(f"OpenAI API key is invalid: {e}") from e
+
         self.model_name = "openai-" + self.model
 
         # Append OpenAI-specific instructions to the base system prompt

hud/cli/build.py

@@ -489,7 +489,7 @@ def build_environment(
         hud_console.warning("Could not retrieve image ID for lock file")
 
     # Remove temp image after we're done
-    subprocess.run(["docker", "rmi", temp_tag], capture_output=True)  # noqa: S603, S607
+    subprocess.run(["docker", "rmi", "-f", temp_tag], capture_output=True)  # noqa: S603, S607
 
     # Add to local registry
     if image_id:

hud/cli/eval.py

@@ -295,7 +295,7 @@ async def run_full_dataset(
     agent_type: Literal["claude", "openai", "vllm"] = "claude",
     model: str | None = None,
     allowed_tools: list[str] | None = None,
-    max_concurrent: int = 50,
+    max_concurrent: int = 30,
     max_steps: int = 10,
     parallel: bool = False,
     max_workers: int | None = None,

hud/datasets/runner.py

@@ -22,7 +22,7 @@ async def run_dataset(
     dataset: str | Dataset | list[dict[str, Any]],
     agent_class: type[MCPAgent],
     agent_config: dict[str, Any] | None = None,
-    max_concurrent: int = 50,
+    max_concurrent: int = 30,
     metadata: dict[str, Any] | None = None,
     max_steps: int = 10,
     split: str = "train",

hud/utils/tests/test_version.py

@@ -5,4 +5,4 @@ def test_import():
     """Test that the package can be imported."""
     import hud
 
-    assert hud.__version__ == "0.4.34"
+    assert hud.__version__ == "0.4.35"

hud/utils/tool_shorthand.py

@@ -10,7 +10,8 @@ def _is_call_like(obj: Any) -> bool:
         return True
     if len(obj) == 1:
         _, v = next(iter(obj.items()))
-        return isinstance(v, dict)
+        if isinstance(v, dict):
+            return "name" in v or (len(v) == 1 and isinstance(next(iter(v.values())), dict))
     return False
 
 
@@ -19,9 +20,9 @@ def _to_call_dict(obj: Any) -> Any:
 
     Rules:
     - If obj is a dict with {name, arguments}: return {name, arguments: recurse(arguments)}
-    - Else if obj is a single-key dict {k: v}: return {name: k, arguments: recurse(v)}
+    - Else if obj is a single-key dict {k: v} where v looks call-like: return {name: k, arguments: recurse(v)}
     - Else: return obj unchanged (leaf arguments/value)
-    """
+    """  # noqa: E501
     if isinstance(obj, dict):
         if "name" in obj and "arguments" in obj:
             args = obj.get("arguments")
@@ -31,8 +32,10 @@ def _to_call_dict(obj: Any) -> Any:
             return {"name": obj.get("name"), "arguments": args}
         if len(obj) == 1:
             k, v = next(iter(obj.items()))
-            if isinstance(v, dict):
+            # Only convert single-key dicts if the value looks like it could be a call
+            if isinstance(v, dict) and _is_call_like(v):
                 return {"name": k, "arguments": _to_call_dict(v)}
+            # Otherwise, leave it as-is (this is the innermost arguments dict)
             return obj
     return obj
 

hud/version.py

@@ -4,4 +4,4 @@ Version information for the HUD SDK.
 
 from __future__ import annotations
 
-__version__ = "0.4.34"
+__version__ = "0.4.35"

{hud_python-0.4.34.dist-info → hud_python-0.4.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hud-python
-Version: 0.4.34
+Version: 0.4.35
 Summary: SDK for the HUD platform.
 Project-URL: Homepage, https://github.com/hud-evals/hud-python
 Project-URL: Bug Tracker, https://github.com/hud-evals/hud-python/issues

{hud_python-0.4.34.dist-info → hud_python-0.4.35.dist-info}/RECORD

@@ -2,13 +2,13 @@ hud/__init__.py,sha256=JMDFUE1pP0J1Xl_miBdt7ERvoffZmTzSFe8yxz512A8,552
 hud/__main__.py,sha256=YR8Dq8OhINOsVfQ55PmRXXg4fEK84Rt_-rMtJ5rvhWo,145
 hud/settings.py,sha256=sMS31iW1m-5VpWk-Blhi5-obLcUA0fwxWE1GgJz-vqU,2708
 hud/types.py,sha256=RtNM2fPU1NAujTmZLOydQIU-ybk3gVRCoJ2TM2hJOlw,10752
-hud/version.py,sha256=yDsOrKQyUBp8e8KHUcrT2U56qKvsk4D-58XSWaUqJMU,105
+hud/version.py,sha256=FINeU2_U4IFvIW-XEPRMxtXONropSKKTWBc10NjEGws,105
 hud/agents/__init__.py,sha256=UoIkljWdbq4bM0LD-mSaw6w826EqdEjOk7r6glNYwYQ,286
 hud/agents/base.py,sha256=_u1zR3gXzZ1RlTCUYdMcvgHqdJBC4-AB1lZt0yBx8lg,35406
-hud/agents/claude.py,sha256=wHiw8iAnjnRmZyKRKcOhagCDQMhz9Z6rlSBWqH1X--M,15781
+hud/agents/claude.py,sha256=TGhm5gE2ltINDAdEsDxKuT9iGMQ5G87R6kmabU3KPt8,16101
 hud/agents/grounded_openai.py,sha256=U-FHjB2Nh1_o0gmlxY5F17lWJ3oHsNRIB2a7z-IKB64,11231
 hud/agents/langchain.py,sha256=1EgCy8jfjunsWxlPC5XfvfLS6_XZVrIF1ZjtHcrvhYw,9584
-hud/agents/openai.py,sha256=ovARRWNuHqKkZ2Q_OCYSVCIZckrh8XY2jUB2p2x1m88,14259
+hud/agents/openai.py,sha256=O1xV1h1l-W8lmnmXqTYr5CwnmnaniMqOxAZbl2CTTng,14576
 hud/agents/openai_chat_generic.py,sha256=7n7timn3fvNRnL2xzWyOTeNTchej2r9cAL1mU6YnFdY,11605
 hud/agents/misc/__init__.py,sha256=BYi4Ytp9b_vycpZFXnr5Oyw6ncKLNNGml8Jrb7bWUb4,136
 hud/agents/misc/response_agent.py,sha256=uMuRDkz5QgaMQliNzBRepond5sb7KyqIiKm3LstjVnw,3753
@@ -21,11 +21,11 @@ hud/agents/tests/test_openai.py,sha256=1S5IZuc3O3moSp70gqVGjc6m-_b49dCfz2fgX5IGv
 hud/cli/__init__.py,sha256=xL1l5MfdWubd9AWe-cpW64WFS1SVsTgI8fdNdTZhIvs,40259
 hud/cli/__main__.py,sha256=fDH7XITyuDITwSDIVwRso06aouADO0CzTHKqp5TOwJE,143
 hud/cli/analyze.py,sha256=4u5oYfJMquOjT9PzzRTYVcTZDxDi0ilNP_g532_hpOU,14716
-hud/cli/build.py,sha256=X8ykInqvDpsBU0rOU_x_sbp27YwCd160hplZLcwZFEg,18479
+hud/cli/build.py,sha256=cCsCgUD-vX7ZL5h14dGadig_PWRdcQKBdj1MV0C9CTk,18485
 hud/cli/clone.py,sha256=AwVDIuhr8mHb1oT2Af2HrD25SiTdwATpE6zd93vzLgA,6099
 hud/cli/debug.py,sha256=jtFW8J5F_3rhq1Hf1_SkJ7aLS3wjnyIs_LsC8k5cnzc,14200
 hud/cli/dev.py,sha256=56vQdH9oe_XGnOcRcFbNIsLEoBnpCl1eANlRFUeddHQ,31734
-hud/cli/eval.py,sha256=W_eY4uoIQwHcSCvxNaQeRfWC10uQA1UhBWiNQzQPuXM,22694
+hud/cli/eval.py,sha256=53Xx2Yv6yJrNqvU242qBb8hs2Twh1RIoizNvYy6dGKY,22694
 hud/cli/get.py,sha256=sksKrdzBGZa7ZuSoQkc0haj-CvOGVSSikoVXeaUd3N4,6274
 hud/cli/init.py,sha256=McZwpxZMXD-It_PXINCUy-SwUaPiQ7jdpSU5-F-caO8,19671
 hud/cli/list_func.py,sha256=EVi2Vc3Lb3glBNJxFx4MPnZknZ4xmuJz1OFg_dc8a_E,7177
@@ -88,7 +88,7 @@ hud/clients/utils/retry.py,sha256=mMs2T_mAlb8AYhSqMR4AmCw7838gqCC4mdG3zjMAYM4,57
 hud/clients/utils/retry_transport.py,sha256=Rsq25eiKKt_pM1bas78QEZvO0illK97X_3opmaS3A3w,6809
 hud/datasets/__init__.py,sha256=-g05iDy76CU4JiRHjKBBhgh3STtiIjmWhUfPqgf5hJE,697
 hud/datasets/parallel.py,sha256=m7_z2QwjaRuM9gJFYyiPIJUwrlTxZSvFMAd9L2IDZEo,25772
-hud/datasets/runner.py,sha256=2KhGEDzYW_qrSCaNJmsKqiAYZE_-h5VaQ7kv8rSe7Fw,4687
+hud/datasets/runner.py,sha256=43Ua1PUQgnb6cdO9YXJM7kxdlmxPeSV4478Azy5HVGU,4687
 hud/datasets/utils.py,sha256=hdZfjWH5l3FVJaWBSHEEpjujAG7DqEam_vHgslL8MLs,4279
 hud/misc/__init__.py,sha256=m_pprQQ-G-Y0Sd0NEiR8MtAMbElnuFZ2OWT8TXrw7c4,43
 hud/misc/claude_plays_pokemon.py,sha256=IthAkjDVr2Q-GNvX-QLJyMzN7-0pHqqJbagGNv2m7yo,10453
@@ -194,17 +194,17 @@ hud/utils/pretty_errors.py,sha256=WGeL4CTHtlA6KgPuV_JSX5l6H4-xbuTp6Y6tw1bkiFg,24
 hud/utils/progress.py,sha256=suikwFM8sdSfkV10nAOEaInDhG4XKgOSvFePg4jSj1A,5927
 hud/utils/tasks.py,sha256=JwFIq0cpPMpMYnICUmx_G4CF6uy9MtiCmmmN7eA6FsA,4682
 hud/utils/telemetry.py,sha256=hrVIx2rUjSGyy9IVxTZ_3Jii83PiHjyFRd5ls2whimM,1863
-hud/utils/tool_shorthand.py,sha256=nWo-Z7D4w8qF1lWKP7TkXMHZiU3vj4jAwfcBXkwrpnE,1833
+hud/utils/tool_shorthand.py,sha256=_haLgK3yazLR2Y0jlEHUUQjw9uZCxi9yTipAwdOAJ70,2148
 hud/utils/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hud/utils/tests/test_async_utils.py,sha256=RkdSnYErRV3Jn7dfg6CPlcE1RSUL__2B627oIqAyy1s,5945
 hud/utils/tests/test_init.py,sha256=2QLQSGgyP9wJhOvPCusm_zjJad0qApOZi1BXpxcdHXQ,383
 hud/utils/tests/test_mcp.py,sha256=0pUa16mL-bqbZDXp5NHBnt1gO5o10BOg7zTMHZ1DNPM,4023
 hud/utils/tests/test_progress.py,sha256=QSF7Kpi03Ff_l3mAeqW9qs1nhK50j9vBiSobZq7T4f4,7394
 hud/utils/tests/test_telemetry.py,sha256=5jl7bEx8C8b-FfFUko5pf4UY-mPOR-9HaeL98dGtVHM,2781
-hud/utils/tests/test_version.py,sha256=YmDs3a476rPIo4rLodsx3zqGeIIeDCXLjenmKK4-rkE,160
+hud/utils/tests/test_version.py,sha256=gVwJvjGLJ5VNZYJPFRHyfTPWah6I0M4JS0sYTWLoGM4,160
 hud/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hud_python-0.4.34.dist-info/METADATA,sha256=8ju6prkWIMJ5ChDPU-evkWqWucKZ1QyU3XTM2717RYA,20861
-hud_python-0.4.34.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hud_python-0.4.34.dist-info/entry_points.txt,sha256=jJbodNFg1m0-CDofe5AHvB4zKBq7sSdP97-ohaQ3ae4,63
-hud_python-0.4.34.dist-info/licenses/LICENSE,sha256=yIzBheVUf86FC1bztAcr7RYWWNxyd3B-UJQ3uddg1HA,1078
-hud_python-0.4.34.dist-info/RECORD,,
+hud_python-0.4.35.dist-info/METADATA,sha256=bSffhIrX5P4LCM-rTGuZz71gwsIPkoqzFYu-wPcH2SE,20861
+hud_python-0.4.35.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hud_python-0.4.35.dist-info/entry_points.txt,sha256=jJbodNFg1m0-CDofe5AHvB4zKBq7sSdP97-ohaQ3ae4,63
+hud_python-0.4.35.dist-info/licenses/LICENSE,sha256=yIzBheVUf86FC1bztAcr7RYWWNxyd3B-UJQ3uddg1HA,1078
+hud_python-0.4.35.dist-info/RECORD,,