hud-python 0.5.2__tar.gz → 0.5.3__tar.gz

{hud_python-0.5.2 → hud_python-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hud-python
-Version: 0.5.2
+Version: 0.5.3
 Summary: SDK for the HUD platform.
 Project-URL: Homepage, https://github.com/hud-evals/hud-python
 Project-URL: Bug Tracker, https://github.com/hud-evals/hud-python/issues

hud_python-0.5.3/hud/agents/__init__.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from typing import Any
+
+from .base import MCPAgent
+from .openai import OpenAIAgent
+from .openai_chat import OpenAIChatAgent
+from .operator import OperatorAgent
+
+__all__ = [
+    "MCPAgent",
+    "OpenAIAgent",
+    "OpenAIChatAgent",
+    "OperatorAgent",
+    "create_agent",
+]
+
+
+def create_agent(model: str, **kwargs: Any) -> MCPAgent:
+    """Create an agent for a gateway model.
+
+    This routes ALL requests through the HUD gateway. For direct API access
+    (using your own API keys), use the agent classes directly.
+
+    Args:
+        model: Model name (e.g., "gpt-4o", "claude-sonnet-4-5").
+        **kwargs: Additional params passed to agent.create().
+
+    Returns:
+        Configured MCPAgent instance with gateway routing.
+
+    Example:
+        ```python
+        # Gateway routing (recommended)
+        agent = create_agent("gpt-4o")
+        agent = create_agent("claude-sonnet-4-5", temperature=0.7)
+
+        # Direct API access (use agent classes)
+        from hud.agents.claude import ClaudeAgent
+
+        agent = ClaudeAgent.create(model="claude-sonnet-4-5")
+        ```
+    """
+    from hud.agents.gateway import build_gateway_client
+    from hud.agents.resolver import resolve_cls
+
+    # Resolve class and gateway info
+    agent_cls, gateway_info = resolve_cls(model)
+
+    # Get model ID from gateway info or use input
+    model_id = model
+    if gateway_info:
+        model_id = gateway_info.get("model") or gateway_info.get("id") or model
+
+    # Determine provider: from gateway info, or infer from agent class
+    if gateway_info:
+        provider = gateway_info.get("provider") or "openai"
+    else:
+        # Map agent class to provider for known types
+        from hud.agents.claude import ClaudeAgent
+        from hud.agents.gemini import GeminiAgent
+
+        _AGENT_TO_PROVIDER = {
+            ClaudeAgent: "anthropic",
+            GeminiAgent: "google",
+        }
+        provider = _AGENT_TO_PROVIDER.get(agent_cls, "openai")
+
+    client = build_gateway_client(provider)
+
+    # Set up kwargs
+    kwargs.setdefault("model", model_id)
+
+    # Use correct client key based on agent type
+    if agent_cls == OpenAIChatAgent:
+        kwargs.setdefault("openai_client", client)
+    else:
+        # Claude and other agents use model_client and validate_api_key
+        kwargs.setdefault("model_client", client)
+        kwargs.setdefault("validate_api_key", False)
+
+    return agent_cls.create(**kwargs)

hud_python-0.5.3/hud/agents/gateway.py

@@ -0,0 +1,42 @@
+"""Gateway client utilities for HUD inference gateway."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def build_gateway_client(provider: str) -> Any:
+    """Build a client configured for HUD gateway routing.
+
+    Args:
+        provider: Provider name ("anthropic", "openai", "gemini", etc.)
+
+    Returns:
+        Configured async client for the provider.
+    """
+    from hud.settings import settings
+
+    provider = provider.lower()
+
+    if provider == "anthropic":
+        from anthropic import AsyncAnthropic
+
+        return AsyncAnthropic(api_key=settings.api_key, base_url=settings.hud_gateway_url)
+
+    if provider == "gemini":
+        from google import genai
+        from google.genai.types import HttpOptions
+
+        return genai.Client(
+            api_key="PLACEHOLDER",
+            http_options=HttpOptions(
+                api_version="v1beta",
+                base_url=settings.hud_gateway_url,
+                headers={"Authorization": f"Bearer {settings.api_key}"},
+            ),
+        )
+
+    # OpenAI-compatible (openai, azure, together, groq, fireworks, etc.)
+    from openai import AsyncOpenAI
+
+    return AsyncOpenAI(api_key=settings.api_key, base_url=settings.hud_gateway_url)

hud_python-0.5.3/hud/agents/resolver.py

@@ -0,0 +1,70 @@
+"""Model resolution - maps model strings to agent classes."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from hud.agents.base import MCPAgent
+
+__all__ = ["resolve_cls"]
+
+_models_cache: list[dict[str, Any]] | None = None
+
+# Provider name → AgentType value (only anthropic differs)
+_PROVIDER_TO_AGENT = {"anthropic": "claude"}
+
+
+def _fetch_gateway_models() -> list[dict[str, Any]]:
+    """Fetch available models from HUD gateway (cached)."""
+    global _models_cache
+    if _models_cache is not None:
+        return _models_cache
+
+    import httpx
+
+    from hud.settings import settings
+
+    if not settings.api_key:
+        return []
+
+    try:
+        resp = httpx.get(
+            f"{settings.hud_gateway_url}/models",
+            headers={"Authorization": f"Bearer {settings.api_key}"},
+            timeout=10.0,
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        _models_cache = data.get("data", data) if isinstance(data, dict) else data
+        return _models_cache or []
+    except Exception:
+        return []
+
+
+def resolve_cls(model: str) -> tuple[type[MCPAgent], dict[str, Any] | None]:
+    """Resolve model string to (agent_class, gateway_info).
+
+    Returns:
+        (agent_class, None) for known AgentTypes
+        (agent_class, gateway_model_info) for gateway models
+    """
+    from hud.types import AgentType
+
+    # Known AgentType → no gateway info
+    try:
+        return AgentType(model).cls, None
+    except ValueError:
+        pass
+
+    # Gateway lookup
+    for m in _fetch_gateway_models():
+        if model in (m.get("id"), m.get("name"), m.get("model")):
+            provider = (m.get("provider") or "openai_compatible").lower()
+            agent_str = _PROVIDER_TO_AGENT.get(provider, provider)
+            try:
+                return AgentType(agent_str).cls, m
+            except ValueError:
+                return AgentType.OPENAI_COMPATIBLE.cls, m
+
+    raise ValueError(f"Model '{model}' not found")

hud_python-0.5.3/hud/agents/tests/test_resolver.py

@@ -0,0 +1,192 @@
+"""Tests for model resolution and create_agent."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from hud.agents import create_agent
+from hud.agents.resolver import resolve_cls
+
+
+class TestResolveCls:
+    """Tests for resolve_cls function."""
+
+    def test_resolves_known_agent_type(self) -> None:
+        """Known AgentType strings resolve to their class."""
+        from hud.agents.claude import ClaudeAgent
+
+        cls, gateway_info = resolve_cls("claude")
+        assert cls == ClaudeAgent
+        assert gateway_info is None
+
+    def test_resolves_openai(self) -> None:
+        """Resolves 'openai' to OpenAIAgent."""
+        from hud.agents import OpenAIAgent
+
+        cls, _gateway_info = resolve_cls("openai")
+        assert cls == OpenAIAgent
+
+    def test_resolves_gemini(self) -> None:
+        """Resolves 'gemini' to GeminiAgent."""
+        from hud.agents.gemini import GeminiAgent
+
+        cls, _gateway_info = resolve_cls("gemini")
+        assert cls == GeminiAgent
+
+    def test_unknown_model_without_gateway_raises(self) -> None:
+        """Unknown model with no gateway models raises ValueError."""
+        with (
+            patch("hud.agents.resolver._fetch_gateway_models", return_value=[]),
+            pytest.raises(ValueError, match="not found"),
+        ):
+            resolve_cls("unknown-model-xyz")
+
+    def test_resolves_gateway_model(self) -> None:
+        """Resolves model found in gateway."""
+        from hud.agents import OpenAIAgent
+
+        mock_models = [
+            {"id": "gpt-4o", "model": "gpt-4o", "provider": "openai"},
+        ]
+
+        with patch("hud.agents.resolver._fetch_gateway_models", return_value=mock_models):
+            cls, info = resolve_cls("gpt-4o")
+            assert cls == OpenAIAgent
+            assert info is not None
+            assert info["id"] == "gpt-4o"
+
+    def test_resolves_anthropic_provider_to_claude(self) -> None:
+        """Provider 'anthropic' maps to ClaudeAgent."""
+        from hud.agents.claude import ClaudeAgent
+
+        mock_models = [
+            {"id": "claude-sonnet", "model": "claude-3-sonnet", "provider": "anthropic"},
+        ]
+
+        with patch("hud.agents.resolver._fetch_gateway_models", return_value=mock_models):
+            cls, _info = resolve_cls("claude-sonnet")
+            assert cls == ClaudeAgent
+
+    def test_resolves_unknown_provider_to_openai_compatible(self) -> None:
+        """Unknown provider maps to OpenAIChatAgent."""
+        from hud.agents.openai_chat import OpenAIChatAgent
+
+        mock_models = [
+            {"id": "custom-model", "model": "custom", "provider": "custom-provider"},
+        ]
+
+        with patch("hud.agents.resolver._fetch_gateway_models", return_value=mock_models):
+            cls, _info = resolve_cls("custom-model")
+            assert cls == OpenAIChatAgent
+
+
+class TestCreateAgent:
+    """Tests for create_agent function - gateway-only."""
+
+    def test_creates_with_gateway_client(self) -> None:
+        """create_agent always uses gateway routing."""
+        from hud.agents import OpenAIAgent
+
+        mock_models = [
+            {"id": "gpt-4o", "model": "gpt-4o", "provider": "openai"},
+        ]
+
+        with (
+            patch("hud.agents.resolver._fetch_gateway_models", return_value=mock_models),
+            patch.object(OpenAIAgent, "create") as mock_create,
+            patch("hud.agents.gateway.build_gateway_client") as mock_build_client,
+        ):
+            mock_client = MagicMock()
+            mock_build_client.return_value = mock_client
+            mock_agent = MagicMock()
+            mock_create.return_value = mock_agent
+
+            agent = create_agent("gpt-4o")
+
+            # Should have set model and model_client
+            call_kwargs = mock_create.call_args.kwargs
+            assert call_kwargs["model"] == "gpt-4o"
+            assert "model_client" in call_kwargs
+            assert agent == mock_agent
+
+    def test_passes_kwargs_to_create(self) -> None:
+        """Extra kwargs are passed to agent.create()."""
+        from hud.agents import OpenAIAgent
+
+        mock_models = [
+            {"id": "gpt-4o", "model": "gpt-4o", "provider": "openai"},
+        ]
+
+        with (
+            patch("hud.agents.resolver._fetch_gateway_models", return_value=mock_models),
+            patch.object(OpenAIAgent, "create") as mock_create,
+            patch("hud.agents.gateway.build_gateway_client"),
+        ):
+            mock_create.return_value = MagicMock()
+
+            create_agent("gpt-4o", temperature=0.5, max_tokens=1000)
+
+            call_kwargs = mock_create.call_args.kwargs
+            assert call_kwargs["temperature"] == 0.5
+            assert call_kwargs["max_tokens"] == 1000
+
+    def test_known_agent_type_also_uses_gateway(self) -> None:
+        """Even 'claude' string uses gateway (it's a gateway shortcut)."""
+        from hud.agents.claude import ClaudeAgent
+
+        with (
+            patch.object(ClaudeAgent, "create") as mock_create,
+            patch("hud.agents.gateway.build_gateway_client") as mock_build_client,
+        ):
+            mock_client = MagicMock()
+            mock_build_client.return_value = mock_client
+            mock_create.return_value = MagicMock()
+
+            create_agent("claude")
+
+            # Should still build gateway client
+            mock_build_client.assert_called_once()
+            call_kwargs = mock_create.call_args.kwargs
+            assert "model_client" in call_kwargs
+
+
+class TestBuildGatewayClient:
+    """Tests for build_gateway_client function."""
+
+    def test_builds_anthropic_client(self) -> None:
+        """Builds AsyncAnthropic for anthropic provider."""
+        from hud.agents.gateway import build_gateway_client
+
+        with patch("hud.settings.settings") as mock_settings:
+            mock_settings.api_key = "test-key"
+            mock_settings.hud_gateway_url = "https://gateway.hud.ai"
+
+            with patch("anthropic.AsyncAnthropic") as mock_client_cls:
+                build_gateway_client("anthropic")
+                mock_client_cls.assert_called_once()
+
+    def test_builds_openai_client_for_openai(self) -> None:
+        """Builds AsyncOpenAI for openai provider."""
+        from hud.agents.gateway import build_gateway_client
+
+        with patch("hud.settings.settings") as mock_settings:
+            mock_settings.api_key = "test-key"
+            mock_settings.hud_gateway_url = "https://gateway.hud.ai"
+
+            with patch("openai.AsyncOpenAI") as mock_client_cls:
+                build_gateway_client("openai")
+                mock_client_cls.assert_called_once()
+
+    def test_builds_openai_client_for_unknown(self) -> None:
+        """Builds AsyncOpenAI for unknown providers (openai-compatible)."""
+        from hud.agents.gateway import build_gateway_client
+
+        with patch("hud.settings.settings") as mock_settings:
+            mock_settings.api_key = "test-key"
+            mock_settings.hud_gateway_url = "https://gateway.hud.ai"
+
+            with patch("openai.AsyncOpenAI") as mock_client_cls:
+                build_gateway_client("together")
+                mock_client_cls.assert_called_once()

{hud_python-0.5.2 → hud_python-0.5.3}/hud/cli/eval.py

@@ -338,47 +338,27 @@ class EvalConfig(BaseModel):
 
         # Configure gateway mode - route LLM API calls through HUD gateway
         if self.gateway:
-            hud_api_key = settings.api_key
-            if not hud_api_key:
+            if not settings.api_key:
                 raise typer.Exit(1)  # Already validated in validate_api_keys()
 
-            if self.agent_type == AgentType.CLAUDE:
-                from anthropic import AsyncAnthropic
-
-                kwargs["model_client"] = AsyncAnthropic(
-                    api_key=hud_api_key,
-                    base_url=settings.hud_gateway_url,
-                )
-                hud_console.info("🌐 Using HUD Gateway for Claude API")
-            elif self.agent_type in (AgentType.OPENAI, AgentType.OPERATOR):
-                from openai import AsyncOpenAI
+            from hud.agents.gateway import build_gateway_client
 
-                kwargs["model_client"] = AsyncOpenAI(
-                    api_key=hud_api_key,
-                    base_url=settings.hud_gateway_url,
-                )
-                hud_console.info("🌐 Using HUD Gateway for OpenAI API")
-            elif self.agent_type == AgentType.OPENAI_COMPATIBLE:
-                from openai import AsyncOpenAI
+            # Map AgentType to provider
+            agent_to_provider = {
+                AgentType.CLAUDE: "anthropic",
+                AgentType.OPENAI: "openai",
+                AgentType.OPERATOR: "openai",
+                AgentType.GEMINI: "gemini",
+                AgentType.GEMINI_CUA: "gemini",
+                AgentType.OPENAI_COMPATIBLE: "openai",
+            }
+            provider = agent_to_provider.get(self.agent_type, "openai")
+            client = build_gateway_client(provider)
 
-                kwargs["openai_client"] = AsyncOpenAI(
-                    api_key=hud_api_key,
-                    base_url=settings.hud_gateway_url,
-                )
-                hud_console.info("🌐 Using HUD Gateway for OpenAI-compatible API")
-            elif self.agent_type in (AgentType.GEMINI, AgentType.GEMINI_CUA):
-                from google import genai
-                from google.genai.types import HttpOptions
-
-                kwargs["model_client"] = genai.Client(
-                    api_key="PLACEHOLDER",
-                    http_options=HttpOptions(
-                        api_version="v1beta",
-                        base_url=settings.hud_gateway_url,
-                        headers={"Authorization": f"Bearer {hud_api_key}"},
-                    ),
-                )
-                hud_console.info("🌐 Using HUD Gateway for Gemini API")
+            # OpenAI-compatible uses openai_client key
+            is_oai_compat = self.agent_type == AgentType.OPENAI_COMPATIBLE
+            kwargs["openai_client" if is_oai_compat else "model_client"] = client
+            hud_console.info(f"🌐 Using HUD Gateway for {provider} API")
 
         return kwargs
 

{hud_python-0.5.2 → hud_python-0.5.3}/hud/datasets/loader.py

@@ -63,7 +63,8 @@ def _load_from_file(path: Path) -> list[Task]:
     from hud.eval.task import Task
 
     raw_items = _load_raw_from_file(path)
-    return [Task(**item) for item in raw_items]
+    # Default args to {} for runnable tasks (None = template)
+    return [Task(**{**item, "args": item.get("args") or {}}) for item in raw_items]
 
 
 def _load_raw_from_huggingface(dataset_name: str) -> list[dict[str, Any]]:
@@ -99,7 +100,8 @@ def _load_from_huggingface(dataset_name: str) -> list[Task]:
     raw_items = _load_raw_from_huggingface(dataset_name)
     from hud.eval.task import Task
 
-    return [Task(**item) for item in raw_items]
+    # Default args to {} for runnable tasks (None = template)
+    return [Task(**{**item, "args": item.get("args") or {}}) for item in raw_items]
 
 
 def _load_raw_from_api(dataset_name: str) -> list[dict[str, Any]]:
@@ -138,7 +140,8 @@ def _load_from_api(dataset_name: str) -> list[Task]:
     from hud.eval.task import Task
 
     raw_items = _load_raw_from_api(dataset_name)
-    return [Task(**item) for item in raw_items]
+    # Default args to {} for runnable tasks (None = template)
+    return [Task(**{**item, "args": item.get("args") or {}}) for item in raw_items]
 
 
 @overload

{hud_python-0.5.2 → hud_python-0.5.3}/hud/datasets/runner.py

@@ -40,7 +40,7 @@ async def run_dataset(
             - A source string (file path, API slug) - loaded via load_tasks()
             - A single TaskInput (Task, LegacyTask, or dict)
             - A list of TaskInput objects
-        agent_type: Type of agent to create (e.g., "claude", "openai", AgentType.CLAUDE).
+        agent_type: Agent type (e.g., "claude", "openai", AgentType.CLAUDE).
         agent_params: Parameters to pass to agent.create().
         max_steps: Maximum steps per task.
         max_concurrent: Maximum concurrent tasks (for parallel execution).
@@ -70,6 +70,10 @@ async def run_dataset(
     from hud.datasets.loader import load_tasks
     from hud.eval.task import Task
 
+    # Normalize agent_type to AgentType enum
+    if isinstance(agent_type, str):
+        agent_type = AgentType(agent_type)
+
     # Normalize tasks to list[Task]
     task_list: list[Task]
     if isinstance(tasks, str):
@@ -86,10 +90,6 @@ async def run_dataset(
     if not task_list:
         raise ValueError("No tasks to run")
 
-    # Resolve agent class
-    agent_type_enum = agent_type if isinstance(agent_type, AgentType) else AgentType(agent_type)
-    agent_cls = agent_type_enum.cls
-
     # Use hud.eval() for both single and parallel execution
     async with hud.eval(
         task_list,
@@ -97,8 +97,8 @@ async def run_dataset(
         max_concurrent=max_concurrent,
         quiet=quiet,
     ) as ctx:
-        # Create agent fresh for each context (ensures correct tool initialization)
-        agent = agent_cls.create(**(agent_params or {}))
+        # Create agent using AgentType.cls.create()
+        agent = agent_type.cls.create(**(agent_params or {}))
         await agent.run(ctx, max_steps=max_steps)
         # Reward is computed by EvalContext.__aexit__ from evaluate tools
 
@@ -198,9 +198,8 @@ async def run_single_task(
         if ctx.system_prompt and "system_prompt" not in final_agent_params:
             final_agent_params["system_prompt"] = ctx.system_prompt
 
-        # Create agent inside ctx so it has access to context-derived values
-        agent_cls = agent_type.cls
-        agent = agent_cls.create(**final_agent_params)
+        # Create agent using AgentType.cls.create()
+        agent = agent_type.cls.create(**final_agent_params)
 
         # Store metadata if provided
         if metadata:

{hud_python-0.5.2 → hud_python-0.5.3}/hud/environment/environment.py

@@ -362,6 +362,22 @@ class Environment(
             await asyncio.gather(*[c.disconnect() for c in self._connections.values()])
         self._router.clear()
 
+    async def run_async(
+        self,
+        transport: Literal["stdio", "http", "sse"] | None = None,
+        show_banner: bool = True,
+        **transport_kwargs: Any,
+    ) -> None:
+        """Run the MCP server, auto-connecting all connectors first.
+
+        This ensures that tools from external MCP servers (via connect_mcp_config)
+        are discovered and available when the server starts.
+        """
+        async with self:  # Connect all connectors via __aenter__
+            await super().run_async(
+                transport=transport, show_banner=show_banner, **transport_kwargs
+            )
+
     async def _build_routing(self) -> None:
         """Build tool routing from local tools and connection caches."""
         # Use get_tools() not list_tools() - it includes mounted servers without
@@ -376,6 +392,27 @@ class Environment(
         # Populate mock schemas for auto-generated mock values
         self._populate_mock_schemas()
 
+    # =========================================================================
+    # MCP Protocol Overrides - Include connector tools in MCP responses
+    # =========================================================================
+
+    def _setup_handlers(self) -> None:
+        """Override FastMCP to register our custom handlers for tools."""
+        # Call parent to set up all standard handlers
+        super()._setup_handlers()
+        # Re-register our custom handlers (overwrites parent's registrations)
+        self._mcp_server.list_tools()(self._env_list_tools)
+        self._mcp_server.call_tool()(self._env_call_tool)
+
+    async def _env_list_tools(self) -> list[mcp_types.Tool]:
+        """Return all tools including those from connectors."""
+        return self._router.tools
+
+    async def _env_call_tool(self, name: str, arguments: dict[str, Any] | None = None) -> list[Any]:
+        """Route tool calls through our router (handles both local and connector tools)."""
+        result = await self._execute_tool(name, arguments or {})
+        return result.content or []
+
     # =========================================================================
     # Tool Operations
     # =========================================================================