hud-python 0.4.45__py3-none-any.whl → 0.5.13__py3-none-any.whl

hud/clients/environment.py

@@ -0,0 +1,51 @@
+"""Environment-based client adapter for agents."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import mcp.types as types
+
+from hud.types import MCPToolCall, MCPToolResult
+
+if TYPE_CHECKING:
+    from hud.environment import Environment
+    from hud.eval.context import EvalContext
+
+__all__ = ["EnvironmentClient"]
+
+
+class EnvironmentClient:
+    """Adapter wrapping Environment/EvalContext as AgentMCPClient."""
+
+    def __init__(self, env: Environment | EvalContext) -> None:
+        self._env = env
+        self._initialized = False
+
+    @property
+    def mcp_config(self) -> dict[str, dict[str, Any]]:
+        return {}
+
+    @property
+    def is_connected(self) -> bool:
+        return self._initialized
+
+    async def initialize(self, mcp_config: dict[str, dict[str, Any]] | None = None) -> None:
+        if not self._initialized:
+            await self._env.list_tools()
+            self._initialized = True
+
+    async def list_tools(self) -> list[types.Tool]:
+        return await self._env.list_tools()
+
+    async def call_tool(self, tool_call: MCPToolCall) -> MCPToolResult:
+        result = await self._env.call_tool(tool_call.name, **(tool_call.arguments or {}))
+        if isinstance(result, MCPToolResult):
+            return result
+        return MCPToolResult(
+            content=[types.TextContent(type="text", text=str(result))],
+            isError=False,
+        )
+
+    async def shutdown(self) -> None:
+        self._initialized = False

hud/clients/fastmcp.py

@@ -12,6 +12,7 @@ from fastmcp import Client as FastMCPClient
 from mcp import Implementation, types
 from mcp.shared.exceptions import McpError
 
+from hud.settings import settings
 from hud.types import MCPToolCall, MCPToolResult
 from hud.version import __version__ as hud_version
 
@@ -73,7 +74,7 @@ class FastMCPHUDClient(BaseHUDClient):
             return
 
         # Create FastMCP client with the custom transport
-        timeout = 10 * 60  # 5 minutes
+        timeout = settings.client_timeout
         os.environ["FASTMCP_CLIENT_INIT_TIMEOUT"] = str(timeout)
 
         # Create custom transport with retry support for HTTP servers
@@ -91,11 +92,11 @@ class FastMCPHUDClient(BaseHUDClient):
                     # Check if connecting to HUD API
                     for server_config in mcp_config.values():
                         url = server_config.get("url", "")
-                        if "mcp.hud.so" in url:
+                        if "mcp.hud.ai" in url:
                             raise RuntimeError(
                                 "Authentication failed for HUD API. "
                                 "Please ensure your HUD_API_KEY environment variable is set correctly."  # noqa: E501
-                                "You can get an API key at https://hud.so"
+                                "You can get an API key at https://hud.ai"
                             ) from e
                     # Generic 401 error
                     raise RuntimeError(
@@ -110,7 +111,7 @@ class FastMCPHUDClient(BaseHUDClient):
                     hasattr(self._client, "_session_state")
                     and self._client._session_state.session is not None
                 ):
-                    self._client._session_state.session._validate_structured_outputs = (
+                    self._client._session_state.session._validate_structured_outputs = (  # type: ignore[attr-defined]
                         self._strict_validation
                     )
             except ImportError:
@@ -124,6 +125,12 @@ class FastMCPHUDClient(BaseHUDClient):
             raise ValueError("Client is not connected, call initialize() first")
         return await self._client.list_tools()
 
+    async def _list_prompts_impl(self) -> list[types.Prompt]:
+        """List all available prompts (FastMCP supports this)."""
+        if self._client is None:
+            raise ValueError("Client is not connected, call initialize() first")
+        return await self._client.list_prompts()
+
     async def _call_tool(self, tool_call: MCPToolCall) -> MCPToolResult:
         """Execute a tool by name."""
         if self._client is None:
@@ -143,8 +150,8 @@ class FastMCPHUDClient(BaseHUDClient):
             structuredContent=result.structured_content,
         )
 
-    async def list_resources(self) -> list[types.Resource]:
-        """List all available resources."""
+    async def _list_resources_impl(self) -> list[types.Resource]:
+        """Implementation of resource listing for FastMCP client."""
         if self._client is None:
             raise ValueError("Client is not connected, call initialize() first")
         return await self._client.list_resources()

hud/clients/mcp_use.py

@@ -9,9 +9,8 @@ from urllib.parse import urlparse
 
 from mcp import Implementation, types
 from mcp.shared.exceptions import McpError
-from mcp_use.client import MCPClient as MCPUseClient
-from mcp_use.session import MCPSession as MCPUseSession
-from mcp_use.types.http import HttpOptions
+from mcp_use.client.client import MCPClient as MCPUseClient
+from mcp_use.client.session import MCPSession as MCPUseSession
 from pydantic import AnyUrl
 
 from hud.settings import settings
@@ -20,7 +19,6 @@ from hud.utils.hud_console import HUDConsole
 from hud.version import __version__ as hud_version
 
 from .base import BaseHUDClient
-from .utils.retry_transport import create_retry_httpx_client
 
 logger = logging.getLogger(__name__)
 hud_console = HUDConsole(logger=logger)
@@ -58,12 +56,6 @@ class MCPUseHUDClient(BaseHUDClient):
             str, tuple[str, types.Tool, types.Tool]
         ] = {}  # server_name, original_tool, prefixed_tool
         self._client: Any | None = None  # Will be MCPUseClient when available
-        # Transport options for MCP-use (disable_sse_fallback, httpx_client_factory, etc.)
-        # Default to retry-enabled HTTPX client if factory not provided
-        self._http_options: HttpOptions = HttpOptions(
-            httpx_client_factory=create_retry_httpx_client,
-            disable_sse_fallback=True,
-        )
 
     async def _connect(self, mcp_config: dict[str, dict[str, Any]]) -> None:
         """Create all sessions for MCP-use client."""
@@ -71,6 +63,16 @@ class MCPUseHUDClient(BaseHUDClient):
             logger.warning("Client is already connected, cannot connect again")
             return
 
+        # Use configurable timeout for SSE read operations to support long-running tool calls.
+        max_request_timeout = 840
+        for server_cfg in mcp_config.values():
+            if "sse_read_timeout" not in server_cfg:
+                server_cfg["sse_read_timeout"] = (
+                    min(settings.client_timeout, max_request_timeout)
+                    if settings.client_timeout > 0
+                    else max_request_timeout
+                )
+
         # If a server target matches HUD's MCP host and no auth is provided,
         # inject the HUD API key as a Bearer token to avoid OAuth browser flow.
         try:
@@ -88,11 +90,13 @@ class MCPUseHUDClient(BaseHUDClient):
         config = {"mcpServers": mcp_config}
         if MCPUseClient is None:
             raise ImportError("MCPUseClient is not available")
-        self._client = MCPUseClient.from_dict(config, http_options=self._http_options)
+        self._client = MCPUseClient.from_dict(config)
         try:
-            assert self._client is not None  # noqa: S101
+            assert self._client is not None
             self._sessions = await self._client.create_all_sessions()
-            hud_console.info(f"Created {len(self._sessions)} MCP sessions")
+            session_count = len(self._sessions)
+            session_text = "session" if session_count == 1 else "sessions"
+            hud_console.info(f"Created {session_count} MCP {session_text}")
 
             # Configure validation for all sessions based on client setting
             try:
@@ -241,8 +245,8 @@ class MCPUseHUDClient(BaseHUDClient):
             structuredContent=result.structuredContent,
         )
 
-    async def list_resources(self) -> list[types.Resource]:
-        """List all available resources."""
+    async def _list_resources_impl(self) -> list[types.Resource]:
+        """Implementation of resource listing for MCP-use client."""
         if self._client is None or not self._sessions:
             raise ValueError("Client is not connected, call initialize() first")
 
@@ -268,6 +272,32 @@ class MCPUseHUDClient(BaseHUDClient):
                 continue
         return []
 
+    async def _list_prompts_impl(self) -> list[types.Prompt]:
+        """Implementation of prompt listing for MCP-use client (best-effort)."""
+        if self._client is None or not self._sessions:
+            raise ValueError("Client is not connected, call initialize() first")
+
+        all_prompts: list[types.Prompt] = []
+        for server_name, session in self._sessions.items():
+            try:
+                if not hasattr(session, "connector") or not hasattr(
+                    session.connector, "client_session"
+                ):
+                    continue
+                if session.connector.client_session is None:
+                    continue
+
+                if not hasattr(session.connector.client_session, "list_prompts"):
+                    continue
+
+                prompts_result = await session.connector.client_session.list_prompts()
+                all_prompts.extend(prompts_result.prompts)
+            except Exception as e:
+                if self.verbose:
+                    hud_console.debug(f"Could not list prompts from server '{server_name}': {e}")
+                continue
+        return all_prompts
+
     async def read_resource(self, uri: str | AnyUrl) -> types.ReadResourceResult | None:
         """Read a resource by URI from any server that provides it."""
         if self._client is None or not self._sessions:

hud/clients/tests/test_analyze_scenarios.py

@@ -0,0 +1,206 @@
+"""Tests for scenario discovery via prompts/resources in analyze_environment()."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import pytest
+from mcp import types
+from pydantic import AnyUrl
+
+from hud.clients.base import BaseHUDClient
+
+if TYPE_CHECKING:
+    from hud.types import MCPToolCall, MCPToolResult
+
+
+class _MockClient(BaseHUDClient):
+    """Minimal BaseHUDClient for testing analyze_environment scenario derivation."""
+
+    def __init__(
+        self,
+        *,
+        prompts: list[types.Prompt],
+        resources: list[types.Resource],
+    ) -> None:
+        super().__init__(mcp_config={"test": {"url": "mock://test"}}, verbose=True)
+        self._mock_prompts = prompts
+        self._mock_resources = resources
+        # Skip initialize() (which fetches telemetry); we just need analyze_environment().
+        self._initialized = True
+
+    async def _connect(self, mcp_config: dict[str, dict[str, Any]]) -> None:  # pragma: no cover
+        return None
+
+    async def list_tools(self) -> list[types.Tool]:
+        return []
+
+    async def _list_resources_impl(self) -> list[types.Resource]:
+        return self._mock_resources
+
+    async def _list_prompts_impl(self) -> list[types.Prompt]:
+        return self._mock_prompts
+
+    async def _call_tool(self, tool_call: MCPToolCall) -> MCPToolResult:  # pragma: no cover
+        raise NotImplementedError
+
+    async def read_resource(self, uri: str) -> types.ReadResourceResult | None:  # pragma: no cover
+        return None
+
+    async def _disconnect(self) -> None:  # pragma: no cover
+        return None
+
+
+@pytest.mark.asyncio
+async def test_analyze_environment_derives_scenarios_from_scenario_prompt_and_resource() -> None:
+    prompts = [
+        types.Prompt(
+            name="my-env:checkout",
+            description="[Setup] Checkout flow",
+            arguments=[],
+        )
+    ]
+    resources = [
+        types.Resource(
+            uri=AnyUrl("my-env:checkout"),
+            name="checkout",
+            description="[Evaluate] Checkout flow",
+        )
+    ]
+
+    client = _MockClient(prompts=prompts, resources=resources)
+    analysis = await client.analyze_environment()
+
+    assert "scenarios" in analysis
+    assert len(analysis["scenarios"]) == 1
+    scenario = analysis["scenarios"][0]
+    assert scenario["id"] == "my-env:checkout"
+    assert scenario["env"] == "my-env"
+    assert scenario["name"] == "checkout"
+    assert scenario["has_setup_prompt"] is True
+    assert scenario["has_evaluate_resource"] is True
+
+
+@pytest.mark.asyncio
+async def test_analyze_environment_scenario_from_setup_only() -> None:
+    prompts = [
+        types.Prompt(
+            name="env-x:only_setup",
+            description="[Setup] Setup only scenario",
+            arguments=[],
+        )
+    ]
+    resources: list[types.Resource] = []
+
+    client = _MockClient(prompts=prompts, resources=resources)
+    analysis = await client.analyze_environment()
+
+    assert len(analysis["scenarios"]) == 1
+    scenario = analysis["scenarios"][0]
+    assert scenario["id"] == "env-x:only_setup"
+    assert scenario["has_setup_prompt"] is True
+    assert scenario["has_evaluate_resource"] is False
+
+
+@pytest.mark.asyncio
+async def test_analyze_environment_scenario_from_evaluate_only() -> None:
+    prompts: list[types.Prompt] = []
+    resources = [
+        types.Resource(
+            uri=AnyUrl("env-y:only_eval"),
+            name="only_eval",
+            description="[Evaluate] Evaluate only scenario",
+        )
+    ]
+
+    client = _MockClient(prompts=prompts, resources=resources)
+    analysis = await client.analyze_environment()
+
+    assert len(analysis["scenarios"]) == 1
+    scenario = analysis["scenarios"][0]
+    assert scenario["id"] == "env-y:only_eval"
+    assert scenario["has_setup_prompt"] is False
+    assert scenario["has_evaluate_resource"] is True
+
+
+@pytest.mark.asyncio
+async def test_analyze_environment_extracts_scenario_code_from_meta() -> None:
+    """Test that scenario code is extracted from the meta field."""
+    scenario_code = """@env.scenario()
+async def checkout(product_id: str):
+    await env.call_tool("navigate", url="/checkout")
+    yield "Complete the checkout"
+    result = await env.call_tool("check_order")
+    yield 1.0 if result else 0.0
+"""
+    # Use model_validate with _meta alias (Pydantic alias for the meta field)
+    prompts = [
+        types.Prompt.model_validate(
+            {
+                "name": "my-env:checkout",
+                "description": "[Setup] Checkout flow",
+                "arguments": [{"name": "product_id", "required": True}],
+                "_meta": {"code": scenario_code},
+            }
+        )
+    ]
+    resources = [
+        types.Resource.model_validate(
+            {
+                "uri": "my-env:checkout",
+                "name": "checkout",
+                "description": "[Evaluate] Checkout flow",
+                "_meta": {"code": scenario_code},
+            }
+        )
+    ]
+
+    client = _MockClient(prompts=prompts, resources=resources)
+    analysis = await client.analyze_environment()
+
+    assert len(analysis["scenarios"]) == 1
+    scenario = analysis["scenarios"][0]
+    assert scenario["id"] == "my-env:checkout"
+    assert "code" in scenario
+    assert scenario["code"] == scenario_code
+    assert "async def checkout" in scenario["code"]
+
+
+@pytest.mark.asyncio
+async def test_analyze_environment_extracts_meta_on_prompts_and_resources() -> None:
+    """Test that meta field is included in prompts and resources analysis."""
+    meta_data = {"code": "test code", "extra": "value"}
+    # Use model_validate with _meta alias (Pydantic alias for the meta field)
+    prompts = [
+        types.Prompt.model_validate(
+            {
+                "name": "test-prompt",
+                "description": "A test prompt",
+                "arguments": [],
+                "_meta": meta_data,
+            }
+        )
+    ]
+    resources = [
+        types.Resource.model_validate(
+            {
+                "uri": "file:///test",
+                "name": "test-resource",
+                "description": "A test resource",
+                "_meta": meta_data,
+            }
+        )
+    ]
+
+    client = _MockClient(prompts=prompts, resources=resources)
+    analysis = await client.analyze_environment()
+
+    # Check prompts have meta
+    assert len(analysis["prompts"]) == 1
+    assert "meta" in analysis["prompts"][0]
+    assert analysis["prompts"][0]["meta"] == meta_data
+
+    # Check resources have meta
+    assert len(analysis["resources"]) == 1
+    assert "meta" in analysis["resources"][0]
+    assert analysis["resources"][0]["meta"] == meta_data

hud/clients/tests/test_protocol.py

@@ -35,9 +35,15 @@ class MockClient(BaseHUDClient):
             raise RuntimeError("Not connected")
         return self._mock_tools
 
-    async def list_resources(self) -> list[types.Resource]:
-        """Minimal list_resources for protocol satisfaction in tests."""
-        return []
+    async def _list_resources_impl(self) -> list[types.Resource]:
+        """Minimal resource listing implementation for tests."""
+        from pydantic import AnyUrl
+
+        return [
+            types.Resource(
+                uri=AnyUrl("telemetry://live"), name="telemetry", description="Live telemetry data"
+            )
+        ]
 
     async def _call_tool(self, tool_call: MCPToolCall) -> MCPToolResult:
         if tool_call.name == "test_tool":

hud/datasets/__init__.py

@@ -1,33 +1,36 @@
 """HUD datasets module.
 
-Provides data models, utilities, and execution functions for working with HUD datasets.
+Provides unified task loading, saving, and execution for HUD evaluations.
+
+Key functions:
+- load_tasks(): Load tasks from JSON, JSONL, HuggingFace, or HUD API
+- save_tasks(): Save tasks to the HUD API
+- run_dataset(): Run an agent on a dataset of tasks
+- submit_rollouts(): Submit tasks for remote execution
+
+Supports both v4 (LegacyTask) and v5 (Task) formats with automatic conversion.
 """
 
-# Data models
-# Execution functions
 from __future__ import annotations
 
-from hud.types import Task
+from hud.eval.display import display_results
 
-from .parallel import (
-    calculate_optimal_workers,
-    run_dataset_parallel,
-    run_dataset_parallel_manual,
+from .loader import load_dataset, load_tasks, save_tasks
+from .runner import run_dataset, run_single_task
+from .utils import (
+    BatchRequest,
+    SingleTaskRequest,
+    submit_rollouts,
 )
-from .runner import run_dataset
-
-# Utilities
-from .utils import fetch_system_prompt_from_dataset, save_tasks
 
 __all__ = [
-    # Core data model
-    "Task",
-    "calculate_optimal_workers",
-    # Utilities
-    "fetch_system_prompt_from_dataset",
-    # Execution
+    "BatchRequest",
+    "SingleTaskRequest",
+    "display_results",
+    "load_dataset",  # Deprecated alias
+    "load_tasks",
     "run_dataset",
-    "run_dataset_parallel",
-    "run_dataset_parallel_manual",
+    "run_single_task",
     "save_tasks",
+    "submit_rollouts",
 ]