hud-python 0.4.45__py3-none-any.whl → 0.5.13__py3-none-any.whl

hud/eval/utils.py

@@ -0,0 +1,194 @@
+"""Utility functions for the eval module."""
+
+from __future__ import annotations
+
+import logging
+import warnings
+from typing import Any
+
+__all__ = ["build_env_from_v4", "is_v4_format", "validate_v4_task"]
+
+logger = logging.getLogger(__name__)
+
+
+def is_v4_format(data: dict[str, Any]) -> bool:
+    """Detect if dict looks like v4 LegacyTask format.
+
+    Used for branching logic. Checks if data has the core v4 fields
+    (prompt AND mcp_config). Does NOT validate completeness.
+
+    Args:
+        data: Dict to check
+
+    Returns:
+        True if looks like v4 format, False otherwise
+    """
+    if not isinstance(data, dict):
+        return False
+
+    # Core v4 detection: prompt + mcp_config
+    return bool(data.get("prompt")) and bool(data.get("mcp_config"))
+
+
+def validate_v4_task(data: dict[str, Any]) -> None:
+    """Validate v4 task has all required fields.
+
+    A valid v4 task must have all three required fields:
+    - prompt: The task instruction
+    - mcp_config: MCP server configuration
+    - evaluate_tool: How to evaluate success
+
+    Call this after is_v4_format() when you need to ensure completeness.
+
+    Args:
+        data: Dict to validate
+
+    Raises:
+        ValueError: If any required fields are missing
+    """
+    missing = []
+    if not data.get("prompt"):
+        missing.append("prompt")
+    if not data.get("mcp_config"):
+        missing.append("mcp_config")
+    if not data.get("evaluate_tool"):
+        missing.append("evaluate_tool")
+
+    if missing:
+        raise ValueError(f"v4 task missing required fields: {', '.join(missing)}")
+
+
+def build_env_from_v4(source: dict[str, Any] | Any) -> dict[str, Any]:
+    """Build Environment from v4 LegacyTask format.
+
+    Creates an Environment configured with the legacy task's fields.
+    Returns a dict ready to be passed to Task() constructor.
+
+    Args:
+        source: dict or LegacyTask with v4 fields (prompt, mcp_config, etc.)
+
+    Returns:
+        Dict with Task fields: env, id, scenario, args, validation, system_prompt, metadata
+
+    Raises:
+        TypeError: If source is not a dict or LegacyTask
+    """
+    from hud.environment import Environment
+    from hud.types import LegacyTask, MCPToolCall
+
+    # Convert dict to LegacyTask if needed
+    if isinstance(source, dict):
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", category=DeprecationWarning)
+            legacy = LegacyTask(**source)
+    elif isinstance(source, LegacyTask):
+        legacy = source
+    else:
+        raise TypeError(f"Expected dict or LegacyTask, got {type(source).__name__}")
+
+    # Warn if using local MCP configs (command without url)
+    _warn_local_mcp(legacy.mcp_config)
+
+    # Extract tool filters from agent_config (v4 style)
+    # These are agent-level filters, not connection-level
+    include_tools: list[str] | None = None
+    exclude_tools: list[str] | None = None
+    if legacy.agent_config:
+        include_tools = legacy.agent_config.allowed_tools
+        exclude_tools = legacy.agent_config.disallowed_tools
+
+    # Convert ["*"] wildcard to None (meaning include all)
+    if include_tools == ["*"]:
+        include_tools = None
+
+    # Create Environment - NO connections made here, just config stored
+    env = Environment(legacy.id or "v4-legacy")
+    env.connect_mcp_config(legacy.mcp_config)
+
+    # Store agent-level tool filters on Environment (applied in as_tools())
+    # This allows Environment to call setup/evaluate while hiding them from agent
+    env._agent_include = include_tools
+    env._agent_exclude = exclude_tools
+
+    # Set the prompt
+    env.prompt = legacy.prompt
+
+    # Add setup_tool calls (stored, not executed)
+    if legacy.setup_tool:
+        setup_calls = legacy.setup_tool
+        if not isinstance(setup_calls, list):
+            setup_calls = [setup_calls]
+        for call in setup_calls:
+            env.setup_tool(call.name, **(call.arguments or {}))
+
+    # Add evaluate_tool calls (stored, not executed)
+    if legacy.evaluate_tool:
+        eval_calls = legacy.evaluate_tool
+        if not isinstance(eval_calls, list):
+            eval_calls = [eval_calls]
+        for call in eval_calls:
+            env.evaluate_tool(call.name, **(call.arguments or {}))
+
+    # Build Task fields dict
+    result: dict[str, Any] = {
+        "env": env,
+        "id": legacy.id,
+        "scenario": None,  # v4 uses prompt, not scenarios
+        "args": {},
+    }
+
+    # Map integration_test_tool → validation (same concept: tool calls to verify)
+    # Also populate _integration_test_calls for IntegrationTestRunner compatibility
+    if legacy.integration_test_tool:
+        int_test = legacy.integration_test_tool
+        if not isinstance(int_test, list):
+            int_test = [int_test]
+        # Convert to MCPToolCall if needed
+        result["validation"] = [
+            call if isinstance(call, MCPToolCall) else MCPToolCall(**call.model_dump())
+            for call in int_test
+        ]
+        # Populate _integration_test_calls on env for IntegrationTestRunner
+        env._integration_test_calls = [(call.name, call.arguments or {}) for call in int_test]
+
+    # Extract agent_config fields that need to be passed through
+    if legacy.agent_config:
+        agent_config_dict: dict[str, Any] = {}
+        if legacy.agent_config.system_prompt:
+            agent_config_dict["system_prompt"] = legacy.agent_config.system_prompt
+        if legacy.agent_config.append_setup_output:
+            agent_config_dict["append_setup_output"] = legacy.agent_config.append_setup_output
+        if legacy.agent_config.append_setup_tool:
+            agent_config_dict["append_setup_tool"] = legacy.agent_config.append_setup_tool
+        if agent_config_dict:
+            result["agent_config"] = agent_config_dict
+
+    # Preserve metadata
+    if legacy.metadata:
+        result["metadata"] = legacy.metadata
+
+    return result
+
+
+def _warn_local_mcp(mcp_config: dict[str, Any] | None) -> None:
+    """Warn if mcp_config uses local MCP servers (command without url).
+
+    Local MCP servers can cause port conflicts when running tasks concurrently.
+    """
+    if not mcp_config:
+        return
+
+    has_local = any(
+        isinstance(server_cfg, dict) and "command" in server_cfg and not server_cfg.get("url")
+        for server_cfg in mcp_config.values()
+        if isinstance(server_cfg, dict)
+    )
+
+    if has_local:
+        warnings.warn(
+            "Task uses local MCP configuration (command without url). "
+            "This may cause port conflicts when running tasks concurrently. "
+            "Consider using remote MCP servers for parallel execution.",
+            UserWarning,
+            stacklevel=4,
+        )

hud/patches/__init__.py

@@ -0,0 +1,19 @@
+"""
+HUD runtime patches for third-party libraries.
+
+This module applies monkey-patches to fix issues in dependencies
+without requiring forked packages.
+"""
+
+from hud.patches.mcp_patches import apply_all_patches, suppress_fastmcp_logging
+from hud.patches.warnings import apply_default_warning_filters, suppress_mcp_use_import_warnings
+
+# Apply patches on import
+apply_all_patches()
+
+__all__ = [
+    "apply_all_patches",
+    "apply_default_warning_filters",
+    "suppress_fastmcp_logging",
+    "suppress_mcp_use_import_warnings",
+]

hud/patches/mcp_patches.py

@@ -0,0 +1,308 @@
+"""
+Runtime patches for the standard mcp package.
+
+These patches apply fixes from the HUD fork without requiring a separate package.
+Import this module early (e.g., in hud/__init__.py) to apply patches.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def patch_streamable_http_error_handling() -> None:
+    """
+    Patch StreamableHTTPTransport.post_writer to handle request errors properly.
+
+    The original implementation doesn't catch errors in handle_request_async,
+    which can cause the client to hang indefinitely. This patch wraps the handler
+    to send a proper JSONRPCError response when transport errors occur (e.g.,
+    ReadTimeout), allowing the waiting caller to receive the error and fail
+    gracefully instead of hanging.
+    """
+    try:
+        from mcp.client.streamable_http import StreamableHTTPTransport
+
+        async def patched_post_writer(
+            self: Any,
+            client: Any,
+            write_stream_reader: Any,
+            read_stream_writer: Any,
+            write_stream: Any,
+            start_get_stream: Any,
+            tg: Any,
+        ) -> None:
+            import asyncio
+            import ssl
+            import time
+
+            import httpx
+            from mcp.client.streamable_http import RequestContext
+            from mcp.shared.message import ClientMessageMetadata, SessionMessage
+            from mcp.types import ErrorData, JSONRPCError, JSONRPCMessage, JSONRPCRequest
+
+            from hud.settings import settings
+
+            async def handle_request_async(ctx: RequestContext, is_resumption: bool) -> None:
+                msg = ctx.session_message.message
+                # Use configured timeout, minimum 30s to prevent instant failures
+                timeout = max(settings.client_timeout, 15.0)
+                deadline = time.monotonic() + timeout
+                retryable = (
+                    httpx.ConnectError,
+                    httpx.ReadError,
+                    httpx.TimeoutException,
+                    ssl.SSLError,
+                )
+
+                async def send_error_response(exc: Exception) -> None:
+                    """Send an error response to the client."""
+                    if isinstance(msg.root, JSONRPCRequest):
+                        error_response = JSONRPCError(
+                            jsonrpc="2.0",
+                            id=msg.root.id,
+                            error=ErrorData(
+                                code=-32000,
+                                message=f"Transport error: {type(exc).__name__}",
+                                data={"error_type": type(exc).__name__, "detail": str(exc)},
+                            ),
+                        )
+                        await ctx.read_stream_writer.send(
+                            SessionMessage(JSONRPCMessage(error_response))
+                        )
+                    else:
+                        await ctx.read_stream_writer.send(exc)
+
+                while True:
+                    try:
+                        if is_resumption:
+                            await self._handle_resumption_request(ctx)
+                        else:
+                            await self._handle_post_request(ctx)
+                        return
+                    except retryable as e:
+                        if time.monotonic() >= deadline:
+                            logger.error("MCP request failed after timeout: %s", e)
+                            await send_error_response(e)
+                            return
+                        logger.warning("Retrying MCP request after error: %s", e)
+                        await asyncio.sleep(2.0)
+                    except asyncio.CancelledError:
+                        raise
+                    except Exception as e:
+                        logger.exception("Request handler error: %s", e)
+                        await send_error_response(e)
+                        return
+
+            try:
+                async with write_stream_reader:
+                    async for session_message in write_stream_reader:
+                        message = session_message.message
+                        metadata = (
+                            session_message.metadata
+                            if isinstance(session_message.metadata, ClientMessageMetadata)
+                            else None
+                        )
+                        is_resumption = bool(metadata and metadata.resumption_token)
+
+                        logger.debug("Sending client message: %s", message)
+
+                        if self._is_initialized_notification(message):
+                            start_get_stream()
+
+                        ctx = RequestContext(
+                            client=client,
+                            headers=self.request_headers,
+                            session_id=self.session_id,
+                            session_message=session_message,
+                            metadata=metadata,
+                            read_stream_writer=read_stream_writer,
+                            sse_read_timeout=self.sse_read_timeout,
+                        )
+
+                        if isinstance(message.root, JSONRPCRequest):
+                            tg.start_soon(handle_request_async, ctx, is_resumption)
+                        else:
+                            await handle_request_async(ctx, is_resumption)
+
+            except Exception:
+                logger.exception("Error in post_writer")
+            finally:
+                await read_stream_writer.aclose()
+                await write_stream.aclose()
+
+        StreamableHTTPTransport.post_writer = patched_post_writer
+        logger.debug("Patched StreamableHTTPTransport.post_writer")
+
+    except ImportError:
+        logger.debug("mcp.client.streamable_http not available, skipping patch")
+    except Exception as e:
+        logger.warning("Failed to patch streamable_http: %s", e)
+
+
+def patch_client_session_validation() -> None:
+    """
+    Patch ClientSession to skip structured output validation.
+
+    The original validation is strict and raises errors for non-conforming
+    but usable responses. We replace it with a no-op.
+    """
+    try:
+        from mcp.client.session import ClientSession
+
+        async def noop_validate(self: Any, name: str, result: Any) -> None:
+            """Skip structured output validation entirely."""
+
+        ClientSession._validate_tool_result = noop_validate
+        logger.debug("Patched ClientSession._validate_tool_result to skip validation")
+
+    except ImportError:
+        logger.debug("mcp.client.session not available, skipping patch")
+    except Exception as e:
+        logger.warning("Failed to patch client session: %s", e)
+
+
+def patch_server_output_validation() -> None:
+    """
+    Patch MCP server to skip structured output validation and auto-generate
+    structuredContent for FastMCP tools with x-fastmcp-wrap-result.
+    """
+    try:
+        import json
+
+        import mcp.types as types
+        from mcp.server.lowlevel.server import Server
+
+        def patched_call_tool(
+            self: Any, validate_input: bool = True, validate_output: bool = False
+        ) -> Any:
+            """Patched call_tool that skips output validation."""
+
+            def decorator(func: Any) -> Any:
+                async def handler(req: types.CallToolRequest) -> Any:
+                    try:
+                        tool_name = req.params.name
+                        arguments = req.params.arguments or {}
+                        tool = await self._get_cached_tool_definition(tool_name)
+
+                        if validate_input and tool:
+                            try:
+                                import jsonschema
+
+                                jsonschema.validate(instance=arguments, schema=tool.inputSchema)
+                            except jsonschema.ValidationError as e:
+                                return self._make_error_result(
+                                    f"Input validation error: {e.message}"
+                                )
+
+                        results = await func(tool_name, arguments)
+
+                        # output normalization
+                        unstructured_content: list[Any]
+                        maybe_structured_content: dict[str, Any] | None
+                        if isinstance(results, types.CallToolResult):
+                            return types.ServerResult(results)
+                        elif isinstance(results, tuple) and len(results) == 2:
+                            unstructured_content, maybe_structured_content = results
+                        elif isinstance(results, dict):
+                            maybe_structured_content = results
+                            text = json.dumps(results, indent=2)
+                            unstructured_content = [types.TextContent(type="text", text=text)]
+                        elif results is None:
+                            # None means success with no content
+                            unstructured_content = []
+                            maybe_structured_content = None
+                        elif isinstance(results, (str, bytes, bytearray, memoryview)):
+                            # Handle string/bytes explicitly before iterable check
+                            # (these are iterable but should not be split into chars/ints)
+                            if isinstance(results, str):
+                                text = results
+                            elif isinstance(results, memoryview):
+                                text = bytes(results).decode("utf-8", errors="replace")
+                            else:
+                                text = bytes(results).decode("utf-8", errors="replace")
+                            unstructured_content = [types.TextContent(type="text", text=text)]
+                            maybe_structured_content = None
+                        elif isinstance(results, (int, float, bool)):
+                            # Primitives -> string representation
+                            unstructured_content = [
+                                types.TextContent(type="text", text=str(results))
+                            ]
+                            maybe_structured_content = None
+                        elif hasattr(results, "__iter__"):
+                            unstructured_content = list(results)
+                            maybe_structured_content = None
+                        else:
+                            return self._make_error_result(
+                                f"Unexpected return type: {type(results).__name__}"
+                            )
+
+                        # Auto-generate structuredContent for FastMCP tools
+                        # FastMCP generates outputSchema but doesn't populate it
+                        if maybe_structured_content is None and tool:
+                            output_schema = getattr(tool, "outputSchema", None)
+                            if output_schema and output_schema.get("x-fastmcp-wrap-result"):
+                                for item in unstructured_content:
+                                    if isinstance(item, types.TextContent):
+                                        try:
+                                            parsed = json.loads(item.text)
+                                            maybe_structured_content = {"result": parsed}
+                                        except json.JSONDecodeError:
+                                            maybe_structured_content = {"result": item.text}
+                                        break
+
+                        return types.ServerResult(
+                            types.CallToolResult(
+                                content=list(unstructured_content),
+                                structuredContent=maybe_structured_content,
+                                isError=False,
+                            )
+                        )
+                    except Exception as e:
+                        return self._make_error_result(str(e))
+
+                self.request_handlers[types.CallToolRequest] = handler
+                return func
+
+            return decorator
+
+        Server.call_tool = patched_call_tool
+        logger.debug("Patched Server.call_tool to skip output validation")
+
+    except ImportError:
+        logger.debug("mcp.server.lowlevel.server not available, skipping patch")
+    except Exception as e:
+        logger.warning("Failed to patch server output validation: %s", e)
+
+
+def suppress_fastmcp_logging(level: int = logging.WARNING) -> None:
+    """
+    Suppress verbose fastmcp logging.
+
+    FastMCP logs a lot of INFO-level messages that clutter output.
+    This sets all fastmcp loggers to the specified level.
+
+    Args:
+        level: Logging level to set (default: WARNING)
+    """
+    loggers_to_suppress = [
+        "fastmcp",
+        "fastmcp.server.server",
+        "fastmcp.server.openapi",
+        "fastmcp.tools.tool_manager",
+    ]
+    for logger_name in loggers_to_suppress:
+        logging.getLogger(logger_name).setLevel(level)
+    logger.debug("Suppressed fastmcp logging to level %s", level)
+
+
+def apply_all_patches() -> None:
+    """Apply all MCP patches."""
+    patch_streamable_http_error_handling()
+    patch_client_session_validation()
+    patch_server_output_validation()
+    suppress_fastmcp_logging()
+    logger.debug("All MCP patches applied")

hud/patches/warnings.py

@@ -0,0 +1,54 @@
+"""
+Centralized warning filters for noisy third-party dependencies.
+
+Keep these helpers here so the rest of the codebase can stay clean and avoid
+scattering warning filters across unrelated modules.
+"""
+
+from __future__ import annotations
+
+import warnings
+from contextlib import contextmanager
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+
+def apply_default_warning_filters(*, verbose: bool) -> None:
+    """Apply our default warning filters for non-verbose CLI/server modes."""
+    if verbose:
+        return
+
+    warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+    # Pydantic v2 emits PydanticDeprecatedSince20 for v1-style config usage in deps.
+    try:
+        from pydantic.warnings import PydanticDeprecatedSince20
+    except Exception:
+        return
+
+    warnings.filterwarnings("ignore", category=PydanticDeprecatedSince20)
+
+
+@contextmanager
+def suppress_mcp_use_import_warnings() -> Iterator[None]:
+    """Suppress known noisy warnings emitted during `mcp_use` imports."""
+    try:
+        from pydantic.warnings import PydanticDeprecatedSince20
+    except Exception:  # pragma: no cover
+        PydanticDeprecatedSince20 = None  # type: ignore[assignment]
+
+    with warnings.catch_warnings():
+        # mcp_use currently emits DeprecationWarning from its package __init__.py.
+        warnings.filterwarnings("ignore", category=DeprecationWarning, module=r"mcp_use(\..*)?$")
+
+        # mcp_use currently defines Pydantic v1-style `class Config` in oauth models.
+        if PydanticDeprecatedSince20 is not None:
+            warnings.filterwarnings(
+                "ignore",
+                category=PydanticDeprecatedSince20,
+                module=r"mcp_use\.client\.auth\.oauth$",
+            )
+
+        yield

hud/samples/browser.py

@@ -7,17 +7,17 @@ from typing import Any
 from pydantic import Field
 
 from hud.settings import settings
-from hud.types import MCPToolCall, Task
+from hud.types import LegacyTask, MCPToolCall
 
 
-class BrowserTask(Task):
-    """Task subclass with browser defaults for BrowserTask(prompt=...)."""
+class BrowserTask(LegacyTask):
+    """LegacyTask subclass with browser defaults for BrowserTask(prompt=...)."""
 
     prompt: str = "Open Google and be ready to search."
     mcp_config: dict[str, Any] = Field(
         default_factory=lambda: {
             "browser": {
-                "url": "https://mcp.hud.so/v3/mcp",
+                "url": settings.hud_mcp_url,
                 "headers": {
                     "Authorization": f"Bearer {settings.api_key}",
                     "Mcp-Image": "hudevals/hud-remote-browser:0.1.1",

hud/server/__init__.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from .router import MCPRouter
 from .server import MCPServer
 
-__all__ = ["MCPServer"]
+__all__ = ["MCPRouter", "MCPServer"]

hud/server/low_level.py

@@ -89,11 +89,12 @@ class LowLevelServerWithInit(_BaseLL):
 
     def __init__(
         self,
+        fastmcp: Any,
         *args: Any,
         init_fn: Callable[[RequestContext], Awaitable[None]] | None = None,
         **kwargs: Any,
     ) -> None:
-        super().__init__(*args, **kwargs)
+        super().__init__(fastmcp, *args, **kwargs)
         self._init_fn = init_fn
 
     async def run(