hud-python 0.4.52__py3-none-any.whl → 0.4.54__py3-none-any.whl

hud/cli/eval.py

@@ -5,13 +5,14 @@ from __future__ import annotations
 import asyncio
 import logging
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal
+from typing import TYPE_CHECKING, Any
 
 import typer
 
 import hud
 from hud.cli.utils.env_check import ensure_built, find_environment_dir
 from hud.settings import settings
+from hud.types import AgentType
 from hud.utils.group_eval import display_group_statistics, run_tasks_grouped
 from hud.utils.hud_console import HUDConsole
 
@@ -68,8 +69,52 @@ def get_available_models() -> list[dict[str, str | None]]:
         return []
 
 
+def _build_vllm_config(
+    vllm_base_url: str | None,
+    model: str | None,
+    allowed_tools: list[str] | None,
+    verbose: bool,
+) -> dict[str, Any]:
+    """Build configuration for vLLM agent.
+
+    Args:
+        vllm_base_url: Optional base URL for vLLM server
+        model: Model name to use
+        allowed_tools: Optional list of allowed tools
+        verbose: Enable verbose output
+
+    Returns:
+        Dictionary with agent configuration
+    """
+    # Determine base URL and API key
+    if vllm_base_url is not None:
+        base_url = vllm_base_url
+        api_key = settings.api_key if base_url.startswith(settings.hud_rl_url) else "token-abc123"
+        hud_console.info(f"Using vLLM server at {base_url}")
+    else:
+        base_url = "http://localhost:8000/v1"
+        api_key = "token-abc123"
+
+    config: dict[str, Any] = {
+        "api_key": api_key,
+        "base_url": base_url,
+        "model_name": model or "served-model",
+        "verbose": verbose,
+        "completion_kwargs": {
+            "temperature": 0.7,
+            "max_tokens": 2048,
+            "tool_choice": "auto",
+        },
+    }
+
+    if allowed_tools:
+        config["allowed_tools"] = allowed_tools
+
+    return config
+
+
 def build_agent(
-    agent_type: Literal["claude", "openai", "vllm", "litellm", "integration_test"],
+    agent_type: AgentType,
     *,
     model: str | None = None,
     allowed_tools: list[str] | None = None,
@@ -79,15 +124,13 @@ def build_agent(
     """Create and return the requested agent type."""
 
     # Import agents lazily to avoid dependency issues
-    if agent_type == "integration_test":
+    if agent_type == AgentType.INTEGRATION_TEST:
         from hud.agents.misc.integration_test_agent import IntegrationTestRunner
 
         return IntegrationTestRunner(verbose=verbose)
-    elif agent_type == "vllm":
+    elif agent_type == AgentType.VLLM:
         # Create a generic OpenAI agent for vLLM server
         try:
-            from openai import AsyncOpenAI
-
             from hud.agents.openai_chat_generic import GenericOpenAIChatAgent
         except ImportError as e:
             hud_console.error(
@@ -96,38 +139,16 @@ def build_agent(
             )
             raise typer.Exit(1) from e
 
-        # Determine the base URL to use
-        if vllm_base_url is not None:
-            # Use the provided vLLM URL (for custom/local servers)
-            base_url = vllm_base_url
-            hud_console.info(f"Using vLLM server at {base_url}")
-            api_key = (
-                settings.api_key if base_url.startswith(settings.hud_rl_url) else "token-abc123"
-            )
-        else:
-            # Default to localhost
-            base_url = "http://localhost:8000/v1"
-            api_key = "token-abc123"
-
-        # Create OpenAI client for vLLM
-        openai_client = AsyncOpenAI(
-            base_url=base_url,
-            api_key=api_key,
-            timeout=30.0,
-        )
-
-        return GenericOpenAIChatAgent(
-            openai_client=openai_client,
-            model_name=model or "served-model",  # Default model name
+        # Use the shared config builder
+        config = _build_vllm_config(
+            vllm_base_url=vllm_base_url,
+            model=model,
+            allowed_tools=allowed_tools,
             verbose=verbose,
-            completion_kwargs={
-                "temperature": 0.7,
-                "max_tokens": 2048,
-                "tool_choice": "required",  # if self.actor_config.force_tool_choice else "auto",
-            },
         )
+        return GenericOpenAIChatAgent(**config)
 
-    elif agent_type == "openai":
+    elif agent_type == AgentType.OPENAI:
         try:
             from hud.agents import OperatorAgent
         except ImportError as e:
@@ -145,7 +166,7 @@ def build_agent(
         else:
             return OperatorAgent(verbose=verbose)
 
-    elif agent_type == "litellm":
+    elif agent_type == AgentType.LITELLM:
         try:
             from hud.agents.lite_llm import LiteAgent
         except ImportError as e:
@@ -189,7 +210,7 @@ def build_agent(
 async def run_single_task(
     source: str,
     *,
-    agent_type: Literal["claude", "openai", "vllm", "litellm", "integration_test"] = "claude",
+    agent_type: AgentType = AgentType.CLAUDE,
     model: str | None = None,
     allowed_tools: list[str] | None = None,
     max_steps: int = 10,
@@ -248,42 +269,34 @@ async def run_single_task(
 
     # Use grouped evaluation if group_size > 1
     agent_config: dict[str, Any] = {}
-    if agent_type == "integration_test":
+    if agent_type == AgentType.INTEGRATION_TEST:
         from hud.agents.misc.integration_test_agent import IntegrationTestRunner
 
         agent_class = IntegrationTestRunner
         agent_config = {"verbose": verbose}
         if allowed_tools:
             agent_config["allowed_tools"] = allowed_tools
-    elif agent_type == "vllm":
+    elif agent_type == AgentType.VLLM:
         # Special handling for vLLM
-        sample_agent = build_agent(
-            agent_type,
+        from hud.agents.openai_chat_generic import GenericOpenAIChatAgent
+
+        agent_class = GenericOpenAIChatAgent
+
+        # Use the shared config builder
+        agent_config = _build_vllm_config(
+            vllm_base_url=vllm_base_url,
             model=model,
             allowed_tools=allowed_tools,
             verbose=verbose,
-            vllm_base_url=vllm_base_url,
         )
-        agent_config = {
-            "openai_client": sample_agent.oai,
-            "model_name": sample_agent.model_name,
-            "verbose": verbose,
-            "completion_kwargs": sample_agent.completion_kwargs,
-        }
-        if allowed_tools:
-            agent_config["allowed_tools"] = allowed_tools
-
-        from hud.agents.openai_chat_generic import GenericOpenAIChatAgent
-
-        agent_class = GenericOpenAIChatAgent
-    elif agent_type == "openai":
+    elif agent_type == AgentType.OPENAI:
         from hud.agents import OperatorAgent
 
         agent_class = OperatorAgent
         agent_config = {"verbose": verbose}
         if allowed_tools:
             agent_config["allowed_tools"] = allowed_tools
-    elif agent_type == "litellm":
+    elif agent_type == AgentType.LITELLM:
         from hud.agents.lite_llm import LiteAgent
 
         agent_class = LiteAgent
@@ -293,7 +306,7 @@ async def run_single_task(
         }
         if allowed_tools:
             agent_config["allowed_tools"] = allowed_tools
-    elif agent_type == "claude":
+    elif agent_type == AgentType.CLAUDE:
         from hud.agents import ClaudeAgent
 
         agent_class = ClaudeAgent
@@ -341,7 +354,7 @@ async def run_single_task(
 async def run_full_dataset(
     source: str,
     *,
-    agent_type: Literal["claude", "openai", "vllm", "litellm", "integration_test"] = "claude",
+    agent_type: AgentType = AgentType.CLAUDE,
     model: str | None = None,
     allowed_tools: list[str] | None = None,
     max_concurrent: int = 30,
@@ -382,12 +395,13 @@ async def run_full_dataset(
     dataset_name = f"Dataset: {path.name}" if path.exists() else source.split("/")[-1]
 
     # Build agent class + config for run_dataset
-    if agent_type == "integration_test":  # --integration-test mode
+    agent_config: dict[str, Any]
+    if agent_type == AgentType.INTEGRATION_TEST:  # --integration-test mode
         from hud.agents.misc.integration_test_agent import IntegrationTestRunner
 
         agent_class = IntegrationTestRunner
         agent_config = {"verbose": verbose}
-    elif agent_type == "vllm":
+    elif agent_type == AgentType.VLLM:
         try:
             from hud.agents.openai_chat_generic import GenericOpenAIChatAgent
 
@@ -399,25 +413,14 @@ async def run_full_dataset(
             )
             raise typer.Exit(1) from e
 
-        # Use build_agent to create a sample agent to get the config
-        sample_agent = build_agent(
-            agent_type,
+        # Use the shared config builder
+        agent_config = _build_vllm_config(
+            vllm_base_url=vllm_base_url,
             model=model,
             allowed_tools=allowed_tools,
             verbose=verbose,
-            vllm_base_url=vllm_base_url,
         )
-
-        # Extract the config from the sample agent
-        agent_config: dict[str, Any] = {
-            "openai_client": sample_agent.oai,
-            "model_name": sample_agent.model_name,
-            "verbose": verbose,
-            "completion_kwargs": sample_agent.completion_kwargs,
-        }
-        if allowed_tools:
-            agent_config["allowed_tools"] = allowed_tools
-    elif agent_type == "openai":
+    elif agent_type == AgentType.OPENAI:
         try:
             from hud.agents import OperatorAgent
 
@@ -433,7 +436,7 @@ async def run_full_dataset(
         if allowed_tools:
             agent_config["allowed_tools"] = allowed_tools
 
-    elif agent_type == "litellm":
+    elif agent_type == AgentType.LITELLM:
         try:
             from hud.agents.lite_llm import LiteAgent
 
@@ -537,8 +540,8 @@ def eval_command(
         "--full",
         help="Run the entire dataset (omit for single-task debug mode)",
     ),
-    agent: Literal["claude", "openai", "vllm", "litellm", "integration_test"] = typer.Option(
-        "claude",
+    agent: AgentType = typer.Option(  # noqa: B008
+        AgentType.CLAUDE,
         "--agent",
         help="Agent backend to use (claude, openai, vllm for local server, or litellm)",
     ),
@@ -630,8 +633,6 @@ def eval_command(
         # Run with verbose output for debugging
         hud eval task.json --verbose
     """
-    from hud.settings import settings
-
     # Always configure basic logging so agent steps can be logged
     # Set to INFO by default for consistency with run_evaluation.py
     if very_verbose:
@@ -648,21 +649,21 @@ def eval_command(
 
     # We pass integration_test as the agent_type
     if integration_test:
-        agent = "integration_test"
+        agent = AgentType.INTEGRATION_TEST
 
     # Check for required API keys
-    if agent == "claude":
+    if agent == AgentType.CLAUDE:
         if not settings.anthropic_api_key:
             hud_console.error("ANTHROPIC_API_KEY is required for Claude agent")
             hud_console.info(
                 "Set it in your environment or run: hud set ANTHROPIC_API_KEY=your-key-here"
             )
             raise typer.Exit(1)
-    elif agent == "openai" and not settings.openai_api_key:
+    elif agent == AgentType.OPENAI and not settings.openai_api_key:
         hud_console.error("OPENAI_API_KEY is required for OpenAI agent")
         hud_console.info("Set it in your environment or run: hud set OPENAI_API_KEY=your-key-here")
         raise typer.Exit(1)
-    elif agent == "vllm":
+    elif agent == AgentType.VLLM:
         if model:
             hud_console.info(f"Using vLLM with model: {model}")
         else:

hud/cli/tests/test_analyze_module.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from hud.cli.analyze import (
+    analyze_environment,
+    analyze_environment_from_config,
+    analyze_environment_from_mcp_config,
+    display_interactive,
+    display_markdown,
+    parse_docker_command,
+)
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+# Mark entire module as asyncio to ensure async tests run with pytest-asyncio
+pytestmark = pytest.mark.asyncio
+
+
+def test_parse_docker_command():
+    cmd = ["docker", "run", "--rm", "-i", "img"]
+    cfg = parse_docker_command(cmd)
+    assert cfg == {"local": {"command": "docker", "args": ["run", "--rm", "-i", "img"]}}
+
+
+@pytest.mark.asyncio
+@patch("hud.cli.analyze.MCPClient")
+@patch("hud.cli.analyze.console")
+async def test_analyze_environment_success_json(mock_console, MockClient):
+    client = AsyncMock()
+    client.initialize.return_value = None
+    client.analyze_environment.return_value = {"tools": [], "resources": []}
+    client.shutdown.return_value = None
+    MockClient.return_value = client
+
+    await analyze_environment(["docker", "run", "img"], output_format="json", verbose=False)
+    assert client.initialize.awaited
+    assert client.analyze_environment.awaited
+    assert client.shutdown.awaited
+    assert mock_console.print_json.called
+
+
+@pytest.mark.asyncio
+@patch("hud.cli.analyze.MCPClient")
+@patch("hud.cli.analyze.console")
+async def test_analyze_environment_failure(mock_console, MockClient):
+    client = AsyncMock()
+    client.initialize.side_effect = RuntimeError("boom")
+    client.shutdown.return_value = None
+    MockClient.return_value = client
+
+    # Should swallow exception and return without raising
+    await analyze_environment(["docker", "run", "img"], output_format="json", verbose=True)
+    assert client.shutdown.awaited
+    assert mock_console.print_json.called is False
+
+
+def test_display_interactive_metadata_only(monkeypatch):
+    import hud.cli.analyze as mod
+
+    monkeypatch.setattr(mod, "console", MagicMock(), raising=False)
+    monkeypatch.setattr(mod, "hud_console", MagicMock(), raising=False)
+
+    analysis = {
+        "image": "img:latest",
+        "status": "cached",
+        "tool_count": 2,
+        "tools": [
+            {"name": "t1", "description": "d1", "inputSchema": {"type": "object"}},
+            {"name": "t2", "description": "d2"},
+        ],
+        "resources": [],
+    }
+    display_interactive(analysis)
+
+
+def test_display_markdown_both_paths(capsys):
+    # metadata-only
+    md_only = {"image": "img:latest", "tool_count": 0, "tools": [], "resources": []}
+    display_markdown(md_only)
+
+    # live metadata
+    live = {"metadata": {"servers": ["s1"], "initialized": True}, "tools": [], "resources": []}
+    display_markdown(live)
+
+    # Check that output was generated
+    captured = capsys.readouterr()
+    assert "MCP Environment Analysis" in captured.out
+
+
+@patch("hud.cli.analyze.MCPClient")
+async def test_analyze_environment_from_config(MockClient, tmp_path: Path):
+    client = AsyncMock()
+    client.initialize.return_value = None
+    client.analyze_environment.return_value = {"tools": [], "resources": []}
+    client.shutdown.return_value = None
+    MockClient.return_value = client
+
+    cfg = tmp_path / "mcp.json"
+    cfg.write_text('{"local": {"command": "docker", "args": ["run", "img"]}}')
+    await analyze_environment_from_config(cfg, output_format="json", verbose=False)
+    assert client.initialize.awaited and client.shutdown.awaited
+
+
+@patch("hud.cli.analyze.MCPClient")
+async def test_analyze_environment_from_mcp_config(MockClient):
+    client = AsyncMock()
+    client.initialize.return_value = None
+    client.analyze_environment.return_value = {"tools": [], "resources": []}
+    client.shutdown.return_value = None
+    MockClient.return_value = client
+
+    mcp_config = {"local": {"command": "docker", "args": ["run", "img"]}}
+    await analyze_environment_from_mcp_config(mcp_config, output_format="json", verbose=False)
+    assert client.initialize.awaited and client.shutdown.awaited

hud/cli/tests/test_build.py

@@ -219,6 +219,17 @@ class TestAnalyzeMcpEnvironment:
         mock_tool.description = "Test tool"
         mock_tool.inputSchema = {"type": "object"}
 
+        # Prefer analyze_environment path (aligns with analyze CLI tests)
+        mock_client.analyze_environment = mock.AsyncMock(
+            return_value={
+                "metadata": {"servers": ["local"], "initialized": True},
+                "tools": [{"name": "test_tool", "description": "Test tool"}],
+                "hub_tools": {},
+                "resources": [],
+                "telemetry": {},
+            }
+        )
+        # Fallback still defined for completeness
         mock_client.list_tools.return_value = [mock_tool]
 
         result = await analyze_mcp_environment("test:latest")
@@ -237,7 +248,9 @@ class TestAnalyzeMcpEnvironment:
         mock_client_class.return_value = mock_client
         mock_client.initialize.side_effect = ConnectionError("Connection failed")
 
-        with pytest.raises(ConnectionError):
+        from hud.shared.exceptions import HudException
+
+        with pytest.raises(HudException, match="Connection failed"):
             await analyze_mcp_environment("test:latest")
 
     @mock.patch("hud.cli.build.MCPClient")
@@ -245,6 +258,15 @@ class TestAnalyzeMcpEnvironment:
         """Test analysis in verbose mode."""
         mock_client = mock.AsyncMock()
         mock_client_class.return_value = mock_client
+        mock_client.analyze_environment = mock.AsyncMock(
+            return_value={
+                "metadata": {"servers": ["local"], "initialized": True},
+                "tools": [],
+                "hub_tools": {},
+                "resources": [],
+                "telemetry": {},
+            }
+        )
         mock_client.list_tools.return_value = []
 
         # Just test that it runs without error in verbose mode
@@ -363,7 +385,7 @@ ENV API_KEY
         mock_run.return_value = mock_result
 
         # Run build
-        build_environment(str(env_dir), "test/env:latest")
+        build_environment(str(env_dir), "test-env:latest")
 
         # Check lock file was created
         lock_file = env_dir / "hud.lock.yaml"

hud/cli/tests/test_build_failure.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from unittest.mock import patch
+
+import pytest
+import typer
+
+from hud.cli.build import build_environment
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+@patch("hud.cli.build.compute_source_hash", return_value="deadbeef")
+@patch(
+    "hud.cli.build.analyze_mcp_environment",
+    return_value={"initializeMs": 10, "toolCount": 0, "tools": []},
+)
+@patch("hud.cli.build.build_docker_image", return_value=True)
+def test_build_label_rebuild_failure(_bd, _an, _hash, tmp_path: Path, monkeypatch):
+    # Minimal environment dir
+    env = tmp_path / "env"
+    env.mkdir()
+    (env / "Dockerfile").write_text("FROM python:3.11")
+
+    # Ensure subprocess.run returns non-zero for the second build (label build)
+    import types
+
+    def run_side_effect(cmd, *a, **k):
+        # Return 0 for first docker build, 1 for label build
+        if isinstance(cmd, list) and cmd[:2] == ["docker", "build"] and "--label" in cmd:
+            return types.SimpleNamespace(returncode=1, stderr="boom")
+        return types.SimpleNamespace(returncode=0, stdout="")
+
+    monkeypatch.setenv("FASTMCP_DISABLE_BANNER", "1")
+    with (
+        patch("hud.cli.build.subprocess.run", side_effect=run_side_effect),
+        pytest.raises(typer.Exit),
+    ):
+        build_environment(str(env), verbose=False)

hud/cli/tests/test_build_module.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from unittest import mock
+
+from hud.cli.build import (
+    extract_env_vars_from_dockerfile,
+    get_docker_image_digest,
+    get_docker_image_id,
+)
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+def test_extract_env_vars_from_dockerfile_complex(tmp_path: Path):
+    dockerfile = tmp_path / "Dockerfile"
+    dockerfile.write_text(
+        """
+FROM python:3.11
+ARG BUILD_TOKEN
+ARG DEFAULTED=1
+ENV RUNTIME_KEY
+ENV FROM_ARG=$BUILD_TOKEN
+ENV WITH_DEFAULT=val
+"""
+    )
+    required, optional = extract_env_vars_from_dockerfile(dockerfile)
+    # BUILD_TOKEN required (ARG without default)
+    assert "BUILD_TOKEN" in required
+    # RUNTIME_KEY required (ENV without value)
+    assert "RUNTIME_KEY" in required
+    # FROM_ARG references BUILD_TOKEN -> required
+    assert "FROM_ARG" in required
+    # DEFAULTED and WITH_DEFAULT should not be marked required by default
+    assert "DEFAULTED" not in required
+    assert "WITH_DEFAULT" not in required
+    assert optional == []
+
+
+@mock.patch("subprocess.run")
+def test_get_docker_image_digest_none(mock_run):
+    mock_run.return_value = mock.Mock(stdout="[]", returncode=0)
+    assert get_docker_image_digest("img") is None
+
+
+@mock.patch("subprocess.run")
+def test_get_docker_image_id_ok(mock_run):
+    mock_run.return_value = mock.Mock(stdout="sha256:abc", returncode=0)
+    assert get_docker_image_id("img") == "sha256:abc"

hud/cli/tests/test_cli_more_wrappers.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from unittest.mock import patch
+
+import hud.cli as cli
+
+
+def test_version_does_not_crash():
+    # Just ensure it runs without raising
+    cli.version()
+
+
+@patch("hud.cli.list_module.list_command")
+def test_list_environments_wrapper(mock_list):
+    cli.list_environments(filter_name=None, json_output=False, show_all=False, verbose=False)
+    assert mock_list.called
+
+
+@patch("hud.cli.clone_repository", return_value=(True, "/tmp/repo"))
+@patch("hud.cli.get_clone_message", return_value={})
+@patch("hud.cli.print_tutorial")
+def test_clone_wrapper(mock_tutorial, _msg, _clone):
+    cli.clone("https://example.com/repo.git")
+    assert mock_tutorial.called
+
+
+@patch("hud.cli.remove_command")
+def test_remove_wrapper(mock_remove):
+    cli.remove(target="all", yes=True, verbose=False)
+    assert mock_remove.called