hud-python 0.3.4__py3-none-any.whl → 0.4.0__py3-none-any.whl

hud/mcp/tests/test_claude.py

@@ -1,294 +0,0 @@
-"""Tests for Claude MCP Agent implementation."""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, cast
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-from anthropic import BadRequestError
-from mcp import types
-from mcp.types import CallToolRequestParams as MCPToolCall
-
-from hud.mcp.claude import (
-    ClaudeMCPAgent,
-    base64_to_content_block,
-    text_to_content_block,
-    tool_use_content_block,
-)
-
-if TYPE_CHECKING:
-    from anthropic.types.beta import BetaImageBlockParam, BetaMessageParam, BetaTextBlockParam
-
-
-class TestClaudeHelperFunctions:
-    """Test helper functions for Claude message formatting."""
-
-    def test_base64_to_content_block(self):
-        """Test base64 image conversion."""
-        base64_data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="  # noqa: E501
-        result = base64_to_content_block(base64_data)
-
-        assert result["type"] == "image"
-        assert result["source"]["type"] == "base64"
-        assert result["source"]["media_type"] == "image/png"
-        assert result["source"]["data"] == base64_data
-
-    def test_text_to_content_block(self):
-        """Test text conversion."""
-        text = "Hello, world!"
-        result = text_to_content_block(text)
-
-        assert result["type"] == "text"
-        assert result["text"] == text
-
-    def test_tool_use_content_block(self):
-        """Test tool result content block creation."""
-        tool_use_id = "tool_123"
-        content: list[BetaTextBlockParam | BetaImageBlockParam] = [
-            text_to_content_block("Result text")
-        ]
-
-        result = tool_use_content_block(tool_use_id, content)
-
-        assert result["type"] == "tool_result"
-        assert result["tool_use_id"] == tool_use_id
-        assert result["content"] == content  # type: ignore
-
-
-class TestClaudeMCPAgent:
-    """Test ClaudeMCPAgent class."""
-
-    @pytest.fixture
-    def mock_mcp_client(self):
-        """Create a mock MCP client."""
-        mcp_client = MagicMock()
-        mcp_client.get_all_active_sessions = MagicMock(return_value={})
-        mcp_client.get_tool_map = MagicMock(return_value={})
-        return mcp_client
-
-    @pytest.fixture
-    def mock_anthropic(self):
-        """Create a mock Anthropic client."""
-        with patch("hud.mcp.claude.AsyncAnthropic") as mock:
-            client = AsyncMock()
-            # Add beta attribute with messages
-            client.beta = AsyncMock()
-            client.beta.messages = AsyncMock()
-            mock.return_value = client
-            yield client
-
-    @pytest.mark.asyncio
-    async def test_init(self, mock_mcp_client, mock_anthropic):
-        """Test agent initialization."""
-        # Test with provided model_client
-        mock_model_client = MagicMock()
-        agent = ClaudeMCPAgent(
-            mcp_client=mock_mcp_client,
-            model_client=mock_model_client,
-            model="claude-3-opus-20240229",
-            max_tokens=1000,
-        )
-
-        assert agent.model_name == "claude-3-opus-20240229"
-        assert agent.max_tokens == 1000
-        assert agent.anthropic_client == mock_model_client
-
-    @pytest.mark.asyncio
-    async def test_init_without_model_client(self, mock_mcp_client):
-        """Test agent initialization without model client."""
-        with patch("hud.mcp.claude.settings.anthropic_api_key", "test_key"):
-            agent = ClaudeMCPAgent(mcp_client=mock_mcp_client, model="claude-3-opus-20240229")
-
-            assert agent.model_name == "claude-3-opus-20240229"
-            assert agent.anthropic_client is not None
-
-    @pytest.mark.asyncio
-    async def test_create_initial_messages(self, mock_mcp_client):
-        """Test creating initial messages."""
-        mock_model_client = MagicMock()
-        agent = ClaudeMCPAgent(mcp_client=mock_mcp_client, model_client=mock_model_client)
-
-        # Test with text only
-        messages = await agent.create_initial_messages("Hello, Claude!")
-        assert len(messages) == 1
-        assert messages[0]["role"] == "user"
-        content = list(messages[0]["content"])
-        assert content[0]["type"] == "text"  # type: ignore
-        assert content[0]["text"] == "Hello, Claude!"  # type: ignore
-
-        # Test with screenshot
-        messages = await agent.create_initial_messages("Look at this", screenshot="base64data")
-        assert len(messages) == 1
-        assert messages[0]["role"] == "user"
-        content = list(messages[0]["content"])
-        assert len(content) == 2
-        # Claude puts text first, then image
-        assert content[0]["type"] == "text"  # type: ignore
-        assert content[0]["text"] == "Look at this"  # type: ignore
-        assert content[1]["type"] == "image"  # type: ignore
-
-    @pytest.mark.asyncio
-    async def test_format_tool_results_method(self, mock_mcp_client):
-        """Test the agent's format_tool_results method."""
-        mock_model_client = MagicMock()
-        agent = ClaudeMCPAgent(mcp_client=mock_mcp_client, model_client=mock_model_client)
-
-        tool_calls = [
-            MCPToolCall(name="test_tool", arguments={}, tool_use_id="id1"),  # type: ignore
-        ]
-
-        tool_results = [
-            types.CallToolResult(
-                content=[types.TextContent(type="text", text="Success")], isError=False
-            ),
-        ]
-
-        messages = await agent.format_tool_results(tool_calls, tool_results)
-
-        # format_tool_results returns a single user message with tool result content
-        assert len(messages) == 1
-        assert messages[0]["role"] == "user"
-        # The content is wrapped in a tool result block
-        content = list(messages[0]["content"])
-        assert len(content) == 1
-        assert content[0]["type"] == "tool_result"  # type: ignore
-        assert content[0]["tool_use_id"] == "id1"  # type: ignore
-        # The actual content is nested inside
-        inner_content = list(content[0]["content"])  # type: ignore
-        assert inner_content[0]["type"] == "text"  # type: ignore
-        assert inner_content[0]["text"] == "Success"  # type: ignore
-
-    @pytest.mark.asyncio
-    async def test_get_model_response(self, mock_mcp_client, mock_anthropic):
-        """Test getting model response from Claude API."""
-        agent = ClaudeMCPAgent(mcp_client=mock_mcp_client, model_client=mock_anthropic)
-
-        # Mock the API response
-        mock_response = MagicMock()
-
-        # Create text block
-        text_block = MagicMock()
-        text_block.type = "text"
-        text_block.text = "Hello!"
-
-        # Create tool use block
-        tool_block = MagicMock()
-        tool_block.type = "tool_use"
-        tool_block.id = "tool_123"
-        tool_block.name = "test_tool"
-        tool_block.input = {"param": "value"}
-
-        mock_response.content = [text_block, tool_block]
-        mock_response.usage = MagicMock(input_tokens=10, output_tokens=20)
-        mock_anthropic.beta.messages.create = AsyncMock(return_value=mock_response)
-
-        messages = [
-            cast("BetaMessageParam", {"role": "user", "content": [{"type": "text", "text": "Hi"}]})
-        ]
-        response = await agent.get_model_response(messages)
-
-        assert response.content == "Hello!"
-        assert len(response.tool_calls) == 1
-        assert response.tool_calls[0].name == "test_tool"
-        assert response.tool_calls[0].arguments == {"param": "value"}
-        # The test was checking for Claude-specific attributes that aren't part of ModelResponse
-        # These would need to be accessed from the original Claude response if needed
-
-        # Verify API was called correctly
-        mock_anthropic.beta.messages.create.assert_called_once()
-
-    @pytest.mark.asyncio
-    async def test_get_model_response_text_only(self, mock_mcp_client, mock_anthropic):
-        """Test getting text-only response."""
-        agent = ClaudeMCPAgent(mcp_client=mock_mcp_client, model_client=mock_anthropic)
-
-        mock_response = MagicMock()
-        # Create text block
-        text_block = MagicMock()
-        text_block.type = "text"
-        text_block.text = "Just text"
-        mock_response.content = [text_block]
-        mock_response.usage = MagicMock(input_tokens=5, output_tokens=10)
-        mock_anthropic.beta.messages.create = AsyncMock(return_value=mock_response)
-
-        messages = [
-            cast("BetaMessageParam", {"role": "user", "content": [{"type": "text", "text": "Hi"}]})
-        ]
-        response = await agent.get_model_response(messages)
-
-        assert response.content == "Just text"
-        assert response.tool_calls == []
-
-    @pytest.mark.asyncio
-    async def test_get_model_response_error(self, mock_mcp_client, mock_anthropic):
-        """Test handling API errors."""
-        agent = ClaudeMCPAgent(mcp_client=mock_mcp_client, model_client=mock_anthropic)
-
-        # Mock API error
-        mock_anthropic.beta.messages.create = AsyncMock(
-            side_effect=BadRequestError(
-                message="Invalid request",
-                response=MagicMock(status_code=400),
-                body={"error": {"message": "Invalid request"}},
-            )
-        )
-
-        messages = [{"role": "user", "content": [{"type": "text", "text": "Hi"}]}]
-
-        with pytest.raises(BadRequestError):
-            await agent.get_model_response(messages)  # type: ignore
-
-    @pytest.mark.asyncio
-    async def test_run_with_tools(self, mock_mcp_client, mock_anthropic):
-        """Test running agent with tool usage."""
-        agent = ClaudeMCPAgent(mcp_client=mock_mcp_client, model_client=mock_anthropic)
-
-        # Mock tool availability
-        agent._available_tools = [
-            types.Tool(name="calculator", description="Calculator", inputSchema={"type": "object"})
-        ]
-        agent._tool_map = {
-            "calculator": (
-                "server1",
-                types.Tool(
-                    name="calculator", description="Calculator", inputSchema={"type": "object"}
-                ),
-            )
-        }
-
-        # Mock initial response with tool use
-        initial_response = MagicMock()
-        # Create tool use block
-        tool_block = MagicMock()
-        tool_block.type = "tool_use"
-        tool_block.id = "calc_123"
-        tool_block.name = "calculator"
-        tool_block.input = {"operation": "add", "a": 2, "b": 3}
-        initial_response.content = [tool_block]
-        initial_response.usage = MagicMock(input_tokens=10, output_tokens=15)
-
-        # Mock follow-up response
-        final_response = MagicMock()
-        text_block = MagicMock()
-        text_block.type = "text"
-        text_block.text = "2 + 3 = 5"
-        final_response.content = [text_block]
-        final_response.usage = MagicMock(input_tokens=20, output_tokens=10)
-
-        mock_anthropic.beta.messages.create = AsyncMock(
-            side_effect=[initial_response, final_response]
-        )
-
-        # Mock tool execution
-        agent.mcp_client.call_tool = AsyncMock(
-            return_value=types.CallToolResult(
-                content=[types.TextContent(type="text", text="5")], isError=False
-            )
-        )
-
-        # Use a string prompt instead of a task
-        result = await agent.run("What is 2 + 3?")
-
-        assert result.content == "2 + 3 = 5"
-        assert result.done is True

hud/task.py

@@ -1,149 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, cast
-
-from pydantic import BaseModel, Field
-
-from hud.types import CustomGym, Gym, MetadataKeys, SensitiveData
-from hud.utils.common import FunctionConfigs
-from hud.utils.deprecation import deprecated
-
-if TYPE_CHECKING:
-    from hud.agent import Agent
-
-
-@deprecated(
-    reason="Task class is being replaced by TaskConfig for better MCP integration",
-    replacement="hud.datasets.TaskConfig",
-    version="0.3.0",
-    removal_version="0.4.0",
-)
-class Task(BaseModel):
-    """A task that can be executed and evaluated.
-
-    A Task represents a specific activity to be performed in an environment.
-    It contains the prompt describing the task and configurations for
-    setting up and evaluating the environment.
-
-    The setup and evaluate configurations can be in several formats:
-    - String (function name): "chrome.maximize"
-    - Tuple (function with args): ("chrome.activate_tab", 5)
-    - Dict: {"function": "chrome.navigate", "args": ["https://example.com"]}
-    - List of the above: ["chrome.maximize", {"function": "chrome.navigate", "args": ["https://example.com"]}]
-
-    Attributes:
-        id: The remote task ID (optional if local-only)
-        prompt: The task prompt or instruction
-        system_prompt: The system prompt for the evalset (optional)
-        setup: Environment setup configuration (optional)
-        evaluate: Configuration for evaluating responses
-        metadata: Additional task metadata
-        sensitive_data: Sensitive data such as API keys, passwords, etc.
-        choices: Multiple choice answer list (for Inspect compatibility)
-        target: Ideal target output (for Inspect compatibility)
-        files: Files that go along with the task (for Inspect compatibility)
-        gym: Environment specification
-    """
-
-    id: str | None = None  # Remote task ID (optional if local-only)
-
-    prompt: str  # Task prompt or instruction
-    system_prompt: str | None = None  # System prompt for the evalset (optional)
-
-    gym: Gym | None = None  # Environment specification
-
-    # Setup and evaluate configurations for the environment (environment specific)
-    setup: FunctionConfigs | None = None
-    evaluate: FunctionConfigs | None = None
-
-    # Overflow configuration for environments that don't conform to the standard
-    config: dict[str, Any] | None = None
-
-    # Sensitive data such as API keys, passwords, etc.
-    sensitive_data: SensitiveData = Field(default_factory=dict)
-
-    # Metadata for the task evaluation, information about the agent (see MetadataKeys)
-    metadata: dict[MetadataKeys, Any] = Field(default_factory=dict)
-
-    # Description of the task, for extra information about its purpose and context
-    description: str | None = None
-
-    # Gold file url for the task
-    gold_file_url: str | None = None
-
-    @classmethod
-    def from_dict(cls, data: dict[str, Any]) -> Task:
-        return cls(**data)
-
-    @classmethod
-    def from_serialized(cls, data: dict[str, Any]) -> Task:
-        gym_data = data.get("gym")
-        parsed_gym: Gym | None = gym_data
-
-        parsed_setup = [(param, entry) for param, entry in data.get("setup", [])]
-        parsed_evaluate = [(param, entry) for param, entry in data.get("evaluate", [])]
-
-        # Convert dict gym data to CustomGym if needed
-        if (
-            isinstance(gym_data, dict)
-            and gym_data.get("type") == "public"
-            and gym_data.get("location") in ("local", "remote")
-            and gym_data.get("image_or_build_context") is not None
-        ):
-            parsed_gym = CustomGym(
-                type=cast("Literal['public']", gym_data["type"]),
-                location=cast("Literal['local', 'remote']", gym_data["location"]),
-                image_or_build_context=Path(gym_data["image_or_build_context"]),
-            )
-
-        return cls(
-            id=data.get("id"),
-            prompt=data.get("prompt", ""),
-            system_prompt=data.get("system_prompt"),
-            setup=parsed_setup,
-            evaluate=parsed_evaluate,
-            gym=parsed_gym,
-            config=data.get("config"),
-            description=data.get("description"),
-            sensitive_data=data.get("sensitive_data", {}),
-            metadata=data.get("metadata", {}),
-            gold_file_url=data.get("gold_file_url"),
-        )
-
-    async def fit(self, agent: Agent | type[Agent]) -> None:
-        if isinstance(agent, type):
-            agent = agent()
-
-        if self.gym is None:
-            return
-        self.gym = agent.transfer_gyms.get(self.gym, self.gym)
-
-    def serialize(self) -> dict[str, Any]:
-        if isinstance(self.setup, list):
-            parsed_setup = [[param, entry] for param, entry in self.setup]
-        else:
-            parsed_setup = self.setup
-        if isinstance(self.evaluate, list):
-            parsed_evaluate = [[param, entry] for param, entry in self.evaluate]
-        else:
-            parsed_evaluate = self.evaluate
-
-        if isinstance(self.gym, CustomGym):
-            parsed_gym = self.gym.model_dump()
-            parsed_gym["image_or_build_context"] = str(parsed_gym["image_or_build_context"])
-        else:  # is ServerGym
-            parsed_gym = self.gym
-
-        return {
-            "id": self.id,
-            "prompt": self.prompt,
-            "config": self.config,
-            "description": self.description,
-            "setup": parsed_setup,
-            "evaluate": parsed_evaluate,
-            "gym": parsed_gym,
-            "sensitive_data": self.sensitive_data,
-            "metadata": self.metadata,
-            "gold_file_url": self.gold_file_url,
-        }

hud/taskset.py

@@ -1,237 +0,0 @@
-from __future__ import annotations
-
-import logging
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, get_args
-
-from pydantic import BaseModel
-
-from hud.env.environment import create_remote_config
-from hud.server import make_request
-from hud.settings import settings
-from hud.task import Task
-from hud.types import CustomGym, ServerGym
-from hud.utils.config import REMOTE_EVALUATE, REMOTE_SETUP
-from hud.utils.deprecation import deprecated
-
-logger = logging.getLogger(__name__)
-
-if TYPE_CHECKING:
-    from collections.abc import Iterator
-
-    from hud.agent import Agent
-
-
-@deprecated(
-    reason="TaskSet class is being replaced by HuggingFace datasets on hud-evals",
-    replacement="Use TaskConfig-based collections or hud.datasets module",
-    version="0.3.0",
-    removal_version="0.4.0",
-)
-class TaskSet(BaseModel):
-    """
-    Collection of related tasks for benchmarking.
-
-    Attributes:
-        id: Unique identifier for the taskset
-        name: Name of the taskset
-        description: Description of the taskset
-        tasks: List of Task objects in the taskset
-    """
-
-    id: str | None = None
-    name: str | None = None
-    description: str | None = None
-    tasks: list[Task] = []
-
-    def __getitem__(self, index: int) -> Task:
-        """
-        Allows accessing tasks by index using square bracket notation.
-
-        Args:
-            index: The index of the task to retrieve
-
-        Returns:
-            Task: The task at the specified index
-
-        Raises:
-            IndexError: If the index is out of range
-        """
-        return self.tasks[index]
-
-    def __len__(self) -> int:
-        """
-        Returns the number of tasks in the taskset.
-
-        Returns:
-            int: The number of tasks in the taskset
-        """
-        return len(self.tasks)
-
-    def __iter__(self) -> Iterator[Task]:
-        """
-        Returns an iterator over the tasks in the taskset.
-        """
-        return iter(self.tasks)
-
-    async def upload(
-        self,
-        name: str | None = None,
-        description: str | None = None,
-        api_key: str | None = None,
-    ) -> None:
-        """
-        Uploads the taskset to the server.
-        """
-        if name is None:
-            name = self.name
-
-        if name is None:
-            raise ValueError("Taskset name is required")
-
-        if api_key is None:
-            api_key = settings.api_key
-
-        # Convert all tasks to expanded configs
-        processed_tasks = []
-        for task in self.tasks:
-            if task.setup is not None:
-                setup_config = (
-                    create_remote_config(None, task.setup, REMOTE_SETUP)[0].args[0].model_dump()
-                )
-            else:
-                setup_config = None
-            if task.evaluate is not None:
-                evaluate_config = (
-                    create_remote_config(None, task.evaluate, REMOTE_EVALUATE)[0]
-                    .args[0]
-                    .model_dump()
-                )
-            else:
-                evaluate_config = None
-
-            if isinstance(task.gym, CustomGym):
-                if isinstance(task.gym.image_or_build_context, Path):
-                    raise ValueError(
-                        "Local build contexts are not supported for "
-                        "remote tasksets, attach an image or existing "
-                        "gym id."
-                    )
-                gym_str = "docker"
-                image_uri = task.gym.image_or_build_context
-            elif isinstance(task.gym, str) and task.gym in get_args(ServerGym):
-                gym_str = task.gym
-                image_uri = None
-            else:
-                raise ValueError(f"Unknown gym type: {type(task.gym)}")
-
-            processed_tasks.append(
-                {
-                    "prompt": task.prompt,
-                    "gym": gym_str,
-                    "setup": setup_config,
-                    "evaluate": evaluate_config,
-                    "config": task.config,
-                    "image_uri": image_uri,
-                    "description": task.description,
-                }
-            )
-
-        await make_request(
-            method="POST",
-            url=f"{settings.base_url}/v2/tasksets",
-            api_key=api_key,
-            json={
-                "name": name,
-                "description": description,
-                "tasks": processed_tasks,
-            },
-        )
-        logger.info(
-            "Taskset %s uploaded successfully, see it on app.hud.so/evalsets/%s", name, name
-        )
-
-    def _apply(self, dict: dict[str, Any]) -> None:
-        """
-        Applies a parameter to all tasks in the taskset.
-        """
-        for task in self.tasks:
-            for key, value in dict.items():
-                setattr(task, key, value)
-
-    def fit(self, agent: Agent | type[Agent]) -> None:
-        """
-        Automatically adapts the taskset to the agent's transfer_gyms.
-        """
-        if isinstance(agent, type):
-            agent = agent()
-
-        for task in self.tasks:
-            if task.gym is None or isinstance(task.gym, CustomGym):
-                continue
-            task.gym = agent.transfer_gyms.get(task.gym, task.gym)
-
-
-@deprecated(
-    reason="load_taskset is being replaced by new dataset loading mechanisms for MCP-based tasks",
-    replacement="Use TaskConfig-based dataset loading from hud.datasets module",
-    version="0.3.0",
-    removal_version="0.4.0",
-)
-async def load_taskset(
-    taskset_id: str,
-    api_key: str | None = None,
-    metadata: dict[str, Any] | None = None,
-    load_custom_as_local: bool = False,
-    system_prompt: str | None = None,
-) -> TaskSet:
-    """
-    Loads a TaskSet by its ID.
-
-    Args:
-        taskset_id: The ID of the taskset to load
-        api_key: Optional API key to use for the request
-        metadata: Optional metadata to apply to the taskset
-        load_custom_as_local: Whether to load custom gyms as local
-        system_prompt: Optional system prompt to override the default
-    Returns:
-        TaskSet: The loaded taskset
-    """
-
-    if api_key is None:
-        api_key = settings.api_key
-
-    data = await make_request(
-        method="GET",
-        url=f"{settings.base_url}/v2/tasksets/{taskset_id}/tasks",
-        api_key=api_key,
-    )
-
-    logger.info("Taskset %s loaded successfully", taskset_id)
-
-    tasks = data["evalset"]
-    for task in tasks:
-        if system_prompt:
-            task["system_prompt"] = system_prompt
-        if task["gym"] == "docker":
-            if "image_uri" not in task:
-                raise ValueError(
-                    "No `image_uri` key found. This taskset may be "
-                    "incompatible with your version of HUD SDK."
-                )
-
-            task["gym"] = CustomGym(
-                location="local" if load_custom_as_local else "remote",
-                image_or_build_context=task["image_uri"],
-            )
-
-    taskset = TaskSet.model_validate(
-        {
-            "id": taskset_id,
-            "tasks": tasks,
-        }
-    )
-
-    taskset._apply({"metadata": metadata})
-
-    return taskset