hud-python 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

hud/evaluators/tests/test_judge.py

@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+import base64
+
+import pytest
+
+from hud.evaluators.base import EvaluationResult
+from hud.evaluators.judge import (
+    _call_eval_endpoint,
+    _evaluate_with_llm,
+    _is_base64_image,
+    _process_input,
+    judge,
+)
+
+
+class _MockLLM:
+    """Mock LLM for testing."""
+
+    def __init__(self, response_text):
+        self.response_text = response_text
+
+    async def ainvoke(self, _prompt: str) -> str:
+        return self.response_text
+
+
+@pytest.mark.parametrize(
+    "input_data, expected_result",
+    [
+        ("Hello world", "Hello world"),
+        (123, "123"),
+        (["Hello", "world"], ["Hello", "world"]),
+        ({"key": "value"}, {"key": "value"}),
+        (b"Hello world", base64.b64encode(b"Hello world").decode("utf-8")),
+    ],
+)
+def test_process_input(input_data, expected_result):
+    """Test processing various input types."""
+    result = _process_input(input_data)
+    assert result == expected_result
+
+
+@pytest.mark.parametrize(
+    "input_data, expected_result",
+    [
+        ("not an image", False),
+        ("data:image/png;base64,abc123", True),
+        (b"not an image", False),
+        (123, False),
+    ],
+)
+def test_is_base64_image(input_data, expected_result):
+    """Test base64 image detection."""
+    assert _is_base64_image(input_data) == expected_result
+
+
+def test_is_base64_image_with_signatures(mocker):
+    """Test base64 image detection with common image signatures."""
+    # Mock base64.b64decode to return different image signatures
+    mock_b64decode = mocker.patch("base64.b64decode")
+
+    # Test JPEG signature
+    mock_b64decode.return_value = b"\xff\xd8\xff" + b"some data"
+    assert _is_base64_image("not_really_base64_but_mocked") is True
+
+    # Test PNG signature
+    mock_b64decode.return_value = b"\x89PNG\r\n\x1a\n" + b"some data"
+    assert _is_base64_image("not_really_base64_but_mocked") is True
+
+    # Test GIF signature
+    mock_b64decode.return_value = b"GIF8" + b"some data"
+    assert _is_base64_image("not_really_base64_but_mocked") is True
+
+    # Test RIFF signature (WebP)
+    mock_b64decode.return_value = b"RIFF" + b"some data"
+    assert _is_base64_image("not_really_base64_but_mocked") is True
+
+
+@pytest.mark.asyncio
+async def test_call_eval_endpoint_success(mocker):
+    """Test successful remote evaluation call."""
+    mock_response = {
+        "score": 0.8,
+        "reason": "Good response",
+        "criteria_scores": {"relevance": 0.9, "accuracy": 0.7},
+    }
+    mock_make_request = mocker.patch(
+        "hud.evaluators.judge.make_request", return_value=mock_response
+    )
+    result = await _call_eval_endpoint("test response", "test answer", [], "LLM")
+    assert result == mock_response
+    mock_make_request.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_call_eval_endpoint_failure(mocker):
+    """Test remote evaluation call failure."""
+    mocker.patch("hud.evaluators.judge.make_request", side_effect=Exception("API error"))
+    result = await _call_eval_endpoint("test response", "test answer", [], "LLM")
+    assert result["score"] == -1.0
+    assert "Remote evaluation failed" in result["reason"]
+    assert result["criteria_scores"] == {}
+
+
+def test_judge_without_llm(mocker):
+    """Test judge function without custom LLM."""
+    mock_result = {
+        "score": 0.9,
+        "reason": "Good answer",
+        "criteria_scores": {"relevance": 1.0},
+    }
+
+    async def mock_endpoint(*args, **kwargs):
+        return mock_result
+
+    mocker.patch("hud.evaluators.judge._call_eval_endpoint", mock_endpoint)
+    result = judge("test response", "test answer")
+
+    assert result.score == 0.9
+    assert result.reason == "Good answer"
+    assert result.mode == "LLM"
+    assert result.criteria_scores == {"relevance": 1.0}
+
+
+def test_judge_with_image_answer(mocker):
+    """Test judge function with an image as the answer."""
+    mock_result = {
+        "score": 0.85,
+        "reason": "Good image analysis",
+        "criteria_scores": {"visual_accuracy": 0.85},
+    }
+
+    async def mock_endpoint(*args, **kwargs):
+        return mock_result
+
+    mocker.patch("hud.evaluators.judge._call_eval_endpoint", mock_endpoint)
+
+    # Create a mock image
+    image_data = b"fake_image_data"
+    base64_image = base64.b64encode(image_data).decode("utf-8")
+    image_uri = f"data:image/jpeg;base64,{base64_image}"
+
+    result = judge("description of image", image_uri)
+
+    assert result.score == 0.85
+    assert result.reason == "Good image analysis"
+    assert result.mode == "VLM"  # Should use VLM mode for images
+    assert result.criteria_scores == {"visual_accuracy": 0.85}
+
+
+def test_judge_with_llm(mocker):
+    """Test judge function with custom LLM."""
+    mock_llm = _MockLLM('{"score": 0.75, "reason": "Pretty good"}')
+    mock_result = EvaluationResult(score=0.75, reason="Pretty good", mode="custom_llm")
+    mocker.patch("hud.evaluators.judge._evaluate_with_llm", return_value=mock_result)
+    result = judge("test response", "test answer", llm=mock_llm)
+    assert result.score == 0.75
+    assert result.reason == "Pretty good"
+    assert result.mode == "custom_llm"
+
+
+def test_evaluate_with_llm_valid_json():
+    """Test _evaluate_with_llm with valid JSON response."""
+    llm = _MockLLM('{"score": 0.85, "reason": "The response is accurate and well-structured."}')
+    result = _evaluate_with_llm("test response", "test answer", llm)
+
+    assert result.score == 0.85
+    assert result.reason == "The response is accurate and well-structured."
+    assert result.mode == "custom_llm"
+
+
+def test_evaluate_with_llm_json_in_text():
+    """Test _evaluate_with_llm with JSON embedded in text."""
+    llm_response = """
+    I've evaluated the response and here's my assessment:
+    
+    {"score": 0.7, "reason": "Good but could be more detailed"}
+    
+    I hope this helps!
+    """
+    llm = _MockLLM(llm_response)
+    result = _evaluate_with_llm("test response", "test answer", llm)
+
+    assert result.score == 0.7
+    assert result.reason == "Good but could be more detailed"
+    assert result.mode == "custom_llm"
+
+
+def test_evaluate_with_llm_invalid_json():
+    """Test _evaluate_with_llm with invalid JSON response."""
+    llm = _MockLLM("This is not a JSON response")
+    result = _evaluate_with_llm("test response", "test answer", llm)
+
+    assert result.score == 0.5  # Default score for unparseable responses
+    assert "Unable to parse LLM response as JSON" in result.reason
+    assert result.mode == "custom_llm"
+
+
+def test_evaluate_with_llm_exception(mocker):
+    """Test _evaluate_with_llm when an exception occurs."""
+    # Mock the LLM to raise an exception
+    failing_llm = _MockLLM("doesn't matter")
+    mocker.patch.object(failing_llm, "ainvoke", side_effect=Exception("LLM API error"))
+
+    result = _evaluate_with_llm("test response", "test answer", failing_llm)
+
+    assert result.score == 0.0  # Zero score for errors
+    assert "LLM evaluation error: LLM API error" in result.reason
+    assert result.mode == "custom_llm"
+
+
+def test_evaluate_with_llm_with_criteria():
+    """Test _evaluate_with_llm with evaluation criteria."""
+    llm = _MockLLM('{"score": 0.9, "reason": "Excellent match on all criteria"}')
+
+    # Test with string criteria
+    string_criteria = ["Accuracy", "Relevance", "Completeness"]
+    result = _evaluate_with_llm("test response", "test answer", llm, criteria=string_criteria)
+
+    assert result.score == 0.9
+    assert result.reason == "Excellent match on all criteria"
+
+    # Test with dict criteria
+    dict_criteria = [
+        {"description": "Factual accuracy", "weight": 0.6},
+        {"description": "Grammar and spelling", "weight": 0.4},
+    ]
+    result = _evaluate_with_llm("test response", "test answer", llm, criteria=dict_criteria)
+
+    assert result.score == 0.9
+    assert result.reason == "Excellent match on all criteria"

hud/evaluators/tests/test_match.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import pytest
+
+from hud.evaluators.match import match_all, match_diff, match_fuzzy, match_regex, match_single
+
+
+@pytest.mark.parametrize(
+    "response, answer, expected_score, expected_reason, expected_mode",
+    [
+        ("Hello, world!", "world", 1.0, "Exact match", "single"),
+        ("Hello, world!", "not world", 0.0, "No exact match found", "single"),
+    ],
+)
+def test_match_single(
+    response: str,
+    answer: str,
+    expected_score: float,
+    expected_reason: str,
+    expected_mode: str,
+):
+    result = match_single(response, answer)
+    assert result.score == expected_score
+    assert result.reason == expected_reason
+    assert result.mode == expected_mode
+
+
+@pytest.mark.parametrize(
+    "response, answers, expected_score, expected_reason, expected_mode",
+    [
+        ("Hello, world!", ["world", "hello"], 1.0, "All 2 expected items found", "all"),
+        ("Hello, world!", ["world", "not hello"], 0.5, "Only 1 of 2 expected items found", "all"),
+    ],
+)
+def test_match_all(
+    response: str,
+    answers: list[str],
+    expected_score: float,
+    expected_reason: str,
+    expected_mode: str,
+):
+    result = match_all(response, answers)
+    assert result.score == expected_score
+    assert result.reason == expected_reason
+    assert result.mode == expected_mode
+
+
+@pytest.mark.parametrize(
+    "response, answer, expected_score, expected_reason, expected_mode",
+    [
+        ("hello world", "hello world", 1.0, "Fuzzy match with 100.0% similarity", "fuzz"),
+        ("hello wrld", "hello world", 0.9, "Fuzzy match with 90.9% similarity", "fuzz"),
+        ("hello", "hello world", 0.45, "Fuzzy match with 45.5% similarity", "fuzz"),
+        ("", "hello world", 0.0, "Fuzzy match with 0.0% similarity", "fuzz"),
+    ],
+)
+def test_match_fuzzy(
+    response: str,
+    answer: str,
+    expected_score: float,
+    expected_reason: str,
+    expected_mode: str,
+):
+    result = match_fuzzy(response, answer)
+    assert result.score == pytest.approx(expected_score, abs=1e-2)
+    assert result.reason == expected_reason
+    assert result.mode == expected_mode
+
+
+@pytest.mark.parametrize(
+    "response, pattern, expected_score, expected_reason, expected_mode",
+    [
+        ("hello world", r"hello.*", 1.0, "Regex pattern matched", "regex"),
+        ("hello world", r"^hello.*$", 1.0, "Regex pattern matched", "regex"),
+        ("hello world", r"world$", 1.0, "Regex pattern matched", "regex"),
+        ("hello world", r"^goodbye.*$", 0.0, "Regex pattern did not match", "regex"),
+        ("hello world", r"[invalid[", 0.0, "Invalid regex pattern", "regex"),
+    ],
+)
+def test_match_regex(
+    response: str,
+    pattern: str,
+    expected_score: float,
+    expected_reason: str,
+    expected_mode: str,
+):
+    result = match_regex(response, pattern)
+    assert result.score == expected_score
+    assert result.reason == expected_reason
+    assert result.mode == expected_mode
+
+
+@pytest.mark.parametrize(
+    "response, answer, expected_score, expected_reason, expected_mode",
+    [
+        ("hello world", "hello world", 1.0, "String difference with 100.0% similarity", "diff"),
+        ("hello", "hello world", 0.625, "String difference with 62.5% similarity", "diff"),
+        ("", "hello world", 0.0, "String difference with 0.0% similarity", "diff"),
+        (100, 100, 1.0, "Numeric difference: 0", "diff"),
+        (90, 100, 0.9, "Numeric difference: 10", "diff"),
+        (0, 100, 0.0, "Numeric difference: 100", "diff"),
+        (-100, 100, 0.0, "Numeric difference: 200", "diff"),
+    ],
+)
+def test_match_diff(
+    response: str | int | float,
+    answer: str | int | float,
+    expected_score: float,
+    expected_reason: str,
+    expected_mode: str,
+):
+    result = match_diff(response, answer)
+    assert result.score == pytest.approx(expected_score, abs=1e-2)
+    assert result.reason == expected_reason
+    assert result.mode == expected_mode

hud/evaluators/tests/test_remote.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import pytest
+
+from hud.evaluators.base import EvaluationResult
+from hud.evaluators.remote import _remote_eval_call, remote_evaluate
+
+
+@pytest.mark.asyncio
+async def test_remote_eval_call_success(mocker):
+    mock_response = {
+        "score": 0.85,
+        "reason": "Good match",
+        "details": {"relevance": 0.9, "correctness": 0.8},
+    }
+    mock_make_request = mocker.patch(
+        "hud.evaluators.remote.make_request", return_value=mock_response
+    )
+
+    result = await _remote_eval_call(
+        response="test response", answer="test answer", eval_type="match"
+    )
+
+    assert result == mock_response
+    mock_make_request.assert_called_once()
+    call_args = mock_make_request.call_args[1]
+    assert call_args["method"] == "POST"
+    assert "evaluations/evaluate" in call_args["url"]
+    assert call_args["json"]["response"] == "test response"
+    assert call_args["json"]["answer"] == "test answer"
+    assert call_args["json"]["type"] == "match"
+
+
+@pytest.mark.asyncio
+async def test_remote_eval_call_with_config(mocker):
+    mock_response = {"score": 0.75, "reason": "Good", "details": {}}
+    mock_make_request = mocker.patch(
+        "hud.evaluators.remote.make_request", return_value=mock_response
+    )
+
+    config = {"threshold": 0.8, "strict": True}
+    result = await _remote_eval_call(
+        response="test response", answer="test answer", eval_type="judge", config=config
+    )
+
+    assert result == mock_response
+    mock_make_request.assert_called_once()
+    call_args = mock_make_request.call_args[1]
+    assert call_args["json"]["config"] == config
+
+
+@pytest.mark.asyncio
+async def test_remote_eval_call_failure(mocker):
+    mocker.patch("hud.evaluators.remote.make_request", side_effect=Exception("API error"))
+
+    result = await _remote_eval_call(
+        response="test response", answer="test answer", eval_type="match"
+    )
+
+    assert result["score"] == -1.0
+    assert "Remote evaluation failed" in result["reason"]
+    assert "API error" in result["reason"]
+    assert result["details"] == {}
+
+
+def test_remote_evaluate(mocker):
+    mock_result = {"score": 0.9, "reason": "Excellent match", "details": {"similarity": 0.95}}
+
+    async def mock_remote_call(*args, **kwargs):
+        return mock_result
+
+    mocker.patch("hud.evaluators.remote._remote_eval_call", side_effect=mock_remote_call)
+
+    result = remote_evaluate(
+        response="test response", answer="test answer", eval_type="custom_eval"
+    )
+
+    assert isinstance(result, EvaluationResult)
+    assert result.score == 0.9
+    assert result.reason == "Excellent match"
+    assert result.mode == "custom_eval"
+    assert result.criteria_scores == {"similarity": 0.95}
+
+
+def test_remote_evaluate_missing_fields(mocker):
+    mock_result = {"score": 0.8}  # Missing reason and details
+
+    async def mock_remote_call(*args, **kwargs):
+        return mock_result
+
+    mocker.patch("hud.evaluators.remote._remote_eval_call", side_effect=mock_remote_call)
+
+    result = remote_evaluate(response="test response", answer="test answer")
+
+    assert result.score == 0.8
+    assert result.reason == "Remote evaluation completed"
+    assert result.mode == "default"
+    assert result.criteria_scores == {}

hud/exceptions.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import httpx
+    from typing_extensions import Self
+
+logger = logging.getLogger(__name__)
+
+
+class HudException(Exception):
+    """Base exception class for all HUD SDK errors.
+
+    This is the parent class for all exceptions raised by the HUD SDK.
+    Consumers should be able to catch this exception to handle any HUD-related error.
+    """
+
+
+class HudRequestError(Exception):
+    """Any request to the HUD API can raise this exception."""
+
+    def __init__(
+        self,
+        message: str,
+        status_code: int | None = None,
+        response_text: str | None = None,
+        response_json: dict[str, Any] | None = None,
+        response_headers: dict[str, str] | None = None,
+    ) -> None:
+        self.message = message
+        self.status_code = status_code
+        self.response_text = response_text
+        self.response_json = response_json
+        self.response_headers = response_headers
+        super().__init__(message)
+
+    def __str__(self) -> str:
+        parts = [self.message]
+
+        if self.status_code:
+            parts.append(f"Status: {self.status_code}")
+
+        if self.response_text:
+            parts.append(f"Response Text: {self.response_text}")
+
+        if self.response_json:
+            parts.append(f"Response JSON: {self.response_json}")
+
+        if self.response_headers:
+            parts.append(f"Headers: {self.response_headers}")
+
+        return " | ".join(parts)
+
+    @classmethod
+    def from_httpx_error(cls, error: httpx.HTTPStatusError, context: str = "") -> Self:
+        """Create a RequestError from an HTTPx error response.
+
+        Args:
+            error: The HTTPx error response.
+            context: Additional context to include in the error message.
+
+        Returns:
+            A RequestError instance.
+        """
+        response = error.response
+        status_code = response.status_code
+        response_text = response.text
+        response_headers = dict(response.headers)
+
+        # Try to get detailed error info from JSON if available
+        response_json = None
+        try:
+            response_json = response.json()
+            detail = response_json.get("detail")
+            if detail:
+                message = f"Request failed: {detail}"
+            else:
+                # If no detail field but we have JSON, include a summary
+                message = f"Request failed with status {status_code}"
+                if len(response_json) <= 5:  # If it's a small object, include it in the message
+                    message += f" - JSON response: {response_json}"
+        except Exception:
+            # Fallback to simple message if JSON parsing fails
+            message = f"Request failed with status {status_code}"
+
+        # Add context if provided
+        if context:
+            message = f"{context}: {message}"
+
+        # Log the error details
+        logger.error(
+            "HTTP error from HUD SDK: %s | URL: %s | Status: %s | Response: %s%s",
+            message,
+            response.url,
+            status_code,
+            response_text[:500],
+            "..." if len(response_text) > 500 else "",
+        )
+        return cls(
+            message=message,
+            status_code=status_code,
+            response_text=response_text,
+            response_json=response_json,
+            response_headers=response_headers,
+        )
+
+
+class HudResponseError(HudException):
+    """Raised when an API response is invalid or missing required data.
+
+    This exception is raised when we receive a successful response (e.g. 200)
+    but the response data is invalid, missing required fields, or otherwise
+    cannot be processed.
+
+    Attributes:
+        message: A human-readable error message
+        response_json: The invalid response data
+    """
+
+    def __init__(
+        self,
+        message: str,
+        response_json: dict[str, Any] | None = None,
+    ) -> None:
+        self.message = message
+        self.response_json = response_json
+        super().__init__(message)
+
+    def __str__(self) -> str:
+        parts = [self.message]
+        if self.response_json:
+            parts.append(f"Response: {self.response_json}")
+        return " | ".join(parts)
+
+
+class HudAuthenticationError(HudException):
+    """Raised when authentication with the HUD API fails.
+
+    This exception is raised when an API key is missing, invalid, or
+    has insufficient permissions for the requested operation.
+    """
+
+
+class HudRateLimitError(HudException):
+    """Raised when the rate limit for the HUD API is exceeded.
+
+    This exception is raised when too many requests are made in a
+    short period of time.
+    """
+
+
+class HudTimeoutError(HudException):
+    """Raised when a request to the HUD API times out.
+
+    This exception is raised when a request takes longer than the
+    configured timeout period.
+    """
+
+
+class HudNetworkError(HudException):
+    """Raised when there is a network-related error.
+
+    This exception is raised when there are issues with the network
+    connection, DNS resolution, or other network-related problems.
+    """

hud/gym.py

@@ -8,15 +8,16 @@ from hud.env.environment import Environment
 from hud.env.local_docker_client import LocalDockerClient
 from hud.env.remote_client import RemoteClient
 from hud.env.remote_docker_client import RemoteDockerClient
-from hud.task import Task
 from hud.types import CustomGym, Gym
 from hud.utils.common import get_gym_id
 
 if TYPE_CHECKING:
     from hud.job import Job
+    from hud.task import Task
 
 logger = logging.getLogger("hud.gym")
 
+
 async def make(
     env_src: Gym | Task,
     *,
@@ -26,7 +27,7 @@ async def make(
 ) -> Environment:
     """
     Create an environment from an environment ID or a Task object.
-    
+
     Args:
         env_src: Environment ID or Task object
         job: Job object to associate with this environment
@@ -35,7 +36,7 @@ async def make(
     """
     if metadata is None:
         metadata = {}
-    
+
     # Handle job parameter
     effective_job_id = None
     if job is not None:
@@ -45,18 +46,19 @@ async def make(
     else:
         # Try to get an active job from the decorator context
         try:
-            from hud.job import get_active_job
-            active_job = get_active_job()
+            import hud.job
+
+            active_job = hud.job.get_active_job()
             if active_job:
                 effective_job_id = active_job.id
         except ImportError:
             pass  # Module not available, skip
-    
+
     gym = None
     task = None
-    if isinstance(env_src, Gym):
+    if isinstance(env_src, str | CustomGym):
         gym = env_src
-    elif isinstance(env_src, Task):
+    else:
         gym = env_src.gym
         task = env_src
 
@@ -77,7 +79,7 @@ async def make(
             )
         else:
             raise ValueError(f"Invalid environment location: {gym.location}")
-            
+
         # Set up the environment with a source path
         if gym.controller_source_dir:
             logger.info("Setting source path")
@@ -101,7 +103,7 @@ async def make(
 
     # Create the environment itself
     environment = Environment(client=client, metadata=metadata, task=task, build_data=build_data)
-    
+
     if task:
         await environment._setup()