hud-python 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

hud/evaluators/match.py

@@ -2,20 +2,27 @@ from __future__ import annotations
 
 import re
 from difflib import SequenceMatcher
-from typing import Any
+from typing import TYPE_CHECKING, Protocol
 
 from textdistance import levenshtein
 
 from hud.evaluators.base import EvaluationResult
 
+if TYPE_CHECKING:
+    from collections.abc import Sequence
 
-def match_single(response: Any, answer: Any) -> EvaluationResult:
+
+class _Stringable(Protocol):
+    def __str__(self) -> str: ...
+
+
+def match_single(response: _Stringable, answer: _Stringable) -> EvaluationResult:
     """Check if the answer is present within the response.
-    
+
     Args:
         response: The response to evaluate
         answer: The expected answer
-        
+
     Returns:
         EvaluationResult with score=1.0 if match, 0.0 otherwise
     """
@@ -23,54 +30,50 @@ def match_single(response: Any, answer: Any) -> EvaluationResult:
     return EvaluationResult(
         score=1.0 if passed else 0.0,
         reason="Exact match" if passed else "No exact match found",
-        mode="single"
+        mode="single",
     )
 
 
-def match_all(response: Any, answers: list) -> EvaluationResult:
+def match_all(response: _Stringable, answers: Sequence[_Stringable]) -> EvaluationResult:
     """Count how many expected answers are in the response.
-    
+
     Args:
         response: The response to evaluate
         answers: List of expected answers
-        
+
     Returns:
         EvaluationResult with score=proportion of matches (0.0-1.0)
     """
     response_str = str(response).lower()
     matches = 0
-    
+
     for answer in answers:
         if str(answer).lower() in response_str:
             matches += 1
-            
+
     score = matches / len(answers) if answers else 0.0
-    
+
     if matches == len(answers):
         reason = f"All {matches} expected items found"
     else:
         reason = f"Only {matches} of {len(answers)} expected items found"
-        
-    return EvaluationResult(
-        score=score,
-        reason=reason,
-        mode="all"
-    )
 
+    return EvaluationResult(score=score, reason=reason, mode="all")
 
-def match_fuzzy(response: Any, answer: Any) -> EvaluationResult:
+
+def match_fuzzy(response: _Stringable, answer: _Stringable) -> EvaluationResult:
     """Calculate similarity using Levenshtein distance.
-    
+
     Args:
         response: The response to evaluate
         answer: The expected answer
-        
+
     Returns:
         EvaluationResult with score=similarity (0.0-1.0)
     """
     s1 = str(response).lower()
     s2 = str(answer).lower()
-    
+
     if s1 == s2:
         score = 1.0
     elif len(s1) == 0 or len(s2) == 0:
@@ -80,21 +83,19 @@ def match_fuzzy(response: Any, answer: Any) -> EvaluationResult:
         distance = levenshtein.distance(s1, s2)
         max_len = max(len(s1), len(s2))
         score = 1.0 - (distance / max_len)
-        
+
     return EvaluationResult(
-        score=score,
-        reason=f"Fuzzy match with {score:.1%} similarity",
-        mode="fuzz"
+        score=score, reason=f"Fuzzy match with {score:.1%} similarity", mode="fuzz"
     )
 
 
-def match_regex(response: Any, pattern: str) -> EvaluationResult:
+def match_regex(response: _Stringable, pattern: str) -> EvaluationResult:
     """Check if response matches regex pattern.
-    
+
     Args:
         response: The response to evaluate
         pattern: Regular expression pattern to match
-        
+
     Returns:
         EvaluationResult with score=1.0 if match, 0.0 otherwise
     """
@@ -104,23 +105,19 @@ def match_regex(response: Any, pattern: str) -> EvaluationResult:
         return EvaluationResult(
             score=1.0 if passed else 0.0,
             reason="Regex pattern matched" if passed else "Regex pattern did not match",
-            mode="regex"
+            mode="regex",
         )
     except re.error:
-        return EvaluationResult(
-            score=0.0,
-            reason="Invalid regex pattern",
-            mode="regex"
-        )
+        return EvaluationResult(score=0.0, reason="Invalid regex pattern", mode="regex")
 
 
-def match_diff(response: Any, answer: Any) -> EvaluationResult:
+def match_diff(response: _Stringable, answer: _Stringable) -> EvaluationResult:
     """Compare difference between response and answer.
-    
+
     Args:
         response: The response to evaluate
         answer: The expected answer
-        
+
     Returns:
         EvaluationResult with score=similarity (0.0-1.0)
     """
@@ -130,34 +127,30 @@ def match_diff(response: Any, answer: Any) -> EvaluationResult:
     else:
         score = _match_string_diff(response, answer)
         reason = f"String difference with {score:.1%} similarity"
-            
-    return EvaluationResult(
-        score=score,
-        reason=reason,
-        mode="diff"
-    )
+
+    return EvaluationResult(score=score, reason=reason, mode="diff")
 
 
-def _match_string_diff(response: Any, answer: Any) -> float:
+def _match_string_diff(response: _Stringable, answer: _Stringable) -> float:
     """Compare difference between response and answer strings."""
     matcher = SequenceMatcher(None, str(response), str(answer))
     return matcher.ratio()
-    
+
 
 def _match_numeric_diff(response: float, answer: float) -> float:
     """Calculate normalized difference between numeric values.
-    
+
     Returns a value between 0 and 1, where 1 means identical and 0 means maximum difference.
     """
     if response == answer:
         return 1.0
-        
+
     # Simple absolute difference normalized to a 0-1 scale
     diff = abs(response - answer)
     max_val = max(abs(response), abs(answer))
-    
+
     if max_val == 0:
         return 1.0  # Both are zero
-        
+
     # Normalize and invert so 1.0 means identical
     return max(0.0, 1.0 - min(1.0, diff / max_val))

hud/evaluators/remote.py

@@ -9,19 +9,16 @@ from hud.settings import settings
 
 
 async def _remote_eval_call(
-    response: Any,
-    answer: Any,
-    eval_type: str,
-    config: dict[str, Any] | None = None
+    response: Any, answer: Any, eval_type: str, config: dict[str, Any] | None = None
 ) -> dict[str, Any]:
     """Send an evaluation request to the remote server.
-    
+
     Args:
         response: The response to evaluate
         answer: The reference answer to compare against
         eval_type: Type of evaluation (e.g., "match", "judge", "agent")
         config: Optional configuration parameters
-        
+
     Returns:
         Dictionary with evaluation results from the server
     """
@@ -33,46 +30,36 @@ async def _remote_eval_call(
                 "response": response,
                 "answer": answer,
                 "type": eval_type,
-                "config": config or {}
+                "config": config or {},
             },
             api_key=settings.api_key,
         )
         return result
     except Exception as e:
-        return {
-            "score": -1.0,
-            "reason": f"Remote evaluation failed: {e!s}",
-            "details": {}
-        }
+        return {"score": -1.0, "reason": f"Remote evaluation failed: {e!s}", "details": {}}
 
 
 def remote_evaluate(
-    response: Any,
-    answer: Any,
-    eval_type: str = "default",
-    config: dict[str, Any] | None = None
+    response: Any, answer: Any, eval_type: str = "default", config: dict[str, Any] | None = None
 ) -> EvaluationResult:
     """Evaluate a response using remote evaluation services.
-    
+
     Args:
         response: The response to evaluate
         answer: The reference answer to compare against
         eval_type: Type of evaluation to perform
         config: Optional configuration for the evaluation
-        
+
     Returns:
         EvaluationResult containing the evaluation results
     """
-    result = asyncio.run(_remote_eval_call(
-        response=response,
-        answer=answer,
-        eval_type=eval_type,
-        config=config
-    ))
-    
+    result = asyncio.run(
+        _remote_eval_call(response=response, answer=answer, eval_type=eval_type, config=config)
+    )
+
     return EvaluationResult(
         score=result.get("score", -1.0),
         reason=result.get("reason", "Remote evaluation completed"),
         mode=eval_type,
-        criteria_scores=result.get("details", {})
+        criteria_scores=result.get("details", {}),
     )

hud/evaluators/tests/test_inspect.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from hud.evaluators.inspect import inspect_evaluate
+
+
+def test_inspect_evaluate_basic():
+    """Test basic functionality of inspect_evaluate."""
+    result = inspect_evaluate("Test response", "Test answer")
+
+    assert result.score == 0.0
+    assert result.reason == "Inspect evaluation not implemented"
+    assert result.mode == "inspect"

hud/evaluators/tests/test_judge.py

@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+import base64
+
+import pytest
+
+from hud.evaluators.base import EvaluationResult
+from hud.evaluators.judge import (
+    _call_eval_endpoint,
+    _evaluate_with_llm,
+    _is_base64_image,
+    _process_input,
+    judge,
+)
+
+
+class _MockLLM:
+    """Mock LLM for testing."""
+
+    def __init__(self, response_text):
+        self.response_text = response_text
+
+    async def ainvoke(self, _prompt: str) -> str:
+        return self.response_text
+
+
+@pytest.mark.parametrize(
+    "input_data, expected_result",
+    [
+        ("Hello world", "Hello world"),
+        (123, "123"),
+        (["Hello", "world"], ["Hello", "world"]),
+        ({"key": "value"}, {"key": "value"}),
+        (b"Hello world", base64.b64encode(b"Hello world").decode("utf-8")),
+    ],
+)
+def test_process_input(input_data, expected_result):
+    """Test processing various input types."""
+    result = _process_input(input_data)
+    assert result == expected_result
+
+
+@pytest.mark.parametrize(
+    "input_data, expected_result",
+    [
+        ("not an image", False),
+        ("data:image/png;base64,abc123", True),
+        (b"not an image", False),
+        (123, False),
+    ],
+)
+def test_is_base64_image(input_data, expected_result):
+    """Test base64 image detection."""
+    assert _is_base64_image(input_data) == expected_result
+
+
+def test_is_base64_image_with_signatures(mocker):
+    """Test base64 image detection with common image signatures."""
+    # Mock base64.b64decode to return different image signatures
+    mock_b64decode = mocker.patch("base64.b64decode")
+
+    # Test JPEG signature
+    mock_b64decode.return_value = b"\xff\xd8\xff" + b"some data"
+    assert _is_base64_image("not_really_base64_but_mocked") is True
+
+    # Test PNG signature
+    mock_b64decode.return_value = b"\x89PNG\r\n\x1a\n" + b"some data"
+    assert _is_base64_image("not_really_base64_but_mocked") is True
+
+    # Test GIF signature
+    mock_b64decode.return_value = b"GIF8" + b"some data"
+    assert _is_base64_image("not_really_base64_but_mocked") is True
+
+    # Test RIFF signature (WebP)
+    mock_b64decode.return_value = b"RIFF" + b"some data"
+    assert _is_base64_image("not_really_base64_but_mocked") is True
+
+
+@pytest.mark.asyncio
+async def test_call_eval_endpoint_success(mocker):
+    """Test successful remote evaluation call."""
+    mock_response = {
+        "score": 0.8,
+        "reason": "Good response",
+        "criteria_scores": {"relevance": 0.9, "accuracy": 0.7},
+    }
+    mock_make_request = mocker.patch(
+        "hud.evaluators.judge.make_request", return_value=mock_response
+    )
+    result = await _call_eval_endpoint("test response", "test answer", [], "LLM")
+    assert result == mock_response
+    mock_make_request.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_call_eval_endpoint_failure(mocker):
+    """Test remote evaluation call failure."""
+    mocker.patch("hud.evaluators.judge.make_request", side_effect=Exception("API error"))
+    result = await _call_eval_endpoint("test response", "test answer", [], "LLM")
+    assert result["score"] == -1.0
+    assert "Remote evaluation failed" in result["reason"]
+    assert result["criteria_scores"] == {}
+
+
+def test_judge_without_llm(mocker):
+    """Test judge function without custom LLM."""
+    mock_result = {
+        "score": 0.9,
+        "reason": "Good answer",
+        "criteria_scores": {"relevance": 1.0},
+    }
+
+    async def mock_endpoint(*args, **kwargs):
+        return mock_result
+
+    mocker.patch("hud.evaluators.judge._call_eval_endpoint", mock_endpoint)
+    result = judge("test response", "test answer")
+
+    assert result.score == 0.9
+    assert result.reason == "Good answer"
+    assert result.mode == "LLM"
+    assert result.criteria_scores == {"relevance": 1.0}
+
+
+def test_judge_with_image_answer(mocker):
+    """Test judge function with an image as the answer."""
+    mock_result = {
+        "score": 0.85,
+        "reason": "Good image analysis",
+        "criteria_scores": {"visual_accuracy": 0.85},
+    }
+
+    async def mock_endpoint(*args, **kwargs):
+        return mock_result
+
+    mocker.patch("hud.evaluators.judge._call_eval_endpoint", mock_endpoint)
+
+    # Create a mock image
+    image_data = b"fake_image_data"
+    base64_image = base64.b64encode(image_data).decode("utf-8")
+    image_uri = f"data:image/jpeg;base64,{base64_image}"
+
+    result = judge("description of image", image_uri)
+
+    assert result.score == 0.85
+    assert result.reason == "Good image analysis"
+    assert result.mode == "VLM"  # Should use VLM mode for images
+    assert result.criteria_scores == {"visual_accuracy": 0.85}
+
+
+def test_judge_with_llm(mocker):
+    """Test judge function with custom LLM."""
+    mock_llm = _MockLLM('{"score": 0.75, "reason": "Pretty good"}')
+    mock_result = EvaluationResult(score=0.75, reason="Pretty good", mode="custom_llm")
+    mocker.patch("hud.evaluators.judge._evaluate_with_llm", return_value=mock_result)
+    result = judge("test response", "test answer", llm=mock_llm)
+    assert result.score == 0.75
+    assert result.reason == "Pretty good"
+    assert result.mode == "custom_llm"
+
+
+def test_evaluate_with_llm_valid_json():
+    """Test _evaluate_with_llm with valid JSON response."""
+    llm = _MockLLM('{"score": 0.85, "reason": "The response is accurate and well-structured."}')
+    result = _evaluate_with_llm("test response", "test answer", llm)
+
+    assert result.score == 0.85
+    assert result.reason == "The response is accurate and well-structured."
+    assert result.mode == "custom_llm"
+
+
+def test_evaluate_with_llm_json_in_text():
+    """Test _evaluate_with_llm with JSON embedded in text."""
+    llm_response = """
+    I've evaluated the response and here's my assessment:
+    
+    {"score": 0.7, "reason": "Good but could be more detailed"}
+    
+    I hope this helps!
+    """
+    llm = _MockLLM(llm_response)
+    result = _evaluate_with_llm("test response", "test answer", llm)
+
+    assert result.score == 0.7
+    assert result.reason == "Good but could be more detailed"
+    assert result.mode == "custom_llm"
+
+
+def test_evaluate_with_llm_invalid_json():
+    """Test _evaluate_with_llm with invalid JSON response."""
+    llm = _MockLLM("This is not a JSON response")
+    result = _evaluate_with_llm("test response", "test answer", llm)
+
+    assert result.score == 0.5  # Default score for unparseable responses
+    assert "Unable to parse LLM response as JSON" in result.reason
+    assert result.mode == "custom_llm"
+
+
+def test_evaluate_with_llm_exception(mocker):
+    """Test _evaluate_with_llm when an exception occurs."""
+    # Mock the LLM to raise an exception
+    failing_llm = _MockLLM("doesn't matter")
+    mocker.patch.object(failing_llm, "ainvoke", side_effect=Exception("LLM API error"))
+
+    result = _evaluate_with_llm("test response", "test answer", failing_llm)
+
+    assert result.score == 0.0  # Zero score for errors
+    assert "LLM evaluation error: LLM API error" in result.reason
+    assert result.mode == "custom_llm"
+
+
+def test_evaluate_with_llm_with_criteria():
+    """Test _evaluate_with_llm with evaluation criteria."""
+    llm = _MockLLM('{"score": 0.9, "reason": "Excellent match on all criteria"}')
+
+    # Test with string criteria
+    string_criteria = ["Accuracy", "Relevance", "Completeness"]
+    result = _evaluate_with_llm("test response", "test answer", llm, criteria=string_criteria)
+
+    assert result.score == 0.9
+    assert result.reason == "Excellent match on all criteria"
+
+    # Test with dict criteria
+    dict_criteria = [
+        {"description": "Factual accuracy", "weight": 0.6},
+        {"description": "Grammar and spelling", "weight": 0.4},
+    ]
+    result = _evaluate_with_llm("test response", "test answer", llm, criteria=dict_criteria)
+
+    assert result.score == 0.9
+    assert result.reason == "Excellent match on all criteria"

hud/evaluators/tests/test_match.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import pytest
+
+from hud.evaluators.match import match_all, match_diff, match_fuzzy, match_regex, match_single
+
+
+@pytest.mark.parametrize(
+    "response, answer, expected_score, expected_reason, expected_mode",
+    [
+        ("Hello, world!", "world", 1.0, "Exact match", "single"),
+        ("Hello, world!", "not world", 0.0, "No exact match found", "single"),
+    ],
+)
+def test_match_single(
+    response: str,
+    answer: str,
+    expected_score: float,
+    expected_reason: str,
+    expected_mode: str,
+):
+    result = match_single(response, answer)
+    assert result.score == expected_score
+    assert result.reason == expected_reason
+    assert result.mode == expected_mode
+
+
+@pytest.mark.parametrize(
+    "response, answers, expected_score, expected_reason, expected_mode",
+    [
+        ("Hello, world!", ["world", "hello"], 1.0, "All 2 expected items found", "all"),
+        ("Hello, world!", ["world", "not hello"], 0.5, "Only 1 of 2 expected items found", "all"),
+    ],
+)
+def test_match_all(
+    response: str,
+    answers: list[str],
+    expected_score: float,
+    expected_reason: str,
+    expected_mode: str,
+):
+    result = match_all(response, answers)
+    assert result.score == expected_score
+    assert result.reason == expected_reason
+    assert result.mode == expected_mode
+
+
+@pytest.mark.parametrize(
+    "response, answer, expected_score, expected_reason, expected_mode",
+    [
+        ("hello world", "hello world", 1.0, "Fuzzy match with 100.0% similarity", "fuzz"),
+        ("hello wrld", "hello world", 0.9, "Fuzzy match with 90.9% similarity", "fuzz"),
+        ("hello", "hello world", 0.45, "Fuzzy match with 45.5% similarity", "fuzz"),
+        ("", "hello world", 0.0, "Fuzzy match with 0.0% similarity", "fuzz"),
+    ],
+)
+def test_match_fuzzy(
+    response: str,
+    answer: str,
+    expected_score: float,
+    expected_reason: str,
+    expected_mode: str,
+):
+    result = match_fuzzy(response, answer)
+    assert result.score == pytest.approx(expected_score, abs=1e-2)
+    assert result.reason == expected_reason
+    assert result.mode == expected_mode
+
+
+@pytest.mark.parametrize(
+    "response, pattern, expected_score, expected_reason, expected_mode",
+    [
+        ("hello world", r"hello.*", 1.0, "Regex pattern matched", "regex"),
+        ("hello world", r"^hello.*$", 1.0, "Regex pattern matched", "regex"),
+        ("hello world", r"world$", 1.0, "Regex pattern matched", "regex"),
+        ("hello world", r"^goodbye.*$", 0.0, "Regex pattern did not match", "regex"),
+        ("hello world", r"[invalid[", 0.0, "Invalid regex pattern", "regex"),
+    ],
+)
+def test_match_regex(
+    response: str,
+    pattern: str,
+    expected_score: float,
+    expected_reason: str,
+    expected_mode: str,
+):
+    result = match_regex(response, pattern)
+    assert result.score == expected_score
+    assert result.reason == expected_reason
+    assert result.mode == expected_mode
+
+
+@pytest.mark.parametrize(
+    "response, answer, expected_score, expected_reason, expected_mode",
+    [
+        ("hello world", "hello world", 1.0, "String difference with 100.0% similarity", "diff"),
+        ("hello", "hello world", 0.625, "String difference with 62.5% similarity", "diff"),
+        ("", "hello world", 0.0, "String difference with 0.0% similarity", "diff"),
+        (100, 100, 1.0, "Numeric difference: 0", "diff"),
+        (90, 100, 0.9, "Numeric difference: 10", "diff"),
+        (0, 100, 0.0, "Numeric difference: 100", "diff"),
+        (-100, 100, 0.0, "Numeric difference: 200", "diff"),
+    ],
+)
+def test_match_diff(
+    response: str | int | float,
+    answer: str | int | float,
+    expected_score: float,
+    expected_reason: str,
+    expected_mode: str,
+):
+    result = match_diff(response, answer)
+    assert result.score == pytest.approx(expected_score, abs=1e-2)
+    assert result.reason == expected_reason
+    assert result.mode == expected_mode