hud-python 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

hud/evaluators/remote.py

@@ -1,65 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-from typing import Any
-
-from hud.evaluators.base import EvaluationResult
-from hud.server import make_request
-from hud.settings import settings
-
-
-async def _remote_eval_call(
-    response: Any, answer: Any, eval_type: str, config: dict[str, Any] | None = None
-) -> dict[str, Any]:
-    """Send an evaluation request to the remote server.
-
-    Args:
-        response: The response to evaluate
-        answer: The reference answer to compare against
-        eval_type: Type of evaluation (e.g., "match", "judge", "agent")
-        config: Optional configuration parameters
-
-    Returns:
-        Dictionary with evaluation results from the server
-    """
-    try:
-        result = await make_request(
-            method="POST",
-            url=f"{settings.base_url}/evaluations/evaluate",
-            json={
-                "response": response,
-                "answer": answer,
-                "type": eval_type,
-                "config": config or {},
-            },
-            api_key=settings.api_key,
-        )
-        return result
-    except Exception as e:
-        return {"score": -1.0, "reason": f"Remote evaluation failed: {e!s}", "details": {}}
-
-
-def remote_evaluate(
-    response: Any, answer: Any, eval_type: str = "default", config: dict[str, Any] | None = None
-) -> EvaluationResult:
-    """Evaluate a response using remote evaluation services.
-
-    Args:
-        response: The response to evaluate
-        answer: The reference answer to compare against
-        eval_type: Type of evaluation to perform
-        config: Optional configuration for the evaluation
-
-    Returns:
-        EvaluationResult containing the evaluation results
-    """
-    result = asyncio.run(
-        _remote_eval_call(response=response, answer=answer, eval_type=eval_type, config=config)
-    )
-
-    return EvaluationResult(
-        score=result.get("score", -1.0),
-        reason=result.get("reason", "Remote evaluation completed"),
-        mode=eval_type,
-        criteria_scores=result.get("details", {}),
-    )

hud/evaluators/tests/test_inspect.py

@@ -1,12 +0,0 @@
-from __future__ import annotations
-
-from hud.evaluators.inspect import inspect_evaluate
-
-
-def test_inspect_evaluate_basic():
-    """Test basic functionality of inspect_evaluate."""
-    result = inspect_evaluate("Test response", "Test answer")
-
-    assert result.score == 0.0
-    assert result.reason == "Inspect evaluation not implemented"
-    assert result.mode == "inspect"

hud/evaluators/tests/test_judge.py

@@ -1,231 +0,0 @@
-from __future__ import annotations
-
-import base64
-
-import pytest
-
-from hud.evaluators.base import EvaluationResult
-from hud.evaluators.judge import (
-    _call_eval_endpoint,
-    _evaluate_with_llm,
-    _is_base64_image,
-    _process_input,
-    judge,
-)
-
-
-class _MockLLM:
-    """Mock LLM for testing."""
-
-    def __init__(self, response_text):
-        self.response_text = response_text
-
-    async def ainvoke(self, _prompt: str) -> str:
-        return self.response_text
-
-
-@pytest.mark.parametrize(
-    "input_data, expected_result",
-    [
-        ("Hello world", "Hello world"),
-        (123, "123"),
-        (["Hello", "world"], ["Hello", "world"]),
-        ({"key": "value"}, {"key": "value"}),
-        (b"Hello world", base64.b64encode(b"Hello world").decode("utf-8")),
-    ],
-)
-def test_process_input(input_data, expected_result):
-    """Test processing various input types."""
-    result = _process_input(input_data)
-    assert result == expected_result
-
-
-@pytest.mark.parametrize(
-    "input_data, expected_result",
-    [
-        ("not an image", False),
-        ("data:image/png;base64,abc123", True),
-        (b"not an image", False),
-        (123, False),
-    ],
-)
-def test_is_base64_image(input_data, expected_result):
-    """Test base64 image detection."""
-    assert _is_base64_image(input_data) == expected_result
-
-
-def test_is_base64_image_with_signatures(mocker):
-    """Test base64 image detection with common image signatures."""
-    # Mock base64.b64decode to return different image signatures
-    mock_b64decode = mocker.patch("base64.b64decode")
-
-    # Test JPEG signature
-    mock_b64decode.return_value = b"\xff\xd8\xff" + b"some data"
-    assert _is_base64_image("not_really_base64_but_mocked") is True
-
-    # Test PNG signature
-    mock_b64decode.return_value = b"\x89PNG\r\n\x1a\n" + b"some data"
-    assert _is_base64_image("not_really_base64_but_mocked") is True
-
-    # Test GIF signature
-    mock_b64decode.return_value = b"GIF8" + b"some data"
-    assert _is_base64_image("not_really_base64_but_mocked") is True
-
-    # Test RIFF signature (WebP)
-    mock_b64decode.return_value = b"RIFF" + b"some data"
-    assert _is_base64_image("not_really_base64_but_mocked") is True
-
-
-@pytest.mark.asyncio
-async def test_call_eval_endpoint_success(mocker):
-    """Test successful remote evaluation call."""
-    mock_response = {
-        "score": 0.8,
-        "reason": "Good response",
-        "criteria_scores": {"relevance": 0.9, "accuracy": 0.7},
-    }
-    mock_make_request = mocker.patch(
-        "hud.evaluators.judge.make_request", return_value=mock_response
-    )
-    result = await _call_eval_endpoint("test response", "test answer", [], "LLM")
-    assert result == mock_response
-    mock_make_request.assert_called_once()
-
-
-@pytest.mark.asyncio
-async def test_call_eval_endpoint_failure(mocker):
-    """Test remote evaluation call failure."""
-    mocker.patch("hud.evaluators.judge.make_request", side_effect=Exception("API error"))
-    result = await _call_eval_endpoint("test response", "test answer", [], "LLM")
-    assert result["score"] == -1.0
-    assert "Remote evaluation failed" in result["reason"]
-    assert result["criteria_scores"] == {}
-
-
-def test_judge_without_llm(mocker):
-    """Test judge function without custom LLM."""
-    mock_result = {
-        "score": 0.9,
-        "reason": "Good answer",
-        "criteria_scores": {"relevance": 1.0},
-    }
-
-    async def mock_endpoint(*args, **kwargs):
-        return mock_result
-
-    mocker.patch("hud.evaluators.judge._call_eval_endpoint", mock_endpoint)
-    result = judge("test response", "test answer")
-
-    assert result.score == 0.9
-    assert result.reason == "Good answer"
-    assert result.mode == "LLM"
-    assert result.criteria_scores == {"relevance": 1.0}
-
-
-def test_judge_with_image_answer(mocker):
-    """Test judge function with an image as the answer."""
-    mock_result = {
-        "score": 0.85,
-        "reason": "Good image analysis",
-        "criteria_scores": {"visual_accuracy": 0.85},
-    }
-
-    async def mock_endpoint(*args, **kwargs):
-        return mock_result
-
-    mocker.patch("hud.evaluators.judge._call_eval_endpoint", mock_endpoint)
-
-    # Create a mock image
-    image_data = b"fake_image_data"
-    base64_image = base64.b64encode(image_data).decode("utf-8")
-    image_uri = f"data:image/jpeg;base64,{base64_image}"
-
-    result = judge("description of image", image_uri)
-
-    assert result.score == 0.85
-    assert result.reason == "Good image analysis"
-    assert result.mode == "VLM"  # Should use VLM mode for images
-    assert result.criteria_scores == {"visual_accuracy": 0.85}
-
-
-def test_judge_with_llm(mocker):
-    """Test judge function with custom LLM."""
-    mock_llm = _MockLLM('{"score": 0.75, "reason": "Pretty good"}')
-    mock_result = EvaluationResult(score=0.75, reason="Pretty good", mode="custom_llm")
-    mocker.patch("hud.evaluators.judge._evaluate_with_llm", return_value=mock_result)
-    result = judge("test response", "test answer", llm=mock_llm)
-    assert result.score == 0.75
-    assert result.reason == "Pretty good"
-    assert result.mode == "custom_llm"
-
-
-def test_evaluate_with_llm_valid_json():
-    """Test _evaluate_with_llm with valid JSON response."""
-    llm = _MockLLM('{"score": 0.85, "reason": "The response is accurate and well-structured."}')
-    result = _evaluate_with_llm("test response", "test answer", llm)
-
-    assert result.score == 0.85
-    assert result.reason == "The response is accurate and well-structured."
-    assert result.mode == "custom_llm"
-
-
-def test_evaluate_with_llm_json_in_text():
-    """Test _evaluate_with_llm with JSON embedded in text."""
-    llm_response = """
-    I've evaluated the response and here's my assessment:
-    
-    {"score": 0.7, "reason": "Good but could be more detailed"}
-    
-    I hope this helps!
-    """
-    llm = _MockLLM(llm_response)
-    result = _evaluate_with_llm("test response", "test answer", llm)
-
-    assert result.score == 0.7
-    assert result.reason == "Good but could be more detailed"
-    assert result.mode == "custom_llm"
-
-
-def test_evaluate_with_llm_invalid_json():
-    """Test _evaluate_with_llm with invalid JSON response."""
-    llm = _MockLLM("This is not a JSON response")
-    result = _evaluate_with_llm("test response", "test answer", llm)
-
-    assert result.score == 0.5  # Default score for unparseable responses
-    assert "Unable to parse LLM response as JSON" in result.reason
-    assert result.mode == "custom_llm"
-
-
-def test_evaluate_with_llm_exception(mocker):
-    """Test _evaluate_with_llm when an exception occurs."""
-    # Mock the LLM to raise an exception
-    failing_llm = _MockLLM("doesn't matter")
-    mocker.patch.object(failing_llm, "ainvoke", side_effect=Exception("LLM API error"))
-
-    result = _evaluate_with_llm("test response", "test answer", failing_llm)
-
-    assert result.score == 0.0  # Zero score for errors
-    assert "LLM evaluation error: LLM API error" in result.reason
-    assert result.mode == "custom_llm"
-
-
-def test_evaluate_with_llm_with_criteria():
-    """Test _evaluate_with_llm with evaluation criteria."""
-    llm = _MockLLM('{"score": 0.9, "reason": "Excellent match on all criteria"}')
-
-    # Test with string criteria
-    string_criteria = ["Accuracy", "Relevance", "Completeness"]
-    result = _evaluate_with_llm("test response", "test answer", llm, criteria=string_criteria)
-
-    assert result.score == 0.9
-    assert result.reason == "Excellent match on all criteria"
-
-    # Test with dict criteria
-    dict_criteria = [
-        {"description": "Factual accuracy", "weight": 0.6},
-        {"description": "Grammar and spelling", "weight": 0.4},
-    ]
-    result = _evaluate_with_llm("test response", "test answer", llm, criteria=dict_criteria)
-
-    assert result.score == 0.9
-    assert result.reason == "Excellent match on all criteria"

hud/evaluators/tests/test_match.py

@@ -1,115 +0,0 @@
-from __future__ import annotations
-
-import pytest
-
-from hud.evaluators.match import match_all, match_diff, match_fuzzy, match_regex, match_single
-
-
-@pytest.mark.parametrize(
-    "response, answer, expected_score, expected_reason, expected_mode",
-    [
-        ("Hello, world!", "world", 1.0, "Exact match", "single"),
-        ("Hello, world!", "not world", 0.0, "No exact match found", "single"),
-    ],
-)
-def test_match_single(
-    response: str,
-    answer: str,
-    expected_score: float,
-    expected_reason: str,
-    expected_mode: str,
-):
-    result = match_single(response, answer)
-    assert result.score == expected_score
-    assert result.reason == expected_reason
-    assert result.mode == expected_mode
-
-
-@pytest.mark.parametrize(
-    "response, answers, expected_score, expected_reason, expected_mode",
-    [
-        ("Hello, world!", ["world", "hello"], 1.0, "All 2 expected items found", "all"),
-        ("Hello, world!", ["world", "not hello"], 0.5, "Only 1 of 2 expected items found", "all"),
-    ],
-)
-def test_match_all(
-    response: str,
-    answers: list[str],
-    expected_score: float,
-    expected_reason: str,
-    expected_mode: str,
-):
-    result = match_all(response, answers)
-    assert result.score == expected_score
-    assert result.reason == expected_reason
-    assert result.mode == expected_mode
-
-
-@pytest.mark.parametrize(
-    "response, answer, expected_score, expected_reason, expected_mode",
-    [
-        ("hello world", "hello world", 1.0, "Fuzzy match with 100.0% similarity", "fuzz"),
-        ("hello wrld", "hello world", 0.9, "Fuzzy match with 90.9% similarity", "fuzz"),
-        ("hello", "hello world", 0.45, "Fuzzy match with 45.5% similarity", "fuzz"),
-        ("", "hello world", 0.0, "Fuzzy match with 0.0% similarity", "fuzz"),
-    ],
-)
-def test_match_fuzzy(
-    response: str,
-    answer: str,
-    expected_score: float,
-    expected_reason: str,
-    expected_mode: str,
-):
-    result = match_fuzzy(response, answer)
-    assert result.score == pytest.approx(expected_score, abs=1e-2)
-    assert result.reason == expected_reason
-    assert result.mode == expected_mode
-
-
-@pytest.mark.parametrize(
-    "response, pattern, expected_score, expected_reason, expected_mode",
-    [
-        ("hello world", r"hello.*", 1.0, "Regex pattern matched", "regex"),
-        ("hello world", r"^hello.*$", 1.0, "Regex pattern matched", "regex"),
-        ("hello world", r"world$", 1.0, "Regex pattern matched", "regex"),
-        ("hello world", r"^goodbye.*$", 0.0, "Regex pattern did not match", "regex"),
-        ("hello world", r"[invalid[", 0.0, "Invalid regex pattern", "regex"),
-    ],
-)
-def test_match_regex(
-    response: str,
-    pattern: str,
-    expected_score: float,
-    expected_reason: str,
-    expected_mode: str,
-):
-    result = match_regex(response, pattern)
-    assert result.score == expected_score
-    assert result.reason == expected_reason
-    assert result.mode == expected_mode
-
-
-@pytest.mark.parametrize(
-    "response, answer, expected_score, expected_reason, expected_mode",
-    [
-        ("hello world", "hello world", 1.0, "String difference with 100.0% similarity", "diff"),
-        ("hello", "hello world", 0.625, "String difference with 62.5% similarity", "diff"),
-        ("", "hello world", 0.0, "String difference with 0.0% similarity", "diff"),
-        (100, 100, 1.0, "Numeric difference: 0", "diff"),
-        (90, 100, 0.9, "Numeric difference: 10", "diff"),
-        (0, 100, 0.0, "Numeric difference: 100", "diff"),
-        (-100, 100, 0.0, "Numeric difference: 200", "diff"),
-    ],
-)
-def test_match_diff(
-    response: str | int | float,
-    answer: str | int | float,
-    expected_score: float,
-    expected_reason: str,
-    expected_mode: str,
-):
-    result = match_diff(response, answer)
-    assert result.score == pytest.approx(expected_score, abs=1e-2)
-    assert result.reason == expected_reason
-    assert result.mode == expected_mode

hud/evaluators/tests/test_remote.py

@@ -1,98 +0,0 @@
-from __future__ import annotations
-
-import pytest
-
-from hud.evaluators.base import EvaluationResult
-from hud.evaluators.remote import _remote_eval_call, remote_evaluate
-
-
-@pytest.mark.asyncio
-async def test_remote_eval_call_success(mocker):
-    mock_response = {
-        "score": 0.85,
-        "reason": "Good match",
-        "details": {"relevance": 0.9, "correctness": 0.8},
-    }
-    mock_make_request = mocker.patch(
-        "hud.evaluators.remote.make_request", return_value=mock_response
-    )
-
-    result = await _remote_eval_call(
-        response="test response", answer="test answer", eval_type="match"
-    )
-
-    assert result == mock_response
-    mock_make_request.assert_called_once()
-    call_args = mock_make_request.call_args[1]
-    assert call_args["method"] == "POST"
-    assert "evaluations/evaluate" in call_args["url"]
-    assert call_args["json"]["response"] == "test response"
-    assert call_args["json"]["answer"] == "test answer"
-    assert call_args["json"]["type"] == "match"
-
-
-@pytest.mark.asyncio
-async def test_remote_eval_call_with_config(mocker):
-    mock_response = {"score": 0.75, "reason": "Good", "details": {}}
-    mock_make_request = mocker.patch(
-        "hud.evaluators.remote.make_request", return_value=mock_response
-    )
-
-    config = {"threshold": 0.8, "strict": True}
-    result = await _remote_eval_call(
-        response="test response", answer="test answer", eval_type="judge", config=config
-    )
-
-    assert result == mock_response
-    mock_make_request.assert_called_once()
-    call_args = mock_make_request.call_args[1]
-    assert call_args["json"]["config"] == config
-
-
-@pytest.mark.asyncio
-async def test_remote_eval_call_failure(mocker):
-    mocker.patch("hud.evaluators.remote.make_request", side_effect=Exception("API error"))
-
-    result = await _remote_eval_call(
-        response="test response", answer="test answer", eval_type="match"
-    )
-
-    assert result["score"] == -1.0
-    assert "Remote evaluation failed" in result["reason"]
-    assert "API error" in result["reason"]
-    assert result["details"] == {}
-
-
-def test_remote_evaluate(mocker):
-    mock_result = {"score": 0.9, "reason": "Excellent match", "details": {"similarity": 0.95}}
-
-    async def mock_remote_call(*args, **kwargs):
-        return mock_result
-
-    mocker.patch("hud.evaluators.remote._remote_eval_call", side_effect=mock_remote_call)
-
-    result = remote_evaluate(
-        response="test response", answer="test answer", eval_type="custom_eval"
-    )
-
-    assert isinstance(result, EvaluationResult)
-    assert result.score == 0.9
-    assert result.reason == "Excellent match"
-    assert result.mode == "custom_eval"
-    assert result.criteria_scores == {"similarity": 0.95}
-
-
-def test_remote_evaluate_missing_fields(mocker):
-    mock_result = {"score": 0.8}  # Missing reason and details
-
-    async def mock_remote_call(*args, **kwargs):
-        return mock_result
-
-    mocker.patch("hud.evaluators.remote._remote_eval_call", side_effect=mock_remote_call)
-
-    result = remote_evaluate(response="test response", answer="test answer")
-
-    assert result.score == 0.8
-    assert result.reason == "Remote evaluation completed"
-    assert result.mode == "default"
-    assert result.criteria_scores == {}