hud-python 0.2.10__py3-none-any.whl → 0.3.1__py3-none-any.whl

hud/evaluators/__init__.py

@@ -1,9 +0,0 @@
-"""
-Evaluators for assessing task responses.
-"""
-
-from __future__ import annotations
-
-from hud.evaluators.base import Evaluator
-
-__all__ = ["Evaluator"]

hud/evaluators/base.py

@@ -1,32 +0,0 @@
-from __future__ import annotations
-
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING
-
-from pydantic import BaseModel, Field
-
-if TYPE_CHECKING:
-    from hud.task import Task
-
-
-class EvaluationResult(BaseModel):
-    """Result of an evaluation.
-
-    Attributes:
-        score: Float score between 0 and 1
-        reason: Explanation of the evaluation
-        mode: Mode used for matching, if applicable
-    """
-
-    score: float
-    reason: str
-    mode: str | None = None
-    criteria_scores: dict[str, float] | None = Field(default_factory=dict)
-
-
-class Evaluator(ABC):
-    """Abstract base class for evaluators."""
-
-    @abstractmethod
-    def evaluate(self, task: Task, response: str) -> EvaluationResult:
-        """Evaluate a task and response."""

hud/evaluators/inspect.py

@@ -1,24 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-from hud.evaluators.base import EvaluationResult
-
-
-def inspect_evaluate(
-    response: Any,
-    answer: Any,
-) -> EvaluationResult:
-    """Evaluate using Inspect-ai's evaluation models.
-
-    Args:
-        response: The response to evaluate
-        answer: The reference answer to compare against
-        model_name: The Inspect model to use
-        prompt: Optional custom prompt for evaluation
-        metrics: Optional list of metrics to evaluate against
-
-    Returns:
-        EvaluationResult with the evaluation results
-    """
-    return EvaluationResult(score=0.0, reason="Inspect evaluation not implemented", mode="inspect")

hud/evaluators/judge.py

@@ -1,189 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import base64
-from typing import Any, Protocol, TypedDict
-
-from hud.evaluators.base import EvaluationResult
-from hud.server import make_request
-from hud.settings import settings
-
-
-class LLM(Protocol):
-    """Protocol for LLM interfaces that can be used for evaluation."""
-
-    async def ainvoke(self, prompt: str, /) -> str: ...
-
-
-class Criterion(TypedDict, total=False):
-    """Criterion for judge-based evaluation."""
-
-    description: str
-    weight: float
-
-
-async def _call_eval_endpoint(
-    response: Any, answer: Any, criteria: list[Any], mode: str
-) -> dict[str, Any]:
-    """Call the run_eval endpoint to evaluate the response."""
-    try:
-        result = await make_request(
-            method="POST",
-            url=f"{settings.base_url}/evaluations/run_eval",
-            json={"response": response, "answer": answer, "criteria": criteria, "mode": mode},
-            api_key=settings.api_key,
-        )
-        return result
-    except Exception as e:
-        # Fallback to local evaluation if remote call fails
-        return {
-            "score": -1.0,
-            "reason": f"Remote evaluation failed: {e!s}. Fallback to default score.",
-            "criteria_scores": {},
-        }
-
-
-def _process_input(data: Any) -> Any:
-    """Process input data, detecting and handling base64 images."""
-    if isinstance(data, bytes):
-        # Convert bytes to base64 string
-        return base64.b64encode(data).decode("utf-8")
-
-    if isinstance(data, str) and _is_base64_image(data):
-        # It's already a base64 string, just return it
-        return data
-
-    if isinstance(data, list) and all(isinstance(item, str) for item in data):
-        # Process list of strings
-        return data
-
-    # For other types, convert to string
-    return str(data) if not isinstance(data, str | dict) else data
-
-
-def _is_base64_image(data: Any) -> bool:
-    """Check if a string is a base64 encoded image."""
-    if not isinstance(data, str):
-        return False
-
-    # Check for common image data URI pattern
-    if data.startswith(("data:image/", "data:application/octet-stream")):
-        return True
-
-    # Check if it's a base64 encoded string with image header
-    try:
-        # First, validate it's base64 decodable
-        padding_needed = len(data) % 4
-        if padding_needed:
-            data += "=" * (4 - padding_needed)
-
-        # Try to decode the first few bytes to check for image signatures
-        sample = base64.b64decode(data[:30])
-
-        # Check for common image format signatures
-        return sample.startswith((b"\xff\xd8\xff", b"\x89PNG\r\n\x1a\n", b"GIF8", b"RIFF"))
-    except Exception:
-        return False
-
-
-def judge(
-    response: Any,
-    answer: Any,
-    llm: LLM | None = None,
-    criteria: list[str] | list[dict] | None = None,
-) -> EvaluationResult:
-    """Judge a response against an answer using an LLM.
-
-    Args:
-        response: The response to evaluate
-        answer: The reference answer to compare against
-        llm: Optional langchain LLM to use for evaluation
-        criteria: Evaluation criteria as strings or dictionaries
-
-    Returns:
-        EvaluationResult with evaluation results
-    """
-    # Process inputs
-    processed_response = _process_input(response)
-    processed_answer = _process_input(answer)
-
-    # If LLM is provided, use it for evaluation
-    if llm:
-        return _evaluate_with_llm(processed_response, processed_answer, llm, criteria)
-
-    # Otherwise, use the remote evaluation service
-    mode = "LLM"
-    if isinstance(answer, bytes) or _is_base64_image(answer):
-        mode = "VLM"
-
-    # Call the eval endpoint synchronously
-    result = asyncio.run(
-        _call_eval_endpoint(
-            response=processed_response, answer=processed_answer, criteria=criteria or [], mode=mode
-        )
-    )
-
-    return EvaluationResult(
-        score=result.get("score", -1.0),
-        reason=result.get("reason", "Response evaluated"),
-        mode=mode,
-        criteria_scores=result.get("criteria_scores", {}),
-    )
-
-
-def _evaluate_with_llm(
-    response: Any, answer: Any, llm: LLM, criteria: list[str] | list[dict] | None = None
-) -> EvaluationResult:
-    """Evaluate a response against an answer using a provided LLM."""
-    criteria_text = ""
-    if criteria:
-        criteria_text = "Use the following criteria:\n"
-        for c in criteria:
-            if isinstance(c, dict) and "description" in c:
-                criteria_text += f"- {c['description']}\n"
-            elif isinstance(c, str):
-                criteria_text += f"- {c}\n"
-
-    prompt = f"""Evaluate the quality of a response given a reference answer.
-
-REFERENCE ANSWER:
-{answer}
-
-RESPONSE TO EVALUATE:
-{response}
-
-{criteria_text}
-Rate the response on a scale from 0.0 to 1.0, where 1.0 is perfect.
-Provide a brief explanation for your rating.
-Format your answer as a JSON object with 'score' (float) and 'reason' (string) fields.
-"""
-
-    try:
-        # Run the evaluation asynchronously
-        result_text = asyncio.run(llm.ainvoke(prompt))
-
-        # Attempt to parse JSON response
-        import json
-        import re
-
-        # Try to extract JSON if wrapped in other text
-        json_match = re.search(r"\{.*?\}", result_text, re.DOTALL)
-        if json_match:
-            json_str = json_match.group(0)
-            result = json.loads(json_str)
-
-            return EvaluationResult(
-                score=float(result.get("score", 0.5)),
-                reason=result.get("reason", "Evaluated with custom LLM"),
-                mode="custom_llm",
-            )
-
-        # If can't parse as JSON, use default values
-        return EvaluationResult(
-            score=0.5,
-            reason=f"Unable to parse LLM response as JSON. Raw response: {result_text[:100]}...",
-            mode="custom_llm",
-        )
-
-    except Exception as e:
-        return EvaluationResult(score=0.0, reason=f"LLM evaluation error: {e!s}", mode="custom_llm")

hud/evaluators/match.py

@@ -1,156 +0,0 @@
-from __future__ import annotations
-
-import re
-from difflib import SequenceMatcher
-from typing import TYPE_CHECKING, Protocol
-
-from textdistance import levenshtein
-
-from hud.evaluators.base import EvaluationResult
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-
-class _Stringable(Protocol):
-    def __str__(self) -> str: ...
-
-
-def match_single(response: _Stringable, answer: _Stringable) -> EvaluationResult:
-    """Check if the answer is present within the response.
-
-    Args:
-        response: The response to evaluate
-        answer: The expected answer
-
-    Returns:
-        EvaluationResult with score=1.0 if match, 0.0 otherwise
-    """
-    passed = str(answer).lower().strip() in str(response).lower().strip()
-    return EvaluationResult(
-        score=1.0 if passed else 0.0,
-        reason="Exact match" if passed else "No exact match found",
-        mode="single",
-    )
-
-
-def match_all(response: _Stringable, answers: Sequence[_Stringable]) -> EvaluationResult:
-    """Count how many expected answers are in the response.
-
-    Args:
-        response: The response to evaluate
-        answers: List of expected answers
-
-    Returns:
-        EvaluationResult with score=proportion of matches (0.0-1.0)
-    """
-    response_str = str(response).lower()
-    matches = 0
-
-    for answer in answers:
-        if str(answer).lower() in response_str:
-            matches += 1
-
-    score = matches / len(answers) if answers else 0.0
-
-    if matches == len(answers):
-        reason = f"All {matches} expected items found"
-    else:
-        reason = f"Only {matches} of {len(answers)} expected items found"
-
-    return EvaluationResult(score=score, reason=reason, mode="all")
-
-
-def match_fuzzy(response: _Stringable, answer: _Stringable) -> EvaluationResult:
-    """Calculate similarity using Levenshtein distance.
-
-    Args:
-        response: The response to evaluate
-        answer: The expected answer
-
-    Returns:
-        EvaluationResult with score=similarity (0.0-1.0)
-    """
-    s1 = str(response).lower()
-    s2 = str(answer).lower()
-
-    if s1 == s2:
-        score = 1.0
-    elif len(s1) == 0 or len(s2) == 0:
-        score = 0.0
-    else:
-        # Use Levenshtein distance
-        distance = levenshtein.distance(s1, s2)
-        max_len = max(len(s1), len(s2))
-        score = 1.0 - (distance / max_len)
-
-    return EvaluationResult(
-        score=score, reason=f"Fuzzy match with {score:.1%} similarity", mode="fuzz"
-    )
-
-
-def match_regex(response: _Stringable, pattern: str) -> EvaluationResult:
-    """Check if response matches regex pattern.
-
-    Args:
-        response: The response to evaluate
-        pattern: Regular expression pattern to match
-
-    Returns:
-        EvaluationResult with score=1.0 if match, 0.0 otherwise
-    """
-    try:
-        regex = re.compile(pattern, re.DOTALL)
-        passed = bool(regex.search(str(response)))
-        return EvaluationResult(
-            score=1.0 if passed else 0.0,
-            reason="Regex pattern matched" if passed else "Regex pattern did not match",
-            mode="regex",
-        )
-    except re.error:
-        return EvaluationResult(score=0.0, reason="Invalid regex pattern", mode="regex")
-
-
-def match_diff(response: _Stringable, answer: _Stringable) -> EvaluationResult:
-    """Compare difference between response and answer.
-
-    Args:
-        response: The response to evaluate
-        answer: The expected answer
-
-    Returns:
-        EvaluationResult with score=similarity (0.0-1.0)
-    """
-    if isinstance(response, int | float) and isinstance(answer, int | float):
-        score = _match_numeric_diff(response, answer)
-        reason = f"Numeric difference: {abs(response - answer)}"
-    else:
-        score = _match_string_diff(response, answer)
-        reason = f"String difference with {score:.1%} similarity"
-
-    return EvaluationResult(score=score, reason=reason, mode="diff")
-
-
-def _match_string_diff(response: _Stringable, answer: _Stringable) -> float:
-    """Compare difference between response and answer strings."""
-    matcher = SequenceMatcher(None, str(response), str(answer))
-    return matcher.ratio()
-
-
-def _match_numeric_diff(response: float, answer: float) -> float:
-    """Calculate normalized difference between numeric values.
-
-    Returns a value between 0 and 1, where 1 means identical and 0 means maximum difference.
-    """
-    if response == answer:
-        return 1.0
-
-    # Simple absolute difference normalized to a 0-1 scale
-    diff = abs(response - answer)
-    max_val = max(abs(response), abs(answer))
-
-    if max_val == 0:
-        return 1.0  # Both are zero
-
-    # Normalize and invert so 1.0 means identical
-    return max(0.0, 1.0 - min(1.0, diff / max_val))

hud/evaluators/remote.py

@@ -1,65 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-from typing import Any
-
-from hud.evaluators.base import EvaluationResult
-from hud.server import make_request
-from hud.settings import settings
-
-
-async def _remote_eval_call(
-    response: Any, answer: Any, eval_type: str, config: dict[str, Any] | None = None
-) -> dict[str, Any]:
-    """Send an evaluation request to the remote server.
-
-    Args:
-        response: The response to evaluate
-        answer: The reference answer to compare against
-        eval_type: Type of evaluation (e.g., "match", "judge", "agent")
-        config: Optional configuration parameters
-
-    Returns:
-        Dictionary with evaluation results from the server
-    """
-    try:
-        result = await make_request(
-            method="POST",
-            url=f"{settings.base_url}/evaluations/evaluate",
-            json={
-                "response": response,
-                "answer": answer,
-                "type": eval_type,
-                "config": config or {},
-            },
-            api_key=settings.api_key,
-        )
-        return result
-    except Exception as e:
-        return {"score": -1.0, "reason": f"Remote evaluation failed: {e!s}", "details": {}}
-
-
-def remote_evaluate(
-    response: Any, answer: Any, eval_type: str = "default", config: dict[str, Any] | None = None
-) -> EvaluationResult:
-    """Evaluate a response using remote evaluation services.
-
-    Args:
-        response: The response to evaluate
-        answer: The reference answer to compare against
-        eval_type: Type of evaluation to perform
-        config: Optional configuration for the evaluation
-
-    Returns:
-        EvaluationResult containing the evaluation results
-    """
-    result = asyncio.run(
-        _remote_eval_call(response=response, answer=answer, eval_type=eval_type, config=config)
-    )
-
-    return EvaluationResult(
-        score=result.get("score", -1.0),
-        reason=result.get("reason", "Remote evaluation completed"),
-        mode=eval_type,
-        criteria_scores=result.get("details", {}),
-    )

hud/evaluators/tests/test_inspect.py

@@ -1,12 +0,0 @@
-from __future__ import annotations
-
-from hud.evaluators.inspect import inspect_evaluate
-
-
-def test_inspect_evaluate_basic():
-    """Test basic functionality of inspect_evaluate."""
-    result = inspect_evaluate("Test response", "Test answer")
-
-    assert result.score == 0.0
-    assert result.reason == "Inspect evaluation not implemented"
-    assert result.mode == "inspect"