evalring 0.2.0__py3-none-any.whl

EvalRing/__init__.py

@@ -0,0 +1,106 @@
+"""
+EvalRing: a unified evaluation framework for agents across LLMs and agent versions.
+
+The public API is re-exported here. Three abstractions carry the framework:
+
+- a :class:`~EvalRing.dataset.base.BaseDataset` supplies
+  :class:`~EvalRing.dataset.base.DataSample` records,
+- a :class:`~EvalRing.agent.base.BaseAgent` turns each sample into an
+  :class:`~EvalRing.agent.base.AgentResponse`,
+- a :class:`~EvalRing.evaluator.base.BaseEvaluator` scores the responses and
+  returns an :class:`~EvalRing.evaluator.base.EvaluationResult`.
+
+Provider credentials are resolved centrally by
+:func:`EvalRing.config.resolve_credentials`; see ``docs/CONFIGURATION.md``.
+"""
+
+__version__ = "0.2.0"
+
+from .agent import (
+    AgentResponse,
+    BaseAgent,
+    ClassificationPrediction,
+    ErrorClass,
+    MockAgent,
+    MultiRoleHostOrchestrator,
+    OpenAIAgent,
+    RoleConfig,
+    RuleBasedAgent,
+    classify_error,
+    resolve_classification_prediction,
+)
+from .config import (
+    MissingCredentialsError,
+    ProviderCredentials,
+    resolve_credentials,
+    resolve_model_name,
+)
+from .dataset import BaseDataset, CSVDataset, DataFrameDataset, DataSample, JSONDataset
+from .evaluator import (
+    BaseEvaluator,
+    ClassificationEvaluator,
+    EvalSteps,
+    EvaluationMetrics,
+    EvaluationResult,
+    JudgeMetric,
+    JudgeTemplate,
+    JudgeVerdict,
+    LLMJudge,
+    LLMJudgeEvaluator,
+    OpenAIJudge,
+    Rubric,
+    RubricLevel,
+    ScoringCriteria,
+)
+from .logging_utils import configure_logging, get_logger
+from .utils import GlobalCache, generate_model_list, generate_suite_visuals, run_suite
+
+__all__ = [
+    "__version__",
+    # Configuration
+    "MissingCredentialsError",
+    "ProviderCredentials",
+    "resolve_credentials",
+    "resolve_model_name",
+    "configure_logging",
+    "get_logger",
+    # Datasets
+    "BaseDataset",
+    "DataSample",
+    "JSONDataset",
+    "CSVDataset",
+    "DataFrameDataset",
+    # Agents
+    "BaseAgent",
+    "AgentResponse",
+    "ClassificationPrediction",
+    "MockAgent",
+    "RuleBasedAgent",
+    "OpenAIAgent",
+    "MultiRoleHostOrchestrator",
+    "RoleConfig",
+    "ErrorClass",
+    "classify_error",
+    "resolve_classification_prediction",
+    # Evaluators
+    "BaseEvaluator",
+    "EvaluationResult",
+    "EvaluationMetrics",
+    "ClassificationEvaluator",
+    # LLM-as-a-judge
+    "Rubric",
+    "RubricLevel",
+    "ScoringCriteria",
+    "JudgeVerdict",
+    "EvalSteps",
+    "JudgeTemplate",
+    "JudgeMetric",
+    "LLMJudge",
+    "OpenAIJudge",
+    "LLMJudgeEvaluator",
+    # Suite tooling
+    "GlobalCache",
+    "generate_model_list",
+    "generate_suite_visuals",
+    "run_suite",
+]

EvalRing/agent/__init__.py

@@ -0,0 +1,33 @@
+"""
+Agent module for EvalRing.
+"""
+
+from .base import AgentResponse, BaseAgent
+from .classification import (
+    ClassificationPrediction,
+    aggregate_base_vs_rest_probabilities,
+    normalize_probability_distribution,
+    parse_json_object,
+    resolve_classification_prediction,
+)
+from .errors import ErrorClass, classify_error, format_exception
+from .implementations import MockAgent, OpenAIAgent, RuleBasedAgent
+from .multi_role import MultiRoleHostOrchestrator, RoleConfig
+
+__all__ = [
+    "BaseAgent",
+    "AgentResponse",
+    "ClassificationPrediction",
+    "aggregate_base_vs_rest_probabilities",
+    "normalize_probability_distribution",
+    "parse_json_object",
+    "resolve_classification_prediction",
+    "MultiRoleHostOrchestrator",
+    "RoleConfig",
+    "MockAgent",
+    "RuleBasedAgent",
+    "OpenAIAgent",
+    "ErrorClass",
+    "classify_error",
+    "format_exception",
+]

EvalRing/agent/base.py

@@ -0,0 +1,122 @@
+"""
+Base classes for agents in the EvalRing framework.
+"""
+
+import json
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class AgentResponse:
+    """Represents an agent's response to an input."""
+
+    input_id: str
+    input_text: str
+    output: Any
+    confidence: float | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    processing_time: float | None = None
+    error: str | None = None
+
+    def is_successful(self) -> bool:
+        """Check if the response was successful."""
+        return self.error is None
+
+
+class BaseAgent(ABC):
+    """
+    Abstract base class for all agents in EvalRing.
+
+    This class provides a standardized interface for different types of agents
+    (LLM-based, rule-based, ensemble, etc.) to ensure compatibility with the
+    evaluation framework.
+    """
+
+    def __init__(self, name: str, version: str = "1.0", description: str | None = None, **kwargs):
+        self.name = name
+        self.version = version
+        self.description = description or f"Agent: {name}"
+        self._metadata = kwargs
+        self._is_initialized = False
+
+    @abstractmethod
+    def initialize(self, **kwargs) -> None:
+        """
+        Initialize the agent with required resources.
+
+        This method should be called before making predictions.
+        """
+        pass
+
+    @abstractmethod
+    def predict(self, input_text: str, **kwargs) -> AgentResponse:
+        """
+        Make a prediction for a single input.
+
+        Args:
+            input_text: The input text to process
+            **kwargs: Additional parameters for prediction
+
+        Returns:
+            AgentResponse containing the prediction and metadata
+        """
+        pass
+
+    def predict_batch(self, inputs: list[str], **kwargs) -> list[AgentResponse]:
+        """
+        Make predictions for multiple inputs.
+
+        Default implementation processes inputs sequentially.
+        Override this method for batch processing optimization.
+
+        Args:
+            inputs: List of input texts to process
+            **kwargs: Additional parameters for prediction
+
+        Returns:
+            List of AgentResponse objects
+        """
+        responses = []
+        for i, input_text in enumerate(inputs):
+            response = self.predict(input_text, **kwargs)
+            response.input_id = str(i)
+            responses.append(response)
+        return responses
+
+    def validate_input(self, input_text: str) -> bool:
+        """
+        Validate input before processing.
+
+        Args:
+            input_text: Input text to validate
+
+        Returns:
+            True if input is valid, False otherwise
+        """
+        return isinstance(input_text, str) and len(input_text.strip()) > 0
+
+    def get_info(self) -> dict[str, Any]:
+        """Get agent information."""
+        return {
+            "name": self.name,
+            "version": self.version,
+            "description": self.description,
+            "type": self.__class__.__name__,
+            "metadata": self._metadata,
+            "is_initialized": self._is_initialized,
+        }
+
+    def save_config(self, filepath: str | Path) -> None:
+        """Save agent configuration to file."""
+        config = self.get_info()
+        with open(filepath, "w") as f:
+            json.dump(config, f, indent=2)
+
+    def __str__(self) -> str:
+        return f"{self.__class__.__name__}(name='{self.name}', version='{self.version}')"
+
+    def __repr__(self) -> str:
+        return self.__str__()

EvalRing/agent/classification.py

@@ -0,0 +1,191 @@
+"""
+Reusable utilities for classification-style outputs.
+
+These helpers let agents return either:
+- a plain class label string, or
+- a structured mapping of class -> confidence score.
+
+The evaluator can then resolve the top class consistently.
+"""
+
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class ClassificationPrediction:
+    """Normalized prediction object used by evaluators."""
+
+    label: str | None
+    confidence: float | None = None
+    class_scores: dict[str, float] | None = None
+
+
+def parse_json_object(raw_text: str) -> dict[str, Any] | None:
+    """
+    Parse a JSON object from raw model text.
+
+    Supports either:
+    - exact JSON object text, or
+    - text that contains one object block.
+    """
+    text = (raw_text or "").strip()
+    if not text:
+        return None
+
+    try:
+        parsed = json.loads(text)
+        return parsed if isinstance(parsed, dict) else None
+    except Exception:
+        pass
+
+    start = text.find("{")
+    end = text.rfind("}")
+    if start == -1 or end == -1 or end <= start:
+        return None
+
+    try:
+        parsed = json.loads(text[start : end + 1])
+        return parsed if isinstance(parsed, dict) else None
+    except Exception:
+        return None
+
+
+def canonicalize_label(raw_label: Any, label_aliases: Mapping[str, str] | None = None) -> str:
+    """Map a raw label to its canonical form (case-insensitive aliases)."""
+    text = str(raw_label).strip()
+    if not text:
+        return ""
+
+    if not label_aliases:
+        return text
+
+    alias_map = {k.lower(): v for k, v in label_aliases.items()}
+    return alias_map.get(text.lower(), text)
+
+
+def normalize_class_scores(
+    output: Mapping[Any, Any],
+    *,
+    label_aliases: Mapping[str, str] | None = None,
+) -> dict[str, float]:
+    """Convert a raw mapping into a canonical class-score dict."""
+    merged: dict[str, float] = {}
+    for raw_label, raw_score in output.items():
+        try:
+            score = float(raw_score)
+        except (TypeError, ValueError):
+            continue
+
+        label = canonicalize_label(raw_label, label_aliases=label_aliases)
+        if not label:
+            continue
+
+        merged[label] = merged.get(label, 0.0) + score
+    return merged
+
+
+def normalize_probability_distribution(scores: Mapping[str, float]) -> dict[str, float]:
+    """Normalize non-negative class scores into a probability distribution."""
+    normalized: dict[str, float] = {}
+    for label, value in scores.items():
+        try:
+            v = float(value)
+        except (TypeError, ValueError):
+            continue
+        normalized[label] = max(0.0, v)
+
+    total = sum(normalized.values())
+    if total <= 0:
+        return dict.fromkeys(normalized, 0.0)
+
+    return {label: value / total for label, value in normalized.items()}
+
+
+def aggregate_base_vs_rest_probabilities(
+    *,
+    base_label: str,
+    target_vs_base_probs: Mapping[str, float],
+    all_labels: list[str] | None = None,
+    epsilon: float = 1e-6,
+) -> dict[str, float]:
+    """
+    Convert pairwise binary probabilities into a full multi-class distribution.
+
+    Expected input per target class is p(target | target vs base).
+    We convert each pairwise probability into odds r_t = p_t / (1-p_t), and use:
+        P(base) = 1 / (1 + sum_t r_t)
+        P(target_t) = r_t * P(base)
+
+    This makes a reusable bridge from base-vs-rest binary runs to multi-class output.
+    """
+    if not base_label:
+        raise ValueError("base_label must be a non-empty string")
+
+    ratios: dict[str, float] = {}
+    for label, probability in target_vs_base_probs.items():
+        if label == base_label:
+            continue
+
+        try:
+            p_target = float(probability)
+        except (TypeError, ValueError):
+            continue
+
+        p_target = min(max(p_target, epsilon), 1.0 - epsilon)
+        ratios[label] = p_target / (1.0 - p_target)
+
+    base_prob = 1.0 / (1.0 + sum(ratios.values()))
+    scores: dict[str, float] = {base_label: base_prob}
+    for label, ratio in ratios.items():
+        scores[label] = ratio * base_prob
+
+    if all_labels:
+        for label in all_labels:
+            scores.setdefault(label, 0.0)
+
+    return normalize_probability_distribution(scores)
+
+
+def resolve_classification_prediction(
+    output: Any,
+    *,
+    valid_labels: list[str] | None = None,
+    label_aliases: Mapping[str, str] | None = None,
+) -> ClassificationPrediction:
+    """
+    Resolve top-class prediction from string or class-score mapping output.
+
+    If output is a mapping, the highest-scoring class is selected.
+    Ties are resolved by valid_labels order (if provided), then alphabetically.
+    """
+    if isinstance(output, Mapping):
+        scores = normalize_class_scores(output, label_aliases=label_aliases)
+
+        if valid_labels:
+            valid_set = set(valid_labels)
+            scores = {k: v for k, v in scores.items() if k in valid_set}
+
+        if not scores:
+            return ClassificationPrediction(label=None, confidence=None, class_scores={})
+
+        order = {label: idx for idx, label in enumerate(valid_labels or [])}
+        best_label, best_score = min(
+            scores.items(),
+            key=lambda kv: (
+                -kv[1],
+                order.get(kv[0], 10**9),
+                kv[0].lower(),
+            ),
+        )
+        return ClassificationPrediction(
+            label=best_label, confidence=float(best_score), class_scores=scores
+        )
+
+    if output is None:
+        return ClassificationPrediction(label=None, confidence=None, class_scores=None)
+
+    label = canonicalize_label(str(output), label_aliases=label_aliases)
+    return ClassificationPrediction(label=label, confidence=None, class_scores=None)

EvalRing/agent/errors.py

@@ -0,0 +1,109 @@
+"""Reusable error formatting and classification for model-backed agents.
+
+Two concerns that every LLM task hits, factored out so tasks share one policy:
+
+- :func:`format_exception` turns a raised exception (often from LiteLLM / the
+  OpenAI SDK) into a *complete* diagnostic string — exception type, full
+  message, and useful provider attributes — instead of just the class name.
+- :func:`classify_error` labels an error message as rate-limited, transient
+  (worth patient retries), and/or terminal (retrying only wastes tokens) so
+  evaluators can share one retry policy.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+def format_exception(e: BaseException) -> str:
+    """Build a complete, diagnosable error string from an exception.
+
+    Captures the exception type and its full message (falling back to ``repr``
+    when the message is empty), plus common LiteLLM/OpenAI attributes
+    (``status_code``, ``llm_provider``, ``model``) when present.
+    """
+    detail = str(e).strip()
+    if not detail:
+        detail = repr(e)
+    parts = [f"{type(e).__name__}: {detail}"]
+    for attr in ("status_code", "llm_provider", "model"):
+        val = getattr(e, attr, None)
+        if val not in (None, ""):
+            parts.append(f"{attr}={val}")
+    return " | ".join(parts)
+
+
+# Substrings (matched against a lower-cased message) for each class.
+_RATE_LIMIT_MARKERS = ("429", "rate limit", "rate_limit", "too many requests")
+_TRANSIENT_MARKERS = (
+    "connection error",
+    "internalservererror",
+    "internal server error",
+    "overloaded",
+    "timeout",
+    "timed out",
+    "temporarily unavailable",
+    "service unavailable",
+    "bad gateway",
+    "gateway timeout",
+    " 500",
+    " 502",
+    " 503",
+    " 504",
+    "502 ",
+    "503 ",
+    "504 ",
+)
+_TERMINAL_MARKERS = (
+    "empty response",
+    "content filter",
+    "content_filter",
+    "invalid request",
+    "invalid_request",
+    "badrequest",
+    "bad request",
+    "context length",
+    "maximum context",
+    "unsupported",
+)
+
+
+@dataclass(frozen=True)
+class ErrorClass:
+    """Retry-relevant classification of an error message."""
+
+    is_rate_limit: bool
+    is_transient: bool
+    is_terminal: bool
+
+
+def classify_error(message: str) -> ErrorClass:
+    """Classify an error message for retry decisions.
+
+    Precedence: an **empty response** (or other terminal condition) is terminal
+    even if its message also mentions "timed out" — retrying such a case only
+    burns another full timeout window of tokens. Rate limits take precedence
+    over generic transient handling so callers can apply a dedicated backoff.
+    """
+    el = (message or "").lower()
+    is_rate_limit = any(m in el for m in _RATE_LIMIT_MARKERS)
+    is_transient = any(m in el for m in _TRANSIENT_MARKERS)
+    # Always-terminal conditions win over transient-looking text: these all
+    # mention "timeout"/"timed out" but retrying only wastes another full
+    # deadline — an empty response, a hard client timeout, and a soft
+    # (in-stream) timeout where the model was simply too slow for the limit.
+    forced_terminal = (
+        "empty response" in el
+        or "hard client timeout" in el
+        or "request timeout and was stopped" in el
+    )
+    is_terminal = forced_terminal or (
+        any(m in el for m in _TERMINAL_MARKERS) and not is_rate_limit and not is_transient
+    )
+    if is_terminal:
+        # A terminal error should never be treated as retryable.
+        is_transient = False
+        is_rate_limit = False
+    return ErrorClass(
+        is_rate_limit=is_rate_limit, is_transient=is_transient, is_terminal=is_terminal
+    )