evalring 0.2.0__py3-none-any.whl

EvalRing/evaluator/llm_judge/judge.py

@@ -0,0 +1,138 @@
+"""
+LLM Judge backends.
+
+An LLMJudge wraps an LLM client and exposes a simple `generate(prompt) -> str`
+interface. The evaluator/metric layer calls the judge to get raw outputs,
+which are then parsed via the schema module.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from ...config import resolve_credentials, resolve_model_name
+
+
+class LLMJudge(ABC):
+    """
+    Abstract base class for an LLM judge backend.
+
+    Subclass this to integrate any LLM provider (OpenAI, Anthropic,
+    local models, etc.) as a judge.
+    """
+
+    def __init__(self, model_name: str, **kwargs):
+        self.model_name = model_name
+        self._config = kwargs
+
+    @abstractmethod
+    def generate(self, prompt: str, **kwargs) -> str:
+        """
+        Generate a text response from the judge LLM.
+
+        Args:
+            prompt: The fully-rendered judge prompt.
+            **kwargs: Provider-specific overrides.
+
+        Returns:
+            Raw string output from the LLM.
+        """
+        pass
+
+    @abstractmethod
+    def get_model_name(self) -> str:
+        """Return the model identifier string."""
+        pass
+
+    def get_info(self) -> dict[str, Any]:
+        """Return metadata about this judge for logging."""
+        return {
+            "model_name": self.model_name,
+            "type": self.__class__.__name__,
+            "config": {k: v for k, v in self._config.items() if k != "api_key"},
+        }
+
+
+class OpenAIJudge(LLMJudge):
+    """
+    LLM judge backed by any OpenAI-compatible Chat Completions endpoint.
+
+    Args:
+        model_name: Judge model identifier. Defaults to ``$EVALRING_MODEL``,
+            then ``"gpt-4o"``.
+        api_key: API key. Resolved from the environment when omitted; see
+            :func:`EvalRing.config.resolve_credentials`.
+        base_url: OpenAI-compatible endpoint. Resolved alongside the API key
+            when omitted; ``None`` means the OpenAI default.
+        temperature: Sampling temperature. Default 0.0 for deterministic judging.
+        max_completion_tokens: Max tokens in judge response. Default 512.
+        **kwargs: Additional parameters forwarded to the API call.
+    """
+
+    def __init__(
+        self,
+        model_name: str | None = None,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        temperature: float = 0.0,
+        max_completion_tokens: int = 512,
+        **kwargs,
+    ):
+        resolved_model = resolve_model_name(model_name, default="gpt-4o") or "gpt-4o"
+        super().__init__(resolved_model, **kwargs)
+        credentials = resolve_credentials(api_key, base_url)
+        self.credentials = credentials
+        self.api_key = credentials.api_key
+        self.base_url = credentials.base_url
+        self.temperature = temperature
+        self.max_completion_tokens = max_completion_tokens
+        self._client: Any = None
+
+    def _ensure_client(self):
+        """Lazily initialize the OpenAI-compatible client."""
+        if self._client is None:
+            try:
+                from openai import OpenAI
+            except ImportError as exc:
+                raise ImportError(
+                    "OpenAIJudge requires the 'openai' package. "
+                    "Install it with: pip install evalring[llm]"
+                ) from exc
+            client_kwargs: dict[str, Any] = {"api_key": self.credentials.require_key()}
+            if self.base_url:
+                client_kwargs["base_url"] = self.base_url
+            self._client = OpenAI(**client_kwargs)
+
+    def generate(self, prompt: str, **kwargs) -> str:
+        """
+        Call the OpenAI Chat Completions API.
+
+        Args:
+            prompt: The judge prompt (sent as a user message).
+            **kwargs: Overrides for temperature, max_completion_tokens, etc.
+
+        Returns:
+            The model's response text.
+        """
+        self._ensure_client()
+
+        temperature = kwargs.pop("temperature", self.temperature)
+        max_tokens = kwargs.pop("max_completion_tokens", self.max_completion_tokens)
+
+        response = self._client.chat.completions.create(
+            model=self.model_name,
+            messages=[
+                {
+                    "role": "system",
+                    "content": "You are an expert evaluation judge. Always respond with valid JSON only.",
+                },
+                {"role": "user", "content": prompt},
+            ],
+            temperature=temperature,
+            max_completion_tokens=max_tokens,
+            **kwargs,
+        )
+
+        return response.choices[0].message.content.strip()
+
+    def get_model_name(self) -> str:
+        return self.model_name

EvalRing/evaluator/llm_judge/metric.py

@@ -0,0 +1,194 @@
+"""
+JudgeMetric: The core evaluation unit in the LLM-as-a-Judge pipeline.
+
+A JudgeMetric defines WHAT to measure. It follows the GEval two-phase pattern:
+  1. (Optional) Auto-generate evaluation steps from a criteria string
+  2. Score a test case using those steps + an optional rubric
+
+Multiple JudgeMetrics can be composed to produce a multi-dimensional
+evaluation (e.g., correctness + reasoning quality + safety).
+"""
+
+import logging
+from typing import Any
+
+from .judge import LLMJudge
+from .rubric import Rubric, ScoringCriteria
+from .schema import JudgeVerdict, parse_eval_steps, parse_verdict
+from .template import JudgeTemplate
+
+logger = logging.getLogger(__name__)
+
+
+class JudgeMetric:
+    """
+    A single LLM-judged evaluation metric.
+
+    Follows the criteria → steps → score pipeline:
+    - User provides a ScoringCriteria (name, description, optional steps/rubric)
+    - If no steps provided, the judge LLM generates them (cached)
+    - For each test case, the steps + rubric are sent to the judge for scoring
+
+    Args:
+        criteria: A ScoringCriteria instance defining what to measure.
+        judge: The LLMJudge backend to use for evaluation.
+        parameters: Which test case fields the judge sees
+                    (e.g., ["input", "output", "ground_truth"]).
+        template: JudgeTemplate class (or subclass) for prompt rendering.
+        strict_mode: If True, scoring is binary (0 or 1).
+        threshold: Score >= threshold is considered a "pass". Default 0.5 (on 0-1 scale).
+    """
+
+    def __init__(
+        self,
+        criteria: ScoringCriteria,
+        judge: LLMJudge,
+        parameters: list[str] | None = None,
+        template: type[JudgeTemplate] = JudgeTemplate,
+        strict_mode: bool = False,
+        threshold: float = 0.5,
+    ):
+        self.criteria = criteria
+        self.judge = judge
+        self.parameters = parameters or ["input", "output", "ground_truth"]
+        self.template = template
+        self.strict_mode = strict_mode
+        self.threshold = threshold
+
+        # Cached evaluation steps (generated once, reused for all samples)
+        self._evaluation_steps: list[str] | None = None
+
+    @property
+    def name(self) -> str:
+        return self.criteria.name
+
+    @property
+    def weight(self) -> float:
+        return self.criteria.weight
+
+    @property
+    def evaluation_steps(self) -> list[str]:
+        """Get evaluation steps, generating them if needed."""
+        if self._evaluation_steps is None:
+            if self.criteria.evaluation_steps:
+                self._evaluation_steps = self.criteria.evaluation_steps
+            else:
+                self._generate_steps()
+        if self._evaluation_steps is None:
+            raise RuntimeError(
+                f"[{self.name}] Judge returned no evaluation steps for the criteria."
+            )
+        return self._evaluation_steps
+
+    def _generate_steps(self) -> None:
+        """
+        Phase 1: Ask the judge LLM to generate evaluation steps from the criteria.
+        These steps are cached and reused for all subsequent scorings.
+        """
+        logger.info(f"[{self.name}] Generating evaluation steps from criteria...")
+
+        prompt = self.template.generate_evaluation_steps(
+            criteria=self.criteria.criteria, parameters=self.parameters
+        )
+
+        raw_output = self.judge.generate(prompt)
+        eval_steps = parse_eval_steps(raw_output)
+        self._evaluation_steps = eval_steps.steps
+
+        logger.info(f"[{self.name}] Generated {len(self._evaluation_steps)} evaluation steps")
+
+    def score(self, test_case: dict[str, str]) -> JudgeVerdict:
+        """
+        Phase 2: Score a single test case using the evaluation steps.
+
+        Args:
+            test_case: Dict mapping parameter names to values.
+                       e.g., {"input": "...", "output": "...", "ground_truth": "..."}
+
+        Returns:
+            JudgeVerdict with score, reason, and metadata.
+        """
+        steps = self.evaluation_steps  # triggers generation if needed
+
+        # Build the scoring prompt
+        if self.strict_mode:
+            prompt = self.template.generate_strict_score(
+                evaluation_steps=steps,
+                test_case=test_case,
+                parameters=self.parameters,
+            )
+            max_score = 1
+        else:
+            score_range = self.criteria.rubric_score_range
+            rubric_text = self.criteria.rubric_text
+
+            prompt = self.template.generate_score(
+                evaluation_steps=steps,
+                test_case=test_case,
+                parameters=self.parameters,
+                rubric_text=rubric_text,
+                score_range=score_range,
+            )
+            max_score = score_range[1]
+
+        # Call the judge
+        raw_output = self.judge.generate(prompt)
+        verdict = parse_verdict(raw_output, max_score=max_score)
+
+        # Add metric metadata
+        verdict.metadata["metric_name"] = self.name
+        verdict.metadata["strict_mode"] = self.strict_mode
+        verdict.metadata["max_score"] = max_score
+
+        return verdict
+
+    def score_simple(
+        self,
+        input_text: str,
+        output: str,
+        ground_truth: str = "",
+    ) -> JudgeVerdict:
+        """
+        Score a single case without building a dict manually.
+
+        Convenience wrapper around :meth:`score` that constructs the
+        test-case dict for you.
+
+        Args:
+            input_text: The original input.
+            output: The agent's output / prediction.
+            ground_truth: The expected correct answer (optional).
+
+        Returns:
+            JudgeVerdict with score, reason, and metadata.
+        """
+        test_case = {
+            "input": input_text,
+            "output": output,
+            "ground_truth": ground_truth,
+        }
+        return self.score(test_case)
+
+    def is_successful(self, verdict: JudgeVerdict) -> bool:
+        """Check if a verdict passes the threshold."""
+        max_score = verdict.metadata.get("max_score", 10)
+        normalized = verdict.score / max_score if max_score > 0 else 0
+        return normalized >= self.threshold
+
+    def get_info(self) -> dict[str, Any]:
+        """Return metric configuration for logging/reproducibility."""
+        return {
+            "name": self.name,
+            "criteria": self.criteria.criteria,
+            "evaluation_steps": self._evaluation_steps,
+            "rubric": (
+                self.criteria.rubric.to_dict()
+                if isinstance(self.criteria.rubric, Rubric)
+                else self.criteria.rubric
+            ),
+            "parameters": self.parameters,
+            "strict_mode": self.strict_mode,
+            "threshold": self.threshold,
+            "weight": self.weight,
+            "judge_model": self.judge.get_model_name(),
+        }

EvalRing/evaluator/llm_judge/rubric.py

@@ -0,0 +1,160 @@
+"""
+Rubric definitions for LLM-as-a-Judge scoring.
+
+A Rubric anchors the judge's scoring by defining discrete grade bands,
+each with a score range and a description of what that score means.
+"""
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class RubricLevel:
+    """
+    A single level/band within a rubric.
+
+    Examples:
+        RubricLevel(score=5, label="Excellent", description="Perfectly matches ground truth...")
+        RubricLevel(score=1, label="Poor", description="Completely incorrect or irrelevant...")
+    """
+
+    score: int
+    label: str
+    description: str
+
+    def to_prompt_string(self) -> str:
+        return f"Score {self.score} ({self.label}): {self.description}"
+
+
+@dataclass
+class Rubric:
+    """
+    A complete scoring rubric composed of ordered levels.
+
+    The rubric is injected into the judge prompt to provide scoring anchors,
+    improving calibration and consistency across evaluations.
+
+    Args:
+        name: Descriptive name for this rubric (e.g., "classification_accuracy")
+        levels: Ordered list of RubricLevel defining the scoring scale
+        score_range: (min, max) score bounds. Auto-derived from levels if not set.
+        description: Optional high-level description of what this rubric measures.
+    """
+
+    name: str
+    levels: list[RubricLevel]
+    score_range: tuple[int, int] | None = None
+    description: str = ""
+
+    def __post_init__(self):
+        if not self.levels:
+            raise ValueError("Rubric must have at least one level")
+        # Sort levels by score
+        self.levels = sorted(self.levels, key=lambda level: level.score)
+        # Auto-derive score range
+        if self.score_range is None:
+            self.score_range = (self.levels[0].score, self.levels[-1].score)
+
+    @property
+    def _range(self) -> tuple[int, int]:
+        """Score range, always populated once ``__post_init__`` has run."""
+        assert self.score_range is not None
+        return self.score_range
+
+    def to_prompt_string(self) -> str:
+        """Render the rubric as a string suitable for injection into a judge prompt."""
+        lines = []
+        if self.description:
+            lines.append(f"Rubric: {self.description}")
+        lines.append(f"Score range: {self._range[0]} to {self._range[1]}")
+        lines.append("")
+        for level in self.levels:
+            lines.append(level.to_prompt_string())
+        return "\n".join(lines)
+
+    @property
+    def max_score(self) -> int:
+        return self._range[1]
+
+    @property
+    def min_score(self) -> int:
+        return self._range[0]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "description": self.description,
+            "score_range": list(self._range),
+            "levels": [
+                {"score": level.score, "label": level.label, "description": level.description}
+                for level in self.levels
+            ],
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "Rubric":
+        levels = [RubricLevel(**level) for level in data["levels"]]
+        return cls(
+            name=data["name"],
+            levels=levels,
+            score_range=tuple(data["score_range"]) if "score_range" in data else None,
+            description=data.get("description", ""),
+        )
+
+
+@dataclass
+class ScoringCriteria:
+    """
+    A named evaluation criterion with an optional rubric.
+
+    This is the user-facing config object: you provide a criteria description
+    (and optionally explicit evaluation_steps), and the framework either uses
+    your steps or asks the judge LLM to generate them.
+
+    Args:
+        name: Short name for the criterion (e.g., "correctness", "relevance")
+        criteria: Natural language description of what this criterion measures.
+        evaluation_steps: Optional pre-defined steps. If None, the judge LLM
+                          will auto-generate them from the criteria string.
+        rubric: Optional Rubric to anchor scoring. If None, a default 0-10
+                scale is used.
+        weight: Weight for this criterion when combining multiple metrics.
+                Defaults to 1.0.
+    """
+
+    name: str
+    criteria: str
+    evaluation_steps: list[str] | None = None
+    rubric: Rubric | str | None = None
+    weight: float = 1.0
+
+    @property
+    def rubric_text(self) -> str | None:
+        """Get the rubric as a prompt-ready string, regardless of input type."""
+        if self.rubric is None:
+            return None
+        if isinstance(self.rubric, str):
+            return self.rubric
+        return self.rubric.to_prompt_string()
+
+    @property
+    def rubric_score_range(self) -> tuple[int, int]:
+        """Get the score range from the rubric, defaulting to (0, 10)."""
+        if isinstance(self.rubric, Rubric):
+            return (self.rubric.min_score, self.rubric.max_score)
+        return (0, 10)
+
+    def to_dict(self) -> dict[str, Any]:
+        rubric_val: dict[str, Any] | str | None = None
+        if isinstance(self.rubric, Rubric):
+            rubric_val = self.rubric.to_dict()
+        elif isinstance(self.rubric, str):
+            rubric_val = self.rubric
+        return {
+            "name": self.name,
+            "criteria": self.criteria,
+            "evaluation_steps": self.evaluation_steps,
+            "rubric": rubric_val,
+            "weight": self.weight,
+        }

EvalRing/evaluator/llm_judge/schema.py

@@ -0,0 +1,147 @@
+"""
+Response schemas for structured LLM judge output.
+
+These define the expected JSON shapes returned by the judge LLM.
+We use simple dataclass parsing (no pydantic dependency) with
+a robust JSON extraction fallback.
+"""
+
+import json
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class JudgeVerdict:
+    """
+    The judge's verdict for a single evaluation.
+
+    Attributes:
+        score: Numeric score (within the rubric's range, e.g., 0-10)
+        reason: The judge's chain-of-thought reasoning for the score.
+        metadata: Any additional data returned by the judge.
+    """
+
+    score: float
+    reason: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def normalized_score(self) -> float:
+        """Score normalized to 0-1 range (assuming 0-10 raw scale)."""
+        return self.score / 10.0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"score": self.score, "reason": self.reason, "metadata": self.metadata}
+
+
+@dataclass
+class EvalSteps:
+    """
+    Auto-generated evaluation steps from a criteria string.
+
+    Attributes:
+        steps: Ordered list of evaluation step descriptions.
+    """
+
+    steps: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"steps": self.steps}
+
+
+def extract_json(text: str) -> dict[str, Any]:
+    """
+    Robustly extract the first JSON object from LLM output.
+
+    Handles:
+    - Clean JSON responses
+    - JSON wrapped in markdown code blocks (```json ... ```)
+    - JSON mixed with other text
+    - Trailing commas (common LLM error)
+
+    Args:
+        text: Raw LLM output string.
+
+    Returns:
+        Parsed dictionary.
+
+    Raises:
+        ValueError: If no valid JSON object can be extracted.
+    """
+    # Strip markdown code fences
+    text = re.sub(r"```(?:json)?\s*", "", text)
+    text = re.sub(r"```\s*$", "", text)
+    text = text.strip()
+
+    # Try direct parse first
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        pass
+
+    # Find the first { ... } block
+    brace_depth = 0
+    start = None
+    for i, ch in enumerate(text):
+        if ch == "{":
+            if brace_depth == 0:
+                start = i
+            brace_depth += 1
+        elif ch == "}":
+            brace_depth -= 1
+            if brace_depth == 0 and start is not None:
+                candidate = text[start : i + 1]
+                # Fix trailing commas before } or ]
+                candidate = re.sub(r",\s*([}\]])", r"\1", candidate)
+                try:
+                    return json.loads(candidate)
+                except json.JSONDecodeError:
+                    start = None
+                    continue
+
+    raise ValueError(f"Could not extract valid JSON from LLM output: {text[:200]}...")
+
+
+def parse_verdict(raw_output: str, max_score: int = 10) -> JudgeVerdict:
+    """
+    Parse a JudgeVerdict from raw LLM judge output.
+
+    Expected JSON format:
+        {"score": <int>, "reason": "<string>"}
+
+    Args:
+        raw_output: Raw string from the judge LLM.
+        max_score: Maximum valid score (for clamping).
+
+    Returns:
+        Parsed JudgeVerdict instance.
+    """
+    data = extract_json(raw_output)
+
+    score = float(data.get("score", 0))
+    score = max(0, min(score, max_score))  # clamp
+
+    reason = str(data.get("reason", "No reason provided."))
+
+    # Capture any extra fields as metadata
+    metadata = {k: v for k, v in data.items() if k not in ("score", "reason")}
+
+    return JudgeVerdict(score=score, reason=reason, metadata=metadata)
+
+
+def parse_eval_steps(raw_output: str) -> EvalSteps:
+    """
+    Parse auto-generated evaluation steps from LLM output.
+
+    Expected JSON format:
+        {"steps": ["step 1", "step 2", ...]}
+    """
+    data = extract_json(raw_output)
+    steps = data.get("steps", [])
+    if isinstance(steps, list):
+        steps = [str(s) for s in steps]
+    else:
+        steps = [str(steps)]
+    return EvalSteps(steps=steps)