evalrun 0.4.0__py3-none-any.whl

framework/core/contracts.py

@@ -0,0 +1,88 @@
+"""Domain-neutral core contracts and data models for evaluation."""
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Literal, Optional
+from framework.models import Benchmark
+
+
+@dataclass
+class Scenario:
+    """Domain-neutral evaluation scenario specification."""
+
+    id: str
+    name: str
+    domain: str  # e.g. "travel", "support_triage", "scheduling"
+    description: str
+    prompt: str
+    constraints: Dict[str, Any]
+    expected_behavior: List[str]
+    evaluation_criteria: Dict[str, List[str]]
+    pass_criteria: List[str]
+    failure_conditions: List[str]
+    notes: Optional[List[str]] = None
+    profile_name: str = "travel-agent"
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def benchmark_id(self) -> str:
+        """Backward-compatibility property for legacy codebase."""
+        return self.id
+
+    @property
+    def profile(self) -> str:
+        """Backward-compatibility property for legacy profile name."""
+        return self.profile_name
+
+
+def to_scenario(benchmark: Benchmark, domain: str = "travel") -> Scenario:
+    """Converts a legacy Benchmark instance into a generic Scenario instance."""
+    return Scenario(
+        id=benchmark.benchmark_id,
+        name=benchmark.name,
+        domain=domain,
+        description=benchmark.description,
+        prompt=benchmark.prompt,
+        constraints=benchmark.constraints,
+        expected_behavior=benchmark.expected_behavior,
+        evaluation_criteria=benchmark.evaluation_criteria,
+        pass_criteria=benchmark.pass_criteria,
+        failure_conditions=benchmark.failure_conditions,
+        notes=benchmark.notes,
+        profile_name=getattr(benchmark, "profile", "travel-agent"),
+    )
+
+
+def to_benchmark(scenario: Scenario) -> Benchmark:
+    """Converts a generic Scenario instance into a legacy Benchmark instance."""
+    return Benchmark(
+        benchmark_id=scenario.id,
+        name=scenario.name,
+        description=scenario.description,
+        prompt=scenario.prompt,
+        constraints=scenario.constraints,
+        expected_behavior=scenario.expected_behavior,
+        evaluation_criteria=scenario.evaluation_criteria,
+        pass_criteria=scenario.pass_criteria,
+        failure_conditions=scenario.failure_conditions,
+        notes=scenario.notes or [],
+        profile=scenario.profile_name,
+    )
+
+
+@dataclass
+class RunTrace:
+    """Execution trace tracking metadata, latency, token usage, and status for an evaluation run."""
+
+    trace_id: str
+    scenario_id: str
+    agent_id: str
+    model_name: str
+    started_at_utc: str
+    finished_at_utc: str
+    latency_seconds: float
+    status: Literal["success", "error", "timeout"]
+    error: Optional[str] = None
+    retries_attempted: int = 0
+    token_usage: Optional[Dict[str, int]] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)

framework/core/suite.py

@@ -0,0 +1,44 @@
+"""EvaluationSuite class managing versioned collections of Scenarios and EvaluationProfiles."""
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+from framework.core.contracts import Scenario
+from framework.models import EvaluationProfile
+
+
+@dataclass
+class EvaluationSuite:
+    """Encapsulates a collection of scenarios and domain evaluation profiles."""
+
+    suite_id: str
+    name: str
+    domain: str
+    version: str
+    scenarios: List[Scenario]
+    profiles: Dict[str, EvaluationProfile] = field(default_factory=dict)
+
+    def get_scenario(self, scenario_id: str) -> Optional[Scenario]:
+        """Looks up a scenario by ID."""
+        for scenario in self.scenarios:
+            if scenario.id == scenario_id:
+                return scenario
+        return None
+
+    def get_profile(self, profile_name: Optional[str] = None) -> EvaluationProfile:
+        """Resolves an evaluation profile by name or default key."""
+        if profile_name:
+            if profile_name in self.profiles:
+                return self.profiles[profile_name]
+            available = list(self.profiles.keys())
+            raise ValueError(
+                f"Profile '{profile_name}' not found in suite '{self.suite_id}'. "
+                f"Available profiles: {available}"
+            )
+        if "default" in self.profiles:
+            return self.profiles["default"]
+        available = list(self.profiles.keys())
+        raise ValueError(
+            f"No profile specified and no 'default' profile registered for suite '{self.suite_id}'. "
+            f"Available profiles: {available}"
+        )

framework/evaluation/__init__.py

@@ -0,0 +1,22 @@
+"""Package containing the evaluation engine, interfaces, and concrete evaluators."""
+
+from framework.evaluation.base import BaseEvaluator
+from framework.evaluation.engine import EvaluationEngine
+from framework.evaluation.runner import BenchmarkRunner
+from framework.llms import BaseLLM, MockLLM, OpenAILLM, GeminiLLM
+from framework.evaluation.evaluators import (
+    ConstraintEvaluator,
+    PlanningQualityEvaluator,
+    PersonalizationEvaluator,
+    AdaptabilityEvaluator,
+    InformationAccuracyEvaluator,
+)
+from framework.evaluation.evaluators.base_llm import BaseLLMEvaluator
+from framework.evaluation.testing import DummyEvaluator
+from framework.evaluation.dimensions import (
+    CONSTRAINT_SATISFACTION,
+    PLANNING_QUALITY,
+    INFORMATION_ACCURACY,
+    PERSONALIZATION,
+    ADAPTABILITY,
+)

framework/evaluation/base.py

@@ -0,0 +1,29 @@
+"""Base interface for all evaluation strategies."""
+
+from abc import ABC, abstractmethod
+from framework.models import AgentOutput, Benchmark, DimensionScore
+
+
+class BaseEvaluator(ABC):
+    """Abstract base class defining the interface for evaluation strategies.
+
+    Evaluators assess an agent's output against a benchmark scenario for a
+    single specific evaluation dimension.
+    """
+
+    @abstractmethod
+    def evaluate(
+        self,
+        benchmark: Benchmark,
+        output: AgentOutput,
+    ) -> DimensionScore:
+        """Evaluates a single dimension of an agent's output against a benchmark.
+
+        Args:
+            benchmark: The scenario benchmark details.
+            output: The response/data generated by the AI agent.
+
+        Returns:
+            A DimensionScore containing the dimension name, score, and reasoning.
+        """
+        pass

framework/evaluation/dimensions.py

@@ -0,0 +1,7 @@
+"""Standard evaluation dimension constants."""
+
+CONSTRAINT_SATISFACTION = "Constraint Satisfaction"
+PLANNING_QUALITY = "Planning Quality"
+INFORMATION_ACCURACY = "Information Accuracy"
+PERSONALIZATION = "Personalization"
+ADAPTABILITY = "Adaptability"

framework/evaluation/engine.py

@@ -0,0 +1,110 @@
+from langfuse import observe
+
+from typing import Dict
+from framework.exceptions import EvaluationError
+from framework.models import (
+    AgentOutput,
+    Benchmark,
+    DimensionScore,
+    EvaluationProfile,
+    EvaluationResult,
+)
+from framework.evaluation.base import BaseEvaluator
+
+
+class EvaluationEngine:
+    """Orchestrates the evaluation of an agent's output against a benchmark scenario.
+
+    Uses an EvaluationProfile to weigh scores across multiple dimensions and
+    calculates the final evaluation outcome.
+    """
+
+    def __init__(self, evaluators: Dict[str, BaseEvaluator]):
+        """Initializes the EvaluationEngine.
+
+        Args:
+            evaluators: A dictionary mapping dimension names to BaseEvaluator instances.
+        """
+        self.evaluators = evaluators
+
+    @observe(name="evaluation-engine")
+    def evaluate(
+        self,
+        benchmark: Benchmark,
+        output: AgentOutput,
+        profile: EvaluationProfile,
+    ) -> EvaluationResult:
+        """Runs the evaluation pipeline for the given benchmark and agent output.
+
+        Args:
+            benchmark: The benchmark scenario.
+            output: The response/data generated by the AI agent.
+            profile: The evaluation profile specifying dimension weights.
+
+        Returns:
+            An EvaluationResult containing the dimension breakdown and final status.
+
+        Raises:
+            EvaluationError: If a dimension specified in the profile has no registered evaluator,
+                is missing from the benchmark criteria, or returns an inconsistent dimension name.
+        """
+        if not profile.weights:
+            raise EvaluationError(
+                f"Evaluation profile '{profile.name}' contains no weights."
+            )
+
+        # Validate that all profile dimensions exist in the benchmark criteria
+        for dimension in profile.weights.keys():
+            if dimension not in benchmark.evaluation_criteria:
+                raise EvaluationError(
+                    f"Dimension '{dimension}' from profile '{profile.name}' "
+                    f"is not defined in benchmark '{benchmark.benchmark_id}' evaluation criteria."
+                )
+
+        dimension_scores = []
+        weighted_score_sum = 0.0
+        total_weight = 0.0
+
+        for dimension, weight in profile.weights.items():
+            evaluator = self.evaluators.get(dimension)
+            if not evaluator:
+                raise EvaluationError(
+                    f"No evaluator registered for dimension '{dimension}' "
+                    f"in profile '{profile.name}'."
+                )
+
+            try:
+                score = evaluator.evaluate(benchmark, output)
+            except Exception as e:
+                raise EvaluationError(
+                    f"Evaluator failed for dimension '{dimension}': {e}"
+                ) from e
+
+            # Safety check: Ensure the evaluator returned the correct dimension score
+            if score.dimension != dimension:
+                raise EvaluationError(
+                    f"Evaluator returned score for dimension '{score.dimension}' "
+                    f"instead of the expected dimension '{dimension}'."
+                )
+
+            dimension_scores.append(score)
+            weighted_score_sum += score.score * weight
+            total_weight += weight
+
+        if total_weight <= 0:
+            raise EvaluationError(
+                "Total weight of evaluation dimensions must be greater than zero."
+            )
+
+        overall_score = weighted_score_sum / total_weight
+        passed = overall_score >= profile.pass_threshold
+
+        eval_result = EvaluationResult(
+            benchmark_id=benchmark.benchmark_id,
+            benchmark_name=benchmark.name,
+            overall_score=overall_score,
+            dimension_scores=dimension_scores,
+            passed=passed,
+        )
+
+        return eval_result

framework/evaluation/evaluators/__init__.py

@@ -0,0 +1,7 @@
+"""Package containing concrete evaluation dimension strategies."""
+
+from framework.evaluation.evaluators.constraint import ConstraintEvaluator
+from framework.evaluation.evaluators.planning import PlanningQualityEvaluator
+from framework.evaluation.evaluators.personalization import PersonalizationEvaluator
+from framework.evaluation.evaluators.adaptability import AdaptabilityEvaluator
+from framework.evaluation.evaluators.information_accuracy import InformationAccuracyEvaluator

framework/evaluation/evaluators/adaptability.py

@@ -0,0 +1,27 @@
+"""Evaluator for adaptability quality using LLM judgment."""
+
+from typing import List
+from framework.models import AgentOutput, Benchmark
+from framework.evaluation.evaluators.base_llm import BaseLLMEvaluator
+from framework.evaluation.prompts import build_adaptability_prompt
+from framework.evaluation.dimensions import ADAPTABILITY
+from framework.llms import Message
+
+
+class AdaptabilityEvaluator(BaseLLMEvaluator):
+    """Evaluates the 'Adaptability' dimension of an agent's output.
+
+    Uses an LLM Judge to check how well the itinerary is modified to respond to changing
+    circumstances while keeping traveler preferences intact.
+    """
+
+    @property
+    def dimension(self) -> str:
+        return ADAPTABILITY
+
+    def build_prompt(
+        self,
+        benchmark: Benchmark,
+        output: AgentOutput,
+    ) -> List[Message]:
+        return build_adaptability_prompt(benchmark, output)

framework/evaluation/evaluators/base_llm.py

@@ -0,0 +1,104 @@
+"""Base class for all LLM-based evaluators."""
+
+from abc import abstractmethod
+import json
+from typing import Dict, List
+from framework.evaluation.base import BaseEvaluator
+from framework.exceptions import EvaluationError
+from framework.models import AgentOutput, Benchmark, DimensionScore
+from framework.llms import BaseLLM, Message
+from ...utils import parse_json_markdown
+
+
+class BaseLLMEvaluator(BaseEvaluator):
+    """Abstract base class implementing orchestration and parsing for LLM judges."""
+
+    def __init__(self, llm: BaseLLM):
+        """Initializes the BaseLLMEvaluator.
+
+        Args:
+            llm: An instance of a BaseLLM provider wrapper.
+        """
+        self.llm = llm
+
+    @property
+    @abstractmethod
+    def dimension(self) -> str:
+        """The dimension name this evaluator evaluates.
+
+        Returns:
+            A string containing the evaluation dimension name.
+        """
+        pass
+
+    @abstractmethod
+    def build_prompt(
+        self, benchmark: Benchmark, output: AgentOutput
+    ) -> List[Message]:
+        """Constructs the prompt messages for the LLM.
+
+        Args:
+            benchmark: The scenario benchmark details.
+            output: The response/data generated by the AI agent.
+
+        Returns:
+            A list of Message objects.
+        """
+        pass
+
+    def _parse_json_response(self, text: str) -> dict:
+        """Extracts and parses a JSON object from raw LLM output.
+
+        Handles wrapping blocks like ```json ... ``` robustly.
+
+        Args:
+            text: Raw text generated by the model.
+
+        Returns:
+            The parsed dictionary.
+
+        Raises:
+            ValueError: If parsing fails.
+        """
+        parsed = parse_json_markdown(text)
+        if not isinstance(parsed, dict):
+            raise ValueError(f"Expected JSON dictionary, got: {type(parsed)}")
+        return parsed
+
+    def evaluate(
+        self,
+        benchmark: Benchmark,
+        output: AgentOutput,
+    ) -> DimensionScore:
+        """Orchestrates LLM prompt execution, response parsing, and score wrapping.
+
+        Args:
+            benchmark: The scenario benchmark details.
+            output: The response/data generated by the AI agent.
+
+        Returns:
+            A DimensionScore containing the final rating.
+
+        Raises:
+            EvaluationError: If the model generation or JSON parsing fails.
+        """
+        messages = self.build_prompt(benchmark, output)
+        response_text = ""
+
+        try:
+            response = self.llm.generate(messages)
+            response_text = response.text
+            parsed = self._parse_json_response(response_text)
+            score = float(parsed["score"])
+            reason = str(parsed["reason"])
+        except Exception as e:
+            raise EvaluationError(
+                f"Failed to generate or parse LLM evaluation response: {e}. "
+                f"Raw response: {response_text}"
+            ) from e
+
+        return DimensionScore(
+            dimension=self.dimension,
+            score=score,
+            reason=reason,
+        )

framework/evaluation/evaluators/constraint.py

@@ -0,0 +1,27 @@
+"""Evaluator for constraint satisfaction using LLM judgment."""
+
+from typing import List
+from framework.models import AgentOutput, Benchmark
+from framework.evaluation.evaluators.base_llm import BaseLLMEvaluator
+from framework.evaluation.prompts import build_constraint_prompt
+from framework.evaluation.dimensions import CONSTRAINT_SATISFACTION
+from framework.llms import Message
+
+
+class ConstraintEvaluator(BaseLLMEvaluator):
+    """Evaluates the 'Constraint Satisfaction' dimension of an agent's output.
+
+    Uses an LLM Judge to determine if all constraints defined in the benchmark
+    scenario are successfully met.
+    """
+
+    @property
+    def dimension(self) -> str:
+        return CONSTRAINT_SATISFACTION
+
+    def build_prompt(
+        self,
+        benchmark: Benchmark,
+        output: AgentOutput,
+    ) -> List[Message]:
+        return build_constraint_prompt(benchmark, output)

framework/evaluation/evaluators/information_accuracy.py

@@ -0,0 +1,41 @@
+"""Evaluator for factual information accuracy using hybrid verifiers."""
+
+from typing import List
+from framework.evaluation.evaluators.base_llm import BaseLLMEvaluator
+from framework.evaluation.prompts.information_accuracy import build_information_accuracy_prompt
+from framework.evaluation.dimensions import INFORMATION_ACCURACY
+from framework.llms import BaseLLM, Message
+from framework.models import AgentOutput, Benchmark
+from framework.verification.pipeline import VerificationPipeline
+
+
+class InformationAccuracyEvaluator(BaseLLMEvaluator):
+    """Evaluates the 'Information Accuracy' dimension of an agent's output.
+
+    Uses an injected VerificationPipeline to check factual claims,
+    and formats them to get graded by the LLM Judge.
+    """
+
+    def __init__(self, llm: BaseLLM, pipeline: VerificationPipeline):
+        """Initializes the InformationAccuracyEvaluator.
+
+        Args:
+            llm: The LLM client for final evidence grading.
+            pipeline: The VerificationPipeline instance to verify claims.
+        """
+        super().__init__(llm)
+        self.pipeline = pipeline
+
+    @property
+    def dimension(self) -> str:
+        return INFORMATION_ACCURACY
+
+    def build_prompt(
+        self,
+        benchmark: Benchmark,
+        output: AgentOutput,
+    ) -> List[Message]:
+        # 1. Run the injected pipeline to extract and verify factual claims
+        report = self.pipeline.run(output)
+        # 2. Format findings context and return prompts list
+        return build_information_accuracy_prompt(benchmark, output, report)

framework/evaluation/evaluators/personalization.py

@@ -0,0 +1,27 @@
+"""Evaluator for personalization quality using LLM judgment."""
+
+from typing import List
+from framework.models import AgentOutput, Benchmark
+from framework.evaluation.evaluators.base_llm import BaseLLMEvaluator
+from framework.evaluation.prompts import build_personalization_prompt
+from framework.evaluation.dimensions import PERSONALIZATION
+from framework.llms import Message
+
+
+class PersonalizationEvaluator(BaseLLMEvaluator):
+    """Evaluates the 'Personalization' dimension of an agent's output.
+
+    Uses an LLM Judge to check if the itinerary is tailored to the traveler's stated
+    interests, hobbies, remote working requirements, and pacing preferences.
+    """
+
+    @property
+    def dimension(self) -> str:
+        return PERSONALIZATION
+
+    def build_prompt(
+        self,
+        benchmark: Benchmark,
+        output: AgentOutput,
+    ) -> List[Message]:
+        return build_personalization_prompt(benchmark, output)

framework/evaluation/evaluators/planning.py

@@ -0,0 +1,27 @@
+"""Evaluator for planning quality using LLM judgment."""
+
+from typing import List
+from framework.models import AgentOutput, Benchmark
+from framework.evaluation.evaluators.base_llm import BaseLLMEvaluator
+from framework.evaluation.prompts import build_planning_prompt
+from framework.evaluation.dimensions import PLANNING_QUALITY
+from framework.llms import Message
+
+
+class PlanningQualityEvaluator(BaseLLMEvaluator):
+    """Evaluates the 'Planning Quality' dimension of an agent's output.
+
+    Uses an LLM Judge to determine if the itinerary follows a logical,
+    efficient route, minimizes accommodation changes, and balances pacing.
+    """
+
+    @property
+    def dimension(self) -> str:
+        return PLANNING_QUALITY
+
+    def build_prompt(
+        self,
+        benchmark: Benchmark,
+        output: AgentOutput,
+    ) -> List[Message]:
+        return build_planning_prompt(benchmark, output)

framework/evaluation/evaluators/support.py

@@ -0,0 +1,81 @@
+"""Support-domain evaluator prompts.
+
+The core dimensions are shared with travel, but their rubrics are not.  Keeping
+these prompts separate prevents a support ticket from being judged as if it
+were an itinerary.
+"""
+
+from typing import List
+
+from framework.evaluation.dimensions import (
+    ADAPTABILITY,
+    INFORMATION_ACCURACY,
+    PERSONALIZATION,
+    PLANNING_QUALITY,
+)
+from framework.evaluation.evaluators.base_llm import BaseLLMEvaluator
+from framework.evaluation.prompts.base import build_llm_judge_prompt
+from framework.llms import Message
+from framework.models import AgentOutput, Benchmark
+from framework.verification.pipeline import VerificationPipeline
+
+
+class _SupportEvaluator(BaseLLMEvaluator):
+    rubric = ""
+
+    def build_prompt(self, benchmark: Benchmark, output: AgentOutput) -> List[Message]:
+        return build_llm_judge_prompt(
+            "You are an objective evaluator for a customer-support triage agent.",
+            self.rubric,
+            benchmark,
+            output,
+        )
+
+
+class SupportPlanningEvaluator(_SupportEvaluator):
+    dimension = PLANNING_QUALITY
+    rubric = (
+        "Evaluate whether the triage workflow is ordered and operationally useful. "
+        "Check acknowledgement, SLA preservation, escalation order, evidence gathering, "
+        "containment actions, and the next customer update. Do not apply travel or itinerary criteria."
+    )
+
+
+class SupportPersonalizationEvaluator(_SupportEvaluator):
+    dimension = PERSONALIZATION
+    rubric = (
+        "Evaluate whether the response is appropriately tailored to this enterprise customer, "
+        "the EU production impact, the imminent launch, and the requested support-owner role. "
+        "Assess empathy, clarity, and usefulness of the customer-facing message."
+    )
+
+
+class SupportAdaptabilityEvaluator(_SupportEvaluator):
+    dimension = ADAPTABILITY
+    rubric = (
+        "Evaluate how well the triage plan handles uncertainty and branching conditions. "
+        "Check the distinction between immediate Payments/Incident Commander escalation and "
+        "conditional EU platform escalation, while avoiding an unverified root-cause claim."
+    )
+
+
+class SupportInformationAccuracyEvaluator(BaseLLMEvaluator):
+    dimension = INFORMATION_ACCURACY
+
+    def __init__(self, llm, pipeline: VerificationPipeline):
+        super().__init__(llm)
+        self.pipeline = pipeline
+
+    def build_prompt(self, benchmark: Benchmark, output: AgentOutput) -> List[Message]:
+        report = self.pipeline.run(output)
+        return build_llm_judge_prompt(
+            "You are an objective evaluator checking factual accuracy in a customer-support triage response.",
+            (
+                "Check that observed incident facts, priority, timestamps, impact, and escalation "
+                "targets match the scenario. Separate facts from hypotheses and do not penalize "
+                "unknown claims merely because they are absent from an unrelated knowledge base.\n\n"
+                f"Verification evidence:\n{report}"
+            ),
+            benchmark,
+            output,
+        )

framework/evaluation/prompts/__init__.py

@@ -0,0 +1,11 @@
+"""Evaluation prompts package."""
+
+from framework.evaluation.prompts.base import (
+    JSON_RESPONSE_SCHEMA,
+    build_llm_judge_prompt,
+)
+from framework.evaluation.prompts.constraint import build_constraint_prompt
+from framework.evaluation.prompts.planning import build_planning_prompt
+from framework.evaluation.prompts.personalization import build_personalization_prompt
+from framework.evaluation.prompts.adaptability import build_adaptability_prompt
+from framework.evaluation.prompts.information_accuracy import build_information_accuracy_prompt