evalrun 0.4.0__py3-none-any.whl

framework/evaluation/runner.py

@@ -0,0 +1,363 @@
+"""Benchmark runner for orchestrating end-to-end agent evaluation pipelines."""
+
+import inspect
+import json
+import os
+import sys
+import time
+import uuid
+from datetime import datetime, timezone
+from typing import Any, Dict
+from framework.core import RunTrace
+from framework.models import AgentOutput, Benchmark, EvaluationResult
+from framework.parser import parse_benchmark
+from framework.profiles import PROFILE_REGISTRY, TRAVEL_PROFILE
+from framework.evaluation.engine import EvaluationEngine
+from framework.evaluation.dimensions import (
+    CONSTRAINT_SATISFACTION,
+    PLANNING_QUALITY,
+    INFORMATION_ACCURACY,
+    PERSONALIZATION,
+    ADAPTABILITY,
+)
+from framework.evaluation.evaluators import (
+    ConstraintEvaluator,
+    PlanningQualityEvaluator,
+    InformationAccuracyEvaluator,
+    PersonalizationEvaluator,
+    AdaptabilityEvaluator,
+)
+from framework.evaluation.evaluators.support import (
+    SupportAdaptabilityEvaluator,
+    SupportInformationAccuracyEvaluator,
+    SupportPersonalizationEvaluator,
+    SupportPlanningEvaluator,
+)
+from framework.verification.extractor import ClaimExtractor
+from framework.verification.local import LocalKnowledgeBaseVerifier
+from framework.verification.pipeline import VerificationPipeline
+
+
+class BenchmarkRunner:
+    """Automates the entire evaluation pipeline for a given agent under test.
+
+    Orchestrates benchmark ingestion, profile lookup, agent execution, evaluation via LLM judges,
+    and report generation.
+    """
+
+    def __init__(
+        self,
+        agent,
+        judge_llm,
+        local_verifier_path: str = "ground_truth/japan_demo.json",
+        output_dir: str = "scratch",
+        auditor: Any = None,
+    ):
+        """Initializes the BenchmarkRunner.
+
+        Args:
+            agent: The agent under test (must expose a run(prompt: str) method returning AgentOutput).
+            judge_llm: The reference judge LLM client.
+            local_verifier_path: Path to the local database file for factual verification.
+            output_dir: Folder to store reports and itineraries.
+            auditor: Optional IndependentBudgetAuditor instance for v3 read-only gate checks.
+        """
+        self.agent = agent
+        self.judge_llm = judge_llm
+        self.local_verifier_path = local_verifier_path
+        self.output_dir = output_dir
+        self.auditor = auditor
+
+        # Initialize the verification pipeline
+        self.verifier = LocalKnowledgeBaseVerifier(local_verifier_path)
+        self.extractor = ClaimExtractor(judge_llm)
+        self.pipeline = VerificationPipeline(self.extractor, self.verifier)
+
+        # Initialize the evaluation engine
+        evaluators = {
+            CONSTRAINT_SATISFACTION: ConstraintEvaluator(judge_llm),
+            PLANNING_QUALITY: PlanningQualityEvaluator(judge_llm),
+            INFORMATION_ACCURACY: InformationAccuracyEvaluator(judge_llm, self.pipeline),
+            PERSONALIZATION: PersonalizationEvaluator(judge_llm),
+            ADAPTABILITY: AdaptabilityEvaluator(judge_llm),
+        }
+
+        # Merge dynamic evaluator plugins from registry
+        try:
+            from framework.profiles.registry import _DYNAMIC_EVALUATOR_REGISTRY
+            for dim_name, plugin_instance in _DYNAMIC_EVALUATOR_REGISTRY.items():
+                evaluators[dim_name] = plugin_instance
+        except Exception:
+            pass
+
+        self.engine = EvaluationEngine(evaluators=evaluators)
+
+    def _configure_profile_evaluators(self, profile) -> None:
+        """Apply domain-specific rubrics after the scenario profile is known."""
+        if getattr(profile, "name", "").lower() != "support-triage":
+            return
+        self.engine.evaluators.update(
+            {
+                PLANNING_QUALITY: SupportPlanningEvaluator(self.judge_llm),
+                INFORMATION_ACCURACY: SupportInformationAccuracyEvaluator(
+                    self.judge_llm, self.pipeline
+                ),
+                PERSONALIZATION: SupportPersonalizationEvaluator(self.judge_llm),
+                ADAPTABILITY: SupportAdaptabilityEvaluator(self.judge_llm),
+            }
+        )
+
+    def run(self, filepath: str) -> EvaluationResult:
+        """Runs the complete evaluation pipeline for a single benchmark file.
+
+        Args:
+            filepath: Path to the benchmark markdown file.
+
+        Returns:
+            The final EvaluationResult containing overall and dimension scores.
+        """
+        # 1. Parse scenario file
+        benchmark = parse_benchmark(filepath)
+
+        # 2. Resolve evaluation profile (Static -> Dynamic -> Fallback/Error)
+        prof_name = benchmark.profile
+        profile = PROFILE_REGISTRY.get(prof_name)
+        if not profile:
+            try:
+                from framework.profiles.registry import get_custom_profile
+                profile = get_custom_profile(prof_name)
+            except Exception:
+                pass
+        if not profile:
+            if prof_name in ("travel-agent", "travel", "default"):
+                profile = TRAVEL_PROFILE
+            else:
+                raise ValueError(
+                    f"Evaluation profile '{prof_name}' not found in static or dynamic profile registries."
+                )
+
+        # Dimension names are shared across domains, but their rubrics are not.
+        # Configure support prompts only after this scenario's profile resolves.
+        self._configure_profile_evaluators(profile)
+
+        # 3. Execute agent with RunTrace instrumentation
+        t0 = time.time()
+        start_time = datetime.now(timezone.utc)
+        run_kwargs = {}
+        run_parameters = inspect.signature(self.agent.run).parameters
+        if "planning_mode" in run_parameters:
+            run_kwargs["planning_mode"] = "closed_world_evaluation"
+        if "validation_scenario_id" in run_parameters:
+            run_kwargs["validation_scenario_id"] = benchmark.benchmark_id
+
+        agent_output = None
+        status = "success"
+        err_msg = None
+
+        try:
+            agent_output = self.agent.run(benchmark.prompt, **run_kwargs)
+
+            # 4. Optional v3 Independent Budget Auditor Gate Pass
+            if self.auditor is not None:
+                audit_report = self.auditor.audit(
+                    scenario_prompt=benchmark.prompt,
+                    itinerary_content=agent_output.content,
+                )
+                agent_output.metadata["audit_report"] = audit_report.to_dict()
+                agent_output.metadata["audit_gate_decision"] = audit_report.status
+
+            # 5. Evaluate using the engine
+            result = self.engine.evaluate(benchmark, agent_output, profile)
+            result.agent_metadata = agent_output.metadata
+
+            latency = time.time() - t0
+            finished_time = datetime.now(timezone.utc)
+
+            # 6. Record RunTrace metadata
+            llm_obj = getattr(self.agent, "llm", None)
+            raw_usage = getattr(llm_obj, "last_token_usage", None) if llm_obj else None
+            token_usage = raw_usage if isinstance(raw_usage, dict) else None
+
+            trace = RunTrace(
+                trace_id=f"tr-{uuid.uuid4().hex[:8]}",
+                scenario_id=benchmark.benchmark_id,
+                agent_id=getattr(self.agent, "agent_name", self.agent.__class__.__name__),
+                model_name=getattr(llm_obj, "model_name", "unknown") if llm_obj else "unknown",
+                started_at_utc=start_time.isoformat(),
+                finished_at_utc=finished_time.isoformat(),
+                latency_seconds=round(latency, 2),
+                status="success",
+                error=None,
+                token_usage=token_usage,
+                metadata=agent_output.metadata,
+            )
+            agent_output.metadata["run_trace"] = {
+                "trace_id": trace.trace_id,
+                "started_at_utc": trace.started_at_utc,
+                "finished_at_utc": trace.finished_at_utc,
+                "latency_seconds": trace.latency_seconds,
+                "status": trace.status,
+                "token_usage": trace.token_usage,
+            }
+
+            # 7. Save reports and outputs
+            self._save_execution_files(benchmark, agent_output, result, profile)
+            return result
+
+        except Exception as e:
+            latency = time.time() - t0
+            finished_time = datetime.now(timezone.utc)
+            status = "timeout" if isinstance(e, (TimeoutError, TimeoutError)) or "timeout" in str(e).lower() else "error"
+            err_msg = str(e)
+
+            llm_obj = getattr(self.agent, "llm", None)
+            raw_usage = getattr(llm_obj, "last_token_usage", None) if llm_obj else None
+            token_usage = raw_usage if isinstance(raw_usage, dict) else None
+
+            agent_name = getattr(self.agent, "agent_name", None) or self.agent.__class__.__name__
+            model_name = getattr(llm_obj, "model_name", "unknown") if llm_obj and hasattr(llm_obj, "model_name") else "unknown"
+
+            trace = RunTrace(
+                trace_id=f"tr-{uuid.uuid4().hex[:8]}",
+                scenario_id=benchmark.benchmark_id,
+                agent_id=str(agent_name),
+                model_name=str(model_name),
+                started_at_utc=start_time.isoformat(),
+                finished_at_utc=finished_time.isoformat(),
+                latency_seconds=round(latency, 2),
+                status=status,
+                error=err_msg,
+                token_usage=token_usage,
+            )
+
+            # Persist error trace metadata file
+            self._save_error_trace_file(benchmark, trace)
+            raise
+
+    def run_directory(self, dirpath: str) -> Dict[str, EvaluationResult]:
+        """Runs evaluations for all benchmark files in the specified directory.
+
+        Args:
+            dirpath: Path to the folder containing benchmark markdown files.
+
+        Returns:
+            A dictionary mapping benchmark ID to the EvaluationResult.
+        """
+        results = {}
+        for filename in sorted(os.listdir(dirpath)):
+            if filename.endswith(".md"):
+                filepath = os.path.join(dirpath, filename)
+                try:
+                    result = self.run(filepath)
+                    results[result.benchmark_id] = result
+                except Exception as e:
+                    print(f"Error evaluating benchmark '{filename}': {e}", file=sys.stderr)
+        return results
+
+    def _save_execution_files(
+        self,
+        benchmark: Benchmark,
+        agent_output: AgentOutput,
+        result: EvaluationResult,
+        profile: Any,
+    ):
+        """Helper to serialize agent output, JSON data reports, and Markdown summaries."""
+        os.makedirs(self.output_dir, exist_ok=True)
+        agent_name = getattr(self.agent, "llm", self.agent).__class__.__name__.lower()
+        if hasattr(self.agent, "llm") and hasattr(self.agent.llm, "model_name"):
+            model_slug = self.agent.llm.model_name.replace("/", "_").replace(".", "_")
+        else:
+            model_slug = agent_name
+
+        base_name = f"{model_slug}_{benchmark.benchmark_id}"
+
+        # 1. Save Raw Agent Output/Itinerary
+        itinerary_path = os.path.join(self.output_dir, f"{base_name}_itinerary.md")
+        with open(itinerary_path, "w", encoding="utf-8") as f:
+            f.write(agent_output.content)
+
+        # 2. Save Detailed JSON Report
+        report_data = {
+            "benchmark_id": result.benchmark_id,
+            "benchmark_name": result.benchmark_name,
+            "agent_model": getattr(self.agent, "llm", self.agent).__class__.__name__,
+            "model_name": getattr(getattr(self.agent, "llm", None), "model_name", "unknown"),
+            "profile": profile.name,
+            "overall_score": result.overall_score,
+            "passed": result.passed,
+            "agent_metadata": agent_output.metadata,
+            "dimension_scores": [
+                {
+                    "dimension": ds.dimension,
+                    "score": ds.score,
+                    "reason": ds.reason,
+                }
+                for ds in result.dimension_scores
+            ],
+        }
+        json_path = os.path.join(self.output_dir, f"{base_name}_report.json")
+        with open(json_path, "w", encoding="utf-8") as f:
+            json.dump(report_data, f, indent=2)
+
+        # 3. Save Summary Markdown Report
+        markdown_path = os.path.join(self.output_dir, f"{base_name}_report.md")
+        with open(markdown_path, "w", encoding="utf-8") as f:
+            f.write(f"# Evaluation Summary Report\n\n")
+            f.write(f"- **Benchmark**: {result.benchmark_name} (`{result.benchmark_id}`)\n")
+            f.write(f"- **Agent Model**: {report_data['model_name']}\n")
+            f.write(f"- **Evaluation Profile**: `{profile.name}`\n")
+            f.write(f"- **Overall Score**: **{result.overall_score:.2f} / 100**\n")
+            f.write(f"- **Outcome**: {'PASSED' if result.passed else 'FAILED'}\n\n")
+
+            # Render Auditor Gate metadata section if present
+            if "audit_report" in agent_output.metadata:
+                audit = agent_output.metadata["audit_report"]
+                f.write(f"## Independent Auditor Gate Report\n\n")
+                f.write(f"- **Gate Decision**: `{'PASS' if audit.get('passed') else 'BLOCK'}`\n")
+                f.write(f"- **Audit Score**: {audit.get('audit_score', 0.0):.2f} / 100\n")
+                f.write(f"- **Audit Confidence**: {audit.get('audit_confidence', 0.0):.2f}\n")
+                f.write(f"- **Reasoning Summary**: {audit.get('reasoning_summary', 'N/A')}\n\n")
+                if audit.get("violations"):
+                    f.write("### Detected Budget Violations\n\n")
+                    for v in audit["violations"]:
+                        f.write(f"- **{v['violation_type']}**: {v['description']} (Est. Discrepancy: ₹{v.get('estimated_discrepancy_inr', 0.0):,.2f})\n")
+                    f.write("\n")
+
+            # Render MCP metadata trace section if present
+            if "mcp_validation" in agent_output.metadata:
+                mcp = agent_output.metadata["mcp_validation"]
+                f.write(f"## MCP Validation Trace\n\n")
+                f.write(f"- **Status**: `{mcp.get('status', 'unknown')}`\n")
+                f.write(f"- **Locked-Booking Check**: `{'valid' if mcp.get('locked_valid') else 'invalid'}`\n")
+                f.write(f"- **Savings Check**: ₹{mcp.get('savings_realized_inr', 0):,} verified (target: ₹{mcp.get('target_savings_inr', 0):,})\n\n")
+
+            f.write(f"## Dimension Breakdown\n\n")
+            f.write(f"| Dimension | Score | Weight |\n")
+            f.write(f"| :--- | :---: | :---: |\n")
+            for ds in result.dimension_scores:
+                weight = profile.weights.get(ds.dimension, 0.0)
+                f.write(f"| {ds.dimension} | {ds.score:.1f} | {weight:.1f}% |\n")
+            f.write(f"\n## Reasoning Details\n\n")
+            for ds in result.dimension_scores:
+                f.write(f"### {ds.dimension} (Score: {ds.score:.1f})\n\n")
+                f.write(f"{ds.reason}\n\n")
+
+    def _save_error_trace_file(self, benchmark: Benchmark, trace: RunTrace) -> None:
+        """Persists an error/timeout trace JSON record when an evaluation fails."""
+        os.makedirs(self.output_dir, exist_ok=True)
+        model_name = trace.model_name.replace("/", "_").replace(".", "_")
+        json_path = os.path.join(self.output_dir, f"{model_name}_{benchmark.benchmark_id}_error_trace.json")
+        trace_data = {
+            "trace_id": trace.trace_id,
+            "scenario_id": trace.scenario_id,
+            "agent_id": trace.agent_id,
+            "model_name": trace.model_name,
+            "started_at_utc": trace.started_at_utc,
+            "finished_at_utc": trace.finished_at_utc,
+            "latency_seconds": trace.latency_seconds,
+            "status": trace.status,
+            "error": trace.error,
+            "token_usage": trace.token_usage,
+        }
+        with open(json_path, "w", encoding="utf-8") as f:
+            json.dump(trace_data, f, indent=2)

framework/evaluation/testing.py

@@ -0,0 +1,25 @@
+"""Mock and testing utilities for evaluation pipelines."""
+
+from framework.evaluation.base import BaseEvaluator
+from framework.models import AgentOutput, Benchmark, DimensionScore
+
+
+class DummyEvaluator(BaseEvaluator):
+    """Simple evaluator returning a static score for profile verification."""
+
+    def __init__(self, dimension: str, score: float, reason: str):
+        """Initializes the DummyEvaluator.
+
+        Args:
+            dimension: Dimension name key this evaluator represents.
+            score: Static numeric score to return.
+            reason: Static reason explanation to return.
+        """
+        self._dimension = dimension
+        self.score = score
+        self.reason = reason
+
+    def evaluate(self, benchmark: Benchmark, output: AgentOutput) -> DimensionScore:
+        return DimensionScore(
+            dimension=self._dimension, score=self.score, reason=self.reason
+        )

framework/exceptions.py

@@ -0,0 +1,49 @@
+"""Common exceptions for the evaluation framework."""
+
+
+class BenchmarkError(Exception):
+    """Base class for all benchmark-related exceptions."""
+
+    pass
+
+
+class FrontmatterParseError(BenchmarkError):
+    """Raised when parsing YAML frontmatter from content fails."""
+
+    pass
+
+
+class MarkdownParseError(BenchmarkError):
+    """Raised when parsing markdown body into sections fails."""
+
+    pass
+
+
+class TransformerError(BenchmarkError):
+    """Raised when transforming raw markdown sections into Python structures fails."""
+
+    pass
+
+
+class MapperError(BenchmarkError):
+    """Raised when mapping parsed content to a Benchmark dataclass fails."""
+
+    pass
+
+
+class EvaluationError(Exception):
+    """Raised when evaluation orchestration or execution fails."""
+
+    pass
+
+
+class VerificationError(EvaluationError):
+    """Base class for factual claim verification errors."""
+
+    pass
+
+
+class KnowledgeBaseLoadError(VerificationError):
+    """Raised when loading or parsing a verifier knowledge base fails."""
+
+    pass

framework/llms/__init__.py

@@ -0,0 +1,8 @@
+"""LLM abstractions package containing wrapper clients and factory methods."""
+
+from framework.llms.base import BaseLLM, Message
+from framework.llms.mock import MockLLM
+from framework.llms.openai import OpenAILLM
+from framework.llms.gemini import GeminiLLM
+from framework.llms.openai_compatible import OpenAICompatibleLLM
+from framework.llms.factory import create_llm

framework/llms/base.py

@@ -0,0 +1,37 @@
+"""Base interfaces and data classes for LLM clients."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Dict, List
+
+
+@dataclass
+class Message:
+    """Represents a chat message sent to/from an LLM."""
+
+    role: str
+    content: str
+
+
+@dataclass
+class LLMResponse:
+    """Represents a structured response from an LLM."""
+
+    text: str
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+class BaseLLM(ABC):
+    """Abstract base class representing an LLM client wrapper."""
+
+    @abstractmethod
+    def generate(self, messages: List[Message]) -> LLMResponse:
+        """Generates a structured completion for the given list of chat messages.
+
+        Args:
+            messages: A list of Message dataclasses.
+
+        Returns:
+            An LLMResponse containing the text outcome.
+        """
+        pass

framework/llms/factory.py

@@ -0,0 +1,38 @@
+"""Factory module for creating LLM client wrappers."""
+
+from typing import Any, Dict, Type
+from framework.llms.base import BaseLLM
+from framework.llms.mock import MockLLM
+from framework.llms.openai import OpenAILLM
+from framework.llms.gemini import GeminiLLM
+
+# Registry of supported providers mapping provider name to class wrapper
+_PROVIDERS: Dict[str, Type[BaseLLM]] = {
+    "openai": OpenAILLM,
+    "gemini": GeminiLLM,
+    "mock": MockLLM,
+}
+
+
+def create_llm(provider: str, **kwargs: Any) -> BaseLLM:
+    """Factory function to instantiate an LLM wrapper by provider name.
+
+    Args:
+        provider: Name of the LLM provider ('openai', 'gemini', 'mock').
+        **kwargs: Arguments passed to the provider class constructor.
+
+    Returns:
+        An instance of BaseLLM.
+
+    Raises:
+        ValueError: If an unknown provider name is supplied.
+    """
+    try:
+        provider_cls = _PROVIDERS[provider.lower()]
+    except KeyError:
+        supported = ", ".join(sorted(_PROVIDERS.keys()))
+        raise ValueError(
+            f"Unknown LLM provider '{provider}'. Supported providers: {supported}"
+        )
+
+    return provider_cls(**kwargs)

framework/llms/gemini.py

@@ -0,0 +1,85 @@
+"""Google Gemini client wrapper implementation."""
+
+from typing import List, Optional
+from framework.llms.base import BaseLLM, LLMResponse, Message
+
+
+class GeminiLLM(BaseLLM):
+    """Google Gemini API client wrapper."""
+
+    def __init__(
+        self, model_name: str = "gemini-1.5-pro", api_key: Optional[str] = None
+    ):
+        """Initializes the GeminiLLM.
+
+        Validates that the 'google-generativeai' library is installed immediately.
+
+        Args:
+            model_name: The target model name.
+            api_key: The Gemini API key. If not provided, configuration is skipped.
+
+        Raises:
+            ImportError: If the 'google-generativeai' library is not installed.
+        """
+        try:
+            # pyrefly: ignore [missing-import]
+            import google.generativeai as genai
+        except ImportError:
+            raise ImportError(
+                "The 'google-generativeai' library is required to use GeminiLLM. "
+                "Please install it using 'pip install google-generativeai'."
+            )
+
+        if api_key:
+            genai.configure(api_key=api_key)
+
+        self._genai = genai
+        # Ensure name is prefixed correctly (e.g. models/gemini-1.5-pro)
+        if model_name and not model_name.startswith("models/"):
+            self.model_name = f"models/{model_name}"
+        else:
+            self.model_name = model_name
+
+    def generate(self, messages: List[Message]) -> LLMResponse:
+        system_instruction = None
+        gemini_contents = []
+
+        for msg in messages:
+            if msg.role == "system":
+                system_instruction = msg.content
+            else:
+                # Map standard roles: user -> user, assistant/model -> model
+                role = "user" if msg.role == "user" else "model"
+                gemini_contents.append(
+                    {"role": role, "parts": [{"text": msg.content}]}
+                )
+
+        config = {}
+        if system_instruction:
+            config["system_instruction"] = system_instruction
+
+        import time
+
+        model = self._genai.GenerativeModel(self.model_name, **config)
+        
+        max_retries = 4
+        for attempt in range(max_retries):
+            try:
+                response = model.generate_content(
+                    gemini_contents,
+                    request_options={"timeout": 300.0},
+                )
+                response_text = response.text or ""
+                return LLMResponse(text=response_text)
+            except Exception as e:
+                err_str = str(e).lower()
+                is_transient = any(
+                    err in err_str
+                    for err in ["504", "deadline", "timeout", "resourceexhausted", "429", "503", "unavailable"]
+                )
+                if is_transient and attempt < max_retries - 1:
+                    sleep_seconds = 5 * (attempt + 1)
+                    print(f"[GeminiLLM] Transient error ({e}). Retrying in {sleep_seconds}s (attempt {attempt + 1}/{max_retries})...")
+                    time.sleep(sleep_seconds)
+                else:
+                    raise e

framework/llms/mock.py

@@ -0,0 +1,25 @@
+"""Mock LLM client wrapper for testing and offline validation."""
+
+from typing import List, Union
+from framework.llms.base import BaseLLM, LLMResponse, Message
+
+
+class MockLLM(BaseLLM):
+    """A mock LLM wrapper for testing and offline execution."""
+
+    def __init__(self, responses: Union[str, List[str]]):
+        """Initializes the MockLLM.
+
+        Args:
+            responses: A single response string or a list of response strings
+                to return sequentially on each generate call.
+        """
+        self.responses = [responses] if isinstance(responses, str) else list(responses)
+        self.call_count = 0
+
+    def generate(self, messages: List[Message]) -> LLMResponse:
+        if not self.responses:
+            return LLMResponse(text="Mock response")
+        response = self.responses[self.call_count % len(self.responses)]
+        self.call_count += 1
+        return LLMResponse(text=response)