evalrun 0.4.0__py3-none-any.whl

agents/support/triage_agent.py

@@ -0,0 +1,45 @@
+"""Customer-support triage agent used by the second-domain benchmark."""
+
+from langfuse import observe
+
+from agents.base import BaseAgent
+from framework.llms import BaseLLM, Message
+from framework.models import AgentOutput
+
+
+SUPPORT_TRIAGE_SYSTEM_PROMPT = """You are a production customer-support triage agent.
+
+Read the ticket carefully and return a concise, structured triage decision. Include:
+1. priority and the SLA deadline;
+2. issue category and a one-sentence evidence-based summary;
+3. immediate containment or next action;
+4. the exact escalation destination and timing;
+5. a safe, empathetic customer-facing response.
+
+Do not invent a root cause, promise an unsupported resolution time, expose secrets,
+or downgrade a high-impact production incident. Separate facts from hypotheses.
+"""
+
+
+class SupportTriageAgent(BaseAgent):
+    """Thin adapter that gives any BaseLLM the support-triage agent contract."""
+
+    def __init__(self, llm: BaseLLM):
+        self.llm = llm
+
+    @observe(name="support-triage-agent")
+    def run(self, prompt: str) -> AgentOutput:
+        response = self.llm.generate(
+            [
+                Message(role="system", content=SUPPORT_TRIAGE_SYSTEM_PROMPT),
+                Message(role="user", content=prompt),
+            ]
+        )
+        return AgentOutput(
+            content=response.text,
+            metadata={
+                "agent": self.__class__.__name__,
+                "llm": type(self.llm).__name__,
+                "domain": "support-triage",
+            },
+        )

agents/travel/__init__.py

@@ -0,0 +1,11 @@
+"""Travel planning agent package."""
+
+from agents.travel.agent import TravelPlanningAgent
+from agents.travel.session import (
+    Booking,
+    RemoteWorkSchedule,
+    UserPreferences,
+    TripConstraints,
+    CurrentTripState,
+    TravelSessionMemory,
+)

agents/travel/agent.py

@@ -0,0 +1,377 @@
+from langfuse import observe
+
+import asyncio
+import json
+from typing import Any, Dict, Optional
+from framework.llms import BaseLLM, Message
+from framework.models import AgentOutput
+from framework.utils import parse_json_markdown
+from framework.memory import BaseSessionMemory
+from agents.base import BaseAgent
+from agents.research.planner import ResearchPlanner
+from framework.mcp.revision_summary import (
+    REVISION_SUMMARY_JSON_TEMPLATE,
+    parse_revision_summary,
+)
+from .prompts import CLOSED_WORLD_EVALUATION_INSTRUCTION, TRAVEL_PLANNING_SYSTEM_PROMPT
+
+
+class TravelPlanningAgent(BaseAgent):
+    """A travel planning assistant agent powered by an LLM.
+
+    Accepts user prompts/scenarios and plans itineraries accordingly,
+    optionally collaborating with a ResearchAgent, ResearchPlanner, and SessionMemory.
+    """
+
+    def __init__(
+        self,
+        llm: BaseLLM,
+        research_agent: Optional[BaseAgent] = None,
+        research_planner: Optional[ResearchPlanner] = None,
+        reflection_agent: Optional[BaseAgent] = None,
+        validation_client: Optional[Any] = None,
+    ):
+        """Initializes the TravelPlanningAgent.
+
+        Args:
+            llm: The LLM client wrapper to generate itineraries.
+            research_agent: An optional research subagent to query for factual details.
+            research_planner: An optional planner to determine what queries to research.
+            reflection_agent: An optional reflection subagent to critique itineraries.
+            validation_client: Optional async MCP client for deterministic
+                replanning checks. It is opt-in so baseline experiments remain
+                unchanged.
+        """
+        self.llm = llm
+        self.research_agent = research_agent
+        self.research_planner = research_planner or ResearchPlanner(llm)
+        self.reflection_agent = reflection_agent
+        self.validation_client = validation_client
+
+    @observe(name="travel-planning-agent")
+    def run(
+        self,
+        prompt: str,
+        session_memory: Optional[BaseSessionMemory] = None,
+        validation_scenario_id: Optional[str] = None,
+        planning_mode: str = "standard",
+    ) -> AgentOutput:
+        """Generates a travel itinerary based on user preferences.
+
+        Args:
+            prompt: The user prompt describing constraints and preferences.
+            session_memory: Optional session memory to provide state context.
+            validation_scenario_id: MCP scenario identifier. When supplied
+                together with ``validation_client`` during replanning, the
+                proposed revision is checked before the final LLM revision.
+            planning_mode: ``closed_world_evaluation`` tells the planner that a
+                benchmark deliberately supplies all planning constraints. The
+                default ``standard`` mode retains the cautious clarification
+                policy for genuinely under-specified user requests.
+
+        Returns:
+            An AgentOutput containing the planned travel itinerary.
+        """
+        research_context = ""
+        research_metadata = []
+
+        if self.research_agent:
+            # Delegate query generation to the external planner component
+            queries = self.research_planner.plan_queries(prompt)
+            if queries:
+                findings = []
+                for query in queries:
+                    try:
+                        research_output = self.research_agent.run(query)
+                        escaped_query = query.replace('"', '\\"')
+                        # Indent multi-line answers to preserve YAML block formatting
+                        indented_answer = research_output.content.replace('\n', '\n    ')
+                        findings.append(
+                            f"- query: \"{escaped_query}\"\n"
+                            f"  answer: |\n"
+                            f"    {indented_answer}"
+                        )
+                        research_metadata.append({
+                            "query": query,
+                            "researcher": research_output.metadata.get("agent"),
+                        })
+                    except Exception as e:
+                        findings.append(f"- query: \"{query}\"\n  answer: \"FAILED: {e}\"")
+                
+                research_context = (
+                    "### FACTUAL RESEARCH FINDINGS (YAML format):\n"
+                    "Research Findings:\n"
+                    + "\n".join(findings)
+                    + "\n==================================================\n\n"
+                )
+
+        memory_context = ""
+        if session_memory:
+            memory_context = (
+                "### CURRENT TRAVELER SESSION STATE (YAML format):\n"
+                f"{session_memory.to_yaml()}\n"
+                "==================================================\n\n"
+            )
+
+        system_prompt = self._system_prompt_for(planning_mode)
+        user_content = ""
+        if memory_context:
+            user_content += memory_context
+        if research_context:
+            user_content += research_context
+        user_content += prompt
+
+        messages = [
+            Message(role="system", content=system_prompt),
+            Message(role="user", content=user_content),
+        ]
+
+        # 1. Generate initial draft plan (v1)
+        response = self.llm.generate(messages)
+        final_itinerary = response.text
+
+        # 2. Invoke reflection loop if a reflection agent is set
+        reflection_critique = None
+        mcp_validation: Optional[Dict[str, Any]] = None
+        revision_triggered = False
+        if self.reflection_agent:
+            if hasattr(self.reflection_agent, "reflect"):
+                reflection_output = self.reflection_agent.reflect(prompt, final_itinerary, session_memory)
+            else:
+                reflection_output = self.reflection_agent.run(
+                    f"Scenario: {prompt}\n\nDraft Itinerary:\n{final_itinerary}"
+                )
+            
+            critique = reflection_output.content
+
+            # v2.1 must validate every replanning draft. Reflection approval
+            # does not prove that hard constraints or savings arithmetic hold.
+            is_replanning = False
+            if session_memory and session_memory.state.current_day > 1:
+                is_replanning = True
+            elif "currently on day" in prompt.lower() or "replanning" in prompt.lower() or "disruption" in prompt.lower():
+                is_replanning = True
+
+            initial_validation = None
+            if is_replanning and self.validation_client and validation_scenario_id:
+                initial_validation = self._validate_replanning_proposal(
+                    system_prompt=system_prompt,
+                    scenario_prompt=prompt,
+                    draft_itinerary=final_itinerary,
+                    critique=critique,
+                    scenario_id=validation_scenario_id,
+                )
+                mcp_validation = {"initial": initial_validation, "final": None}
+
+            reflection_requires_revision = "ITINERARY APPROVED" not in critique
+            mcp_requires_revision = self._validation_requires_revision(initial_validation)
+            revision_triggered = reflection_requires_revision or mcp_requires_revision
+
+            if reflection_requires_revision:
+                reflection_critique = critique
+
+            if revision_triggered:
+                # Increment plan version in memory
+                if session_memory:
+                    session_memory.state.current_plan_version += 1
+                
+                # Regenerate memory context with updated plan version
+                memory_context_v2 = ""
+                if session_memory:
+                    memory_context_v2 = (
+                        "### CURRENT TRAVELER SESSION STATE (YAML format):\n"
+                        f"{session_memory.to_yaml()}\n"
+                        "==================================================\n\n"
+                    )
+
+                if is_replanning:
+                    revision_rules = (
+                        "1. PRESERVE every part of the itinerary that was NOT criticized. Do NOT rewrite or shift unaffected days.\n"
+                        "2. Do NOT modify, delete, or rearrange locked bookings (e.g., booked flights, non-refundable hotel stays).\n"
+                        "3. Make ONLY the minimum necessary adjustments needed to resolve the critique points."
+                    )
+                else:
+                    revision_rules = (
+                        "1. You are free to re-sequence, compress, or optimize the entire itinerary globally to satisfy the constraints (e.g., total duration, must-visit destinations) and address the critiques.\n"
+                        "2. Do NOT exceed the total duration limit specified in the original request.\n"
+                        "3. Make adjustments to address the critiques while keeping the travel style and preferences consistent."
+                    )
+
+                # Formulate revision prompt
+                validation_context = ""
+                if initial_validation:
+                    validation_context = (
+                        "\n\n### DETERMINISTIC MCP VALIDATION REPORT:\n"
+                        f"{json.dumps(initial_validation, indent=2)}\n"
+                        "Treat every violation or unmet savings target in this report as a hard "
+                        "requirement for the final revision. Do not claim a saving unless it is "
+                        "represented by a concrete itinerary change.\n"
+                    )
+                revision_prompt = (
+                    f"### ORIGINAL SCENARIO REQUEST:\n{prompt}\n\n"
+                    f"### INITIAL DRAFT PLAN:\n{final_itinerary}\n\n"
+                    f"### REFLECTION CRITIQUE:\n{critique}\n\n"
+                    "You are instructed to revise the initial draft plan to resolve the critiques listed above.\n"
+                    "You MUST strictly follow these rules during the revision:\n"
+                    f"{revision_rules}\n"
+                    "4. If a critique point directly conflicts with an existing hard constraint in the original request, prioritize and preserve the hard constraint.\n"
+                    "5. You MUST generate a complete, full day-by-day traveler-facing itinerary. Do NOT output a list of questions, deferrals, or requests for information in place of the itinerary."
+                    f"{validation_context}"
+                )
+
+                user_content_v2 = ""
+                if memory_context_v2:
+                    user_content_v2 += memory_context_v2
+                if research_context:
+                    user_content_v2 += research_context
+                user_content_v2 += revision_prompt
+
+                messages_v2 = [
+                    Message(role="system", content=system_prompt),
+                    Message(role="user", content=user_content_v2),
+                ]
+                # Generate revised plan (v2)
+                response = self.llm.generate(messages_v2)
+                if response.text.strip():
+                    final_itinerary = response.text
+
+                if is_replanning and self.validation_client and validation_scenario_id:
+                    mcp_validation["final"] = self._validate_replanning_proposal(
+                        system_prompt=system_prompt,
+                        scenario_prompt=prompt,
+                        draft_itinerary=final_itinerary,
+                        critique="Validate the final revised itinerary against the original constraints.",
+                        scenario_id=validation_scenario_id,
+                    )
+
+        # 3. Assemble metadata
+        metadata = {
+            "agent": self.__class__.__name__,
+            "llm": type(self.llm).__name__,
+            "reflection_approved": not bool(reflection_critique),
+            "revision_triggered": revision_triggered,
+        }
+        if research_metadata:
+            metadata["research_steps"] = research_metadata
+        if revision_triggered:
+            metadata["reflection_critique"] = reflection_critique
+            metadata["final_plan_version"] = session_memory.state.current_plan_version if session_memory else 2
+        if mcp_validation:
+            metadata["mcp_validation"] = mcp_validation
+
+        return AgentOutput(
+            content=final_itinerary,
+            metadata=metadata,
+        )
+
+    def _validate_replanning_proposal(
+        self,
+        system_prompt: str,
+        scenario_prompt: str,
+        draft_itinerary: str,
+        critique: str,
+        scenario_id: str,
+    ) -> Dict[str, Any]:
+        """Gets explicit revision claims and validates them through MCP.
+
+        The model does not use this call to write the final traveler-facing
+        answer. It only declares the concrete changes it intends to make, which
+        lets MCP verify locked bookings and arithmetic before the final rewrite.
+        A malformed summary or unavailable server is reported to the final
+        revision prompt rather than silently treated as a passing check.
+        """
+        # Retrieve exact locked constraints to ground the model on valid booking IDs
+        locked_info = {}
+        if self.validation_client:
+            try:
+                locked_info = self._call_validation_tool(
+                    "get_locked_constraints", {"scenario_id": scenario_id}
+                )
+            except Exception:
+                pass
+
+        immutable_ids = locked_info.get("immutable_booking_ids", ["kyoto-hostel", "narita-return-flight"])
+
+        summary_prompt = (
+            "Return only one JSON object describing the proposed replanning revision. "
+            "Do not write an itinerary and do not use Markdown outside the JSON.\n\n"
+            f"The scenario_id must be exactly: {scenario_id}\n"
+            f"The exact booking_id values to preserve in booking_actions MUST be: {json.dumps(immutable_ids)}. "
+            "Do NOT add suffixes or alter these booking_id strings.\n"
+            "List every itinerary day you intend to change. For every locked booking, "
+            "declare preserve, move, cancel, or modify. Itemize each concrete saving "
+            "in INR; do not estimate a total without its components.\n\n"
+            f"Required JSON shape:\n{REVISION_SUMMARY_JSON_TEMPLATE}\n\n"
+            f"### ORIGINAL SCENARIO REQUEST:\n{scenario_prompt}\n\n"
+            f"### INITIAL DRAFT PLAN:\n{draft_itinerary}\n\n"
+            f"### REFLECTION CRITIQUE:\n{critique}\n"
+        )
+        summary_response = self.llm.generate(
+            [
+                Message(role="system", content=system_prompt),
+                Message(role="user", content=summary_prompt),
+            ]
+        )
+
+        try:
+            summary = parse_revision_summary(summary_response.text)
+            if summary.scenario_id != scenario_id:
+                raise ValueError(
+                    "Revision summary scenario_id does not match the requested scenario."
+                )
+            tool_arguments = summary.to_tool_arguments()
+            return {
+                "status": "completed",
+                "revision_summary": tool_arguments,
+                "locked_constraints": locked_info or self._call_validation_tool(
+                    "get_locked_constraints", {"scenario_id": scenario_id}
+                ),
+                "revision_check": self._call_validation_tool(
+                    "validate_revision",
+                    {
+                        "scenario_id": scenario_id,
+                        "booking_actions": tool_arguments["booking_actions"],
+                        "changed_days": tool_arguments["changed_days"],
+                    },
+                ),
+                "savings_check": self._call_validation_tool(
+                    "calculate_savings",
+                    {
+                        "scenario_id": scenario_id,
+                        "savings_items": tool_arguments["savings_items"],
+                    },
+                ),
+            }
+        except (RuntimeError, ValueError) as error:
+            return {
+                "status": "unavailable",
+                "error": str(error),
+                "raw_revision_summary": summary_response.text,
+            }
+
+    def _call_validation_tool(self, name: str, arguments: Dict[str, Any]) -> Dict[str, Any]:
+        """Runs an async MCP call from this currently synchronous agent API."""
+        return asyncio.run(self.validation_client.call_tool(name, arguments))
+
+    @staticmethod
+    def _validation_requires_revision(validation: Optional[Dict[str, Any]]) -> bool:
+        """Returns whether an MCP result requires a constrained revision."""
+        if validation is None:
+            return False
+        if validation.get("status") != "completed":
+            return True
+        return (
+            not validation.get("revision_check", {}).get("valid", False)
+            or not validation.get("savings_check", {}).get("target_met", False)
+        )
+
+    @staticmethod
+    def _system_prompt_for(planning_mode: str) -> str:
+        """Returns the planner policy for normal or controlled benchmark runs."""
+        if planning_mode == "standard":
+            return TRAVEL_PLANNING_SYSTEM_PROMPT
+        if planning_mode == "closed_world_evaluation":
+            return TRAVEL_PLANNING_SYSTEM_PROMPT + "\n" + CLOSED_WORLD_EVALUATION_INSTRUCTION
+        raise ValueError(
+            "planning_mode must be 'standard' or 'closed_world_evaluation'."
+        )

agents/travel/prompts.py

@@ -0,0 +1,30 @@
+TRAVEL_PLANNING_SYSTEM_PROMPT = (
+    "You are a professional travel agent assistant.\n"
+    "Your task is to plan a detailed, well-structured travel itinerary "
+    "based on the user's preferences, budget limits, duration, backpacking style, "
+    "and other specified constraints.\n"
+    "Be precise, geographically efficient, and mention exact travel logistics, closed days, and pricing "
+    "where appropriate.\n\n"
+    "CRITICAL RULE ON UNCERTAINTY:\n"
+    "If the user's request lacks critical details needed to build a realistic itinerary (such as travel dates, "
+    "trip duration, specific destinations, or seasonal timing dependencies like cherry blossom bloom forecasts), "
+    "you must NOT guess, assume, or hallucinate a sample or final itinerary. Instead, you must:\n"
+    "1. Defer generating the final day-by-day itinerary.\n"
+    "2. Ask the user clarifying questions to obtain the missing details (e.g., travel style, departure airport).\n"
+    "3. Output structured tool requests in a YAML block specifying what external search queries are required to gather "
+    "the necessary data first (forecasts, festival dates, pricing). Use this exact format:\n\n"
+    "Tool Requests:\n"
+    "  - Search:\n"
+    "      query: \"<search query>\"\n"
+)
+
+
+CLOSED_WORLD_EVALUATION_INSTRUCTION = (
+    "EVALUATION SCENARIO MODE:\n"
+    "The scenario supplied in the user message is a complete, closed-world test "
+    "case. Produce a concrete provisional itinerary using its stated days, "
+    "constraints, and disruptions. Do not ask clarifying questions or defer the "
+    "itinerary merely because calendar dates, live availability, or exact prices "
+    "are absent. Do not invent external facts: label any operational detail that "
+    "would require confirmation as conditional, while still completing the plan.\n"
+)

agents/travel/session.py

@@ -0,0 +1,110 @@
+"""Travel-specific session memory implementation."""
+
+from dataclasses import dataclass, field, asdict
+from typing import Any, Dict, List, Optional
+import yaml
+import json
+from framework.memory import BaseSessionMemory
+
+
+@dataclass
+class Booking:
+    """Represents a structured locked booking."""
+    booking_id: str
+    booking_type: str  # e.g., "flight", "hotel", "activity", "train"
+    location: str
+    start_day: int
+    end_day: int
+    refundable: bool = True
+    notes: str = ""
+
+
+@dataclass
+class RemoteWorkSchedule:
+    """Represents a remote work commitment and timezone constraints."""
+    timezone: str = "UTC"
+    availability_start: str = "09:00"
+    availability_end: str = "17:00"
+    meeting_start: str = ""
+    meeting_end: str = ""
+    flexible_hours: int = 0
+
+
+@dataclass
+class UserPreferences:
+    """Represents purely static user profiling and travel preferences."""
+    interests: List[str] = field(default_factory=list)
+    travel_style: str = "balanced"
+    accommodation_preference: str = "hotel"
+    walking_tolerance: str = "moderate"
+
+
+@dataclass
+class TripConstraints:
+    """Represents hard trip limits, schedules, and locked bookings."""
+    total_budget: float = 0.0
+    duration_days: int = 1
+    destinations: List[str] = field(default_factory=list)
+    remote_work_schedule: Optional[RemoteWorkSchedule] = None
+    locked_bookings: List[Booking] = field(default_factory=list)
+    hard_constraints: List[str] = field(default_factory=list)
+
+
+@dataclass
+class CurrentTripState:
+    """Tracks active planning state and execution progress."""
+    current_day: int = 1
+    remaining_budget: float = 0.0
+    completed_destinations: List[str] = field(default_factory=list)
+    current_plan_version: int = 1
+
+
+class TravelSessionMemory(BaseSessionMemory):
+    """Holds and serializes working session memory for travel planning and reflection."""
+
+    def __init__(
+        self,
+        session_id: str,
+        version: int = 1,
+        preferences: Optional[UserPreferences] = None,
+        constraints: Optional[TripConstraints] = None,
+        state: Optional[CurrentTripState] = None,
+    ):
+        """Initializes the TravelSessionMemory.
+
+        Args:
+            session_id: A unique string identifier for the planning session.
+            version: Schema/memory state version.
+            preferences: Optional initial UserPreferences.
+            constraints: Optional initial TripConstraints.
+            state: Optional initial CurrentTripState.
+        """
+        self.session_id = session_id
+        self.version = version
+        self.preferences = preferences or UserPreferences()
+        self.constraints = constraints or TripConstraints()
+        self.state = state or CurrentTripState()
+
+    def to_yaml(self) -> str:
+        """Serializes session state to structured YAML for LLM context injection.
+
+        Excludes empty lists or empty dicts to conserve context tokens.
+        """
+        data = {
+            "session_id": self.session_id,
+            "version": self.version,
+            "preferences": {k: v for k, v in asdict(self.preferences).items() if v},
+            "constraints": {k: v for k, v in asdict(self.constraints).items() if v},
+            "state": {k: v for k, v in asdict(self.state).items() if v},
+        }
+        return yaml.dump(data, sort_keys=False, default_flow_style=False)
+
+    def to_json(self) -> str:
+        """Returns JSON representation of memory state."""
+        return json.dumps({
+            "session_id": self.session_id,
+            "version": self.version,
+            "preferences": asdict(self.preferences),
+            "constraints": asdict(self.constraints),
+            "state": asdict(self.state),
+        }, indent=2)

cli/__init__.py

@@ -0,0 +1,6 @@
+"""evalrun CLI package exports."""
+
+from cli.main import main, create_parser
+from cli.resolver import resolve_agent
+
+__all__ = ["main", "create_parser", "resolve_agent"]

cli/demo.py

@@ -0,0 +1,47 @@
+"""Offline, zero-credential EvalRun product demonstration."""
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from cli.formatter import format_terminal_summary
+from cli.html_reporter import generate_html_report
+from framework.models import DimensionScore, EvaluationResult
+
+
+def run_demo(output_dir: str = "results/demo") -> int:
+    """Create a deterministic sample report without contacting a model endpoint."""
+    out = Path(output_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    result = EvaluationResult(
+        benchmark_id="evalrun-demo-scenario",
+        benchmark_name="EvalRun Offline Demo",
+        overall_score=88.0,
+        dimension_scores=[
+            DimensionScore("Constraint Satisfaction", 90.0, "All demo constraints were satisfied."),
+            DimensionScore("Planning Quality", 86.0, "The demo output is coherent and complete."),
+        ],
+        passed=True,
+        agent_metadata={"demo": True, "audit_gate_decision": "PASS"},
+    )
+    manifest = {
+        "run_id": "evalrun-offline-demo",
+        "timestamp_utc": datetime.now(timezone.utc).isoformat(),
+        "target_agent_spec": "built-in offline demo",
+        "target_model": {"model_name": "offline-demo", "base_url": "local", "api_key": "EMPTY"},
+        "judge_model": {"model_name": "offline-demo", "base_url": "local", "api_key": "EMPTY"},
+        "output_dir": str(out),
+        "total_scenarios": 1,
+        "overall_passed": True,
+        "scenarios": [{"scenario_id": result.benchmark_id, "scenario_name": result.benchmark_name,
+                       "overall_score": result.overall_score, "passed": result.passed,
+                       "audit_gate_decision": "PASS"}],
+    }
+    with open(out / "manifest.json", "w", encoding="utf-8") as handle:
+        json.dump(manifest, handle, indent=2)
+    with open(out / "evalrun-demo-output.txt", "w", encoding="utf-8") as handle:
+        handle.write("This is a deterministic offline preview. No model endpoint was contacted.\n")
+    generate_html_report([result], manifest, str(out))
+    print(format_terminal_summary([result], manifest))
+    print(f"\nOffline demo artifacts: {out.resolve()}")
+    return 0