evalrun 0.4.0__py3-none-any.whl

agents/__init__.py

@@ -0,0 +1,6 @@
+"""Agents package representing subjects under evaluation."""
+
+from agents.base import BaseAgent
+from agents.research.agent import ResearchAgent
+from agents.research.planner import ResearchPlanner
+from agents.reflection.agent import ReflectionAgent

agents/auditor/__init__.py

@@ -0,0 +1,13 @@
+"""Independent Budget Auditor package exports."""
+
+from agents.auditor.schema import AuditReport, BudgetViolation, FailureCode
+from agents.auditor.parser import AuditParser
+from agents.auditor.budget_auditor import IndependentBudgetAuditor
+
+__all__ = [
+    "AuditReport",
+    "AuditParser",
+    "BudgetViolation",
+    "FailureCode",
+    "IndependentBudgetAuditor",
+]

agents/auditor/budget_auditor.py

@@ -0,0 +1,91 @@
+"""Independent Budget Auditor implementation for financial verification."""
+
+from typing import Optional, Any
+
+from agents.base import BaseAgent
+from agents.auditor.schema import AuditReport, BudgetViolation, FailureCode
+from agents.auditor.prompts import AUDITOR_SYSTEM_PROMPT, format_auditor_user_prompt
+from agents.auditor.parser import AuditParser
+from framework.llms.base import BaseLLM, Message
+
+
+class IndependentBudgetAuditor(BaseAgent):
+    """An independent budget auditing agent with zero shared memory of reflection state."""
+
+    def __init__(self, llm: BaseLLM, max_retries: int = 2):
+        self.llm = llm
+        self.max_retries = max_retries
+
+    def audit(
+        self,
+        scenario_prompt: str,
+        itinerary_content: str,
+        total_budget_inr: Optional[float] = None,
+        daily_budget_jpy: Optional[float] = None,
+    ) -> AuditReport:
+        """Independently audits an itinerary against scenario budget constraints.
+
+        Args:
+            scenario_prompt: Original scenario specification prompt.
+            itinerary_content: Finalized itinerary content to audit.
+            total_budget_inr: Optional overall total budget limit in INR.
+            daily_budget_jpy: Optional daily spend allowance limit in JPY.
+
+        Returns:
+            An AuditReport instance detailing PASS/BLOCK status and violations.
+        """
+        user_content = format_auditor_user_prompt(
+            scenario_prompt=scenario_prompt,
+            itinerary_content=itinerary_content,
+            total_budget_inr=total_budget_inr,
+            daily_budget_jpy=daily_budget_jpy,
+        )
+
+        messages = [
+            Message(role="system", content=AUDITOR_SYSTEM_PROMPT),
+            Message(role="user", content=user_content),
+        ]
+
+        last_error = ""
+        last_raw_response = ""
+
+        for attempt in range(self.max_retries + 1):
+            response = self.llm.generate(messages)
+            last_raw_response = response.text
+            report, parse_err = AuditParser.parse_and_validate(response.text)
+            if report is not None:
+                report.retries_attempted = attempt
+                return report
+
+            last_error = parse_err or "Unknown validation error"
+            if attempt < self.max_retries:
+                messages.append(Message(role="assistant", content=response.text))
+                messages.append(
+                    Message(
+                        role="user",
+                        content=f"Your previous response was rejected due to validation failure: '{last_error}'. Please correct your output and return ONLY a valid JSON object matching the exact schema inside ```json ... ```.",
+                    )
+                )
+
+        # Fallback default report if all retries fail
+        return AuditReport(
+            status="BLOCK",
+            audit_score=0.0,
+            violations=[
+                BudgetViolation(
+                    violation_type=FailureCode.MATH_HALLUCINATION,
+                    description=f"Auditor failed output validation: {last_error}",
+                )
+            ],
+            reasoning_summary=f"Audit failed due to model output validation failure: {last_error}",
+            audit_confidence=0.0,
+            parse_error=last_error,
+            retries_attempted=self.max_retries + 1,
+            raw_model_response=last_raw_response,
+        )
+
+    def run(self, prompt: str, **kwargs) -> Any:
+        """Standard AgentOutput wrapper for pipeline runner compatibility."""
+        itinerary = kwargs.get("itinerary_content", prompt)
+        scenario = kwargs.get("scenario_prompt", prompt)
+        return self.audit(scenario_prompt=scenario, itinerary_content=itinerary)

agents/auditor/parser.py

@@ -0,0 +1,139 @@
+"""Structured JSON output parser and validator for AuditReport."""
+
+import json
+import re
+from typing import Optional, Tuple, List, Dict, Any
+
+from agents.auditor.schema import AuditReport, BudgetViolation, FailureCode
+
+
+class AuditParser:
+    """Parser and validator for Independent Budget Auditor LLM responses."""
+
+    @staticmethod
+    def parse_and_validate(text: str) -> Tuple[Optional[AuditReport], Optional[str]]:
+        """Strictly parses and validates AuditReport from raw LLM output.
+
+        Args:
+            text: Raw response string from the model.
+
+        Returns:
+            Tuple of (AuditReport, None) on success, or (None, error_message) on failure.
+        """
+        try:
+            # 1. Extract JSON block or brace contents
+            match = re.search(r"```json\s*(\{.*?\})\s*```", text, re.DOTALL)
+            if match:
+                raw_json = match.group(1)
+            else:
+                match_brace = re.search(r"(\{.*\})", text, re.DOTALL)
+                if match_brace:
+                    raw_json = match_brace.group(1)
+                else:
+                    return None, "No JSON codeblock or object found in model output"
+
+            # 2. Parse JSON syntax
+            try:
+                data = json.loads(raw_json)
+            except Exception as ex:
+                return None, f"JSON syntax error: {str(ex)}"
+
+            if not isinstance(data, dict):
+                return None, "Root JSON payload must be an object"
+
+            # 3. Check mandatory keys
+            required_keys = [
+                "status",
+                "audit_score",
+                "violations",
+                "total_estimated_spend_inr",
+                "budget_limit_inr",
+                "variance_inr",
+                "audit_confidence",
+                "reasoning_summary",
+            ]
+            missing_keys = [k for k in required_keys if k not in data]
+            if missing_keys:
+                return None, f"Missing required top-level JSON keys: {missing_keys}"
+
+            # 4. Validate status
+            status = str(data["status"]).upper().strip()
+            if status not in ("PASS", "BLOCK"):
+                return None, f"Invalid status '{data['status']}'; must be 'PASS' or 'BLOCK'"
+
+            # 5. Validate numeric ranges
+            try:
+                audit_score = float(data["audit_score"])
+                if not (0.0 <= audit_score <= 100.0):
+                    return None, f"audit_score {audit_score} out of bounds [0.0, 100.0]"
+            except (ValueError, TypeError):
+                return None, "audit_score must be a numeric float"
+
+            try:
+                audit_confidence = float(data["audit_confidence"])
+                if not (0.0 <= audit_confidence <= 1.0):
+                    return None, f"audit_confidence {audit_confidence} out of bounds [0.0, 1.0]"
+            except (ValueError, TypeError):
+                return None, "audit_confidence must be a numeric float"
+
+            # 6. Validate violations list and failure codes
+            raw_violations = data["violations"]
+            if not isinstance(raw_violations, list):
+                return None, "'violations' must be a JSON array"
+
+            violations: List[BudgetViolation] = []
+            for idx, v in enumerate(raw_violations):
+                if not isinstance(v, dict):
+                    return None, f"violation at index {idx} must be an object"
+
+                v_type = v.get("violation_type")
+                if not v_type or v_type not in FailureCode.ALL_CODES:
+                    return None, f"violation at index {idx} has invalid type '{v_type}'; must be one of {FailureCode.ALL_CODES}"
+
+                description = v.get("description")
+                if not description or not str(description).strip():
+                    return None, f"violation at index {idx} is missing a description"
+
+                try:
+                    disc = float(v.get("estimated_discrepancy_inr", 0.0))
+                except (ValueError, TypeError):
+                    disc = 0.0
+
+                affected_days = []
+                if "affected_days" in v and isinstance(v["affected_days"], list):
+                    for d in v["affected_days"]:
+                        try:
+                            affected_days.append(int(d))
+                        except (ValueError, TypeError):
+                            pass
+
+                violations.append(
+                    BudgetViolation(
+                        violation_type=v_type,
+                        description=str(description).strip(),
+                        estimated_discrepancy_inr=disc,
+                        affected_days=affected_days,
+                    )
+                )
+
+            # 7. Validate Status vs Violations Consistency
+            if status == "PASS" and len(violations) > 0:
+                return None, f"Status is 'PASS' but violations array is non-empty ({len(violations)} violations specified)"
+            if status == "BLOCK" and len(violations) == 0:
+                return None, "Status is 'BLOCK' but violations array is empty (must list at least 1 violation)"
+
+            return (
+                AuditReport(
+                    status=status,
+                    audit_score=audit_score,
+                    violations=violations,
+                    total_estimated_spend_inr=float(data.get("total_estimated_spend_inr", 0.0)),
+                    budget_limit_inr=float(data.get("budget_limit_inr", 0.0)),
+                    variance_inr=float(data.get("variance_inr", 0.0)),
+                    audit_confidence=audit_confidence,
+                    reasoning_summary=str(data.get("reasoning_summary", "")),
+                ),
+                None,
+            )
+        except Exception as e:
+            return None, f"Unexpected error parsing audit JSON: {str(e)}"

agents/auditor/prompts.py

@@ -0,0 +1,64 @@
+"""System and user prompts for the Independent Budget Auditor."""
+
+AUDITOR_SYSTEM_PROMPT = """You are an Independent Financial Budget Auditor for travel itineraries.
+Your sole job is to independently inspect a finalized travel itinerary against the given scenario requirements and verify financial compliance.
+
+CRITICAL RULES:
+1. You are READ-ONLY. You MUST NOT rewrite, edit, or summarize the itinerary.
+2. You MUST NOT negotiate constraints or assume unstated discounts.
+3. Be strict, precise, and objective.
+
+You must check for four specific failure types:
+1. MATH_HALLUCINATION: The itinerary claims to save money (e.g. "taking highway bus saves ₹20,000") without providing verifiable pricing arithmetic or explicit itemized breakdowns.
+2. DAILY_OVERRUN: The sum of food, local transit, and activities on any given day exceeds the specified daily budget allowance limit.
+3. HIDDEN_OVERHEAD: Essential travel expenses (e.g. airport return transfer, luggage lockers, mandatory temple/attraction entry fees, local IC card top-ups) are omitted to artificially appear under budget.
+4. ANCHOR_MUTATION: Pre-paid, locked, or non-refundable bookings (e.g. specific hotel stays or return flights) are altered, canceled, or replaced without authorization.
+
+OUTPUT FORMAT:
+You MUST respond ONLY with a valid JSON object wrapped in ```json ... ``` with the following keys:
+{
+  "status": "PASS" | "BLOCK",
+  "audit_score": <float between 0.0 and 100.0>,
+  "violations": [
+    {
+      "violation_type": "MATH_HALLUCINATION" | "DAILY_OVERRUN" | "HIDDEN_OVERHEAD" | "ANCHOR_MUTATION",
+      "description": "<detailed explanation of the violation>",
+      "estimated_discrepancy_inr": <estimated discrepancy in INR float>,
+      "affected_days": [<list of integer day numbers>]
+    }
+  ],
+  "total_estimated_spend_inr": <float>,
+  "budget_limit_inr": <float>,
+  "variance_inr": <float, positive means under budget, negative means overspend>,
+  "audit_confidence": <float between 0.0 and 1.0>,
+  "reasoning_summary": "<brief summary of findings>"
+}
+"""
+
+
+def format_auditor_user_prompt(
+    scenario_prompt: str,
+    itinerary_content: str,
+    total_budget_inr: float = None,
+    daily_budget_jpy: float = None,
+) -> str:
+    """Formats the user input text for the auditor.
+
+    Args:
+        scenario_prompt: Scenario specification prompt.
+        itinerary_content: Finalized itinerary to audit.
+        total_budget_inr: Optional overall total budget in INR.
+        daily_budget_jpy: Optional daily spend allowance limit in JPY.
+
+    Returns:
+        Formatted user prompt string.
+    """
+    user_content = f"### SCENARIO PROMPT\n{scenario_prompt}\n\n"
+    if total_budget_inr is not None:
+        user_content += f"TOTAL BUDGET LIMIT (INR): ₹{total_budget_inr:,.2f}\n"
+    if daily_budget_jpy is not None:
+        user_content += f"DAILY ALLOWANCE LIMIT (JPY): ¥{daily_budget_jpy:,.2f}\n"
+
+    user_content += f"\n### FINAL ITINERARY TO AUDIT\n{itinerary_content}\n\n"
+    user_content += "Perform your independent budget audit and return JSON only."
+    return user_content

agents/auditor/schema.py

@@ -0,0 +1,67 @@
+"""Data contracts for the Independent Budget Auditor."""
+
+from dataclasses import dataclass, field
+from typing import List, Optional, Dict, Any
+
+
+class FailureCode:
+    MATH_HALLUCINATION = "MATH_HALLUCINATION"
+    DAILY_OVERRUN = "DAILY_OVERRUN"
+    HIDDEN_OVERHEAD = "HIDDEN_OVERHEAD"
+    ANCHOR_MUTATION = "ANCHOR_MUTATION"
+
+    ALL_CODES = [MATH_HALLUCINATION, DAILY_OVERRUN, HIDDEN_OVERHEAD, ANCHOR_MUTATION]
+
+
+@dataclass
+class BudgetViolation:
+    violation_type: str  # One of FailureCode.ALL_CODES
+    description: str
+    estimated_discrepancy_inr: float = 0.0
+    affected_days: List[int] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "violation_type": self.violation_type,
+            "description": self.description,
+            "estimated_discrepancy_inr": self.estimated_discrepancy_inr,
+            "affected_days": self.affected_days,
+        }
+
+
+@dataclass
+class AuditReport:
+    status: str  # "PASS" or "BLOCK"
+    audit_score: float  # 0.0 to 100.0
+    violations: List[BudgetViolation] = field(default_factory=list)
+    total_estimated_spend_inr: float = 0.0
+    budget_limit_inr: float = 0.0
+    variance_inr: float = 0.0
+    audit_confidence: float = 1.0
+    reasoning_summary: str = ""
+    parse_error: Optional[str] = None
+    retries_attempted: int = 0
+    raw_model_response: Optional[str] = None
+
+    @property
+    def passed(self) -> bool:
+        return self.status.upper() == "PASS"
+
+    def to_dict(self) -> Dict[str, Any]:
+        res = {
+            "status": self.status,
+            "passed": self.passed,
+            "audit_score": self.audit_score,
+            "violations": [v.to_dict() for v in self.violations],
+            "total_estimated_spend_inr": self.total_estimated_spend_inr,
+            "budget_limit_inr": self.budget_limit_inr,
+            "variance_inr": self.variance_inr,
+            "audit_confidence": self.audit_confidence,
+            "reasoning_summary": self.reasoning_summary,
+            "retries_attempted": self.retries_attempted,
+        }
+        if self.parse_error:
+            res["parse_error"] = self.parse_error
+        if self.raw_model_response:
+            res["raw_model_response"] = self.raw_model_response
+        return res

agents/base.py

@@ -0,0 +1,20 @@
+"""Abstract base agent class definition."""
+
+from abc import ABC, abstractmethod
+from framework.models import AgentOutput
+
+
+class BaseAgent(ABC):
+    """Abstract base class defining the standard interface for all agents."""
+
+    @abstractmethod
+    def run(self, prompt: str) -> AgentOutput:
+        """Executes the agent's primary decision/generation task.
+
+        Args:
+            prompt: The user request/scenario prompt.
+
+        Returns:
+            An AgentOutput dataclass containing generated content and metadata.
+        """
+        pass

agents/reflection/__init__.py

@@ -0,0 +1,3 @@
+"""Reflection agent package."""
+
+from agents.reflection.agent import ReflectionAgent

agents/reflection/agent.py

@@ -0,0 +1,77 @@
+"""Reflection agent implementation for auditing and critiquing planned itineraries."""
+
+from typing import Optional
+from langfuse import observe
+from framework.llms import BaseLLM, Message
+from framework.models import AgentOutput
+from framework.memory import BaseSessionMemory
+from agents.base import BaseAgent
+from .prompts import REFLECTION_SYSTEM_PROMPT
+
+
+class ReflectionAgent(BaseAgent):
+    """An agent that critiques travel itineraries against traveler preferences and constraints."""
+
+    def __init__(self, llm: BaseLLM):
+        """Initializes the ReflectionAgent.
+
+        Args:
+            llm: The LLM client wrapper to use for auditing the itinerary.
+        """
+        self.llm = llm
+
+    @observe(name="reflection-agent")
+    def run(self, prompt: str) -> AgentOutput:
+        """Fallback wrapper to satisfy BaseAgent abstract interface.
+
+        Args:
+            prompt: A combined text containing request details and itinerary to critique.
+
+        Returns:
+            An AgentOutput containing the critique text.
+        """
+        messages = [
+            Message(role="system", content=REFLECTION_SYSTEM_PROMPT),
+            Message(role="user", content=prompt),
+        ]
+        response = self.llm.generate(messages)
+
+        return AgentOutput(
+            content=response.text,
+            metadata={
+                "agent": self.__class__.__name__,
+                "llm": type(self.llm).__name__,
+            },
+        )
+
+    def reflect(
+        self,
+        prompt: str,
+        itinerary: str,
+        session_memory: Optional[BaseSessionMemory] = None,
+    ) -> AgentOutput:
+        """Audits a travel plan and returns structured critiques.
+
+        Args:
+            prompt: The original user travel request/scenario prompt.
+            itinerary: The draft itinerary to evaluate.
+            session_memory: Optional active session memory context.
+
+        Returns:
+            An AgentOutput containing critiques or 'ITINERARY APPROVED'.
+        """
+        memory_str = ""
+        if session_memory:
+            memory_str = (
+                "### CURRENT TRAVELER SESSION STATE:\n"
+                f"{session_memory.to_yaml()}\n"
+                "==================================================\n\n"
+            )
+
+        user_content = (
+            f"{memory_str}"
+            f"### ORIGINAL USER PROMPT:\n{prompt}\n\n"
+            f"### DRAFT ITINERARY TO AUDIT:\n{itinerary}\n"
+        )
+
+        return self.run(user_content)

agents/reflection/prompts.py

@@ -0,0 +1,18 @@
+"""System prompt instructions for the Reflection Agent."""
+
+REFLECTION_SYSTEM_PROMPT = (
+    "You are a critical travel reflection assistant.\n"
+    "Your job is to evaluate draft travel itineraries against the user's constraints, preferences, "
+    "and work schedules, identifying any issues, inefficiencies, or rule violations.\n\n"
+    "Analyze the plan for:\n"
+    "1. Constraint Violations: Exceeding budget caps, duration mismatch, missing destinations.\n"
+    "2. Schedule Clashes: Commits during remote work/meeting hours or fails to account for timezone shifts.\n"
+    "3. Routing Inefficiencies: Geographic backtracking or excessive travel times.\n"
+    "4. Personalization Gaps: Mismatch with traveler interests, walking tolerance, or housing styles.\n\n"
+    "CRITICAL RULES:\n"
+    "- Do NOT request revisions for minor cosmetic, stylistic, or wording preferences.\n"
+    "- If the itinerary satisfies all hard constraints, budget limits, schedule blocks, must-visit destinations, and contains only cosmetic improvements, you MUST reply ONLY with 'ITINERARY APPROVED'.\n"
+    "- Only raise critiques for true, concrete violations (e.g. over-budget, meeting conflicts, missing cities, severe backtracking/routing errors).\n\n"
+    "Structure your feedback clearly, listing critical issues that must be corrected. "
+    "If the itinerary is fully correct and satisfies all constraints, reply ONLY with 'ITINERARY APPROVED'."
+)

agents/research/__init__.py

@@ -0,0 +1,4 @@
+"""Research agent package."""
+
+from agents.research.agent import ResearchAgent
+from agents.research.planner import ResearchPlanner

agents/research/agent.py

@@ -0,0 +1,45 @@
+from langfuse import observe
+
+from framework.llms import BaseLLM, Message
+from framework.models import AgentOutput
+from agents.base import BaseAgent
+from .prompts import RESEARCH_SYSTEM_PROMPT
+
+
+class ResearchAgent(BaseAgent):
+    """A travel research assistant agent designed to retrieve factual answers."""
+
+    def __init__(self, llm: BaseLLM):
+        """Initializes the ResearchAgent.
+
+        Args:
+            llm: The LLM client wrapper to use for reasoning and answering questions.
+        """
+        self.llm = llm
+
+    @observe(name="research-agent")
+    def run(self, prompt: str) -> AgentOutput:
+        """Runs the research agent to answer a specific factual query.
+
+        Args:
+            prompt: The specific question/query (e.g. transit time, price, schedule).
+
+        Returns:
+            An AgentOutput containing the research answer.
+        """
+        messages = [
+            Message(role="system", content=RESEARCH_SYSTEM_PROMPT),
+            Message(role="user", content=prompt),
+        ]
+
+        response = self.llm.generate(messages)
+
+        metadata = {
+            "agent": self.__class__.__name__,
+            "llm": type(self.llm).__name__,
+        }
+
+        return AgentOutput(
+            content=response.text,
+            metadata=metadata,
+        )

agents/research/planner.py

@@ -0,0 +1,52 @@
+"""Research planner for generating travel fact queries to research."""
+
+from framework.llms import BaseLLM, Message
+from framework.utils import parse_json_markdown
+
+from langfuse import observe
+
+
+class ResearchPlanner:
+    """Decides what factual details need to be researched based on a user's travel request."""
+
+    def __init__(self, llm: BaseLLM):
+        """Initializes the ResearchPlanner.
+
+        Args:
+            llm: The LLM client wrapper to use for generating queries.
+        """
+        self.llm = llm
+
+    @observe(name="research-planner")
+    def plan_queries(self, prompt: str) -> list[str]:
+        """Generates up to 3 factual queries to guide research.
+
+        Args:
+            prompt: The user request/scenario prompt.
+
+        Returns:
+            A list of query strings.
+        """
+        system_instruction = (
+            "You are a helpful travel planning assistant.\n"
+            "Given the user's travel request, identify up to 3 critical factual questions "
+            "(e.g., transit times between cities, lodging prices, attraction opening hours or closures, "
+            "seasonal forecasts) that should be researched first to make the itinerary realistic.\n"
+            "Respond ONLY with a valid JSON list of query strings (e.g., [\"Kyoto to Tokyo train duration\", \"Shinjuku Gyoen opening hours\"]). "
+            "If no specific factual research is required, respond with []."
+        )
+        messages = [
+            Message(role="system", content=system_instruction),
+            Message(role="user", content=prompt),
+        ]
+        
+        queries = []
+        try:
+            response = self.llm.generate(messages)
+            parsed = parse_json_markdown(response.text)
+            if isinstance(parsed, list):
+                queries = [str(q) for q in parsed]
+        except Exception:
+            pass
+
+        return queries

agents/research/prompts.py

@@ -0,0 +1,14 @@
+"""System prompt instructions for the Research Agent."""
+
+RESEARCH_SYSTEM_PROMPT = (
+    "You are a professional travel research assistant.\n"
+    "Your primary goal is to answer travel research questions as accurately, factually, "
+    "and realistically as possible based on historical data and travel knowledge.\n"
+    "Focus on answering questions about:\n"
+    "1. Transit options, durations, routes, and connections (e.g. Kyoto to Hiroshima trains).\n"
+    "2. Lodging prices, hostel locations, and standard booking policies.\n"
+    "3. Attraction details, ticket prices, opening hours, and scheduled closures.\n"
+    "4. Weather patterns, seasonal advice, and local safety updates.\n\n"
+    "Be precise and cite standard sources or estimates where appropriate. Keep your response "
+    "concise and focused on answering the query factually."
+)

agents/support/__init__.py

@@ -0,0 +1,5 @@
+"""Support-domain agents."""
+
+from .triage_agent import SupportTriageAgent
+
+__all__ = ["SupportTriageAgent"]