evalrun 0.4.0__py3-none-any.whl

framework/evaluation/prompts/adaptability.py

@@ -0,0 +1,57 @@
+"""Prompt builder for Adaptability dimension."""
+
+from typing import List
+from framework.models import AgentOutput, Benchmark
+from framework.evaluation.prompts.base import build_llm_judge_prompt
+from framework.llms import Message
+
+# Detailed evaluation rubric for Adaptability
+ADAPTABILITY_RUBRIC = (
+    "Main Question: When circumstances change, how well does the agent revise "
+    "the plan while preserving the overall experience?\n\n"
+    "Note on scope: Evaluate only how well the agent adapts the itinerary to changes, "
+    "disruptions, or new constraints introduced mid-travel. Do not evaluate baseline "
+    "planning quality, factual correctness of attractions, or static user interests, "
+    "as those are evaluated separately in other dimensions.\n\n"
+    "Evaluate the adaptability quality of the travel itinerary across these three core aspects:\n\n"
+    "1. Constraint Adaptation:\n"
+    "   Assess how effectively the agent revises the itinerary to accommodate modified or new constraints.\n"
+    "   Favor revisions that make the smallest necessary changes while preserving the overall itinerary whenever practical.\n"
+    "   Consider factors such as: adapting to budget reductions, timeline shifts, or transit cancellations.\n\n"
+    "2. Experience Preservation:\n"
+    "   Assess how well the revised plan preserves the original travel goals, style, and interests of the traveler.\n"
+    "   Consider factors such as: finding alternative activities of similar theme/vibe rather than replacing them with generic tourist options.\n\n"
+    "3. Impact Mitigation:\n"
+    "   Assess whether the revisions minimize cascading disruptions to the rest of the itinerary.\n"
+    "   Consider factors such as: avoiding unnecessary cancellations of existing bookings and keeping alterations localized where possible.\n\n"
+    "### Scoring Guidance:\n"
+    "- 90–100: Excellent adaptability, smoothly resolving changes with zero friction and high experience preservation.\n"
+    "- 75–89: Good adaptability resolving disruptions with minor inefficiencies or pacing compromises.\n"
+    "- 50–74: Moderate adaptability; recovers from changes but sacrifices multiple travel goals or introduces excessive pacing stress.\n"
+    "- 25–49: Poor adaptability; fails to meet new constraints or completely loses the traveler's preferences during revision.\n"
+    "- 0–24: Not adaptable; fails to address changed circumstances entirely.\n\n"
+    "Assign the score holistically across all three aspects. "
+    "Do not average the aspects mechanically unless the evidence clearly supports doing so."
+)
+
+
+def build_adaptability_prompt(
+    benchmark: Benchmark, output: AgentOutput
+) -> List[Message]:
+    """Generates LLM judge messages for adaptability quality assessment.
+
+    Args:
+        benchmark: The scenario benchmark details.
+        output: The agent response output.
+
+    Returns:
+        List of prompt Message structures.
+    """
+    system_objective = (
+        "You are an objective AI evaluator checking the adaptability quality "
+        "of an agent's output itinerary against a benchmark scenario."
+    )
+
+    return build_llm_judge_prompt(
+        system_objective, ADAPTABILITY_RUBRIC, benchmark, output
+    )

framework/evaluation/prompts/base.py

@@ -0,0 +1,52 @@
+"""Base template components and helpers for building evaluation prompts."""
+
+from typing import List
+from framework.models import AgentOutput, Benchmark
+from framework.llms import Message
+
+# Standard JSON schema instructions enforced across all LLM judges
+JSON_RESPONSE_SCHEMA = (
+    "Return a JSON object containing exactly these fields:\n"
+    "{\n"
+    '  "score": <integer from 0 to 100 representing the grade>,\n'
+    '  "reason": "<detailed justification detailing your assessment>"\n'
+    "}\n"
+    "Do not include any markdown syntax, explanations, or code blocks outside the JSON."
+)
+
+
+def build_llm_judge_prompt(
+    system_objective: str,
+    criteria_rubric: str,
+    benchmark: Benchmark,
+    output: AgentOutput,
+) -> List[Message]:
+    """Universal prompt builder for LLM judges.
+
+    Ensures consistent formatting and JSON output constraints.
+
+    Args:
+        system_objective: The role and primary mission of the judge.
+        criteria_rubric: Rubric details specific to the evaluation dimension.
+        benchmark: The benchmark scenario details.
+        output: The agent response output.
+
+    Returns:
+        A list of chat Message structures.
+    """
+    system_instruction = (
+        f"{system_objective}\n\n"
+        f"### Rubric / Criteria:\n{criteria_rubric}\n\n"
+        f"{JSON_RESPONSE_SCHEMA}"
+    )
+
+    user_prompt = (
+        f"Benchmark Name: {benchmark.name}\n"
+        f"Benchmark Description: {benchmark.description}\n\n"
+        f"### Agent Output To Evaluate:\n{output.content}\n"
+    )
+
+    return [
+        Message(role="system", content=system_instruction),
+        Message(role="user", content=user_prompt),
+    ]

framework/evaluation/prompts/constraint.py

@@ -0,0 +1,41 @@
+"""Prompt builder for Constraint Satisfaction dimension."""
+
+from typing import List
+import yaml
+from framework.models import AgentOutput, Benchmark
+from framework.evaluation.prompts.base import build_llm_judge_prompt
+from framework.llms import Message
+
+
+def build_constraint_prompt(
+    benchmark: Benchmark, output: AgentOutput
+) -> List[Message]:
+    """Generates LLM judge messages for constraint satisfaction checking.
+
+    Args:
+        benchmark: The scenario benchmark details.
+        output: The agent response output.
+
+    Returns:
+        List of prompt Message structures.
+    """
+    system_objective = (
+        "You are an objective AI evaluator checking if an agent's output satisfies "
+        "the explicit constraints of a benchmark scenario."
+    )
+
+    constraints_str = (
+        yaml.dump(benchmark.constraints, default_flow_style=False)
+        if benchmark.constraints
+        else "No explicit constraints defined."
+    )
+
+    criteria_rubric = (
+        "Analyze the agent output and determine if every constraint listed "
+        "below is satisfied.\n"
+        f"Explicit constraints to verify:\n{constraints_str}"
+    )
+
+    return build_llm_judge_prompt(
+        system_objective, criteria_rubric, benchmark, output
+    )

framework/evaluation/prompts/information_accuracy.py

@@ -0,0 +1,79 @@
+"""Prompt builder for Information Accuracy dimension."""
+
+from typing import List
+from framework.models import AgentOutput, Benchmark
+from framework.evaluation.prompts.base import JSON_RESPONSE_SCHEMA
+from framework.llms import Message
+from framework.verification.models import VerificationReport
+
+# Detailed evaluation rubric for Information Accuracy
+INFORMATION_ACCURACY_RUBRIC = (
+    "Main Question: Considering only information accuracy, can the factual claims "
+    "in this itinerary be trusted?\n\n"
+    "Note on scope: Evaluate only the truthfulness, precision, and validity of factual claims "
+    "(e.g., hotel prices, attraction existence, opening hours, transit times) based on the provided "
+    "verification evidence. Do not evaluate planning quality, constraint satisfaction, personalization, "
+    "or adaptability, as those are evaluated separately in other dimensions.\n\n"
+    "Evaluate the information accuracy of the travel itinerary based on this verification evidence across three core aspects:\n\n"
+    "1. Verification Accuracy:\n"
+    "   Assess the proportion of claims that were verified as true.\n"
+    "   Consider factors such as: the number of refuted claims, particularly those that are critical to the trip.\n\n"
+    "2. Severity of Refutations:\n"
+    "   Assess the real-world impact of incorrect claims on the traveler's experience.\n"
+    "   Treat critical factual inaccuracies—such as incorrect visa/entry rules, non-existent attractions, "
+    "   closed attractions, or severe routing/geographical errors—as significantly more critical "
+    "   than minor discrepancies (like small price or budget differences) when assessing the scoring deductions.\n\n"
+    "3. Handling of Unverifiable or Missing Facts:\n"
+    "   Assess how the verifier findings (unknown or not found claims) impact the overall trustworthiness of the itinerary.\n\n"
+    "### Scoring Guidance:\n"
+    "- 90–100: Highly accurate. All claims are verified, or only minor, trivial discrepancies exist.\n"
+    "- 75–89: Generally accurate. Minor price or timing errors that do not disrupt the trip flow.\n"
+    "- 50–74: Noticeable factual errors. Multiple refuted claims or a single critical refutation (e.g., closed attraction).\n"
+    "- 25–49: Major inaccuracies. Key transport links or multiple attractions are closed/incorrect.\n"
+    "- 0–24: Highly untrustworthy. Multiple severe refutations, non-existent locations, or major fictional data.\n\n"
+    "Assign the score holistically across all three aspects. "
+    "Do not average the aspects mechanically unless the evidence clearly supports doing so."
+)
+
+
+def build_information_accuracy_prompt(
+    benchmark: Benchmark,
+    output: AgentOutput,
+    report: VerificationReport,
+) -> List[Message]:
+    """Generates LLM judge messages for information accuracy assessment.
+
+    Args:
+        benchmark: The scenario benchmark details.
+        output: The agent response output.
+        report: The VerificationReport containing factual verification results.
+
+    Returns:
+        List of prompt Message structures.
+    """
+    system_instruction = (
+        "You are an objective AI evaluator checking the information accuracy "
+        "of an agent's output itinerary against a benchmark scenario and "
+        "provided verification evidence.\n\n"
+        f"### Rubric / Criteria:\n{INFORMATION_ACCURACY_RUBRIC}\n\n"
+        f"{JSON_RESPONSE_SCHEMA}"
+    )
+
+    evidence_context = report.to_markdown()
+
+    user_prompt = (
+        f"VERIFICATION EVIDENCE FINDINGS:\n"
+        f"================================\n"
+        f"{evidence_context}\n"
+        f"================================\n\n"
+        f"Please evaluate the original itinerary below considering this factual evidence.\n\n"
+        f"Benchmark Name: {benchmark.name}\n"
+        f"Benchmark Description: {benchmark.description}\n\n"
+        f"### Agent Output To Evaluate:\n{output.content}\n"
+    )
+
+    return [
+        Message(role="system", content=system_instruction),
+        Message(role="user", content=user_prompt),
+    ]
+

framework/evaluation/prompts/personalization.py

@@ -0,0 +1,57 @@
+"""Prompt builder for Personalization dimension."""
+
+from typing import List
+from framework.models import AgentOutput, Benchmark
+from framework.evaluation.prompts.base import build_llm_judge_prompt
+from framework.llms import Message
+
+# Detailed evaluation rubric for Personalization
+PERSONALIZATION_RUBRIC = (
+    "Main Question: Considering only personalization, does the itinerary feel "
+    "specifically tailored to this traveler's stated preferences, interests, and profile?\n\n"
+    "Note on scope: Evaluate only how well the itinerary aligns with the traveler's stated preferences, "
+    "goals, interests, and style. Do not evaluate route efficiency, daily pacing, factual correctness "
+    "of attractions, or strict duration constraints, as those are evaluated separately in other dimensions.\n\n"
+    "Evaluate the personalization quality of the travel itinerary across these three core aspects:\n\n"
+    "1. Preference Alignment:\n"
+    "   Assess whether the activities, sightseeing spots, and recommendations match the traveler's listed interests.\n"
+    "   Consider both explicit preferences stated by the traveler and reasonable implications of those preferences "
+    "   (e.g., backpacking implies budget lodging and public transit), but do not invent unsupported interests.\n"
+    "   Consider factors such as: incorporating stated hobbies (e.g., photography, thrift shopping, café culture) and aligning with the backpacking travel style.\n\n"
+    "2. Context & Constraints Integration:\n"
+    "   Assess whether lodging selection and pacing integrate cleanly with the traveler's daily constraints.\n"
+    "   Consider factors such as: scheduling around remote work windows and respecting walking preferences or physical limits.\n\n"
+    "3. Recommendation Relevance:\n"
+    "   Assess whether the recommendations are highly relevant to this specific traveler's profile rather than boilerplate tourist stops.\n"
+    "   Consider factors such as: localized recommendations that directly serve the traveler's stated and implied interests.\n\n"
+    "### Scoring Guidance:\n"
+    "- 90–100: Excellent personalization, highly tailored to the traveler's profile with deep alignment.\n"
+    "- 75–89: Good personalization matching major interests, with minor generic elements.\n"
+    "- 50–74: Moderate personalization; feels somewhat generic or overlooks multiple stated preferences.\n"
+    "- 25–49: Poor personalization; ignores key interests or remote work requirements.\n"
+    "- 0–24: Not personalized at all; entirely boilerplate itinerary.\n\n"
+    "Assign the score holistically across all three aspects. "
+    "Do not average the aspects mechanically unless the evidence clearly supports doing so."
+)
+
+
+def build_personalization_prompt(
+    benchmark: Benchmark, output: AgentOutput
+) -> List[Message]:
+    """Generates LLM judge messages for personalization quality assessment.
+
+    Args:
+        benchmark: The scenario benchmark details.
+        output: The agent response output.
+
+    Returns:
+        List of prompt Message structures.
+    """
+    system_objective = (
+        "You are an objective AI evaluator checking the personalization quality "
+        "of an agent's output itinerary against a benchmark scenario."
+    )
+
+    return build_llm_judge_prompt(
+        system_objective, PERSONALIZATION_RUBRIC, benchmark, output
+    )

framework/evaluation/prompts/planning.py

@@ -0,0 +1,61 @@
+"""Prompt builder for Planning Quality dimension."""
+
+from typing import List
+from framework.models import AgentOutput, Benchmark
+from framework.evaluation.prompts.base import build_llm_judge_prompt
+from framework.llms import Message
+
+# Detailed evaluation rubric for Planning Quality
+PLANNING_RUBRIC = (
+    "Main Question: Considering only planning quality, is this itinerary organized in the best possible way for the traveler?\n\n"
+    "Note on scope: Evaluate only the organization and structure of the itinerary. "
+    "Do not consider factual accuracy, personalization, or whether explicit user constraints were satisfied, "
+    "as those are evaluated separately in other dimensions.\n\n"
+    "Evaluate the planning quality of the travel itinerary across these six core aspects:\n\n"
+    "1. Route Logic & Geographic Efficiency:\n"
+    "   Assess whether the itinerary minimizes unnecessary travel while maintaining a logical progression.\n"
+    "   Consider factors such as: unnecessary backtracking, revisiting cities, or inefficient long-distance jumps.\n\n"
+    "2. Accommodation Stability:\n"
+    "   Assess whether stays are grouped effectively to avoid unnecessary overhead.\n"
+    "   Consider factors such as: changing hotels/guesthouses every night, daily packing/unpacking, or not grouping activities around the active lodging node.\n\n"
+    "3. Daily Pacing & Balance:\n"
+    "   Assess whether the activities are realistic and pacing feels balanced.\n"
+    "   Consider factors such as: overloading single days with 5+ major sites, not leaving transit buffers, or conflicting with remote work hours.\n\n"
+    "4. Time & Transit Utilization:\n"
+    "   Assess whether transit schedules are optimized for convenience and cost.\n"
+    "   Consider factors such as: losing full days to slow daylight transits, choosing complex routes with tight/unrealistic transfer connections.\n\n"
+    "5. Trip Flow & Cohesion:\n"
+    "   Assess whether the sequence of cities, theme, and travel flow feel natural and coherent rather than disjointed, scattered, or random.\n\n"
+    "6. Trade-off Optimization:\n"
+    "   Assess whether the plan demonstrates balanced decisions between travel convenience, fatigue, cost, and traveler experience.\n\n"
+    "### Scoring Guidance:\n"
+    "- 90–100: Excellent planning with only minor improvements possible.\n"
+    "- 75–89: Good planning with some inefficiencies.\n"
+    "- 50–74: Noticeable planning problems that reduce the overall experience.\n"
+    "- 25–49: Major inefficiencies or unrealistic daily pacing.\n"
+    "- 0–24: Poorly organized itinerary.\n\n"
+    "Assign the score holistically across all six aspects. "
+    "Do not average the aspects mechanically unless the evidence clearly supports doing so."
+)
+
+
+def build_planning_prompt(
+    benchmark: Benchmark, output: AgentOutput
+) -> List[Message]:
+    """Generates LLM judge messages for planning quality assessment.
+
+    Args:
+        benchmark: The scenario benchmark details.
+        output: The agent response output.
+
+    Returns:
+        List of prompt Message structures.
+    """
+    system_objective = (
+        "You are an objective AI evaluator checking the planning quality "
+        "of an agent's output itinerary against a benchmark scenario."
+    )
+
+    return build_llm_judge_prompt(
+        system_objective, PLANNING_RUBRIC, benchmark, output
+    )