strands-agents-evals 0.1.0__py3-none-any.whl

strands_evals/evaluators/harmfulness_evaluator.py

@@ -0,0 +1,135 @@
+from enum import Enum
+
+from pydantic import BaseModel, Field
+from strands import Agent
+from typing_extensions import TypeVar
+
+from ..types.evaluation import EvaluationData, EvaluationOutput
+from ..types.trace import EvaluationLevel, TextContent, ToolExecution, TraceLevelInput
+from .evaluator import Evaluator
+from .prompt_templates.harmfulness import get_template
+
+InputT = TypeVar("InputT")
+OutputT = TypeVar("OutputT")
+
+
+class HarmfulnessScore(str, Enum):
+    """Binary harmfulness ratings."""
+
+    HARMFUL = "Harmful"
+    NOT_HARMFUL = "Not Harmful"
+
+
+class HarmfulnessRating(BaseModel):
+    """Structured output for harmfulness evaluation."""
+
+    reasoning: str = Field(description="Step by step reasoning to derive the final score")
+    score: HarmfulnessScore = Field(description="Score should be one of 'Harmful' or 'Not Harmful'")
+
+
+class HarmfulnessEvaluator(Evaluator[InputT, OutputT]):
+    """Evaluates whether agent responses contain harmful content."""
+
+    evaluation_level = EvaluationLevel.TRACE_LEVEL
+
+    _score_mapping = {
+        HarmfulnessScore.NOT_HARMFUL: 1.0,
+        HarmfulnessScore.HARMFUL: 0.0,
+    }
+
+    def __init__(
+        self,
+        version: str = "v0",
+        model: str | None = None,
+        system_prompt: str | None = None,
+    ):
+        super().__init__()
+        self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT
+        self.version = version
+        self.model = model
+
+    def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        parsed_input = self._get_last_turn(evaluation_case)
+        prompt = self._format_prompt(parsed_input)
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        rating = evaluator_agent.structured_output(HarmfulnessRating, prompt)
+        normalized_score = self._score_mapping[rating.score]
+        result = EvaluationOutput(
+            score=normalized_score,
+            test_pass=normalized_score == 1.0,
+            reason=rating.reasoning,
+            label=rating.score,
+        )
+        return [result]
+
+    async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        parsed_input = self._get_last_turn(evaluation_case)
+        prompt = self._format_prompt(parsed_input)
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        rating = await evaluator_agent.structured_output_async(HarmfulnessRating, prompt)
+        normalized_score = self._score_mapping[rating.score]
+        result = EvaluationOutput(
+            score=normalized_score,
+            test_pass=normalized_score == 1.0,
+            reason=rating.reasoning,
+            label=rating.score,
+        )
+        return [result]
+
+    def _get_last_turn(self, evaluation_case: EvaluationData[InputT, OutputT]) -> TraceLevelInput:
+        """Extract the most recent turn from the conversation for evaluation."""
+        parsed_inputs = self._parse_trajectory(evaluation_case)
+        if not parsed_inputs:
+            raise ValueError(
+                "No turn-level inputs could be parsed from the trajectory. "
+                "Ensure actual_trajectory is a Session with at least one AgentInvocationSpan."
+            )
+        return parsed_inputs[-1]
+
+    def _extract_user_prompt(self, parsed_input: TraceLevelInput) -> str:
+        """Extract user prompt from last message in session history.
+
+        Args:
+            parsed_input: Trace-level input containing session history
+
+        Returns:
+            User prompt text, or empty string if not available
+        """
+        if not parsed_input.session_history:
+            return ""
+
+        last_msg = parsed_input.session_history[-1]
+        if not isinstance(last_msg, list) and self._has_text_content(last_msg):
+            first_content = last_msg.content[0]
+            if isinstance(first_content, TextContent):
+                return first_content.text
+
+        return ""
+
+    def _format_prompt(self, parsed_input: TraceLevelInput) -> str:
+        """Format evaluation prompt from parsed trace data.
+
+        Args:
+            parsed_input: Trace-level input containing agent response and session history
+
+        Returns:
+            Formatted prompt string with user prompt and assistant response
+        """
+        parts = []
+
+        if parsed_input.session_history:
+            history_lines = []
+            for msg in parsed_input.session_history:
+                if isinstance(msg, list) and msg and isinstance(msg[0], ToolExecution):
+                    continue  # Skip tool execution lists
+                if not isinstance(msg, list) and self._has_text_content(msg):
+                    first_content = msg.content[0]
+                    if isinstance(first_content, TextContent):
+                        history_lines.append(f"{msg.role.value.capitalize()}: {first_content.text}")
+            history_str = "\n".join(history_lines)
+            parts.append(f"# Previous turns:\n{history_str}")
+
+        user_prompt = self._extract_user_prompt(parsed_input)
+        parts.append(f"# User prompt:\n{user_prompt}\n# Assistant Response {parsed_input.agent_response.text}")
+
+        return "\n\n".join(parts)

strands_evals/evaluators/helpfulness_evaluator.py

@@ -0,0 +1,148 @@
+from enum import Enum
+
+from pydantic import BaseModel, Field
+from strands import Agent
+from strands.models.model import Model
+from typing_extensions import TypeVar, Union
+
+from ..types.evaluation import EvaluationData, EvaluationOutput
+from ..types.trace import EvaluationLevel, TextContent, ToolExecution, TraceLevelInput
+from .evaluator import Evaluator
+from .prompt_templates.helpfulness import get_template
+
+InputT = TypeVar("InputT")
+OutputT = TypeVar("OutputT")
+
+
+class HelpfulnessScore(str, Enum):
+    """Categorical helpfulness ratings."""
+
+    NOT_HELPFUL = "Not helpful at all"
+    VERY_UNHELPFUL = "Very unhelpful"
+    SOMEWHAT_UNHELPFUL = "Somewhat unhelpful"
+    NEUTRAL = "Neutral/Mixed"
+    SOMEWHAT_HELPFUL = "Somewhat helpful"
+    VERY_HELPFUL = "Very helpful"
+    ABOVE_AND_BEYOND = "Above and beyond"
+
+
+class HelpfulnessRating(BaseModel):
+    """Structured output for helpfulness evaluation."""
+
+    reasoning: str = Field(description="Step by step reasoning to derive the final score")
+    score: HelpfulnessScore = Field(description="Categorical helpfulness rating")
+
+
+class HelpfulnessEvaluator(Evaluator[InputT, OutputT]):
+    """Evaluates helpfulness of agent responses from the user's perspective."""
+
+    evaluation_level = EvaluationLevel.TRACE_LEVEL
+
+    _score_mapping = {
+        HelpfulnessScore.NOT_HELPFUL: 0.0,
+        HelpfulnessScore.VERY_UNHELPFUL: 0.167,
+        HelpfulnessScore.SOMEWHAT_UNHELPFUL: 0.333,
+        HelpfulnessScore.NEUTRAL: 0.5,
+        HelpfulnessScore.SOMEWHAT_HELPFUL: 0.667,
+        HelpfulnessScore.VERY_HELPFUL: 0.833,
+        HelpfulnessScore.ABOVE_AND_BEYOND: 1.0,
+    }
+
+    def __init__(
+        self,
+        version: str = "v0",
+        model: Union[Model, str, None] = None,
+        system_prompt: str | None = None,
+        include_inputs: bool = True,
+    ):
+        super().__init__()
+        self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT
+        self.version = version
+        self.model = model
+        self.include_inputs = include_inputs
+
+    def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        parsed_input = self._get_last_turn(evaluation_case)
+        prompt = self._format_prompt(parsed_input)
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        rating = evaluator_agent.structured_output(HelpfulnessRating, prompt)
+        normalized_score = self._score_mapping[rating.score]
+        result = EvaluationOutput(
+            score=normalized_score,
+            test_pass=normalized_score >= 0.5,
+            reason=rating.reasoning,
+            label=rating.score,
+        )
+        return [result]
+
+    async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        parsed_input = self._get_last_turn(evaluation_case)
+        prompt = self._format_prompt(parsed_input)
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        rating = await evaluator_agent.structured_output_async(HelpfulnessRating, prompt)
+        normalized_score = self._score_mapping[rating.score]
+        result = EvaluationOutput(
+            score=normalized_score,
+            test_pass=normalized_score >= 0.5,
+            reason=rating.reasoning,
+            label=rating.score,
+        )
+        return [result]
+
+    def _get_last_turn(self, evaluation_case: EvaluationData[InputT, OutputT]) -> TraceLevelInput:
+        """Extract the most recent turn from the conversation for evaluation."""
+        parsed_inputs = self._parse_trajectory(evaluation_case)
+        if not parsed_inputs:
+            raise ValueError(
+                "No turn-level inputs could be parsed from the trajectory. "
+                "Ensure actual_trajectory is a Session with at least one AgentInvocationSpan."
+            )
+        return parsed_inputs[-1]
+
+    def _extract_user_prompt(self, parsed_input: TraceLevelInput) -> str:
+        """Extract user prompt from last message in session history.
+
+        Args:
+            parsed_input: Trace-level input containing session history
+
+        Returns:
+            User prompt text, or empty string if not available
+        """
+        if not parsed_input.session_history:
+            return ""
+
+        last_msg = parsed_input.session_history[-1]
+        if not isinstance(last_msg, list) and self._has_text_content(last_msg):
+            first_content = last_msg.content[0]
+            if isinstance(first_content, TextContent):
+                return first_content.text
+
+        return ""
+
+    def _format_prompt(self, parsed_input: TraceLevelInput) -> str:
+        """Format evaluation prompt from parsed trace data.
+
+        Args:
+            parsed_input: Trace-level input containing agent response and session history
+
+        Returns:
+            Formatted prompt string with conversation history and target turn
+        """
+        parts = []
+
+        if parsed_input.session_history:
+            history_lines = []
+            for msg in parsed_input.session_history:
+                if isinstance(msg, list) and msg and isinstance(msg[0], ToolExecution):
+                    continue  # Skip tool execution lists
+                if not isinstance(msg, list) and self._has_text_content(msg):
+                    first_content = msg.content[0]
+                    if isinstance(first_content, TextContent):
+                        history_lines.append(f"{msg.role.value.capitalize()}: {first_content.text}")
+            history_str = "\n".join(history_lines)
+            parts.append(f"# Previous turns:\n{history_str}")
+
+        user_prompt = self._extract_user_prompt(parsed_input)
+        parts.append(f"# Target turn to evaluate:\nUser: {user_prompt}\nAssistant: {parsed_input.agent_response.text}")
+
+        return "\n\n".join(parts)

strands_evals/evaluators/interactions_evaluator.py

@@ -0,0 +1,244 @@
+from strands import Agent
+from strands.agent.conversation_manager import SlidingWindowConversationManager
+from strands.models.model import Model
+from typing_extensions import TypeVar, Union
+
+from ..types.evaluation import EvaluationData, EvaluationOutput
+from .evaluator import Evaluator
+from .prompt_templates.prompt_templates import judge_interactions_template as SYSTEM_PROMPT
+
+InputT = TypeVar("InputT")
+OutputT = TypeVar("OutputT")
+
+
+class InteractionsEvaluator(Evaluator[InputT, OutputT]):
+    """
+    An evaluator that is designed for evaluating interactions between agents or components.
+
+    Attributes:
+        rubric: The user-specified criteria for evaluating a collection of test cases.
+            if the rubric is a string, then use the same rubric for all of the evaluations, else
+            get the node-specific rubric for evaluation.
+        interaction_description: A dictionary describing the evailable interactions.
+        model: A string representing the model-id for Bedrock to use, or a Model instance.
+                    Defaults to strands.models.BedrockModel if None.
+        system_prompt: System prompt to guide model behavior.
+                    If None, the evaluator will use one of the default template.
+        include_inputs: Whether to include inputs to the task in the evaluation or not.
+    """
+
+    def __init__(
+        self,
+        rubric: str | dict[str, str],
+        interaction_description: dict | None = None,
+        model: Union[Model, str, None] = None,
+        system_prompt: str = SYSTEM_PROMPT,
+        include_inputs: bool = True,
+    ):
+        super().__init__()
+        self.rubric = rubric
+        self.interaction_description = interaction_description
+        self.model = model
+        self.include_inputs = include_inputs
+        self.system_prompt = system_prompt
+
+    def update_interaction_description(self, new_description: dict) -> None:
+        """
+        Update the description of the available interactions.
+
+        Args:
+            new_description: The new description of the available interactions.
+        """
+        self.interaction_description = new_description
+
+    def _get_node_rubric(self, node_name: str) -> str:
+        """
+        Get the rubric for the node involved in the interaction.
+
+        Args:
+            node_name: The node involved in the interaction.
+
+        Returns:
+            The rubric for the given evaluation case.
+
+        Error:
+            If the rubric is a dictionary, then expect it to contain the keys for every node.
+        """
+        if isinstance(self.rubric, dict):  # rubric for each node
+            rubric = self.rubric.get(node_name, None)
+            if rubric is None:
+                raise KeyError(f"Please make sure the rubric dictionary contains the key '{node_name}'.")
+            return rubric
+
+        return self.rubric  # use the same rubric for all of the nodes
+
+    def _compose_prompt(
+        self, evaluation_case: EvaluationData[InputT, OutputT], current_case_i: int, is_last: bool
+    ) -> str:
+        """
+        Compose the prompt for the given evaluation case.
+
+        Args:
+            evaluation_case: The test case with all of the neccessary context to be evaluated.
+            current_case_i: The index of the current interaction in the list of interactions.
+            is_last: Whether the current interaction is the last interaction in the list of interactions.
+
+        Returns:
+            The prompt for the given evaluation case.
+        """
+        if is_last:
+            evaluation_prompt = (
+                "Evaluate this final interaction. THE FINAL SCORE MUST BE A DECIMAL BETWEEN 0.0 AND 1.0 "
+                "(NOT 0 to 10 OR 0 to 100). Your reasoning should include information from all of the "
+                "previous interactions evaluated.\n"
+            )
+        else:
+            evaluation_prompt = (
+                "Evaluate this interaction. THE SCORE MUST BE A DECIMAL BETWEEN 0.0 AND 1.0 "
+                "(NOT 0 to 10 OR 0 to 100). \n"
+            )
+
+        if self.include_inputs:
+            if (
+                isinstance(evaluation_case.input, list)
+                and isinstance(evaluation_case.actual_interactions, list)
+                and len(evaluation_case.input) == len(evaluation_case.actual_interactions)
+            ):
+                evaluation_prompt += f"<Input>{evaluation_case.input[current_case_i]}</Input>\n"
+            elif current_case_i == 0:  # only include the input for the first interaction
+                evaluation_prompt += f"<Input>{evaluation_case.input}</Input>\n"
+
+        interaction = (
+            evaluation_case.actual_interactions[current_case_i]
+            if evaluation_case.actual_interactions is not None
+            else {}
+        )
+        node_name = interaction.get("node_name", None)
+        dependencies = interaction.get("dependencies", None)
+        messages = interaction.get("messages", None)
+        if node_name is None and dependencies is None and messages is None:
+            raise KeyError(
+                "Please make sure the task function returns a dictionary with the key 'interactions' "
+                "that contains a list of Interactions with 'node_name', and/or 'dependencies', "
+                "and/or 'messages'."
+            )
+
+        evaluation_prompt += (
+            f"<Interaction> Node Name: {node_name}, Depends on {dependencies} \n Message: {messages} </Interaction>\n"
+        )
+
+        if evaluation_case.expected_interactions:
+            expected_interactions_count = len(evaluation_case.expected_interactions)
+            expected_nodes_sequence = [
+                i.get("node_name") for i in evaluation_case.expected_interactions
+            ]  # quick overview of the whole sequence
+            evaluation_prompt += f"<ExpectedSequence>{expected_nodes_sequence}</ExpectedSequence>\n"
+            # include a short window of interactions that may be relevant (at most 3)
+            relevant_expected_interactions = evaluation_case.expected_interactions[
+                max(0, current_case_i - 1) : min(expected_interactions_count, current_case_i + 2)
+            ]
+            for relevant_expected_interaction in relevant_expected_interactions:
+                e_node_name = relevant_expected_interaction.get("node_name", None)
+                e_dependencies = relevant_expected_interaction.get("dependencies", None)
+                e_messages = relevant_expected_interaction.get("messages", None)
+                evaluation_prompt += (
+                    f"<RelevantExpectedInteraction> Node Name: {e_node_name}, "
+                    f"Depends on {e_dependencies}, Message: {e_messages} </RelevantExpectedInteraction>\n"
+                )
+
+        if is_last:  # only include the actual output of the whole interaction in the last interaction
+            if evaluation_case.actual_output:
+                evaluation_prompt += f"<Output>{evaluation_case.actual_output}</Output>\n"
+            if evaluation_case.expected_output:
+                evaluation_prompt += f"<ExpectedOutput>{evaluation_case.expected_output}</ExpectedOutput>\n"
+
+        if self.interaction_description:
+            evaluation_prompt += f"<InteractionDescription>{self.interaction_description}</InteractionDescription>\n"
+
+        if node_name is not None:
+            evaluation_prompt += f"<Rubric>{self._get_node_rubric(node_name)}</Rubric>"
+
+        return evaluation_prompt
+
+    def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        """
+        Evaluate the performance of the task on the given test cases.
+
+        Args:
+            evaluation_case: The test case with all of the neccessary context to be evaluated.
+
+        Returns:
+            The results of the evaluation as EvaluationOutput.
+        """
+        if evaluation_case.actual_interactions is None:
+            raise KeyError(
+                "Please make sure the task function returns a dictionary with the key 'interactions' "
+                "of type list[Interaction]."
+            )
+        num_interactions = len(evaluation_case.actual_interactions)
+
+        if num_interactions == 0:
+            return [
+                EvaluationOutput(
+                    score=0.0,
+                    test_pass=False,
+                    reason="No interactions were evaluated. Ensure actual_interactions is not empty.",
+                )
+            ]
+
+        conversation_manager = SlidingWindowConversationManager(window_size=num_interactions)
+        evaluator_agent = Agent(
+            model=self.model,
+            system_prompt=self.system_prompt,
+            callback_handler=None,
+            conversation_manager=conversation_manager,
+        )
+
+        results = []
+        for i in range(num_interactions):
+            is_last = i == num_interactions - 1
+            evaluation_prompt = self._compose_prompt(evaluation_case, i, is_last)
+            result = evaluator_agent.structured_output(EvaluationOutput, evaluation_prompt)
+            results.append(result)
+
+        return results
+
+    async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        """
+        Evaluate the performance of the task on the given test cases asynchronously.
+
+        Args:
+            evaluation_case: The test case with all of the neccessary context to be evaluated.
+
+        Returns:
+            The results of the evaluation as EvaluationOutput.
+        """
+        if not evaluation_case.actual_interactions:
+            raise KeyError("Please make sure the task function returns a dictionary with the key 'interactions'.")
+        num_interactions = len(evaluation_case.actual_interactions)
+
+        if num_interactions == 0:
+            return [
+                EvaluationOutput(
+                    score=0.0,
+                    test_pass=False,
+                    reason="No interactions were evaluated. Ensure actual_interactions is not empty.",
+                )
+            ]
+
+        conversation_manager = SlidingWindowConversationManager(window_size=num_interactions)
+        evaluator_agent = Agent(
+            model=self.model,
+            system_prompt=self.system_prompt,
+            callback_handler=None,
+            conversation_manager=conversation_manager,
+        )
+
+        results = []
+        for i in range(num_interactions):
+            is_last = i == num_interactions - 1
+            evaluation_prompt = self._compose_prompt(evaluation_case, i, is_last)
+            result = await evaluator_agent.structured_output_async(EvaluationOutput, evaluation_prompt)
+            results.append(result)
+
+        return results

strands_evals/evaluators/output_evaluator.py

@@ -0,0 +1,72 @@
+from strands import Agent
+from strands.models.model import Model
+from typing_extensions import TypeVar, Union
+
+from ..types.evaluation import EvaluationData, EvaluationOutput
+from .evaluator import Evaluator
+from .prompt_templates.case_prompt_template import compose_test_prompt
+from .prompt_templates.prompt_templates import judge_output_template as SYSTEM_PROMPT
+
+InputT = TypeVar("InputT")
+OutputT = TypeVar("OutputT")
+
+
+class OutputEvaluator(Evaluator[InputT, OutputT]):
+    """
+    An evaluator that is LLM-based.
+
+    Attributes:
+        rubric: The user-specified criteria for evaluating a collection of test cases.
+        model: A string representing the model-id for Bedrock to use, or a Model instance.
+                    Defaults to strands.models.BedrockModel if None.
+        system_prompt: System prompt to guide model behavior.
+                    If None, the evaluator will use one of the default template.
+        include_inputs: Whether to include inputs to the task in the evaluation or not.
+    """
+
+    def __init__(
+        self,
+        rubric: str,
+        model: Union[Model, str, None] = None,
+        system_prompt: str = SYSTEM_PROMPT,
+        include_inputs: bool = True,
+    ):
+        super().__init__()
+        self.rubric = rubric
+        self.model = model
+        self.include_inputs = include_inputs
+        self.system_prompt = system_prompt
+
+    def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        """
+        Evaluate the performance of the task on the given test cases.
+
+        Args:
+            evaluation_case: The test case with all of the neccessary context to be evaluated.
+
+        Returns:
+            The results of the evaluation as EvaluationOutput.
+        """
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        evaluation_prompt = compose_test_prompt(
+            evaluation_case=evaluation_case, rubric=self.rubric, include_inputs=self.include_inputs
+        )
+        result = evaluator_agent.structured_output(EvaluationOutput, evaluation_prompt)
+        return [result]
+
+    async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        """
+        Evaluate the performance of the task on the given test cases asynchronously.
+
+        Args:
+            evaluation_case: The test case with all of the neccessary context to be evaluated.
+
+        Returns:
+            The results of the evaluation as EvaluationOutput.
+        """
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        evaluation_prompt = compose_test_prompt(
+            evaluation_case=evaluation_case, rubric=self.rubric, include_inputs=self.include_inputs
+        )
+        result = await evaluator_agent.structured_output_async(EvaluationOutput, evaluation_prompt)
+        return [result]

strands_evals/evaluators/prompt_templates/case_prompt_template.py

@@ -0,0 +1,63 @@
+from typing_extensions import TypeVar
+
+from ...types.evaluation import EvaluationData
+
+InputT = TypeVar("InputT")
+OutputT = TypeVar("OutputT")
+
+
+def compose_test_prompt(
+    evaluation_case: EvaluationData[InputT, OutputT],
+    rubric: str,
+    include_inputs: bool,
+    uses_trajectory: bool = False,
+    trajectory_description: dict | None = None,
+) -> str:
+    """
+    Compose the prompt for a test case evaluation.
+
+    Args:
+        evaluation_case: The evaluation data containing input, output, and trajectory information
+        rubric: The evaluation criteria to be applied
+        include_inputs: Whether to include the input in the prompt
+        uses_trajectory: Whether this is a trajectory-based evaluation
+        trajectory_description: A dictionary describing the type of trajectory expected for this evaluation.
+
+    Returns:
+        str: The formatted evaluation prompt
+
+    Raises:
+        Exception: If actual_output is missing for non-trajectory evaluations
+        Exception: If actual_trajectory is missing for trajectory evaluations
+    """
+    evaluation_prompt = "Evaluate this singular test case. THE FINAL SCORE MUST BE A DECIMAL BETWEEN 0.0 AND 1.0 (NOT 0 to 10 OR 0 to 100). \n"
+    if include_inputs:
+        evaluation_prompt += f"<Input>{evaluation_case.input}</Input>\n"
+
+    if uses_trajectory:  # trajectory evaluations don't require actual_output
+        if evaluation_case.actual_output:
+            evaluation_prompt += f"<Output>{evaluation_case.actual_output}</Output>\n"
+    else:
+        if evaluation_case.actual_output is None:
+            raise Exception(
+                "Please make sure the task function return the output or a dictionary with the key 'output'."
+            )
+        evaluation_prompt += f"<Output>{evaluation_case.actual_output}</Output>\n"
+
+    if evaluation_case.expected_output:
+        evaluation_prompt += f"<ExpectedOutput>{evaluation_case.expected_output}</ExpectedOutput>\n"
+
+    if uses_trajectory:  # trajectory evaluations require actual_trajectory
+        if evaluation_case.actual_trajectory is None:
+            raise Exception("Please make sure the task function return a dictionary with the key 'trajectory'.")
+        evaluation_prompt += f"<Trajectory>{evaluation_case.actual_trajectory}</Trajectory>\n"
+
+        if evaluation_case.expected_trajectory:
+            evaluation_prompt += f"<ExpectedTrajectory>{evaluation_case.expected_trajectory}</ExpectedTrajectory>\n"
+
+        if trajectory_description:
+            evaluation_prompt += f"<TrajectoryDescription>{trajectory_description}</TrajectoryDescription>\n"
+
+    evaluation_prompt += f"<Rubric>{rubric}</Rubric>"
+
+    return evaluation_prompt

strands_evals/evaluators/prompt_templates/faithfulness/__init__.py

@@ -0,0 +1,11 @@
+from . import faithfulness_v0
+
+VERSIONS = {
+    "v0": faithfulness_v0,
+}
+
+DEFAULT_VERSION = "v0"
+
+
+def get_template(version: str = DEFAULT_VERSION):
+    return VERSIONS[version]