strands-agents-evals 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

strands_evals/evaluators/response_relevance_evaluator.py

@@ -0,0 +1,144 @@
+from enum import Enum
+from typing import cast
+
+from pydantic import BaseModel, Field
+from strands import Agent
+from strands.agent.agent_result import AgentResult
+from strands.models.model import Model
+from typing_extensions import TypeVar, Union
+
+from ..types.evaluation import EvaluationData, EvaluationOutput
+from ..types.trace import EvaluationLevel, TextContent, ToolExecution, TraceLevelInput
+from .evaluator import Evaluator
+from .prompt_templates.response_relevance import get_template
+
+InputT = TypeVar("InputT")
+OutputT = TypeVar("OutputT")
+
+
+class ResponseRelevanceScore(str, Enum):
+    """Categorical response relevance ratings."""
+
+    NOT_AT_ALL = "Not At All"
+    NOT_GENERALLY = "Not Generally"
+    NEUTRAL_MIXED = "Neutral/Mixed"
+    GENERALLY_YES = "Generally Yes"
+    COMPLETELY_YES = "Completely Yes"
+
+
+class ResponseRelevanceRating(BaseModel):
+    """Structured output for response relevance evaluation."""
+
+    reasoning: str = Field(description="Step by step reasoning to derive the final score")
+    score: ResponseRelevanceScore = Field(description="Categorical response relevance rating")
+
+
+class ResponseRelevanceEvaluator(Evaluator[InputT, OutputT]):
+    """Evaluates the relevance of agent responses to user questions."""
+
+    evaluation_level = EvaluationLevel.TRACE_LEVEL
+
+    _score_mapping = {
+        ResponseRelevanceScore.NOT_AT_ALL: 0.0,
+        ResponseRelevanceScore.NOT_GENERALLY: 0.25,
+        ResponseRelevanceScore.NEUTRAL_MIXED: 0.5,
+        ResponseRelevanceScore.GENERALLY_YES: 0.75,
+        ResponseRelevanceScore.COMPLETELY_YES: 1.0,
+    }
+
+    def __init__(
+        self,
+        version: str = "v0",
+        model: Union[Model, str, None] = None,
+        system_prompt: str | None = None,
+        include_inputs: bool = True,
+    ):
+        super().__init__()
+        self.system_prompt = system_prompt or get_template(version).SYSTEM_PROMPT
+        self.version = version
+        self.model = model
+        self.include_inputs = include_inputs
+
+    def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        parsed_input = self._get_last_turn(evaluation_case)
+        prompt = self._format_prompt(parsed_input)
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        result = evaluator_agent(prompt, structured_output_model=ResponseRelevanceRating)
+        return self._create_evaluation_output(result)
+
+    async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        parsed_input = self._get_last_turn(evaluation_case)
+        prompt = self._format_prompt(parsed_input)
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        result = await evaluator_agent.invoke_async(prompt, structured_output_model=ResponseRelevanceRating)
+        return self._create_evaluation_output(result)
+
+    def _create_evaluation_output(self, result: AgentResult) -> list[EvaluationOutput]:
+        rating = cast(ResponseRelevanceRating, result.structured_output)
+        normalized_score = self._score_mapping[rating.score]
+        return [
+            EvaluationOutput(
+                score=normalized_score,
+                test_pass=normalized_score >= 0.5,
+                reason=rating.reasoning,
+                label=rating.score,
+            )
+        ]
+
+    def _get_last_turn(self, evaluation_case: EvaluationData[InputT, OutputT]) -> TraceLevelInput:
+        """Extract the most recent turn from the conversation for evaluation."""
+        parsed_inputs = self._parse_trajectory(evaluation_case)
+        if not parsed_inputs:
+            raise ValueError(
+                "No turn-level inputs could be parsed from the trajectory. "
+                "Ensure actual_trajectory is a Session with at least one AgentInvocationSpan."
+            )
+        return parsed_inputs[-1]
+
+    def _extract_user_prompt(self, parsed_input: TraceLevelInput) -> str:
+        """Extract user prompt from last message in session history.
+
+        Args:
+            parsed_input: Trace-level input containing session history
+
+        Returns:
+            User prompt text, or empty string if not available
+        """
+        if not parsed_input.session_history:
+            return ""
+
+        last_msg = parsed_input.session_history[-1]
+        if not isinstance(last_msg, list) and self._has_text_content(last_msg):
+            first_content = last_msg.content[0]
+            if isinstance(first_content, TextContent):
+                return first_content.text
+
+        return ""
+
+    def _format_prompt(self, parsed_input: TraceLevelInput) -> str:
+        """Format evaluation prompt from parsed trace data.
+
+        Args:
+            parsed_input: Trace-level input containing agent response and session history
+
+        Returns:
+            Formatted prompt string with conversation context and target response
+        """
+        parts = []
+
+        if parsed_input.session_history:
+            history_lines = []
+            for msg in parsed_input.session_history:
+                if isinstance(msg, list) and msg and isinstance(msg[0], ToolExecution):
+                    continue  # Skip tool execution lists
+                if not isinstance(msg, list) and self._has_text_content(msg):
+                    first_content = msg.content[0]
+                    if isinstance(first_content, TextContent):
+                        history_lines.append(f"{msg.role.value.capitalize()}: {first_content.text}")
+            history_str = "\n".join(history_lines)
+            parts.append(f"# Previous turns:\n{history_str}")
+
+        user_prompt = self._extract_user_prompt(parsed_input)
+        parts.append(f"# Target turn to evaluate:\nUser: {user_prompt}\nAssistant: {parsed_input.agent_response.text}")
+
+        return "\n\n".join(parts)

strands_evals/evaluators/tool_parameter_accuracy_evaluator.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import cast
 
 from pydantic import BaseModel, Field
 from strands import Agent
@@ -56,12 +57,17 @@ class ToolParameterAccuracyEvaluator(Evaluator[InputT, OutputT]):
         for tool_input in tool_inputs:
             prompt = self._format_prompt(tool_input)
             evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
-            rating = evaluator_agent.structured_output(ToolParameterAccuracyRating, prompt)
+            result = evaluator_agent(prompt, structured_output_model=ToolParameterAccuracyRating)
+            rating = cast(ToolParameterAccuracyRating, result.structured_output)
             normalized_score = self._score_mapping[rating.score]
-            result = EvaluationOutput(
-                score=normalized_score, test_pass=normalized_score == 1.0, reason=rating.reasoning, label=rating.score
+            results.append(
+                EvaluationOutput(
+                    score=normalized_score,
+                    test_pass=normalized_score == 1.0,
+                    reason=rating.reasoning,
+                    label=rating.score,
+                )
             )
-            results.append(result)
 
         return results
 
@@ -72,12 +78,17 @@ class ToolParameterAccuracyEvaluator(Evaluator[InputT, OutputT]):
         for tool_input in tool_inputs:
             prompt = self._format_prompt(tool_input)
             evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
-            rating = await evaluator_agent.structured_output_async(ToolParameterAccuracyRating, prompt)
+            result = await evaluator_agent.invoke_async(prompt, structured_output_model=ToolParameterAccuracyRating)
+            rating = cast(ToolParameterAccuracyRating, result.structured_output)
             normalized_score = self._score_mapping[rating.score]
-            result = EvaluationOutput(
-                score=normalized_score, test_pass=normalized_score == 1.0, reason=rating.reasoning, label=rating.score
+            results.append(
+                EvaluationOutput(
+                    score=normalized_score,
+                    test_pass=normalized_score == 1.0,
+                    reason=rating.reasoning,
+                    label=rating.score,
+                )
             )
-            results.append(result)
 
         return results
 

strands_evals/evaluators/tool_selection_accuracy_evaluator.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import cast
 
 from pydantic import BaseModel, Field
 from strands import Agent
@@ -56,12 +57,17 @@ class ToolSelectionAccuracyEvaluator(Evaluator[InputT, OutputT]):
         for tool_input in tool_inputs:
             prompt = self._format_prompt(tool_input)
             evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
-            rating = evaluator_agent.structured_output(ToolSelectionRating, prompt)
+            result = evaluator_agent(prompt, structured_output_model=ToolSelectionRating)
+            rating = cast(ToolSelectionRating, result.structured_output)
             normalized_score = self._score_mapping[rating.score]
-            result = EvaluationOutput(
-                score=normalized_score, test_pass=normalized_score == 1.0, reason=rating.reasoning, label=rating.score
+            results.append(
+                EvaluationOutput(
+                    score=normalized_score,
+                    test_pass=normalized_score == 1.0,
+                    reason=rating.reasoning,
+                    label=rating.score,
+                )
             )
-            results.append(result)
 
         return results
 
@@ -72,12 +78,17 @@ class ToolSelectionAccuracyEvaluator(Evaluator[InputT, OutputT]):
         for tool_input in tool_inputs:
             prompt = self._format_prompt(tool_input)
             evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
-            rating = await evaluator_agent.structured_output_async(ToolSelectionRating, prompt)
+            result = await evaluator_agent.invoke_async(prompt, structured_output_model=ToolSelectionRating)
+            rating = cast(ToolSelectionRating, result.structured_output)
             normalized_score = self._score_mapping[rating.score]
-            result = EvaluationOutput(
-                score=normalized_score, test_pass=normalized_score == 1.0, reason=rating.reasoning, label=rating.score
+            results.append(
+                EvaluationOutput(
+                    score=normalized_score,
+                    test_pass=normalized_score == 1.0,
+                    reason=rating.reasoning,
+                    label=rating.score,
+                )
             )
-            results.append(result)
 
         return results
 

strands_evals/evaluators/trajectory_evaluator.py

@@ -1,3 +1,5 @@
+from typing import cast
+
 from strands import Agent
 from strands.models.model import Model
 from typing_extensions import Any, TypeVar, Union
@@ -74,8 +76,8 @@ class TrajectoryEvaluator(Evaluator[InputT, OutputT]):
             include_inputs=self.include_inputs,
             uses_trajectory=True,
         )
-        result = evaluator_agent.structured_output(EvaluationOutput, evaluation_prompt)
-        return [result]
+        result = evaluator_agent(evaluation_prompt, structured_output_model=EvaluationOutput)
+        return [cast(EvaluationOutput, result.structured_output)]
 
     async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         """
@@ -96,5 +98,5 @@ class TrajectoryEvaluator(Evaluator[InputT, OutputT]):
             include_inputs=self.include_inputs,
             uses_trajectory=True,
         )
-        result = await evaluator_agent.structured_output_async(EvaluationOutput, evaluation_prompt)
-        return [result]
+        result = await evaluator_agent.invoke_async(evaluation_prompt, structured_output_model=EvaluationOutput)
+        return [cast(EvaluationOutput, result.structured_output)]