strands-agents-evals 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

{strands_agents_evals-0.1.4.dist-info → strands_agents_evals-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: strands-agents-evals
-Version: 0.1.4
+Version: 0.1.5
 Summary: Evaluation framework for Strands
 Author-email: AWS <opensource@amazon.com>
 License: Apache-2.0
@@ -15,6 +15,7 @@ Requires-Dist: pydantic<3.0.0,>=2.0.0
 Requires-Dist: rich<15.0.0,>=14.0.0
 Requires-Dist: strands-agents-tools<1.0.0,>=0.1.0
 Requires-Dist: strands-agents>=1.0.0
+Requires-Dist: tenacity<10.0.0,>=8.0.0
 Requires-Dist: typing-extensions>=4.0
 Provides-Extra: dev
 Requires-Dist: hatch<2.0.0,>=1.0.0; extra == 'dev'

{strands_agents_evals-0.1.4.dist-info → strands_agents_evals-0.1.5.dist-info}/RECORD

@@ -1,20 +1,25 @@
 strands_evals/__init__.py,sha256=WnYsQGtkatrCKM8v_i_oCtBHNJfPaTrOg2ThUlf55Pk,485
 strands_evals/case.py,sha256=KWAL947NkmNzg9FFdTsL6KI9AFLQ8IcFjaOjcs9x5to,2131
-strands_evals/experiment.py,sha256=yySXFW5p9xkDSvkxHBBDncxXKiuj0aDFY7iKoUyprwc,28745
+strands_evals/experiment.py,sha256=d9SWLjEnWjxqTfTB0fivPZRkVD8xFagS5r_OMxaqxmc,37723
+strands_evals/utils.py,sha256=a8mkCjtyPTcz4YtcbC9La8OnNGE_Tl9Lqxf06ZbVfCA,1175
 strands_evals/display/display_console.py,sha256=bOTr6RepgnifALz2DgXnnk3c4Jjxu_mA68-pFr7xry0,5932
-strands_evals/evaluators/__init__.py,sha256=OfZU5RkYewHOAnEjPKdxiEvPnfOOWNZc_9nQpAfARfI,887
-strands_evals/evaluators/evaluator.py,sha256=XEesDeT83H93B1X_w8s0Nsb1KKHy26QO8b99Hi6vKbc,7466
-strands_evals/evaluators/faithfulness_evaluator.py,sha256=i6oLgG58BxYAv-lottapn4XfSBncvGyYNIXNRqSILAQ,4742
-strands_evals/evaluators/goal_success_rate_evaluator.py,sha256=ZKP2Us62_cwVwUVng9QlOytkrMXC7ObOlp7xr-obOw8,3373
-strands_evals/evaluators/harmfulness_evaluator.py,sha256=odKugWJUbVGuPzU3gEQjdumqIkmbKRIyPEKLESQt-vQ,5315
-strands_evals/evaluators/helpfulness_evaluator.py,sha256=7lRLhDsr1PSbPAILmDG015lfxn6iKa_dQleuFzlzMcQ,5922
-strands_evals/evaluators/interactions_evaluator.py,sha256=-JB85oXiEGyCr7oUH5nzwJpPw4GBWcmMKNzOHjErRSo,10694
-strands_evals/evaluators/output_evaluator.py,sha256=XEul2qc7cArl192cojKeB4BZ9EX8pCbpAQv7xwIqJAk,2949
-strands_evals/evaluators/tool_parameter_accuracy_evaluator.py,sha256=jLHA2hR3E23N3fOZEeOR_KqqkYwzP2FWNnTZOd55h9E,4642
-strands_evals/evaluators/tool_selection_accuracy_evaluator.py,sha256=bDa_-k3ye9v80urdyFIpsURSmdK6g7muZ_w0NfN_E3Y,4575
-strands_evals/evaluators/trajectory_evaluator.py,sha256=MIq0dxGxMBw-cOt8zc80jrVnO-McqtBQ5E4_0An2ka4,3989
+strands_evals/evaluators/__init__.py,sha256=IHDzg31Od5lkwonbb329KcPHbn3FsKi8VallqddmF7E,1074
+strands_evals/evaluators/conciseness_evaluator.py,sha256=pqO1CTR-NTyOU2vuhZZrIW9mpq3XstZysoKdNuPOoZo,5566
+strands_evals/evaluators/evaluator.py,sha256=iW8A62wG0ZjXSz95eYraOTZ4FbhkhOOHIyj7Puol9fw,7619
+strands_evals/evaluators/faithfulness_evaluator.py,sha256=dICHVCYa3_lj5FKzS7dnepDk7IgN2CEh0rY64OitPgQ,4957
+strands_evals/evaluators/goal_success_rate_evaluator.py,sha256=jglDcwNu9vEZLYwq7XsQrvvIWiTPQl_Eqs0BxDfY4r0,3586
+strands_evals/evaluators/harmfulness_evaluator.py,sha256=OeEjTogThr6FIzPFwlNiDiXzNhP2ET3hzw7NfDgcOMs,5528
+strands_evals/evaluators/helpfulness_evaluator.py,sha256=5PvTTqOU1Q3Wvuxxa4KZvE_es_LtdL5EGvhhKU305NA,6135
+strands_evals/evaluators/interactions_evaluator.py,sha256=j1zCrLGQxSzFe02eM4qkWHyJtokKWFqhzKz2_AzVTLE,10822
+strands_evals/evaluators/output_evaluator.py,sha256=0pfM4oJ9UOTeJ1NYweBufUpgCLUSPs5Xy9xrKXGyCIg,3077
+strands_evals/evaluators/response_relevance_evaluator.py,sha256=SJf4Q65_LWEMHre4cupFdC3hMCvLwYiXPP2IS20jMFY,5885
+strands_evals/evaluators/tool_parameter_accuracy_evaluator.py,sha256=YYBsMxS8o2UUcK75-6zBqZGwDzCmj7ymZrVEmzPB7xI,4989
+strands_evals/evaluators/tool_selection_accuracy_evaluator.py,sha256=RdzM1peqGmawfwwYO6J5U27y2_lNp8FM-qmTKw-M8LM,4906
+strands_evals/evaluators/trajectory_evaluator.py,sha256=YJSjsy_4E0TJl-ERmelKObF-yF0npVExKWlTPDeLmQQ,4117
 strands_evals/evaluators/prompt_templates/case_prompt_template.py,sha256=NQH3flsOxQOp4sLAQ0g6Q4YjkpuIWzNJqZ8-bSIn78c,2687
 strands_evals/evaluators/prompt_templates/prompt_templates.py,sha256=tx4auXcHo-wxsQtJ9wCJGQbbURmhsRkB4LS-CzmAwwA,14468
+strands_evals/evaluators/prompt_templates/conciseness/__init__.py,sha256=G6L1Jw4fAPS9R0CwPcCQ77v-O9NDya5gx3Kwt-MTsvo,176
+strands_evals/evaluators/prompt_templates/conciseness/conciseness_v0.py,sha256=02anpXMdfLS1X21oZPr2-2-M_-Cm7LeOsxMdMRxoUUM,583
 strands_evals/evaluators/prompt_templates/faithfulness/__init__.py,sha256=_18J1msOfOikbvyYefAmOJEFaBJ2G75ybkTyQeHNvRA,178
 strands_evals/evaluators/prompt_templates/faithfulness/faithfulness_v0.py,sha256=0NVqDYnppwg-pBb6PW6PGaOuBwlYz7bVJLu0EgkI4dk,1410
 strands_evals/evaluators/prompt_templates/goal_success_rate/__init__.py,sha256=cfOeRMakkG_FL7_1ewJl3AuCJGT13ZSWM7tnInLJetw,188
@@ -23,6 +28,8 @@ strands_evals/evaluators/prompt_templates/harmfulness/__init__.py,sha256=Y2KQcYK
 strands_evals/evaluators/prompt_templates/harmfulness/harmfulness_v0.py,sha256=uGvZlfcPxfdHpUR_If2-xpMCX6-ynFlT0MnjZzqH3xA,1108
 strands_evals/evaluators/prompt_templates/helpfulness/__init__.py,sha256=8j55Lwo3qmkVDflZWXwxjauot7IAayN5ua1X9yQS1vM,176
 strands_evals/evaluators/prompt_templates/helpfulness/helpfulness_v0.py,sha256=MUlK5B0RDdQdcL5Ke_cp5V6sbuVZcwdlBnUoC3Ju9B4,1569
+strands_evals/evaluators/prompt_templates/response_relevance/__init__.py,sha256=Yn7xcYvWmsby4oE6z0WYhLT-IU-z7hQ-QYNM1EwGGaE,190
+strands_evals/evaluators/prompt_templates/response_relevance/response_relevance_v0.py,sha256=KXfNZHnQx7L4qu3o1rg7rvfnD0UjI9fIWOsMAcrDaRY,1663
 strands_evals/evaluators/prompt_templates/tool_parameter_accuracy/__init__.py,sha256=lFx-tDGveafaqAAEvVuYU3-Pj9G0-14GlSrMkq80wX0,200
 strands_evals/evaluators/prompt_templates/tool_parameter_accuracy/tool_parameter_accuracy_v0.py,sha256=D_m1aJN7AXeA8Z2iIysxvwx52TtiozGZFWvnXvG8Tms,2259
 strands_evals/evaluators/prompt_templates/tool_selection_accuracy/__init__.py,sha256=AiMdoIS4-MqGeeNQkzPkjBnOxW5W5-jw20Qbn6fOjik,200
@@ -31,7 +38,7 @@ strands_evals/extractors/__init__.py,sha256=Jmlrk-m8sSS_LwmCVSloIkg3BjOgRzNEezja
 strands_evals/extractors/graph_extractor.py,sha256=TeT-58JB9roqSvy2ELz1kg8WF5YO-cfLlGZTO0F9s_4,1105
 strands_evals/extractors/swarm_extractor.py,sha256=Sm1XFCkAGVdF3XDyO3iF-20I8C6sAQ8JPNP5fgotOFU,2682
 strands_evals/extractors/tools_use_extractor.py,sha256=emLL63LKldL2IA2u5wZL0ZhklZJqX0KLr5xFRt-S4i4,6600
-strands_evals/extractors/trace_extractor.py,sha256=TJKl0OdjFhh-htlV1Wxzem8TQdb0rxa-efkq_e0pAdo,7287
+strands_evals/extractors/trace_extractor.py,sha256=Qbxi5UE9KgqQdJR1HIpqLAXdze_M9lJ_ASSaB5MOzk8,7259
 strands_evals/generators/__init__.py,sha256=B1F30DAIf0kPyBdE4PAZvSby-dTelqb_7hFJoATqVb0,89
 strands_evals/generators/experiment_generator.py,sha256=6wLTL0iG2b0YAiu0w8dDiaBxOIy7p_Fs7l3hCjgQc0w,22655
 strands_evals/generators/topic_planner.py,sha256=FtgTVDlV9hWJyO8E4Z__nEWvvrOJzmTW4y6yZ9Alx1A,2436
@@ -61,8 +68,8 @@ strands_evals/types/evaluation_report.py,sha256=vT86zO4Qn9CQbULo3aziGMdG-1qWLdcB
 strands_evals/types/trace.py,sha256=BFoEylzAlENyPH702T5MDz-_H21-Wfx-FFTSXX1tDfY,4844
 strands_evals/types/simulation/__init__.py,sha256=-mz5lW6qFfIMm4dJGaP9pXY3xeiefLbB0XevjdFykkU,133
 strands_evals/types/simulation/actor.py,sha256=ESTV8165c3Ad5QT4yYmjm-A-oZdwZ0Rf0Lq7zokjTPo,1163
-strands_agents_evals-0.1.4.dist-info/METADATA,sha256=VQm_tm1Umm3fi_HfujW0Ovm_XyvQQCjEJrAL4-dGjKQ,17721
-strands_agents_evals-0.1.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-strands_agents_evals-0.1.4.dist-info/licenses/LICENSE,sha256=yIWKWnZEC7ykhOE0z330Y4XQkN0YssWOZQ-TkliALls,10141
-strands_agents_evals-0.1.4.dist-info/licenses/NOTICE,sha256=Eg13ogOmcI7JpMjxniFnKG81vwU3X8X7P_IlpvVg5RU,66
-strands_agents_evals-0.1.4.dist-info/RECORD,,
+strands_agents_evals-0.1.5.dist-info/METADATA,sha256=cj7lVVMEKiOonsr3Zm4Qbjf-AHgZa0TsmFYvFiBpGVU,17760
+strands_agents_evals-0.1.5.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+strands_agents_evals-0.1.5.dist-info/licenses/LICENSE,sha256=yIWKWnZEC7ykhOE0z330Y4XQkN0YssWOZQ-TkliALls,10141
+strands_agents_evals-0.1.5.dist-info/licenses/NOTICE,sha256=Eg13ogOmcI7JpMjxniFnKG81vwU3X8X7P_IlpvVg5RU,66
+strands_agents_evals-0.1.5.dist-info/RECORD,,

strands_evals/evaluators/__init__.py

@@ -1,3 +1,4 @@
+from .conciseness_evaluator import ConcisenessEvaluator
 from .evaluator import Evaluator
 from .faithfulness_evaluator import FaithfulnessEvaluator
 from .goal_success_rate_evaluator import GoalSuccessRateEvaluator
@@ -5,6 +6,7 @@ from .harmfulness_evaluator import HarmfulnessEvaluator
 from .helpfulness_evaluator import HelpfulnessEvaluator
 from .interactions_evaluator import InteractionsEvaluator
 from .output_evaluator import OutputEvaluator
+from .response_relevance_evaluator import ResponseRelevanceEvaluator
 from .tool_parameter_accuracy_evaluator import ToolParameterAccuracyEvaluator
 from .tool_selection_accuracy_evaluator import ToolSelectionAccuracyEvaluator
 from .trajectory_evaluator import TrajectoryEvaluator
@@ -18,6 +20,8 @@ __all__ = [
     "HarmfulnessEvaluator",
     "GoalSuccessRateEvaluator",
     "FaithfulnessEvaluator",
+    "ResponseRelevanceEvaluator",
     "ToolSelectionAccuracyEvaluator",
     "ToolParameterAccuracyEvaluator",
+    "ConcisenessEvaluator",
 ]

strands_evals/evaluators/conciseness_evaluator.py

@@ -0,0 +1,139 @@
+from enum import Enum
+from typing import cast
+
+from pydantic import BaseModel, Field
+from strands import Agent
+from strands.models.model import Model
+from typing_extensions import TypeVar, Union
+
+from ..types.evaluation import EvaluationData, EvaluationOutput
+from ..types.trace import EvaluationLevel, TextContent, ToolExecution, TraceLevelInput
+from .evaluator import Evaluator
+from .prompt_templates.conciseness import get_template
+
+InputT = TypeVar("InputT")
+OutputT = TypeVar("OutputT")
+
+
+class ConcisenessScore(str, Enum):
+    """Categorical conciseness ratings."""
+
+    NOT_CONCISE = "Not Concise"
+    PARTIALLY_CONCISE = "Partially Concise"
+    PERFECTLY_CONCISE = "Perfectly Concise"
+
+
+class ConcisenessRating(BaseModel):
+    """Structured output for conciseness evaluation."""
+
+    reasoning: str = Field(description="Step by step reasoning to derive the final score")
+    score: ConcisenessScore = Field(description="Categorical conciseness rating")
+
+
+class ConcisenessEvaluator(Evaluator[InputT, OutputT]):
+    """Evaluates how concise the assistant's response is."""
+
+    evaluation_level = EvaluationLevel.TRACE_LEVEL
+
+    _score_mapping = {
+        ConcisenessScore.NOT_CONCISE: 0.0,
+        ConcisenessScore.PARTIALLY_CONCISE: 0.5,
+        ConcisenessScore.PERFECTLY_CONCISE: 1.0,
+    }
+
+    def __init__(
+        self,
+        version: str = "v0",
+        model: Union[Model, str, None] = None,
+        system_prompt: str | None = None,
+        include_inputs: bool = True,
+    ):
+        super().__init__()
+        self.system_prompt = system_prompt or get_template(version).SYSTEM_PROMPT
+        self.version = version
+        self.model = model
+        self.include_inputs = include_inputs
+
+    def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        parsed_input = self._get_last_turn(evaluation_case)
+        prompt = self._format_prompt(parsed_input)
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        result = evaluator_agent(prompt, structured_output_model=ConcisenessRating)
+        return self._create_evaluation_output(result)
+
+    async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        parsed_input = self._get_last_turn(evaluation_case)
+        prompt = self._format_prompt(parsed_input)
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        result = await evaluator_agent.invoke_async(prompt, structured_output_model=ConcisenessRating)
+        return self._create_evaluation_output(result)
+
+    def _create_evaluation_output(self, result) -> list[EvaluationOutput]:
+        rating = cast(ConcisenessRating, result.structured_output)
+        normalized_score = self._score_mapping[rating.score]
+        return [
+            EvaluationOutput(
+                score=normalized_score,
+                test_pass=normalized_score >= 0.5,
+                reason=rating.reasoning,
+                label=rating.score,
+            )
+        ]
+
+    def _get_last_turn(self, evaluation_case: EvaluationData[InputT, OutputT]) -> TraceLevelInput:
+        """Extract the most recent turn from the conversation for evaluation."""
+        parsed_inputs = self._parse_trajectory(evaluation_case)
+        if not parsed_inputs:
+            raise ValueError(
+                "No turn-level inputs could be parsed from the trajectory. "
+                "Ensure actual_trajectory is a Session with at least one AgentInvocationSpan."
+            )
+        return parsed_inputs[-1]
+
+    def _extract_user_prompt(self, parsed_input: TraceLevelInput) -> str:
+        """Extract user prompt from last message in session history.
+
+        Args:
+            parsed_input: Trace-level input containing session history
+
+        Returns:
+            User prompt text, or empty string if not available
+        """
+        if not parsed_input.session_history:
+            return ""
+
+        last_msg = parsed_input.session_history[-1]
+        if not isinstance(last_msg, list) and self._has_text_content(last_msg):
+            first_content = last_msg.content[0]
+            if isinstance(first_content, TextContent):
+                return first_content.text
+
+        return ""
+
+    def _format_prompt(self, parsed_input: TraceLevelInput) -> str:
+        """Format evaluation prompt from parsed trace data.
+
+        Args:
+            parsed_input: Trace-level input containing agent response and session history
+
+        Returns:
+            Formatted prompt string with conversation history and target turn
+        """
+        parts = []
+
+        if parsed_input.session_history:
+            history_lines = []
+            for msg in parsed_input.session_history:
+                if isinstance(msg, list) and msg and isinstance(msg[0], ToolExecution):
+                    continue  # Skip tool execution lists
+                if not isinstance(msg, list) and self._has_text_content(msg):
+                    first_content = msg.content[0]
+                    if isinstance(first_content, TextContent):
+                        history_lines.append(f"{msg.role.value.capitalize()}: {first_content.text}")
+            history_str = "\n".join(history_lines)
+            parts.append(f"# Previous turns:\n{history_str}")
+
+        user_prompt = self._extract_user_prompt(parsed_input)
+        parts.append(f"# Target turn to evaluate:\nUser: {user_prompt}\nAssistant: {parsed_input.agent_response.text}")
+
+        return "\n\n".join(parts)

strands_evals/evaluators/evaluator.py

@@ -63,6 +63,10 @@ class Evaluator(Generic[InputT, OutputT]):
 
     @staticmethod
     def _default_aggregator(outputs: list[EvaluationOutput]) -> tuple[float, bool, str]:
+        # Handle empty outputs list to avoid division by zero
+        if not outputs:
+            return (0.0, False, "No evaluation outputs produced")
+
         avg_score = sum(o.score for o in outputs) / len(outputs)
         all_pass = all(o.test_pass for o in outputs)
         combined_reason = " | ".join(o.reason for o in outputs if o.reason)

strands_evals/evaluators/faithfulness_evaluator.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import cast
 
 from pydantic import BaseModel, Field
 from strands import Agent
@@ -59,29 +60,33 @@ class FaithfulnessEvaluator(Evaluator[InputT, OutputT]):
         parsed_input = self._get_last_turn(evaluation_case)
         prompt = self._format_prompt(parsed_input)
         evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
-        rating = evaluator_agent.structured_output(FaithfulnessRating, prompt)
+        result = evaluator_agent(prompt, structured_output_model=FaithfulnessRating)
+        rating = cast(FaithfulnessRating, result.structured_output)
         normalized_score = self._score_mapping[rating.score]
-        result = EvaluationOutput(
-            score=normalized_score,
-            test_pass=normalized_score >= 0.5,
-            reason=rating.reasoning,
-            label=rating.score,
-        )
-        return [result]
+        return [
+            EvaluationOutput(
+                score=normalized_score,
+                test_pass=normalized_score >= 0.5,
+                reason=rating.reasoning,
+                label=rating.score,
+            )
+        ]
 
     async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
         prompt = self._format_prompt(parsed_input)
         evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
-        rating = await evaluator_agent.structured_output_async(FaithfulnessRating, prompt)
+        result = await evaluator_agent.invoke_async(prompt, structured_output_model=FaithfulnessRating)
+        rating = cast(FaithfulnessRating, result.structured_output)
         normalized_score = self._score_mapping[rating.score]
-        result = EvaluationOutput(
-            score=normalized_score,
-            test_pass=normalized_score >= 0.5,
-            reason=rating.reasoning,
-            label=rating.score,
-        )
-        return [result]
+        return [
+            EvaluationOutput(
+                score=normalized_score,
+                test_pass=normalized_score >= 0.5,
+                reason=rating.reasoning,
+                label=rating.score,
+            )
+        ]
 
     def _get_last_turn(self, evaluation_case: EvaluationData[InputT, OutputT]) -> TraceLevelInput:
         """Extract the most recent turn from the conversation for evaluation."""

strands_evals/evaluators/goal_success_rate_evaluator.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import cast
 
 from pydantic import BaseModel, Field
 from strands import Agent
@@ -53,29 +54,33 @@ class GoalSuccessRateEvaluator(Evaluator[InputT, OutputT]):
         session_input = self._parse_trajectory(evaluation_case)
         prompt = self._format_prompt(session_input)
         evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
-        rating = evaluator_agent.structured_output(GoalSuccessRating, prompt)
+        result = evaluator_agent(prompt, structured_output_model=GoalSuccessRating)
+        rating = cast(GoalSuccessRating, result.structured_output)
         normalized_score = self._score_mapping[rating.score]
-        result = EvaluationOutput(
-            score=normalized_score,
-            test_pass=normalized_score >= 1.0,
-            reason=rating.reasoning,
-            label=rating.score,
-        )
-        return [result]
+        return [
+            EvaluationOutput(
+                score=normalized_score,
+                test_pass=normalized_score >= 1.0,
+                reason=rating.reasoning,
+                label=rating.score,
+            )
+        ]
 
     async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         session_input = self._parse_trajectory(evaluation_case)
         prompt = self._format_prompt(session_input)
         evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
-        rating = await evaluator_agent.structured_output_async(GoalSuccessRating, prompt)
+        result = await evaluator_agent.invoke_async(prompt, structured_output_model=GoalSuccessRating)
+        rating = cast(GoalSuccessRating, result.structured_output)
         normalized_score = self._score_mapping[rating.score]
-        result = EvaluationOutput(
-            score=normalized_score,
-            test_pass=normalized_score >= 1.0,
-            reason=rating.reasoning,
-            label=rating.score,
-        )
-        return [result]
+        return [
+            EvaluationOutput(
+                score=normalized_score,
+                test_pass=normalized_score >= 1.0,
+                reason=rating.reasoning,
+                label=rating.score,
+            )
+        ]
 
     def _format_prompt(self, session_input: SessionLevelInput) -> str:
         """Format evaluation prompt from session-level input."""

strands_evals/evaluators/harmfulness_evaluator.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import cast
 
 from pydantic import BaseModel, Field
 from strands import Agent
@@ -52,29 +53,33 @@ class HarmfulnessEvaluator(Evaluator[InputT, OutputT]):
         parsed_input = self._get_last_turn(evaluation_case)
         prompt = self._format_prompt(parsed_input)
         evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
-        rating = evaluator_agent.structured_output(HarmfulnessRating, prompt)
+        result = evaluator_agent(prompt, structured_output_model=HarmfulnessRating)
+        rating = cast(HarmfulnessRating, result.structured_output)
         normalized_score = self._score_mapping[rating.score]
-        result = EvaluationOutput(
-            score=normalized_score,
-            test_pass=normalized_score == 1.0,
-            reason=rating.reasoning,
-            label=rating.score,
-        )
-        return [result]
+        return [
+            EvaluationOutput(
+                score=normalized_score,
+                test_pass=normalized_score == 1.0,
+                reason=rating.reasoning,
+                label=rating.score,
+            )
+        ]
 
     async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
         prompt = self._format_prompt(parsed_input)
         evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
-        rating = await evaluator_agent.structured_output_async(HarmfulnessRating, prompt)
+        result = await evaluator_agent.invoke_async(prompt, structured_output_model=HarmfulnessRating)
+        rating = cast(HarmfulnessRating, result.structured_output)
         normalized_score = self._score_mapping[rating.score]
-        result = EvaluationOutput(
-            score=normalized_score,
-            test_pass=normalized_score == 1.0,
-            reason=rating.reasoning,
-            label=rating.score,
-        )
-        return [result]
+        return [
+            EvaluationOutput(
+                score=normalized_score,
+                test_pass=normalized_score == 1.0,
+                reason=rating.reasoning,
+                label=rating.score,
+            )
+        ]
 
     def _get_last_turn(self, evaluation_case: EvaluationData[InputT, OutputT]) -> TraceLevelInput:
         """Extract the most recent turn from the conversation for evaluation."""

strands_evals/evaluators/helpfulness_evaluator.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from typing import cast
 
 from pydantic import BaseModel, Field
 from strands import Agent
@@ -65,29 +66,33 @@ class HelpfulnessEvaluator(Evaluator[InputT, OutputT]):
         parsed_input = self._get_last_turn(evaluation_case)
         prompt = self._format_prompt(parsed_input)
         evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
-        rating = evaluator_agent.structured_output(HelpfulnessRating, prompt)
+        result = evaluator_agent(prompt, structured_output_model=HelpfulnessRating)
+        rating = cast(HelpfulnessRating, result.structured_output)
         normalized_score = self._score_mapping[rating.score]
-        result = EvaluationOutput(
-            score=normalized_score,
-            test_pass=normalized_score >= 0.5,
-            reason=rating.reasoning,
-            label=rating.score,
-        )
-        return [result]
+        return [
+            EvaluationOutput(
+                score=normalized_score,
+                test_pass=normalized_score >= 0.5,
+                reason=rating.reasoning,
+                label=rating.score,
+            )
+        ]
 
     async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
         prompt = self._format_prompt(parsed_input)
         evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
-        rating = await evaluator_agent.structured_output_async(HelpfulnessRating, prompt)
+        result = await evaluator_agent.invoke_async(prompt, structured_output_model=HelpfulnessRating)
+        rating = cast(HelpfulnessRating, result.structured_output)
         normalized_score = self._score_mapping[rating.score]
-        result = EvaluationOutput(
-            score=normalized_score,
-            test_pass=normalized_score >= 0.5,
-            reason=rating.reasoning,
-            label=rating.score,
-        )
-        return [result]
+        return [
+            EvaluationOutput(
+                score=normalized_score,
+                test_pass=normalized_score >= 0.5,
+                reason=rating.reasoning,
+                label=rating.score,
+            )
+        ]
 
     def _get_last_turn(self, evaluation_case: EvaluationData[InputT, OutputT]) -> TraceLevelInput:
         """Extract the most recent turn from the conversation for evaluation."""

strands_evals/evaluators/interactions_evaluator.py

@@ -1,3 +1,5 @@
+from typing import cast
+
 from strands import Agent
 from strands.agent.conversation_manager import SlidingWindowConversationManager
 from strands.models.model import Model
@@ -198,8 +200,8 @@ class InteractionsEvaluator(Evaluator[InputT, OutputT]):
         for i in range(num_interactions):
             is_last = i == num_interactions - 1
             evaluation_prompt = self._compose_prompt(evaluation_case, i, is_last)
-            result = evaluator_agent.structured_output(EvaluationOutput, evaluation_prompt)
-            results.append(result)
+            result = evaluator_agent(evaluation_prompt, structured_output_model=EvaluationOutput)
+            results.append(cast(EvaluationOutput, result.structured_output))
 
         return results
 
@@ -238,7 +240,7 @@ class InteractionsEvaluator(Evaluator[InputT, OutputT]):
         for i in range(num_interactions):
             is_last = i == num_interactions - 1
             evaluation_prompt = self._compose_prompt(evaluation_case, i, is_last)
-            result = await evaluator_agent.structured_output_async(EvaluationOutput, evaluation_prompt)
-            results.append(result)
+            result = await evaluator_agent.invoke_async(evaluation_prompt, structured_output_model=EvaluationOutput)
+            results.append(cast(EvaluationOutput, result.structured_output))
 
         return results

strands_evals/evaluators/output_evaluator.py

@@ -1,3 +1,5 @@
+from typing import cast
+
 from strands import Agent
 from strands.models.model import Model
 from typing_extensions import TypeVar, Union
@@ -51,8 +53,8 @@ class OutputEvaluator(Evaluator[InputT, OutputT]):
         evaluation_prompt = compose_test_prompt(
             evaluation_case=evaluation_case, rubric=self.rubric, include_inputs=self.include_inputs
         )
-        result = evaluator_agent.structured_output(EvaluationOutput, evaluation_prompt)
-        return [result]
+        result = evaluator_agent(evaluation_prompt, structured_output_model=EvaluationOutput)
+        return [cast(EvaluationOutput, result.structured_output)]
 
     async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         """
@@ -68,5 +70,5 @@ class OutputEvaluator(Evaluator[InputT, OutputT]):
         evaluation_prompt = compose_test_prompt(
             evaluation_case=evaluation_case, rubric=self.rubric, include_inputs=self.include_inputs
         )
-        result = await evaluator_agent.structured_output_async(EvaluationOutput, evaluation_prompt)
-        return [result]
+        result = await evaluator_agent.invoke_async(evaluation_prompt, structured_output_model=EvaluationOutput)
+        return [cast(EvaluationOutput, result.structured_output)]

strands_evals/evaluators/prompt_templates/conciseness/__init__.py

@@ -0,0 +1,11 @@
+from . import conciseness_v0
+
+VERSIONS = {
+    "v0": conciseness_v0,
+}
+
+DEFAULT_VERSION = "v0"
+
+
+def get_template(version: str = DEFAULT_VERSION):
+    return VERSIONS[version]

strands_evals/evaluators/prompt_templates/conciseness/conciseness_v0.py

@@ -0,0 +1,9 @@
+SYSTEM_PROMPT = """You are evaluating how concise the Assistant's response is.
+A concise response provides exactly what was requested using the minimum necessary words, without extra explanations, pleasantries, or repetition unless specifically asked for.
+
+## Scoring
+- Perfectly Concise: delivers exactly what was asked with no unnecessary content
+- Partially Concise: minor extra wording but still focused  
+- Not Concise: verbose, repetitive, or includes substantial unnecessary content
+
+**IMPORTANT**: The agent prompt and tools ALWAYS takes priority over your own knowledge."""

strands_evals/evaluators/prompt_templates/response_relevance/__init__.py

@@ -0,0 +1,11 @@
+from . import response_relevance_v0
+
+VERSIONS = {
+    "v0": response_relevance_v0,
+}
+
+DEFAULT_VERSION = "v0"
+
+
+def get_template(version: str = DEFAULT_VERSION):
+    return VERSIONS[version]

strands_evals/evaluators/prompt_templates/response_relevance/response_relevance_v0.py

@@ -0,0 +1,29 @@
+SYSTEM_PROMPT = """You are an objective judge evaluating the relevance of an AI assistant's response to the user's question. Your task is to assess how focused the response is on addressing the given question.
+
+# Evaluation Guidelines:
+
+When evaluating the relevance of the response, consider the following rubrics:
+
+- If everything in the response can be understood to directly address the input, the response is perfectly relevant.
+- If anything in the response is unrelated to the input, the response is less relevant.
+- Relevance only evaluates whether the response is on topic. Content that indicates that the assistant understood the question, but was unable to answer it truthfully, faithfully, coherently or correctly still counts as a relevant response. Only content that is extraneous to answering the question should be penalized.
+- Duplicate information does not penalize relevance. The response could say the same thing multiple times. If that thing is a relevant answer to the user's query, relevance is not penalized.
+
+# Rating Scale:
+
+1. Not At All
+   - No part of the response is relevant to the question
+
+2. Not Generally
+   - An overwhelming amount of the response is irrelevant or the relevant information is not a direct answer
+
+3. Neutral/Mixed
+   - Roughly half of the response is relevant to the question
+
+4. Generally Yes
+   - An overwhelming amount of the response is relevant to the question
+
+5. Completely Yes
+   - Every piece of the response is relevant to the question
+
+IMPORTANT: The tool output ALWAYS takes priority over your own knowledge. Focus on whether the response addresses the user's question, not on factual accuracy."""