strands-agents-evals 0.1.0__py3-none-any.whl

strands_evals/__init__.py

@@ -0,0 +1,22 @@
+__version__ = "0.1.0"
+
+from . import evaluators, extractors, generators, simulation, telemetry, types
+from .case import Case
+from .experiment import Experiment
+from .simulation import ActorSimulator, UserSimulator
+from .telemetry import StrandsEvalsTelemetry, get_tracer
+
+__all__ = [
+    "Experiment",
+    "Case",
+    "evaluators",
+    "extractors",
+    "types",
+    "generators",
+    "simulation",
+    "telemetry",
+    "StrandsEvalsTelemetry",
+    "get_tracer",
+    "ActorSimulator",
+    "UserSimulator",
+]

strands_evals/case.py

@@ -0,0 +1,53 @@
+import uuid
+
+from pydantic import BaseModel, Field
+from typing_extensions import Any, Generic, TypeVar
+
+from .types.evaluation import Interaction
+
+InputT = TypeVar("InputT")
+OutputT = TypeVar("OutputT")
+
+
+class Case(BaseModel, Generic[InputT, OutputT]):
+    """
+    A single test case, representing a row in an Experiment.
+
+    Each test case represents a single test scenario with inputs to test.
+    Optionally, a test case may contains a name, expected outputs, expected trajectory, expected interactions
+    and arbitrary metadata.
+
+    Attributes:
+        input: The input to the task. eg. the query to the agent
+        name: The name of the test case. This will be used to identify the test in the summary report.
+        session_id: The session ID for the test case. Automatically generates a UUID4 if not provided.
+        expected_output: The expected response given the input. eg. the agent's response
+        expected_trajectory: The expected trajectory of a task given the input. eg. sequence of tools
+        expected_interactions: The expected interaction sequence given the input (ideal for multi-agent systems).
+        metadata: Additional information about the test case.
+
+    Example:
+        case = Case[str,str](name="Simple Math",
+                        input="What is 2x2?",
+                        expected_output="2x2 is 4.",
+                        expected_trajectory=["calculator],
+                        metadata={"category": "math"})
+
+        simple_test_case = Case(input="What is 2x2?")
+
+        case_with_interaction = Case(
+                    input="What is 2x2?",
+                    expected_interactions=[
+                        {"agent_1":"Hello, what would you like to do?"},
+                        {"agent_2":"What is 2x2?"}
+                    ]
+                )
+    """
+
+    name: str | None = None
+    session_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    input: InputT
+    expected_output: OutputT | None = None
+    expected_trajectory: list[Any] | None = None
+    expected_interactions: list[Interaction] | None = None
+    metadata: dict[str, Any] | None = None

strands_evals/display/display_console.py

@@ -0,0 +1,150 @@
+from rich.console import Console
+from rich.panel import Panel
+from rich.prompt import Prompt
+from rich.table import Table
+from rich.tree import Tree
+
+console = Console()
+
+
+class CollapsibleTableReportDisplay:
+    """
+    Interactive console display for evaluation reports with expandable/collapsible test case details.
+
+    This class provides an interactive rich console interface for displaying evaluation results
+    with the ability to expand or collapse individual test cases to show or hide details.
+
+    Attributes:
+        items: Dictionary of test cases with their details and expansion state
+        overall_score: The overall evaluation score
+        include_input: Whether to display input values in the table
+        include_output: Whether to display output values in the table
+
+
+    items should follow the following structure:
+        {
+        "0": {"details": {
+                "name": str,
+                "score": float,
+                "test_pass: bool,
+                "reason": str,
+                ... # will display everything that's given like actual_output etc.
+                },
+                "detailed_results": list[EvaluationOutput],
+            },
+            "expanded": bool
+        }
+    """
+
+    def __init__(self, items: dict, overall_score: float):
+        """
+        Initialize the collapsible table display.
+
+        Args:
+            items: Dictionary of test cases with their details and expansion state
+            overall_score: The overall evaluation score
+        """
+        self.items = items
+        self.overall_score = overall_score
+
+    def display_items(self):
+        """
+        Display the evaluation report as a rich table with expandable/collapsible rows.
+
+        Renders a table showing test case results with expansion indicators.
+        Expanded rows show full details, while collapsed rows show minimal information.
+        """
+        overall_score_string = f"[bold blue]Overall Score: {self.overall_score:.2f}[/bold blue]"
+        pass_count = sum([1 if case["details"]["test_pass"] else 0 for case in self.items.values()])
+        pass_rate = pass_count / len(self.items)
+        overall_pass_rate = f"[bold blue]Pass Rate: {pass_rate}[/bold blue]"
+        spacing = "           "
+        console.print(Panel(f"{overall_score_string}{spacing}{overall_pass_rate}", title="📊 Evaluation Report"))
+
+        # Create Table and headers
+        table = Table(title="Test Case Results", show_lines=True)
+        colors_mapping = {
+            "index": "cyan",
+            "name": "magenta",
+            "score": "green",
+        }
+        headers = ["index"] + list(self.items["0"]["details"].keys())
+        for header in headers:
+            if header in colors_mapping:
+                table.add_column(header, style=colors_mapping[header])
+            else:
+                table.add_column(header, style="yellow")
+
+        for key, item in self.items.items():
+            symbol = "▼" if item["expanded"] else "▶"
+            case = item["details"]
+            pass_status = "✅" if case["test_pass"] else "❌"
+            other_fields = list(case.values())[3:]
+            if item["expanded"]:  # We always to render at least the index, name, score, test_pass, and reason
+                renderables = [
+                    f"{symbol} {key}",
+                    case.get("name", f"Test {key}"),
+                    case.get("score"),
+                    pass_status,
+                ] + other_fields
+            else:
+                renderables = [
+                    f"{symbol} {key}",
+                    case.get("name", f"Test {key}"),
+                    case.get("score"),
+                    pass_status,
+                ] + len(other_fields) * ["..."]
+            table.add_row(*renderables)
+
+        console.print(table)
+
+        for key, item in self.items.items():
+            if item["expanded"] and item.get("detailed_results"):
+                detailed_results = item["detailed_results"]
+                if len(detailed_results) > 1:  # Only show if multiple metrics
+                    tree = Tree(f"[bold cyan]📋 Detailed Metrics for Case {key}[/bold cyan]")
+                    for i, result in enumerate(detailed_results):
+                        status = "✅" if result.test_pass else "❌"
+                        metric_node = tree.add(f"[yellow]Metric {i + 1}[/yellow]: Score={result.score:.2f} {status}")
+                        if result.reason:
+                            metric_node.add(f"[dim]{result.reason}[/dim]")
+                    console.print(tree)
+                    console.print()
+
+    def run(self, static: bool = False):
+        """
+        Run the interactive display loop. If static, then the terminal will only display the report.
+
+        Args:
+            static: Whether to display only or allow interaction with the report.
+
+        Provides an interactive console interface where users can:
+        - Expand/collapse individual test cases by entering their number
+        - Expand all test cases with 'o'
+        - Collapse all test cases with 'c'
+        - Quit the interactive view with 'q'
+        """
+        while True:
+            console.clear()
+            self.display_items()
+
+            if static:
+                return
+
+            choice = Prompt.ask(
+                "\nEnter the test case number to expand/collapse it, o to expand all, "
+                "and c to collapse all (q to quit)."
+            )
+
+            if choice.lower() == "q":
+                break
+
+            if choice.lower() == "o":
+                for key in self.items:
+                    self.items[key]["expanded"] = True
+            elif choice.lower() == "c":
+                for key in self.items:
+                    self.items[key]["expanded"] = False
+            else:
+                if choice in self.items:
+                    self.items[choice]["expanded"] = not self.items[choice]["expanded"]

strands_evals/evaluators/__init__.py

@@ -0,0 +1,23 @@
+from .evaluator import Evaluator
+from .faithfulness_evaluator import FaithfulnessEvaluator
+from .goal_success_rate_evaluator import GoalSuccessRateEvaluator
+from .harmfulness_evaluator import HarmfulnessEvaluator
+from .helpfulness_evaluator import HelpfulnessEvaluator
+from .interactions_evaluator import InteractionsEvaluator
+from .output_evaluator import OutputEvaluator
+from .tool_parameter_accuracy_evaluator import ToolParameterAccuracyEvaluator
+from .tool_selection_accuracy_evaluator import ToolSelectionAccuracyEvaluator
+from .trajectory_evaluator import TrajectoryEvaluator
+
+__all__ = [
+    "Evaluator",
+    "OutputEvaluator",
+    "TrajectoryEvaluator",
+    "InteractionsEvaluator",
+    "HelpfulnessEvaluator",
+    "HarmfulnessEvaluator",
+    "GoalSuccessRateEvaluator",
+    "FaithfulnessEvaluator",
+    "ToolSelectionAccuracyEvaluator",
+    "ToolParameterAccuracyEvaluator",
+]

strands_evals/evaluators/evaluator.py

@@ -0,0 +1,182 @@
+import inspect
+import logging
+
+from strands.models.model import Model
+from typing_extensions import Any, Generic, TypeGuard, TypeVar, Union
+
+from ..extractors import TraceExtractor
+from ..types.evaluation import EvaluationData, EvaluationOutput
+from ..types.trace import AssistantMessage, Context, EvaluationLevel, Session, TextContent, ToolConfig, UserMessage
+
+logger = logging.getLogger(__name__)
+
+InputT = TypeVar("InputT")
+OutputT = TypeVar("OutputT")
+
+DEFAULT_BEDROCK_MODEL_ID = "us.anthropic.claude-sonnet-4-20250514-v1:0"
+
+
+class Evaluator(Generic[InputT, OutputT]):
+    """
+    Base class for evaluators.
+
+    Evaluators can assess the performance of a task on all test cases.
+    Subclasses must implement the `evaluate` method.
+    """
+
+    # Optional: subclasses can set this to enable trace parsing
+    evaluation_level: EvaluationLevel | None = None
+    _trace_extractor: TraceExtractor | None = None
+
+    def __init__(self, trace_extractor: TraceExtractor | None = None):
+        """Initialize evaluator with optional custom trace extractor.
+
+        Args:
+            trace_extractor: Custom trace extractor. If None and evaluation_level is set,
+                           a default TraceExtractor will be created.
+        """
+        self.aggregator = self._default_aggregator
+        if trace_extractor:
+            self._trace_extractor = trace_extractor
+        elif self.evaluation_level:
+            self._trace_extractor = TraceExtractor(self.evaluation_level)
+
+    def _get_model_id(self, model: Union[Model, str, None]) -> str:
+        """Extract model_id from a Model instance or string for serialization.
+
+        This helper method should be called in subclass __init__ methods that accept a model parameter.
+
+        Args:
+            model: Model instance, string model ID, or None
+
+        Returns:
+            The model ID string, DEFAULT_BEDROCK_MODEL_ID if None, or empty string for invalid types
+        """
+        if isinstance(model, str):
+            return model
+        elif isinstance(model, Model) and hasattr(model, "config") and isinstance(model.config, dict):
+            return model.config.get("model_id", "")
+        elif model is None:
+            return DEFAULT_BEDROCK_MODEL_ID
+        else:
+            return ""
+
+    @staticmethod
+    def _default_aggregator(outputs: list[EvaluationOutput]) -> tuple[float, bool, str]:
+        avg_score = sum(o.score for o in outputs) / len(outputs)
+        all_pass = all(o.test_pass for o in outputs)
+        combined_reason = " | ".join(o.reason for o in outputs if o.reason)
+        return avg_score, all_pass, combined_reason
+
+    def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        """
+        Evaluate the performance of the task on the given test cases.
+
+        Args:
+            evaluation_case: The test case with all of the neccessary context to be evaluated.
+
+        Raises:
+            NotImplementedError: This method is not implemented in the base class.
+        """
+        raise NotImplementedError("This method should be implemented in subclasses.")
+
+    async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        """
+        Evaluate the performance of the task on the given test cases asynchronously.
+
+        Args:
+            evaluation_case: The test case with all of the neccessary context to be evaluated.
+
+        Raises:
+            NotImplementedError: This method is not implemented in the base class.
+        """
+        raise NotImplementedError(
+            "This method should be implemented in subclasses, especially if you want to run evaluations asynchronously."
+        )
+
+    def _parse_trajectory(self, evaluation_case: EvaluationData[InputT, OutputT]) -> Any:
+        """Parse Session trajectory using TraceExtractor."""
+        if not self._trace_extractor:
+            raise ValueError("No trace extractor configured. Set evaluation_level or provide trace_extractor.")
+
+        trajectory = evaluation_case.actual_trajectory
+        if not isinstance(trajectory, Session):
+            raise TypeError(
+                f"Trace parsing requires actual_trajectory to be a Session object, got {type(trajectory).__name__}."
+            )
+
+        return self._trace_extractor.extract(trajectory)
+
+    def _format_tools(self, tools: list[ToolConfig]) -> str:
+        """Format available tools for prompt display."""
+        return "\n".join([f"- {tool.name}: {tool.description or 'No description'}" for tool in tools])
+
+    def _format_session_history(self, contexts: list[Context]) -> str:
+        """Format session history with tool executions for prompt display."""
+        lines = []
+        for ctx in contexts:
+            lines.append(f"User: {ctx.user_prompt.text}")
+            if ctx.tool_execution_history:
+                for tool_exec in ctx.tool_execution_history:
+                    lines.append(f"Action: {tool_exec.tool_call.name}({tool_exec.tool_call.arguments})")
+                    lines.append(f"Tool: {tool_exec.tool_result.content}")
+            lines.append(f"Assistant: {ctx.agent_response.text}")
+        return "\n".join(lines)
+
+    def _has_text_content(self, msg: UserMessage | AssistantMessage) -> TypeGuard[UserMessage | AssistantMessage]:
+        """Check if a message object has accessible text content.
+
+        Args:
+            msg: Message object to check (UserMessage or AssistantMessage)
+
+        Returns:
+            True if msg has content attribute with at least one item that is TextContent
+        """
+        return (
+            hasattr(msg, "content")
+            and bool(msg.content)
+            and len(msg.content) > 0
+            and isinstance(msg.content[0], TextContent)
+        )
+
+    @classmethod
+    def get_type_name(cls) -> str:
+        """
+        Get the name of the evaluator type.
+
+        Returns:
+            str: The name of the evaluator type.
+        """
+        return cls.__name__
+
+    def to_dict(self) -> dict:
+        """
+        Convert the evaluator into a dictionary.
+
+        Returns:
+            dict: A dictionary containing the evaluator's information. Omit private attributes
+            (attributes starting with '_') and attributes with default values.
+        """
+
+        _dict = {"evaluator_type": self.get_type_name()}
+
+        # Get default values from __init__ signature
+        sig = inspect.signature(self.__class__.__init__)
+        defaults = {k: v.default for k, v in sig.parameters.items() if v.default != inspect.Parameter.empty}
+        exclude_attrs = {"aggregator"}
+        for k, v in self.__dict__.items():
+            if not k.startswith("_") and k not in exclude_attrs:
+                # Handle model attribute specially
+                if k == "model":
+                    if isinstance(v, Model):
+                        # Serialize Model instance to model_id
+                        _dict["model_id"] = self._get_model_id(v)
+                    elif v is None and "model" in defaults and defaults["model"] is None:
+                        # model=None is default, serialize as model_id with default value
+                        _dict["model_id"] = self._get_model_id(None)
+                    elif v is not None:
+                        # String model ID, include as-is
+                        _dict[k] = v
+                elif k not in defaults or v != defaults[k]:
+                    _dict[k] = v
+        return _dict

strands_evals/evaluators/faithfulness_evaluator.py

@@ -0,0 +1,116 @@
+from enum import Enum
+
+from pydantic import BaseModel, Field
+from strands import Agent
+from strands.models.model import Model
+from typing_extensions import TypeVar, Union
+
+from ..types.evaluation import EvaluationData, EvaluationOutput
+from ..types.trace import EvaluationLevel, TraceLevelInput
+from .evaluator import Evaluator
+from .prompt_templates.faithfulness import get_template
+
+InputT = TypeVar("InputT")
+OutputT = TypeVar("OutputT")
+
+
+class FaithfulnessScore(str, Enum):
+    """Categorical faithfulness ratings."""
+
+    NOT_AT_ALL = "Not At All"
+    NOT_GENERALLY = "Not Generally"
+    NEUTRAL = "Neutral/Mixed"
+    GENERALLY_YES = "Generally Yes"
+    COMPLETELY_YES = "Completely Yes"
+
+
+class FaithfulnessRating(BaseModel):
+    """Structured output for faithfulness evaluation."""
+
+    reasoning: str = Field(description="Step by step reasoning to derive the final score")
+    score: FaithfulnessScore = Field(description="Categorical faithfulness rating")
+
+
+class FaithfulnessEvaluator(Evaluator[InputT, OutputT]):
+    """Evaluates faithfulness of agent responses against conversation history."""
+
+    evaluation_level = EvaluationLevel.TRACE_LEVEL
+
+    _score_mapping = {
+        FaithfulnessScore.NOT_AT_ALL: 0.0,
+        FaithfulnessScore.NOT_GENERALLY: 0.25,
+        FaithfulnessScore.NEUTRAL: 0.5,
+        FaithfulnessScore.GENERALLY_YES: 0.75,
+        FaithfulnessScore.COMPLETELY_YES: 1.0,
+    }
+
+    def __init__(
+        self,
+        version: str = "v0",
+        model: Union[Model, str, None] = None,
+        system_prompt: str | None = None,
+    ):
+        super().__init__()
+        self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT
+        self.version = version
+        self.model = model
+
+    def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        parsed_input = self._get_last_turn(evaluation_case)
+        prompt = self._format_prompt(parsed_input)
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        rating = evaluator_agent.structured_output(FaithfulnessRating, prompt)
+        normalized_score = self._score_mapping[rating.score]
+        result = EvaluationOutput(
+            score=normalized_score,
+            test_pass=normalized_score >= 0.5,
+            reason=rating.reasoning,
+            label=rating.score,
+        )
+        return [result]
+
+    async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        parsed_input = self._get_last_turn(evaluation_case)
+        prompt = self._format_prompt(parsed_input)
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        rating = await evaluator_agent.structured_output_async(FaithfulnessRating, prompt)
+        normalized_score = self._score_mapping[rating.score]
+        result = EvaluationOutput(
+            score=normalized_score,
+            test_pass=normalized_score >= 0.5,
+            reason=rating.reasoning,
+            label=rating.score,
+        )
+        return [result]
+
+    def _get_last_turn(self, evaluation_case: EvaluationData[InputT, OutputT]) -> TraceLevelInput:
+        """Extract the most recent turn from the conversation for evaluation."""
+        parsed_inputs = self._parse_trajectory(evaluation_case)
+        if not parsed_inputs:
+            raise ValueError(
+                "No turn-level inputs could be parsed from the trajectory. "
+                "Ensure actual_trajectory is a Session with at least one AgentInvocationSpan."
+            )
+        return parsed_inputs[-1]
+
+    def _format_prompt(self, parsed_input: TraceLevelInput) -> str:
+        """Format evaluation prompt from parsed turn data."""
+        parts = []
+
+        if parsed_input.session_history:
+            history_lines = []
+            for msg in parsed_input.session_history:
+                if isinstance(msg, list):
+                    # Handle tool execution lists
+                    for tool_exec in msg:
+                        history_lines.append(f"Action: {tool_exec.tool_call.name}({tool_exec.tool_call.arguments})")
+                        history_lines.append(f"Tool: {tool_exec.tool_result.content}")
+                else:
+                    text = msg.content[0].text if msg.content and hasattr(msg.content[0], "text") else ""
+                    history_lines.append(f"{msg.role.value.capitalize()}: {text}")
+            history_str = "\n".join(history_lines)
+            parts.append(f"# Conversation History:\n{history_str}")
+
+        parts.append(f"# Assistant's Response:\n{parsed_input.agent_response.text}")
+
+        return "\n\n".join(parts)

strands_evals/evaluators/goal_success_rate_evaluator.py

@@ -0,0 +1,90 @@
+from enum import Enum
+
+from pydantic import BaseModel, Field
+from strands import Agent
+from strands.models.model import Model
+from typing_extensions import TypeVar, Union
+
+from ..types.evaluation import EvaluationData, EvaluationOutput
+from ..types.trace import EvaluationLevel, SessionLevelInput
+from .evaluator import Evaluator
+from .prompt_templates.goal_success_rate import get_template
+
+InputT = TypeVar("InputT")
+OutputT = TypeVar("OutputT")
+
+
+class GoalSuccessScore(str, Enum):
+    """Binary goal success ratings."""
+
+    YES = "Yes"
+    NO = "No"
+
+
+class GoalSuccessRating(BaseModel):
+    """Structured output for goal success evaluation."""
+
+    reasoning: str = Field(description="Step by step reasoning to derive the final score")
+    score: GoalSuccessScore = Field(description="Score should be one of 'Yes' or 'No'")
+
+
+class GoalSuccessRateEvaluator(Evaluator[InputT, OutputT]):
+    """Evaluates whether all user goals were successfully achieved in a conversation."""
+
+    evaluation_level = EvaluationLevel.SESSION_LEVEL
+
+    _score_mapping = {
+        GoalSuccessScore.YES: 1.0,
+        GoalSuccessScore.NO: 0.0,
+    }
+
+    def __init__(
+        self,
+        version: str = "v0",
+        model: Union[Model, str, None] = None,
+        system_prompt: str | None = None,
+    ):
+        super().__init__()
+        self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT
+        self.version = version
+        self.model = model
+
+    def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        session_input = self._parse_trajectory(evaluation_case)
+        prompt = self._format_prompt(session_input)
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        rating = evaluator_agent.structured_output(GoalSuccessRating, prompt)
+        normalized_score = self._score_mapping[rating.score]
+        result = EvaluationOutput(
+            score=normalized_score,
+            test_pass=normalized_score >= 1.0,
+            reason=rating.reasoning,
+            label=rating.score,
+        )
+        return [result]
+
+    async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
+        session_input = self._parse_trajectory(evaluation_case)
+        prompt = self._format_prompt(session_input)
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        rating = await evaluator_agent.structured_output_async(GoalSuccessRating, prompt)
+        normalized_score = self._score_mapping[rating.score]
+        result = EvaluationOutput(
+            score=normalized_score,
+            test_pass=normalized_score >= 1.0,
+            reason=rating.reasoning,
+            label=rating.score,
+        )
+        return [result]
+
+    def _format_prompt(self, session_input: SessionLevelInput) -> str:
+        """Format evaluation prompt from session-level input."""
+        parts = []
+
+        if session_input.available_tools:
+            parts.append(f"# Available tools\n{self._format_tools(session_input.available_tools)}")
+
+        if session_input.session_history:
+            parts.append(f"# Conversation record\n{self._format_session_history(session_input.session_history)}")
+
+        return "\n\n".join(parts)