strands-agents-evals 0.1.0__py3-none-any.whl

strands_evals/tools/evaluation_tools.py

@@ -0,0 +1,67 @@
+from strands import tool
+
+
+@tool
+def exact_match_scorer(actual_trajectory: list, expected_trajectory: list) -> float:
+    """
+    Score the trajectory based on exact match. A score of 0 indicates 0 steps matched and 1
+    indicates a perfect match.
+
+    Args:
+        actual_trajectory: The actual trajectory.
+        expected_trajectory: The expected trajectory.
+
+    Returns:
+        The score of the trajectory.
+    """
+    correct = 0
+    for actual, expected in zip(actual_trajectory, expected_trajectory, strict=False):
+        if actual == expected:
+            correct += 1
+
+    return correct / len(expected_trajectory)
+
+
+@tool
+def in_order_match_scorer(actual_trajectory: list, expected_trajectory: list) -> float:
+    """
+    Score based on correct actions in correct order, allows extra actions.
+
+    Args:
+        actual_trajectory: The actual trajectory.
+        expected_trajectory: The expected trajectory.
+
+    Returns:
+        The score of the trajectory.
+    """
+    if not expected_trajectory:
+        return 1.0
+
+    expected_idx = 0
+    for action in actual_trajectory:
+        if expected_idx < len(expected_trajectory) and action == expected_trajectory[expected_idx]:
+            expected_idx += 1
+
+    return expected_idx / len(expected_trajectory)
+
+
+@tool
+def any_order_match_scorer(actual_trajectory: list, expected_trajectory: list) -> float:
+    """
+    Score based on correct actions in any order, allows extra actions.
+
+    Args:
+        actual_trajectory: The actual trajectory.
+        expected_trajectory: The expected trajectory.
+
+    Returns:
+        The score of the trajectory.
+    """
+    if not expected_trajectory:
+        return 1.0
+
+    expected_set = set(expected_trajectory)
+    actual_set = set(actual_trajectory)
+    matched = len(expected_set.intersection(actual_set))
+
+    return matched / len(expected_trajectory)

strands_evals/types/__init__.py

@@ -0,0 +1,11 @@
+from .evaluation import EvaluationData, EvaluationOutput, Interaction, TaskOutput
+from .simulation import ActorProfile, ActorResponse
+
+__all__ = [
+    "Interaction",
+    "TaskOutput",
+    "EvaluationData",
+    "EvaluationOutput",
+    "ActorProfile",
+    "ActorResponse",
+]

strands_evals/types/evaluation.py

@@ -0,0 +1,105 @@
+from pydantic import BaseModel
+from typing_extensions import Any, Generic, TypedDict, TypeVar, Union
+
+from .trace import Session
+
+InputT = TypeVar("InputT")
+OutputT = TypeVar("OutputT")
+
+
+class Interaction(TypedDict, total=False):
+    """
+    Represents a single interaction in a multi-agent or multi-step system.
+
+    Used to capture the communication flow and dependencies between different
+    components (agents, tools, or processing nodes) during task execution.
+    All fields are optional to accommodate different interaction patterns.
+
+    Attributes:
+        node_name: Identifier for the agent, tool, or component involved in this interaction
+        dependencies: List of other nodes/components this interaction depends on or references
+        messages: Sequence of messages, responses, or communication exchanged during this interaction
+
+    Example:
+        interaction = {
+            "node_name": "calculator_agent",
+            "dependencies": ["input_parser", "math_validator"],
+            "messages": ["Calculate 2+2"]
+        }
+    """
+
+    node_name: str
+    dependencies: list
+    messages: list
+
+
+class TaskOutput(TypedDict, total=False):
+    """
+    Structured output format for task functions that return complex results.
+
+    Used when task functions need to return more than just the output response,
+    such as trajectory or interaction history. All fields are optional
+    to support different task complexity levels.
+
+    Attributes:
+        output: The primary response or result from the task
+        trajectory: Sequence of steps, tools, or actions taken during task execution
+        interactions: Communication flow between agents or components during execution
+        input: A new input to replace the original in the evaluation, will not mutate the original test case
+
+    Example:
+        task_result = {
+            "output": "The answer is 42",
+            "trajectory": ["calculator", "validator"],
+            "interactions": [{"node_name": "math_agent", "messages": ["Computing..."]}]
+        }
+    """
+
+    output: Any
+    trajectory: list[Any]
+    interactions: list[Interaction]
+    input: Any
+
+
+class EvaluationData(BaseModel, Generic[InputT, OutputT]):
+    """
+    A record of all of the context for the evaluator to evaluate a test case.
+
+    Attributes:
+        input: The input to the task. eg. the query to the agent
+        actual_output: The actual response given the input.
+        expected_output: The expected response given the input.
+        actual_trajectory: The actual trajectory of a task given the input.
+        expected_trajectory: The expected trajectory of a task given the input.
+        name: The name of the test case. This will be used to identify the test in the summary report.
+        metadata: Additional information about the test case.
+        actual_interactions: The actual interaction sequence given the input.
+        expected_interactions: The expected interaction sequence given the input.
+    """
+
+    input: InputT
+    actual_output: OutputT | None = None
+    name: str | None = None
+    expected_output: OutputT | None = None
+    expected_trajectory: Union[list[Any], Session, None] = None
+    actual_trajectory: Union[list[Any], Session, None] = None
+    metadata: dict[str, Any] | None = None
+    actual_interactions: list[Interaction] | None = None
+    expected_interactions: list[Interaction] | None = None
+
+
+class EvaluationOutput(BaseModel):
+    """
+    Structured output for LLM-based judge.
+
+    Attributes:
+        score: The score of the test case.
+        test_pass: Whether the test pass or fail.
+        reason: The reason for the score for each test case.
+        label: The categorical label corresponding to the score.
+    """
+
+    score: float
+    test_pass: bool
+    reason: str | None = None
+    label: str | None = None

strands_evals/types/evaluation_report.py

@@ -0,0 +1,244 @@
+import json
+from pathlib import Path
+
+from pydantic import BaseModel
+from typing_extensions import TypeVar
+
+from ..display.display_console import CollapsibleTableReportDisplay
+from ..types.evaluation import EvaluationOutput
+
+InputT = TypeVar("InputT")
+OutputT = TypeVar("OutputT")
+
+
+class EvaluationReport(BaseModel):
+    """
+    A report of the evaluation of a task.
+
+    Attributes:
+        overall_score: The overall score of the task.
+        scores: A list of the score for each test case in order.
+        cases: A list of records for each test case.
+        test_passes: A list of booleans indicating whether the test pass or fail.
+        reasons: A list of reason for each test case.
+    """
+
+    overall_score: float
+    scores: list[float]
+    cases: list[dict]
+    test_passes: list[bool]
+    reasons: list[str] = []
+    detailed_results: list[list[EvaluationOutput]] = []
+
+    def _display(
+        self,
+        static: bool = True,
+        include_input: bool = True,
+        include_actual_output: bool = False,
+        include_expected_output: bool = False,
+        include_expected_trajectory: bool = False,
+        include_actual_trajectory: bool = False,
+        include_actual_interactions: bool = False,
+        include_expected_interactions: bool = False,
+        include_meta: bool = False,
+    ):
+        """
+        Render an interface of the report with as much details as configured using Rich.
+
+        Args:
+            static: Whether to render the interface as interactive or static.
+            include_input (Defaults to True): Include the input in the display.
+            include_actual_output (Defaults to False): Include the actual output in the display.
+            include_expected_output (Defaults to False): Include the expected output in the display.
+            include_expected_trajectory (Defaults to False): Include the expected trajectory in the display.
+            include_actual_trajectory (Defaults to False): Include the actual trajectory in the display.
+            include_actual_interactions (Defaults to False): Include the actual interactions in the display.
+            include_expected_interactions (Defaults to False): Include the expected interactions in the display.
+            include_meta (Defaults to False): Include metadata in the display.
+
+        Note:
+            This method provides an interactive console interface where users can expand or collapse
+            individual test cases to view more or less detail.
+        """
+        report_data = {}
+        for i in range(len(self.scores)):
+            name = self.cases[i].get("name", f"Test {i + 1}")
+            reason = self.reasons[i] if i < len(self.reasons) else "N/A"
+            details_dict = {
+                "name": name,
+                "score": f"{self.scores[i]:.2f}",
+                "test_pass": self.test_passes[i],
+                "reason": reason,
+            }
+            if include_input:
+                details_dict["input"] = str(self.cases[i].get("input"))
+            if include_actual_output:
+                details_dict["actual_output"] = str(self.cases[i].get("actual_output"))
+            if include_expected_output:
+                details_dict["expected_output"] = str(self.cases[i].get("expected_output"))
+            if include_actual_trajectory:
+                details_dict["actual_trajectory"] = str(self.cases[i].get("actual_trajectory"))
+            if include_expected_trajectory:
+                details_dict["expected_trajectory"] = str(self.cases[i].get("expected_trajectory"))
+            if include_actual_interactions:
+                details_dict["actual_interactions"] = str(self.cases[i].get("actual_interactions"))
+            if include_expected_interactions:
+                details_dict["expected_interactions"] = str(self.cases[i].get("expected_interactions"))
+            if include_meta:
+                details_dict["metadata"] = str(self.cases[i].get("metadata"))
+
+            report_data[str(i)] = {
+                "details": details_dict,
+                "detailed_results": self.detailed_results[i] if i < len(self.detailed_results) else [],  # NEW
+                "expanded": False,
+            }
+
+        display_console = CollapsibleTableReportDisplay(items=report_data, overall_score=self.overall_score)
+        display_console.run(static=static)
+
+    def display(
+        self,
+        include_input: bool = True,
+        include_actual_output: bool = False,
+        include_expected_output: bool = False,
+        include_expected_trajectory: bool = False,
+        include_actual_trajectory: bool = False,
+        include_actual_interactions: bool = False,
+        include_expected_interactions: bool = False,
+        include_meta: bool = False,
+    ):
+        """
+        Render the report with as much details as configured using Rich. Use run_display if want
+        to interact with the table.
+
+        Args:
+            include_input: Whether to include the input in the display. Defaults to True.
+            include_actual_output (Defaults to False): Include the actual output in the display.
+            include_expected_output (Defaults to False): Include the expected output in the display.
+            include_expected_trajectory (Defaults to False): Include the expected trajectory in the display.
+            include_actual_trajectory (Defaults to False): Include the actual trajectory in the display.
+            include_actual_interactions (Defaults to False): Include the actual interactions in the display.
+            include_expected_interactions (Defaults to False): Include the expected interactions in the display.
+            include_meta (Defaults to False): Include metadata in the display.
+        """
+        self._display(
+            static=True,
+            include_input=include_input,
+            include_actual_output=include_actual_output,
+            include_expected_output=include_expected_output,
+            include_expected_trajectory=include_expected_trajectory,
+            include_actual_trajectory=include_actual_trajectory,
+            include_actual_interactions=include_actual_interactions,
+            include_expected_interactions=include_expected_interactions,
+            include_meta=include_meta,
+        )
+
+    def run_display(
+        self,
+        include_input: bool = True,
+        include_actual_output: bool = False,
+        include_expected_output: bool = False,
+        include_expected_trajectory: bool = False,
+        include_actual_trajectory: bool = False,
+        include_actual_interactions: bool = False,
+        include_expected_interactions: bool = False,
+        include_meta: bool = False,
+    ):
+        """
+        Render the report interactively with as much details as configured using Rich.
+
+        Args:
+            include_input: Whether to include the input in the display. Defaults to True.
+            include_actual_output (Defaults to False): Include the actual output in the display.
+            include_expected_output (Defaults to False): Include the expected output in the display.
+            include_expected_trajectory (Defaults to False): Include the expected trajectory in the display.
+            include_actual_trajectory (Defaults to False): Include the actual trajectory in the display.
+            include_actual_interactions (Defaults to False): Include the actual interactions in the display.
+            include_expected_interactions (Defaults to False): Include the expected interactions in the display.
+            include_meta (Defaults to False): Include metadata in the display.
+        """
+        self._display(
+            static=False,
+            include_input=include_input,
+            include_actual_output=include_actual_output,
+            include_expected_output=include_expected_output,
+            include_expected_trajectory=include_expected_trajectory,
+            include_actual_trajectory=include_actual_trajectory,
+            include_actual_interactions=include_actual_interactions,
+            include_expected_interactions=include_expected_interactions,
+            include_meta=include_meta,
+        )
+
+    def to_dict(self):
+        """
+        Returns a dictionary representation of the report.
+        """
+        return self.model_dump()
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        """
+        Create an EvaluationReport instance from a dictionary.
+
+        Args:
+            data: A dictionary containing the report data.
+        """
+        return cls.model_validate(data)
+
+    def to_file(self, path: str):
+        """
+        Write the report to a JSON file.
+
+        Args:
+            path: The file path where the report will be saved. Can be:
+                  - A filename only (e.g., "foo.json" or "foo") - saves in current working directory
+                  - A relative path (e.g., "relative_path/foo.json") - saves relative to current working directory
+                  - An absolute path (e.g., "/path/to/dir/foo.json") - saves in exact directory
+
+                  If no extension is provided, ".json" will be added automatically.
+                  Only .json format is supported.
+
+        Raises:
+            ValueError: If the path has a non-JSON extension.
+        """
+        file_path = Path(path)
+
+        if file_path.suffix:
+            if file_path.suffix != ".json":
+                raise ValueError(
+                    f"Only .json format is supported. Got path with extension: {path}. "
+                    f"Please use a .json extension or provide a path without an extension."
+                )
+        else:
+            file_path = file_path.with_suffix(".json")
+
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        with open(file_path, "w") as f:
+            json.dump(self.to_dict(), f, indent=2)
+
+    @classmethod
+    def from_file(cls, path: str):
+        """
+        Create an EvaluationReport instance from a JSON file.
+
+        Args:
+            path: Path to the JSON file.
+
+        Return:
+            An EvaluationReport object.
+
+        Raises:
+            ValueError: If the file does not have a .json extension.
+        """
+        file_path = Path(path)
+
+        if file_path.suffix != ".json":
+            raise ValueError(
+                f"Only .json format is supported. Got file: {path}. Please provide a path with .json extension."
+            )
+
+        with open(file_path, "r") as f:
+            data = json.load(f)
+
+        return cls.from_dict(data)

strands_evals/types/simulation/__init__.py

@@ -0,0 +1,5 @@
+"""Data models for actor simulation."""
+
+from .actor import ActorProfile, ActorResponse
+
+__all__ = ["ActorProfile", "ActorResponse"]

strands_evals/types/simulation/actor.py

@@ -0,0 +1,34 @@
+from pydantic import BaseModel, Field
+from typing_extensions import Any
+
+
+class ActorProfile(BaseModel):
+    """
+    Profile for actor simulation.
+
+    Attributes:
+        traits: Dictionary of actor characteristics and attributes.
+        context: Supplementary background information about the actor.
+        actor_goal: What the actor ultimately wants to achieve in the interaction.
+    """
+
+    traits: dict[str, Any] = Field(..., description="Actor traits for simulation")
+    context: str = Field(..., description="Supplementary actor background details")
+    actor_goal: str = Field(
+        ...,
+        description="What the actor ultimately wants to achieve in this interaction - "
+        "should be specific, actionable, and written from the actor's perspective",
+    )
+
+
+class ActorResponse(BaseModel):
+    """
+    Structured response from an actor.
+
+    Attributes:
+        reasoning: Internal reasoning process for the response.
+        message: The actual message content from the actor.
+    """
+
+    reasoning: str = Field(..., description="Reasoning for the actor's response")
+    message: str = Field(..., description="Message from the actor")

strands_evals/types/trace.py

@@ -0,0 +1,205 @@
+"""
+Generic trace types for agent observability.
+
+These types represent standard observability primitives for agents.
+"""
+
+from datetime import datetime, timezone
+from enum import Enum
+
+from pydantic import BaseModel, field_serializer
+from typing_extensions import Mapping, Sequence, TypeAlias, Union
+
+
+class Role(str, Enum):
+    USER = "user"
+    ASSISTANT = "assistant"
+
+
+class ContentType(str, Enum):
+    TEXT = "text"
+    TOOL_USE = "tool_use"
+    TOOL_RESULT = "tool_result"
+
+
+class SpanType(str, Enum):
+    INFERENCE = "inference"
+    TOOL_EXECUTION = "execute_tool"
+    AGENT_INVOCATION = "invoke_agent"
+
+
+class EvaluationLevel(str, Enum):
+    """Type of evaluation based on trace granularity."""
+
+    SESSION_LEVEL = "Session"
+    TRACE_LEVEL = "Trace"
+    TOOL_LEVEL = "ToolCall"
+
+
+class ToolCall(BaseModel):
+    name: str
+    arguments: dict
+    tool_call_id: str | None = None
+
+
+class ToolResult(BaseModel):
+    content: str
+    error: str | None = None
+    tool_call_id: str | None = None
+
+
+class ToolConfig(BaseModel):
+    name: str
+    description: str | None = None
+    parameters: dict | None = None
+
+
+class TextContent(BaseModel):
+    content_type: ContentType = ContentType.TEXT
+    text: str
+
+
+class ToolCallContent(ToolCall):
+    content_type: ContentType = ContentType.TOOL_USE
+
+
+class ToolResultContent(ToolResult):
+    content_type: ContentType = ContentType.TOOL_RESULT
+
+
+class UserMessage(BaseModel):
+    role: Role = Role.USER
+    content: list[Union[TextContent, ToolResultContent]]
+
+
+class AssistantMessage(BaseModel):
+    role: Role = Role.ASSISTANT
+    content: list[Union[TextContent, ToolCallContent]]
+
+
+class SpanInfo(BaseModel):
+    trace_id: str | None = None
+    span_id: str | None = None
+    session_id: str
+    parent_span_id: str | None = None
+    start_time: datetime
+    end_time: datetime
+
+    @field_serializer("start_time", "end_time")
+    def serialize_datetime_utc(self, dt: datetime) -> str:
+        """Serialize datetime fields in UTC timezone with ISO format."""
+        # Convert to UTC if timezone-aware, otherwise assume it's already UTC
+        if dt.tzinfo is not None:
+            utc_dt = dt.astimezone(timezone.utc)
+        else:
+            utc_dt = dt.replace(tzinfo=timezone.utc)
+        # Return ISO format string with 'Z' suffix for UTC
+        return utc_dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+class BaseSpan(BaseModel):
+    span_info: SpanInfo
+    metadata: dict | None = {}
+
+
+class InferenceSpan(BaseSpan):
+    span_type: SpanType = SpanType.INFERENCE
+    messages: list[Union[UserMessage, AssistantMessage]]
+
+
+class ToolExecutionSpan(BaseSpan):
+    span_type: SpanType = SpanType.TOOL_EXECUTION
+    tool_call: ToolCall
+    tool_result: ToolResult
+
+
+class AgentInvocationSpan(BaseSpan):
+    span_type: SpanType = SpanType.AGENT_INVOCATION
+    user_prompt: str
+    agent_response: str
+    available_tools: list[ToolConfig]
+
+
+SpanUnion: TypeAlias = Union[InferenceSpan, ToolExecutionSpan, AgentInvocationSpan]
+
+
+class Trace(BaseModel):
+    spans: list[SpanUnion]
+    trace_id: str
+    session_id: str
+
+
+class Session(BaseModel):
+    traces: list[Trace]
+    session_id: str
+
+
+class BaseEvaluationInput(BaseModel):
+    """Base class for all evaluation inputs"""
+
+    span_info: SpanInfo
+
+
+class ToolExecution(BaseModel):
+    tool_call: ToolCall
+    tool_result: ToolResult
+
+
+class Context(BaseModel):
+    user_prompt: TextContent
+    agent_response: TextContent
+    tool_execution_history: list[ToolExecution] | None = None
+
+
+class SessionLevelInput(BaseEvaluationInput):
+    """Input for session-level evaluators"""
+
+    session_history: list[Context]
+    available_tools: list[ToolConfig] | None = None
+
+
+class TraceLevelInput(BaseEvaluationInput):
+    """Input for trace-level evaluators"""
+
+    agent_response: TextContent
+    session_history: list[Union[UserMessage, list[ToolExecution], AssistantMessage]]
+
+
+class ToolLevelInput(BaseEvaluationInput):
+    """Input for tool-level evaluators"""
+
+    available_tools: list[ToolConfig]
+    tool_execution_details: ToolExecutionSpan
+    session_history: list[Union[UserMessage, list[ToolExecution], AssistantMessage]]
+
+
+class EvaluatorScore(BaseModel):
+    explanation: str
+    value: Union[int, float] | None = None
+    error: str | None = None
+
+
+class TokenUsage(BaseModel):
+    cache_read_input_tokens: int
+    cache_creation_input_tokens: int
+    input_tokens: int
+    output_tokens: int
+    total_tokens: int
+
+
+class EvaluatorResult(BaseModel):
+    span_info: SpanInfo
+    evaluator_name: str
+    score: EvaluatorScore
+    token_usage: TokenUsage | None = None
+
+
+class EvaluationResponse(BaseModel):
+    evaluator_results: list[EvaluatorResult]
+
+
+AttributeValue = Mapping[
+    str, str | bool | int | float | Sequence[str] | Sequence[bool] | Sequence[int] | Sequence[float]
+]
+
+Attributes = Mapping[str, AttributeValue] | None