strands-agents-evals 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

strands_evals/experiment.py

@@ -4,8 +4,16 @@ import logging
 import os
 from collections.abc import Callable
 from pathlib import Path
+from typing import cast
 
 from opentelemetry.trace import format_trace_id
+from tenacity import (
+    RetryError,
+    retry,
+    retry_if_exception,
+    stop_after_attempt,
+    wait_exponential,
+)
 from typing_extensions import Any, Generic, TypeVar
 
 from .case import Case
@@ -17,6 +25,7 @@ from .telemetry import get_tracer, serialize
 from .telemetry._cloudwatch_logger import _send_to_cloudwatch
 from .types.evaluation import EvaluationData
 from .types.evaluation_report import EvaluationReport
+from .utils import is_throttling_error
 
 InputT = TypeVar("InputT")
 OutputT = TypeVar("OutputT")
@@ -24,6 +33,11 @@ OutputT = TypeVar("OutputT")
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)
 
+# Retry configuration for handling throttling
+_MAX_RETRY_ATTEMPTS = 6
+_INITIAL_RETRY_DELAY = 4
+_MAX_RETRY_DELAY = 240  # 4 minutes
+
 
 def _get_label_from_score(evaluator: Evaluator, score: float) -> str:
     """
@@ -139,6 +153,34 @@ class Experiment(Generic[InputT, OutputT]):
         """
         self._evaluators = new_evaluators
 
+    def _record_evaluator_result(
+        self,
+        evaluator_data: dict[str, dict[str, list]],
+        eval_name: str,
+        case_data: dict,
+        test_pass: bool,
+        score: float,
+        reason: str,
+        detailed_results: list,
+    ):
+        """
+        Record a single evaluator result in the evaluator_data dictionary.
+
+        Args:
+            evaluator_data: Dictionary to store evaluator results
+            eval_name: Name of the evaluator
+            case_data: Case data (already serialized as dict)
+            test_pass: Whether the test passed
+            score: Evaluation score
+            reason: Reason/explanation for the result
+            detailed_results: Detailed evaluation outputs
+        """
+        evaluator_data[eval_name]["cases"].append(case_data)
+        evaluator_data[eval_name]["test_passes"].append(test_pass)
+        evaluator_data[eval_name]["scores"].append(score)
+        evaluator_data[eval_name]["reasons"].append(reason)
+        evaluator_data[eval_name]["detailed_results"].append(detailed_results)
+
     def _run_task(
         self, task: Callable[[Case[InputT, OutputT]], OutputT | dict[str, Any]], case: Case[InputT, OutputT]
     ) -> EvaluationData[InputT, OutputT]:
@@ -240,92 +282,166 @@ class Experiment(Generic[InputT, OutputT]):
             trace_id = None
 
             try:
-                with self._tracer.start_as_current_span(
-                    f"execute_case {case_name}",
-                ) as case_span:
-                    evaluation_context = await self._run_task_async(task, case)
-                    case_span.set_attributes(
-                        {
-                            "gen_ai.evaluation.data.input": serialize(evaluation_context.input),
-                            "gen_ai.evaluation.data.expected_output": serialize(evaluation_context.expected_output),
-                            "gen_ai.evaluation.data.actual_output": serialize(evaluation_context.actual_output),
-                            "gen_ai.evaluation.data.has_trajectory": (evaluation_context.actual_trajectory is not None),
-                            "gen_ai.evaluation.data.has_interactions": (
-                                evaluation_context.actual_interactions is not None
-                            ),
-                        }
+
+                @retry(
+                    retry=retry_if_exception(is_throttling_error),
+                    stop=stop_after_attempt(_MAX_RETRY_ATTEMPTS),
+                    wait=wait_exponential(multiplier=_INITIAL_RETRY_DELAY, max=_MAX_RETRY_DELAY),
+                    reraise=True,
+                )
+                async def _run_task_with_retry(task=task, case=case):
+                    return await self._run_task_async(task, case)
+
+                try:
+                    with self._tracer.start_as_current_span(
+                        f"execute_case {case_name}",
+                    ) as case_span:
+                        evaluation_context = await _run_task_with_retry()
+                        case_span.set_attributes(
+                            {
+                                "gen_ai.evaluation.data.input": serialize(evaluation_context.input),
+                                "gen_ai.evaluation.data.expected_output": serialize(evaluation_context.expected_output),
+                                "gen_ai.evaluation.data.actual_output": serialize(evaluation_context.actual_output),
+                                "gen_ai.evaluation.data.has_trajectory": (
+                                    evaluation_context.actual_trajectory is not None
+                                ),
+                                "gen_ai.evaluation.data.has_interactions": (
+                                    evaluation_context.actual_interactions is not None
+                                ),
+                            }
+                        )
+                        trace_id = format_trace_id(case_span.get_span_context().trace_id)
+                except RetryError as e:
+                    # Max retries exceeded
+                    original_exception = e.last_attempt.exception()
+                    if original_exception is None:
+                        original_exception = Exception(f"Task execution failed after {_MAX_RETRY_ATTEMPTS} retries")
+                    logger.error(
+                        f"Max retry attempts ({_MAX_RETRY_ATTEMPTS}) exceeded for task execution "
+                        f"on case {case_name}. Last error: {str(original_exception)}"
                     )
-                    trace_id = format_trace_id(case_span.get_span_context().trace_id)
+                    raise original_exception from e
 
                 # Evaluate with each evaluator
                 evaluator_results = []
                 for evaluator in self._evaluators:
-                    with self._tracer.start_as_current_span(
-                        f"evaluator {evaluator.get_type_name()}",
-                    ) as eval_span:
-                        evaluation_outputs = await evaluator.evaluate_async(evaluation_context)
-                        (aggregate_score, aggregate_pass, aggregate_reason) = evaluator.aggregator(evaluation_outputs)
 
-                        try:
-                            label = _get_label_from_score(evaluator, aggregate_score)
-                        except Exception:
-                            label = "UNKNOWN"
+                    @retry(
+                        retry=retry_if_exception(is_throttling_error),
+                        stop=stop_after_attempt(_MAX_RETRY_ATTEMPTS),
+                        wait=wait_exponential(multiplier=_INITIAL_RETRY_DELAY, max=_MAX_RETRY_DELAY),
+                        reraise=True,
+                    )
+                    async def _evaluate_with_retry(evaluator=evaluator, evaluation_context=evaluation_context):
+                        outputs = await evaluator.evaluate_async(evaluation_context)
+                        (score, passed, reason) = evaluator.aggregator(outputs)
+                        return outputs, float(score), passed, reason
+
+                    try:
+                        with self._tracer.start_as_current_span(
+                            f"evaluator {evaluator.get_type_name()}",
+                        ) as eval_span:
+                            (
+                                evaluation_outputs,
+                                aggregate_score,
+                                aggregate_pass,
+                                aggregate_reason,
+                            ) = await _evaluate_with_retry()
+
+                            try:
+                                label = _get_label_from_score(evaluator, aggregate_score)
+                            except Exception:
+                                label = "UNKNOWN"
+
+                            eval_span.set_attributes(
+                                {
+                                    "gen_ai.evaluation.score.label": label,
+                                    "gen_ai.evaluation.score.value": str(aggregate_score),
+                                    "gen_ai.evaluation.test_pass": aggregate_pass,
+                                    "gen_ai.evaluation.explanation": aggregate_reason or "",
+                                }
+                            )
+
+                            evaluator_results.append(
+                                {
+                                    "evaluator_name": evaluator.get_type_name(),
+                                    "test_pass": aggregate_pass,
+                                    "score": aggregate_score,
+                                    "reason": aggregate_reason or "",
+                                    "detailed_results": evaluation_outputs,
+                                }
+                            )
+
+                            # CloudWatch logging for this evaluator
+                            try:
+                                evaluator_full_name = f"Custom.{evaluator.get_type_name()}"
+                                region = os.environ.get("AWS_REGION", "us-east-1")
+                                _config_arn = (
+                                    f"arn:aws:strands:{region}::strands-evaluation-empty-config/{self._config_id}"
+                                )
+                                _evaluator_arn = f"arn:aws:strands-evals:::evaluator/{evaluator_full_name}"
 
-                        eval_span.set_attributes(
+                                log_data = {
+                                    "gen_ai.evaluation.name": evaluator_full_name,
+                                    "gen_ai.evaluation.score.value": str(aggregate_score),
+                                    "gen_ai.evaluation.explanation": aggregate_reason or "",
+                                    "gen_ai.evaluation.score.label": label,
+                                    "gen_ai.response.id": trace_id,
+                                    "aws.bedrock_agentcore.evaluator.rating_scale": "Numerical",
+                                    "aws.bedrock_agentcore.evaluation_level": evaluator.evaluation_level or "Trace",
+                                    "event.name": "gen_ai.evaluation.result",
+                                    "aws.bedrock_agentcore.online_evaluation_config.arn": _config_arn,
+                                    "aws.bedrock_agentcore.online_evaluation_config.name": "strands-local-evaluation",
+                                    "aws.bedrock_agentcore.evaluator.arn": _evaluator_arn,
+                                    "session.id": case.session_id,
+                                }
+
+                                agent_observability_enabled = os.environ.get("AGENT_OBSERVABILITY_ENABLED", "")
+                                if agent_observability_enabled:
+                                    _send_to_cloudwatch(
+                                        message="gen_ai.evaluation.result",
+                                        log_data=log_data,
+                                        trace_id=trace_id,
+                                        evaluator_name=evaluator_full_name,
+                                        score=cast(float, aggregate_score),
+                                        config_id=self._config_id,
+                                        label=label,
+                                    )
+                            except Exception as e:
+                                logger.debug(f"Skipping CloudWatch logging: {str(e)}")
+
+                    except RetryError as e:
+                        # Max retries exceeded
+                        original_exception = e.last_attempt.exception()
+                        if original_exception is None:
+                            original_exception = Exception(
+                                f"Evaluator {evaluator.get_type_name()} failed after {_MAX_RETRY_ATTEMPTS} retries"
+                            )
+                        logger.error(
+                            f"Max retry attempts ({_MAX_RETRY_ATTEMPTS}) exceeded for evaluator "
+                            f"{evaluator.get_type_name()} on case {case_name}. Last error: {str(original_exception)}"
+                        )
+                        evaluator_results.append(
                             {
-                                "gen_ai.evaluation.score.label": label,
-                                "gen_ai.evaluation.score.value": str(aggregate_score),
-                                "gen_ai.evaluation.test_pass": aggregate_pass,
-                                "gen_ai.evaluation.explanation": aggregate_reason or "",
+                                "evaluator_name": evaluator.get_type_name(),
+                                "test_pass": False,
+                                "score": 0,
+                                "reason": f"Evaluator error: {str(original_exception)}",
+                                "detailed_results": [],
                             }
                         )
-
+                    except Exception as e:
+                        # Catch non-throttling errors and record as failure (error isolation)
                         evaluator_results.append(
                             {
                                 "evaluator_name": evaluator.get_type_name(),
-                                "test_pass": aggregate_pass,
-                                "score": aggregate_score,
-                                "reason": aggregate_reason or "",
-                                "detailed_results": evaluation_outputs,
+                                "test_pass": False,
+                                "score": 0,
+                                "reason": f"Evaluator error: {str(e)}",
+                                "detailed_results": [],
                             }
                         )
 
-                        # CloudWatch logging for this evaluator
-                        try:
-                            evaluator_full_name = f"Custom.{evaluator.get_type_name()}"
-                            region = os.environ.get("AWS_REGION", "us-east-1")
-                            _config_arn = f"arn:aws:strands:{region}::strands-evaluation-empty-config/{self._config_id}"
-                            _evaluator_arn = f"arn:aws:strands-evals:::evaluator/{evaluator_full_name}"
-
-                            log_data = {
-                                "gen_ai.evaluation.name": evaluator_full_name,
-                                "gen_ai.evaluation.score.value": str(aggregate_score),
-                                "gen_ai.evaluation.explanation": aggregate_reason or "",
-                                "gen_ai.evaluation.score.label": label,
-                                "gen_ai.response.id": trace_id,
-                                "aws.bedrock_agentcore.evaluator.rating_scale": "Numerical",
-                                "aws.bedrock_agentcore.evaluation_level": evaluator.evaluation_level or "Trace",
-                                "event.name": "gen_ai.evaluation.result",
-                                "aws.bedrock_agentcore.online_evaluation_config.arn": _config_arn,
-                                "aws.bedrock_agentcore.online_evaluation_config.name": "strands-local-evaluation",
-                                "aws.bedrock_agentcore.evaluator.arn": _evaluator_arn,
-                                "session.id": case.session_id,
-                            }
-
-                            agent_observability_enabled = os.environ.get("AGENT_OBSERVABILITY_ENABLED", "")
-                            if agent_observability_enabled:
-                                _send_to_cloudwatch(
-                                    message="gen_ai.evaluation.result",
-                                    log_data=log_data,
-                                    trace_id=trace_id,
-                                    evaluator_name=evaluator_full_name,
-                                    score=aggregate_score,
-                                    config_id=self._config_id,
-                                    label=label,
-                                )
-                        except Exception as e:
-                            logger.debug(f"Skipping CloudWatch logging: {str(e)}")
-
                 # Store results
                 results.append(
                     {
@@ -391,7 +507,16 @@ class Experiment(Generic[InputT, OutputT]):
                     "gen_ai.evaluation.case.input": serialize(case.input),
                 },
             ) as case_span:
-                # Task execution span - execute once
+                # Task execution with retry logic
+                @retry(
+                    retry=retry_if_exception(is_throttling_error),
+                    stop=stop_after_attempt(_MAX_RETRY_ATTEMPTS),
+                    wait=wait_exponential(multiplier=_INITIAL_RETRY_DELAY, max=_MAX_RETRY_DELAY),
+                    reraise=True,
+                )
+                def _run_task_with_retry(task=task, case=case):
+                    return self._run_task(task, case)
+
                 try:
                     with self._tracer.start_as_current_span(
                         "task_execution",
@@ -400,7 +525,7 @@ class Experiment(Generic[InputT, OutputT]):
                             "gen_ai.evaluation.case.name": case_name,
                         },
                     ) as task_span:
-                        evaluation_context = self._run_task(task, case)
+                        evaluation_context = _run_task_with_retry()
                         task_span.set_attributes(
                             {
                                 "gen_ai.evaluation.data.input": serialize(evaluation_context.input),
@@ -414,20 +539,59 @@ class Experiment(Generic[InputT, OutputT]):
                                 ),
                             }
                         )
+                except RetryError as e:
+                    # Max retries exceeded
+                    original_exception = e.last_attempt.exception()
+                    if original_exception is None:
+                        original_exception = Exception(f"Task execution failed after {_MAX_RETRY_ATTEMPTS} retries")
+                    logger.error(
+                        f"Max retry attempts ({_MAX_RETRY_ATTEMPTS}) exceeded for task execution "
+                        f"on case {case_name}. Last error: {str(original_exception)}"
+                    )
+                    case_span.record_exception(original_exception)
+                    for evaluator in self._evaluators:
+                        eval_name = evaluator.get_type_name()
+                        self._record_evaluator_result(
+                            evaluator_data=evaluator_data,
+                            eval_name=eval_name,
+                            case_data=case.model_dump(),
+                            test_pass=False,
+                            score=0,
+                            reason=f"Task execution error: {str(original_exception)}",
+                            detailed_results=[],
+                        )
+                    continue
                 except Exception as e:
                     case_span.record_exception(e)
                     for evaluator in self._evaluators:
                         eval_name = evaluator.get_type_name()
-                        evaluator_data[eval_name]["cases"].append(case.model_dump())
-                        evaluator_data[eval_name]["test_passes"].append(False)
-                        evaluator_data[eval_name]["scores"].append(0)
-                        evaluator_data[eval_name]["reasons"].append(f"Task execution error: {str(e)}")
-                        evaluator_data[eval_name]["detailed_results"].append([])
+                        self._record_evaluator_result(
+                            evaluator_data=evaluator_data,
+                            eval_name=eval_name,
+                            case_data=case.model_dump(),
+                            test_pass=False,
+                            score=0,
+                            reason=f"Task execution error: {str(e)}",
+                            detailed_results=[],
+                        )
                     continue
 
                 # Evaluate with each evaluator using the same task output
                 for evaluator in self._evaluators:
                     eval_name = evaluator.get_type_name()
+
+                    # Evaluator execution with retry logic
+                    @retry(
+                        retry=retry_if_exception(is_throttling_error),
+                        stop=stop_after_attempt(_MAX_RETRY_ATTEMPTS),
+                        wait=wait_exponential(multiplier=_INITIAL_RETRY_DELAY, max=_MAX_RETRY_DELAY),
+                        reraise=True,
+                    )
+                    def _evaluate_with_retry(evaluator=evaluator, evaluation_context=evaluation_context):
+                        outputs = evaluator.evaluate(evaluation_context)
+                        (score, passed, reason) = evaluator.aggregator(outputs)
+                        return outputs, float(score), passed, reason
+
                     try:
                         with self._tracer.start_as_current_span(
                             f"evaluator {evaluator.get_type_name()}",
@@ -436,9 +600,8 @@ class Experiment(Generic[InputT, OutputT]):
                                 "gen_ai.evaluation.case.name": case_name,
                             },
                         ) as eval_span:
-                            evaluation_outputs = evaluator.evaluate(evaluation_context)
-                            (aggregate_score, aggregate_pass, aggregate_reason) = evaluator.aggregator(
-                                evaluation_outputs
+                            evaluation_outputs, aggregate_score, aggregate_pass, aggregate_reason = (
+                                _evaluate_with_retry()
                             )
                             eval_span.set_attributes(
                                 {
@@ -448,17 +611,45 @@ class Experiment(Generic[InputT, OutputT]):
                                 }
                             )
 
-                            evaluator_data[eval_name]["cases"].append(evaluation_context.model_dump())
-                            evaluator_data[eval_name]["test_passes"].append(aggregate_pass)
-                            evaluator_data[eval_name]["scores"].append(aggregate_score)
-                            evaluator_data[eval_name]["reasons"].append(aggregate_reason or "")
-                            evaluator_data[eval_name]["detailed_results"].append(evaluation_outputs)
+                            self._record_evaluator_result(
+                                evaluator_data=evaluator_data,
+                                eval_name=eval_name,
+                                case_data=evaluation_context.model_dump(),
+                                test_pass=aggregate_pass,
+                                score=aggregate_score,
+                                reason=aggregate_reason or "",
+                                detailed_results=evaluation_outputs,
+                            )
+                    except RetryError as e:
+                        # Max retries exceeded
+                        original_exception = e.last_attempt.exception()
+                        if original_exception is None:
+                            original_exception = Exception(
+                                f"Evaluator {evaluator.get_type_name()} failed after {_MAX_RETRY_ATTEMPTS} retries"
+                            )
+                        logger.error(
+                            f"Max retry attempts ({_MAX_RETRY_ATTEMPTS}) exceeded for evaluator "
+                            f"{evaluator.get_type_name()} on case {case_name}. Last error: {str(original_exception)}"
+                        )
+                        self._record_evaluator_result(
+                            evaluator_data=evaluator_data,
+                            eval_name=eval_name,
+                            case_data=evaluation_context.model_dump(),
+                            test_pass=False,
+                            score=0,
+                            reason=f"Evaluator error: {str(original_exception)}",
+                            detailed_results=[],
+                        )
                     except Exception as e:
-                        evaluator_data[eval_name]["cases"].append(evaluation_context.model_dump())
-                        evaluator_data[eval_name]["test_passes"].append(False)
-                        evaluator_data[eval_name]["scores"].append(0)
-                        evaluator_data[eval_name]["reasons"].append(f"Evaluator error: {str(e)}")
-                        evaluator_data[eval_name]["detailed_results"].append([])
+                        self._record_evaluator_result(
+                            evaluator_data=evaluator_data,
+                            eval_name=eval_name,
+                            case_data=evaluation_context.model_dump(),
+                            test_pass=False,
+                            score=0,
+                            reason=f"Evaluator error: {str(e)}",
+                            detailed_results=[],
+                        )
 
         reports = []
         for evaluator in self._evaluators:

strands_evals/extractors/trace_extractor.py

@@ -45,9 +45,11 @@ class TraceExtractor:
     def _extract_trace_level(self, session: Session) -> list[TraceLevelInput]:
         """Extract trace-level inputs with session history up to each turn."""
         evaluation_inputs: list[TraceLevelInput] = []
-        previous_turns: list[Union[UserMessage, AssistantMessage]] = []
+        previous_turns: list[Union[UserMessage, list[ToolExecution], AssistantMessage]] = []
 
         for trace in session.traces:
+            tool_spans = self._find_tool_execution_spans(trace)
+
             for span in trace.spans:
                 if not isinstance(span, AgentInvocationSpan):
                     continue
@@ -59,6 +61,16 @@ class TraceExtractor:
                     logger.warning(f"Failed to create user message: {e}")
                     continue
 
+                # Include tool executions in session history
+                if tool_spans:
+                    try:
+                        tool_executions = [
+                            ToolExecution(tool_call=ts.tool_call, tool_result=ts.tool_result) for ts in tool_spans
+                        ]
+                        previous_turns.append(tool_executions)
+                    except (AttributeError, TypeError, ValueError) as e:
+                        logger.warning(f"Failed to create tool executions: {e}")
+
                 trace_input = TraceLevelInput(
                     span_info=span.span_info,
                     agent_response=TextContent(text=span.agent_response),

strands_evals/utils.py

@@ -0,0 +1,37 @@
+from botocore.exceptions import ClientError
+from strands.types.exceptions import EventLoopException, ModelThrottledException
+
+THROTTLING_ERROR_CODES = {
+    "ThrottlingException",
+    "TooManyRequestsException",
+    "RequestLimitExceeded",
+    "ServiceUnavailable",
+    "ProvisionedThroughputExceededException",
+}
+
+
+def is_throttling_error(exception: BaseException) -> bool:
+    """
+    Check if an exception is a throttling/rate limiting error.
+
+    Args:
+        exception: The exception to check
+
+    Returns:
+        True if the exception indicates throttling, False otherwise
+    """
+    # Check for Strands-specific throttling exceptions
+    if isinstance(exception, (ModelThrottledException, EventLoopException)):
+        return True
+
+    # Check for botocore.errorfactory.ThrottlingException (dynamically generated)
+    if type(exception).__name__ == "ThrottlingException":
+        return True
+
+    # Check for botocore ClientError with throttling error codes
+    if isinstance(exception, ClientError):
+        error_code = exception.response.get("Error", {}).get("Code", "")
+        if error_code in THROTTLING_ERROR_CODES:
+            return True
+
+    return False