strands-agents-evals 0.1.0__py3-none-any.whl

strands_evals/simulation/prompt_templates/actor_system_prompt.py

@@ -0,0 +1,64 @@
+"""
+Default system prompt for actor simulation.
+
+This module contains the default system prompt that configures the actor's behavior,
+communication style, and response protocols for realistic conversation simulation.
+"""
+
+from textwrap import dedent
+
+DEFAULT_USER_SIMULATOR_PROMPT_TEMPLATE = dedent("""## User Simulation
+
+Core Identity:
+- You are simulating a user seeking assistance from an AI assistant
+- You speak in first person only
+- You strictly follow your defined User Goal and User Profile throughout the conversation
+
+## User Profile
+{actor_profile}
+
+
+Response Protocols:
+ When assistant requests information:
+   - Provide brief, specific information
+   - Maximum 2-3 sentences
+
+ When assistant provides solutions/answers:
+   - Ask follow-ups, seek clarification, or express satisfaction. Do no deviate from the User Goal.
+   - While following up, do not increase the conversation scope beyond your User Goal.
+
+Communication Rules:
+1. STRICT maximum response length: 2-3 sentences
+2. You are seeking help, NOT providing help - never give solutions!
+3. Maintain your user profile and expertise level consistently
+4. Express more of your user profile - let your background, expertise level, and personality
+   shine through in your responses
+5. Don't break character by mentioning "assistant" or "AI" explicitly
+6. Address AI assistant responses in second person ("Your suggestion..." not "The assistant's suggestion...")
+7. Do not explicitly mention conversation redirection
+8. Never include meta-references or self-instructions in your responses. These reveal you
+   are a simulator and is not how a real human would communicate. Don't write phrases like:
+   -  I need to respond as the user would ...
+   -  As the simulated user, I should ...
+   -  Here's how the user might respond ...
+   -  Based on my user goal, I need to ...
+9. Use the Exit Conditions strictly to stick to User Goal.
+10. Use all relevant tools first to ground your responses, and then respond
+
+Exit Conditions:
+1. Use get_conversation_goal_completion tool to check if your User Goal is met. When your User Goal is met:
+   - Just generate "<stop/>" to terminate conversation
+2. If conversation becomes unproductive or unsafe:
+   - Naturally steer back towards your User Goal
+   - If this becomes impossible, just generate: "<stop/>" to terminate conversation
+
+CRITICAL BEHAVIORAL CONSTRAINTS:
+- You are ONLY a user seeking assistance, NEVER the one providing assistance.
+- NEVER generate comprehensive responses, detailed plans, or extensive information.
+- NEVER solve problems yourself - that's the assistant's job. Under no circumstances,
+  you can use your tools to solve your user goal/sub goals.
+- If you find yourself writing more than 3 sentences, you're doing it wrong.
+- Generate only "<stop/>" to terminate conversation
+
+Response Format:
+Generate ONLY the next SHORT message (1-3 sentences). No explanations, no solutions, no comprehensive information.""")

strands_evals/simulation/prompt_templates/goal_completion.py

@@ -0,0 +1,27 @@
+"""
+Goal completion assessment prompt template for actor simulation.
+
+This module contains the prompt template used to evaluate whether a conversation
+has successfully achieved the actor's initial goals using a 3-point assessment scale.
+"""
+
+from textwrap import dedent
+
+GOAL_COMPLETION_PROMPT = dedent(
+    """Please evaluate the following conversation against its intended goals using this
+3-point assessment scale:
+
+1 = Does not meet the goal at all
+2 = Partially meets the goal with significant gaps
+3 = Fully meets the goal
+
+Initial Goal:
+{initial_goal}
+
+Conversation to evaluate:
+{conversation}
+
+Please provide:
+- A score (1-3)
+- Brief one line justification"""
+)

strands_evals/simulation/tools/__init__.py

@@ -0,0 +1,5 @@
+"""Tools for actor simulation."""
+
+from .goal_completion import get_conversation_goal_completion
+
+__all__ = ["get_conversation_goal_completion"]

strands_evals/simulation/tools/goal_completion.py

@@ -0,0 +1,93 @@
+import logging
+
+from strands import Agent, tool
+from typing_extensions import Any
+
+from strands_evals.simulation.prompt_templates.goal_completion import GOAL_COMPLETION_PROMPT
+
+logger = logging.getLogger(__name__)
+
+
+@tool
+def get_conversation_goal_completion(initial_goal: str, conversation: list[dict[str, str]]) -> str:
+    """
+    Evaluate conversation goal completion using a 3-point assessment scale.
+
+    Analyzes the conversation against the actor's initial goal and provides a score
+    with justification.
+
+    Args:
+        initial_goal: The actor's original goal or objective.
+        conversation: List of conversation turns, each with 'role' and 'content' keys.
+
+    Returns:
+        Assessment string with score (1-3) and brief justification.
+
+    Raises:
+        ValueError: If the conversation format is invalid.
+    """
+    # Format conversation for the prompt
+    conversation_text = _format_conversation_for_assessment(conversation)
+
+    # Create the assessment prompt
+    prompt = GOAL_COMPLETION_PROMPT.format(initial_goal=initial_goal, conversation=conversation_text)
+
+    goal_completion_agent = Agent(callback_handler=None)
+    response = goal_completion_agent(prompt)
+    logger.info("Successfully completed goal completion assessment")
+    return str(response)
+
+
+def _format_conversation_for_assessment(conversation: list[dict[str, Any]]) -> str:
+    """
+    Format conversation history for goal completion assessment.
+
+    Args:
+        conversation: List of conversation turns with 'role' and 'content' keys.
+            Content can be either a string or a list of content blocks.
+
+    Returns:
+        Formatted conversation string with each turn on a separate line.
+
+    Raises:
+        ValueError: If conversation format is invalid.
+    """
+    try:
+        formatted_turns = []
+
+        for i, turn in enumerate(conversation):
+            if not isinstance(turn, dict):
+                raise ValueError(f"Conversation turn {i} must be a dictionary")
+
+            role = turn.get("role", "").strip()
+            content_raw = turn.get("content", "")
+
+            # Handle both string format and list of content blocks
+            if isinstance(content_raw, str):
+                content = content_raw.strip()
+            elif isinstance(content_raw, list):
+                content_parts = []
+                for block in content_raw:
+                    if isinstance(block, dict) and "text" in block:
+                        content_parts.append(block["text"])
+                content = " ".join(content_parts).strip()
+            else:
+                logger.warning(f"Skipping conversation turn {i} with invalid content type: {type(content_raw)}")
+                continue
+
+            if not role or not content:
+                logger.warning(f"Skipping conversation turn {i} with missing role or content")
+                continue
+
+            formatted_turn = f"{role.upper()}: {content}"
+            formatted_turns.append(formatted_turn)
+
+        if not formatted_turns:
+            raise ValueError("No valid conversation turns found")
+
+        return "\n\n".join(formatted_turns)
+
+    except ValueError:
+        raise
+    except Exception as e:
+        raise ValueError("Error formatting conversation") from e

strands_evals/telemetry/__init__.py

@@ -0,0 +1,15 @@
+"""
+Tracing module for strands_evals.
+
+This module provides OpenTelemetry-based tracing capabilities for evaluation workflows,
+allowing detailed observability into evaluators, evaluation data, and agent execution.
+"""
+
+from .config import StrandsEvalsTelemetry
+from .tracer import get_tracer, serialize
+
+__all__ = [
+    "StrandsEvalsTelemetry",
+    "get_tracer",
+    "serialize",
+]

strands_evals/telemetry/_cloudwatch_logger.py

@@ -0,0 +1,209 @@
+"""
+CloudWatch logging utilities for sending evaluation results directly to CloudWatch Logs.
+
+NOTE: This module is temporary and will be removed in the future once ADOT integration is complete.
+"""
+
+import json
+import logging
+import os
+import time
+
+import boto3
+
+logger = logging.getLogger(__name__)
+
+# Module-level CloudWatch client (lazy initialization)
+_cloudwatch_client = None
+
+
+def _get_cloudwatch_client():
+    """
+    Get or create the CloudWatch Logs client (singleton pattern).
+
+    Returns:
+        boto3 CloudWatch Logs client
+    """
+    global _cloudwatch_client
+    if _cloudwatch_client is None:
+        region = os.environ.get("AWS_REGION", "us-east-1")
+        _cloudwatch_client = boto3.client("logs", region_name=region)
+    return _cloudwatch_client
+
+
+def _parse_log_config_from_env(config_id: str):
+    """
+    Parse log group and stream from environment variables.
+
+    Args:
+        config_id: The config ID (not used for log group, only for ARN)
+
+    Returns:
+        Tuple of (destination_log_group, log_stream_name, service_name, resource_log_group)
+        - destination_log_group: From EVALUATION_RESULTS_LOG_GROUP env var (actual destination where logs are sent)
+        - resource_log_group: The log group from OTEL_RESOURCE_ATTRIBUTES for resource attributes in EMF
+    """
+    # Parse from OTEL_EXPORTER_OTLP_LOGS_HEADERS
+    logs_headers = os.environ.get("OTEL_EXPORTER_OTLP_LOGS_HEADERS", "")
+    log_stream = "default"
+
+    if logs_headers:
+        for header in logs_headers.split(","):
+            if "=" in header:
+                key, value = header.split("=", 1)
+                if key.strip() == "x-aws-log-stream":
+                    log_stream = value.strip()
+
+    # Destination log group is from EVALUATION_RESULTS_LOG_GROUP environment variable
+    destination_log_group = os.environ.get("EVALUATION_RESULTS_LOG_GROUP", "default_strands_evals_results")
+    destination_log_group = f"/aws/bedrock-agentcore/evaluations/results/{destination_log_group}"
+
+    # Get resource log group from OTEL_RESOURCE_ATTRIBUTES (for EMF resource attributes)
+    resource_log_group = None
+    resource_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
+    for attr in resource_attrs.split(","):
+        if "=" in attr:
+            key, value = attr.split("=", 1)
+            if key.strip() == "aws.log.group.names":
+                resource_log_group = value.strip()
+
+    # Get service name from OTEL_RESOURCE_ATTRIBUTES
+    service_name = None
+    for attr in resource_attrs.split(","):
+        if "=" in attr:
+            key, value = attr.split("=", 1)
+            if key.strip() == "service.name":
+                service_name = value.strip()
+                break
+
+    if not service_name:
+        raise ValueError("service.name must be set in OTEL_RESOURCE_ATTRIBUTES environment variable")
+
+    return destination_log_group, log_stream, service_name, resource_log_group
+
+
+def _send_to_cloudwatch(
+    message: str,
+    log_data: dict,
+    trace_id: str,
+    evaluator_name: str,
+    score: float,
+    config_id: str,
+    label: str = "",
+):
+    """
+    Send log event directly to CloudWatch Logs using boto3 in EMF format.
+
+    Args:
+        message: The log message
+        log_data: Dictionary containing the log event data
+        trace_id: The OpenTelemetry trace ID
+        evaluator_name: The name of the evaluator (e.g., "Custom.HelpfulnessEvaluator")
+        score: The evaluation score
+        config_id: The config ID for log group path
+        label: The evaluation label (optional)
+    """
+    try:
+        destination_log_group, log_stream, service_name, resource_log_group = _parse_log_config_from_env(config_id)
+        if not destination_log_group:
+            logger.warning("No destination log group configured, skipping CloudWatch logging")
+            return
+
+        # Get the singleton CloudWatch client
+        cloudwatch_client = _get_cloudwatch_client()
+
+        # Ensure destination log group exists
+        try:
+            cloudwatch_client.create_log_group(logGroupName=destination_log_group)
+        except cloudwatch_client.exceptions.ResourceAlreadyExistsException:
+            pass
+        except Exception as e:
+            logger.warning(f"Failed to create log group: {str(e)}")
+
+        # Ensure log stream exists
+        try:
+            cloudwatch_client.create_log_stream(logGroupName=destination_log_group, logStreamName=log_stream)
+        except cloudwatch_client.exceptions.ResourceAlreadyExistsException:
+            pass
+        except Exception as e:
+            logger.warning(f"Failed to create log stream: {str(e)}")
+
+        # Get sequence token for the log stream
+        sequence_token = None
+        try:
+            response = cloudwatch_client.describe_log_streams(
+                logGroupName=destination_log_group, logStreamNamePrefix=log_stream
+            )
+            if response["logStreams"]:
+                sequence_token = response["logStreams"][0].get("uploadSequenceToken")
+        except Exception as e:
+            logger.warning(f"Failed to get sequence token: {str(e)}")
+
+        # Get current timestamp
+        current_time_ns = time.time_ns()
+        current_time_ms = int(current_time_ns / 1_000_000)
+
+        # Extract online evaluation config ID from ARN
+        online_eval_config_id = ""
+        if "aws.bedrock_agentcore.online_evaluation_config.arn" in log_data:
+            arn = log_data["aws.bedrock_agentcore.online_evaluation_config.arn"]
+            if "/" in arn:
+                online_eval_config_id = arn.split("/")[-1]
+
+        emf_log = {
+            "resource": {
+                "attributes": {
+                    "aws.local.service": service_name,
+                    "aws.service.type": "gen_ai_agent",
+                    "telemetry.sdk.language": "python",
+                    "service.name": service_name,
+                    "aws.log.group.names": resource_log_group if resource_log_group else destination_log_group,
+                }
+            },
+            "timeUnixNano": current_time_ns,
+            "observedTimeUnixNano": current_time_ns,
+            "severityText": "INFO",
+            "body": message,
+            "attributes": {
+                "otelServiceName": service_name,
+                "otelTraceID": trace_id,
+                "otelTraceSampled": True,
+                **log_data,
+            },
+            "flags": 1,
+            "traceId": trace_id,
+            "onlineEvaluationConfigId": online_eval_config_id,
+            evaluator_name: score,
+        }
+
+        # Add label if provided
+        emf_log["label"] = label or "YES"
+        emf_log["service.name"] = service_name
+
+        # Add EMF metadata for CloudWatch metrics
+        emf_log["_aws"] = {
+            "Timestamp": current_time_ms,
+            "CloudWatchMetrics": [
+                {
+                    "Namespace": "Bedrock-AgentCore/Evaluations",
+                    "Dimensions": [
+                        ["service.name"],
+                        ["label", "service.name"],
+                        ["service.name", "onlineEvaluationConfigId"],
+                        ["label", "service.name", "onlineEvaluationConfigId"],
+                    ],
+                    "Metrics": [{"Name": evaluator_name, "Unit": "None"}],
+                }
+            ],
+        }
+
+        log_event = {"timestamp": current_time_ms, "message": json.dumps(emf_log)}
+        put_log_params = {"logGroupName": destination_log_group, "logStreamName": log_stream, "logEvents": [log_event]}
+
+        if sequence_token:
+            put_log_params["sequenceToken"] = sequence_token
+
+        cloudwatch_client.put_log_events(**put_log_params)
+
+    except Exception as e:
+        logger.warning(f"Failed to send log to CloudWatch: {str(e)}")

strands_evals/telemetry/config.py

@@ -0,0 +1,207 @@
+"""OpenTelemetry configuration and setup utilities for strands_evals.
+
+This module provides centralized configuration and initialization functionality
+for OpenTelemetry tracing in evaluation workflows.
+"""
+
+import logging
+from typing import Any
+
+import opentelemetry.trace as trace_api
+from opentelemetry import propagate
+from opentelemetry.baggage.propagation import W3CBaggagePropagator
+from opentelemetry.propagators.composite import CompositePropagator
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider as SDKTracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter, SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from opentelemetry.trace import TracerProvider
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+
+logger = logging.getLogger(__name__)
+
+
+def get_otel_resource() -> Resource:
+    """Create a standard OpenTelemetry resource with service information.
+
+    Returns:
+        Resource object with standard service information.
+    """
+    resource = Resource.create(
+        {
+            "service.name": "strands-evals",
+            "service.version": "0.1.0",
+            "telemetry.sdk.name": "opentelemetry",
+            "telemetry.sdk.language": "python",
+        }
+    )
+
+    return resource
+
+
+class StrandsEvalsTelemetry:
+    """OpenTelemetry configuration and setup for strands_evals.
+
+    Automatically initializes a tracer provider with text map propagators.
+    Trace exporters (console, OTLP) can be set up individually using dedicated methods
+    that support method chaining for convenient configuration.
+
+    Args:
+        tracer_provider: Optional pre-configured SDKTracerProvider. If None,
+            a new one will be created and set as the global tracer provider.
+
+    Environment Variables:
+        Environment variables are handled by the underlying OpenTelemetry SDK:
+        - OTEL_EXPORTER_OTLP_ENDPOINT: OTLP endpoint URL
+        - OTEL_EXPORTER_OTLP_HEADERS: Headers for OTLP requests
+
+    Examples:
+        Quick setup with method chaining:
+        >>> StrandsEvalsTelemetry().setup_console_exporter().setup_otlp_exporter()
+
+        Using a custom tracer provider:
+        >>> StrandsEvalsTelemetry(tracer_provider=my_provider).setup_console_exporter()
+
+        Step-by-step configuration:
+        >>> telemetry = StrandsEvalsTelemetry()
+        >>> telemetry.setup_console_exporter()
+        >>> telemetry.setup_otlp_exporter()
+
+    Note:
+        - The tracer provider is automatically initialized upon instantiation
+        - When no tracer_provider is provided, the instance sets itself as the global provider
+        - Exporters must be explicitly configured using the setup methods
+        - Failed exporter configurations are logged but do not raise exceptions
+        - All setup methods return self to enable method chaining
+    """
+
+    def __init__(
+        self,
+        tracer_provider: SDKTracerProvider | None = None,
+    ) -> None:
+        """Initialize the StrandsEvalsTelemetry instance.
+
+        Args:
+            tracer_provider: Optional pre-configured tracer provider.
+                If None, a new one will be created and set as global.
+
+        The instance is ready to use immediately after initialization, though
+        trace exporters must be configured separately using the setup methods.
+        """
+        self.resource = get_otel_resource()
+        self._in_memory_exporter: InMemorySpanExporter | None = None
+        self.tracer_provider: TracerProvider
+
+        if tracer_provider:
+            self.tracer_provider = tracer_provider
+        else:
+            self._initialize_tracer()
+
+    def _initialize_tracer(self) -> None:
+        """Initialize the OpenTelemetry tracer."""
+        logger.info("Initializing tracer for strands-evals")
+
+        # Create tracer provider
+        tracer_provider = SDKTracerProvider(resource=self.resource)
+
+        # Set as global tracer provider
+        trace_api.set_tracer_provider(tracer_provider)
+
+        # Get the global tracer provider (may be wrapped in a proxy)
+        self.tracer_provider = trace_api.get_tracer_provider()
+
+        # Set up propagators
+        propagate.set_global_textmap(
+            CompositePropagator(
+                [
+                    W3CBaggagePropagator(),
+                    TraceContextTextMapPropagator(),
+                ]
+            )
+        )
+
+    @property
+    def in_memory_exporter(self) -> InMemorySpanExporter:
+        """Get the in-memory exporter instance.
+
+        Returns:
+            The configured in-memory span exporter.
+
+        Raises:
+            RuntimeError: If setup_in_memory_exporter() has not been called.
+        """
+        if self._in_memory_exporter is None:
+            raise RuntimeError(
+                "In-memory exporter is not configured. Call setup_in_memory_exporter() before accessing this property."
+            )
+        return self._in_memory_exporter
+
+    def setup_console_exporter(self, **kwargs: Any) -> "StrandsEvalsTelemetry":
+        """Set up console exporter for the tracer provider.
+
+        Args:
+            **kwargs: Optional keyword arguments passed directly to
+                OpenTelemetry's ConsoleSpanExporter initializer.
+
+        Returns:
+            self: Enables method chaining.
+
+        This method configures a SimpleSpanProcessor with a ConsoleSpanExporter,
+        allowing trace data to be output to the console. Any additional keyword
+        arguments provided will be forwarded to the ConsoleSpanExporter.
+        """
+        try:
+            logger.info("Enabling console export for strands-evals")
+            console_processor = SimpleSpanProcessor(ConsoleSpanExporter(**kwargs))
+            self.tracer_provider.add_span_processor(console_processor)
+        except Exception as e:
+            logger.exception("error=<%s> | Failed to configure console exporter", e)
+        return self
+
+    def setup_in_memory_exporter(self, **kwargs: Any) -> "StrandsEvalsTelemetry":
+        """Set up in-memory exporter for the tracer provider.
+
+        Args:
+            **kwargs: Optional keyword arguments passed directly to
+                OpenTelemetry's InMemorySpanExporter initializer.
+
+        Returns:
+            self: Enables method chaining.
+
+        This method configures a SimpleSpanProcessor with an InMemorySpanExporter,
+        allowing trace data to be stored in memory for testing and debugging purposes.
+        Any additional keyword arguments provided will be forwarded to the InMemorySpanExporter.
+        """
+        try:
+            logger.info("Enabling in-memory export for strands-evals")
+            self._in_memory_exporter = InMemorySpanExporter()
+            span_processor = SimpleSpanProcessor(self._in_memory_exporter)
+            self.tracer_provider.add_span_processor(span_processor)
+        except Exception as e:
+            logger.exception("error=<%s> | Failed to configure console exporter", e)
+        return self
+
+    def setup_otlp_exporter(self, **kwargs: Any) -> "StrandsEvalsTelemetry":
+        """Set up OTLP exporter for the tracer provider.
+
+        Args:
+            **kwargs: Optional keyword arguments passed directly to
+                OpenTelemetry's OTLPSpanExporter initializer.
+
+        Returns:
+            self: Enables method chaining.
+
+        This method configures a BatchSpanProcessor with an OTLPSpanExporter,
+        allowing trace data to be exported to an OTLP endpoint. Any additional
+        keyword arguments provided will be forwarded to the OTLPSpanExporter.
+        """
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+
+        try:
+            otlp_exporter = OTLPSpanExporter(**kwargs)
+            batch_processor = BatchSpanProcessor(otlp_exporter)
+            self.tracer_provider.add_span_processor(batch_processor)
+            logger.info("OTLP exporter configured for strands-evals")
+        except Exception as e:
+            logger.exception("error=<%s> | Failed to configure OTLP exporter", e)
+        return self

strands_evals/telemetry/tracer.py

@@ -0,0 +1,38 @@
+"""OpenTelemetry tracing for strands_evals.
+
+This module provides a simple way to get the OpenTelemetry tracer
+for evaluation workflows.
+"""
+
+import json
+import logging
+from typing import Any
+
+import opentelemetry.trace as trace_api
+from opentelemetry.sdk.trace import TracerProvider as SDKTracerProvider
+
+logger = logging.getLogger(__name__)
+
+
+def get_tracer() -> trace_api.Tracer:
+    """Get the OpenTelemetry tracer for strands_evals.
+
+    Returns:
+        OpenTelemetry Tracer instance from the global tracer provider
+    """
+    tracer_provider = trace_api.get_tracer_provider()
+    if not isinstance(tracer_provider, SDKTracerProvider):
+        tracer_provider = trace_api.NoOpTracerProvider()
+    return tracer_provider.get_tracer("strands-evals")
+
+
+def serialize(obj: Any) -> str:
+    """Serialize an object to JSON string.
+
+    Args:
+        obj: The object to serialize
+
+    Returns:
+        JSON string representation
+    """
+    return json.dumps(obj, ensure_ascii=False, default=str)