strands-agents-evals 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

{strands_agents_evals-0.1.0.dist-info → strands_agents_evals-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: strands-agents-evals
-Version: 0.1.0
+Version: 0.1.2
 Summary: Evaluation framework for Strands
 Author-email: AWS <opensource@amazon.com>
 License: Apache-2.0
@@ -68,6 +68,7 @@ Strands Evaluation is a powerful framework for evaluating AI agents and LLM appl
 ## Feature Overview
 
 - **Multiple Evaluation Types**: Output evaluation, trajectory analysis, tool usage assessment, and interaction evaluation
+- **Dynamic Simulators**: Multi-turn conversation simulation with realistic user behavior and goal-oriented interactions
 - **LLM-as-a-Judge**: Built-in evaluators using language models for sophisticated assessment with structured scoring
 - **Trace-based Evaluation**: Analyze agent behavior through OpenTelemetry execution traces
 - **Automated Experiment Generation**: Generate comprehensive test suites from context descriptions
@@ -226,6 +227,73 @@ reports = experiment.run_evaluations(user_task_function)
 reports[0].run_display()
 ```
 
+### Multi-turn Conversation Simulation
+
+Simulate realistic user interactions with dynamic, goal-oriented conversations using ActorSimulator:
+
+```python
+from strands import Agent
+from strands_evals import Case, Experiment, ActorSimulator
+from strands_evals.evaluators import HelpfulnessEvaluator, GoalSuccessRateEvaluator
+from strands_evals.mappers import StrandsInMemorySessionMapper
+from strands_evals.telemetry import StrandsEvalsTelemetry
+
+# Setup telemetry
+telemetry = StrandsEvalsTelemetry().setup_in_memory_exporter()
+memory_exporter = telemetry.in_memory_exporter
+
+def task_function(case: Case) -> dict:
+    # Create simulator to drive conversation
+    simulator = ActorSimulator.from_case_for_user_simulator(
+        case=case,
+        max_turns=10
+    )
+
+    # Create agent to evaluate
+    agent = Agent(
+        trace_attributes={
+            "gen_ai.conversation.id": case.session_id,
+            "session.id": case.session_id
+        },
+        callback_handler=None
+    )
+
+    # Run multi-turn conversation
+    all_spans = []
+    user_message = case.input
+
+    while simulator.has_next():
+        memory_exporter.clear()
+        agent_response = agent(user_message)
+        turn_spans = list(memory_exporter.get_finished_spans())
+        all_spans.extend(turn_spans)
+
+        user_result = simulator.act(str(agent_response))
+        user_message = str(user_result.structured_output.message)
+
+    # Map to session for evaluation
+    mapper = StrandsInMemorySessionMapper()
+    session = mapper.map_to_session(all_spans, session_id=case.session_id)
+
+    return {"output": str(agent_response), "trajectory": session}
+
+# Use evaluators to assess simulated conversations
+evaluators = [
+    HelpfulnessEvaluator(),
+    GoalSuccessRateEvaluator()
+]
+
+experiment = Experiment(cases=test_cases, evaluators=evaluators)
+reports = experiment.run_evaluations(task_function)
+```
+
+**Key Benefits:**
+- **Dynamic Interactions**: Simulator adapts responses based on agent behavior
+- **Goal-Oriented Testing**: Verify agents can complete user objectives through dialogue
+- **Realistic Conversations**: Generate authentic multi-turn interaction patterns
+- **No Predefined Scripts**: Test agents without hardcoded conversation paths
+- **Comprehensive Evaluation**: Combine with trace-based evaluators for full assessment
+
 ### Automated Experiment Generation
 
 Generate comprehensive test suites automatically from context descriptions:
@@ -388,8 +456,9 @@ reports[0].run_display()  # Interactive display with metrics breakdown
 
 For detailed guidance & examples, explore our documentation:
 
-- [User Guide](https://strandsagents.com/latest//user-guide/evals-sdk/quickstart.md)
-- [Evaluator Reference](https://strandsagents.com/latest/user-guide/evals-sdk/evaluators/)
+- [User Guide](https://strandsagents.com/latest/documentation/docs/user-guide/evals-sdk/quickstart/)
+- [Evaluator Reference](https://strandsagents.com/latest/documentation/docs/user-guide/evals-sdk/evaluators/)
+- [Simulators Guide](https://strandsagents.com/latest/documentation/docs/user-guide/evals-sdk/simulators/)
 
 ## Contributing ❤️
 

{strands_agents_evals-0.1.0.dist-info → strands_agents_evals-0.1.2.dist-info}/RECORD

@@ -1,6 +1,6 @@
-strands_evals/__init__.py,sha256=LzUchWCMqfkXf4ca1p-S8YGBG5yTvy8ePTbdoVlqO8E,508
+strands_evals/__init__.py,sha256=WnYsQGtkatrCKM8v_i_oCtBHNJfPaTrOg2ThUlf55Pk,485
 strands_evals/case.py,sha256=KWAL947NkmNzg9FFdTsL6KI9AFLQ8IcFjaOjcs9x5to,2131
-strands_evals/experiment.py,sha256=H_XJnOPufqg_3ZhSnjFFMFEzo4CqRtoHbHdFqLXR4iQ,28183
+strands_evals/experiment.py,sha256=yySXFW5p9xkDSvkxHBBDncxXKiuj0aDFY7iKoUyprwc,28745
 strands_evals/display/display_console.py,sha256=bOTr6RepgnifALz2DgXnnk3c4Jjxu_mA68-pFr7xry0,5932
 strands_evals/evaluators/__init__.py,sha256=OfZU5RkYewHOAnEjPKdxiEvPnfOOWNZc_9nQpAfARfI,887
 strands_evals/evaluators/evaluator.py,sha256=XEesDeT83H93B1X_w8s0Nsb1KKHy26QO8b99Hi6vKbc,7466
@@ -30,7 +30,7 @@ strands_evals/evaluators/prompt_templates/tool_selection_accuracy/tool_selection
 strands_evals/extractors/__init__.py,sha256=Jmlrk-m8sSS_LwmCVSloIkg3BjOgRzNEezjaAGMw5rw,74
 strands_evals/extractors/graph_extractor.py,sha256=TeT-58JB9roqSvy2ELz1kg8WF5YO-cfLlGZTO0F9s_4,1105
 strands_evals/extractors/swarm_extractor.py,sha256=Sm1XFCkAGVdF3XDyO3iF-20I8C6sAQ8JPNP5fgotOFU,2682
-strands_evals/extractors/tools_use_extractor.py,sha256=XTD7dDIWMiuaIYjUCRnZP45ly629n2XtkrbBwMlKU-s,6190
+strands_evals/extractors/tools_use_extractor.py,sha256=o2e9ZuPqQ_hdcrDkl1Rw9h7Ipfc-CsLNkWemxcRvglg,6409
 strands_evals/extractors/trace_extractor.py,sha256=l7gk5rUFoUcxQduPJz49OX66SdgeK1MLt81aF1yr4Lc,6653
 strands_evals/generators/__init__.py,sha256=B1F30DAIf0kPyBdE4PAZvSby-dTelqb_7hFJoATqVb0,89
 strands_evals/generators/experiment_generator.py,sha256=6wLTL0iG2b0YAiu0w8dDiaBxOIy7p_Fs7l3hCjgQc0w,22655
@@ -61,8 +61,8 @@ strands_evals/types/evaluation_report.py,sha256=vT86zO4Qn9CQbULo3aziGMdG-1qWLdcB
 strands_evals/types/trace.py,sha256=BFoEylzAlENyPH702T5MDz-_H21-Wfx-FFTSXX1tDfY,4844
 strands_evals/types/simulation/__init__.py,sha256=-mz5lW6qFfIMm4dJGaP9pXY3xeiefLbB0XevjdFykkU,133
 strands_evals/types/simulation/actor.py,sha256=ESTV8165c3Ad5QT4yYmjm-A-oZdwZ0Rf0Lq7zokjTPo,1163
-strands_agents_evals-0.1.0.dist-info/METADATA,sha256=M51KkWtamOJjOpBeVocDA2A1np_X17BZmmWyLGTHvUE,15145
-strands_agents_evals-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-strands_agents_evals-0.1.0.dist-info/licenses/LICENSE,sha256=yIWKWnZEC7ykhOE0z330Y4XQkN0YssWOZQ-TkliALls,10141
-strands_agents_evals-0.1.0.dist-info/licenses/NOTICE,sha256=Eg13ogOmcI7JpMjxniFnKG81vwU3X8X7P_IlpvVg5RU,66
-strands_agents_evals-0.1.0.dist-info/RECORD,,
+strands_agents_evals-0.1.2.dist-info/METADATA,sha256=eO-nDiGzAJxDmmwBwPVvNfTdds0408trSL4cKqLR8b4,17721
+strands_agents_evals-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+strands_agents_evals-0.1.2.dist-info/licenses/LICENSE,sha256=yIWKWnZEC7ykhOE0z330Y4XQkN0YssWOZQ-TkliALls,10141
+strands_agents_evals-0.1.2.dist-info/licenses/NOTICE,sha256=Eg13ogOmcI7JpMjxniFnKG81vwU3X8X7P_IlpvVg5RU,66
+strands_agents_evals-0.1.2.dist-info/RECORD,,

strands_evals/__init__.py

@@ -1,5 +1,3 @@
-__version__ = "0.1.0"
-
 from . import evaluators, extractors, generators, simulation, telemetry, types
 from .case import Case
 from .experiment import Experiment

strands_evals/experiment.py

@@ -391,8 +391,8 @@ class Experiment(Generic[InputT, OutputT]):
                     "gen_ai.evaluation.case.input": serialize(case.input),
                 },
             ) as case_span:
+                # Task execution span - execute once
                 try:
-                    # Task execution span - execute once
                     with self._tracer.start_as_current_span(
                         "task_execution",
                         attributes={
@@ -414,9 +414,21 @@ class Experiment(Generic[InputT, OutputT]):
                                 ),
                             }
                         )
-
-                    # Evaluate with each evaluator using the same task output
+                except Exception as e:
+                    case_span.record_exception(e)
                     for evaluator in self._evaluators:
+                        eval_name = evaluator.get_type_name()
+                        evaluator_data[eval_name]["cases"].append(case.model_dump())
+                        evaluator_data[eval_name]["test_passes"].append(False)
+                        evaluator_data[eval_name]["scores"].append(0)
+                        evaluator_data[eval_name]["reasons"].append(f"Task execution error: {str(e)}")
+                        evaluator_data[eval_name]["detailed_results"].append([])
+                    continue
+
+                # Evaluate with each evaluator using the same task output
+                for evaluator in self._evaluators:
+                    eval_name = evaluator.get_type_name()
+                    try:
                         with self._tracer.start_as_current_span(
                             f"evaluator {evaluator.get_type_name()}",
                             attributes={
@@ -436,21 +448,16 @@ class Experiment(Generic[InputT, OutputT]):
                                 }
                             )
 
-                            eval_name = evaluator.get_type_name()
                             evaluator_data[eval_name]["cases"].append(evaluation_context.model_dump())
                             evaluator_data[eval_name]["test_passes"].append(aggregate_pass)
                             evaluator_data[eval_name]["scores"].append(aggregate_score)
                             evaluator_data[eval_name]["reasons"].append(aggregate_reason or "")
                             evaluator_data[eval_name]["detailed_results"].append(evaluation_outputs)
-
-                except Exception as e:
-                    case_span.record_exception(e)
-                    for evaluator in self._evaluators:
-                        eval_name = evaluator.get_type_name()
-                        evaluator_data[eval_name]["cases"].append(case.model_dump())
+                    except Exception as e:
+                        evaluator_data[eval_name]["cases"].append(evaluation_context.model_dump())
                         evaluator_data[eval_name]["test_passes"].append(False)
                         evaluator_data[eval_name]["scores"].append(0)
-                        evaluator_data[eval_name]["reasons"].append(f"An error occured : {str(e)}")
+                        evaluator_data[eval_name]["reasons"].append(f"Evaluator error: {str(e)}")
                         evaluator_data[eval_name]["detailed_results"].append([])
 
         reports = []
@@ -577,8 +584,8 @@ class Experiment(Generic[InputT, OutputT]):
 
         file_path.parent.mkdir(parents=True, exist_ok=True)
 
-        with open(file_path, "w") as f:
-            json.dump(self.to_dict(), f, indent=2)
+        with open(file_path, "w", encoding="utf-8") as f:
+            json.dump(self.to_dict(), f, indent=2, ensure_ascii=False)
 
     @classmethod
     def from_dict(cls, data: dict, custom_evaluators: list[type[Evaluator]] | None = None):
@@ -646,7 +653,7 @@ class Experiment(Generic[InputT, OutputT]):
                 f"Only .json format is supported. Got file: {path}. Please provide a path with .json extension."
             )
 
-        with open(file_path, "r") as f:
+        with open(file_path, "r", encoding="utf-8") as f:
             data = json.load(f)
 
         return cls.from_dict(data, custom_evaluators)

strands_evals/extractors/tools_use_extractor.py

@@ -33,6 +33,7 @@ def extract_agent_tools_used_from_messages(agent_messages):
                     tool_id = tool.get("toolUseId")
                     # get the tool result from the next message
                     tool_result = None
+                    is_error = False
                     next_message_i = i + 1
                     while next_message_i < len(agent_messages):
                         next_message = agent_messages[next_message_i]
@@ -42,13 +43,16 @@ def extract_agent_tools_used_from_messages(agent_messages):
                             content = next_message.get("content")
                             if content:
                                 tool_result_dict = content[0].get("toolResult")
-                                if tool_result_dict.get("toolUseId") == tool_id:
+                                if tool_result_dict and tool_result_dict.get("toolUseId") == tool_id:
                                     tool_result_content = tool_result_dict.get("content", [])
                                     if len(tool_result_content) > 0:
                                         tool_result = tool_result_content[0].get("text")
+                                        is_error = tool_result_dict.get("status") == "error"
                                         break
 
-                    tools_used.append({"name": tool_name, "input": tool_input, "tool_result": tool_result})
+                    tools_used.append(
+                        {"name": tool_name, "input": tool_input, "tool_result": tool_result, "is_error": is_error}
+                    )
     return tools_used