strands-agents-evals 0.1.1__tar.gz → 0.1.3__tar.gz

{strands_agents_evals-0.1.1 → strands_agents_evals-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: strands-agents-evals
-Version: 0.1.1
+Version: 0.1.3
 Summary: Evaluation framework for Strands
 Author-email: AWS <opensource@amazon.com>
 License: Apache-2.0

{strands_agents_evals-0.1.1 → strands_agents_evals-0.1.3}/src/strands_evals/experiment.py

@@ -391,8 +391,8 @@ class Experiment(Generic[InputT, OutputT]):
                     "gen_ai.evaluation.case.input": serialize(case.input),
                 },
             ) as case_span:
+                # Task execution span - execute once
                 try:
-                    # Task execution span - execute once
                     with self._tracer.start_as_current_span(
                         "task_execution",
                         attributes={
@@ -414,9 +414,21 @@ class Experiment(Generic[InputT, OutputT]):
                                 ),
                             }
                         )
-
-                    # Evaluate with each evaluator using the same task output
+                except Exception as e:
+                    case_span.record_exception(e)
                     for evaluator in self._evaluators:
+                        eval_name = evaluator.get_type_name()
+                        evaluator_data[eval_name]["cases"].append(case.model_dump())
+                        evaluator_data[eval_name]["test_passes"].append(False)
+                        evaluator_data[eval_name]["scores"].append(0)
+                        evaluator_data[eval_name]["reasons"].append(f"Task execution error: {str(e)}")
+                        evaluator_data[eval_name]["detailed_results"].append([])
+                    continue
+
+                # Evaluate with each evaluator using the same task output
+                for evaluator in self._evaluators:
+                    eval_name = evaluator.get_type_name()
+                    try:
                         with self._tracer.start_as_current_span(
                             f"evaluator {evaluator.get_type_name()}",
                             attributes={
@@ -436,21 +448,16 @@ class Experiment(Generic[InputT, OutputT]):
                                 }
                             )
 
-                            eval_name = evaluator.get_type_name()
                             evaluator_data[eval_name]["cases"].append(evaluation_context.model_dump())
                             evaluator_data[eval_name]["test_passes"].append(aggregate_pass)
                             evaluator_data[eval_name]["scores"].append(aggregate_score)
                             evaluator_data[eval_name]["reasons"].append(aggregate_reason or "")
                             evaluator_data[eval_name]["detailed_results"].append(evaluation_outputs)
-
-                except Exception as e:
-                    case_span.record_exception(e)
-                    for evaluator in self._evaluators:
-                        eval_name = evaluator.get_type_name()
-                        evaluator_data[eval_name]["cases"].append(case.model_dump())
+                    except Exception as e:
+                        evaluator_data[eval_name]["cases"].append(evaluation_context.model_dump())
                         evaluator_data[eval_name]["test_passes"].append(False)
                         evaluator_data[eval_name]["scores"].append(0)
-                        evaluator_data[eval_name]["reasons"].append(f"An error occured : {str(e)}")
+                        evaluator_data[eval_name]["reasons"].append(f"Evaluator error: {str(e)}")
                         evaluator_data[eval_name]["detailed_results"].append([])
 
         reports = []

{strands_agents_evals-0.1.1 → strands_agents_evals-0.1.3}/src/strands_evals/extractors/tools_use_extractor.py

@@ -22,37 +22,39 @@ def extract_agent_tools_used_from_messages(agent_messages):
         if message.get("role") == "assistant":
             message_info = message.get("content")
             if len(message_info) > 0:
-                tool = None
+                tools = []
                 for message in message_info:
                     if "toolUse" in message:
+                        tools.append(message.get("toolUse"))
+
+                for tool in tools:
+                    if tool:
+                        tool_name = tool.get("name")
+                        tool_input = tool.get("input")
+                        tool_id = tool.get("toolUseId")
+                        # get the tool result from the next message
+                        tool_result = None
+                        is_error = False
+                        next_message_i = i + 1
+                        while next_message_i < len(agent_messages):
+                            next_message = agent_messages[next_message_i]
+                            next_message_i += 1
+
+                            if next_message.get("role") == "user":
+                                content = next_message.get("content")
+                                if content:
+                                    tool_result_dict = content[0].get("toolResult")
+                                    if tool_result_dict and tool_result_dict.get("toolUseId") == tool_id:
+                                        tool_result_content = tool_result_dict.get("content", [])
+                                        if len(tool_result_content) > 0:
+                                            tool_result = tool_result_content[0].get("text")
+                                            is_error = tool_result_dict.get("status") == "error"
+                                            break
+
+                        tools_used.append(
+                            {"name": tool_name, "input": tool_input, "tool_result": tool_result, "is_error": is_error}
+                        )
                         tool = message.get("toolUse")
-
-                if tool:
-                    tool_name = tool.get("name")
-                    tool_input = tool.get("input")
-                    tool_id = tool.get("toolUseId")
-                    # get the tool result from the next message
-                    tool_result = None
-                    is_error = False
-                    next_message_i = i + 1
-                    while next_message_i < len(agent_messages):
-                        next_message = agent_messages[next_message_i]
-                        next_message_i += 1
-
-                        if next_message.get("role") == "user":
-                            content = next_message.get("content")
-                            if content:
-                                tool_result_dict = content[0].get("toolResult")
-                                if tool_result_dict.get("toolUseId") == tool_id:
-                                    tool_result_content = tool_result_dict.get("content", [])
-                                    if len(tool_result_content) > 0:
-                                        tool_result = tool_result_content[0].get("text")
-                                        is_error = tool_result_dict.get("status") == "error"
-                                        break
-
-                    tools_used.append(
-                        {"name": tool_name, "input": tool_input, "tool_result": tool_result, "is_error": is_error}
-                    )
     return tools_used
 
 

{strands_agents_evals-0.1.1 → strands_agents_evals-0.1.3}/tests/strands_evals/extractors/test_tools_use_extractor.py

@@ -48,6 +48,70 @@ def test_tools_use_extractor_extract_from_messages_with_tools():
     assert result[0]["is_error"] is False
 
 
+def test_tools_use_extractor_extract_from_messages_with_multiple_tools():
+    """Test extracting multiple tool usages from messages"""
+    messages = [
+        {"role": "user", "content": [{"text": "Calculate 2+2 and search for weather"}]},
+        {
+            "role": "assistant",
+            "content": [
+                {"text": "I'll calculate and search for you."},
+                {
+                    "toolUse": {
+                        "toolUseId": "tool1",
+                        "name": "calculator",
+                        "input": {"expression": "2+2"},
+                    }
+                },
+                {
+                    "toolUse": {
+                        "toolUseId": "tool2",
+                        "name": "web_search",
+                        "input": {"query": "current weather"},
+                    }
+                },
+            ],
+        },
+        {
+            "role": "user",
+            "content": [
+                {
+                    "toolResult": {
+                        "status": "success",
+                        "content": [{"text": "Result: 4"}],
+                        "toolUseId": "tool1",
+                    }
+                }
+            ],
+        },
+        {
+            "role": "user",
+            "content": [
+                {
+                    "toolResult": {
+                        "status": "success",
+                        "content": [{"text": "Sunny, 25°C"}],
+                        "toolUseId": "tool2",
+                    }
+                }
+            ],
+        },
+        {"role": "assistant", "content": [{"text": "Results: 4 and sunny weather."}]},
+    ]
+
+    result = extract_agent_tools_used_from_messages(messages)
+
+    assert len(result) == 2
+    assert result[0]["name"] == "calculator"
+    assert result[0]["input"] == {"expression": "2+2"}
+    assert result[0]["tool_result"] == "Result: 4"
+    assert result[0]["is_error"] is False
+    assert result[1]["name"] == "web_search"
+    assert result[1]["input"] == {"query": "current weather"}
+    assert result[1]["tool_result"] == "Sunny, 25°C"
+    assert result[1]["is_error"] is False
+
+
 def test_tools_use_extractor_extract_from_messages_no_tools():
     """Test extracting tool usage from messages without tool usage"""
     messages = [
@@ -243,3 +307,38 @@ def test_tools_use_extractor_extract_tools_description_empty():
     result = extract_tools_description(mock_agent, is_short=True)
 
     assert result == {}
+
+
+def test_tools_use_extractor_extract_from_messages_user_message_without_tool_result():
+    """Test extracting tool usage when user message content lacks toolResult key."""
+    messages = [
+        {
+            "role": "assistant",
+            "content": [
+                {"toolUse": {"toolUseId": "tool_abc", "name": "calculator", "input": {"expression": "5+5"}}},
+            ],
+        },
+        {
+            "role": "user",
+            "content": [{"text": "Some user text without toolResult"}],  # No toolResult key
+        },
+        {
+            "role": "user",
+            "content": [
+                {
+                    "toolResult": {
+                        "status": "success",
+                        "content": [{"text": "Result: 10"}],
+                        "toolUseId": "tool_abc",
+                    }
+                }
+            ],
+        },
+    ]
+    result = extract_agent_tools_used_from_messages(messages)
+
+    assert len(result) == 1
+    assert result[0]["name"] == "calculator"
+    assert result[0]["input"] == {"expression": "5+5"}
+    assert result[0]["tool_result"] == "Result: 10"
+    assert result[0]["is_error"] is False

{strands_agents_evals-0.1.1 → strands_agents_evals-0.1.3}/tests/strands_evals/test_experiment.py

@@ -34,6 +34,16 @@ class MockEvaluator2(Evaluator[str, str]):
         return [EvaluationOutput(score=0.5, test_pass=True, reason="Async test evaluation 2")]
 
 
+class ThrowingEvaluator(Evaluator[str, str]):
+    """Evaluator that always throws an exception - used to test error isolation"""
+
+    def evaluate(self, evaluation_case: EvaluationData[str, str]) -> list[EvaluationOutput]:
+        raise RuntimeError("Evaluator exploded")
+
+    async def evaluate_async(self, evaluation_case: EvaluationData[str, str]) -> list[EvaluationOutput]:
+        raise RuntimeError("Async evaluator exploded")
+
+
 @pytest.fixture
 def mock_evaluator():
     return MockEvaluator()
@@ -1052,3 +1062,29 @@ def test_experiment_run_evaluations_multiple_cases(mock_span, simple_task):
         assert len(reports) == 1
         assert len(reports[0].scores) == 2
         assert all(score == 1.0 for score in reports[0].scores)
+
+
+def test_experiment_run_evaluations_evaluator_error_isolated():
+    """Test that one evaluator failing doesn't affect other evaluators."""
+    case = Case(name="test", input="hello", expected_output="hello")
+
+    # MockEvaluator succeeds, ThrowingEvaluator fails
+    experiment = Experiment(cases=[case], evaluators=[MockEvaluator(), ThrowingEvaluator()])
+
+    def echo_task(c):
+        return c.input
+
+    reports = experiment.run_evaluations(echo_task)
+
+    assert len(reports) == 2
+
+    # First evaluator (MockEvaluator) should succeed
+    assert reports[0].scores[0] == 1.0
+    assert reports[0].test_passes[0] is True
+    assert reports[0].reasons[0] == "Mock evaluation"
+
+    # Second evaluator (ThrowingEvaluator) should fail with error message
+    assert reports[1].scores[0] == 0
+    assert reports[1].test_passes[0] is False
+    assert "Evaluator error" in reports[1].reasons[0]
+    assert "Evaluator exploded" in reports[1].reasons[0]