gooddata-eval 1.70.1.dev1__tar.gz → 1.71.0__tar.gz

{gooddata_eval-1.70.1.dev1 → gooddata_eval-1.71.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-eval
-Version: 1.70.1.dev1
+Version: 1.71.0
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.70.1.dev1
+Requires-Dist: gooddata-sdk~=1.71.0
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6

{gooddata_eval-1.70.1.dev1 → gooddata_eval-1.71.0}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.70.1.dev1"
+version = "1.71.0"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.70.1.dev1",
+    "gooddata-sdk~=1.71.0",
     "httpx>=0.27,<1.0",
     "orjson>=3.9.15,<4.0.0",
     "pydantic>=2.6,<3.0",

{gooddata_eval-1.70.1.dev1 → gooddata_eval-1.71.0}/src/gooddata_eval/core/evaluators/alert_skill.py

@@ -25,6 +25,20 @@ def _extract_metric_id(metric_str: str) -> str | None:
     return match.group(1) if match else None
 
 
+def _extract_automation_id(tool_event) -> str | None:
+    """Real server-assigned id of the automation this call created.
+
+    Mirrors ``core/agentic/alert_skill.py``'s ``_extract_alert_call`` — the id
+    lives in the tool *result* (what the server assigned), not the tool call
+    *arguments* (what the agent asked for).
+    """
+    result_data = tool_event.parsed_result()
+    if not isinstance(result_data, dict):
+        return None
+    nested_data = result_data.get("data")
+    return result_data.get("id") or (nested_data.get("id") if isinstance(nested_data, dict) else None)
+
+
 def _check_threshold(expected: dict, actual_args: dict) -> bool:
     operator = expected.get("Operator", "")
     if operator == "ANOMALY":
@@ -58,10 +72,11 @@ class AlertSkillEvaluator:
             return ItemEvaluation(
                 passed=False,
                 rank_key=(False,) * 7,
-                detail={"alert_created": False},
+                detail={"alert_created": False, "automation_id": None},
             )
 
         args = tool_event.parsed_arguments()
+        automation_id = _extract_automation_id(tool_event)
 
         operator_correct = True
         threshold_correct = True
@@ -126,5 +141,9 @@ class AlertSkillEvaluator:
                 "filters_correct": filters_correct,
                 "metric_correct": metric_correct,
                 "recipients_correct": recipients_correct,
+                # Real server-assigned id of the automation this run created —
+                # lets a caller (e.g. a cleanup step) delete the exact object
+                # created instead of diffing the workspace catalog before/after.
+                "automation_id": automation_id,
             },
         )

{gooddata_eval-1.70.1.dev1 → gooddata_eval-1.71.0}/src/gooddata_eval/core/evaluators/metric_skill.py

@@ -28,7 +28,7 @@ class MetricSkillEvaluator:
             return ItemEvaluation(
                 passed=False,
                 rank_key=(False, False, False),
-                detail={"metric_created": False, "maql_correct": False, "format_correct": False},
+                detail={"metric_created": False, "maql_correct": False, "format_correct": False, "metric_id": None},
             )
 
         result = tool_event.parsed_result()
@@ -54,5 +54,10 @@ class MetricSkillEvaluator:
                 "actual_maql": actual_maql,
                 "expected_format": expected_format,
                 "actual_format": actual_format,
+                # The real id of the metric this run created, straight from the
+                # create_metric tool result — lets a caller (e.g. a cleanup step)
+                # delete the exact object created instead of diffing the workspace
+                # catalog before/after and guessing by name.
+                "metric_id": payload.get("metric_id"),
             },
         )

{gooddata_eval-1.70.1.dev1 → gooddata_eval-1.71.0}/tests/test_alert_skill_evaluator.py

@@ -15,11 +15,15 @@ def _item(expected_output: dict) -> DatasetItem:
     )
 
 
-def _chat_with_alert(args: dict) -> ChatResult:
+def _chat_with_alert(args: dict, result: dict | None = None) -> ChatResult:
     return ChatResult.model_validate(
         {
             "toolCallEvents": [
-                {"functionName": "create_metric_alert", "functionArguments": json.dumps(args), "result": "{}"}
+                {
+                    "functionName": "create_metric_alert",
+                    "functionArguments": json.dumps(args),
+                    "result": json.dumps(result if result is not None else {}),
+                }
             ]
         }
     )
@@ -81,3 +85,40 @@ def test_alert_evaluator_fails_when_no_tool_call():
     )
     assert result.passed is False
     assert result.detail["alert_created"] is False
+    assert result.detail["automation_id"] is None
+
+
+def test_alert_evaluator_extracts_automation_id_from_top_level_result():
+    expected = {"Operator": "LESS_THAN", "Threshold": "20000"}
+    actual_args = {"operator": "LESS_THAN", "threshold": 20000}
+    result = get_evaluator("alert_skill").evaluate(
+        _item(expected), _chat_with_alert(actual_args, result={"id": "automation-abc-123"})
+    )
+    assert result.detail["automation_id"] == "automation-abc-123"
+
+
+def test_alert_evaluator_extracts_automation_id_from_nested_data_result():
+    expected = {"Operator": "LESS_THAN", "Threshold": "20000"}
+    actual_args = {"operator": "LESS_THAN", "threshold": 20000}
+    result = get_evaluator("alert_skill").evaluate(
+        _item(expected), _chat_with_alert(actual_args, result={"data": {"id": "automation-xyz-789"}})
+    )
+    assert result.detail["automation_id"] == "automation-xyz-789"
+
+
+def test_alert_evaluator_automation_id_none_when_result_has_no_id():
+    expected = {"Operator": "LESS_THAN", "Threshold": "20000"}
+    actual_args = {"operator": "LESS_THAN", "threshold": 20000}
+    result = get_evaluator("alert_skill").evaluate(_item(expected), _chat_with_alert(actual_args, result={}))
+    assert result.detail["automation_id"] is None
+
+
+def test_alert_evaluator_automation_id_none_when_nested_data_is_not_a_mapping():
+    # Malformed/unexpected payload shape: "data" present but not a dict -> must
+    # not crash, just report automation_id=None (CodeRabbit review on PR #1694).
+    expected = {"Operator": "LESS_THAN", "Threshold": "20000"}
+    actual_args = {"operator": "LESS_THAN", "threshold": 20000}
+    result = get_evaluator("alert_skill").evaluate(
+        _item(expected), _chat_with_alert(actual_args, result={"data": "not-a-mapping"})
+    )
+    assert result.detail["automation_id"] is None

{gooddata_eval-1.70.1.dev1 → gooddata_eval-1.71.0}/tests/test_metric_skill_evaluator.py

@@ -32,6 +32,7 @@ def test_metric_evaluator_passes_on_exact_match():
     assert result.passed is True
     assert result.detail["maql_correct"] is True
     assert result.detail["format_correct"] is True
+    assert result.detail["metric_id"] == "avg_order_value"
 
 
 def test_metric_evaluator_fails_wrong_maql():
@@ -48,3 +49,4 @@ def test_metric_evaluator_fails_when_no_tool_call():
     result = ev.evaluate(_item(), empty)
     assert result.passed is False
     assert result.detail["metric_created"] is False
+    assert result.detail["metric_id"] is None