gooddata-eval 1.75.1.dev4__tar.gz → 1.76.1.dev1__tar.gz

{gooddata_eval-1.75.1.dev4 → gooddata_eval-1.76.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.75.1.dev4
+Version: 1.76.1.dev1
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.75.1.dev4
+Requires-Dist: gooddata-sdk~=1.76.1.dev1
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: jsonpatch<2.0,>=1.33
 Requires-Dist: orjson<4.0.0,>=3.9.15

{gooddata_eval-1.75.1.dev4 → gooddata_eval-1.76.1.dev1}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.75.1.dev4"
+version = "1.76.1.dev1"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.75.1.dev4",
+    "gooddata-sdk~=1.76.1.dev1",
     "httpx>=0.27,<1.0",
     "jsonpatch>=1.33,<2.0",
     "orjson>=3.9.15,<4.0.0",

{gooddata_eval-1.75.1.dev4 → gooddata_eval-1.76.1.dev1}/src/gooddata_eval/cli/agentic_runner.py

@@ -12,12 +12,14 @@ from gooddata_eval.core.agentic._gate import DEFAULT_GATE, EvalGate, normalize_g
 from gooddata_eval.core.agentic._langfuse import make_langfuse_client
 from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline
 from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill
+from gooddata_eval.core.agentic.anomaly_detection import evaluate_agentic_anomaly_detection
 from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation
 from gooddata_eval.core.agentic.dashboard_skill import evaluate_agentic_dashboard_skill
 from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question
 from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail
 from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill
 from gooddata_eval.core.agentic.metric_skill import evaluate_agentic_metric_skill
+from gooddata_eval.core.agentic.report_skill import evaluate_agentic_report_skill
 from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
 from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if
@@ -43,11 +45,13 @@ AGENTIC_TEST_KINDS = frozenset(
         "agentic_metric_skill",
         "agentic_alert_skill",
         "agentic_dashboard_skill",
+        "agentic_report_skill",
         "agentic_search",
         "agentic_general_question",
         "agentic_guardrail",
         "agentic_conversation",
         "agentic_kda_skill",
+        "agentic_anomaly_detection",
         "agentic_what_if",
     }
 )
@@ -92,7 +96,8 @@ PARALLEL_SAFE_TEST_KINDS = frozenset(
 #
 # agentic_dashboard_skill is absent by default rather than by evidence: gen-ai holds the draft and
 # any chart it authors in conversation state and writes neither until a user saves from the UI, so
-# it is a candidate for the allowlist once the dataset has runs behind it.
+# it is a candidate for the allowlist once the dataset has runs behind it. agentic_report_skill is
+# absent for the same reason: the report draft stays in conversation state until a user saves it.
 WORKSPACE_MUTATING_TEST_KINDS = frozenset(AGENTIC_TEST_KINDS) - PARALLEL_SAFE_TEST_KINDS
 
 
@@ -203,6 +208,18 @@ def _dispatch_agentic(
             agent_id=agent_id,
             **lf_kw,
         )
+    elif kind == "agentic_report_skill":
+        return evaluate_agentic_report_skill(
+            host=host,
+            token=token,
+            workspace_id=workspace_id,
+            question=item.question,
+            expected_output=eo,
+            k=k,
+            gate=gate,
+            agent_id=agent_id,
+            **lf_kw,
+        )
     elif kind == "agentic_alert_skill":
         return evaluate_agentic_alert_skill(
             host=host,
@@ -278,6 +295,18 @@ def _dispatch_agentic(
             agent_id=agent_id,
             **lf_kw,
         )
+    elif kind == "agentic_anomaly_detection":
+        return evaluate_agentic_anomaly_detection(
+            host=host,
+            token=token,
+            workspace_id=workspace_id,
+            question=item.question,
+            expected_output=eo if isinstance(eo, dict) else {},
+            k=k,
+            gate=gate,
+            agent_id=agent_id,
+            **lf_kw,
+        )
     elif kind == "agentic_conversation":
         fixture_data = eo.get("fixture") or eo if isinstance(eo, dict) else {}
         return evaluate_agentic_conversation(

{gooddata_eval-1.75.1.dev4 → gooddata_eval-1.76.1.dev1}/src/gooddata_eval/core/agentic/__init__.py

@@ -53,6 +53,14 @@ from gooddata_eval.core.agentic.metric_skill import (
     evaluate_agentic_metric_skill,
     run_agentic_metric_skill,
 )
+from gooddata_eval.core.agentic.report_skill import (
+    AgenticReportSummary,
+    ReportEvaluation,
+    ReportRunResult,
+    ReportSkillAssertionError,
+    evaluate_agentic_report_skill,
+    run_agentic_report_skill,
+)
 from gooddata_eval.core.agentic.search_tool import (
     AgenticSearchSummary,
     SearchResult,
@@ -75,6 +83,7 @@ __all__ = [
     "AgenticGuardrailSummary",
     "AgenticKdaSummary",
     "AgenticMetricSummary",
+    "AgenticReportSummary",
     "AgenticSearchSummary",
     "AgenticRunSummary",
     "AlertEvaluation",
@@ -95,6 +104,9 @@ __all__ = [
     "KdaSkillAssertionError",
     "MetricRunResult",
     "MetricSkillAssertionError",
+    "ReportEvaluation",
+    "ReportRunResult",
+    "ReportSkillAssertionError",
     "RunResult",
     "SearchResult",
     "SearchToolAssertionError",
@@ -108,6 +120,7 @@ __all__ = [
     "evaluate_agentic_guardrail",
     "evaluate_agentic_kda_skill",
     "evaluate_agentic_metric_skill",
+    "evaluate_agentic_report_skill",
     "evaluate_agentic_search_tool",
     "evaluate_agentic_visualization",
     "run_agentic_alert_skill",
@@ -117,6 +130,7 @@ __all__ = [
     "run_agentic_guardrail",
     "run_agentic_kda_skill",
     "run_agentic_metric_skill",
+    "run_agentic_report_skill",
     "run_agentic_search_tool",
     "run_agentic_visualization",
 ]

{gooddata_eval-1.75.1.dev4 → gooddata_eval-1.76.1.dev1}/src/gooddata_eval/core/agentic/alert_skill.py

@@ -30,11 +30,12 @@ from gooddata_eval.core.agentic._trace_linker import (
     utc_now,
 )
 from gooddata_eval.core.chat.render import render_answer_text
-from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.chat.sse_client import ChatClient, ChatError
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import (
     AgenticAssertionError,
     AgenticEvalOutcome,
+    LoopExit,
     ReasoningStepEvent,
     ToolCallEvent,
     shift_and_index_events,
@@ -483,6 +484,12 @@ class AlertRunResult:
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
     reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
+    # Why the simulated-user loop stopped, and how many turns it took. Without these a run
+    # that ran out of turns is indistinguishable from one that refused: both land on
+    # alert_created=False, and every downstream check is `alert_created and ...`, so both
+    # also report operator/threshold/metric/recipients as False.
+    exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED
+    turns_used: int = 0
 
 
 @dataclass
@@ -681,8 +688,35 @@ def run_agentic_alert_skill(
             turns = 0
             steps = 0
 
+            # Defaults to BUDGET_EXHAUSTED: every other exit sets it explicitly, so a loop
+            # that simply runs out of range() is correctly labelled without a trailing else.
+            exit_reason = LoopExit.BUDGET_EXHAUSTED
+            turns_used = 0
             for _iteration in range(max_iterations):
-                chat_result = client.send_message(conv_id, current_question)
+                turns_used = _iteration + 1
+                try:
+                    chat_result = client.send_message(conv_id, current_question)
+                except ChatError as exc:
+                    # Without this the exception escapes run_agentic_alert_skill entirely,
+                    # discarding every K-run already completed along with any exit_reason.
+                    print(f"[CHAT] send_message failed for conversation {conv_id}: {exc}")
+                    # The stream can break AFTER create_metric_alert already succeeded
+                    # server-side. That id only ever reached alert_id_to_delete from the
+                    # normal path below, so breaking here left the alert in the workspace for
+                    # the `finally` cleanup to miss -- a real object leaking out of a failed
+                    # run, not a reporting gap.
+                    partial = exc.partial_result
+                    if partial is not None:
+                        reasoning_steps.extend(partial.reasoning_steps or [])
+                        response_id = partial.response_id or response_id
+                        partial_alert_id, _, partial_tool_called = _extract_alert_call(partial.tool_call_events or [])
+                        if partial_tool_called:
+                            alert_id_to_delete = partial_alert_id
+                    exit_reason = LoopExit.CHAT_ERROR
+                    break
+                # `turns_used` counts attempts (set above, so a failed send still shows the
+                # loop reached this iteration); `turns`/`steps` count completed work, so they
+                # advance only once send_message has actually returned.
                 turns += 1
                 steps += chat_result.reasoning_step_count
                 reasoning_steps.extend(chat_result.reasoning_steps or [])
@@ -698,6 +732,7 @@ def run_agentic_alert_skill(
                 alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or [])
                 if tool_called:
                     alert_id_to_delete = alert_id
+                    exit_reason = LoopExit.SUCCESS
                     break
                 response_text = (chat_result.text_response or "").strip()
                 if not response_text and chat_result.alert_proposals:
@@ -706,13 +741,24 @@ def run_agentic_alert_skill(
                     response_text = render_answer_text(chat_result)
                 # Stop if agent gave a completely empty response (stuck)
                 if not response_text and not chat_result.tool_call_events:
+                    exit_reason = LoopExit.AGENT_SILENT
                     break
                 # Stop before generating a follow-up for the last iteration
                 if _iteration >= max_iterations - 1:
                     break
-                follow_up = generate_simulated_alert_response(
-                    response_text, expected, conversation_history, question=question
-                )
+                # Recorded rather than raised, matching metric_skill and kda_skill. Letting it
+                # propagate did keep a harness fault from being scored as a content failure,
+                # but it also discarded the K-runs already completed -- and SIMULATED_USER_FAILED
+                # achieves the same separation while keeping them, since reporting reads the
+                # exit reason to classify the run as an error rather than an agent failure.
+                try:
+                    follow_up = generate_simulated_alert_response(
+                        response_text, expected, conversation_history, question=question
+                    )
+                except Exception as exc:  # noqa: BLE001 -- harness-side fault; end only this run
+                    print(f"[SIM-USER] Simulated reply failed for conversation {conv_id}: {exc}")
+                    exit_reason = LoopExit.SIMULATED_USER_FAILED
+                    break
                 # Record this exchange so the next call has full history
                 conversation_history.append({"role": "assistant", "content": response_text})
                 conversation_history.append({"role": "user", "content": follow_up})
@@ -740,6 +786,8 @@ def run_agentic_alert_skill(
                 response_id=response_id,
                 tool_call_events=all_tool_call_events,
                 reasoning_step_events=all_reasoning_step_events,
+                exit_reason=exit_reason,
+                turns_used=turns_used,
             )
         finally:
             if alert_id_to_delete:
@@ -908,6 +956,11 @@ def evaluate_agentic_alert_skill(
         "attributes_correct": ev.attributes_correct,
         "granularity_correct": ev.granularity_correct,
         "actual_alert_arguments": best.actual_alert_arguments,
+        # Why the loop stopped. alert_created=False alone cannot tell a refusal from a run
+        # that hit max_iterations while still on track -- see LoopExit.
+        "exit_reason": best.exit_reason.value,
+        "turns_used": best.turns_used,
+        "max_iterations": max_iterations,
         **timeline_detail(best.tool_call_events, best.reasoning_step_events),
     }