gooddata-eval 1.72.1.dev5__tar.gz → 1.73.0__tar.gz

{gooddata_eval-1.72.1.dev5 → gooddata_eval-1.73.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.72.1.dev5
+Version: 1.73.0
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.72.1.dev5
+Requires-Dist: gooddata-sdk~=1.73.0
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6
@@ -90,6 +90,40 @@ When the same model id is offered by multiple providers, use the
 
 Both provider name and provider id are accepted as the prefix.
 
+### Targeting a specific AI Hub agent
+
+GoodData has no admin-settable "default agent": when a conversation doesn't
+name one, the platform picks whichever agent was last used or last edited in
+that workspace. If your org has several AI Hub agents configured (e.g. one
+scoped to visualization only, another with every skill enabled), evaluating
+without `--agent-id` can silently exercise the wrong one — a
+`metric_skill`/`alert_skill` item run against a visualization-only agent will
+never pass, no matter how well-formed the question is.
+
+```bash
+export GD_EVAL_AGENT_ID='eval-all-skills'
+
+gd-eval run \
+  --host  https://your.gooddata.cloud \
+  --workspace  ecommerce_demo \
+  --dataset  ./my-dataset \
+  --model  gpt-5.2 \
+  --runs  1 \
+  --json  results.json
+```
+
+Or pass it explicitly instead of via the env var:
+
+```bash
+gd-eval run \
+  --host  https://your.gooddata.cloud \
+  --workspace  ecommerce_demo \
+  --dataset  ./my-dataset \
+  --agent-id  eval-all-skills \
+  --model  gpt-5.2 \
+  --runs  1
+```
+
 ### All flags
 
 #### Connection
@@ -100,6 +134,7 @@ Both provider name and provider id are accepted as the prefix.
 | `--token TOKEN` | `GOODDATA_TOKEN` | API token. Pass via flag or env var. |
 | `--profile NAME` | — | Profile name in `~/.gooddata/profiles.yaml` (same file as the `gdc` CLI). |
 | `--workspace ID` | — | **Required.** Workspace id to evaluate against. |
+| `--agent-id ID` | `GD_EVAL_AGENT_ID` | AI Hub agent every conversation should target. GoodData has no admin-settable default agent — without this, each conversation falls back to whichever agent the platform's last-used/last-edited heuristic resolves, which may not have every skill under test enabled. |
 
 #### Dataset source (pick one)
 

{gooddata_eval-1.72.1.dev5 → gooddata_eval-1.73.0}/README.md

@@ -62,6 +62,40 @@ When the same model id is offered by multiple providers, use the
 
 Both provider name and provider id are accepted as the prefix.
 
+### Targeting a specific AI Hub agent
+
+GoodData has no admin-settable "default agent": when a conversation doesn't
+name one, the platform picks whichever agent was last used or last edited in
+that workspace. If your org has several AI Hub agents configured (e.g. one
+scoped to visualization only, another with every skill enabled), evaluating
+without `--agent-id` can silently exercise the wrong one — a
+`metric_skill`/`alert_skill` item run against a visualization-only agent will
+never pass, no matter how well-formed the question is.
+
+```bash
+export GD_EVAL_AGENT_ID='eval-all-skills'
+
+gd-eval run \
+  --host  https://your.gooddata.cloud \
+  --workspace  ecommerce_demo \
+  --dataset  ./my-dataset \
+  --model  gpt-5.2 \
+  --runs  1 \
+  --json  results.json
+```
+
+Or pass it explicitly instead of via the env var:
+
+```bash
+gd-eval run \
+  --host  https://your.gooddata.cloud \
+  --workspace  ecommerce_demo \
+  --dataset  ./my-dataset \
+  --agent-id  eval-all-skills \
+  --model  gpt-5.2 \
+  --runs  1
+```
+
 ### All flags
 
 #### Connection
@@ -72,6 +106,7 @@ Both provider name and provider id are accepted as the prefix.
 | `--token TOKEN` | `GOODDATA_TOKEN` | API token. Pass via flag or env var. |
 | `--profile NAME` | — | Profile name in `~/.gooddata/profiles.yaml` (same file as the `gdc` CLI). |
 | `--workspace ID` | — | **Required.** Workspace id to evaluate against. |
+| `--agent-id ID` | `GD_EVAL_AGENT_ID` | AI Hub agent every conversation should target. GoodData has no admin-settable default agent — without this, each conversation falls back to whichever agent the platform's last-used/last-edited heuristic resolves, which may not have every skill under test enabled. |
 
 #### Dataset source (pick one)
 

{gooddata_eval-1.72.1.dev5 → gooddata_eval-1.73.0}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.72.1.dev5"
+version = "1.73.0"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.72.1.dev5",
+    "gooddata-sdk~=1.73.0",
     "httpx>=0.27,<1.0",
     "orjson>=3.9.15,<4.0.0",
     "pydantic>=2.6,<3.0",

{gooddata_eval-1.72.1.dev5 → gooddata_eval-1.73.0}/src/gooddata_eval/cli/agentic_runner.py

@@ -16,7 +16,7 @@ from gooddata_eval.core.agentic.metric_skill import evaluate_agentic_metric_skil
 from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
 from gooddata_eval.core.config import ReasoningEffort
-from gooddata_eval.core.models import CreatedVisualization, DatasetItem
+from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 
 
@@ -85,8 +85,14 @@ def _dispatch_agentic(
     run_ts: str,
     model_version_override: str | None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> None:
-    """Call the appropriate evaluate_agentic_* function for the item's test_kind."""
+    agent_id: str | None = None,
+) -> AgenticEvalOutcome | list[str] | None:
+    """Call the appropriate evaluate_agentic_* function for the item's test_kind.
+
+    Returns whatever that function returns -- alert_skill/metric_skill/conversation return
+    an AgenticEvalOutcome; the rest still return None
+    (unchanged).
+    """
     kind = item.test_kind
     eo = item.expected_output
     lf_kw: _LfKw = {
@@ -99,66 +105,72 @@ def _dispatch_agentic(
     }
 
     if kind in ("vis_agentic", "agentic_visualization"):
-        evaluate_agentic_visualization(
+        return evaluate_agentic_visualization(
             host=host,
             token=token,
             workspace_id=workspace_id,
             question=item.question,
             expected_outputs=_parse_visualization_expected(eo),
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_metric_skill":
-        evaluate_agentic_metric_skill(
+        return evaluate_agentic_metric_skill(
             host=host,
             token=token,
             workspace_id=workspace_id,
             question=item.question,
             expected_output=eo if isinstance(eo, (dict, list)) else {},
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_alert_skill":
-        evaluate_agentic_alert_skill(
+        return evaluate_agentic_alert_skill(
             host=host,
             token=token,
             workspace_id=workspace_id,
             question=item.question,
             expected_output=eo if isinstance(eo, dict) else {},
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_search":
         eo_dict = eo if isinstance(eo, dict) else {}
         tool_call = eo_dict.get("tool_call", {})
         expected_args = tool_call.get("function_arguments", eo_dict)
-        evaluate_agentic_search_tool(
+        return evaluate_agentic_search_tool(
             host=host,
             token=token,
             workspace_id=workspace_id,
             question=item.question,
             expected_tool_call=expected_args,
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_general_question":
-        evaluate_agentic_general_question(
+        return evaluate_agentic_general_question(
             host=host,
             token=token,
             workspace_id=workspace_id,
             question=item.question,
             expected_output=eo if isinstance(eo, str) else str(eo),
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_guardrail":
-        evaluate_agentic_guardrail(
+        return evaluate_agentic_guardrail(
             host=host,
             token=token,
             workspace_id=workspace_id,
             question=item.question,
             expected_output=eo if isinstance(eo, str) else str(eo),
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_kda_skill":
@@ -173,11 +185,12 @@ def _dispatch_agentic(
         )
     elif kind == "agentic_conversation":
         fixture_data = eo.get("fixture") or eo if isinstance(eo, dict) else {}
-        evaluate_agentic_conversation(
+        return evaluate_agentic_conversation(
             host=host,
             token=token,
             workspace_id=workspace_id,
             fixture=ConversationFixture.model_validate(fixture_data),
+            agent_id=agent_id,
             **lf_kw,
         )
     else:
@@ -197,6 +210,7 @@ def run_agentic_items(
     run_ts: str,
     on_item_start: Any = None,
     on_item_done: Any = None,
+    agent_id: str | None = None,
 ) -> EvalReport:
     """Run agentic items through evaluate_agentic_* and return an EvalReport."""
     langfuse = make_langfuse_client() if use_langfuse else None
@@ -219,12 +233,26 @@ def run_agentic_items(
         )
         t0 = time.perf_counter()
         try:
-            _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort)
+            outcome = _dispatch_agentic(
+                item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort, agent_id
+            )
+            if isinstance(outcome, AgenticEvalOutcome):
+                reasoning_steps = outcome.reasoning_steps
+                conversation_id = outcome.conversation_id
+                response_id = outcome.response_id
+            else:
+                reasoning_steps, conversation_id, response_id = outcome, None, None
             item_report.pass_at_k = True
             item_report.runs = k
+            item_report.reasoning_steps = reasoning_steps or []
+            item_report.conversation_id = conversation_id
+            item_report.response_id = response_id
         except AssertionError as exc:
             item_report.pass_at_k = False
             item_report.runs = k
+            item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or []
+            item_report.conversation_id = getattr(exc, "conversation_id", None)
+            item_report.response_id = getattr(exc, "response_id", None)
             print(f"[agentic] {item.id} FAIL: {exc}", flush=True)
         except Exception as exc:
             item_report.error = f"{type(exc).__name__}: {exc}"

{gooddata_eval-1.72.1.dev5 → gooddata_eval-1.73.0}/src/gooddata_eval/cli/main.py

@@ -2,6 +2,7 @@
 """`gd-eval` command-line entry point."""
 
 import argparse
+import os
 import sys
 import threading
 from datetime import datetime, timezone
@@ -117,6 +118,16 @@ def _build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).",
     )
+    run.add_argument(
+        "--agent-id",
+        dest="agent_id",
+        help=(
+            "AI Hub agent id every conversation should target (or set GD_EVAL_AGENT_ID). "
+            "GoodData has no admin-settable default agent -- without this, each conversation "
+            "falls back to whichever agent the platform's last-used/last-edited heuristic "
+            "resolves, which may not have every skill under test enabled."
+        ),
+    )
     models_cmd = sub.add_parser("models", help="List LLM providers and models configured in the org.")
     models_cmd.add_argument("--host", help="GoodData host URL.")
     models_cmd.add_argument("--token", help="API token (or set GOODDATA_TOKEN).")
@@ -347,6 +358,7 @@ def _run(config: RunConfig) -> int:
                     run_ts=run_ts,
                     on_item_start=on_item_start,
                     on_item_done=on_item_done,
+                    agent_id=config.agent_id,
                 )
 
             # --- non-agentic items (single-turn, use Evaluator) ---
@@ -357,6 +369,7 @@ def _run(config: RunConfig) -> int:
                     workspace_id=config.workspace_id,
                     preserve_failed=config.preserve_failed,
                     reasoning_effort=config.reasoning_effort,
+                    agent_id=config.agent_id,
                 ),
                 SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
             )
@@ -449,6 +462,7 @@ def main(argv: list[str] | None = None) -> int:
             kind=args.kind,
             preserve_failed=args.preserve_failed,
             reasoning_effort=args.reasoning_effort,
+            agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"),
         )
         return _run(config)
     except (

{gooddata_eval-1.72.1.dev5 → gooddata_eval-1.73.0}/src/gooddata_eval/core/agentic/alert_skill.py

@@ -6,7 +6,7 @@ from __future__ import annotations
 import json
 import os
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any
 
 from gooddata_sdk import GoodDataSdk
@@ -14,7 +14,7 @@ from gooddata_sdk import GoodDataSdk
 from gooddata_eval.core.agentic._catalog import CatalogMetricAlert
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
-from gooddata_eval.core.models import ToolCallEvent
+from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent
 
 try:
     from openai import OpenAI as _OpenAI
@@ -119,7 +119,31 @@ def _check_metric(expected: CatalogMetricAlert, actual_args: dict) -> bool:
     return expected.metric_id == act_metric
 
 
-def _check_recipients(expected: CatalogMetricAlert, actual_args: dict) -> bool:
+def _resolve_internal_recipient_ids(sdk: GoodDataSdk, emails: list[str]) -> set[str]:
+    """Best-effort map of expected recipient emails to internal GoodData user ids.
+
+    Some notification channels are workspace-restricted to internal users --
+    `create_metric_alert` then addresses the alert by internal user id
+    (`internal_recipients`), never by email, so an expected email has to be
+    resolved before it can be compared against that field. Failures (no
+    matching user, no permission, network error) are swallowed: the caller
+    treats an empty result the same as "this delivery path doesn't match",
+    which is correct -- it doesn't mean the alert itself failed.
+    """
+    if not emails:
+        return set()
+    try:
+        # RSQL quoted-string escaping: backslash first, then the enclosing quote char,
+        # or an email like o'hara@example.com breaks the filter into invalid RSQL.
+        escaped = [email.replace("\\", "\\\\").replace("'", "\\'") for email in emails]
+        quoted = ",".join(f"'{email}'" for email in escaped)
+        resp = sdk._client.entities_api.get_all_entities_users(filter=f"email=in=({quoted})")
+        return {u.id for u in (resp.data or [])}
+    except Exception:
+        return set()
+
+
+def _check_recipients(expected: CatalogMetricAlert, actual_args: dict, sdk: GoodDataSdk | None = None) -> bool:
     if not expected.recipients:
         return True
     act_recip_raw = actual_args.get("recipients", actual_args.get("external_recipients"))
@@ -134,7 +158,14 @@ def _check_recipients(expected: CatalogMetricAlert, actual_args: dict) -> bool:
         act_recip = act_recip_raw
     else:
         act_recip = []
-    return set(expected.recipients) == set(act_recip or [])
+    if set(expected.recipients) == set(act_recip or []):
+        return True
+    act_internal = actual_args.get("internal_recipients")
+    if sdk is not None and isinstance(act_internal, list) and act_internal:
+        internal_recipient_ids = _resolve_internal_recipient_ids(sdk, expected.recipients)
+        if internal_recipient_ids & set(act_internal):
+            return True
+    return False
 
 
 def generate_simulated_alert_response(
@@ -302,6 +333,8 @@ class AlertRunResult:
     alert_id: str | None
     eval: AlertEvaluation
     actual_alert_arguments: dict
+    reasoning_steps: list[str] = field(default_factory=list)
+    response_id: str | None = None
 
 
 @dataclass
@@ -434,11 +467,14 @@ def run_agentic_alert_skill(
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
     reasoning_effort: ReasoningEffort | None = None,
+    agent_id: str | None = None,
 ) -> AgenticAlertSummary:
     """Run the alert-skill agentic evaluation K times and return a summary."""
     expected = _normalize_expected_output(expected_output)
     run_results: list[AlertRunResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
+    client = ChatClient(
+        host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id
+    )
     sdk = GoodDataSdk.create(host, token)
 
     def _run_once(conv_id: str) -> AlertRunResult:
@@ -447,6 +483,8 @@ def run_agentic_alert_skill(
             alert_id: str | None = None
             actual_args: dict = {}
             tool_called = False
+            reasoning_steps: list[str] = []
+            response_id: str | None = None
             # conversation_history stores prior turns for GPT-4o context.
             # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply.
             conversation_history: list = []
@@ -454,6 +492,8 @@ def run_agentic_alert_skill(
 
             for _iteration in range(max_iterations):
                 chat_result = client.send_message(conv_id, current_question)
+                reasoning_steps.extend(chat_result.reasoning_steps or [])
+                response_id = chat_result.response_id or response_id
                 alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or [])
                 if tool_called:
                     alert_id_to_delete = alert_id
@@ -482,13 +522,15 @@ def run_agentic_alert_skill(
                 trigger_correct=tool_called and _check_trigger(expected, actual_args),
                 filters_correct=tool_called and _check_filters(expected, actual_args),
                 metric_correct=tool_called and _check_metric(expected, actual_args),
-                recipients_correct=tool_called and _check_recipients(expected, actual_args),
+                recipients_correct=tool_called and _check_recipients(expected, actual_args, sdk=sdk),
             )
             return AlertRunResult(
                 conversation_id=conv_id,
                 alert_id=alert_id,
                 eval=ev,
                 actual_alert_arguments=actual_args,
+                reasoning_steps=reasoning_steps,
+                response_id=response_id,
             )
         finally:
             if alert_id_to_delete:
@@ -539,6 +581,9 @@ class AlertSkillAssertionError(AssertionError):
     """Raised when an alert-skill evaluation fails."""
 
     __tracebackhide__ = True
+    reasoning_steps: list[str]
+    conversation_id: str
+    response_id: str | None
 
 
 def evaluate_agentic_alert_skill(
@@ -550,6 +595,7 @@ def evaluate_agentic_alert_skill(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    agent_id: str | None = None,
     langfuse: object | None = None,
     dataset_item_id: str = "",
     dataset_name: str = "alert_skill",
@@ -557,8 +603,16 @@ def evaluate_agentic_alert_skill(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> None:
-    """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
+) -> AgenticEvalOutcome:
+    """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.
+
+    Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an
+    AgenticEvalOutcome on success; on failure the same three values are attached to the
+    raised exception as
+    ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the
+    `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them
+    either way.
+    """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
 
@@ -577,6 +631,7 @@ def evaluate_agentic_alert_skill(
         max_iterations=max_iterations,
         initial_conversation_id=initial_conversation_id,
         reasoning_effort=reasoning_effort,
+        agent_id=agent_id,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -631,7 +686,7 @@ def evaluate_agentic_alert_skill(
     if not summary.pass_at_k:
         best = summary.best
         ev = best.eval
-        raise AlertSkillAssertionError(
+        exc = AlertSkillAssertionError(
             f"Alert skill assertion failed. strict_pass={ev.strict_pass}. "
             f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, "
             f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, "
@@ -639,3 +694,12 @@ def evaluate_agentic_alert_skill(
             f"recipients_correct={ev.recipients_correct}. "
             f"Actual args: {best.actual_alert_arguments}"
         )
+        exc.reasoning_steps = best.reasoning_steps
+        exc.conversation_id = best.conversation_id
+        exc.response_id = best.response_id
+        raise exc
+    return AgenticEvalOutcome(
+        reasoning_steps=summary.best.reasoning_steps,
+        conversation_id=summary.best.conversation_id,
+        response_id=summary.best.response_id,
+    )