gooddata-eval 1.68.1.dev4__tar.gz → 1.69.1.dev1__tar.gz

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-eval
-Version: 1.68.1.dev4
+Version: 1.69.1.dev1
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.68.1.dev4
+Requires-Dist: gooddata-sdk~=1.69.1.dev1
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.68.1.dev4"
+version = "1.69.1.dev1"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.68.1.dev4",
+    "gooddata-sdk~=1.69.1.dev1",
     "httpx>=0.27,<1.0",
     "orjson>=3.9.15,<4.0.0",
     "pydantic>=2.6,<3.0",
@@ -45,6 +45,7 @@ dev = [
 test = [
     "pytest~=8.3.4",
     "pytest-cov~=6.0.0",
+    "pytest-json-report==1.5.0",
     "pytest-mock>=3.14.0",
 ]
 

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/cli/agentic_runner.py

@@ -6,7 +6,7 @@ from __future__ import annotations
 import time
 from typing import Any, TypedDict
 
-from gooddata_eval.core.agentic._langfuse import HttpxLangfuseClient, make_langfuse_client
+from gooddata_eval.core.agentic._langfuse import make_langfuse_client
 from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill
 from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation
 from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question
@@ -17,17 +17,14 @@ from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualizat
 from gooddata_eval.core.models import CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 
-_LfKw = TypedDict(
-    "_LfKw",
-    {
-        "langfuse": Any,
-        "dataset_item_id": str,
-        "dataset_name": str,
-        "run_timestamp": str,
-        "model_version_override": str | None,
-    },
-    total=False,
-)
+
+class _LfKw(TypedDict, total=False):
+    langfuse: Any
+    dataset_item_id: str
+    dataset_name: str
+    run_timestamp: str
+    model_version_override: str | None
+
 
 AGENTIC_TEST_KINDS = frozenset(
     {

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/cli/main.py

@@ -3,6 +3,7 @@
 
 import argparse
 import sys
+import threading
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -11,6 +12,7 @@ from gooddata_api_client.exceptions import ApiException
 from rich.console import Console
 from rich.table import Table
 
+from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import RunConfig
 from gooddata_eval.core.connection import ConnectionError_, resolve_connection
@@ -19,7 +21,6 @@ from gooddata_eval.core.langfuse.sink import LangfuseSink
 from gooddata_eval.core.models import ChatResult, DatasetItem
 from gooddata_eval.core.reporting.console import render_comparison, render_console
 from gooddata_eval.core.reporting.json_report import write_multi_model_report
-from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items
 from gooddata_eval.core.runner import ItemReport, run_items
 from gooddata_eval.core.summary.http_client import SummaryClient
 from gooddata_eval.core.workspace import ModelResolutionError, WorkspaceModelController
@@ -97,6 +98,12 @@ def _build_parser() -> argparse.ArgumentParser:
     )
     run.add_argument("--json", dest="json_path", help="Write a JSON report to this path.")
     run.add_argument("--quiet", action="store_true", help="Suppress per-item progress output.")
+    run.add_argument(
+        "--preserve-failed",
+        action="store_true",
+        dest="preserve_failed",
+        help="Keep failed conversations on the server for post-mortem inspection.",
+    )
     run.add_argument(
         "--langfuse",
         action="store_true",
@@ -139,14 +146,24 @@ def _parse_model_arg(val: str) -> tuple[str | None, str]:
 
 
 def _make_progress_callbacks(console: Console):
-    """Build (on_item_start, on_run_done, on_item_done) callbacks that stream progress."""
+    """Build (on_item_start, on_run_done, on_item_done) callbacks that stream progress.
+
+    A threading lock guards all console.print() calls so that concurrent
+    ``--concurrency 2+`` workers do not deadlock when stdout is piped
+    (e.g. running in a background process).
+    """
+    _print_lock = threading.Lock()
 
     def on_item_start(index: int, total: int, item: DatasetItem) -> None:
-        console.print(f"[dim]\\[{index}/{total}][/dim] [cyan]{item.id}[/cyan] {_truncate(item.question)}")
+        with _print_lock:
+            console.print(f"[dim]\\[{index}/{total}][/dim] [cyan]{item.id}[/cyan] {_truncate(item.question)}")
 
     def on_run_done(index: int, total: int, run_index: int, runs: int, passed: bool, latency: float) -> None:
         tag = "[green]pass[/green]" if passed else "[red]fail[/red]"
-        console.print(f"[dim]\\[{index}/{total}][/dim]     run {run_index}/{runs}  {tag}  [dim]{latency:.2f}s[/dim]")
+        with _print_lock:
+            console.print(
+                f"[dim]\\[{index}/{total}][/dim]     run {run_index}/{runs}  {tag}  [dim]{latency:.2f}s[/dim]"
+            )
 
     def on_item_done(index: int, total: int, report: ItemReport) -> None:
         if report.skipped:
@@ -165,7 +182,8 @@ def _make_progress_callbacks(console: Console):
                 f" [dim]({report.latency_s:.2f}s total, {report.avg_latency_s:.2f}s avg, "
                 f"quality={quality_str}, {report.runs} run(s))[/dim]"
             )
-        console.print(f"[dim]\\[{index}/{total}][/dim]   -> {tag} [cyan]{report.id}[/cyan]{suffix}")
+        with _print_lock:
+            console.print(f"[dim]\\[{index}/{total}][/dim]   -> {tag} [cyan]{report.id}[/cyan]{suffix}")
 
     return on_item_start, on_run_done, on_item_done
 
@@ -319,7 +337,12 @@ def _run(config: RunConfig) -> int:
 
             # --- non-agentic items (single-turn, use Evaluator) ---
             backend = _RoutingBackend(
-                ChatClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
+                ChatClient(
+                    host=config.host,
+                    token=config.token,
+                    workspace_id=config.workspace_id,
+                    preserve_failed=config.preserve_failed,
+                ),
                 SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
             )
             try:
@@ -409,6 +432,7 @@ def main(argv: list[str] | None = None) -> int:
             log_to_langfuse=args.langfuse,
             quiet=args.quiet,
             kind=args.kind,
+            preserve_failed=args.preserve_failed,
         )
         return _run(config)
     except (

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/alert_skill.py

@@ -7,12 +7,10 @@ import json
 import os
 import re
 from dataclasses import dataclass
-
 from typing import Any
 
-from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.agentic._catalog import CatalogMetricAlert
-
+from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.models import ToolCallEvent
 
 try:
@@ -80,7 +78,11 @@ def _check_trigger(expected: CatalogMetricAlert, actual_args: dict) -> bool:
     if expected.operator == "ANOMALY":
         return True
     exp_trigger = expected.trigger
-    act_trigger = actual_args.get("trigger", actual_args.get("triggerMode", "ALWAYS"))
+    # A missing OR explicit-null trigger means "unset" -> the product persists the
+    # default ALWAYS ("Every time"). `.get(k, default)` only returns the default when
+    # the key is ABSENT, but create_metric_alert serialises unset params as
+    # `trigger: null`, so chain with `or` to also cover the present-but-None case.
+    act_trigger = actual_args.get("trigger") or actual_args.get("triggerMode") or "ALWAYS"
     if exp_trigger in _ALWAYS_TRIGGER_VALUES:
         return act_trigger in {"ALWAYS", "Every time"}
     act_api = _TRIGGER_DISPLAY_TO_API.get(act_trigger, act_trigger)
@@ -438,7 +440,9 @@ def evaluate_agentic_alert_skill(
     model_version_override: str | None = None,
 ) -> None:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/conversation.py

@@ -386,7 +386,9 @@ def evaluate_agentic_conversation(
     model_version_override: str | None = None,
 ) -> None:
     """Run conversation evaluation, log to Langfuse, and raise on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/general_question.py

@@ -154,7 +154,9 @@ def evaluate_agentic_general_question(
     model_version_override: str | None = None,
 ) -> None:
     """Run general-question evaluation, log to Langfuse, and raise on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/guardrail.py

@@ -151,7 +151,9 @@ def evaluate_agentic_guardrail(
     model_version_override: str | None = None,
 ) -> None:
     """Run guardrail evaluation, log to Langfuse, and raise on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/metric_skill.py

@@ -3,11 +3,10 @@
 
 from __future__ import annotations
 
-from typing import Any
-
 import os
 import re
 from dataclasses import dataclass
+from typing import Any
 
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.models import ToolCallEvent
@@ -247,7 +246,9 @@ def evaluate_agentic_metric_skill(
     model_version_override: str | None = None,
 ) -> None:
     """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/search_tool.py

@@ -145,7 +145,9 @@ def evaluate_agentic_search_tool(
     model_version_override: str | None = None,
 ) -> None:
     """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/visualization.py

@@ -267,7 +267,9 @@ def evaluate_agentic_visualization(
 ) -> None:
     """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure."""
     import json as _json  # noqa: PLC0415
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/chat/sse_client.py

@@ -106,6 +106,7 @@ class _SseAccumulator:
     call_id_to_event_index: dict[str, int] = field(default_factory=dict)
     reasoning_steps: list[dict[str, Any]] = field(default_factory=list)
     adhoc_viz_args: list[dict[str, Any]] = field(default_factory=list)
+    response_id: str | None = None
 
 
 def _handle_text(content: dict[str, Any], acc: _SseAccumulator) -> None:
@@ -176,7 +177,9 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult:
             "objects": [acc.adhoc_viz_args[-1]],
             "reasoning": "\n".join(acc.viz_reasoning_parts),
         }
-    return ChatResult.model_validate(payload)
+    result = ChatResult.model_validate(payload)
+    result.response_id = acc.response_id
+    return result
 
 
 def parse_sse_lines(lines: Iterable[str]) -> ChatResult:
@@ -204,9 +207,13 @@ def parse_sse_lines(lines: Iterable[str]) -> ChatResult:
             if code in _RETRYABLE_STATUS_CODES:
                 raise TransientChatError(message, status_code=code, detail=detail)
             raise ChatError(message, status_code=code, detail=detail)
+        if event_data.get("responseId") and not acc.response_id:
+            acc.response_id = event_data["responseId"]
         item = event_data.get("item")
         if not item:
             continue
+        if item.get("responseId") and not acc.response_id:
+            acc.response_id = item["responseId"]
         role = item.get("role")
         content: dict[str, Any] = item.get("content") or {}
         ctype = content.get("type")
@@ -227,10 +234,13 @@ def parse_sse_lines(lines: Iterable[str]) -> ChatResult:
 class ChatClient:
     """Single-turn AI chat client over the GoodData AI conversation endpoints."""
 
-    def __init__(self, host: str, token: str, workspace_id: str, *, timeout: float = 300.0):
+    def __init__(
+        self, host: str, token: str, workspace_id: str, *, timeout: float = 300.0, preserve_failed: bool = False
+    ):
         self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations"
         self._auth = {"Authorization": f"Bearer {token}"}
         self._client = httpx.Client(timeout=timeout)
+        self._preserve_failed = preserve_failed
 
     def create_conversation(self) -> str:
         def _do() -> str:
@@ -264,12 +274,29 @@ class ChatClient:
         return _retry_transient(_do, is_retryable=_is_retryable_exc)
 
     def ask(self, item: DatasetItem) -> ChatResult:
-        """Run one single-turn conversation: create, send, parse, clean up."""
+        """Run one conversation: create, send, parse, clean up.
+
+        The conversation_id is attached to the returned ChatResult for tracing.
+        When ``preserve_failed`` is set, failed conversations are kept on the
+        server so they can be inspected after the run; the conversation_id is
+        attached to the raised exception as well.
+        """
         conversation_id = self.create_conversation()
+        success = False
         try:
-            return self.send_message(conversation_id, item.question)
+            result = self.send_message(conversation_id, item.question)
+            result.conversation_id = conversation_id
+            success = True
+            return result
+        except Exception as exc:
+            try:
+                object.__setattr__(exc, "conversation_id", conversation_id)
+            except TypeError:
+                pass  # C-extension exception that rejects __setattr__
+            raise
         finally:
-            self.delete_conversation(conversation_id)
+            if success or not self._preserve_failed:
+                self.delete_conversation(conversation_id)
 
     def close(self) -> None:
         self._client.close()

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/config.py

@@ -19,3 +19,4 @@ class RunConfig:
     log_to_langfuse: bool = False
     quiet: bool = False
     kind: str = "visualization"
+    preserve_failed: bool = False

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/evaluators/alert_skill.py

@@ -78,7 +78,9 @@ class AlertSkillEvaluator:
 
         if "Trigger" in expected:
             expected_trigger = _TRIGGER_MAP.get(expected["Trigger"], expected["Trigger"])
-            trigger_correct = args.get("trigger") == expected_trigger
+            # Omitted/null trigger persists as the product default ALWAYS ("Every time").
+            actual_trigger = args.get("trigger") or "ALWAYS"
+            trigger_correct = actual_trigger == expected_trigger
 
         if "Filters" in expected:
             actual_filters = args.get("filters") or []

gooddata_eval-1.69.1.dev1/src/gooddata_eval/core/evaluators/search_tool.py

@@ -0,0 +1,59 @@
+# (C) 2026 GoodData Corporation
+"""Evaluator for search_tool: agent must call the catalog search with expected parameters."""
+
+from gooddata_eval.core.evaluators.base import ItemEvaluation
+from gooddata_eval.core.models import ChatResult, DatasetItem
+
+
+def _normalize_str_list(value: object, *, lowercase: bool = False) -> list[str]:
+    # Arguments come from raw model-emitted JSON. The search_objects schema
+    # declares keywords/object_types as list[str], but a malformed tool call may
+    # send a non-list or non-string entries. Drop the offending entries defensively
+    # so bad input can't raise (.lower()/sorted() on a non-str) and abort the whole
+    # evaluation run; a non-list collapses to [] and the surviving strings are
+    # still compared normally.
+    if not isinstance(value, list):
+        return []
+    items = [item for item in value if isinstance(item, str)]
+    return sorted(item.lower() if lowercase else item for item in items)
+
+
+def _args_match(actual_args: dict, expected_args: dict) -> bool:
+    # Only keywords and object_types determine semantic correctness.
+    # limit is optional with a server-side default; emit_widget was renamed to
+    # user_requested_search in the tool schema — neither affects search quality.
+    actual_kw = _normalize_str_list(actual_args.get("keywords"), lowercase=True)
+    expected_kw = _normalize_str_list(expected_args.get("keywords"), lowercase=True)
+    if actual_kw != expected_kw:
+        return False
+    # object_types is compared case-sensitively (no lowercase=True): they are
+    # controlled ObjectType StrEnum values the model emits verbatim ("metric",
+    # "dashboard"), so a case mismatch is a genuine error, not a formatting quirk.
+    return _normalize_str_list(actual_args.get("object_types")) == _normalize_str_list(
+        expected_args.get("object_types")
+    )
+
+
+class SearchToolEvaluator:
+    test_kind = "search_tool"
+
+    def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation:
+        expected_call = (item.expected_output or {}).get("tool_call", {})
+        expected_fn = expected_call.get("function_name", "search_objects")
+        expected_args = expected_call.get("function_arguments", {})
+
+        matching_events = [ev for ev in chat_result.tool_call_events if ev.function_name == expected_fn]
+        tool_selection = len(matching_events) > 0
+        tool_correctness = any(_args_match(ev.parsed_arguments(), expected_args) for ev in matching_events)
+
+        # tool_selection is the hard gate; tool_correctness is scored but not blocking
+        return ItemEvaluation(
+            passed=tool_selection,
+            rank_key=(int(tool_selection), int(tool_correctness)),
+            detail={
+                "tool_selection": tool_selection,
+                "tool_correctness": tool_correctness,
+                "expected_function": expected_fn,
+                "calls_found": len(matching_events),
+            },
+        )

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/models.py

@@ -94,6 +94,8 @@ class ChatResult(BaseModel):
     created_visualizations: CreatedVisualizations | None = Field(default=None, alias="createdVisualizations")
     tool_call_events: list[ToolCallEvent] = Field(default_factory=list, alias="toolCallEvents")
     reasoning_step_count: int = Field(default=0, alias="reasoningStepCount")
+    conversation_id: str | None = Field(default=None, alias="conversationId")
+    response_id: str | None = Field(default=None, alias="responseId")
 
 
 class SummaryInput(BaseModel):

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/reporting/json_report.py

@@ -34,6 +34,8 @@ def _build_run_dict(report: EvalReport) -> dict:
                 "latency_s": round(item.latency_s, 3),
                 "avg_latency_s": round(item.avg_latency_s, 3),
                 "detail": item.best_detail,
+                "conversation_id": item.conversation_id,
+                "response_id": item.response_id,
             }
             for item in report.items
         },

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/runner.py

@@ -31,6 +31,8 @@ class ItemReport:
     runs: int = 0
     latency_s: float = 0.0  # total wall-clock across this item's runs
     best_detail: dict = field(default_factory=dict)
+    conversation_id: str | None = None
+    response_id: str | None = None
 
     @property
     def avg_latency_s(self) -> float:
@@ -112,6 +114,8 @@ def _run_one_item(
         for run_index in range(1, runs + 1):
             t0 = time.perf_counter()
             chat_result = backend.ask(item)
+            report.conversation_id = getattr(chat_result, "conversation_id", None) or report.conversation_id
+            report.response_id = getattr(chat_result, "response_id", None) or report.response_id
             evaluation = evaluator.evaluate(item, chat_result)
             latency = time.perf_counter() - t0
             report.runs += 1
@@ -123,7 +127,9 @@ def _run_one_item(
             if on_run_done is not None:
                 on_run_done(run_index, runs, evaluation.passed, latency)
     except Exception as e:  # agent/network/parse failure for this item
-        report.error = f"{type(e).__name__}: {e}"
+        conv_id = getattr(e, "conversation_id", None)
+        report.conversation_id = conv_id or report.conversation_id
+        report.error = f"{type(e).__name__}: {e}" + (f" [conversation_id={conv_id}]" if conv_id else "")
         if best is not None:
             report.best_detail = best.detail
         return report

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/tests/test_agentic_alert_skill.py

@@ -3,10 +3,10 @@
 from unittest.mock import MagicMock, patch
 
 from gooddata_eval.core.agentic.alert_skill import (
-    AgenticAlertSummary,
     AlertEvaluation,
-    AlertRunResult,
+    _check_trigger,
     _deep_subset,
+    _normalize_expected_output,
     _to_number,
     run_agentic_alert_skill,
 )
@@ -33,6 +33,24 @@ def test_deep_subset_missing_key():
     assert _deep_subset({"a": 1, "c": 3}, {"a": 1}) is False
 
 
+def test_check_trigger_missing_or_null_defaults_to_always():
+    # "Every time" is the product default -> the agent may omit the trigger arg
+    # entirely, or serialise it as null. Both must count as ALWAYS, not a mismatch.
+    expected = _normalize_expected_output({"Operator": "GREATER_THAN", "Trigger": "Every time"})
+    assert _check_trigger(expected, {"operator": "GREATER_THAN"}) is True  # key absent
+    assert _check_trigger(expected, {"trigger": None}) is True  # present-but-null (the bug)
+    assert _check_trigger(expected, {"trigger": "ALWAYS"}) is True
+
+
+def test_check_trigger_once_needs_explicit_once():
+    # A "One time" expectation must still require an explicit ONCE - the null-default
+    # fix must not turn a wrong/absent trigger into a pass here.
+    expected = _normalize_expected_output({"Operator": "LESS_THAN", "Trigger": "One time"})
+    assert _check_trigger(expected, {"trigger": "ONCE"}) is True
+    assert _check_trigger(expected, {"trigger": None}) is False  # null != ONCE
+    assert _check_trigger(expected, {"trigger": "ONCE_PER_INTERVAL"}) is False  # real model error stays a fail
+
+
 def test_alert_evaluation_strict_pass():
     ev = AlertEvaluation(
         alert_created=True,

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/tests/test_agentic_conversation.py

@@ -4,13 +4,12 @@ from unittest.mock import MagicMock, patch
 
 from gooddata_eval.core.agentic.conversation import (
     ConversationFixture,
-    ConversationResult,
     TurnDefinition,
     TurnResult,
     _resolve_refs,
     run_agentic_conversation,
 )
-from gooddata_eval.core.models import ChatResult, ToolCallEvent
+from gooddata_eval.core.models import ToolCallEvent
 
 
 def test_turn_definition_model():

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/tests/test_agentic_general_question.py

@@ -3,11 +3,9 @@
 from unittest.mock import MagicMock, patch
 
 from gooddata_eval.core.agentic.general_question import (
-    AgenticGeneralQuestionSummary,
     GeneralQuestionResult,
     run_agentic_general_question,
 )
-from gooddata_eval.core.models import ChatResult
 
 
 def test_general_question_result_fields():

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/tests/test_agentic_guardrail.py

@@ -3,11 +3,9 @@
 from unittest.mock import MagicMock, patch
 
 from gooddata_eval.core.agentic.guardrail import (
-    AgenticGuardrailSummary,
     GuardrailResult,
     run_agentic_guardrail,
 )
-from gooddata_eval.core.models import ChatResult
 
 
 def test_guardrail_result_fields():

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/tests/test_agentic_search_tool.py

@@ -3,8 +3,6 @@
 from unittest.mock import MagicMock, patch
 
 from gooddata_eval.core.agentic.search_tool import (
-    AgenticSearchSummary,
-    SearchResult,
     _tool_correctness,
     _tool_selection,
     run_agentic_search_tool,

{gooddata_eval-1.68.1.dev4 → gooddata_eval-1.69.1.dev1}/tests/test_agentic_visualization.py

@@ -4,14 +4,9 @@
 All tests mock ChatClient so no network is needed.
 """
 
-from dataclasses import dataclass
 from unittest.mock import MagicMock, call, patch
 
-import pytest
-
 from gooddata_eval.core.agentic.visualization import (
-    AgenticRunSummary,
-    RunResult,
     _execute_single_run,
     run_agentic_visualization,
 )
@@ -154,7 +149,8 @@ def test_run_agentic_visualization_uses_initial_conversation_for_run_0():
     # create_conversation should NOT be called for run 0
     instance.create_conversation.assert_not_called()
     instance.send_message.assert_called_once_with("existing-conv", "Show revenue")
-    instance.delete_conversation.assert_called_once_with("existing-conv")
+    # the caller-supplied conversation is left intact; the function only deletes conversations it created
+    instance.delete_conversation.assert_not_called()
     assert len(summary.run_results) == 1
 
 
@@ -176,7 +172,8 @@ def test_run_agentic_visualization_creates_fresh_conversations_for_remaining_run
         )
 
     assert instance.create_conversation.call_count == 1  # only for run 1
-    assert instance.delete_conversation.call_count == 2  # existing-conv + fresh-1
+    # only the self-created fresh-1 is deleted; the caller-supplied existing-conv is left intact
+    instance.delete_conversation.assert_called_once_with("fresh-1")
     assert len(summary.run_results) == 2
 
 
@@ -233,7 +230,7 @@ def test_run_agentic_visualization_creates_conversation_when_no_initial_id():
         instance.create_conversation.side_effect = ["new-0", "new-1"]
         instance.send_message.return_value = _chat_with_viz()
 
-        summary = run_agentic_visualization(
+        run_agentic_visualization(
             host="https://example.com",
             token="tok",
             workspace_id="ws",