gooddata-eval 1.69.0__tar.gz → 1.69.1.dev1__tar.gz

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-eval
-Version: 1.69.0
+Version: 1.69.1.dev1
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.69.0
+Requires-Dist: gooddata-sdk~=1.69.1.dev1
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.69.0"
+version = "1.69.1.dev1"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.69.0",
+    "gooddata-sdk~=1.69.1.dev1",
     "httpx>=0.27,<1.0",
     "orjson>=3.9.15,<4.0.0",
     "pydantic>=2.6,<3.0",
@@ -45,6 +45,7 @@ dev = [
 test = [
     "pytest~=8.3.4",
     "pytest-cov~=6.0.0",
+    "pytest-json-report==1.5.0",
     "pytest-mock>=3.14.0",
 ]
 

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/cli/agentic_runner.py

@@ -6,7 +6,7 @@ from __future__ import annotations
 import time
 from typing import Any, TypedDict
 
-from gooddata_eval.core.agentic._langfuse import HttpxLangfuseClient, make_langfuse_client
+from gooddata_eval.core.agentic._langfuse import make_langfuse_client
 from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill
 from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation
 from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question
@@ -17,17 +17,14 @@ from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualizat
 from gooddata_eval.core.models import CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 
-_LfKw = TypedDict(
-    "_LfKw",
-    {
-        "langfuse": Any,
-        "dataset_item_id": str,
-        "dataset_name": str,
-        "run_timestamp": str,
-        "model_version_override": str | None,
-    },
-    total=False,
-)
+
+class _LfKw(TypedDict, total=False):
+    langfuse: Any
+    dataset_item_id: str
+    dataset_name: str
+    run_timestamp: str
+    model_version_override: str | None
+
 
 AGENTIC_TEST_KINDS = frozenset(
     {

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/cli/main.py

@@ -12,6 +12,7 @@ from gooddata_api_client.exceptions import ApiException
 from rich.console import Console
 from rich.table import Table
 
+from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import RunConfig
 from gooddata_eval.core.connection import ConnectionError_, resolve_connection
@@ -20,7 +21,6 @@ from gooddata_eval.core.langfuse.sink import LangfuseSink
 from gooddata_eval.core.models import ChatResult, DatasetItem
 from gooddata_eval.core.reporting.console import render_comparison, render_console
 from gooddata_eval.core.reporting.json_report import write_multi_model_report
-from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items
 from gooddata_eval.core.runner import ItemReport, run_items
 from gooddata_eval.core.summary.http_client import SummaryClient
 from gooddata_eval.core.workspace import ModelResolutionError, WorkspaceModelController
@@ -98,6 +98,12 @@ def _build_parser() -> argparse.ArgumentParser:
     )
     run.add_argument("--json", dest="json_path", help="Write a JSON report to this path.")
     run.add_argument("--quiet", action="store_true", help="Suppress per-item progress output.")
+    run.add_argument(
+        "--preserve-failed",
+        action="store_true",
+        dest="preserve_failed",
+        help="Keep failed conversations on the server for post-mortem inspection.",
+    )
     run.add_argument(
         "--langfuse",
         action="store_true",
@@ -331,7 +337,12 @@ def _run(config: RunConfig) -> int:
 
             # --- non-agentic items (single-turn, use Evaluator) ---
             backend = _RoutingBackend(
-                ChatClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
+                ChatClient(
+                    host=config.host,
+                    token=config.token,
+                    workspace_id=config.workspace_id,
+                    preserve_failed=config.preserve_failed,
+                ),
                 SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
             )
             try:
@@ -421,6 +432,7 @@ def main(argv: list[str] | None = None) -> int:
             log_to_langfuse=args.langfuse,
             quiet=args.quiet,
             kind=args.kind,
+            preserve_failed=args.preserve_failed,
         )
         return _run(config)
     except (

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/alert_skill.py

@@ -7,12 +7,10 @@ import json
 import os
 import re
 from dataclasses import dataclass
-
 from typing import Any
 
-from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.agentic._catalog import CatalogMetricAlert
-
+from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.models import ToolCallEvent
 
 try:
@@ -80,7 +78,11 @@ def _check_trigger(expected: CatalogMetricAlert, actual_args: dict) -> bool:
     if expected.operator == "ANOMALY":
         return True
     exp_trigger = expected.trigger
-    act_trigger = actual_args.get("trigger", actual_args.get("triggerMode", "ALWAYS"))
+    # A missing OR explicit-null trigger means "unset" -> the product persists the
+    # default ALWAYS ("Every time"). `.get(k, default)` only returns the default when
+    # the key is ABSENT, but create_metric_alert serialises unset params as
+    # `trigger: null`, so chain with `or` to also cover the present-but-None case.
+    act_trigger = actual_args.get("trigger") or actual_args.get("triggerMode") or "ALWAYS"
     if exp_trigger in _ALWAYS_TRIGGER_VALUES:
         return act_trigger in {"ALWAYS", "Every time"}
     act_api = _TRIGGER_DISPLAY_TO_API.get(act_trigger, act_trigger)
@@ -438,7 +440,9 @@ def evaluate_agentic_alert_skill(
     model_version_override: str | None = None,
 ) -> None:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/conversation.py

@@ -386,7 +386,9 @@ def evaluate_agentic_conversation(
     model_version_override: str | None = None,
 ) -> None:
     """Run conversation evaluation, log to Langfuse, and raise on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/general_question.py

@@ -154,7 +154,9 @@ def evaluate_agentic_general_question(
     model_version_override: str | None = None,
 ) -> None:
     """Run general-question evaluation, log to Langfuse, and raise on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/guardrail.py

@@ -151,7 +151,9 @@ def evaluate_agentic_guardrail(
     model_version_override: str | None = None,
 ) -> None:
     """Run guardrail evaluation, log to Langfuse, and raise on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/metric_skill.py

@@ -3,11 +3,10 @@
 
 from __future__ import annotations
 
-from typing import Any
-
 import os
 import re
 from dataclasses import dataclass
+from typing import Any
 
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.models import ToolCallEvent
@@ -247,7 +246,9 @@ def evaluate_agentic_metric_skill(
     model_version_override: str | None = None,
 ) -> None:
     """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/search_tool.py

@@ -145,7 +145,9 @@ def evaluate_agentic_search_tool(
     model_version_override: str | None = None,
 ) -> None:
     """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/agentic/visualization.py

@@ -267,7 +267,9 @@ def evaluate_agentic_visualization(
 ) -> None:
     """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure."""
     import json as _json  # noqa: PLC0415
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/chat/sse_client.py

@@ -106,6 +106,7 @@ class _SseAccumulator:
     call_id_to_event_index: dict[str, int] = field(default_factory=dict)
     reasoning_steps: list[dict[str, Any]] = field(default_factory=list)
     adhoc_viz_args: list[dict[str, Any]] = field(default_factory=list)
+    response_id: str | None = None
 
 
 def _handle_text(content: dict[str, Any], acc: _SseAccumulator) -> None:
@@ -176,7 +177,9 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult:
             "objects": [acc.adhoc_viz_args[-1]],
             "reasoning": "\n".join(acc.viz_reasoning_parts),
         }
-    return ChatResult.model_validate(payload)
+    result = ChatResult.model_validate(payload)
+    result.response_id = acc.response_id
+    return result
 
 
 def parse_sse_lines(lines: Iterable[str]) -> ChatResult:
@@ -204,9 +207,13 @@ def parse_sse_lines(lines: Iterable[str]) -> ChatResult:
             if code in _RETRYABLE_STATUS_CODES:
                 raise TransientChatError(message, status_code=code, detail=detail)
             raise ChatError(message, status_code=code, detail=detail)
+        if event_data.get("responseId") and not acc.response_id:
+            acc.response_id = event_data["responseId"]
         item = event_data.get("item")
         if not item:
             continue
+        if item.get("responseId") and not acc.response_id:
+            acc.response_id = item["responseId"]
         role = item.get("role")
         content: dict[str, Any] = item.get("content") or {}
         ctype = content.get("type")
@@ -227,10 +234,13 @@ def parse_sse_lines(lines: Iterable[str]) -> ChatResult:
 class ChatClient:
     """Single-turn AI chat client over the GoodData AI conversation endpoints."""
 
-    def __init__(self, host: str, token: str, workspace_id: str, *, timeout: float = 300.0):
+    def __init__(
+        self, host: str, token: str, workspace_id: str, *, timeout: float = 300.0, preserve_failed: bool = False
+    ):
         self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations"
         self._auth = {"Authorization": f"Bearer {token}"}
         self._client = httpx.Client(timeout=timeout)
+        self._preserve_failed = preserve_failed
 
     def create_conversation(self) -> str:
         def _do() -> str:
@@ -264,12 +274,29 @@ class ChatClient:
         return _retry_transient(_do, is_retryable=_is_retryable_exc)
 
     def ask(self, item: DatasetItem) -> ChatResult:
-        """Run one single-turn conversation: create, send, parse, clean up."""
+        """Run one conversation: create, send, parse, clean up.
+
+        The conversation_id is attached to the returned ChatResult for tracing.
+        When ``preserve_failed`` is set, failed conversations are kept on the
+        server so they can be inspected after the run; the conversation_id is
+        attached to the raised exception as well.
+        """
         conversation_id = self.create_conversation()
+        success = False
         try:
-            return self.send_message(conversation_id, item.question)
+            result = self.send_message(conversation_id, item.question)
+            result.conversation_id = conversation_id
+            success = True
+            return result
+        except Exception as exc:
+            try:
+                object.__setattr__(exc, "conversation_id", conversation_id)
+            except TypeError:
+                pass  # C-extension exception that rejects __setattr__
+            raise
         finally:
-            self.delete_conversation(conversation_id)
+            if success or not self._preserve_failed:
+                self.delete_conversation(conversation_id)
 
     def close(self) -> None:
         self._client.close()

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/config.py

@@ -19,3 +19,4 @@ class RunConfig:
     log_to_langfuse: bool = False
     quiet: bool = False
     kind: str = "visualization"
+    preserve_failed: bool = False

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/evaluators/alert_skill.py

@@ -78,7 +78,9 @@ class AlertSkillEvaluator:
 
         if "Trigger" in expected:
             expected_trigger = _TRIGGER_MAP.get(expected["Trigger"], expected["Trigger"])
-            trigger_correct = args.get("trigger") == expected_trigger
+            # Omitted/null trigger persists as the product default ALWAYS ("Every time").
+            actual_trigger = args.get("trigger") or "ALWAYS"
+            trigger_correct = actual_trigger == expected_trigger
 
         if "Filters" in expected:
             actual_filters = args.get("filters") or []

gooddata_eval-1.69.1.dev1/src/gooddata_eval/core/evaluators/search_tool.py

@@ -0,0 +1,59 @@
+# (C) 2026 GoodData Corporation
+"""Evaluator for search_tool: agent must call the catalog search with expected parameters."""
+
+from gooddata_eval.core.evaluators.base import ItemEvaluation
+from gooddata_eval.core.models import ChatResult, DatasetItem
+
+
+def _normalize_str_list(value: object, *, lowercase: bool = False) -> list[str]:
+    # Arguments come from raw model-emitted JSON. The search_objects schema
+    # declares keywords/object_types as list[str], but a malformed tool call may
+    # send a non-list or non-string entries. Drop the offending entries defensively
+    # so bad input can't raise (.lower()/sorted() on a non-str) and abort the whole
+    # evaluation run; a non-list collapses to [] and the surviving strings are
+    # still compared normally.
+    if not isinstance(value, list):
+        return []
+    items = [item for item in value if isinstance(item, str)]
+    return sorted(item.lower() if lowercase else item for item in items)
+
+
+def _args_match(actual_args: dict, expected_args: dict) -> bool:
+    # Only keywords and object_types determine semantic correctness.
+    # limit is optional with a server-side default; emit_widget was renamed to
+    # user_requested_search in the tool schema — neither affects search quality.
+    actual_kw = _normalize_str_list(actual_args.get("keywords"), lowercase=True)
+    expected_kw = _normalize_str_list(expected_args.get("keywords"), lowercase=True)
+    if actual_kw != expected_kw:
+        return False
+    # object_types is compared case-sensitively (no lowercase=True): they are
+    # controlled ObjectType StrEnum values the model emits verbatim ("metric",
+    # "dashboard"), so a case mismatch is a genuine error, not a formatting quirk.
+    return _normalize_str_list(actual_args.get("object_types")) == _normalize_str_list(
+        expected_args.get("object_types")
+    )
+
+
+class SearchToolEvaluator:
+    test_kind = "search_tool"
+
+    def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation:
+        expected_call = (item.expected_output or {}).get("tool_call", {})
+        expected_fn = expected_call.get("function_name", "search_objects")
+        expected_args = expected_call.get("function_arguments", {})
+
+        matching_events = [ev for ev in chat_result.tool_call_events if ev.function_name == expected_fn]
+        tool_selection = len(matching_events) > 0
+        tool_correctness = any(_args_match(ev.parsed_arguments(), expected_args) for ev in matching_events)
+
+        # tool_selection is the hard gate; tool_correctness is scored but not blocking
+        return ItemEvaluation(
+            passed=tool_selection,
+            rank_key=(int(tool_selection), int(tool_correctness)),
+            detail={
+                "tool_selection": tool_selection,
+                "tool_correctness": tool_correctness,
+                "expected_function": expected_fn,
+                "calls_found": len(matching_events),
+            },
+        )

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/models.py

@@ -94,6 +94,8 @@ class ChatResult(BaseModel):
     created_visualizations: CreatedVisualizations | None = Field(default=None, alias="createdVisualizations")
     tool_call_events: list[ToolCallEvent] = Field(default_factory=list, alias="toolCallEvents")
     reasoning_step_count: int = Field(default=0, alias="reasoningStepCount")
+    conversation_id: str | None = Field(default=None, alias="conversationId")
+    response_id: str | None = Field(default=None, alias="responseId")
 
 
 class SummaryInput(BaseModel):

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/reporting/json_report.py

@@ -34,6 +34,8 @@ def _build_run_dict(report: EvalReport) -> dict:
                 "latency_s": round(item.latency_s, 3),
                 "avg_latency_s": round(item.avg_latency_s, 3),
                 "detail": item.best_detail,
+                "conversation_id": item.conversation_id,
+                "response_id": item.response_id,
             }
             for item in report.items
         },

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/src/gooddata_eval/core/runner.py

@@ -31,6 +31,8 @@ class ItemReport:
     runs: int = 0
     latency_s: float = 0.0  # total wall-clock across this item's runs
     best_detail: dict = field(default_factory=dict)
+    conversation_id: str | None = None
+    response_id: str | None = None
 
     @property
     def avg_latency_s(self) -> float:
@@ -112,6 +114,8 @@ def _run_one_item(
         for run_index in range(1, runs + 1):
             t0 = time.perf_counter()
             chat_result = backend.ask(item)
+            report.conversation_id = getattr(chat_result, "conversation_id", None) or report.conversation_id
+            report.response_id = getattr(chat_result, "response_id", None) or report.response_id
             evaluation = evaluator.evaluate(item, chat_result)
             latency = time.perf_counter() - t0
             report.runs += 1
@@ -123,7 +127,9 @@ def _run_one_item(
             if on_run_done is not None:
                 on_run_done(run_index, runs, evaluation.passed, latency)
     except Exception as e:  # agent/network/parse failure for this item
-        report.error = f"{type(e).__name__}: {e}"
+        conv_id = getattr(e, "conversation_id", None)
+        report.conversation_id = conv_id or report.conversation_id
+        report.error = f"{type(e).__name__}: {e}" + (f" [conversation_id={conv_id}]" if conv_id else "")
         if best is not None:
             report.best_detail = best.detail
         return report

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/tests/test_agentic_alert_skill.py

@@ -3,10 +3,10 @@
 from unittest.mock import MagicMock, patch
 
 from gooddata_eval.core.agentic.alert_skill import (
-    AgenticAlertSummary,
     AlertEvaluation,
-    AlertRunResult,
+    _check_trigger,
     _deep_subset,
+    _normalize_expected_output,
     _to_number,
     run_agentic_alert_skill,
 )
@@ -33,6 +33,24 @@ def test_deep_subset_missing_key():
     assert _deep_subset({"a": 1, "c": 3}, {"a": 1}) is False
 
 
+def test_check_trigger_missing_or_null_defaults_to_always():
+    # "Every time" is the product default -> the agent may omit the trigger arg
+    # entirely, or serialise it as null. Both must count as ALWAYS, not a mismatch.
+    expected = _normalize_expected_output({"Operator": "GREATER_THAN", "Trigger": "Every time"})
+    assert _check_trigger(expected, {"operator": "GREATER_THAN"}) is True  # key absent
+    assert _check_trigger(expected, {"trigger": None}) is True  # present-but-null (the bug)
+    assert _check_trigger(expected, {"trigger": "ALWAYS"}) is True
+
+
+def test_check_trigger_once_needs_explicit_once():
+    # A "One time" expectation must still require an explicit ONCE - the null-default
+    # fix must not turn a wrong/absent trigger into a pass here.
+    expected = _normalize_expected_output({"Operator": "LESS_THAN", "Trigger": "One time"})
+    assert _check_trigger(expected, {"trigger": "ONCE"}) is True
+    assert _check_trigger(expected, {"trigger": None}) is False  # null != ONCE
+    assert _check_trigger(expected, {"trigger": "ONCE_PER_INTERVAL"}) is False  # real model error stays a fail
+
+
 def test_alert_evaluation_strict_pass():
     ev = AlertEvaluation(
         alert_created=True,

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/tests/test_agentic_conversation.py

@@ -4,13 +4,12 @@ from unittest.mock import MagicMock, patch
 
 from gooddata_eval.core.agentic.conversation import (
     ConversationFixture,
-    ConversationResult,
     TurnDefinition,
     TurnResult,
     _resolve_refs,
     run_agentic_conversation,
 )
-from gooddata_eval.core.models import ChatResult, ToolCallEvent
+from gooddata_eval.core.models import ToolCallEvent
 
 
 def test_turn_definition_model():

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/tests/test_agentic_general_question.py

@@ -3,11 +3,9 @@
 from unittest.mock import MagicMock, patch
 
 from gooddata_eval.core.agentic.general_question import (
-    AgenticGeneralQuestionSummary,
     GeneralQuestionResult,
     run_agentic_general_question,
 )
-from gooddata_eval.core.models import ChatResult
 
 
 def test_general_question_result_fields():

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/tests/test_agentic_guardrail.py

@@ -3,11 +3,9 @@
 from unittest.mock import MagicMock, patch
 
 from gooddata_eval.core.agentic.guardrail import (
-    AgenticGuardrailSummary,
     GuardrailResult,
     run_agentic_guardrail,
 )
-from gooddata_eval.core.models import ChatResult
 
 
 def test_guardrail_result_fields():

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/tests/test_agentic_search_tool.py

@@ -3,8 +3,6 @@
 from unittest.mock import MagicMock, patch
 
 from gooddata_eval.core.agentic.search_tool import (
-    AgenticSearchSummary,
-    SearchResult,
     _tool_correctness,
     _tool_selection,
     run_agentic_search_tool,

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/tests/test_agentic_visualization.py

@@ -4,14 +4,9 @@
 All tests mock ChatClient so no network is needed.
 """
 
-from dataclasses import dataclass
 from unittest.mock import MagicMock, call, patch
 
-import pytest
-
 from gooddata_eval.core.agentic.visualization import (
-    AgenticRunSummary,
-    RunResult,
     _execute_single_run,
     run_agentic_visualization,
 )
@@ -154,7 +149,8 @@ def test_run_agentic_visualization_uses_initial_conversation_for_run_0():
     # create_conversation should NOT be called for run 0
     instance.create_conversation.assert_not_called()
     instance.send_message.assert_called_once_with("existing-conv", "Show revenue")
-    instance.delete_conversation.assert_called_once_with("existing-conv")
+    # the caller-supplied conversation is left intact; the function only deletes conversations it created
+    instance.delete_conversation.assert_not_called()
     assert len(summary.run_results) == 1
 
 
@@ -176,7 +172,8 @@ def test_run_agentic_visualization_creates_fresh_conversations_for_remaining_run
         )
 
     assert instance.create_conversation.call_count == 1  # only for run 1
-    assert instance.delete_conversation.call_count == 2  # existing-conv + fresh-1
+    # only the self-created fresh-1 is deleted; the caller-supplied existing-conv is left intact
+    instance.delete_conversation.assert_called_once_with("fresh-1")
     assert len(summary.run_results) == 2
 
 
@@ -233,7 +230,7 @@ def test_run_agentic_visualization_creates_conversation_when_no_initial_id():
         instance.create_conversation.side_effect = ["new-0", "new-1"]
         instance.send_message.return_value = _chat_with_viz()
 
-        summary = run_agentic_visualization(
+        run_agentic_visualization(
             host="https://example.com",
             token="tok",
             workspace_id="ws",

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/tests/test_alert_skill_evaluator.py

@@ -58,6 +58,23 @@ def test_alert_evaluator_skips_absent_field():
     assert result.passed is True
 
 
+def test_alert_evaluator_defaults_missing_trigger_to_always():
+    # The agent may omit `trigger` (or send null) -> backend default is ALWAYS,
+    # so an "Every time" expectation must still pass instead of a false-negative.
+    expected = {"Operator": "GREATER_THAN", "Threshold": "150", "Trigger": "Every time"}
+    ev = get_evaluator("alert_skill")
+
+    absent = ev.evaluate(_item(expected), _chat_with_alert({"operator": "GREATER_THAN", "threshold": 150}))
+    assert absent.passed is True
+    assert absent.detail["trigger_correct"] is True
+
+    null = ev.evaluate(
+        _item(expected), _chat_with_alert({"operator": "GREATER_THAN", "threshold": 150, "trigger": None})
+    )
+    assert null.passed is True
+    assert null.detail["trigger_correct"] is True
+
+
 def test_alert_evaluator_fails_when_no_tool_call():
     result = get_evaluator("alert_skill").evaluate(
         _item({"Operator": "LESS_THAN"}), ChatResult.model_validate({"textResponse": "here is the alert"})

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/tests/test_cli.py

@@ -1,4 +1,7 @@
 # (C) 2026 GoodData Corporation
+import io
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
 import httpx
 import orjson
 import pytest
@@ -86,7 +89,8 @@ def test_cli_run_end_to_end(monkeypatch, tmp_path, fixtures_dir):
         ]
     )
     assert exit_code == 0
-    assert orjson.loads(out.read_bytes())["runs"]["gpt-5.2"]["summary"]["passed"] == 1
+    # run keys are provider-prefixed (provider_name/model) to stay collision-free across providers
+    assert orjson.loads(out.read_bytes())["runs"]["Test Provider/gpt-5.2"]["summary"]["passed"] == 1
 
 
 def test_cli_operational_error_exits_nonzero(monkeypatch, fixtures_dir):
@@ -405,9 +409,10 @@ def test_cli_multimodel_writes_nested_json(monkeypatch, tmp_path, fixtures_dir):
         ]
     )
     data = orjson.loads(out.read_bytes())
-    assert data["models"] == ["gpt-5.2", "gpt-4o"]
+    # keys are provider-prefixed (provider_name/model); provider_name is "P" here
+    assert data["models"] == ["P/gpt-5.2", "P/gpt-4o"]
     assert "runs" in data and "comparison" in data
-    assert data["comparison"]["gpt-5.2"]["passed"] == 1
+    assert data["comparison"]["P/gpt-5.2"]["passed"] == 1
 
 
 def test_cli_restore_fires_even_when_model_loop_raises(monkeypatch, fixtures_dir):
@@ -512,6 +517,62 @@ def test_cli_rejects_zero_concurrency(monkeypatch, fixtures_dir):
     assert exit_code == 2
 
 
+def test_cli_preserve_failed_flag_parsed(monkeypatch, fixtures_dir):
+    """--preserve-failed sets preserve_failed=True in RunConfig and is passed to ChatClient."""
+    monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok"))
+    captured_kwargs: dict = {}
+
+    class _FakeController:
+        def __init__(self, *a, **k): ...
+        def get_active(self):
+            return ActiveLlmProvider(provider_id="p", default_model_id="gpt-5.2")
+
+        def resolve_and_activate(self, requested, provider=None):
+            return ResolvedModel(provider_id="p", model_id="gpt-5.2", switched=False, provider_name="P")
+
+        def restore(self, original): ...
+        def close(self): ...
+
+    monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController)
+
+    original_chat_client = cli_main.ChatClient
+
+    def _capture_chat_client(**kwargs):
+        captured_kwargs.update(kwargs)
+        return object()
+
+    monkeypatch.setattr(cli_main, "ChatClient", _capture_chat_client)
+
+    def _fake_run(items, backend, *, runs, model, workspace_id, **kw):
+        return EvalReport(
+            model=model,
+            workspace_id=workspace_id,
+            items=[
+                ItemReport(id="i1", dataset_name="d", test_kind="visualization", question="q", pass_at_k=True, runs=1)
+            ],
+        )
+
+    monkeypatch.setattr(cli_main, "run_items", _fake_run)
+
+    exit_code = cli_main.main(
+        [
+            "run",
+            "--host",
+            "https://h",
+            "--token",
+            "tok",
+            "--workspace",
+            "ws1",
+            "--dataset",
+            str(fixtures_dir / "sample_dataset"),
+            "--preserve-failed",
+            "--quiet",
+        ]
+    )
+    assert exit_code == 0
+    assert captured_kwargs.get("preserve_failed") is True
+
+
 def test_cli_rejects_negative_concurrency(monkeypatch, fixtures_dir):
     monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok"))
     exit_code = cli_main.main(
@@ -534,10 +595,6 @@ def test_cli_rejects_negative_concurrency(monkeypatch, fixtures_dir):
 
 def test_progress_callbacks_thread_safe():
     """Verify progress callbacks can be called from multiple threads without error."""
-    import io
-    import threading
-    from concurrent.futures import ThreadPoolExecutor, as_completed
-
     console = Console(file=io.StringIO(), force_terminal=False)
     on_item_start, on_run_done, on_item_done = cli_main._make_progress_callbacks(console)
 
@@ -554,7 +611,9 @@ def test_progress_callbacks_thread_safe():
             )
             on_item_start(index, 100, item)
             on_run_done(index, 100, 1, 1, index % 2 == 0, 1.5)
-            report = ItemReport(id=f"test-{index}", dataset_name="test", test_kind="general_question")
+            report = ItemReport(
+                id=f"test-{index}", dataset_name="test", test_kind="general_question", question=f"Question {index}"
+            )
             report.runs = 1
             report.latency_s = 1.5
             report.pass_at_k = index % 2 == 0

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/tests/test_langfuse_source.py

@@ -19,7 +19,8 @@ def test_item_from_raw_dict_input():
     item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization")
     assert item.id == "lf-1"
     assert item.question == "Show revenue"
-    assert item.test_kind == "visualization"
+    # expected_output carries a "visualization" key, so _infer_test_kind classifies it as production agentic vis
+    assert item.test_kind == "vis_agentic"
     assert item.dataset_name == "ds"
 
 

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/tests/test_runner.py

@@ -233,6 +233,31 @@ def test_run_items_concurrency_callbacks_fire_for_all_items():
     assert sorted(done_ids) == [f"c{i}" for i in range(5)]
 
 
+def test_run_items_error_report_includes_conversation_id():
+    """When an exception has a conversation_id, it appears in the error report."""
+
+    class _ConvIdBackend:
+        def ask(self, item: DatasetItem) -> ChatResult:
+            exc = RuntimeError("agent error")
+            exc.conversation_id = "conv-xyz"  # type: ignore[attr-defined]
+            raise exc
+
+    report = run_items([_item()], _ConvIdBackend(), runs=1)
+    assert report.errored == 1
+    assert "conversation_id=conv-xyz" in report.items[0].error
+
+
+def test_run_items_error_report_omits_conversation_id_when_absent():
+    """When no conversation_id is set, the error string has no bracket suffix."""
+
+    class _PlainBackend:
+        def ask(self, item: DatasetItem) -> ChatResult:
+            raise RuntimeError("plain error")
+
+    report = run_items([_item()], _PlainBackend(), runs=1)
+    assert "conversation_id" not in report.items[0].error
+
+
 def test_run_items_callback_exception_is_logged_not_swallowed(capsys):
     """A raising callback prints a traceback to stderr but the run continues."""
     backend = _FakeBackend([_chat_with(_viz_obj())] * 2)

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev1}/tests/test_sse_client.py

@@ -5,6 +5,7 @@ import httpx
 import pytest
 from gooddata_eval.core.chat import sse_client as sse_mod
 from gooddata_eval.core.chat.sse_client import ChatClient, ChatError, TransientChatError, parse_sse_lines
+from gooddata_eval.core.models import DatasetItem
 
 
 def test_parse_sse_lines_collects_text_and_visualization(fixtures_dir):
@@ -222,3 +223,120 @@ def test_float_env_uses_default_when_unset(monkeypatch):
 def test_float_env_reads_override(monkeypatch):
     monkeypatch.setenv("GD_TEST_FLOAT", "1.5")
     assert sse_mod._float_env("GD_TEST_FLOAT", 5.0) == 1.5
+
+
+def test_parse_sse_lines_captures_response_id_from_event_data():
+    """response_id is captured from the top-level event_data."""
+    lines = [
+        'data: {"responseId": "resp-123", "item": {"role": "assistant", "content": {"type": "text", "text": "hi"}}}',
+    ]
+    result = parse_sse_lines(lines)
+    assert result.response_id == "resp-123"
+
+
+def test_parse_sse_lines_captures_response_id_from_item():
+    """response_id is captured from item when not present at top level."""
+    lines = [
+        'data: {"item": {"role": "assistant", "responseId": "resp-456", "content": {"type": "text", "text": "hi"}}}',
+    ]
+    result = parse_sse_lines(lines)
+    assert result.response_id == "resp-456"
+
+
+def test_parse_sse_lines_first_response_id_wins():
+    """Only the first responseId encountered is kept."""
+    lines = [
+        'data: {"responseId": "first", "item": {"role": "assistant", "content": {"type": "text", "text": "a"}}}',
+        'data: {"responseId": "second", "item": {"role": "assistant", "content": {"type": "text", "text": "b"}}}',
+    ]
+    result = parse_sse_lines(lines)
+    assert result.response_id == "first"
+
+
+def test_ask_attaches_conversation_id_to_result(monkeypatch):
+    """ChatClient.ask() sets conversation_id on the returned ChatResult."""
+    calls = []
+
+    def handler(request):
+        if request.method == "POST" and request.url.path.endswith("/conversations"):
+            return httpx.Response(200, json={"conversationId": "conv-abc"})
+        if request.method == "POST" and "messages" in str(request.url):
+            return httpx.Response(200, content=_OK_SSE)
+        if request.method == "DELETE":
+            calls.append("delete")
+            return httpx.Response(204)
+        return httpx.Response(404)
+
+    client = _client_with_handler(handler)
+    item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={})
+    result = client.ask(item)
+    assert result.conversation_id == "conv-abc"
+    assert "delete" in calls  # conversation cleaned up on success
+
+
+def test_ask_preserve_failed_keeps_conversation_on_error(monkeypatch):
+    """With preserve_failed=True, failed conversations are not deleted."""
+    calls = []
+
+    def handler(request):
+        if request.method == "POST" and request.url.path.endswith("/conversations"):
+            return httpx.Response(200, json={"conversationId": "conv-fail"})
+        if request.method == "POST" and "messages" in str(request.url):
+            return httpx.Response(200, content=_NONRETRY_SSE)
+        if request.method == "DELETE":
+            calls.append("delete")
+            return httpx.Response(204)
+        return httpx.Response(404)
+
+    monkeypatch.setattr(sse_mod.time, "sleep", lambda s: None)
+    client = ChatClient(host="https://example.invalid", token="t", workspace_id="w", preserve_failed=True)
+    client._client = httpx.Client(transport=httpx.MockTransport(handler))
+
+    item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={})
+    with pytest.raises(ChatError):
+        client.ask(item)
+    assert "delete" not in calls  # conversation preserved
+
+
+def test_ask_without_preserve_failed_deletes_on_error(monkeypatch):
+    """Without preserve_failed, conversations are deleted even on error."""
+    calls = []
+
+    def handler(request):
+        if request.method == "POST" and request.url.path.endswith("/conversations"):
+            return httpx.Response(200, json={"conversationId": "conv-del"})
+        if request.method == "POST" and "messages" in str(request.url):
+            return httpx.Response(200, content=_NONRETRY_SSE)
+        if request.method == "DELETE":
+            calls.append("delete")
+            return httpx.Response(204)
+        return httpx.Response(404)
+
+    monkeypatch.setattr(sse_mod.time, "sleep", lambda s: None)
+    client = _client_with_handler(handler)
+
+    item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={})
+    with pytest.raises(ChatError):
+        client.ask(item)
+    assert "delete" in calls  # conversation deleted
+
+
+def test_ask_attaches_conversation_id_to_exception(monkeypatch):
+    """On failure, conversation_id is attached to the raised exception."""
+
+    def handler(request):
+        if request.method == "POST" and request.url.path.endswith("/conversations"):
+            return httpx.Response(200, json={"conversationId": "conv-exc"})
+        if request.method == "POST" and "messages" in str(request.url):
+            return httpx.Response(200, content=_NONRETRY_SSE)
+        if request.method == "DELETE":
+            return httpx.Response(204)
+        return httpx.Response(404)
+
+    monkeypatch.setattr(sse_mod.time, "sleep", lambda s: None)
+    client = _client_with_handler(handler)
+
+    item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={})
+    with pytest.raises(ChatError) as ei:
+        client.ask(item)
+    assert ei.value.conversation_id == "conv-exc"

gooddata_eval-1.69.1.dev1/tox.ini

@@ -0,0 +1,16 @@
+# (C) 2026 GoodData Corporation
+[tox]
+envlist = py3{10,11,12,13,14}
+
+[testenv]
+runner = uv-venv-lock-runner
+package = wheel
+wheel_build_env = .pkg
+extras =
+    llm-judge
+dependency_groups =
+    test
+setenv =
+    COVERAGE_CORE=sysmon
+commands =
+    pytest -v --cov --cov-report=xml tests {posargs} --json-report --json-report-file=.json-report-{envname}.json

gooddata_eval-1.69.0/src/gooddata_eval/core/evaluators/search_tool.py

@@ -1,40 +0,0 @@
-# (C) 2026 GoodData Corporation
-"""Evaluator for search_tool: agent must call the catalog search with expected parameters."""
-
-from gooddata_eval.core.evaluators.base import ItemEvaluation
-from gooddata_eval.core.models import ChatResult, DatasetItem
-
-
-def _args_match(actual_args: dict, expected_args: dict) -> bool:
-    if sorted(actual_args.get("keywords") or []) != sorted(expected_args.get("keywords") or []):
-        return False
-    if sorted(actual_args.get("object_types") or []) != sorted(expected_args.get("object_types") or []):
-        return False
-    if actual_args.get("limit") != expected_args.get("limit"):
-        return False
-    return actual_args.get("emit_widget") == expected_args.get("emit_widget")
-
-
-class SearchToolEvaluator:
-    test_kind = "search_tool"
-
-    def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation:
-        expected_call = (item.expected_output or {}).get("tool_call", {})
-        expected_fn = expected_call.get("function_name", "search_objects")
-        expected_args = expected_call.get("function_arguments", {})
-
-        matching_events = [ev for ev in chat_result.tool_call_events if ev.function_name == expected_fn]
-        tool_selection = len(matching_events) > 0
-        tool_correctness = any(_args_match(ev.parsed_arguments(), expected_args) for ev in matching_events)
-
-        # tool_selection is the hard gate; tool_correctness is scored but not blocking
-        return ItemEvaluation(
-            passed=tool_selection,
-            rank_key=(int(tool_selection), int(tool_correctness)),
-            detail={
-                "tool_selection": tool_selection,
-                "tool_correctness": tool_correctness,
-                "expected_function": expected_fn,
-                "calls_found": len(matching_events),
-            },
-        )