gooddata-eval 1.69.0__tar.gz → 1.69.1.dev2__tar.gz

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-eval
-Version: 1.69.0
+Version: 1.69.1.dev2
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.69.0
+Requires-Dist: gooddata-sdk~=1.69.1.dev2
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev2}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.69.0"
+version = "1.69.1.dev2"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.69.0",
+    "gooddata-sdk~=1.69.1.dev2",
     "httpx>=0.27,<1.0",
     "orjson>=3.9.15,<4.0.0",
     "pydantic>=2.6,<3.0",
@@ -45,6 +45,7 @@ dev = [
 test = [
     "pytest~=8.3.4",
     "pytest-cov~=6.0.0",
+    "pytest-json-report==1.5.0",
     "pytest-mock>=3.14.0",
 ]
 

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev2}/src/gooddata_eval/cli/agentic_runner.py

@@ -6,7 +6,7 @@ from __future__ import annotations
 import time
 from typing import Any, TypedDict
 
-from gooddata_eval.core.agentic._langfuse import HttpxLangfuseClient, make_langfuse_client
+from gooddata_eval.core.agentic._langfuse import make_langfuse_client
 from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill
 from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation
 from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question
@@ -17,17 +17,14 @@ from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualizat
 from gooddata_eval.core.models import CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 
-_LfKw = TypedDict(
-    "_LfKw",
-    {
-        "langfuse": Any,
-        "dataset_item_id": str,
-        "dataset_name": str,
-        "run_timestamp": str,
-        "model_version_override": str | None,
-    },
-    total=False,
-)
+
+class _LfKw(TypedDict, total=False):
+    langfuse: Any
+    dataset_item_id: str
+    dataset_name: str
+    run_timestamp: str
+    model_version_override: str | None
+
 
 AGENTIC_TEST_KINDS = frozenset(
     {

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev2}/src/gooddata_eval/cli/main.py

@@ -12,6 +12,7 @@ from gooddata_api_client.exceptions import ApiException
 from rich.console import Console
 from rich.table import Table
 
+from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import RunConfig
 from gooddata_eval.core.connection import ConnectionError_, resolve_connection
@@ -20,7 +21,6 @@ from gooddata_eval.core.langfuse.sink import LangfuseSink
 from gooddata_eval.core.models import ChatResult, DatasetItem
 from gooddata_eval.core.reporting.console import render_comparison, render_console
 from gooddata_eval.core.reporting.json_report import write_multi_model_report
-from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items
 from gooddata_eval.core.runner import ItemReport, run_items
 from gooddata_eval.core.summary.http_client import SummaryClient
 from gooddata_eval.core.workspace import ModelResolutionError, WorkspaceModelController
@@ -98,6 +98,12 @@ def _build_parser() -> argparse.ArgumentParser:
     )
     run.add_argument("--json", dest="json_path", help="Write a JSON report to this path.")
     run.add_argument("--quiet", action="store_true", help="Suppress per-item progress output.")
+    run.add_argument(
+        "--preserve-failed",
+        action="store_true",
+        dest="preserve_failed",
+        help="Keep failed conversations on the server for post-mortem inspection.",
+    )
     run.add_argument(
         "--langfuse",
         action="store_true",
@@ -331,7 +337,12 @@ def _run(config: RunConfig) -> int:
 
             # --- non-agentic items (single-turn, use Evaluator) ---
             backend = _RoutingBackend(
-                ChatClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
+                ChatClient(
+                    host=config.host,
+                    token=config.token,
+                    workspace_id=config.workspace_id,
+                    preserve_failed=config.preserve_failed,
+                ),
                 SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
             )
             try:
@@ -421,6 +432,7 @@ def main(argv: list[str] | None = None) -> int:
             log_to_langfuse=args.langfuse,
             quiet=args.quiet,
             kind=args.kind,
+            preserve_failed=args.preserve_failed,
         )
         return _run(config)
     except (

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev2}/src/gooddata_eval/core/agentic/_langfuse.py

@@ -305,7 +305,7 @@ def observe(
                 run_name=run_name,
                 dataset_item_id=dataset_item_id,
                 trace_id=trace_id,
-                metadata=run_metadata or {"testing_framework": "tavern-e2e"},
+                metadata=run_metadata or {},
                 run_description="",
             )
             _log.debug(
@@ -383,14 +383,31 @@ def build_run_context(
     dataset_name: str,
     run_timestamp: str | None,
     model_version_override: str | None,
+    run_metadata_extra: dict[str, Any] | None = None,
 ) -> tuple[str, dict[str, Any]]:
-    """Return (run_name_base, run_metadata) with model version resolved from workspace API."""
+    """Return (run_name_base, run_metadata) with model version resolved from workspace API.
+
+    Args:
+        host: GoodData host URL.
+        token: API token used to resolve the workspace model version.
+        workspace_id: Workspace whose active LLM provider is read when no override is given.
+        dataset_name: Langfuse dataset name; used as the run-name prefix.
+        run_timestamp: Shared run timestamp for the run name (falls back to the current time).
+        model_version_override: Explicit model-version tag; when set it wins over the
+            workspace lookup.
+        run_metadata_extra: Optional extra key/values added to the dataset-run metadata
+            (e.g. a testing-framework tag or a CI run id for scoping). Default None keeps
+            behavior unchanged. The SDK-derived model_version is applied last and cannot
+            be overwritten by this dict.
+    """
     model = get_model_version(host, token, workspace_id, model_version_override)
     ts = run_timestamp or datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
     base = f"{dataset_name}_{ts}"
     if model:
         base = f"{base}_{model}"
-    metadata: dict[str, Any] = {"testing_framework": "tavern-e2e"}
+    # Caller supplies its own run tags (e.g. testing_framework); model_version is applied
+    # last so the SDK-derived value cannot be overwritten by run_metadata_extra.
+    metadata: dict[str, Any] = dict(run_metadata_extra) if run_metadata_extra else {}
     if model:
         metadata["model_version"] = model
     return base, metadata

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev2}/src/gooddata_eval/core/agentic/alert_skill.py

@@ -7,12 +7,12 @@ import json
 import os
 import re
 from dataclasses import dataclass
-
 from typing import Any
 
-from gooddata_eval.core.chat.sse_client import ChatClient
-from gooddata_eval.core.agentic._catalog import CatalogMetricAlert
+from gooddata_sdk import GoodDataSdk
 
+from gooddata_eval.core.agentic._catalog import CatalogMetricAlert
+from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.models import ToolCallEvent
 
 try:
@@ -80,7 +80,11 @@ def _check_trigger(expected: CatalogMetricAlert, actual_args: dict) -> bool:
     if expected.operator == "ANOMALY":
         return True
     exp_trigger = expected.trigger
-    act_trigger = actual_args.get("trigger", actual_args.get("triggerMode", "ALWAYS"))
+    # A missing OR explicit-null trigger means "unset" -> the product persists the
+    # default ALWAYS ("Every time"). `.get(k, default)` only returns the default when
+    # the key is ABSENT, but create_metric_alert serialises unset params as
+    # `trigger: null`, so chain with `or` to also cover the present-but-None case.
+    act_trigger = actual_args.get("trigger") or actual_args.get("triggerMode") or "ALWAYS"
     if exp_trigger in _ALWAYS_TRIGGER_VALUES:
         return act_trigger in {"ALWAYS", "Every time"}
     act_api = _TRIGGER_DISPLAY_TO_API.get(act_trigger, act_trigger)
@@ -181,11 +185,14 @@ def generate_simulated_alert_response(
     return response.choices[0].message.content or ""
 
 
-def _delete_alert(client: ChatClient, workspace_id: str, alert_id: str) -> None:
-    host = str(client._base).split("/api/")[0]
-    url = f"{host}/api/v1/entities/workspaces/{workspace_id}/automations/{alert_id}"
+def _delete_alert(sdk: GoodDataSdk, workspace_id: str, alert_id: str) -> None:
+    """Best-effort delete of an alert (automation) created during evaluation.
+
+    Uses the GoodData SDK entities API rather than reimplementing the REST call.
+    Failures are logged, not raised.
+    """
     try:
-        client._client.delete(url, headers=client._auth)
+        sdk._client.entities_api.delete_entity_automations(workspace_id, alert_id)
     except Exception as exc:
         print(f"[CLEANUP] Failed to delete alert {alert_id}: {exc}")
 
@@ -325,6 +332,7 @@ def run_agentic_alert_skill(
     expected = _normalize_expected_output(expected_output)
     run_results: list[AlertRunResult] = []
     client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    sdk = GoodDataSdk.create(host, token)
 
     def _run_once(conv_id: str) -> AlertRunResult:
         alert_id_to_delete: str | None = None
@@ -373,7 +381,7 @@ def run_agentic_alert_skill(
             )
         finally:
             if alert_id_to_delete:
-                _delete_alert(client, workspace_id, alert_id_to_delete)
+                _delete_alert(sdk, workspace_id, alert_id_to_delete)
 
     try:
         conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
@@ -436,9 +444,12 @@ def evaluate_agentic_alert_skill(
     dataset_name: str = "alert_skill",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:
@@ -471,6 +482,7 @@ def evaluate_agentic_alert_skill(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev2}/src/gooddata_eval/core/agentic/conversation.py

@@ -8,8 +8,10 @@ import re
 from dataclasses import dataclass
 from typing import Literal
 
+from gooddata_sdk import GoodDataSdk
 from pydantic import BaseModel
 
+from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.models import ChatResult, ToolCallEvent
 from gooddata_eval.core.scoring import (
@@ -283,11 +285,16 @@ def run_agentic_conversation(
     replies before the agent produces the expected output.
     """
     client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    sdk = GoodDataSdk.create(host, token)
     turn_results: list[TurnResult] = []
     turn_outputs: dict[str, dict] = {}
     total_clarification_turns = 0
     conversation_id: str = ""
     owns_conversation = False
+    # Metrics created during this conversation, deleted after it completes so they do
+    # not persist in the (shared) workspace and get reused by a later test. Deferred to
+    # the end — a later turn may $ref a metric an earlier turn created.
+    created_metric_ids: list[str] = []
 
     try:
         if initial_conversation_id is not None:
@@ -335,6 +342,11 @@ def run_agentic_conversation(
                 if metric_data:
                     turn_outputs[turn.turn_id] = metric_data
 
+            # Track every metric created this turn (any turn may create one) for cleanup.
+            for metric_id in _extract_created_metric_ids(all_tool_calls):
+                if metric_id not in created_metric_ids:
+                    created_metric_ids.append(metric_id)
+
             turn_results.append(
                 TurnResult(
                     turn_id=turn.turn_id,
@@ -351,6 +363,8 @@ def run_agentic_conversation(
     finally:
         if owns_conversation and conversation_id:
             client.delete_conversation(conversation_id)
+        for metric_id in created_metric_ids:
+            _delete_metric(sdk, workspace_id, metric_id)
         client.close()
 
     activated_all = {skill for tr in turn_results for skill in tr.activated_skills}
@@ -384,9 +398,12 @@ def evaluate_agentic_conversation(
     dataset_name: str = "conversation",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run conversation evaluation, log to Langfuse, and raise on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:
@@ -417,6 +434,7 @@ def evaluate_agentic_conversation(
             dataset_name or fixture.dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev2}/src/gooddata_eval/core/agentic/general_question.py

@@ -152,9 +152,12 @@ def evaluate_agentic_general_question(
     dataset_name: str = "general_question",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run general-question evaluation, log to Langfuse, and raise on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:
@@ -186,6 +189,7 @@ def evaluate_agentic_general_question(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev2}/src/gooddata_eval/core/agentic/guardrail.py

@@ -149,9 +149,12 @@ def evaluate_agentic_guardrail(
     dataset_name: str = "guardrail",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run guardrail evaluation, log to Langfuse, and raise on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:
@@ -183,6 +186,7 @@ def evaluate_agentic_guardrail(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev2}/src/gooddata_eval/core/agentic/metric_skill.py

@@ -3,11 +3,12 @@
 
 from __future__ import annotations
 
-from typing import Any
-
 import os
 import re
 from dataclasses import dataclass
+from typing import Any
+
+from gooddata_sdk import GoodDataSdk
 
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.models import ToolCallEvent
@@ -128,6 +129,41 @@ def _extract_metric_result(tool_call_events: list[ToolCallEvent]) -> dict | None
     return None
 
 
+def _extract_created_metric_ids(tool_call_events: list[ToolCallEvent]) -> list[str]:
+    """Ids of every metric created by ``create_metric`` calls (a turn may create more than one).
+
+    Used for cleanup so no created metric leaks — unlike ``_extract_metric_result``, which
+    returns only the first result for MAQL evaluation. Shared with conversation evaluation.
+    """
+    metric_ids: list[str] = []
+    for tc in tool_call_events:
+        if tc.function_name != "create_metric" or not tc.result:
+            continue
+        result_data = tc.parsed_result()
+        if not result_data:
+            continue
+        data = result_data.get("data", result_data)
+        metric_id = data.get("metric_id") if isinstance(data, dict) else None
+        if metric_id and metric_id not in metric_ids:
+            metric_ids.append(metric_id)
+    return metric_ids
+
+
+def _delete_metric(sdk: GoodDataSdk, workspace_id: str, metric_id: str) -> None:
+    """Delete a metric created during evaluation.
+
+    Eval runs share a persistent workspace, so a metric left behind is picked up by
+    a later test — the agent reuses it (returning ``SELECT {id}`` instead of full
+    MAQL) and the assertion fails. Deleting the created metric on the way out keeps
+    the workspace clean for the next run. Best-effort: failures are logged, not raised.
+    Mirrors ``alert_skill._delete_alert``.
+    """
+    try:
+        sdk._client.entities_api.delete_entity_metrics(workspace_id, metric_id)
+    except Exception as exc:
+        print(f"[CLEANUP] Failed to delete metric {metric_id}: {exc}")
+
+
 def _is_asking_clarification(text: str) -> bool:
     if not text:
         return False
@@ -137,41 +173,54 @@ def _is_asking_clarification(text: str) -> bool:
 
 def _execute_single_metric_run(
     client: ChatClient,
+    sdk: GoodDataSdk,
+    workspace_id: str,
     conversation_id: str,
     question: str,
     expected_outputs: list[dict],
     max_iterations: int,
 ) -> MetricRunResult:
-    """Drive one full multi-turn metric-skill conversation and evaluate the result."""
+    """Drive one full multi-turn metric-skill conversation and evaluate the result.
+
+    Any metric the agent creates during this run is deleted on the way out (see
+    ``_delete_metric``) so it cannot leak into — and be reused by — a later test
+    sharing the workspace.
+    """
     primary_expected = expected_outputs[0] if expected_outputs else {}
     metric_result: dict | None = None
+    metric_id_to_delete: str | None = None
     turns = 0
     current_question = question
 
-    for _iteration in range(max_iterations):
-        turns += 1
-        chat_result = client.send_message(conversation_id, current_question)
-        candidate = _extract_metric_result(chat_result.tool_call_events or [])
-        if candidate is not None:
-            metric_result = candidate
-            break
-        response_text = (chat_result.text_response or "").strip()
-        if _is_asking_clarification(response_text):
-            current_question = generate_simulated_response(response_text, primary_expected)
-        else:
-            break
-
-    actual_maql = (metric_result or {}).get("maql", "")
-    metric_created = metric_result is not None
-    maql_correct, _ = _best_maql_match(actual_maql, expected_outputs) if metric_created else (False, "")
-    return MetricRunResult(
-        conversation_id=conversation_id,
-        metric_result=metric_result,
-        metric_created=metric_created,
-        actual_maql=actual_maql,
-        maql_correct=maql_correct,
-        total_turns=float(turns),
-    )
+    try:
+        for _iteration in range(max_iterations):
+            turns += 1
+            chat_result = client.send_message(conversation_id, current_question)
+            candidate = _extract_metric_result(chat_result.tool_call_events or [])
+            if candidate is not None:
+                metric_result = candidate
+                metric_id_to_delete = candidate.get("metric_id")
+                break
+            response_text = (chat_result.text_response or "").strip()
+            if _is_asking_clarification(response_text):
+                current_question = generate_simulated_response(response_text, primary_expected)
+            else:
+                break
+
+        actual_maql = (metric_result or {}).get("maql", "")
+        metric_created = metric_result is not None
+        maql_correct, _ = _best_maql_match(actual_maql, expected_outputs) if metric_created else (False, "")
+        return MetricRunResult(
+            conversation_id=conversation_id,
+            metric_result=metric_result,
+            metric_created=metric_created,
+            actual_maql=actual_maql,
+            maql_correct=maql_correct,
+            total_turns=float(turns),
+        )
+    finally:
+        if metric_id_to_delete:
+            _delete_metric(sdk, workspace_id, metric_id_to_delete)
 
 
 def run_agentic_metric_skill(
@@ -192,12 +241,15 @@ def run_agentic_metric_skill(
     expected_outputs: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output]
     run_results: list[MetricRunResult] = []
     client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    sdk = GoodDataSdk.create(host, token)
 
     try:
         conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
         try:
             run_results.append(
-                _execute_single_metric_run(client, conv_id_0, question, expected_outputs, max_iterations)
+                _execute_single_metric_run(
+                    client, sdk, workspace_id, conv_id_0, question, expected_outputs, max_iterations
+                )
             )
         finally:
             if initial_conversation_id is None:  # only delete conversations we created
@@ -207,7 +259,9 @@ def run_agentic_metric_skill(
             conv_id = client.create_conversation()
             try:
                 run_results.append(
-                    _execute_single_metric_run(client, conv_id, question, expected_outputs, max_iterations)
+                    _execute_single_metric_run(
+                        client, sdk, workspace_id, conv_id, question, expected_outputs, max_iterations
+                    )
                 )
             finally:
                 client.delete_conversation(conv_id)
@@ -245,9 +299,12 @@ def evaluate_agentic_metric_skill(
     dataset_name: str = "metric_skill",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:
@@ -280,6 +337,7 @@ def evaluate_agentic_metric_skill(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev2}/src/gooddata_eval/core/agentic/search_tool.py

@@ -143,9 +143,12 @@ def evaluate_agentic_search_tool(
     dataset_name: str = "search",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure."""
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:
@@ -177,6 +180,7 @@ def evaluate_agentic_search_tool(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.69.0 → gooddata_eval-1.69.1.dev2}/src/gooddata_eval/core/agentic/visualization.py

@@ -263,11 +263,14 @@ def evaluate_agentic_visualization(
     dataset_name: str = "visualization",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
     record_output_path: str | None = None,
 ) -> None:
     """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure."""
     import json as _json  # noqa: PLC0415
-    from datetime import datetime as _dt, timezone as _tz  # noqa: PLC0415
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
     from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
 
     if langfuse is None:
@@ -300,6 +303,7 @@ def evaluate_agentic_visualization(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         K = len(summary.run_results)
         traces_by_conv = find_traces_per_conversation(