gooddata-eval 1.71.0__tar.gz → 1.71.1.dev2__tar.gz

{gooddata_eval-1.71.0 → gooddata_eval-1.71.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-eval
-Version: 1.71.0
+Version: 1.71.1.dev2
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.71.0
+Requires-Dist: gooddata-sdk~=1.71.1.dev2
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6
@@ -120,6 +120,7 @@ Both provider name and provider id are accepted as the prefix.
 |---|---|---|
 | `--runs K` | `2` | Independent runs per item (pass@K). An item passes if any run passes. |
 | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests. Progress output interleaves when K > 1. |
+| `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` items go through the summary endpoint, which has no such option. |
 
 #### Output
 
@@ -132,7 +133,7 @@ Both provider name and provider id are accepted as the prefix.
 
 | Flag | Description |
 |---|---|
-| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. |
+| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`, suffixed `-effort-{level}` when `--reasoning-effort` is set so runs differing only by effort stay separate). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. |
 
 ### JSON report shape
 

{gooddata_eval-1.71.0 → gooddata_eval-1.71.1.dev2}/README.md

@@ -92,6 +92,7 @@ Both provider name and provider id are accepted as the prefix.
 |---|---|---|
 | `--runs K` | `2` | Independent runs per item (pass@K). An item passes if any run passes. |
 | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests. Progress output interleaves when K > 1. |
+| `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` items go through the summary endpoint, which has no such option. |
 
 #### Output
 
@@ -104,7 +105,7 @@ Both provider name and provider id are accepted as the prefix.
 
 | Flag | Description |
 |---|---|
-| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. |
+| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`, suffixed `-effort-{level}` when `--reasoning-effort` is set so runs differing only by effort stay separate). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. |
 
 ### JSON report shape
 

{gooddata_eval-1.71.0 → gooddata_eval-1.71.1.dev2}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.71.0"
+version = "1.71.1.dev2"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.71.0",
+    "gooddata-sdk~=1.71.1.dev2",
     "httpx>=0.27,<1.0",
     "orjson>=3.9.15,<4.0.0",
     "pydantic>=2.6,<3.0",

{gooddata_eval-1.71.0 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/cli/agentic_runner.py

@@ -14,6 +14,7 @@ from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail
 from gooddata_eval.core.agentic.metric_skill import evaluate_agentic_metric_skill
 from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 
@@ -24,6 +25,7 @@ class _LfKw(TypedDict, total=False):
     dataset_name: str
     run_timestamp: str
     model_version_override: str | None
+    reasoning_effort: ReasoningEffort | None
 
 
 AGENTIC_TEST_KINDS = frozenset(
@@ -80,6 +82,7 @@ def _dispatch_agentic(
     langfuse: Any,
     run_ts: str,
     model_version_override: str | None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Call the appropriate evaluate_agentic_* function for the item's test_kind."""
     kind = item.test_kind
@@ -90,6 +93,7 @@ def _dispatch_agentic(
         "dataset_name": item.dataset_name,
         "run_timestamp": run_ts,
         "model_version_override": model_version_override,
+        "reasoning_effort": reasoning_effort,
     }
 
     if kind in ("vis_agentic", "agentic_visualization"):
@@ -176,6 +180,7 @@ def run_agentic_items(
     *,
     k: int = 2,
     model_version: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
     use_langfuse: bool = False,
     run_ts: str,
     on_item_start: Any = None,
@@ -202,7 +207,7 @@ def run_agentic_items(
         )
         t0 = time.perf_counter()
         try:
-            _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version)
+            _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort)
             item_report.pass_at_k = True
             item_report.runs = k
         except AssertionError as exc:

{gooddata_eval-1.71.0 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/cli/main.py

@@ -6,6 +6,7 @@ import sys
 import threading
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import get_args
 
 import httpx
 from gooddata_api_client.exceptions import ApiException
@@ -14,7 +15,7 @@ from rich.table import Table
 
 from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items
 from gooddata_eval.core.chat.sse_client import ChatClient
-from gooddata_eval.core.config import RunConfig
+from gooddata_eval.core.config import ReasoningEffort, RunConfig
 from gooddata_eval.core.connection import ConnectionError_, resolve_connection
 from gooddata_eval.core.dataset.local import load_local_dataset
 from gooddata_eval.core.langfuse.sink import LangfuseSink
@@ -104,6 +105,13 @@ def _build_parser() -> argparse.ArgumentParser:
         dest="preserve_failed",
         help="Keep failed conversations on the server for post-mortem inspection.",
     )
+    run.add_argument(
+        "--reasoning-effort",
+        dest="reasoning_effort",
+        choices=list(get_args(ReasoningEffort)),
+        help="Reasoning effort requested per message. Requires the enableGenAiReasoningEffort "
+        "feature flag on the target organization; without it the server ignores the value.",
+    )
     run.add_argument(
         "--langfuse",
         action="store_true",
@@ -292,6 +300,10 @@ def _run(config: RunConfig) -> int:
                     progress_console.print(f"Provider={provider_display}, model={resolved.model_id}{switched}")
 
             run_name = f"gd-eval-{run_ts}-{resolved.model_id}"
+            if config.reasoning_effort:
+                # Without this two runs differing only by effort share a name and are
+                # indistinguishable in the report, which is the comparison this exists for.
+                run_name = f"{run_name}-effort-{config.reasoning_effort.lower()}"
             if progress_console and config.log_to_langfuse:
                 progress_console.print(f"Logging to Langfuse run '{run_name}'...")
 
@@ -307,6 +319,7 @@ def _run(config: RunConfig) -> int:
                     run_name=run_name,
                     model_id=resolved.model_id,
                     provider_type=resolved.provider_type,
+                    reasoning_effort=config.reasoning_effort,
                 )
 
                 def on_langfuse_item_done(
@@ -329,6 +342,7 @@ def _run(config: RunConfig) -> int:
                     workspace_id=config.workspace_id,
                     k=config.runs,
                     model_version=resolved.model_id,
+                    reasoning_effort=config.reasoning_effort,
                     use_langfuse=config.log_to_langfuse,
                     run_ts=run_ts,
                     on_item_start=on_item_start,
@@ -342,6 +356,7 @@ def _run(config: RunConfig) -> int:
                     token=config.token,
                     workspace_id=config.workspace_id,
                     preserve_failed=config.preserve_failed,
+                    reasoning_effort=config.reasoning_effort,
                 ),
                 SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
             )
@@ -433,6 +448,7 @@ def main(argv: list[str] | None = None) -> int:
             quiet=args.quiet,
             kind=args.kind,
             preserve_failed=args.preserve_failed,
+            reasoning_effort=args.reasoning_effort,
         )
         return _run(config)
     except (

{gooddata_eval-1.71.0 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/_langfuse.py

@@ -15,6 +15,8 @@ from typing import Any
 
 import httpx
 
+from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort
+
 _log = logging.getLogger(__name__)
 
 # ---------------------------------------------------------------------------
@@ -384,6 +386,7 @@ def build_run_context(
     run_timestamp: str | None,
     model_version_override: str | None,
     run_metadata_extra: dict[str, Any] | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> tuple[str, dict[str, Any]]:
     """Return (run_name_base, run_metadata) with model version resolved from workspace API.
 
@@ -399,15 +402,24 @@ def build_run_context(
             (e.g. a testing-framework tag or a CI run id for scoping). Default None keeps
             behavior unchanged. The SDK-derived model_version is applied last and cannot
             be overwritten by this dict.
+        reasoning_effort: Effort the run requested, stamped into both the run name and
+            the metadata so effort-varying runs stay comparable side by side.
     """
+    effort = normalize_reasoning_effort(reasoning_effort)
     model = get_model_version(host, token, workspace_id, model_version_override)
     ts = run_timestamp or datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
     base = f"{dataset_name}_{ts}"
     if model:
         base = f"{base}_{model}"
+    # Part of the run name, not just metadata: two runs that differ only by effort would
+    # otherwise collide on the same name and be indistinguishable in the report.
+    if effort:
+        base = f"{base}_effort-{effort.lower()}"
     # Caller supplies its own run tags (e.g. testing_framework); model_version is applied
     # last so the SDK-derived value cannot be overwritten by run_metadata_extra.
     metadata: dict[str, Any] = dict(run_metadata_extra) if run_metadata_extra else {}
+    if effort:
+        metadata["reasoning_effort"] = effort
     if model:
         metadata["model_version"] = model
     return base, metadata

{gooddata_eval-1.71.0 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/alert_skill.py

@@ -13,6 +13,7 @@ from gooddata_sdk import GoodDataSdk
 
 from gooddata_eval.core.agentic._catalog import CatalogMetricAlert
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import ToolCallEvent
 
 try:
@@ -311,11 +312,26 @@ def _extract_alert_call(tool_call_events: list[ToolCallEvent]) -> tuple[str | No
     return None, {}, False
 
 
-def _is_asking_clarification(text: str) -> bool:
-    if not text:
-        return False
-    t = text.lower()
-    return "?" in t or "could you" in t or "please" in t or "clarif" in t
+def render_alert_proposal(proposal: dict) -> str:
+    """Render an alert-proposal part as the text the simulated user reacts to.
+
+    The alert skill's confirmation step deliberately emits no text part (GDAI-2032) — the
+    prompt and the CTA live only in the proposal payload, which the frontend renders as a
+    widget. Dumping the payload (rather than prose) keeps recipients, condition, trigger and
+    dashboard visible so the simulated user can still verify them against its goal, and does
+    not need updating whenever ``AlertProposal`` grows a field.
+    """
+    cta = proposal.get("cta") or "Should I create this alert?"
+    summary = {k: v for k, v in proposal.items() if k != "cta"}
+    alert = dict(summary.get("alert") or {})
+    # The AFM execution block is opaque wire dicts — noise that would crowd out the fields
+    # the simulated user actually has to check.
+    alert.pop("execution", None)
+    if "alert" in summary:
+        # Key off presence, not truthiness: an alert whose only key was `execution` must
+        # still be replaced, otherwise the original (execution-bearing) dict survives.
+        summary["alert"] = alert
+    return f"{cta}\n\nAlert proposal:\n{json.dumps(summary, indent=2, sort_keys=True)}"
 
 
 def run_agentic_alert_skill(
@@ -327,11 +343,12 @@ def run_agentic_alert_skill(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticAlertSummary:
     """Run the alert-skill agentic evaluation K times and return a summary."""
     expected = _normalize_expected_output(expected_output)
     run_results: list[AlertRunResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     sdk = GoodDataSdk.create(host, token)
 
     def _run_once(conv_id: str) -> AlertRunResult:
@@ -352,6 +369,8 @@ def run_agentic_alert_skill(
                     alert_id_to_delete = alert_id
                     break
                 response_text = (chat_result.text_response or "").strip()
+                if not response_text and chat_result.alert_proposals:
+                    response_text = render_alert_proposal(chat_result.alert_proposals[-1])
                 # Stop if agent gave a completely empty response (stuck)
                 if not response_text and not chat_result.tool_call_events:
                     break
@@ -445,6 +464,7 @@ def evaluate_agentic_alert_skill(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -464,6 +484,7 @@ def evaluate_agentic_alert_skill(
         k=k,
         max_iterations=max_iterations,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -483,6 +504,7 @@ def evaluate_agentic_alert_skill(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.71.0 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/conversation.py

@@ -11,8 +11,10 @@ from typing import Literal
 from gooddata_sdk import GoodDataSdk
 from pydantic import BaseModel
 
+from gooddata_eval.core.agentic.alert_skill import render_alert_proposal
 from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import ChatResult, ToolCallEvent
 from gooddata_eval.core.scoring import (
     check_filters,
@@ -277,6 +279,7 @@ def run_agentic_conversation(
     fixture: ConversationFixture,
     max_clarification_turns: int = 20,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> ConversationResult:
     """Run a multi-turn, multi-skill conversation evaluation (no K-runs).
 
@@ -284,7 +287,7 @@ def run_agentic_conversation(
     trigger up to *max_clarification_turns* additional rounds of simulated-user
     replies before the agent produces the expected output.
     """
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     sdk = GoodDataSdk.create(host, token)
     turn_results: list[TurnResult] = []
     turn_outputs: dict[str, dict] = {}
@@ -322,7 +325,10 @@ def run_agentic_conversation(
                     break
 
                 response_text = (chat_result.text_response or "").strip()
-                if _is_asking_clarification(response_text) and clarification_turns < max_clarification_turns:
+                if not response_text and chat_result.alert_proposals:
+                    response_text = render_alert_proposal(chat_result.alert_proposals[-1])
+                asking = _is_asking_clarification(response_text) or bool(chat_result.alert_proposals)
+                if asking and clarification_turns < max_clarification_turns:
                     clarification_turns += 1
                     total_clarification_turns += 1
                     current_message = _get_sim_user_response(response_text, resolved_turn, resolved_expected)
@@ -399,6 +405,7 @@ def evaluate_agentic_conversation(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run conversation evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -416,6 +423,7 @@ def evaluate_agentic_conversation(
         fixture=fixture,
         max_clarification_turns=max_clarification_turns,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -435,6 +443,7 @@ def evaluate_agentic_conversation(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.71.0 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/general_question.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 from dataclasses import dataclass
 
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import LLMJudge
 
 _DEFAULT_K = 1
@@ -71,10 +72,11 @@ def run_agentic_general_question(
     expected_output: str,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticGeneralQuestionSummary:
     """Run the general-question agentic evaluation K times and return a summary."""
     run_results: list[GeneralQuestionResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     judge = LLMJudge(_GENERAL_QUESTION_EVALUATION_STEPS, model="gpt-4o")
 
     try:
@@ -153,6 +155,7 @@ def evaluate_agentic_general_question(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run general-question evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -171,6 +174,7 @@ def evaluate_agentic_general_question(
         expected_output=expected_output,
         k=k,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -190,6 +194,7 @@ def evaluate_agentic_general_question(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.71.0 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/guardrail.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 from dataclasses import dataclass
 
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import LLMJudge
 
 _DEFAULT_K = 1
@@ -68,10 +69,11 @@ def run_agentic_guardrail(
     expected_output: str,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticGuardrailSummary:
     """Run the guardrail agentic evaluation K times and return a summary."""
     run_results: list[GuardrailResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     judge = LLMJudge(_GUARDRAIL_EVALUATION_STEPS, model="gpt-4o")
 
     try:
@@ -150,6 +152,7 @@ def evaluate_agentic_guardrail(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run guardrail evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -168,6 +171,7 @@ def evaluate_agentic_guardrail(
         expected_output=expected_output,
         k=k,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -187,6 +191,7 @@ def evaluate_agentic_guardrail(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.71.0 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/metric_skill.py

@@ -11,6 +11,7 @@ from typing import Any
 from gooddata_sdk import GoodDataSdk
 
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import ToolCallEvent
 
 try:
@@ -232,6 +233,7 @@ def run_agentic_metric_skill(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticMetricSummary:
     """Run the metric-skill agentic evaluation K times and return a summary.
 
@@ -240,7 +242,7 @@ def run_agentic_metric_skill(
     """
     expected_outputs: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output]
     run_results: list[MetricRunResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     sdk = GoodDataSdk.create(host, token)
 
     try:
@@ -300,6 +302,7 @@ def evaluate_agentic_metric_skill(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -319,6 +322,7 @@ def evaluate_agentic_metric_skill(
         k=k,
         max_iterations=max_iterations,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -338,6 +342,7 @@ def evaluate_agentic_metric_skill(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.71.0 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/search_tool.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 from dataclasses import dataclass
 
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import ToolCallEvent
 
 _DEFAULT_K = 1
@@ -67,11 +68,12 @@ def run_agentic_search_tool(
     expected_tool_call: dict,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticSearchSummary:
     """Run the search-tool agentic evaluation K times (single-turn each)."""
     run_results: list[SearchResult] = []
 
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     try:
         conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
         try:
@@ -144,6 +146,7 @@ def evaluate_agentic_search_tool(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -162,6 +165,7 @@ def evaluate_agentic_search_tool(
         expected_tool_call=expected_tool_call,
         k=k,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -181,6 +185,7 @@ def evaluate_agentic_search_tool(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.71.0 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/visualization.py

@@ -11,6 +11,7 @@ import os
 from dataclasses import dataclass
 
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators.visualization import (
     EvaluationResult,
     _check_visualization_skill_activated,
@@ -203,6 +204,7 @@ def run_agentic_visualization(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticRunSummary:
     """Run K independent conversations and return evaluation results.
 
@@ -211,7 +213,7 @@ def run_agentic_visualization(
     fresh conversations. Caller-supplied conversations are not deleted; all
     conversations created by this function are deleted on completion.
     """
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     run_results: list[RunResult] = []
 
     try:
@@ -265,6 +267,7 @@ def evaluate_agentic_visualization(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     record_output_path: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure."""
     import json as _json  # noqa: PLC0415
@@ -285,6 +288,7 @@ def evaluate_agentic_visualization(
         k=k,
         max_iterations=max_iterations,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -304,6 +308,7 @@ def evaluate_agentic_visualization(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         K = len(summary.run_results)
         traces_by_conv = find_traces_per_conversation(