gooddata-eval 1.71.1.dev1__tar.gz → 1.71.1.dev2__tar.gz

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-eval
-Version: 1.71.1.dev1
+Version: 1.71.1.dev2
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.71.1.dev1
+Requires-Dist: gooddata-sdk~=1.71.1.dev2
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6
@@ -120,6 +120,7 @@ Both provider name and provider id are accepted as the prefix.
 |---|---|---|
 | `--runs K` | `2` | Independent runs per item (pass@K). An item passes if any run passes. |
 | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests. Progress output interleaves when K > 1. |
+| `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` items go through the summary endpoint, which has no such option. |
 
 #### Output
 
@@ -132,7 +133,7 @@ Both provider name and provider id are accepted as the prefix.
 
 | Flag | Description |
 |---|---|
-| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. |
+| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`, suffixed `-effort-{level}` when `--reasoning-effort` is set so runs differing only by effort stay separate). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. |
 
 ### JSON report shape
 

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/README.md

@@ -92,6 +92,7 @@ Both provider name and provider id are accepted as the prefix.
 |---|---|---|
 | `--runs K` | `2` | Independent runs per item (pass@K). An item passes if any run passes. |
 | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests. Progress output interleaves when K > 1. |
+| `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` items go through the summary endpoint, which has no such option. |
 
 #### Output
 
@@ -104,7 +105,7 @@ Both provider name and provider id are accepted as the prefix.
 
 | Flag | Description |
 |---|---|
-| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. |
+| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`, suffixed `-effort-{level}` when `--reasoning-effort` is set so runs differing only by effort stay separate). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. |
 
 ### JSON report shape
 

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.71.1.dev1"
+version = "1.71.1.dev2"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.71.1.dev1",
+    "gooddata-sdk~=1.71.1.dev2",
     "httpx>=0.27,<1.0",
     "orjson>=3.9.15,<4.0.0",
     "pydantic>=2.6,<3.0",

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/cli/agentic_runner.py

@@ -14,6 +14,7 @@ from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail
 from gooddata_eval.core.agentic.metric_skill import evaluate_agentic_metric_skill
 from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 
@@ -24,6 +25,7 @@ class _LfKw(TypedDict, total=False):
     dataset_name: str
     run_timestamp: str
     model_version_override: str | None
+    reasoning_effort: ReasoningEffort | None
 
 
 AGENTIC_TEST_KINDS = frozenset(
@@ -80,6 +82,7 @@ def _dispatch_agentic(
     langfuse: Any,
     run_ts: str,
     model_version_override: str | None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Call the appropriate evaluate_agentic_* function for the item's test_kind."""
     kind = item.test_kind
@@ -90,6 +93,7 @@ def _dispatch_agentic(
         "dataset_name": item.dataset_name,
         "run_timestamp": run_ts,
         "model_version_override": model_version_override,
+        "reasoning_effort": reasoning_effort,
     }
 
     if kind in ("vis_agentic", "agentic_visualization"):
@@ -176,6 +180,7 @@ def run_agentic_items(
     *,
     k: int = 2,
     model_version: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
     use_langfuse: bool = False,
     run_ts: str,
     on_item_start: Any = None,
@@ -202,7 +207,7 @@ def run_agentic_items(
         )
         t0 = time.perf_counter()
         try:
-            _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version)
+            _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort)
             item_report.pass_at_k = True
             item_report.runs = k
         except AssertionError as exc:

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/cli/main.py

@@ -6,6 +6,7 @@ import sys
 import threading
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import get_args
 
 import httpx
 from gooddata_api_client.exceptions import ApiException
@@ -14,7 +15,7 @@ from rich.table import Table
 
 from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items
 from gooddata_eval.core.chat.sse_client import ChatClient
-from gooddata_eval.core.config import RunConfig
+from gooddata_eval.core.config import ReasoningEffort, RunConfig
 from gooddata_eval.core.connection import ConnectionError_, resolve_connection
 from gooddata_eval.core.dataset.local import load_local_dataset
 from gooddata_eval.core.langfuse.sink import LangfuseSink
@@ -104,6 +105,13 @@ def _build_parser() -> argparse.ArgumentParser:
         dest="preserve_failed",
         help="Keep failed conversations on the server for post-mortem inspection.",
     )
+    run.add_argument(
+        "--reasoning-effort",
+        dest="reasoning_effort",
+        choices=list(get_args(ReasoningEffort)),
+        help="Reasoning effort requested per message. Requires the enableGenAiReasoningEffort "
+        "feature flag on the target organization; without it the server ignores the value.",
+    )
     run.add_argument(
         "--langfuse",
         action="store_true",
@@ -292,6 +300,10 @@ def _run(config: RunConfig) -> int:
                     progress_console.print(f"Provider={provider_display}, model={resolved.model_id}{switched}")
 
             run_name = f"gd-eval-{run_ts}-{resolved.model_id}"
+            if config.reasoning_effort:
+                # Without this two runs differing only by effort share a name and are
+                # indistinguishable in the report, which is the comparison this exists for.
+                run_name = f"{run_name}-effort-{config.reasoning_effort.lower()}"
             if progress_console and config.log_to_langfuse:
                 progress_console.print(f"Logging to Langfuse run '{run_name}'...")
 
@@ -307,6 +319,7 @@ def _run(config: RunConfig) -> int:
                     run_name=run_name,
                     model_id=resolved.model_id,
                     provider_type=resolved.provider_type,
+                    reasoning_effort=config.reasoning_effort,
                 )
 
                 def on_langfuse_item_done(
@@ -329,6 +342,7 @@ def _run(config: RunConfig) -> int:
                     workspace_id=config.workspace_id,
                     k=config.runs,
                     model_version=resolved.model_id,
+                    reasoning_effort=config.reasoning_effort,
                     use_langfuse=config.log_to_langfuse,
                     run_ts=run_ts,
                     on_item_start=on_item_start,
@@ -342,6 +356,7 @@ def _run(config: RunConfig) -> int:
                     token=config.token,
                     workspace_id=config.workspace_id,
                     preserve_failed=config.preserve_failed,
+                    reasoning_effort=config.reasoning_effort,
                 ),
                 SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
             )
@@ -433,6 +448,7 @@ def main(argv: list[str] | None = None) -> int:
             quiet=args.quiet,
             kind=args.kind,
             preserve_failed=args.preserve_failed,
+            reasoning_effort=args.reasoning_effort,
         )
         return _run(config)
     except (

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/_langfuse.py

@@ -15,6 +15,8 @@ from typing import Any
 
 import httpx
 
+from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort
+
 _log = logging.getLogger(__name__)
 
 # ---------------------------------------------------------------------------
@@ -384,6 +386,7 @@ def build_run_context(
     run_timestamp: str | None,
     model_version_override: str | None,
     run_metadata_extra: dict[str, Any] | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> tuple[str, dict[str, Any]]:
     """Return (run_name_base, run_metadata) with model version resolved from workspace API.
 
@@ -399,15 +402,24 @@ def build_run_context(
             (e.g. a testing-framework tag or a CI run id for scoping). Default None keeps
             behavior unchanged. The SDK-derived model_version is applied last and cannot
             be overwritten by this dict.
+        reasoning_effort: Effort the run requested, stamped into both the run name and
+            the metadata so effort-varying runs stay comparable side by side.
     """
+    effort = normalize_reasoning_effort(reasoning_effort)
     model = get_model_version(host, token, workspace_id, model_version_override)
     ts = run_timestamp or datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
     base = f"{dataset_name}_{ts}"
     if model:
         base = f"{base}_{model}"
+    # Part of the run name, not just metadata: two runs that differ only by effort would
+    # otherwise collide on the same name and be indistinguishable in the report.
+    if effort:
+        base = f"{base}_effort-{effort.lower()}"
     # Caller supplies its own run tags (e.g. testing_framework); model_version is applied
     # last so the SDK-derived value cannot be overwritten by run_metadata_extra.
     metadata: dict[str, Any] = dict(run_metadata_extra) if run_metadata_extra else {}
+    if effort:
+        metadata["reasoning_effort"] = effort
     if model:
         metadata["model_version"] = model
     return base, metadata

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/alert_skill.py

@@ -13,6 +13,7 @@ from gooddata_sdk import GoodDataSdk
 
 from gooddata_eval.core.agentic._catalog import CatalogMetricAlert
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import ToolCallEvent
 
 try:
@@ -342,11 +343,12 @@ def run_agentic_alert_skill(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticAlertSummary:
     """Run the alert-skill agentic evaluation K times and return a summary."""
     expected = _normalize_expected_output(expected_output)
     run_results: list[AlertRunResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     sdk = GoodDataSdk.create(host, token)
 
     def _run_once(conv_id: str) -> AlertRunResult:
@@ -462,6 +464,7 @@ def evaluate_agentic_alert_skill(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -481,6 +484,7 @@ def evaluate_agentic_alert_skill(
         k=k,
         max_iterations=max_iterations,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -500,6 +504,7 @@ def evaluate_agentic_alert_skill(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/conversation.py

@@ -14,6 +14,7 @@ from pydantic import BaseModel
 from gooddata_eval.core.agentic.alert_skill import render_alert_proposal
 from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import ChatResult, ToolCallEvent
 from gooddata_eval.core.scoring import (
     check_filters,
@@ -278,6 +279,7 @@ def run_agentic_conversation(
     fixture: ConversationFixture,
     max_clarification_turns: int = 20,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> ConversationResult:
     """Run a multi-turn, multi-skill conversation evaluation (no K-runs).
 
@@ -285,7 +287,7 @@ def run_agentic_conversation(
     trigger up to *max_clarification_turns* additional rounds of simulated-user
     replies before the agent produces the expected output.
     """
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     sdk = GoodDataSdk.create(host, token)
     turn_results: list[TurnResult] = []
     turn_outputs: dict[str, dict] = {}
@@ -403,6 +405,7 @@ def evaluate_agentic_conversation(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run conversation evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -420,6 +423,7 @@ def evaluate_agentic_conversation(
         fixture=fixture,
         max_clarification_turns=max_clarification_turns,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -439,6 +443,7 @@ def evaluate_agentic_conversation(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/general_question.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 from dataclasses import dataclass
 
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import LLMJudge
 
 _DEFAULT_K = 1
@@ -71,10 +72,11 @@ def run_agentic_general_question(
     expected_output: str,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticGeneralQuestionSummary:
     """Run the general-question agentic evaluation K times and return a summary."""
     run_results: list[GeneralQuestionResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     judge = LLMJudge(_GENERAL_QUESTION_EVALUATION_STEPS, model="gpt-4o")
 
     try:
@@ -153,6 +155,7 @@ def evaluate_agentic_general_question(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run general-question evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -171,6 +174,7 @@ def evaluate_agentic_general_question(
         expected_output=expected_output,
         k=k,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -190,6 +194,7 @@ def evaluate_agentic_general_question(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/guardrail.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 from dataclasses import dataclass
 
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import LLMJudge
 
 _DEFAULT_K = 1
@@ -68,10 +69,11 @@ def run_agentic_guardrail(
     expected_output: str,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticGuardrailSummary:
     """Run the guardrail agentic evaluation K times and return a summary."""
     run_results: list[GuardrailResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     judge = LLMJudge(_GUARDRAIL_EVALUATION_STEPS, model="gpt-4o")
 
     try:
@@ -150,6 +152,7 @@ def evaluate_agentic_guardrail(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run guardrail evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -168,6 +171,7 @@ def evaluate_agentic_guardrail(
         expected_output=expected_output,
         k=k,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -187,6 +191,7 @@ def evaluate_agentic_guardrail(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/metric_skill.py

@@ -11,6 +11,7 @@ from typing import Any
 from gooddata_sdk import GoodDataSdk
 
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import ToolCallEvent
 
 try:
@@ -232,6 +233,7 @@ def run_agentic_metric_skill(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticMetricSummary:
     """Run the metric-skill agentic evaluation K times and return a summary.
 
@@ -240,7 +242,7 @@ def run_agentic_metric_skill(
     """
     expected_outputs: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output]
     run_results: list[MetricRunResult] = []
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     sdk = GoodDataSdk.create(host, token)
 
     try:
@@ -300,6 +302,7 @@ def evaluate_agentic_metric_skill(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -319,6 +322,7 @@ def evaluate_agentic_metric_skill(
         k=k,
         max_iterations=max_iterations,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -338,6 +342,7 @@ def evaluate_agentic_metric_skill(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/search_tool.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 from dataclasses import dataclass
 
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.models import ToolCallEvent
 
 _DEFAULT_K = 1
@@ -67,11 +68,12 @@ def run_agentic_search_tool(
     expected_tool_call: dict,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticSearchSummary:
     """Run the search-tool agentic evaluation K times (single-turn each)."""
     run_results: list[SearchResult] = []
 
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     try:
         conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
         try:
@@ -144,6 +146,7 @@ def evaluate_agentic_search_tool(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -162,6 +165,7 @@ def evaluate_agentic_search_tool(
         expected_tool_call=expected_tool_call,
         k=k,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -181,6 +185,7 @@ def evaluate_agentic_search_tool(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/agentic/visualization.py

@@ -11,6 +11,7 @@ import os
 from dataclasses import dataclass
 
 from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators.visualization import (
     EvaluationResult,
     _check_visualization_skill_activated,
@@ -203,6 +204,7 @@ def run_agentic_visualization(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> AgenticRunSummary:
     """Run K independent conversations and return evaluation results.
 
@@ -211,7 +213,7 @@ def run_agentic_visualization(
     fresh conversations. Caller-supplied conversations are not deleted; all
     conversations created by this function are deleted on completion.
     """
-    client = ChatClient(host=host, token=token, workspace_id=workspace_id)
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
     run_results: list[RunResult] = []
 
     try:
@@ -265,6 +267,7 @@ def evaluate_agentic_visualization(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     record_output_path: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
 ) -> None:
     """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure."""
     import json as _json  # noqa: PLC0415
@@ -285,6 +288,7 @@ def evaluate_agentic_visualization(
         k=k,
         max_iterations=max_iterations,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:
@@ -304,6 +308,7 @@ def evaluate_agentic_visualization(
             run_timestamp,
             model_version_override,
             run_metadata_extra,
+            reasoning_effort,
         )
         K = len(summary.run_results)
         traces_by_conv = find_traces_per_conversation(

{gooddata_eval-1.71.1.dev1 → gooddata_eval-1.71.1.dev2}/src/gooddata_eval/core/chat/sse_client.py

@@ -22,6 +22,7 @@ from typing import Any, Callable, Iterable, TypeVar
 
 import httpx
 
+from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort
 from gooddata_eval.core.models import ChatResult, DatasetItem
 
 _log = logging.getLogger(__name__)
@@ -242,12 +243,28 @@ class ChatClient:
     """Single-turn AI chat client over the GoodData AI conversation endpoints."""
 
     def __init__(
-        self, host: str, token: str, workspace_id: str, *, timeout: float = 300.0, preserve_failed: bool = False
+        self,
+        host: str,
+        token: str,
+        workspace_id: str,
+        *,
+        timeout: float = 300.0,
+        preserve_failed: bool = False,
+        reasoning_effort: ReasoningEffort | None = None,
     ):
+        """Create a chat client bound to one workspace.
+
+        ``reasoning_effort`` (``LOW``/``MEDIUM``/``HIGH``) is sent as
+        ``options.reasoningEffort`` on every message; when None the key is omitted
+        entirely and the server keeps its own default. The server honours it only
+        while the ``enableGenAiReasoningEffort`` feature flag is on for the
+        organization, so setting it is a request rather than a guarantee.
+        """
         self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations"
         self._auth = {"Authorization": f"Bearer {token}"}
         self._client = httpx.Client(timeout=timeout)
         self._preserve_failed = preserve_failed
+        self._reasoning_effort = normalize_reasoning_effort(reasoning_effort)
 
     def create_conversation(self) -> str:
         def _do() -> str:
@@ -271,7 +288,9 @@ class ChatClient:
     def send_message(self, conversation_id: str, question: str) -> ChatResult:
         url = f"{self._base}/{conversation_id}/messages"
         headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json"}
-        body = {"item": {"role": "user", "content": {"type": "text", "text": question}}}
+        body: dict[str, Any] = {"item": {"role": "user", "content": {"type": "text", "text": question}}}
+        if self._reasoning_effort is not None:
+            body["options"] = {"reasoningEffort": self._reasoning_effort}
 
         def _do() -> ChatResult:
             with self._client.stream("POST", url, json=body, headers=headers) as resp:

gooddata_eval-1.71.1.dev2/src/gooddata_eval/core/config.py

@@ -0,0 +1,46 @@
+# (C) 2026 GoodData Corporation
+"""Validated run configuration produced by the CLI and consumed by the runner."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal, cast, get_args
+
+ReasoningEffort = Literal["LOW", "MEDIUM", "HIGH"]
+"""Effort values the AI chat endpoint accepts, uppercase as the server enum requires."""
+
+
+def normalize_reasoning_effort(value: str | None) -> ReasoningEffort | None:
+    """Canonical effort, or None when unset.
+
+    The `Literal` above only constrains static callers, so normalize once at the
+    boundary: without it a lowercase value reaches the endpoint and is rejected as
+    an out-of-enum request, while an empty string is sent yet skipped by the
+    truthiness checks in the Langfuse writers — leaving a run whose recorded
+    identity disagrees with what it actually requested.
+    """
+    if value is None:
+        return None
+    candidate = value.strip().upper()
+    if not candidate:
+        return None
+    if candidate not in get_args(ReasoningEffort):
+        raise ValueError(f"Invalid reasoning effort {value!r}; expected one of {', '.join(get_args(ReasoningEffort))}.")
+    return cast("ReasoningEffort", candidate)
+
+
+@dataclass
+class RunConfig:
+    host: str
+    token: str
+    workspace_id: str
+    dataset_folder: Path | None = None
+    langfuse_dataset: str | None = None
+    models: list[str] = field(default_factory=list)
+    runs: int = 2
+    concurrency: int = 1
+    json_path: Path | None = None
+    log_to_langfuse: bool = False
+    quiet: bool = False
+    kind: str = "visualization"
+    preserve_failed: bool = False
+    reasoning_effort: ReasoningEffort | None = None