gooddata-eval 1.74.1.dev3__tar.gz → 1.74.1.dev5__tar.gz

{gooddata_eval-1.74.1.dev3 → gooddata_eval-1.74.1.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.74.1.dev3
+Version: 1.74.1.dev5
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.74.1.dev3
+Requires-Dist: gooddata-sdk~=1.74.1.dev5
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6

{gooddata_eval-1.74.1.dev3 → gooddata_eval-1.74.1.dev5}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.74.1.dev3"
+version = "1.74.1.dev5"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.74.1.dev3",
+    "gooddata-sdk~=1.74.1.dev5",
     "httpx>=0.27,<1.0",
     "orjson>=3.9.15,<4.0.0",
     "pydantic>=2.6,<3.0",

{gooddata_eval-1.74.1.dev3 → gooddata_eval-1.74.1.dev5}/src/gooddata_eval/core/agentic/alert_skill.py

@@ -477,6 +477,8 @@ class AlertRunResult:
     alert_id: str | None
     eval: AlertEvaluation
     actual_alert_arguments: dict
+    total_turns: int = 0
+    total_steps: int = 0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -676,9 +678,13 @@ def run_agentic_alert_skill(
             # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply.
             conversation_history: list = []
             current_question = question
+            turns = 0
+            steps = 0
 
             for _iteration in range(max_iterations):
                 chat_result = client.send_message(conv_id, current_question)
+                turns += 1
+                steps += chat_result.reasoning_step_count
                 reasoning_steps.extend(chat_result.reasoning_steps or [])
                 response_id = chat_result.response_id or response_id
                 turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
@@ -728,6 +734,8 @@ def run_agentic_alert_skill(
                 alert_id=alert_id,
                 eval=ev,
                 actual_alert_arguments=actual_args,
+                total_turns=turns,
+                total_steps=steps,
                 reasoning_steps=reasoning_steps,
                 response_id=response_id,
                 tool_call_events=all_tool_call_events,
@@ -851,6 +859,8 @@ def evaluate_agentic_alert_skill(
                 with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid:
                     for score_name, value in strict_checks.items():
                         ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN")
+                    ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC")
+                    ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC")
                     log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k)
                     ctx.quality(
                         tid,

{gooddata_eval-1.74.1.dev3 → gooddata_eval-1.74.1.dev5}/src/gooddata_eval/core/agentic/conversation.py

@@ -336,6 +336,7 @@ class ConversationResult:
     full_skill_coverage: bool
     conversation_success: bool
     total_clarification_turns: int
+    total_steps: int = 0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -365,6 +366,7 @@ def run_agentic_conversation(
     turn_results: list[TurnResult] = []
     turn_outputs: dict[str, dict] = {}
     total_clarification_turns = 0
+    total_steps = 0
     conversation_id: str = ""
     owns_conversation = False
     # Metrics created during this conversation, deleted after it completes so they do
@@ -434,6 +436,7 @@ def run_agentic_conversation(
             for _iter in range(max_clarification_turns + 1):
                 chat_result = client.send_message(conversation_id, current_message)
                 final_result = chat_result
+                total_steps += chat_result.reasoning_step_count
                 turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
                     chat_result,
                     turn_offset=turn_offset,
@@ -522,6 +525,7 @@ def run_agentic_conversation(
         full_skill_coverage=full_skill_coverage,
         conversation_success=conversation_success,
         total_clarification_turns=total_clarification_turns,
+        total_steps=total_steps,
         reasoning_steps=reasoning_steps,
         response_id=response_id,
         tool_call_events=conversation_tool_call_events,
@@ -611,6 +615,22 @@ def evaluate_agentic_conversation(
                     value=float(result.full_skill_coverage),
                     data_type="BOOLEAN",
                 )
+                # One turn per fixture turn, plus every simulated-user round the agent triggered.
+                # The clarification count alone hides how much of the conversation the fixture
+                # asked for, so the comparison needs the total.
+                ctx.score(
+                    tid,
+                    name="turns",
+                    value=len(result.turn_results) + result.total_clarification_turns,
+                    data_type="NUMERIC",
+                )
+                ctx.score(tid, name="steps", value=result.total_steps, data_type="NUMERIC")
+                ctx.score(
+                    tid,
+                    name="clarification_turns",
+                    value=result.total_clarification_turns,
+                    data_type="NUMERIC",
+                )
                 for tr in result.turn_results:
                     ctx.score(
                         tid,

{gooddata_eval-1.74.1.dev3 → gooddata_eval-1.74.1.dev5}/src/gooddata_eval/core/agentic/kda_skill.py

@@ -185,6 +185,8 @@ class KdaRunResult:
     # Wall-clock time of the turn that called create (None if create never happened) --
     # not any earlier disambiguation turn. See run_agentic_kda_skill's _run_once.
     turn_wall_clock_sec: float | None = None
+    total_turns: int = 0
+    total_steps: int = 0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -276,6 +278,9 @@ def run_agentic_kda_skill(
             all_tool_call_events.extend(result.tool_call_events or [])
             all_reasoning_step_events.extend(result.reasoning_step_events or [])
 
+        turns = 0
+        steps = 0
+
         for iteration in range(max_iterations):
             try:
                 chat_result = client.send_message(conv_id, current_question)
@@ -291,6 +296,8 @@ def run_agentic_kda_skill(
                         turn_wall_clock_sec = partial.turn_wall_clock_sec
                 turn_completed = False
                 break
+            turns += 1
+            steps += chat_result.reasoning_step_count
             reasoning_steps.extend(chat_result.reasoning_steps or [])
             response_id = chat_result.response_id or response_id
             _accumulate(chat_result)
@@ -330,6 +337,8 @@ def run_agentic_kda_skill(
             actual_create_args=create_args,
             actual_execute_result=execute_result,
             turn_wall_clock_sec=turn_wall_clock_sec,
+            total_turns=turns,
+            total_steps=steps,
             reasoning_steps=reasoning_steps,
             response_id=response_id,
             tool_call_events=all_tool_call_events,
@@ -442,6 +451,8 @@ def evaluate_agentic_kda_skill(
                     for score_name, value in strict_checks.items():
                         ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN")
                     ctx.score(tid, name="kda_disambiguated", value=float(ev.disambiguated), data_type="BOOLEAN")
+                    ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC")
+                    ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC")
                     if turn_wall_clock_sec is not None:
                         # combo_report.py reads this score directly -- no trace re-resolution needed.
                         ctx.score(

{gooddata_eval-1.74.1.dev3 → gooddata_eval-1.74.1.dev5}/src/gooddata_eval/core/agentic/metric_skill.py

@@ -162,7 +162,8 @@ class MetricRunResult:
     metric_created: bool
     actual_maql: str
     maql_correct: bool
-    total_turns: float
+    total_turns: int
+    total_steps: int = 0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -253,6 +254,7 @@ def _execute_single_metric_run(
     metric_result: dict | None = None
     created_metric_ids: list[str] = []
     turns = 0
+    steps = 0
     current_question = question
     reasoning_steps: list[str] = []
     response_id: str | None = None
@@ -280,6 +282,7 @@ def _execute_single_metric_run(
             )
             all_tool_call_events.extend(chat_result.tool_call_events or [])
             all_reasoning_step_events.extend(chat_result.reasoning_step_events or [])
+            steps += chat_result.reasoning_step_count
             for metric_id in _extract_created_metric_ids(chat_result.tool_call_events or []):
                 if metric_id not in created_metric_ids:
                     created_metric_ids.append(metric_id)
@@ -330,7 +333,8 @@ def _execute_single_metric_run(
             metric_created=metric_created,
             actual_maql=actual_maql,
             maql_correct=maql_correct,
-            total_turns=float(turns),
+            total_turns=turns,
+            total_steps=steps,
             reasoning_steps=reasoning_steps,
             response_id=response_id,
             tool_call_events=all_tool_call_events,
@@ -466,6 +470,8 @@ def evaluate_agentic_metric_skill(
                 ) as tid:
                     ctx.score(tid, name="metric_created", value=float(run.metric_created), data_type="BOOLEAN")
                     ctx.score(tid, name="maql_correct", value=float(run.maql_correct), data_type="BOOLEAN")
+                    ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC")
+                    ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC")
                     log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k)
                     ctx.quality(
                         tid,

{gooddata_eval-1.74.1.dev3 → gooddata_eval-1.74.1.dev5}/src/gooddata_eval/core/agentic/visualization.py

@@ -59,8 +59,8 @@ class RunResult:
     actual_output: CreatedVisualization | None
     eval_result: EvaluationResult
     best_expected: CreatedVisualization
-    total_turns: float
-    total_steps: float
+    total_turns: int
+    total_steps: int
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -186,8 +186,8 @@ def _execute_single_run(
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
 ) -> RunResult:
     """Drive one full multi-turn conversation and evaluate the result."""
-    total_turns = 0.0
-    total_steps = 0.0
+    total_turns = 0
+    total_steps = 0
     all_tool_call_events: list[ToolCallEvent] = []
     all_reasoning_step_events: list[ReasoningStepEvent] = []
     reasoning_steps: list[str] = []
@@ -200,8 +200,8 @@ def _execute_single_run(
     current_result = client.send_message(conversation_id, question)
 
     for iteration in range(max_iterations):
-        total_turns += 1.0
-        total_steps += float(current_result.reasoning_step_count)
+        total_turns += 1
+        total_steps += current_result.reasoning_step_count
         turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
             current_result,
             turn_offset=turn_offset,

{gooddata_eval-1.74.1.dev3 → gooddata_eval-1.74.1.dev5}/tests/test_agentic_alert_skill.py

@@ -996,3 +996,67 @@ def test_every_gen_ai_interval_is_accepted():
         assert AnomalyDetectionGranularity.parse(value.lower()) is AnomalyDetectionGranularity(value)
     assert AnomalyDetectionGranularity.parse(None) is None
     assert AnomalyDetectionGranularity.parse("  ") is None
+
+
+def test_run_agentic_alert_skill_counts_the_turns_and_reasoning_steps_it_used():
+    """QA-29110: the effort comparison reads these. A refusal still took a turn, and the turn
+    count is what separates a wrong answer from a run max_iterations cut short."""
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.return_value = _no_alert_chat_result()
+    mock_client._base = "http://host/api/v1/actions/workspaces/ws1/ai"
+    mock_client._auth = {"Authorization": "Bearer tok"}
+
+    with _patched(mock_client, simulated_reply="Yes please"):
+        summary = run_agentic_alert_skill(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            question="Create alert",
+            expected_output={"operator": "GREATER_THAN", "threshold": 100},
+            k=1,
+            max_iterations=2,
+        )
+
+    # _no_alert_chat_result has no tool calls and non-empty text, so the run replies once and
+    # stops at max_iterations: 2 turns, 1 reasoning step each.
+    assert summary.best.total_turns == 2
+    assert summary.best.total_steps == 2
+
+
+def test_alert_skill_writes_the_turn_and_step_counts_to_langfuse():
+    """The counters exist to reach Langfuse; asserting only the dataclass would pass even if
+    the scores were never written."""
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.return_value = _no_alert_chat_result()
+    mock_client._base = "http://host/api/v1/actions/workspaces/ws1/ai"
+    mock_client._auth = {"Authorization": "Bearer tok"}
+    captured = {}
+
+    def _capture(_submit, _identity, **kwargs):
+        captured["write_scores"] = kwargs["write_scores"]
+
+    with (
+        _patched(mock_client),
+        patch("gooddata_eval.core.agentic.alert_skill.submit_trace_scoring", _capture),
+        pytest.raises(AlertSkillAssertionError),
+    ):
+        evaluate_agentic_alert_skill(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            question="Create alert",
+            expected_output={"operator": "GREATER_THAN", "threshold": 100},
+            k=1,
+            max_iterations=1,
+            langfuse=MagicMock(),
+            dataset_item_id="item-1",
+        )
+
+    ctx = MagicMock()
+    captured["write_scores"](ctx)
+    scores = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list}
+
+    assert scores["turns"] == 1
+    assert scores["steps"] == 1

{gooddata_eval-1.74.1.dev3 → gooddata_eval-1.74.1.dev5}/tests/test_agentic_conversation.py

@@ -1025,3 +1025,107 @@ def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_
         ],
         "latency_breakdown": [],
     }
+
+
+def test_run_agentic_conversation_sums_the_reasoning_steps_of_every_turn():
+    """QA-29110: the effort comparison reads `steps`. A clarification round is part of the
+    work the effort setting changes, so its steps count with the rest."""
+    proposal_turn = ChatResult.model_validate(
+        {
+            "text_response": None,
+            "alertProposals": [{"cta": "Should I create this alert?", "recipients": [{"email": "a@b.com"}]}],
+            "reasoningStepCount": 2,
+            "toolCallEvents": [
+                {"functionName": "set_skills", "functionArguments": '{"skills": ["alert"]}', "result": None},
+                {"functionName": "prepare_metric_alert_proposal", "functionArguments": "{}", "result": None},
+            ],
+        }
+    )
+    created_turn = ChatResult.model_validate(
+        {
+            "text_response": "Alert created.",
+            "reasoningStepCount": 3,
+            "toolCallEvents": [
+                {"functionName": "create_metric_alert", "functionArguments": "{}", "result": '{"id": "alert-1"}'}
+            ],
+        }
+    )
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.side_effect = [proposal_turn, created_turn]
+
+    with (
+        patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client),
+        patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"),
+        patch(
+            "gooddata_eval.core.agentic.conversation._get_sim_user_response",
+            return_value="Yes, please create it.",
+        ),
+    ):
+        result = run_agentic_conversation(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            fixture=_alert_turn_fixture(),
+        )
+
+    assert result.total_steps == 5
+    assert result.total_clarification_turns == 1
+
+
+def test_conversation_writes_the_turn_step_and_clarification_counts_to_langfuse():
+    """`turns` is not the clarification count: it is one per fixture turn plus every
+    simulated-user round, so a test has to pin the sum rather than either half."""
+    proposal_turn = ChatResult.model_validate(
+        {
+            "text_response": None,
+            "alertProposals": [{"cta": "Should I create this alert?", "recipients": [{"email": "a@b.com"}]}],
+            "reasoningStepCount": 2,
+            "toolCallEvents": [
+                {"functionName": "set_skills", "functionArguments": '{"skills": ["alert"]}', "result": None},
+                {"functionName": "prepare_metric_alert_proposal", "functionArguments": "{}", "result": None},
+            ],
+        }
+    )
+    created_turn = ChatResult.model_validate(
+        {
+            "text_response": "Alert created.",
+            "reasoningStepCount": 3,
+            "toolCallEvents": [
+                {"functionName": "create_metric_alert", "functionArguments": "{}", "result": '{"id": "alert-1"}'}
+            ],
+        }
+    )
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.side_effect = [proposal_turn, created_turn]
+    captured = {}
+
+    def _capture(_submit, _identity, **kwargs):
+        captured["write_scores"] = kwargs["write_scores"]
+
+    with (
+        patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client),
+        patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"),
+        patch("gooddata_eval.core.agentic.conversation.submit_trace_scoring", _capture),
+        patch(
+            "gooddata_eval.core.agentic.conversation._get_sim_user_response",
+            return_value="Yes, please create it.",
+        ),
+    ):
+        evaluate_agentic_conversation(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            fixture=_alert_turn_fixture(),
+            langfuse=MagicMock(),
+            dataset_item_id="item-1",
+        )
+
+    ctx = MagicMock()
+    captured["write_scores"](ctx)
+    scores = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list}
+
+    assert scores["clarification_turns"] == 1
+    assert scores["turns"] == 2  # 1 fixture turn + 1 clarification round
+    assert scores["steps"] == 5

{gooddata_eval-1.74.1.dev3 → gooddata_eval-1.74.1.dev5}/tests/test_agentic_kda_skill.py

@@ -1195,3 +1195,88 @@ def test_evaluate_agentic_kda_skill_preserves_reasoning_from_a_chat_error_partia
 
     assert exc_info.value.reasoning_steps == ["analyzing before cutoff"]
     assert exc_info.value.response_id == "resp-3"
+
+
+def test_run_agentic_kda_skill_counts_the_turns_and_reasoning_steps_it_used():
+    """QA-29110: the effort comparison reads these. A binary pass/fail cannot separate two
+    efforts on a nightly's sample, while the reasoning-step count moves with the effort."""
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    # Turn 1 asks for clarification, turn 2 runs the analysis: 2 turns, 1 step each.
+    mock_client.send_message.side_effect = [
+        _no_kda_chat_result("Which metric did you mean?"),
+        _kda_chat_result(success=True),
+    ]
+
+    with (
+        patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client),
+        patch("gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", return_value="Revenue"),
+    ):
+        summary = run_agentic_kda_skill(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            question="What drove the change?",
+            expected_output=_EXPECTED,
+            k=1,
+            max_iterations=2,
+        )
+
+    assert summary.best.total_turns == 2
+    assert summary.best.total_steps == 2
+
+
+def test_run_agentic_kda_skill_reports_no_turns_when_the_first_send_fails():
+    """A run that never got a reply must not report a turn it did not take."""
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.side_effect = RuntimeError("stream died")
+
+    with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client):
+        summary = run_agentic_kda_skill(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            question="What drove the change?",
+            expected_output=_EXPECTED,
+            k=1,
+            max_iterations=1,
+        )
+
+    assert summary.best.total_turns == 0
+    assert summary.best.total_steps == 0
+
+
+def test_kda_skill_writes_the_turn_and_step_counts_to_langfuse():
+    """The counters exist to reach Langfuse; asserting only the dataclass would pass even if
+    the scores were never written."""
+    mock_client = MagicMock()
+    mock_client.create_conversation.return_value = "conv-1"
+    mock_client.send_message.return_value = _kda_chat_result(success=True)
+    captured = {}
+
+    def _capture(_submit, _identity, **kwargs):
+        captured["write_scores"] = kwargs["write_scores"]
+
+    with (
+        patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client),
+        patch("gooddata_eval.core.agentic.kda_skill.submit_trace_scoring", _capture),
+    ):
+        evaluate_agentic_kda_skill(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            question="What drove the change?",
+            expected_output=_EXPECTED,
+            k=1,
+            max_iterations=1,
+            langfuse=MagicMock(),
+            dataset_item_id="item-1",
+        )
+
+    ctx = MagicMock()
+    captured["write_scores"](ctx)
+    scores = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list}
+
+    assert scores["turns"] == 1
+    assert scores["steps"] == 1

{gooddata_eval-1.74.1.dev3 → gooddata_eval-1.74.1.dev5}/tests/test_agentic_metric_skill.py

@@ -582,7 +582,7 @@ def test_run_agentic_metric_skill_fails_the_run_when_the_simulated_reply_cannot_
 
     assert summary.pass_at_k is False
     assert summary.best.metric_created is False
-    assert summary.best.total_turns == 1.0
+    assert summary.best.total_turns == 1
     mock_client.close.assert_called_once()
     mock_sim.assert_called_once_with(
         "Which brand field should I count?", [{"maql": "SELECT {metric/foo}"}], "Create metric foo"
@@ -764,3 +764,90 @@ def test_no_timer_output_by_default(monkeypatch, capsys):
     assert "[timer]" not in capsys.readouterr().out
     # Silenced, not un-measured.
     assert summary.run_results[0].timings.agent_s == 3.0
+
+
+def test_run_agentic_metric_skill_counts_the_turns_and_reasoning_steps_it_used():
+    """QA-29110: the effort comparison reads these. A clarification round is part of the work
+    the effort setting changes, so its steps count with the rest."""
+    clarify_turn = ChatResult.model_validate(
+        {"textResponse": "Which foo?", "toolCallEvents": [], "reasoningStepCount": 2}
+    )
+    created_turn = ChatResult.model_validate(
+        {
+            "textResponse": "done",
+            "reasoningStepCount": 3,
+            "toolCallEvents": [
+                {
+                    "functionName": "create_metric",
+                    "functionArguments": "{}",
+                    "result": '{"data": {"maql": "SELECT {metric/foo}"}}',
+                }
+            ],
+        }
+    )
+    mock_client = _client()
+    mock_client.send_message.side_effect = [clarify_turn, created_turn]
+
+    with _patched(mock_client, simulated_reply="It's foo"):
+        summary = run_agentic_metric_skill(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            question="Create metric foo",
+            expected_output={"maql": "SELECT {metric/foo}"},
+            k=1,
+            max_iterations=2,
+        )
+
+    assert summary.best.total_turns == 2
+    assert summary.best.total_steps == 5
+
+
+def test_metric_skill_writes_the_turn_and_step_counts_to_langfuse():
+    """The counters exist to reach Langfuse; asserting only the dataclass would pass even if
+    the scores were never written."""
+    mock_client = _client()
+    mock_client.send_message.return_value = ChatResult.model_validate(
+        {
+            "textResponse": "done",
+            "reasoningStepCount": 4,
+            "toolCallEvents": [
+                {
+                    "functionName": "create_metric",
+                    "functionArguments": "{}",
+                    "result": '{"data": {"maql": "SELECT {metric/foo}"}}',
+                }
+            ],
+        }
+    )
+    captured = {}
+
+    def _capture(_submit, _identity, **kwargs):
+        captured["write_scores"] = kwargs["write_scores"]
+
+    with (
+        _patched(mock_client),
+        patch("gooddata_eval.core.agentic.metric_skill.submit_trace_scoring", _capture),
+    ):
+        evaluate_agentic_metric_skill(
+            host="http://host/api/v1/actions/workspaces/ws1/ai",
+            token="tok",
+            workspace_id="ws1",
+            question="Create metric foo",
+            expected_output={"maql": "SELECT {metric/foo}"},
+            k=1,
+            max_iterations=1,
+            langfuse=MagicMock(),
+            dataset_item_id="item-1",
+        )
+
+    ctx = MagicMock()
+    captured["write_scores"](ctx)
+    scores = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list}
+
+    assert scores["turns"] == 1
+    assert scores["steps"] == 4
+    # `==` does not separate 1 from 1.0, so the counts need their type pinned separately:
+    # they are counts, and a float reads as though a fraction of a turn were possible.
+    assert isinstance(scores["turns"], int)
+    assert isinstance(scores["steps"], int)

{gooddata_eval-1.74.1.dev3 → gooddata_eval-1.74.1.dev5}/tests/test_agentic_visualization.py

@@ -71,8 +71,8 @@ def test_execute_single_run_viz_on_first_turn():
 
     assert result.eval_result.visualization_created is True
     assert result.eval_result.strict_pass is True
-    assert result.total_turns == 1.0
-    assert result.total_steps == 2.0
+    assert result.total_turns == 1
+    assert result.total_steps == 2
     assert result.conversation_id == "conv-1"
     client.send_message.assert_called_once_with("conv-1", "Show revenue")
 
@@ -93,7 +93,7 @@ def test_execute_single_run_clarification_then_viz(monkeypatch):
     result = _execute_single_run(client, "conv-1", "Show me a chart", [_expected()])
 
     assert result.eval_result.visualization_created is True
-    assert result.total_turns == 2.0
+    assert result.total_turns == 2
     assert client.send_message.call_count == 2
     assert client.send_message.call_args_list[1] == call("conv-1", "Revenue please")
 
@@ -111,7 +111,7 @@ def test_execute_single_run_no_viz_no_text():
     result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()])
 
     assert result.eval_result.visualization_created is False
-    assert result.total_turns == 1.0
+    assert result.total_turns == 1
 
 
 def test_execute_single_run_max_iterations_stops_loop(monkeypatch):