gooddata-eval 1.74.1.dev4__py3-none-any.whl → 1.74.1.dev5__py3-none-any.whl

gooddata_eval/core/agentic/alert_skill.py

@@ -477,6 +477,8 @@ class AlertRunResult:
     alert_id: str | None
     eval: AlertEvaluation
     actual_alert_arguments: dict
+    total_turns: int = 0
+    total_steps: int = 0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -676,9 +678,13 @@ def run_agentic_alert_skill(
             # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply.
             conversation_history: list = []
             current_question = question
+            turns = 0
+            steps = 0
 
             for _iteration in range(max_iterations):
                 chat_result = client.send_message(conv_id, current_question)
+                turns += 1
+                steps += chat_result.reasoning_step_count
                 reasoning_steps.extend(chat_result.reasoning_steps or [])
                 response_id = chat_result.response_id or response_id
                 turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
@@ -728,6 +734,8 @@ def run_agentic_alert_skill(
                 alert_id=alert_id,
                 eval=ev,
                 actual_alert_arguments=actual_args,
+                total_turns=turns,
+                total_steps=steps,
                 reasoning_steps=reasoning_steps,
                 response_id=response_id,
                 tool_call_events=all_tool_call_events,
@@ -851,6 +859,8 @@ def evaluate_agentic_alert_skill(
                 with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid:
                     for score_name, value in strict_checks.items():
                         ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN")
+                    ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC")
+                    ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC")
                     log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k)
                     ctx.quality(
                         tid,

gooddata_eval/core/agentic/conversation.py

@@ -336,6 +336,7 @@ class ConversationResult:
     full_skill_coverage: bool
     conversation_success: bool
     total_clarification_turns: int
+    total_steps: int = 0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -365,6 +366,7 @@ def run_agentic_conversation(
     turn_results: list[TurnResult] = []
     turn_outputs: dict[str, dict] = {}
     total_clarification_turns = 0
+    total_steps = 0
     conversation_id: str = ""
     owns_conversation = False
     # Metrics created during this conversation, deleted after it completes so they do
@@ -434,6 +436,7 @@ def run_agentic_conversation(
             for _iter in range(max_clarification_turns + 1):
                 chat_result = client.send_message(conversation_id, current_message)
                 final_result = chat_result
+                total_steps += chat_result.reasoning_step_count
                 turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
                     chat_result,
                     turn_offset=turn_offset,
@@ -522,6 +525,7 @@ def run_agentic_conversation(
         full_skill_coverage=full_skill_coverage,
         conversation_success=conversation_success,
         total_clarification_turns=total_clarification_turns,
+        total_steps=total_steps,
         reasoning_steps=reasoning_steps,
         response_id=response_id,
         tool_call_events=conversation_tool_call_events,
@@ -611,6 +615,22 @@ def evaluate_agentic_conversation(
                     value=float(result.full_skill_coverage),
                     data_type="BOOLEAN",
                 )
+                # One turn per fixture turn, plus every simulated-user round the agent triggered.
+                # The clarification count alone hides how much of the conversation the fixture
+                # asked for, so the comparison needs the total.
+                ctx.score(
+                    tid,
+                    name="turns",
+                    value=len(result.turn_results) + result.total_clarification_turns,
+                    data_type="NUMERIC",
+                )
+                ctx.score(tid, name="steps", value=result.total_steps, data_type="NUMERIC")
+                ctx.score(
+                    tid,
+                    name="clarification_turns",
+                    value=result.total_clarification_turns,
+                    data_type="NUMERIC",
+                )
                 for tr in result.turn_results:
                     ctx.score(
                         tid,

gooddata_eval/core/agentic/kda_skill.py

@@ -185,6 +185,8 @@ class KdaRunResult:
     # Wall-clock time of the turn that called create (None if create never happened) --
     # not any earlier disambiguation turn. See run_agentic_kda_skill's _run_once.
     turn_wall_clock_sec: float | None = None
+    total_turns: int = 0
+    total_steps: int = 0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -276,6 +278,9 @@ def run_agentic_kda_skill(
             all_tool_call_events.extend(result.tool_call_events or [])
             all_reasoning_step_events.extend(result.reasoning_step_events or [])
 
+        turns = 0
+        steps = 0
+
         for iteration in range(max_iterations):
             try:
                 chat_result = client.send_message(conv_id, current_question)
@@ -291,6 +296,8 @@ def run_agentic_kda_skill(
                         turn_wall_clock_sec = partial.turn_wall_clock_sec
                 turn_completed = False
                 break
+            turns += 1
+            steps += chat_result.reasoning_step_count
             reasoning_steps.extend(chat_result.reasoning_steps or [])
             response_id = chat_result.response_id or response_id
             _accumulate(chat_result)
@@ -330,6 +337,8 @@ def run_agentic_kda_skill(
             actual_create_args=create_args,
             actual_execute_result=execute_result,
             turn_wall_clock_sec=turn_wall_clock_sec,
+            total_turns=turns,
+            total_steps=steps,
             reasoning_steps=reasoning_steps,
             response_id=response_id,
             tool_call_events=all_tool_call_events,
@@ -442,6 +451,8 @@ def evaluate_agentic_kda_skill(
                     for score_name, value in strict_checks.items():
                         ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN")
                     ctx.score(tid, name="kda_disambiguated", value=float(ev.disambiguated), data_type="BOOLEAN")
+                    ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC")
+                    ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC")
                     if turn_wall_clock_sec is not None:
                         # combo_report.py reads this score directly -- no trace re-resolution needed.
                         ctx.score(

gooddata_eval/core/agentic/metric_skill.py

@@ -162,7 +162,8 @@ class MetricRunResult:
     metric_created: bool
     actual_maql: str
     maql_correct: bool
-    total_turns: float
+    total_turns: int
+    total_steps: int = 0
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -253,6 +254,7 @@ def _execute_single_metric_run(
     metric_result: dict | None = None
     created_metric_ids: list[str] = []
     turns = 0
+    steps = 0
     current_question = question
     reasoning_steps: list[str] = []
     response_id: str | None = None
@@ -280,6 +282,7 @@ def _execute_single_metric_run(
             )
             all_tool_call_events.extend(chat_result.tool_call_events or [])
             all_reasoning_step_events.extend(chat_result.reasoning_step_events or [])
+            steps += chat_result.reasoning_step_count
             for metric_id in _extract_created_metric_ids(chat_result.tool_call_events or []):
                 if metric_id not in created_metric_ids:
                     created_metric_ids.append(metric_id)
@@ -330,7 +333,8 @@ def _execute_single_metric_run(
             metric_created=metric_created,
             actual_maql=actual_maql,
             maql_correct=maql_correct,
-            total_turns=float(turns),
+            total_turns=turns,
+            total_steps=steps,
             reasoning_steps=reasoning_steps,
             response_id=response_id,
             tool_call_events=all_tool_call_events,
@@ -466,6 +470,8 @@ def evaluate_agentic_metric_skill(
                 ) as tid:
                     ctx.score(tid, name="metric_created", value=float(run.metric_created), data_type="BOOLEAN")
                     ctx.score(tid, name="maql_correct", value=float(run.maql_correct), data_type="BOOLEAN")
+                    ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC")
+                    ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC")
                     log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k)
                     ctx.quality(
                         tid,

gooddata_eval/core/agentic/visualization.py

@@ -59,8 +59,8 @@ class RunResult:
     actual_output: CreatedVisualization | None
     eval_result: EvaluationResult
     best_expected: CreatedVisualization
-    total_turns: float
-    total_steps: float
+    total_turns: int
+    total_steps: int
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
@@ -186,8 +186,8 @@ def _execute_single_run(
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
 ) -> RunResult:
     """Drive one full multi-turn conversation and evaluate the result."""
-    total_turns = 0.0
-    total_steps = 0.0
+    total_turns = 0
+    total_steps = 0
     all_tool_call_events: list[ToolCallEvent] = []
     all_reasoning_step_events: list[ReasoningStepEvent] = []
     reasoning_steps: list[str] = []
@@ -200,8 +200,8 @@ def _execute_single_run(
     current_result = client.send_message(conversation_id, question)
 
     for iteration in range(max_iterations):
-        total_turns += 1.0
-        total_steps += float(current_result.reasoning_step_count)
+        total_turns += 1
+        total_steps += current_result.reasoning_step_count
         turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
             current_result,
             turn_offset=turn_offset,

{gooddata_eval-1.74.1.dev4.dist-info → gooddata_eval-1.74.1.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.74.1.dev4
+Version: 1.74.1.dev5
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.74.1.dev4
+Requires-Dist: gooddata-sdk~=1.74.1.dev5
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6

{gooddata_eval-1.74.1.dev4.dist-info → gooddata_eval-1.74.1.dev5.dist-info}/RECORD

@@ -17,14 +17,14 @@ gooddata_eval/core/agentic/_catalog.py,sha256=bzDGtiXVHdtJSSMCzV8qYp4p_BxmO7f7Gi
 gooddata_eval/core/agentic/_gate.py,sha256=irI4eWOCfmsbKd6l-HdpoAlnWkq2wjo4b5crGU_Er1o,2631
 gooddata_eval/core/agentic/_langfuse.py,sha256=7V1PFQbWpRnFz0a3wklcI-ROYjGC1wGQv2QCfQqpdws,22604
 gooddata_eval/core/agentic/_trace_linker.py,sha256=iXMTxIXUsF5WNSen7v_K8JJ2j1_37lh-2VAGqQxxmkI,13538
-gooddata_eval/core/agentic/alert_skill.py,sha256=AhETD0k2pnEjrHI6G35v3nHkz-qLU28mKczAgkqsfqE,41312
-gooddata_eval/core/agentic/conversation.py,sha256=JUbRZaU6G-MIMfOE1T22LSosRE8Bw3dv-GoxU6V8WXc,29137
+gooddata_eval/core/agentic/alert_skill.py,sha256=NxTQREQmcjfGcAgGLWRzEXlBH1GK5Ay1OHfxsyfLXr4,41747
+gooddata_eval/core/agentic/conversation.py,sha256=Kasie-7Cg7DHNgRvgtaYDWUCGjRSIMpuGIexfkKsWGQ,30079
 gooddata_eval/core/agentic/general_question.py,sha256=Jp3FaS4Tvmfo0dx9iceiZ3Fr1QzJKDl7fVZZDetS22I,14684
 gooddata_eval/core/agentic/guardrail.py,sha256=9TAZk6rudrm5hAUtr4s2pIvkGJitkLfwv6mpdjsr5l0,13056
-gooddata_eval/core/agentic/kda_skill.py,sha256=uPbeYtYHDs1ebt6jwqNo06xOls8-VriAc16gBdgYbrE,22309
-gooddata_eval/core/agentic/metric_skill.py,sha256=R1trDIxqfqbrR836orsYXtUT9l4CtYtOtPOgrQI0gY0,22736
+gooddata_eval/core/agentic/kda_skill.py,sha256=P_pR0iC12b_hIwnLzBmH0IBEjsO_CxZrlfkmu10cVWE,22721
+gooddata_eval/core/agentic/metric_skill.py,sha256=Y6KNCIMyGCh2pBX9O_J3kDvXP5RbOHo2hZdES3WUdBk,23037
 gooddata_eval/core/agentic/search_tool.py,sha256=qZDYpYFLf4-TbpEgVCgAYEUruR9_FG1AYLTKwS8Klbk,11038
-gooddata_eval/core/agentic/visualization.py,sha256=P-2srl3kIi2z7ktxoaLEwOiSFnnfuFI9KISbvtfzc_w,21527
+gooddata_eval/core/agentic/visualization.py,sha256=rBco49wdUhfV5xqe3FnxiNL7jr7WrVKM2aaQE6t6i30,21510
 gooddata_eval/core/chat/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/chat/render.py,sha256=CqsKN8__DlaiJj_bmcPZ1poc5LfUqtz7EzKRDbks4gU,2082
 gooddata_eval/core/chat/sse_client.py,sha256=RqeQA45mCo5MLvRhhyqTQsB9kpU0eUcmaxN9zidA9qo,19417
@@ -56,8 +56,8 @@ gooddata_eval/core/reporting/console.py,sha256=gwewYDusdWX2J9YB1jCUkZcaJjkZ0E1hA
 gooddata_eval/core/reporting/json_report.py,sha256=q3Jirc_Zhvzg5lkG_kM6JdnViNLe6CXX9WpMKuoM6EM,5228
 gooddata_eval/core/summary/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/summary/http_client.py,sha256=dPArJ6zoCu1W9xmYXFA1WMDNsSOkhn3gaCpYgCUp3gk,2174
-gooddata_eval-1.74.1.dev4.dist-info/METADATA,sha256=f31QYSz8ri62wCcSq0wdROHIIYQmDESIzkeHIL46KNE,24240
-gooddata_eval-1.74.1.dev4.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-gooddata_eval-1.74.1.dev4.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
-gooddata_eval-1.74.1.dev4.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
-gooddata_eval-1.74.1.dev4.dist-info/RECORD,,
+gooddata_eval-1.74.1.dev5.dist-info/METADATA,sha256=ZUFZ0MjJv4pHuMs-q7OWOAwM_jjHg73g5b4PZhBTDH4,24240
+gooddata_eval-1.74.1.dev5.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+gooddata_eval-1.74.1.dev5.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
+gooddata_eval-1.74.1.dev5.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
+gooddata_eval-1.74.1.dev5.dist-info/RECORD,,