gooddata-eval 1.77.1.dev2__py3-none-any.whl → 1.78.1.dev1__py3-none-any.whl

gooddata_eval/cli/agentic_runner.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 import sys
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timezone
 from typing import Any, TypedDict
 
 from gooddata_eval.core.agentic._gate import DEFAULT_GATE, EvalGate, normalize_gate
@@ -28,6 +29,7 @@ from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
 from gooddata_eval.core.evaluators.visualization import requires_execution_of
 from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
+from gooddata_eval.core.timing import TraceRecorder
 
 
 class _LfKw(TypedDict, total=False):
@@ -375,6 +377,18 @@ def _apply_failed_runs(item_report: ItemReport, source: Any) -> None:
         item_report.failed_runs = list(failed_runs)
 
 
+def _apply_run_latencies(item_report: ItemReport, source: Any) -> None:
+    """Copy the per-run wall-clock times off an outcome or a failure, if the kind timed them.
+
+    Read from both like ``_apply_failed_runs``: a failing item reaches the report through the
+    exception. A kind that records none leaves the empty default, so the item adds no sample
+    to the latency distribution rather than a zero.
+    """
+    run_latencies = getattr(source, "run_latencies_s", None)
+    if run_latencies:
+        item_report.run_latencies_s = list(run_latencies)
+
+
 def _apply_timings(item_report: ItemReport, timings: Any) -> None:
     """Copy an outcome's phase breakdown onto the item report, if the kind recorded one.
 
@@ -429,6 +443,7 @@ def run_agentic_items(
     # latency. Drained below before this function returns, so every score is written
     # before the caller renders a report or decides an exit code.
     linker = BackgroundTraceLinker()
+    report.started_at = datetime.now(timezone.utc)
     _t0 = time.perf_counter()
 
     def _process_item(index: int, item: DatasetItem) -> ItemReport:
@@ -447,6 +462,10 @@ def run_agentic_items(
         # None, not False, for the kinds _dispatch_agentic passes no gate to: ItemReport.passed
         # then falls back to pass_at_k and gate_passed keeps meaning "a gate ran".
         gated = item.test_kind not in UNGATED_AGENTIC_TEST_KINDS
+        # Started in this worker, where the item runs, so a concurrent item cannot write into it.
+        # Every kind reaches the chat client the same way, which is why the per-trace latencies are
+        # collected here once rather than handed out by each evaluator.
+        traces = TraceRecorder()
         t0 = time.perf_counter()
         try:
             outcome = _dispatch_agentic(
@@ -482,6 +501,7 @@ def run_agentic_items(
             _apply_timings(item_report, getattr(outcome, "timings", None))
             _apply_run_counts(item_report, outcome)
             _apply_failed_runs(item_report, outcome)
+            _apply_run_latencies(item_report, outcome)
         except AssertionError as exc:
             item_report.gate_passed = False if gated else None
             item_report.runs = k
@@ -492,6 +512,7 @@ def run_agentic_items(
             _apply_timings(item_report, getattr(exc, "timings", None))
             _apply_run_counts(item_report, exc)
             _apply_failed_runs(item_report, exc)
+            _apply_run_latencies(item_report, exc)
             # Read off the counts, not off the gate: pass^K fails items where runs did pass,
             # and reporting those as pass_at_k False would contradict the Langfuse score of
             # the same name. Kinds that report no count read as 0, i.e. a clean failure.
@@ -512,6 +533,7 @@ def run_agentic_items(
             _apply_timings(item_report, getattr(exc, "timings", None))
             _apply_run_counts(item_report, exc)
             _apply_failed_runs(item_report, exc)
+            _apply_run_latencies(item_report, exc)
             # Every run ungraded, by definition of this error -- said explicitly rather than
             # left to _apply_run_counts, whose source is detail["unscored_runs"] and which a
             # kind that attaches no diagnostics would leave at 0.
@@ -525,8 +547,15 @@ def run_agentic_items(
             # judge broke should not also report the agent as costing 0s. Kinds that
             # attach no timings to the exception keep their 0.0 defaults.
             _apply_timings(item_report, getattr(exc, "timings", None))
+            # Phase timings stay, but the per-turn response times do not: this item reports no
+            # runs (and so no run latencies), and keeping its turns would put them in the
+            # response population with no attempt beside them. An item that completed 2 of 3
+            # runs and then raised would otherwise show up in the per-response table and not in
+            # the per-conversation one.
+            traces.discard()
         finally:
             item_report.latency_s = time.perf_counter() - t0
+            item_report.trace_latencies_s = traces.stop()
 
         if on_item_done is not None:
             try:
@@ -607,6 +636,7 @@ def run_agentic_items(
             flush=True,
         )
     report.wall_clock_s = time.perf_counter() - _t0
+    report.finished_at = datetime.now(timezone.utc)
 
     if langfuse is not None:
         try:

gooddata_eval/cli/main.py

@@ -7,7 +7,7 @@ import sys
 import threading
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import get_args
+from typing import Any, get_args
 
 import httpx
 from gooddata_api_client.exceptions import ApiException, ApiTypeError
@@ -36,32 +36,31 @@ from gooddata_eval.core.reporting.html_report import load_report_files, write_ht
 from gooddata_eval.core.reporting.json_report import build_multi_model_report, write_multi_model_report
 from gooddata_eval.core.runner import ItemReport, run_items
 from gooddata_eval.core.summary.http_client import SummaryClient
+from gooddata_eval.core.takeaway.http_client import TakeawayClient
 from gooddata_eval.core.timing import TIMERS_ENV_VAR
 from gooddata_eval.core.workspace import ModelResolutionError, WorkspaceModelController
 
 _EXIT_OK = 0
 _EXIT_OPERATIONAL_ERROR = 2
-_SUMMARY_TEST_KIND = "dashboard_summary"
 
 
 class _RoutingBackend:
     """Dispatch each item to the right backend by test_kind.
 
-    `dashboard_summary` items go to the dedicated summary endpoint; everything
-    else uses the conversational chat endpoint.
+    Kinds in `by_kind` go to their dedicated endpoint client (`dashboard_summary` to the
+    summary endpoint, `document_takeaway` to the takeaway endpoint); everything else uses
+    the conversational chat endpoint.
     """
 
-    def __init__(self, chat: ChatClient, summary: SummaryClient):
+    def __init__(self, chat: ChatClient, by_kind: dict[str, Any]):
         self._chat = chat
-        self._summary = summary
+        self._by_kind = by_kind
 
     def ask(self, item: DatasetItem) -> ChatResult:
-        if item.test_kind == _SUMMARY_TEST_KIND:
-            return self._summary.ask(item)
-        return self._chat.ask(item)
+        return self._by_kind.get(item.test_kind, self._chat).ask(item)
 
     def close(self) -> None:
-        for backend in (self._chat, self._summary):
+        for backend in (self._chat, *self._by_kind.values()):
             if hasattr(backend, "close"):
                 backend.close()
 
@@ -614,7 +613,14 @@ def _run(config: RunConfig) -> int:
                     turn_timeout_s=config.turn_timeout_s,
                     item_timeout_s=config.item_timeout_s,
                 ),
-                SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
+                {
+                    "dashboard_summary": SummaryClient(
+                        host=config.host, token=config.token, workspace_id=config.workspace_id
+                    ),
+                    "document_takeaway": TakeawayClient(
+                        host=config.host, token=config.token, workspace_id=config.workspace_id
+                    ),
+                },
             )
             try:
                 single_report = run_items(
@@ -649,6 +655,10 @@ def _run(config: RunConfig) -> int:
                 report.items.extend(agentic_report.items)
             report.items.extend(single_report.items)
             report.wall_clock_s = (agentic_report.wall_clock_s if agentic_report else 0.0) + single_report.wall_clock_s
+            # The two phases run back to back, so the merged run spans the earliest start to the latest finish.
+            phases = [r for r in (agentic_report, single_report) if r is not None]
+            report.started_at = min((r.started_at for r in phases if r.started_at), default=None)
+            report.finished_at = max((r.finished_at for r in phases if r.finished_at), default=None)
 
             skipped_kinds = sorted({i.test_kind for i in report.items if i.skipped})
             if skipped_kinds:

gooddata_eval/core/_retry.py

@@ -0,0 +1,71 @@
+# (C) 2026 GoodData Corporation
+"""Bounded exponential-backoff retry shared by the gen-ai HTTP clients.
+
+The budget is read from the ``GOODDATA_EVAL_CHAT_*`` environment on every call, so the chat
+and takeaway clients retune together.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import time
+from collections.abc import Callable
+from typing import TypeVar
+
+_log = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+# 500 is here on evidence, not on principle: in one visualization eval batch it hard-failed
+# 10 of 56 runs with zero retry attempts, and every affected question scored normally when the
+# same question/model ran again the next day -- i.e. transient gen-ai faults, not deterministic
+# server bugs. A genuinely deterministic 500 still terminates, just after the bounded backoff.
+RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429, 500, 502, 503, 504})
+
+
+def int_env(name: str, default: int) -> int:
+    """Read an int from the environment, falling back to ``default`` when unset or blank."""
+    raw = os.getenv(name)
+    return int(raw) if raw else default
+
+
+def float_env(name: str, default: float) -> float:
+    """Read a float from the environment, falling back to ``default`` when unset or blank."""
+    raw = os.getenv(name)
+    return float(raw) if raw else default
+
+
+# Retry budget defaults, giving a ~2 min worst-case cap per send (5/10/20/40/60s).
+# Each is overridable via env so CI can retune without cutting a new release --
+# read per call rather than at import, so an exported value cannot silently
+# rewrite what a test that patches these attributes expects.
+_MAX_RETRIES_DEFAULT = 5
+_INITIAL_BACKOFF_S_DEFAULT = 5.0
+_BACKOFF_FACTOR_DEFAULT = 2.0
+_MAX_BACKOFF_S_DEFAULT = 60.0
+
+
+def retry_transient(operation: Callable[[], T], *, is_retryable: Callable[[Exception], bool]) -> T:
+    """Run ``operation``; retry retryable failures with bounded exponential backoff."""
+    max_retries = int_env("GOODDATA_EVAL_CHAT_MAX_RETRIES", _MAX_RETRIES_DEFAULT)
+    delay = float_env("GOODDATA_EVAL_CHAT_INITIAL_BACKOFF_S", _INITIAL_BACKOFF_S_DEFAULT)
+    factor = float_env("GOODDATA_EVAL_CHAT_BACKOFF_FACTOR", _BACKOFF_FACTOR_DEFAULT)
+    max_backoff = float_env("GOODDATA_EVAL_CHAT_MAX_BACKOFF_S", _MAX_BACKOFF_S_DEFAULT)
+    for attempt in range(max_retries + 1):  # 0..N => N retries + 1 initial attempt
+        try:
+            return operation()
+        except Exception as exc:  # noqa: PERF203 — retry loop: per-attempt try/except is intentional
+            if attempt == max_retries or not is_retryable(exc):
+                raise
+            sleep_s = min(delay, max_backoff)
+            _log.warning(
+                "Transient gen-ai error (attempt %d/%d): %s; retrying in %.0fs",
+                attempt + 1,
+                max_retries + 1,
+                exc,
+                sleep_s,
+            )
+            time.sleep(sleep_s)
+            delay *= factor
+    raise AssertionError("unreachable")  # loop either returns or raises

gooddata_eval/core/agentic/_conversation_context.py

@@ -37,14 +37,36 @@ CONFIRMATION_REPLY = "Yes, go ahead."
 # Asked to pick from a list (which dashboard to bind an alert to), a real user picks one.
 NO_ANSWER_REPLY = "Use your best judgement and go ahead. If you need me to pick from a list, take the first option."
 
-# Part of the metric and alert skills' designed flow, not a clarification.
+# Part of the metric and alert skills' designed flow, not a clarification; so is a retry of a
+# corrected proposal. Keep `use` and `run` out: "Which one should I use?" is a choice question.
+_RETRY_VERBS = r"retry|try\s+(?:(?:it|this|that)\s+)?(?:again|once\s+more)|resubmit"
+_APPROVAL_VERBS = rf"go\s+ahead|proceed|create|save|update|apply|{_RETRY_VERBS}"
+_SHOULD_I = r"\b(?:should|shall|may|can)\s+i\s+"
+_WANT_ME_TO = r"\b(?:do\s+you\s+want|would\s+you\s+like)\s+me\s+to\s+"
 _CONFIRMATION_RE = re.compile(
-    r"\b(?:should|shall|may|can)\s+i\s+(?:go\s+ahead|proceed|create|save|update|apply|add|set\s+(?:it|this)\s+up)"
-    r"|\bdo\s+you\s+want\s+me\s+to\s+(?:go\s+ahead|proceed|create|save|update|apply)"
-    r"|\bwould\s+you\s+like\s+me\s+to\s+(?:go\s+ahead|proceed|create|save|update|apply)"
+    rf"{_SHOULD_I}(?:{_APPROVAL_VERBS}|add|set\s+(?:it|this)\s+up)\b"
+    rf"|{_WANT_ME_TO}(?:{_APPROVAL_VERBS})\b"
     r"|\bplease\s+confirm\b|\bconfirm\s+(?:if|whether|that)\b",
     re.IGNORECASE,
 )
+# "Should I retry with gross or net revenue?" offers a choice, so it stays a question, as does
+# a second option asked right after ("Should I retry? Or adjust the formula first?"). The
+# clause runs to the sentence end: a dot inside a token ("2.5") does not end it. Only a
+# prose "or"/"Or" counts: MAQL writes its OR in uppercase. An "or" that hands the turn back
+# to the user ("..., or would you like to adjust it?", "or not?") is not a second option.
+_NOT_AN_OPTION = (
+    r"not\b|(?:would|do)\s+you\s+(?:like|want|prefer)\s+to\b|(?:is|are)\s+there\b"
+    r"|anything\b|something\b|if\s+you\b|let\s+me\s+know\b"
+)
+_RETRY_CHOICE_RE = re.compile(
+    rf"(?:{_SHOULD_I}|{_WANT_ME_TO})(?:{_RETRY_VERBS})\b(?:[^?.!]|\.(?=\S))*"
+    rf"(?:[?.!]\s+)?(?:(?-i:\b[Oo]r\b)|\balternatively\b)(?!\s+(?:{_NOT_AN_OPTION}))",
+    re.IGNORECASE,
+)
+# Inline code, {type/id} references and quoted literals are the proposal itself; an OR
+# inside a formula is not a choice offered to the user. A single quote counts only outside a
+# word, so the apostrophes in "don't ... it's" do not mask the text between them.
+_FORMULA_SPAN_RE = re.compile(r"`+[^`]*`+|\{[^{}]*\}|\"[^\"\n]*\"|(?<!\w)'[^'\n]*'(?!\w)")
 
 # Checked after _CONFIRMATION_RE, so "please confirm" stays a confirmation.
 _REQUEST_RE = re.compile(
@@ -77,7 +99,7 @@ def classify_reply(chat_result: ChatResult, rendered_text: str) -> ReplyKind:
     text = (rendered_text or "").strip()
     if not text:
         return "no_action"
-    if _CONFIRMATION_RE.search(text):
+    if _CONFIRMATION_RE.search(text) and not _RETRY_CHOICE_RE.search(_FORMULA_SPAN_RE.sub(" ", text)):
         return "confirmation"
     if "?" in text or _REQUEST_RE.search(text):
         return "question"
@@ -238,6 +260,33 @@ class ClarificationJudge:
 
 _SIM_USER_MODEL = "gpt-4o"
 
+# The simulated user plays a business user: choosing the formula is the assistant's job.
+_NO_DEFINITIONS = (
+    "Never write MAQL, SQL, formulas or object definitions. If asked to approve one, reply that it is fine to go ahead."
+)
+# An uppercase SELECT counts once a reference, `(`, `*`, DISTINCT or a call follows it, so
+# "SELECT the second option" and "Select the first one" are left alone; in any case it counts
+# once an aggregation call follows, as in "select sum(revenue)". An
+# aggregation call is a formula even without SELECT: in uppercase ("SUM (units)") or in any
+# case with the parenthesis attached ("sum(units) / count(orders)"), so prose such as
+# "Count (all) is fine" is left alone. Lowercase SQL counts by its shape (`*`, a column list,
+# or a table followed by a clause), so "Select the first option from the list" is left alone.
+_MAQL_RE = re.compile(
+    r"\bSELECT\s+(?:[{<(*]|DISTINCT\b|\w+\s*\()|(?i:\bselect\s+(?:sum|count|avg|min|max|median|distinct)\s*\()"
+    r"|(?i:\bselect\s+(?:\*|[\w.]+(?:\s*,\s*[\w.]+)+)\s+from\b)"
+    r"|(?i:\bselect\b[^?!]*?\bfrom\s+[\w.]+\s+(?:where|group\s+by|order\s+by|join)\b)"
+    r"|\b(?:SUM|COUNT|AVG|MIN|MAX|MEDIAN)\s*\(|(?i:\b(?:sum|count|avg|min|max|median)\()"
+    r"|[{<](?:metric|fact|label|attribute|dataset)/"
+)
+
+
+def _without_maql(reply: str | None) -> str | None:
+    """A reply that slipped MAQL in past the prompt leaves the choice to the assistant.
+
+    Both callers answer a question, so the fallback must answer one too, not just approve.
+    """
+    return NO_ANSWER_REPLY if reply and _MAQL_RE.search(reply) else reply
+
 
 def _chat(prompt_system: str, prompt_user: str) -> str | None:
     try:
@@ -271,12 +320,12 @@ def reply_from_facts(transcript: str, question: str, facts: list[str]) -> str:
         return NO_ANSWER_REPLY
     reply = _chat(
         "You play a business user talking to an analytics assistant. Answer its question briefly, "
-        "using ONLY the facts listed below. Never invent requirements. If the facts do not answer "
-        f"the question, reply exactly: {NO_ANSWER_REPLY}",
+        f"using ONLY the facts listed below. Never invent requirements. {_NO_DEFINITIONS} If the facts "
+        f"do not answer the question, reply exactly: {NO_ANSWER_REPLY}",
         f"Conversation so far:\n{transcript}\n\nThe assistant now asks:\n{question}\n\n"
         "Facts you know:\n" + "\n".join(f"- {f}" for f in facts),
     )
-    return reply or NO_ANSWER_REPLY
+    return _without_maql(reply) or NO_ANSWER_REPLY
 
 
 def reply_restating(transcript: str, question: str) -> str:
@@ -289,7 +338,7 @@ def reply_restating(transcript: str, question: str) -> str:
         "You play a business user talking to an analytics assistant. The assistant just asked for "
         "something that was already said or found earlier in the conversation. Reply briefly, "
         "starting with 'As I said earlier,' and repeat only that information, taken from the "
-        "conversation. Do not add anything new.",
+        f"conversation. Do not add anything new. {_NO_DEFINITIONS}",
         f"Conversation so far:\n{transcript}\n\nThe assistant now asks:\n{question}",
     )
-    return reply or "As I said earlier in this conversation — please use what we already established."
+    return _without_maql(reply) or "As I said earlier in this conversation — please use what we already established."

gooddata_eval/core/agentic/alert_skill.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 import json
 import os
 import re
+import time
 from dataclasses import dataclass, field
 from typing import Any
 
@@ -587,6 +588,8 @@ class AlertRunResult:
     # also report operator/threshold/metric/recipients as False.
     exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED
     turns_used: int = 0
+    # This run's own wall time, from just before its chat call to the end of its scoring.
+    run_latency_s: float = 0.0
 
 
 @dataclass
@@ -802,6 +805,7 @@ def run_agentic_alert_skill(
             # that simply runs out of range() is correctly labelled without a trailing else.
             exit_reason = LoopExit.BUDGET_EXHAUSTED
             turns_used = 0
+            run_started = time.monotonic()
             for _iteration in range(max_iterations):
                 turns_used = _iteration + 1
                 try:
@@ -903,6 +907,7 @@ def run_agentic_alert_skill(
                 reasoning_step_events=all_reasoning_step_events,
                 exit_reason=exit_reason,
                 turns_used=turns_used,
+                run_latency_s=time.monotonic() - run_started,
             )
         finally:
             if alert_id_to_delete:
@@ -1122,6 +1127,7 @@ def evaluate_agentic_alert_skill(
         exc.runs_passed = runs_passed
         exc.runs_effective = runs_effective
         exc.failed_runs = failed_runs
+        exc.run_latencies_s = [r.run_latency_s for r in summary.run_results]
         raise exc
     return AgenticEvalOutcome(
         runs_passed=runs_passed,
@@ -1133,4 +1139,5 @@ def evaluate_agentic_alert_skill(
         # Also on the success path: pass@K clears the gate with one passing run, so a
         # 1/3 item reports success while two of its runs failed for reasons worth reading.
         failed_runs=failed_runs,
+        run_latencies_s=[r.run_latency_s for r in summary.run_results],
     )

gooddata_eval/core/agentic/anomaly_detection.py

@@ -28,6 +28,7 @@ from __future__ import annotations
 import logging
 import os
 import re
+import time
 from dataclasses import dataclass, field
 from typing import Any
 
@@ -284,6 +285,8 @@ class AnomalyRunResult:
     response_id: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
     reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
+    # This run's own wall time, from just before its chat call to the end of its scoring.
+    run_latency_s: float = 0.0
 
 
 @dataclass
@@ -393,6 +396,7 @@ def run_agentic_anomaly_detection(
             all_tool_call_events.extend(result.tool_call_events or [])
             all_reasoning_step_events.extend(result.reasoning_step_events or [])
 
+        run_started = time.monotonic()
         for iteration in range(max_iterations):
             try:
                 chat_result = client.send_message(conv_id, current_question)
@@ -440,6 +444,7 @@ def run_agentic_anomaly_detection(
             response_id=response_id,
             tool_call_events=all_tool_call_events,
             reasoning_step_events=all_reasoning_step_events,
+            run_latency_s=time.monotonic() - run_started,
         )
 
     try:
@@ -651,6 +656,7 @@ def evaluate_agentic_anomaly_detection(
         exc.runs_passed = runs_passed
         exc.runs_effective = len(summary.run_results)
         exc.failed_runs = failed_runs
+        exc.run_latencies_s = [r.run_latency_s for r in summary.run_results]
         raise exc
 
     return AgenticEvalOutcome(
@@ -661,4 +667,5 @@ def evaluate_agentic_anomaly_detection(
         response_id=best.response_id,
         detail=detail,
         failed_runs=failed_runs,
+        run_latencies_s=[r.run_latency_s for r in summary.run_results],
     )

gooddata_eval/core/agentic/conversation.py

@@ -19,6 +19,7 @@ from __future__ import annotations
 import json
 import os
 import re
+import time
 from dataclasses import dataclass, field
 from datetime import date
 from typing import ClassVar, Literal
@@ -612,6 +613,9 @@ class ConversationResult:
     stalled_turns: int = 0
     mode: ConversationMode = "legacy"
     judge_model: str | None = None
+    # The whole conversation's wall time, from just before its first message to the end of its
+    # last turn's scoring; conversation create/delete is not part of it.
+    run_latency_s: float = 0.0
 
 
 def _metric_creations(tool_call_events: list[ToolCallEvent]) -> list[tuple[str, bool]]:
@@ -771,6 +775,7 @@ def run_agentic_conversation(
     tool_index_offset = 0
     reasoning_index_offset = 0
     current_context = user_context
+    run_latency_s = 0.0
 
     try:
         if initial_conversation_id is not None:
@@ -779,10 +784,14 @@ def run_agentic_conversation(
             conversation_id = client.create_conversation()
             owns_conversation = True
 
+        run_started = time.monotonic()
+        conversation_swap_s = 0.0  # fresh_conversation_per_turn's delete/create, which is not the run's work
         for turn_index, turn in enumerate(fixture.turns):
             if fresh_conversation_per_turn and turn_index > 0:
+                swap_started = time.monotonic()
                 client.delete_conversation(conversation_id)
                 conversation_id = client.create_conversation()
+                conversation_swap_s += time.monotonic() - swap_started
                 transcript = []
                 active_skills = set()
             # Before the skip below: a turn whose expectation cannot resolve still changed what
@@ -1006,6 +1015,8 @@ def run_agentic_conversation(
                 )
             )
 
+        # Taken here, not after the finally: its cleanup is not part of the run.
+        run_latency_s = time.monotonic() - run_started - conversation_swap_s
     finally:
         if owns_conversation and conversation_id:
             client.delete_conversation(conversation_id)
@@ -1039,6 +1050,7 @@ def run_agentic_conversation(
         stalled_turns=stalled,
         mode=resolved_mode,
         judge_model=judge.model_name if judged else None,
+        run_latency_s=run_latency_s,
     )
 
 
@@ -1246,6 +1258,7 @@ def evaluate_agentic_conversation(
         # for. Saying so explicitly stops the report claiming K runs that never happened.
         exc.runs_passed = 0
         exc.runs_effective = 1
+        exc.run_latencies_s = [result.run_latency_s]
         raise exc
     return AgenticEvalOutcome(
         reasoning_steps=result.reasoning_steps,
@@ -1254,4 +1267,5 @@ def evaluate_agentic_conversation(
         detail=detail,
         runs_passed=1,
         runs_effective=1,
+        run_latencies_s=[result.run_latency_s],
     )

gooddata_eval/core/agentic/dashboard_skill.py

@@ -1167,6 +1167,8 @@ class DashboardRunResult:
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
     reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
     timings: PhaseTimings = field(default_factory=PhaseTimings)
+    # This run's own wall time, from just before its chat call to the end of its scoring.
+    run_latency_s: float = 0.0
 
 
 @dataclass
@@ -1399,6 +1401,7 @@ def _execute_single_dashboard_run(
     tool_index_offset = 0
     reasoning_index_offset = 0
 
+    run_started = time.monotonic()
     for iteration in range(max_iterations):
         turns += 1
         agent_started = time.monotonic()
@@ -1475,6 +1478,7 @@ def _execute_single_dashboard_run(
         tool_call_events=all_tool_call_events,
         reasoning_step_events=all_reasoning_step_events,
         timings=timings,
+        run_latency_s=time.monotonic() - run_started,
     )
 
 
@@ -1703,6 +1707,7 @@ def evaluate_agentic_dashboard_skill(
         exc.runs_passed = runs_passed
         exc.runs_effective = runs_effective
         exc.failed_runs = failed_runs
+        exc.run_latencies_s = [r.run_latency_s for r in summary.run_results]
         raise exc
     return AgenticEvalOutcome(
         runs_passed=runs_passed,
@@ -1713,4 +1718,5 @@ def evaluate_agentic_dashboard_skill(
         detail=detail,
         timings=item_timings,
         failed_runs=failed_runs,
+        run_latencies_s=[r.run_latency_s for r in summary.run_results],
     )

gooddata_eval/core/agentic/document_skill.py

@@ -638,6 +638,8 @@ class DocumentRunResult:
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
     reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
     timings: PhaseTimings = field(default_factory=PhaseTimings)
+    # This run's own wall time, from just before its chat call to the end of its scoring.
+    run_latency_s: float = 0.0
 
     @property
     def diagnostics(self) -> dict[str, bool]:
@@ -711,6 +713,7 @@ def _execute_single_document_run(
     reasoning_index_offset = 0
     first_turn_asked = False
 
+    run_started = time.monotonic()
     for iteration in range(max_iterations):
         turns += 1
         agent_started = time.monotonic()
@@ -781,6 +784,7 @@ def _execute_single_document_run(
         tool_call_events=all_tool_call_events,
         reasoning_step_events=all_reasoning_step_events,
         timings=timings,
+        run_latency_s=time.monotonic() - run_started,
     )
 
 
@@ -980,6 +984,7 @@ def evaluate_agentic_document_skill(
             + " | ".join(unscored)
         )
         exc_judge.timings = item_timings
+        exc_judge.run_latencies_s = [r.run_latency_s for r in summary.run_results]
         raise exc_judge
 
     runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass)
@@ -1033,6 +1038,7 @@ def evaluate_agentic_document_skill(
         exc.runs_passed = runs_passed
         exc.runs_effective = runs_effective
         exc.failed_runs = failed_runs
+        exc.run_latencies_s = [r.run_latency_s for r in summary.run_results]
         raise exc
     return AgenticEvalOutcome(
         runs_passed=runs_passed,
@@ -1043,4 +1049,5 @@ def evaluate_agentic_document_skill(
         detail=detail,
         timings=item_timings,
         failed_runs=failed_runs,
+        run_latencies_s=[r.run_latency_s for r in summary.run_results],
     )

gooddata_eval/core/agentic/general_question.py

@@ -91,6 +91,8 @@ class GeneralQuestionResult:
     judge_error: str | None = None
     tool_call_events: list[ToolCallEvent] = field(default_factory=list)
     reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
+    # This run's own wall time, from just before its chat call to the end of its scoring.
+    run_latency_s: float = 0.0
 
 
 @dataclass
@@ -153,6 +155,9 @@ def _run_single_general_question(
         judge_error=verdict.error,
         tool_call_events=list(chat_result.tool_call_events or []),
         reasoning_step_events=list(chat_result.reasoning_step_events or []),
+        # item_started is taken just before the chat call and total_elapsed right after
+        # scoring, so this is the whole run without a second clock read.
+        run_latency_s=total_elapsed,
     )
 
 
@@ -224,6 +229,7 @@ def _attach_diagnostics(
     failed_runs: list[dict],
     runs_passed: int,
     runs_effective: int,
+    run_latencies_s: list[float],
 ) -> None:
     """Hang the report payload on a raised error, whichever error it is.
 
@@ -238,6 +244,7 @@ def _attach_diagnostics(
     error.runs_passed = runs_passed
     error.runs_effective = runs_effective
     error.failed_runs = failed_runs
+    error.run_latencies_s = run_latencies_s
 
 
 def _run_detail(run: GeneralQuestionResult) -> dict:
@@ -363,6 +370,7 @@ def evaluate_agentic_general_question(
     # `runs_passed` is then 0, so nothing here needs a graded run to exist.
     runs_passed = sum(1 for r in summary.scored_run_results if r.passed)
     runs_effective = len(summary.run_results)
+    run_latencies_s = [r.run_latency_s for r in summary.run_results]
 
     best = summary.best
     detail = {
@@ -389,7 +397,7 @@ def evaluate_agentic_general_question(
             f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): "
             + " | ".join(unscored)
         )
-        _attach_diagnostics(error, best, detail, failed_runs, runs_passed, runs_effective)
+        _attach_diagnostics(error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s)
         error.timings = item_timings
         raise error
 
@@ -398,7 +406,7 @@ def evaluate_agentic_general_question(
         exc = GeneralQuestionAssertionError(
             f"General question assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}"
         )
-        _attach_diagnostics(exc, best, detail, failed_runs, runs_passed, runs_effective)
+        _attach_diagnostics(exc, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s)
         exc.timings = item_timings
         raise exc
     return AgenticEvalOutcome(
@@ -412,4 +420,5 @@ def evaluate_agentic_general_question(
         # Also on the success path: pass@K clears the gate with one passing run, so a
         # 1/3 item reports success while two of its runs failed for reasons worth reading.
         failed_runs=failed_runs,
+        run_latencies_s=run_latencies_s,
     )