gooddata-eval 1.73.0__tar.gz → 1.73.1.dev2__tar.gz

{gooddata_eval-1.73.0 → gooddata_eval-1.73.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.73.0
+Version: 1.73.1.dev2
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.73.0
+Requires-Dist: gooddata-sdk~=1.73.1.dev2
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6

{gooddata_eval-1.73.0 → gooddata_eval-1.73.1.dev2}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.73.0"
+version = "1.73.1.dev2"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.73.0",
+    "gooddata-sdk~=1.73.1.dev2",
     "httpx>=0.27,<1.0",
     "orjson>=3.9.15,<4.0.0",
     "pydantic>=2.6,<3.0",

{gooddata_eval-1.73.0 → gooddata_eval-1.73.1.dev2}/src/gooddata_eval/cli/agentic_runner.py

@@ -86,12 +86,12 @@ def _dispatch_agentic(
     model_version_override: str | None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
-) -> AgenticEvalOutcome | list[str] | None:
+) -> AgenticEvalOutcome:
     """Call the appropriate evaluate_agentic_* function for the item's test_kind.
 
-    Returns whatever that function returns -- alert_skill/metric_skill/conversation return
-    an AgenticEvalOutcome; the rest still return None
-    (unchanged).
+    Every evaluate_agentic_* function returns an AgenticEvalOutcome (reasoning_steps,
+    conversation_id, response_id, detail) on success and attaches the same four attributes
+    to its raised *AssertionError on failure -- no kind is exempt.
     """
     kind = item.test_kind
     eo = item.expected_output
@@ -174,13 +174,14 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_kda_skill":
-        evaluate_agentic_kda_skill(
+        return evaluate_agentic_kda_skill(
             host=host,
             token=token,
             workspace_id=workspace_id,
             question=item.question,
             expected_output=eo if isinstance(eo, dict) else {},
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_conversation":
@@ -240,19 +241,22 @@ def run_agentic_items(
                 reasoning_steps = outcome.reasoning_steps
                 conversation_id = outcome.conversation_id
                 response_id = outcome.response_id
+                detail = outcome.detail
             else:
-                reasoning_steps, conversation_id, response_id = outcome, None, None
+                reasoning_steps, conversation_id, response_id, detail = outcome, None, None, {}
             item_report.pass_at_k = True
             item_report.runs = k
             item_report.reasoning_steps = reasoning_steps or []
             item_report.conversation_id = conversation_id
             item_report.response_id = response_id
+            item_report.best_detail = detail or {}
         except AssertionError as exc:
             item_report.pass_at_k = False
             item_report.runs = k
             item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or []
             item_report.conversation_id = getattr(exc, "conversation_id", None)
             item_report.response_id = getattr(exc, "response_id", None)
+            item_report.best_detail = getattr(exc, "detail", None) or {}
             print(f"[agentic] {item.id} FAIL: {exc}", flush=True)
         except Exception as exc:
             item_report.error = f"{type(exc).__name__}: {exc}"

{gooddata_eval-1.73.0 → gooddata_eval-1.73.1.dev2}/src/gooddata_eval/core/agentic/alert_skill.py

@@ -14,7 +14,7 @@ from gooddata_sdk import GoodDataSdk
 from gooddata_eval.core.agentic._catalog import CatalogMetricAlert
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
-from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent
+from gooddata_eval.core.models import AgenticEvalOutcome, ReasoningStepEvent, ToolCallEvent, build_latency_breakdown
 
 try:
     from openai import OpenAI as _OpenAI
@@ -160,8 +160,18 @@ def _check_recipients(expected: CatalogMetricAlert, actual_args: dict, sdk: Good
         act_recip = []
     if set(expected.recipients) == set(act_recip or []):
         return True
-    act_internal = actual_args.get("internal_recipients")
-    if sdk is not None and isinstance(act_internal, list) and act_internal:
+    act_internal_raw = actual_args.get("internal_recipients")
+    # internal_recipients is declared `anyOf: [array of string, string, null]` in the
+    # create_metric_alert tool schema -- a single id as a bare string is schema-legal,
+    # not a malformed call, so it needs the same string/list normalization already
+    # applied to recipients/external_recipients above.
+    if isinstance(act_internal_raw, str):
+        act_internal = [act_internal_raw]
+    elif isinstance(act_internal_raw, list):
+        act_internal = act_internal_raw
+    else:
+        act_internal = []
+    if sdk is not None and act_internal:
         internal_recipient_ids = _resolve_internal_recipient_ids(sdk, expected.recipients)
         if internal_recipient_ids & set(act_internal):
             return True
@@ -335,6 +345,8 @@ class AlertRunResult:
     actual_alert_arguments: dict
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
+    tool_call_events: list[ToolCallEvent] = field(default_factory=list)
+    reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
 
 
 @dataclass
@@ -485,6 +497,11 @@ def run_agentic_alert_skill(
             tool_called = False
             reasoning_steps: list[str] = []
             response_id: str | None = None
+            all_tool_call_events: list[ToolCallEvent] = []
+            all_reasoning_step_events: list[ReasoningStepEvent] = []
+            turn_offset = 0.0  # each turn's call_ts/ts restarts near 0 -- shift by prior turns' wall time
+            tool_index_offset = 0
+            reasoning_index_offset = 0
             # conversation_history stores prior turns for GPT-4o context.
             # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply.
             conversation_history: list = []
@@ -494,6 +511,21 @@ def run_agentic_alert_skill(
                 chat_result = client.send_message(conv_id, current_question)
                 reasoning_steps.extend(chat_result.reasoning_steps or [])
                 response_id = chat_result.response_id or response_id
+                for tc in chat_result.tool_call_events or []:
+                    if tc.call_ts is not None:
+                        tc.call_ts += turn_offset
+                    if tc.result_ts is not None:
+                        tc.result_ts += turn_offset
+                    if tc.index is not None:
+                        tc.index += tool_index_offset
+                for rs in chat_result.reasoning_step_events or []:
+                    rs.ts += turn_offset
+                    rs.index += reasoning_index_offset
+                all_tool_call_events.extend(chat_result.tool_call_events or [])
+                all_reasoning_step_events.extend(chat_result.reasoning_step_events or [])
+                tool_index_offset += len(chat_result.tool_call_events or [])
+                reasoning_index_offset += len(chat_result.reasoning_step_events or [])
+                turn_offset += chat_result.turn_wall_clock_sec or 0.0
                 alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or [])
                 if tool_called:
                     alert_id_to_delete = alert_id
@@ -531,6 +563,8 @@ def run_agentic_alert_skill(
                 actual_alert_arguments=actual_args,
                 reasoning_steps=reasoning_steps,
                 response_id=response_id,
+                tool_call_events=all_tool_call_events,
+                reasoning_step_events=all_reasoning_step_events,
             )
         finally:
             if alert_id_to_delete:
@@ -584,6 +618,7 @@ class AlertSkillAssertionError(AssertionError):
     reasoning_steps: list[str]
     conversation_id: str
     response_id: str | None
+    detail: dict
 
 
 def evaluate_agentic_alert_skill(
@@ -697,9 +732,33 @@ def evaluate_agentic_alert_skill(
         exc.reasoning_steps = best.reasoning_steps
         exc.conversation_id = best.conversation_id
         exc.response_id = best.response_id
+        exc.detail = {
+            "alert_created": ev.alert_created,
+            "operator_correct": ev.operator_correct,
+            "threshold_correct": ev.threshold_correct,
+            "trigger_correct": ev.trigger_correct,
+            "filters_correct": ev.filters_correct,
+            "metric_correct": ev.metric_correct,
+            "recipients_correct": ev.recipients_correct,
+            "actual_alert_arguments": best.actual_alert_arguments,
+            "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
+        }
         raise exc
+    best = summary.best
+    ev = best.eval
     return AgenticEvalOutcome(
-        reasoning_steps=summary.best.reasoning_steps,
-        conversation_id=summary.best.conversation_id,
-        response_id=summary.best.response_id,
+        reasoning_steps=best.reasoning_steps,
+        conversation_id=best.conversation_id,
+        response_id=best.response_id,
+        detail={
+            "alert_created": ev.alert_created,
+            "operator_correct": ev.operator_correct,
+            "threshold_correct": ev.threshold_correct,
+            "trigger_correct": ev.trigger_correct,
+            "filters_correct": ev.filters_correct,
+            "metric_correct": ev.metric_correct,
+            "recipients_correct": ev.recipients_correct,
+            "actual_alert_arguments": best.actual_alert_arguments,
+            "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
+        },
     )

{gooddata_eval-1.73.0 → gooddata_eval-1.73.1.dev2}/src/gooddata_eval/core/agentic/conversation.py

@@ -12,10 +12,16 @@ from gooddata_sdk import GoodDataSdk
 from pydantic import BaseModel
 
 from gooddata_eval.core.agentic.alert_skill import render_alert_proposal
-from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids
+from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids, _extract_metric_result
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
-from gooddata_eval.core.models import AgenticEvalOutcome, ChatResult, ToolCallEvent
+from gooddata_eval.core.models import (
+    AgenticEvalOutcome,
+    ChatResult,
+    ReasoningStepEvent,
+    ToolCallEvent,
+    build_latency_breakdown,
+)
 from gooddata_eval.core.scoring import (
     check_filters,
     check_viz_type,
@@ -120,7 +126,7 @@ def _check_output_present(turn: TurnDefinition, chat_result: ChatResult) -> bool
             and getattr(chat_result.created_visualizations, "objects", chat_result.created_visualizations)
         )
     if otype == "metric":
-        return any(tc.function_name == "create_metric" for tc in (chat_result.tool_call_events or []))
+        return _extract_metric_result(chat_result.tool_call_events or []) is not None
     if otype == "tool_call":
         expected_tool = turn.expected_tool_name
         if not expected_tool:
@@ -129,19 +135,6 @@ def _check_output_present(turn: TurnDefinition, chat_result: ChatResult) -> bool
     return False
 
 
-def _extract_metric_from_turn(tool_call_events: list[ToolCallEvent]) -> dict | None:
-    """Extract the result payload from the create_metric tool call, if present."""
-    for tc in tool_call_events:
-        if tc.function_name != "create_metric":
-            continue
-        if not tc.result:
-            continue
-        result_data = tc.parsed_result()
-        if result_data is not None:
-            return result_data.get("data", result_data)
-    return None
-
-
 def _check_output_correct(turn: TurnDefinition, chat_result: ChatResult) -> bool | None:
     """Check output correctness against expected_output when defined.
 
@@ -186,7 +179,7 @@ def _check_output_correct(turn: TurnDefinition, chat_result: ChatResult) -> bool
         return all(results) if results else None
 
     if otype == "metric":
-        metric_result = _extract_metric_from_turn(chat_result.tool_call_events or [])
+        metric_result = _extract_metric_result(chat_result.tool_call_events or [])
         if not metric_result:
             return False
         return _normalize_maql(metric_result.get("maql", "")) == _normalize_maql(expected.get("maql", ""))
@@ -267,6 +260,8 @@ class ConversationResult:
     total_clarification_turns: int
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
+    tool_call_events: list[ToolCallEvent] = field(default_factory=list)
+    reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
 
 
 def run_agentic_conversation(
@@ -300,6 +295,14 @@ def run_agentic_conversation(
     created_metric_ids: list[str] = []
     reasoning_steps: list[str] = []
     response_id: str | None = None
+    conversation_tool_call_events: list[ToolCallEvent] = []
+    conversation_reasoning_step_events: list[ReasoningStepEvent] = []
+    # Every send_message() call (across every logical turn AND every clarification
+    # sub-turn within it) restarts call_ts/ts near 0 -- these run across the whole
+    # conversation, not reset per logical turn, so every one of those calls shifts them.
+    turn_offset = 0.0
+    tool_index_offset = 0
+    reasoning_index_offset = 0
 
     try:
         if initial_conversation_id is not None:
@@ -335,7 +338,22 @@ def run_agentic_conversation(
             for _iter in range(max_clarification_turns + 1):
                 chat_result = client.send_message(conversation_id, current_message)
                 final_result = chat_result
+                for tc in chat_result.tool_call_events or []:
+                    if tc.call_ts is not None:
+                        tc.call_ts += turn_offset
+                    if tc.result_ts is not None:
+                        tc.result_ts += turn_offset
+                    if tc.index is not None:
+                        tc.index += tool_index_offset
+                for rs in chat_result.reasoning_step_events or []:
+                    rs.ts += turn_offset
+                    rs.index += reasoning_index_offset
                 all_tool_calls.extend(chat_result.tool_call_events or [])
+                conversation_tool_call_events.extend(chat_result.tool_call_events or [])
+                conversation_reasoning_step_events.extend(chat_result.reasoning_step_events or [])
+                tool_index_offset += len(chat_result.tool_call_events or [])
+                reasoning_index_offset += len(chat_result.reasoning_step_events or [])
+                turn_offset += chat_result.turn_wall_clock_sec or 0.0
                 reasoning_steps.extend(chat_result.reasoning_steps or [])
                 response_id = chat_result.response_id or response_id
 
@@ -362,7 +380,7 @@ def run_agentic_conversation(
 
             # Capture metric output for $ref resolution in subsequent turns.
             if final_result and turn.expected_output_type == "metric":
-                metric_data = _extract_metric_from_turn(all_tool_calls)
+                metric_data = _extract_metric_result(all_tool_calls)
                 if metric_data:
                     turn_outputs[turn.turn_id] = metric_data
 
@@ -403,9 +421,30 @@ def run_agentic_conversation(
         total_clarification_turns=total_clarification_turns,
         reasoning_steps=reasoning_steps,
         response_id=response_id,
+        tool_call_events=conversation_tool_call_events,
+        reasoning_step_events=conversation_reasoning_step_events,
     )
 
 
+def _conversation_detail(result: ConversationResult) -> dict:
+    return {
+        "full_skill_coverage": result.full_skill_coverage,
+        "total_clarification_turns": result.total_clarification_turns,
+        "turns": [
+            {
+                "turn_id": tr.turn_id,
+                "expected_skill": tr.expected_skill,
+                "skill_routing": tr.skill_routing,
+                "output_present": tr.output_present,
+                "output_correct": tr.output_correct,
+                "activated_skills": tr.activated_skills,
+            }
+            for tr in result.turn_results
+        ],
+        "latency_breakdown": build_latency_breakdown(result.tool_call_events, result.reasoning_step_events),
+    }
+
+
 class ConversationAssertionError(AssertionError):
     """Raised when a conversation evaluation fails."""
 
@@ -413,6 +452,7 @@ class ConversationAssertionError(AssertionError):
     reasoning_steps: list[str]
     conversation_id: str
     response_id: str | None
+    detail: dict
 
 
 def evaluate_agentic_conversation(
@@ -524,9 +564,11 @@ def evaluate_agentic_conversation(
         exc.reasoning_steps = result.reasoning_steps
         exc.conversation_id = result.conversation_id
         exc.response_id = result.response_id
+        exc.detail = _conversation_detail(result)
         raise exc
     return AgenticEvalOutcome(
         reasoning_steps=result.reasoning_steps,
         conversation_id=result.conversation_id,
         response_id=result.response_id,
+        detail=_conversation_detail(result),
     )

{gooddata_eval-1.73.0 → gooddata_eval-1.73.1.dev2}/src/gooddata_eval/core/agentic/general_question.py

@@ -3,11 +3,12 @@
 
 from __future__ import annotations
 
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import LLMJudge
+from gooddata_eval.core.models import AgenticEvalOutcome
 
 _DEFAULT_K = 1
 
@@ -52,6 +53,8 @@ class GeneralQuestionResult:
     passed: bool
     llm_judge_score: float
     reasoning: str
+    reasoning_steps: list[str] = field(default_factory=list)
+    response_id: str | None = None
 
 
 @dataclass
@@ -98,6 +101,8 @@ def run_agentic_general_question(
                     passed=passed,
                     llm_judge_score=llm_judge_score,
                     reasoning=reasoning,
+                    reasoning_steps=list(chat_result.reasoning_steps or []),
+                    response_id=chat_result.response_id,
                 )
             )
         finally:
@@ -120,6 +125,8 @@ def run_agentic_general_question(
                         passed=passed,
                         llm_judge_score=llm_judge_score,
                         reasoning=reasoning,
+                        reasoning_steps=list(chat_result.reasoning_steps or []),
+                        response_id=chat_result.response_id,
                     )
                 )
             finally:
@@ -142,6 +149,10 @@ class GeneralQuestionAssertionError(AssertionError):
     """Raised when a general-question evaluation fails."""
 
     __tracebackhide__ = True
+    reasoning_steps: list[str]
+    conversation_id: str
+    response_id: str | None
+    detail: dict
 
 
 def evaluate_agentic_general_question(
@@ -160,8 +171,13 @@ def evaluate_agentic_general_question(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> None:
-    """Run general-question evaluation, log to Langfuse, and raise on failure."""
+) -> AgenticEvalOutcome:
+    """Run general-question evaluation, log to Langfuse, and raise GeneralQuestionAssertionError on failure.
+
+    Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an
+    AgenticEvalOutcome on success; on failure the same three values are attached to the
+    raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id``.
+    """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
 
@@ -223,6 +239,26 @@ def evaluate_agentic_general_question(
 
     if not summary.pass_at_k:
         best = summary.best
-        raise GeneralQuestionAssertionError(
+        exc = GeneralQuestionAssertionError(
             f"General question assertion failed. passed={best.passed}. Reasoning: {best.reasoning}"
         )
+        exc.reasoning_steps = best.reasoning_steps
+        exc.conversation_id = best.conversation_id
+        exc.response_id = best.response_id
+        exc.detail = {
+            "judge_passed": best.passed,
+            "judge_reasoning": best.reasoning,
+            "actual_output": best.actual_output,
+        }
+        raise exc
+    best = summary.best
+    return AgenticEvalOutcome(
+        reasoning_steps=best.reasoning_steps,
+        conversation_id=best.conversation_id,
+        response_id=best.response_id,
+        detail={
+            "judge_passed": best.passed,
+            "judge_reasoning": best.reasoning,
+            "actual_output": best.actual_output,
+        },
+    )

{gooddata_eval-1.73.0 → gooddata_eval-1.73.1.dev2}/src/gooddata_eval/core/agentic/guardrail.py

@@ -3,11 +3,12 @@
 
 from __future__ import annotations
 
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import LLMJudge
+from gooddata_eval.core.models import AgenticEvalOutcome, ReasoningStepEvent, ToolCallEvent, build_latency_breakdown
 
 _DEFAULT_K = 1
 
@@ -49,6 +50,10 @@ class GuardrailResult:
     passed: bool
     llm_judge_score: float
     reasoning: str
+    reasoning_steps: list[str] = field(default_factory=list)
+    response_id: str | None = None
+    tool_call_events: list[ToolCallEvent] = field(default_factory=list)
+    reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
 
 
 @dataclass
@@ -95,6 +100,10 @@ def run_agentic_guardrail(
                     passed=passed,
                     llm_judge_score=llm_judge_score,
                     reasoning=reasoning,
+                    reasoning_steps=list(chat_result.reasoning_steps or []),
+                    response_id=chat_result.response_id,
+                    tool_call_events=list(chat_result.tool_call_events or []),
+                    reasoning_step_events=list(chat_result.reasoning_step_events or []),
                 )
             )
         finally:
@@ -117,6 +126,10 @@ def run_agentic_guardrail(
                         passed=passed,
                         llm_judge_score=llm_judge_score,
                         reasoning=reasoning,
+                        reasoning_steps=list(chat_result.reasoning_steps or []),
+                        response_id=chat_result.response_id,
+                        tool_call_events=list(chat_result.tool_call_events or []),
+                        reasoning_step_events=list(chat_result.reasoning_step_events or []),
                     )
                 )
             finally:
@@ -139,6 +152,10 @@ class GuardrailAssertionError(AssertionError):
     """Raised when a guardrail evaluation fails."""
 
     __tracebackhide__ = True
+    reasoning_steps: list[str]
+    conversation_id: str
+    response_id: str | None
+    detail: dict
 
 
 def evaluate_agentic_guardrail(
@@ -157,8 +174,14 @@ def evaluate_agentic_guardrail(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> None:
-    """Run guardrail evaluation, log to Langfuse, and raise on failure."""
+) -> AgenticEvalOutcome:
+    """Run guardrail evaluation, log to Langfuse, and raise GuardrailAssertionError on failure.
+
+    Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an
+    AgenticEvalOutcome on success; on failure the same three values are attached to the
+    raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors
+    `evaluate_agentic_metric_skill`'s idiom) so callers can retrieve them either way.
+    """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
 
@@ -220,4 +243,26 @@ def evaluate_agentic_guardrail(
 
     if not summary.pass_at_k:
         best = summary.best
-        raise GuardrailAssertionError(f"Guardrail assertion failed. passed={best.passed}. Reasoning: {best.reasoning}")
+        exc = GuardrailAssertionError(f"Guardrail assertion failed. passed={best.passed}. Reasoning: {best.reasoning}")
+        exc.reasoning_steps = best.reasoning_steps
+        exc.conversation_id = best.conversation_id
+        exc.response_id = best.response_id
+        exc.detail = {
+            "judge_passed": best.passed,
+            "judge_reasoning": best.reasoning,
+            "actual_output": best.actual_output,
+            "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
+        }
+        raise exc
+    best = summary.best
+    return AgenticEvalOutcome(
+        reasoning_steps=best.reasoning_steps,
+        conversation_id=best.conversation_id,
+        response_id=best.response_id,
+        detail={
+            "judge_passed": best.passed,
+            "judge_reasoning": best.reasoning,
+            "actual_output": best.actual_output,
+            "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
+        },
+    )