gooddata-eval 1.72.1.dev2__tar.gz → 1.72.1.dev4__tar.gz

{gooddata_eval-1.72.1.dev2 → gooddata_eval-1.72.1.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.72.1.dev2
+Version: 1.72.1.dev4
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.72.1.dev2
+Requires-Dist: gooddata-sdk~=1.72.1.dev4
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6

{gooddata_eval-1.72.1.dev2 → gooddata_eval-1.72.1.dev4}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.72.1.dev2"
+version = "1.72.1.dev4"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.72.1.dev2",
+    "gooddata-sdk~=1.72.1.dev4",
     "httpx>=0.27,<1.0",
     "orjson>=3.9.15,<4.0.0",
     "pydantic>=2.6,<3.0",

{gooddata_eval-1.72.1.dev2 → gooddata_eval-1.72.1.dev4}/src/gooddata_eval/cli/agentic_runner.py

@@ -11,6 +11,7 @@ from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill
 from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation
 from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question
 from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail
+from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill
 from gooddata_eval.core.agentic.metric_skill import evaluate_agentic_metric_skill
 from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
@@ -38,6 +39,7 @@ AGENTIC_TEST_KINDS = frozenset(
         "agentic_general_question",
         "agentic_guardrail",
         "agentic_conversation",
+        "agentic_kda_skill",
     }
 )
 
@@ -159,6 +161,16 @@ def _dispatch_agentic(
             k=k,
             **lf_kw,
         )
+    elif kind == "agentic_kda_skill":
+        evaluate_agentic_kda_skill(
+            host=host,
+            token=token,
+            workspace_id=workspace_id,
+            question=item.question,
+            expected_output=eo if isinstance(eo, dict) else {},
+            k=k,
+            **lf_kw,
+        )
     elif kind == "agentic_conversation":
         fixture_data = eo.get("fixture") or eo if isinstance(eo, dict) else {}
         evaluate_agentic_conversation(

{gooddata_eval-1.72.1.dev2 → gooddata_eval-1.72.1.dev4}/src/gooddata_eval/core/agentic/__init__.py

@@ -30,6 +30,14 @@ from gooddata_eval.core.agentic.guardrail import (
     evaluate_agentic_guardrail,
     run_agentic_guardrail,
 )
+from gooddata_eval.core.agentic.kda_skill import (
+    AgenticKdaSummary,
+    KdaEvaluation,
+    KdaRunResult,
+    KdaSkillAssertionError,
+    evaluate_agentic_kda_skill,
+    run_agentic_kda_skill,
+)
 from gooddata_eval.core.agentic.metric_skill import (
     AgenticMetricSummary,
     MetricRunResult,
@@ -56,6 +64,7 @@ __all__ = [
     "AgenticAlertSummary",
     "AgenticGeneralQuestionSummary",
     "AgenticGuardrailSummary",
+    "AgenticKdaSummary",
     "AgenticMetricSummary",
     "AgenticSearchSummary",
     "AgenticRunSummary",
@@ -69,6 +78,9 @@ __all__ = [
     "GeneralQuestionResult",
     "GuardrailAssertionError",
     "GuardrailResult",
+    "KdaEvaluation",
+    "KdaRunResult",
+    "KdaSkillAssertionError",
     "MetricRunResult",
     "MetricSkillAssertionError",
     "RunResult",
@@ -81,6 +93,7 @@ __all__ = [
     "evaluate_agentic_conversation",
     "evaluate_agentic_general_question",
     "evaluate_agentic_guardrail",
+    "evaluate_agentic_kda_skill",
     "evaluate_agentic_metric_skill",
     "evaluate_agentic_search_tool",
     "evaluate_agentic_visualization",
@@ -88,6 +101,7 @@ __all__ = [
     "run_agentic_conversation",
     "run_agentic_general_question",
     "run_agentic_guardrail",
+    "run_agentic_kda_skill",
     "run_agentic_metric_skill",
     "run_agentic_search_tool",
     "run_agentic_visualization",

gooddata_eval-1.72.1.dev4/src/gooddata_eval/core/agentic/kda_skill.py

@@ -0,0 +1,422 @@
+# (C) 2026 GoodData Corporation. All rights reserved.
+"""Agentic KDA (Key Driver Analysis)-skill evaluation runner."""
+
+from __future__ import annotations
+
+import logging
+import os
+from dataclasses import dataclass
+
+from gooddata_eval.core.chat.sse_client import ChatClient
+from gooddata_eval.core.config import ReasoningEffort
+from gooddata_eval.core.models import ToolCallEvent
+
+_log = logging.getLogger(__name__)
+
+_DEFAULT_K = 1
+# Disambiguation safety net only (create+execute always run together in the same turn) --
+# 3 real questions' worth (metric, period, +1 slack) since a simulated reply is now sent on
+# every non-final turn (see run_agentic_kda_skill), not just ones classified as a question.
+_DEFAULT_MAX_ITERATIONS = 4
+
+
+def _build_period_hint(expected_output: dict) -> str | None:
+    """Build a period hint from whichever of expected_output's Date Attribute/Analyzed
+    Period/Reference Period are present -- a question about only one of them (e.g. "which
+    date dimension?") must still get an answerable hint, not None just because the other
+    two are absent.
+    """
+    date_attr = expected_output.get("Date Attribute")
+    analyzed = expected_output.get("Analyzed Period")
+    reference_period = expected_output.get("Reference Period")
+    if not (date_attr or analyzed or reference_period):
+        return None
+    parts = []
+    if date_attr:
+        parts.append(date_attr)
+    if analyzed and reference_period:
+        parts.append(f"comparing {analyzed} to {reference_period}")
+    elif analyzed:
+        parts.append(f"period {analyzed}")
+    elif reference_period:
+        parts.append(f"compared to {reference_period}")
+    return ", ".join(parts)
+
+
+def _build_clarification_prompt(
+    agent_message: str, measure_candidates: dict | list[dict] | None, period_hint: str | None
+) -> str:
+    """Build the simulated-user prompt, referencing only whatever candidates/period-hint
+    are actually usable -- an empty/None candidate must drop the "acceptable metric/fact"
+    clause entirely rather than assert a literal "None" as if it were a real option.
+    """
+    candidates = [
+        c for c in (measure_candidates if isinstance(measure_candidates, list) else [measure_candidates]) if c
+    ]
+    reference = ""
+    if candidates:
+        candidate_desc = "; or ".join(
+            f"{c.get('type')} '{c.get('id')}'" + (f" (aggregation {c['aggregation']})" if c.get("aggregation") else "")
+            for c in candidates
+        )
+        reference = f"an acceptable metric/fact is {candidate_desc}"
+    if period_hint:
+        reference = (
+            f"{reference}; the intended time period is {period_hint}"
+            if reference
+            else f"the intended time period is {period_hint}"
+        )
+    return (
+        f"You are simulating a user in a conversation with a BI assistant that runs key driver "
+        f"analysis. The assistant asked: '{agent_message}'. "
+        + (f"For reference, {reference}. " if reference else "")
+        + "Reply briefly as the user, answering whichever of those the assistant actually asked about."
+    )
+
+
+def generate_simulated_kda_response(
+    agent_message: str,
+    measure_candidates: dict | list[dict] | None,
+    period_hint: str | None = None,
+) -> str:
+    """Generate a user reply to keep the KDA-skill conversation going (gpt-4o-mini).
+
+    Called on any turn that didn't trigger KDA, whatever the agent's response actually
+    said -- most often a clarifying question about the measure, the period, or both, so
+    both are given as reference and the reply answers whichever was actually asked.
+    Scope only needs KDA to trigger, not the resulting measure/period to be exactly
+    right. Always OpenAI regardless of the combo's own provider -- this is
+    test-harness plumbing, not the system under test.
+    """
+    try:
+        from openai import OpenAI  # noqa: PLC0415
+    except ImportError as exc:
+        raise RuntimeError("openai package is required for generate_simulated_kda_response") from exc
+
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        raise OSError("OPENAI_API_KEY environment variable is not set")
+
+    client = OpenAI(api_key=api_key)
+    prompt = _build_clarification_prompt(agent_message, measure_candidates, period_hint)
+    response = client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[{"role": "user", "content": prompt}],
+        max_tokens=150,
+        timeout=30,
+    )
+    return response.choices[0].message.content or "Please proceed with either option."
+
+
+def _extract_kda_calls(tool_call_events: list[ToolCallEvent]) -> tuple[dict | None, dict | None]:
+    """Return (create_args, execute_result) for the LAST create/execute pair in this turn's
+    tool calls -- not the last create and last execute picked independently. A new create
+    call clears any earlier execute_result -- it belongs to the create it followed, not to
+    this one.
+    """
+    create_args: dict | None = None
+    execute_result: dict | None = None
+    for tc in tool_call_events:
+        if tc.function_name == "create_key_driver_analysis":
+            create_args = tc.parsed_arguments()
+            execute_result = None
+        elif tc.function_name == "execute_key_driver_analysis" and tc.result:
+            execute_result = tc.parsed_result()
+    return create_args, execute_result
+
+
+@dataclass
+class KdaEvaluation:
+    """Evaluation scores for a single KDA-skill run.
+
+    Scope: asserts only that the KDA process runs to completion -- the tool chain
+    triggers, executes successfully, and the chat turn ends cleanly with a non-empty
+    response (``turn_completed`` requires both gen-ai's stream-ended signal and a
+    non-empty ``text_response`` -- a stream that ends cleanly but delivers nothing to the
+    user isn't a completed turn either).
+    """
+
+    triggered: bool
+    executed: bool
+    success: bool
+    turn_completed: bool
+    disambiguated: bool = False
+
+    @property
+    def strict_pass(self) -> bool:
+        return all([self.triggered, self.executed, self.success, self.turn_completed])
+
+
+@dataclass
+class KdaRunResult:
+    """Outcome of one run (one conversation, up to max_iterations messages) for a KDA case."""
+
+    conversation_id: str
+    evaluation: KdaEvaluation
+    actual_create_args: dict | None
+    actual_execute_result: dict | None
+    # Wall-clock time of the turn that called create (None if create never happened) --
+    # not any earlier disambiguation turn. See run_agentic_kda_skill's _run_once.
+    turn_wall_clock_sec: float | None = None
+
+
+@dataclass
+class AgenticKdaSummary:
+    """Aggregated outcome of K runs for a KDA case."""
+
+    run_results: list[KdaRunResult]
+    pass_at_k: bool
+    pass_power_k: bool
+    best: KdaRunResult
+
+
+def _evaluate_run(
+    create_args: dict | None,
+    execute_result: dict | None,
+    turn_completed: bool,
+    disambiguated: bool = False,
+) -> KdaEvaluation:
+    triggered = create_args is not None
+    executed = execute_result is not None
+    success = executed and execute_result.get("success") is True
+    return KdaEvaluation(
+        triggered=triggered,
+        executed=executed,
+        success=success,
+        turn_completed=turn_completed,
+        disambiguated=disambiguated,
+    )
+
+
+def run_agentic_kda_skill(
+    host: str,
+    token: str,
+    workspace_id: str,
+    question: str,
+    expected_output: dict,
+    k: int = _DEFAULT_K,
+    max_iterations: int = _DEFAULT_MAX_ITERATIONS,
+    initial_conversation_id: str | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
+) -> AgenticKdaSummary:
+    """Run the KDA-skill agentic evaluation K times and return a summary.
+
+    Each run is normally one message, one turn -- create and execute are always called
+    together in the same turn (the skill's own system prompt: "NO confirmation needed").
+    A run only extends past turn 1, up to ``max_iterations``, when the agent's response
+    has no create call and isn't empty; a simulated user reply is then always sent, with
+    no attempt to classify whether the text was actually asking for input (matching
+    visualization.py/alert_skill.py's own break conditions) -- missing a genuine
+    clarifying question hard-fails the run, while sending one after an unrecognized final
+    answer only costs one harmless extra turn, so the asymmetry favors never guessing.
+    """
+    if k < 1:
+        # k=0 or negative would otherwise silently run once, indistinguishable from k=1.
+        raise ValueError(f"k must be >= 1, got {k}")
+    run_results: list[KdaRunResult] = []
+    client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort)
+
+    def _run_once(conv_id: str) -> KdaRunResult:
+        create_args: dict | None = None
+        execute_result: dict | None = None
+        turn_wall_clock_sec: float | None = None
+        turn_completed = False
+        disambiguated = False
+        current_question = question
+
+        for iteration in range(max_iterations):
+            try:
+                chat_result = client.send_message(conv_id, current_question)
+            except Exception as exc:  # noqa: BLE001 -- end this run, not the whole assertion
+                _log.warning("KDA send_message failed for conversation %s: %s", conv_id, exc)
+                partial = getattr(exc, "partial_result", None)
+                if partial is not None:
+                    create_args, execute_result = _extract_kda_calls(partial.tool_call_events or [])
+                    if create_args is not None:
+                        turn_wall_clock_sec = partial.turn_wall_clock_sec
+                turn_completed = False
+                break
+            create_args, execute_result = _extract_kda_calls(chat_result.tool_call_events or [])
+            response_text = (chat_result.text_response or "").strip()
+            turn_completed = chat_result.stream_ended and bool(response_text)
+            if create_args is not None:
+                # This turn's own time -- the turn that called create, not any earlier
+                # disambiguation turn or the simulated-reply generation. create and execute
+                # are always called together in the same turn (or not at all), so this is
+                # final either way -- execute_result may still be None (e.g. the skill's
+                # execute tool isn't available at all when data-sharing is off for the org).
+                turn_wall_clock_sec = chat_result.turn_wall_clock_sec
+                break
+            if not response_text:
+                break
+            if iteration >= max_iterations - 1:
+                break
+            # No text classification -- matches visualization.py/alert_skill.py: break only on
+            # the goal signal (create_args set) or an empty response, otherwise always send a
+            # simulated reply. A false positive (agent had already given a final answer) costs
+            # one harmless extra turn; a false negative (missing a genuine clarifying question)
+            # would hard-fail the run, so the asymmetry favors never trying to tell them apart.
+            measure_candidates = expected_output.get("Measure") if isinstance(expected_output, dict) else None
+            period_hint = _build_period_hint(expected_output) if isinstance(expected_output, dict) else None
+            try:
+                current_question = generate_simulated_kda_response(response_text, measure_candidates, period_hint)
+                disambiguated = True
+            except Exception as exc:  # noqa: BLE001 -- safety net, not the assertion; end only this run
+                _log.warning("Simulated KDA user reply failed for conversation %s: %s", conv_id, exc)
+                break
+
+        ev = _evaluate_run(create_args, execute_result, turn_completed, disambiguated)
+        return KdaRunResult(
+            conversation_id=conv_id,
+            evaluation=ev,
+            actual_create_args=create_args,
+            actual_execute_result=execute_result,
+            turn_wall_clock_sec=turn_wall_clock_sec,
+        )
+
+    try:
+        conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
+        try:
+            run_results.append(_run_once(conv_id_0))
+        finally:
+            if initial_conversation_id is None:  # only delete conversations we created
+                client.delete_conversation(conv_id_0)
+
+        for _ in range(1, k):
+            conv_id = client.create_conversation()
+            try:
+                run_results.append(_run_once(conv_id))
+            finally:
+                client.delete_conversation(conv_id)
+    finally:
+        client.close()
+
+    pass_at_k = any(r.evaluation.strict_pass for r in run_results)
+    pass_power_k = all(r.evaluation.strict_pass for r in run_results)
+    best = max(
+        run_results,
+        key=lambda r: sum(
+            [r.evaluation.triggered, r.evaluation.executed, r.evaluation.success, r.evaluation.turn_completed]
+        ),
+    )
+    return AgenticKdaSummary(
+        run_results=run_results,
+        pass_at_k=pass_at_k,
+        pass_power_k=pass_power_k,
+        best=best,
+    )
+
+
+class KdaSkillAssertionError(AssertionError):
+    """Raised when a KDA-skill evaluation fails."""
+
+    __tracebackhide__ = True
+
+
+def evaluate_agentic_kda_skill(
+    host: str,
+    token: str,
+    workspace_id: str,
+    question: str,
+    expected_output: dict,
+    k: int = _DEFAULT_K,
+    max_iterations: int = _DEFAULT_MAX_ITERATIONS,
+    initial_conversation_id: str | None = None,
+    langfuse: object | None = None,
+    dataset_item_id: str = "",
+    dataset_name: str = "kda_skill",
+    run_timestamp: str | None = None,
+    model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
+    reasoning_effort: ReasoningEffort | None = None,
+) -> None:
+    """Run KDA-skill evaluation, log to Langfuse, and raise KdaSkillAssertionError on failure."""
+    from datetime import datetime as _dt  # noqa: PLC0415
+    from datetime import timezone as _tz  # noqa: PLC0415
+
+    from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client  # noqa: PLC0415
+
+    if langfuse is None:
+        langfuse = try_make_langfuse_client()
+    window_start = _dt.now(_tz.utc)
+    summary = run_agentic_kda_skill(
+        host=host,
+        token=token,
+        workspace_id=workspace_id,
+        question=question,
+        expected_output=expected_output,
+        k=k,
+        max_iterations=max_iterations,
+        initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
+    )
+
+    if langfuse is not None and dataset_item_id:
+        from gooddata_eval.core.agentic._langfuse import (  # noqa: PLC0415
+            build_run_context,
+            find_traces_per_conversation,
+            log_quality_and_value_scores,
+            observe,
+            score_safe,
+        )
+
+        run_name_base, run_metadata = build_run_context(
+            host,
+            token,
+            workspace_id,
+            dataset_name,
+            run_timestamp,
+            model_version_override,
+            run_metadata_extra,
+            reasoning_effort,
+        )
+        # No custom selector -- same default (max-latency) as every other skill; harmless
+        # here since latency comes from run.turn_wall_clock_sec below, not this trace.
+        traces_by_conv = find_traces_per_conversation(
+            langfuse,
+            [r.conversation_id for r in summary.run_results],
+            window_start,
+        )
+        suffix_needed = len(summary.run_results) > 1
+        for run_idx, run in enumerate(summary.run_results):
+            pt = traces_by_conv.get(run.conversation_id)
+            run_name = f"{run_name_base}_run{run_idx}" if suffix_needed else run_name_base
+            ev = run.evaluation
+            # Gates strict_pass -- current scope is completion only (see KdaEvaluation docstring).
+            strict_checks = {
+                "kda_triggered": ev.triggered,
+                "kda_executed": ev.executed,
+                "kda_success": ev.success,
+                "kda_turn_completed": ev.turn_completed,
+            }
+            # Not pt.latency: pt can be any trace of the conversation, not necessarily the KDA turn.
+            turn_wall_clock_sec = run.turn_wall_clock_sec
+            _log.info("[kda-report] %s: strict_pass=%s latency_sec=%s", run_name, ev.strict_pass, turn_wall_clock_sec)
+            with observe(langfuse, pt.id if pt else None, dataset_item_id, run_name, run_metadata) as tid:
+                for score_name, value in strict_checks.items():
+                    score_safe(langfuse, tid, name=score_name, value=float(value), data_type="BOOLEAN")
+                score_safe(langfuse, tid, name="kda_disambiguated", value=float(ev.disambiguated), data_type="BOOLEAN")
+                if turn_wall_clock_sec is not None:
+                    # combo_report.py reads this score directly -- no trace re-resolution needed.
+                    score_safe(
+                        langfuse, tid, name="kda_turn_wall_clock_sec", value=turn_wall_clock_sec, data_type="NUMERIC"
+                    )
+                log_quality_and_value_scores(
+                    langfuse,
+                    tid,
+                    strict_checks=strict_checks,
+                    latency_sec=turn_wall_clock_sec,
+                    cost_usd=pt.total_cost if pt and ev.triggered else None,
+                )
+
+    if not summary.pass_at_k:
+        best = summary.best
+        ev = best.evaluation
+        message = (
+            f"KDA skill assertion failed. strict_pass={ev.strict_pass} "
+            f"(triggered={ev.triggered}, executed={ev.executed}, "
+            f"success={ev.success}, turn_completed={ev.turn_completed}). "
+            f"Actual create args: {best.actual_create_args}. "
+            f"Actual execute result: {best.actual_execute_result}."
+        )
+        raise KdaSkillAssertionError(message)

{gooddata_eval-1.72.1.dev2 → gooddata_eval-1.72.1.dev4}/src/gooddata_eval/core/chat/sse_client.py

@@ -28,18 +28,34 @@ from gooddata_eval.core.models import ChatResult, DatasetItem
 _log = logging.getLogger(__name__)
 
 SSE_DATA_PREFIX = "data: "
+SSE_EVENT_PREFIX = "event: "
+# gen-ai's last event, only if at least one item was already emitted (conversations_controller.py).
+_RESPONSE_ENDED_EVENT = "response_ended"
 
 _RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429, 502, 503, 504})
 _METADATA_SYNC_MARKER = "METADATA_SYNC_IN_PROGRESS"
 
 
 class ChatError(RuntimeError):
-    """Non-retryable error reported by the chat SSE stream."""
+    """Non-retryable error reported by the chat SSE stream.
 
-    def __init__(self, message: str, *, status_code: int | None = None, detail: str | None = None) -> None:
+    ``partial_result`` carries whatever the accumulator captured before the error fired
+    (tool calls included). Callers must not assume it's complete -- fields like
+    ``stream_ended`` reflect the state at the moment of the error, not a finished turn.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        status_code: int | None = None,
+        detail: str | None = None,
+        partial_result: ChatResult | None = None,
+    ) -> None:
         super().__init__(message)
         self.status_code = status_code
         self.detail = detail
+        self.partial_result = partial_result
 
 
 class TransientChatError(ChatError):
@@ -109,6 +125,7 @@ class _SseAccumulator:
     reasoning_steps: list[dict[str, Any]] = field(default_factory=list)
     adhoc_viz_args: list[dict[str, Any]] = field(default_factory=list)
     response_id: str | None = None
+    stream_ended: bool = False
 
 
 def _handle_text(content: dict[str, Any], acc: _SseAccumulator) -> None:
@@ -187,15 +204,37 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult:
         }
     result = ChatResult.model_validate(payload)
     result.response_id = acc.response_id
+    result.stream_ended = acc.stream_ended
     return result
 
 
 def parse_sse_lines(lines: Iterable[str]) -> ChatResult:
     """Parse an SSE stream (iterable of decoded lines) into a ChatResult."""
     acc = _SseAccumulator()
-    for raw_line in lines:
+    current_event = "message"  # SSE default in the absence of an explicit "event: " line
+    it = iter(lines)
+    while True:
+        try:
+            raw_line = next(it)
+        except StopIteration:
+            break
+        except Exception as exc:
+            # Only a transport-level failure (e.g. connection drop mid-stream) is rescued
+            # here -- a bug in the processing below must propagate uncaught, not get
+            # mislabeled as a network error.
+            raise ChatError(f"SSE stream error: {exc}", partial_result=_build_chat_result(acc)) from exc
         line = raw_line.decode("utf-8") if isinstance(raw_line, bytes) else raw_line
-        if not line or line.startswith("event: ") or not line.startswith(SSE_DATA_PREFIX):
+        if not line:
+            current_event = "message"  # blank line ends one event block per the SSE spec
+            continue
+        if line.startswith(SSE_EVENT_PREFIX):
+            current_event = line[len(SSE_EVENT_PREFIX) :].strip()
+            if current_event == _RESPONSE_ENDED_EVENT:
+                acc.stream_ended = True
+            continue
+        if not line.startswith(SSE_DATA_PREFIX):
+            continue
+        if current_event == _RESPONSE_ENDED_EVENT:
             continue
         data_str = line[len(SSE_DATA_PREFIX) :]
         if _METADATA_SYNC_MARKER in data_str:
@@ -203,6 +242,7 @@ def parse_sse_lines(lines: Iterable[str]) -> ChatResult:
                 f"SSE transient error: {_METADATA_SYNC_MARKER}",
                 status_code=None,
                 detail=None,
+                partial_result=_build_chat_result(acc),
             )
         try:
             event_data = json.loads(data_str)
@@ -213,8 +253,10 @@ def parse_sse_lines(lines: Iterable[str]) -> ChatResult:
             detail = event_data.get("detail")
             message = f"SSE error {code}: {detail}"
             if code in _RETRYABLE_STATUS_CODES:
-                raise TransientChatError(message, status_code=code, detail=detail)
-            raise ChatError(message, status_code=code, detail=detail)
+                raise TransientChatError(
+                    message, status_code=code, detail=detail, partial_result=_build_chat_result(acc)
+                )
+            raise ChatError(message, status_code=code, detail=detail, partial_result=_build_chat_result(acc))
         if event_data.get("responseId") and not acc.response_id:
             acc.response_id = event_data["responseId"]
         item = event_data.get("item")
@@ -293,9 +335,20 @@ class ChatClient:
             body["options"] = {"reasoningEffort": self._reasoning_effort}
 
         def _do() -> ChatResult:
+            # Set fresh on every retry attempt (before opening this attempt's stream, so its
+            # own connection setup time counts) -- excludes not just the sleep backoff between
+            # attempts, but the entire duration of any earlier failed attempt.
+            t0 = time.monotonic()
             with self._client.stream("POST", url, json=body, headers=headers) as resp:
                 resp.raise_for_status()
-                return parse_sse_lines(resp.iter_lines())
+                try:
+                    result = parse_sse_lines(resp.iter_lines())
+                except ChatError as exc:
+                    if exc.partial_result is not None:
+                        exc.partial_result.turn_wall_clock_sec = time.monotonic() - t0
+                    raise
+                result.turn_wall_clock_sec = time.monotonic() - t0
+                return result
 
         return _retry_transient(_do, is_retryable=_is_retryable_exc)
 

{gooddata_eval-1.72.1.dev2 → gooddata_eval-1.72.1.dev4}/src/gooddata_eval/core/models.py

@@ -100,6 +100,10 @@ class ChatResult(BaseModel):
     reasoning_step_count: int = Field(default=0, alias="reasoningStepCount")
     conversation_id: str | None = Field(default=None, alias="conversationId")
     response_id: str | None = Field(default=None, alias="responseId")
+    # True once gen-ai's response_ended event arrived.
+    stream_ended: bool = False
+    # Wall-clock seconds for the whole chat turn, timed by the client.
+    turn_wall_clock_sec: float | None = None
 
 
 class SummaryInput(BaseModel):