gooddata-eval 1.78.0__py3-none-any.whl → 1.78.1.dev2__py3-none-any.whl

gooddata_eval/cli/agentic_runner.py

@@ -26,7 +26,7 @@ from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualizat
 from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
-from gooddata_eval.core.evaluators.visualization import requires_execution_of
+from gooddata_eval.core.evaluators.visualization import expectation_options
 from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 from gooddata_eval.core.timing import TraceRecorder
@@ -119,6 +119,8 @@ def _parse_visualization_expected(expected_output: Any) -> list[CreatedVisualiza
       {"visualization": {...}} or {"visualization": [{...}]}  <- single/multi candidate
       [{"visualization": {...}}, ...]                          <- bare list
     """
+    if isinstance(expected_output, dict) and expected_output.get("expects_visualization") is False:
+        return []
     if isinstance(expected_output, dict):
         raw_list = expected_output.get("expected_outputs")
         if raw_list is not None:
@@ -187,7 +189,7 @@ def _dispatch_agentic(
             workspace_id=workspace_id,
             question=item.question,
             expected_outputs=_parse_visualization_expected(eo),
-            requires_execution=requires_execution_of(eo),
+            options=expectation_options(eo),
             k=k,
             gate=gate,
             agent_id=agent_id,

gooddata_eval/cli/main.py

@@ -7,7 +7,7 @@ import sys
 import threading
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import get_args
+from typing import Any, get_args
 
 import httpx
 from gooddata_api_client.exceptions import ApiException, ApiTypeError
@@ -36,32 +36,31 @@ from gooddata_eval.core.reporting.html_report import load_report_files, write_ht
 from gooddata_eval.core.reporting.json_report import build_multi_model_report, write_multi_model_report
 from gooddata_eval.core.runner import ItemReport, run_items
 from gooddata_eval.core.summary.http_client import SummaryClient
+from gooddata_eval.core.takeaway.http_client import TakeawayClient
 from gooddata_eval.core.timing import TIMERS_ENV_VAR
 from gooddata_eval.core.workspace import ModelResolutionError, WorkspaceModelController
 
 _EXIT_OK = 0
 _EXIT_OPERATIONAL_ERROR = 2
-_SUMMARY_TEST_KIND = "dashboard_summary"
 
 
 class _RoutingBackend:
     """Dispatch each item to the right backend by test_kind.
 
-    `dashboard_summary` items go to the dedicated summary endpoint; everything
-    else uses the conversational chat endpoint.
+    Kinds in `by_kind` go to their dedicated endpoint client (`dashboard_summary` to the
+    summary endpoint, `document_takeaway` to the takeaway endpoint); everything else uses
+    the conversational chat endpoint.
     """
 
-    def __init__(self, chat: ChatClient, summary: SummaryClient):
+    def __init__(self, chat: ChatClient, by_kind: dict[str, Any]):
         self._chat = chat
-        self._summary = summary
+        self._by_kind = by_kind
 
     def ask(self, item: DatasetItem) -> ChatResult:
-        if item.test_kind == _SUMMARY_TEST_KIND:
-            return self._summary.ask(item)
-        return self._chat.ask(item)
+        return self._by_kind.get(item.test_kind, self._chat).ask(item)
 
     def close(self) -> None:
-        for backend in (self._chat, self._summary):
+        for backend in (self._chat, *self._by_kind.values()):
             if hasattr(backend, "close"):
                 backend.close()
 
@@ -614,7 +613,14 @@ def _run(config: RunConfig) -> int:
                     turn_timeout_s=config.turn_timeout_s,
                     item_timeout_s=config.item_timeout_s,
                 ),
-                SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
+                {
+                    "dashboard_summary": SummaryClient(
+                        host=config.host, token=config.token, workspace_id=config.workspace_id
+                    ),
+                    "document_takeaway": TakeawayClient(
+                        host=config.host, token=config.token, workspace_id=config.workspace_id
+                    ),
+                },
             )
             try:
                 single_report = run_items(

gooddata_eval/core/_retry.py

@@ -0,0 +1,71 @@
+# (C) 2026 GoodData Corporation
+"""Bounded exponential-backoff retry shared by the gen-ai HTTP clients.
+
+The budget is read from the ``GOODDATA_EVAL_CHAT_*`` environment on every call, so the chat
+and takeaway clients retune together.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import time
+from collections.abc import Callable
+from typing import TypeVar
+
+_log = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+# 500 is here on evidence, not on principle: in one visualization eval batch it hard-failed
+# 10 of 56 runs with zero retry attempts, and every affected question scored normally when the
+# same question/model ran again the next day -- i.e. transient gen-ai faults, not deterministic
+# server bugs. A genuinely deterministic 500 still terminates, just after the bounded backoff.
+RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429, 500, 502, 503, 504})
+
+
+def int_env(name: str, default: int) -> int:
+    """Read an int from the environment, falling back to ``default`` when unset or blank."""
+    raw = os.getenv(name)
+    return int(raw) if raw else default
+
+
+def float_env(name: str, default: float) -> float:
+    """Read a float from the environment, falling back to ``default`` when unset or blank."""
+    raw = os.getenv(name)
+    return float(raw) if raw else default
+
+
+# Retry budget defaults, giving a ~2 min worst-case cap per send (5/10/20/40/60s).
+# Each is overridable via env so CI can retune without cutting a new release --
+# read per call rather than at import, so an exported value cannot silently
+# rewrite what a test that patches these attributes expects.
+_MAX_RETRIES_DEFAULT = 5
+_INITIAL_BACKOFF_S_DEFAULT = 5.0
+_BACKOFF_FACTOR_DEFAULT = 2.0
+_MAX_BACKOFF_S_DEFAULT = 60.0
+
+
+def retry_transient(operation: Callable[[], T], *, is_retryable: Callable[[Exception], bool]) -> T:
+    """Run ``operation``; retry retryable failures with bounded exponential backoff."""
+    max_retries = int_env("GOODDATA_EVAL_CHAT_MAX_RETRIES", _MAX_RETRIES_DEFAULT)
+    delay = float_env("GOODDATA_EVAL_CHAT_INITIAL_BACKOFF_S", _INITIAL_BACKOFF_S_DEFAULT)
+    factor = float_env("GOODDATA_EVAL_CHAT_BACKOFF_FACTOR", _BACKOFF_FACTOR_DEFAULT)
+    max_backoff = float_env("GOODDATA_EVAL_CHAT_MAX_BACKOFF_S", _MAX_BACKOFF_S_DEFAULT)
+    for attempt in range(max_retries + 1):  # 0..N => N retries + 1 initial attempt
+        try:
+            return operation()
+        except Exception as exc:  # noqa: PERF203 — retry loop: per-attempt try/except is intentional
+            if attempt == max_retries or not is_retryable(exc):
+                raise
+            sleep_s = min(delay, max_backoff)
+            _log.warning(
+                "Transient gen-ai error (attempt %d/%d): %s; retrying in %.0fs",
+                attempt + 1,
+                max_retries + 1,
+                exc,
+                sleep_s,
+            )
+            time.sleep(sleep_s)
+            delay *= factor
+    raise AssertionError("unreachable")  # loop either returns or raises

gooddata_eval/core/agentic/_conversation_context.py

@@ -37,14 +37,36 @@ CONFIRMATION_REPLY = "Yes, go ahead."
 # Asked to pick from a list (which dashboard to bind an alert to), a real user picks one.
 NO_ANSWER_REPLY = "Use your best judgement and go ahead. If you need me to pick from a list, take the first option."
 
-# Part of the metric and alert skills' designed flow, not a clarification.
+# Part of the metric and alert skills' designed flow, not a clarification; so is a retry of a
+# corrected proposal. Keep `use` and `run` out: "Which one should I use?" is a choice question.
+_RETRY_VERBS = r"retry|try\s+(?:(?:it|this|that)\s+)?(?:again|once\s+more)|resubmit"
+_APPROVAL_VERBS = rf"go\s+ahead|proceed|create|save|update|apply|{_RETRY_VERBS}"
+_SHOULD_I = r"\b(?:should|shall|may|can)\s+i\s+"
+_WANT_ME_TO = r"\b(?:do\s+you\s+want|would\s+you\s+like)\s+me\s+to\s+"
 _CONFIRMATION_RE = re.compile(
-    r"\b(?:should|shall|may|can)\s+i\s+(?:go\s+ahead|proceed|create|save|update|apply|add|set\s+(?:it|this)\s+up)"
-    r"|\bdo\s+you\s+want\s+me\s+to\s+(?:go\s+ahead|proceed|create|save|update|apply)"
-    r"|\bwould\s+you\s+like\s+me\s+to\s+(?:go\s+ahead|proceed|create|save|update|apply)"
+    rf"{_SHOULD_I}(?:{_APPROVAL_VERBS}|add|set\s+(?:it|this)\s+up)\b"
+    rf"|{_WANT_ME_TO}(?:{_APPROVAL_VERBS})\b"
     r"|\bplease\s+confirm\b|\bconfirm\s+(?:if|whether|that)\b",
     re.IGNORECASE,
 )
+# "Should I retry with gross or net revenue?" offers a choice, so it stays a question, as does
+# a second option asked right after ("Should I retry? Or adjust the formula first?"). The
+# clause runs to the sentence end: a dot inside a token ("2.5") does not end it. Only a
+# prose "or"/"Or" counts: MAQL writes its OR in uppercase. An "or" that hands the turn back
+# to the user ("..., or would you like to adjust it?", "or not?") is not a second option.
+_NOT_AN_OPTION = (
+    r"not\b|(?:would|do)\s+you\s+(?:like|want|prefer)\s+to\b|(?:is|are)\s+there\b"
+    r"|anything\b|something\b|if\s+you\b|let\s+me\s+know\b"
+)
+_RETRY_CHOICE_RE = re.compile(
+    rf"(?:{_SHOULD_I}|{_WANT_ME_TO})(?:{_RETRY_VERBS})\b(?:[^?.!]|\.(?=\S))*"
+    rf"(?:[?.!]\s+)?(?:(?-i:\b[Oo]r\b)|\balternatively\b)(?!\s+(?:{_NOT_AN_OPTION}))",
+    re.IGNORECASE,
+)
+# Inline code, {type/id} references and quoted literals are the proposal itself; an OR
+# inside a formula is not a choice offered to the user. A single quote counts only outside a
+# word, so the apostrophes in "don't ... it's" do not mask the text between them.
+_FORMULA_SPAN_RE = re.compile(r"`+[^`]*`+|\{[^{}]*\}|\"[^\"\n]*\"|(?<!\w)'[^'\n]*'(?!\w)")
 
 # Checked after _CONFIRMATION_RE, so "please confirm" stays a confirmation.
 _REQUEST_RE = re.compile(
@@ -77,7 +99,7 @@ def classify_reply(chat_result: ChatResult, rendered_text: str) -> ReplyKind:
     text = (rendered_text or "").strip()
     if not text:
         return "no_action"
-    if _CONFIRMATION_RE.search(text):
+    if _CONFIRMATION_RE.search(text) and not _RETRY_CHOICE_RE.search(_FORMULA_SPAN_RE.sub(" ", text)):
         return "confirmation"
     if "?" in text or _REQUEST_RE.search(text):
         return "question"
@@ -238,6 +260,33 @@ class ClarificationJudge:
 
 _SIM_USER_MODEL = "gpt-4o"
 
+# The simulated user plays a business user: choosing the formula is the assistant's job.
+_NO_DEFINITIONS = (
+    "Never write MAQL, SQL, formulas or object definitions. If asked to approve one, reply that it is fine to go ahead."
+)
+# An uppercase SELECT counts once a reference, `(`, `*`, DISTINCT or a call follows it, so
+# "SELECT the second option" and "Select the first one" are left alone; in any case it counts
+# once an aggregation call follows, as in "select sum(revenue)". An
+# aggregation call is a formula even without SELECT: in uppercase ("SUM (units)") or in any
+# case with the parenthesis attached ("sum(units) / count(orders)"), so prose such as
+# "Count (all) is fine" is left alone. Lowercase SQL counts by its shape (`*`, a column list,
+# or a table followed by a clause), so "Select the first option from the list" is left alone.
+_MAQL_RE = re.compile(
+    r"\bSELECT\s+(?:[{<(*]|DISTINCT\b|\w+\s*\()|(?i:\bselect\s+(?:sum|count|avg|min|max|median|distinct)\s*\()"
+    r"|(?i:\bselect\s+(?:\*|[\w.]+(?:\s*,\s*[\w.]+)+)\s+from\b)"
+    r"|(?i:\bselect\b[^?!]*?\bfrom\s+[\w.]+\s+(?:where|group\s+by|order\s+by|join)\b)"
+    r"|\b(?:SUM|COUNT|AVG|MIN|MAX|MEDIAN)\s*\(|(?i:\b(?:sum|count|avg|min|max|median)\()"
+    r"|[{<](?:metric|fact|label|attribute|dataset)/"
+)
+
+
+def _without_maql(reply: str | None) -> str | None:
+    """A reply that slipped MAQL in past the prompt leaves the choice to the assistant.
+
+    Both callers answer a question, so the fallback must answer one too, not just approve.
+    """
+    return NO_ANSWER_REPLY if reply and _MAQL_RE.search(reply) else reply
+
 
 def _chat(prompt_system: str, prompt_user: str) -> str | None:
     try:
@@ -271,12 +320,12 @@ def reply_from_facts(transcript: str, question: str, facts: list[str]) -> str:
         return NO_ANSWER_REPLY
     reply = _chat(
         "You play a business user talking to an analytics assistant. Answer its question briefly, "
-        "using ONLY the facts listed below. Never invent requirements. If the facts do not answer "
-        f"the question, reply exactly: {NO_ANSWER_REPLY}",
+        f"using ONLY the facts listed below. Never invent requirements. {_NO_DEFINITIONS} If the facts "
+        f"do not answer the question, reply exactly: {NO_ANSWER_REPLY}",
         f"Conversation so far:\n{transcript}\n\nThe assistant now asks:\n{question}\n\n"
         "Facts you know:\n" + "\n".join(f"- {f}" for f in facts),
     )
-    return reply or NO_ANSWER_REPLY
+    return _without_maql(reply) or NO_ANSWER_REPLY
 
 
 def reply_restating(transcript: str, question: str) -> str:
@@ -289,7 +338,7 @@ def reply_restating(transcript: str, question: str) -> str:
         "You play a business user talking to an analytics assistant. The assistant just asked for "
         "something that was already said or found earlier in the conversation. Reply briefly, "
         "starting with 'As I said earlier,' and repeat only that information, taken from the "
-        "conversation. Do not add anything new.",
+        f"conversation. Do not add anything new. {_NO_DEFINITIONS}",
         f"Conversation so far:\n{transcript}\n\nThe assistant now asks:\n{question}",
     )
-    return reply or "As I said earlier in this conversation — please use what we already established."
+    return _without_maql(reply) or "As I said earlier in this conversation — please use what we already established."

gooddata_eval/core/agentic/dashboard_skill.py

@@ -40,6 +40,7 @@ from gooddata_eval.core.models import (
     build_latency_breakdown,
     shift_and_index_events,
 )
+from gooddata_eval.core.scoring import AAC_VISUALIZATION_TYPES, same_viz_type
 from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings
 
 _DEFAULT_K = 1
@@ -582,6 +583,64 @@ def _check_date_bindings(widgets: list[dict], new_ids: set[str], expected: list[
     return failures
 
 
+def _has_chart_types(expected_output: dict) -> bool:
+    """Whether any expected chart states the type it must be placed as."""
+    return any(isinstance(v, dict) and "chart_type" in v for v in expected_output.get("visualizations") or [])
+
+
+def _bodies_by_id(*references: dict) -> dict[str, dict]:
+    """Every chart body the response's references carry, existing and authored, by chart id."""
+    bodies: dict[str, dict] = {}
+    for refs in references:
+        for key in ("visualizations", "new_visualizations"):
+            for body in refs.get(key) or []:
+                if isinstance(body, dict) and body.get("id"):
+                    bodies[body["id"]] = body
+    return bodies
+
+
+def _check_chart_types(
+    widgets: list[dict], new_ids: set[str], expected: list[dict], bodies: dict[str, dict]
+) -> tuple[list[str], list[str]]:
+    """Every widget of a chart that states a ``chart_type`` must show a chart of exactly that type.
+
+    A widget names only the chart id, so the type is read from the chart body the references
+    carry. The title alone cannot catch a chart authored as another type than the one asked
+    for, which is the silent swap GDAI-2054 rules out.
+
+    Returns the failures and the notes. A widget whose chart has no body in the references
+    cannot be checked: gen-ai carries a body only when it could resolve one, so a missing
+    body is a platform fault, noted as ``_check_references`` notes a missing id, and not
+    scored as the agent's.
+    """
+    failures: list[str] = []
+    notes: list[str] = []
+    for exp in expected:
+        if not isinstance(exp, dict) or "chart_type" not in exp:
+            continue
+        wanted = exp["chart_type"]
+        label = exp.get("title") if exp.get("id") is None else exp.get("id")
+        candidates = _matching_widgets(widgets, new_ids, exp)
+        if not candidates:
+            failures.append(f"cannot check the type of chart {label!r}: it is not on the dashboard")
+            continue
+        types: list[str] = []
+        for widget in candidates:
+            chart_type = (bodies.get(widget.get("visualization")) or {}).get("type")
+            if isinstance(chart_type, str):
+                types.append(chart_type)
+            else:
+                notes.append(
+                    f"cannot check the type of chart {label!r}: the response's references carry no body "
+                    f"for {widget.get('visualization')!r}"
+                )
+        # Every placement, as for date bindings: one right placement must not hide a wrong one.
+        if not all(same_viz_type(wanted, t) for t in types):
+            found = ", ".join(repr(t) for t in types)
+            failures.append(f"chart {label!r} is {found}, expected {wanted!r}")
+    return failures, notes
+
+
 def _tabs_of(dashboard: dict) -> list[dict]:
     """The dashboard's tabs, reading a document with root sections as its single tab."""
     tabs = dashboard.get("tabs")
@@ -936,6 +995,12 @@ def _validate_layout_expectations(expected_output: dict) -> None:
     for chart in expected_output.get("visualizations") or []:
         if isinstance(chart, dict) and "date_dataset" in chart and not _is_dataset_or_null(chart["date_dataset"]):
             raise ValueError(f"date_dataset must name a date dataset or be null, got {chart!r}")
+        if (
+            isinstance(chart, dict)
+            and "chart_type" in chart
+            and (not isinstance(chart["chart_type"], str) or chart["chart_type"] not in AAC_VISUALIZATION_TYPES)
+        ):
+            raise ValueError(f"chart_type must be an AAC visualization type, got {chart!r}")
 
     if "tabs" in expected_output:
         tabs = expected_output["tabs"]
@@ -1037,6 +1102,7 @@ class _Applies:
     tab_filters: bool = False
     date_filters: bool = False
     preserved_widgets: bool = False
+    chart_types: bool = False
 
     @classmethod
     def of(cls, expected_output: dict) -> _Applies:
@@ -1049,6 +1115,7 @@ class _Applies:
             tab_filters="tab_filters" in expected_output,
             date_filters="date_filters" in expected_output,
             preserved_widgets="preserved_widgets" in expected_output,
+            chart_types=_has_chart_types(expected_output),
         )
 
 
@@ -1089,6 +1156,7 @@ class DashboardEvaluation:
     tab_filters_correct: bool = False
     date_filters_correct: bool = False
     widgets_preserved: bool = False
+    chart_types_correct: bool = False
     failures: list[str] = field(default_factory=list)
     notes: list[str] = field(default_factory=list)
 
@@ -1129,6 +1197,8 @@ class DashboardEvaluation:
             checks["dashboard_date_filters_correct"] = self.date_filters_correct
         if self.applies.preserved_widgets:
             checks["dashboard_widgets_preserved"] = self.widgets_preserved
+        if self.applies.chart_types:
+            checks["dashboard_chart_types_correct"] = self.chart_types_correct
         return checks
 
     @property
@@ -1280,12 +1350,14 @@ def evaluate_dashboard_response(
             | {v.get("id") for v in patch_references.get("visualizations") or [] if v.get("id")}
             | new_ids
         )
+        bodies = _bodies_by_id(base_references, patch_references)
     else:
         if actual_saved_id is not None:
             saved_failures.append(f"a new draft must not be saved yet, but it reports id {actual_saved_id!r}")
         document = base
         new_ids = {v.get("id") for v in base_references.get("new_visualizations") or [] if v.get("id")}
         known_ids = {v.get("id") for v in base_references.get("visualizations") or [] if v.get("id")} | new_ids
+        bodies = _bodies_by_id(base_references)
 
     if document is None:
         return DashboardEvaluation(
@@ -1337,6 +1409,11 @@ def evaluate_dashboard_response(
     preserved_failures = (
         _check_preserved_widgets(widgets, expected_output["preserved_widgets"]) if applies.preserved_widgets else []
     )
+    chart_type_failures, chart_type_notes = (
+        _check_chart_types(widgets, new_ids, expected_output.get("visualizations") or [], bodies)
+        if applies.chart_types
+        else ([], [])
+    )
 
     return DashboardEvaluation(
         drafted=True,
@@ -1356,6 +1433,7 @@ def evaluate_dashboard_response(
         tab_filters_correct=not tab_filter_failures,
         date_filters_correct=not date_filter_failures,
         widgets_preserved=not preserved_failures,
+        chart_types_correct=not chart_type_failures,
         failures=[
             *chart_failures,
             *saved_failures,
@@ -1366,9 +1444,10 @@ def evaluate_dashboard_response(
             *tab_filter_failures,
             *date_filter_failures,
             *preserved_failures,
+            *chart_type_failures,
             *new_failures,
         ],
-        notes=[*title_notes, *reference_notes],
+        notes=[*title_notes, *reference_notes, *chart_type_notes],
     )