gooddata-eval 1.78.1.dev1__py3-none-any.whl → 1.78.1.dev3__py3-none-any.whl

gooddata_eval/cli/agentic_runner.py

@@ -26,7 +26,7 @@ from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualizat
 from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
-from gooddata_eval.core.evaluators.visualization import requires_execution_of
+from gooddata_eval.core.evaluators.visualization import expectation_options
 from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 from gooddata_eval.core.timing import TraceRecorder
@@ -119,6 +119,8 @@ def _parse_visualization_expected(expected_output: Any) -> list[CreatedVisualiza
       {"visualization": {...}} or {"visualization": [{...}]}  <- single/multi candidate
       [{"visualization": {...}}, ...]                          <- bare list
     """
+    if isinstance(expected_output, dict) and expected_output.get("expects_visualization") is False:
+        return []
     if isinstance(expected_output, dict):
         raw_list = expected_output.get("expected_outputs")
         if raw_list is not None:
@@ -187,7 +189,7 @@ def _dispatch_agentic(
             workspace_id=workspace_id,
             question=item.question,
             expected_outputs=_parse_visualization_expected(eo),
-            requires_execution=requires_execution_of(eo),
+            options=expectation_options(eo),
             k=k,
             gate=gate,
             agent_id=agent_id,

gooddata_eval/core/agentic/_catalog.py

@@ -67,6 +67,12 @@ class CatalogMetricAlert:
     """Expected group-by attributes; ``None`` means the fixture states no expectation."""
     granularity: AnomalyDetectionGranularity | None = None
     """Detection interval for an ANOMALY alert (DAY/WEEK/MONTH/...). Not a date filter."""
+    comparison: dict[str, str | int | float | None] | None = None
+    """Expected period comparison as ``create_metric_alert`` argument names; ``None`` means unasserted."""
+    title: str | None = None
+    """Alert name the user asked for; ``None`` means unasserted."""
+    dashboard_id: str | None = None
+    """Dashboard the alert must be bound to; ``None`` means unasserted."""
 
     @classmethod
     def from_dict(cls, d: dict) -> CatalogMetricAlert:
@@ -87,4 +93,7 @@ class CatalogMetricAlert:
             filters=d.get("filters"),
             attributes=d.get("attributes"),
             granularity=AnomalyDetectionGranularity.parse(d.get("granularity")),
+            comparison=d.get("comparison"),
+            title=d.get("title"),
+            dashboard_id=d.get("dashboard_id"),
         )

gooddata_eval/core/agentic/alert_skill.py

@@ -58,6 +58,37 @@ _TRIGGER_DISPLAY_TO_API = {"Every time": "ALWAYS", "One time": "ONCE"}
 _ALWAYS_TRIGGER_VALUES = {"Every time", "ALWAYS", "not specified"}
 _RELATIVE_OPERATORS = ("INCREASES_BY", "DECREASES_BY", "CHANGES_BY")
 
+# Fixture spelling of each period-comparison property -> the create_metric_alert argument.
+_COMPARISON_ARGS = {
+    "kind": "comparison_kind",
+    "periods_ago": "comparison_periods_ago",
+    "date_dataset_id": "comparison_date_dataset_id",
+    "date_attribute_id": "comparison_date_attribute_id",
+    "arithmetic_operator": "arithmetic_operator",
+}
+# Compared verbatim: object ids are case-sensitive, unlike the enum-valued properties.
+_COMPARISON_ID_ARGS = ("comparison_date_dataset_id", "comparison_date_attribute_id")
+# What create_metric_alert applies when the argument is left out, per its own contract. A
+# metric-to-metric comparison has no periods, so periods_ago defaults only for a period kind.
+_COMPARISON_ARG_DEFAULTS: dict[str, object] = {"comparison_periods_ago": 1, "arithmetic_operator": "DIFFERENCE"}
+_PERIOD_COMPARISON_KINDS = ("PREVIOUS_PERIOD", "SAME_PERIOD_PRIOR_YEAR")
+_NO_COMPARISON_MARKERS = ("none", "no comparison")
+
+# AlertEvaluation checks, in report order. Every run is scored on the first group; the second only
+# when the item states the matching expectation.
+_ALERT_CHECKS = (
+    "alert_created",
+    "operator_correct",
+    "threshold_correct",
+    "trigger_correct",
+    "filters_correct",
+    "metric_correct",
+    "recipients_correct",
+    "attributes_correct",
+    "granularity_correct",
+)
+_CONDITIONAL_ALERT_CHECKS = ("comparison_correct", "title_correct", "dashboard_correct")
+
 # Expected-recipient spellings that mean the owner of the API token.
 _CURRENT_USER_RECIPIENT = "me"
 _CURRENT_USER_MARKERS = {"me", "myself", "current user", "current_user"}
@@ -94,6 +125,29 @@ def _parse_recipients(recipients_str: str | None) -> list[str] | None:
     return [r.strip() for r in recipients_str.replace(";", ",").split(",") if r.strip()]
 
 
+def _recipient_list(value: object, *, split_bare: bool) -> list[str]:
+    """A recipient argument as a list, decoded the way gen-ai does before calling the server.
+
+    The tool schema declares both recipient fields ``anyOf: [array of string, string, null]``,
+    and some models send the list JSON-encoded as a string (``'["admin"]'``). Empty or
+    ``"null"`` means none; any other string that is not a JSON list of strings is a bare
+    value -- one user id, or with ``split_bare`` a comma/semicolon-separated email list
+    (gen-ai forwards a bare string as one entry; the split is this checker's own leniency).
+    A non-string list entry is kept as its ``str()`` so it compares unequal instead of raising.
+    """
+    if isinstance(value, list):
+        return [str(item) for item in value]
+    if not isinstance(value, str) or value.strip() in ("", "null"):
+        return []
+    try:
+        parsed = json.loads(value)
+    except ValueError:
+        parsed = None
+    if isinstance(parsed, list) and all(isinstance(item, str) for item in parsed):
+        return parsed
+    return (_parse_recipients(value) or []) if split_bare else [value]
+
+
 def _deep_subset(expected: object, actual: object) -> bool:
     """Return True if expected is a recursive subset of actual."""
     if isinstance(expected, dict) and isinstance(actual, dict):
@@ -234,6 +288,66 @@ def _check_granularity(expected: CatalogMetricAlert, actual_args: dict) -> bool:
     return str(actual).strip().upper() == expected.granularity.value
 
 
+def _canonical_comparison_value(arg: str, value: object) -> str | int | float | None:
+    """Fold one comparison value so fixture and tool-argument spellings compare equal."""
+    if value is None or str(value).strip() == "":
+        return None
+    if arg == "comparison_periods_ago":
+        return _to_number(value)
+    if arg in _COMPARISON_ID_ARGS:
+        return str(value).strip()
+    return str(value).strip().upper()
+
+
+def _check_comparison(expected: CatalogMetricAlert, actual_args: dict) -> bool:
+    """Compare only the comparison properties the fixture states.
+
+    ``None`` means unasserted. A stated ``comparison_kind`` of ``None`` asserts the alert has
+    no comparison at all. ``comparison_kind`` is read as ``create_metric_alert`` resolves it:
+    left out next to a ``compare_metric_id`` it is METRIC. Any other argument the agent left out
+    on an alert that does compare counts as the value ``create_metric_alert`` applies for it, so
+    omitting ``comparison_periods_ago`` matches 1 on a period comparison; a METRIC comparison has
+    no periods, and with no comparison there is nothing to default.
+    The comparison period itself (DAY, MONTH, ...) is not checked here: it is the granularity
+    of the comparison date filter, which ``filters`` asserts.
+    """
+    if expected.comparison is None:
+        return True
+    kind = actual_args.get("comparison_kind") or ("METRIC" if actual_args.get("compare_metric_id") else None)
+    for arg, exp_value in expected.comparison.items():
+        act_raw = kind if arg == "comparison_kind" else actual_args.get(arg)
+        defaults_apply = arg != "comparison_periods_ago" or str(kind).upper() in _PERIOD_COMPARISON_KINDS
+        if act_raw is None and kind is not None and defaults_apply:
+            act_raw = _COMPARISON_ARG_DEFAULTS.get(arg)
+        # Only the fixture side of an object id is trimmed: a padded id is a different id.
+        actual = act_raw if arg in _COMPARISON_ID_ARGS else _canonical_comparison_value(arg, act_raw)
+        if actual != exp_value:
+            return False
+    return True
+
+
+def _check_title(expected: CatalogMetricAlert, actual_args: dict) -> bool:
+    """Exact match on the alert name when the fixture states one.
+
+    Only the fixture side is trimmed: a name saved with padding is not the name the user asked for.
+
+    An omitted ``title`` fails: ``create_metric_alert`` then saves an auto-generated name, even
+    when ``automation_id`` carries a slug of the requested one.
+    """
+    if expected.title is None:
+        return True
+    actual = actual_args.get("title")
+    return isinstance(actual, str) and actual == expected.title
+
+
+def _check_dashboard(expected: CatalogMetricAlert, actual_args: dict) -> bool:
+    """Compare the bound dashboard id when the fixture states one; the actual id verbatim."""
+    if expected.dashboard_id is None:
+        return True
+    actual = actual_args.get("dashboard_id")
+    return isinstance(actual, str) and actual == expected.dashboard_id
+
+
 def _check_metric(expected: CatalogMetricAlert, actual_args: dict) -> bool:
     if not expected.metric_id:
         return True
@@ -305,35 +419,14 @@ def _check_recipients(
 ) -> bool:
     if not expected.recipients:
         return True
-    act_recip_raw = actual_args.get("recipients", actual_args.get("external_recipients"))
-    if isinstance(act_recip_raw, str):
-        # external_recipients is JSON-encoded (e.g. '["email@example.com"]')
-        try:
-            parsed = json.loads(act_recip_raw)
-            act_recip = parsed if isinstance(parsed, list) else _parse_recipients(act_recip_raw)
-        except (json.JSONDecodeError, ValueError):
-            act_recip = _parse_recipients(act_recip_raw)
-    elif isinstance(act_recip_raw, list):
-        act_recip = act_recip_raw
-    else:
-        act_recip = []
-    act_internal_raw = actual_args.get("internal_recipients")
-    # internal_recipients is declared `anyOf: [array of string, string, null]` in the
-    # create_metric_alert tool schema -- a single id as a bare string is schema-legal,
-    # not a malformed call, so it needs the same string/list normalization already
-    # applied to recipients/external_recipients above.
-    if isinstance(act_internal_raw, str):
-        act_internal = [act_internal_raw]
-    elif isinstance(act_internal_raw, list):
-        act_internal = act_internal_raw
-    else:
-        act_internal = []
+    act_recip = _recipient_list(actual_args.get("recipients", actual_args.get("external_recipients")), split_bare=True)
+    act_internal = _recipient_list(actual_args.get("internal_recipients"), split_bare=False)
     wants_current_user, expected_emails = _split_current_user(expected.recipients)
     if wants_current_user:
         if current_user_id is None or current_user_id not in act_internal:
             return False
         other_internal = set(act_internal) - {current_user_id}
-        external = set(act_recip or [])
+        external = set(act_recip)
         if external - set(expected_emails):
             return False
         # An email not sent externally must arrive as its user's id (workspace-restricted channel).
@@ -345,7 +438,7 @@ def _check_recipients(
         # One lookup per email: a bulk lookup returns only the users that exist, hiding a missing one.
         ids_per_email = [_resolve_internal_recipient_ids(sdk, [email]) for email in unmatched]
         return all(ids_per_email) and other_internal == set().union(*ids_per_email)
-    if set(expected_emails) == set(act_recip or []):
+    if set(expected_emails) == set(act_recip):
         return True
     if sdk is not None and act_internal:
         internal_recipient_ids = _resolve_internal_recipient_ids(sdk, expected_emails)
@@ -489,6 +582,15 @@ def generate_simulated_alert_response(
 
     original_request = f'Your original request to the agent was: "{question}"\n' if question else ""
 
+    # Only a stated title is enforced: an unstated one is the agent's to choose, and telling the
+    # sim-user nothing keeps it from objecting to a generated name.
+    title_rule = (
+        f'8. The alert must be named exactly "{expected.title}". If a summary or proposal shows a '
+        "different name, do NOT confirm: state that exact name and ask the agent to use it.\n"
+        if expected.title
+        else ""
+    )
+
     system_prompt = (
         "You are a user requesting creation of an alert for a metric from an AI agent. "
         "Respond naturally but always steer toward the exact values you were given.\n"
@@ -509,7 +611,10 @@ def generate_simulated_alert_response(
         + recipients_rule
         + filters_rule
         + f"6. Proactively state how often you want to be alerted in your first reply: {trigger_request}. "
-        "   Repeat it if the agent proposes a different cadence.\n" + period_rule + "Reply concisely and directly."
+        "   Repeat it if the agent proposes a different cadence.\n"
+        + period_rule
+        + title_rule
+        + "Reply concisely and directly."
     )
 
     messages: list = [{"role": "system", "content": system_prompt}]
@@ -550,22 +655,28 @@ class AlertEvaluation:
     recipients_correct: bool
     attributes_correct: bool = True
     granularity_correct: bool = True
+    # ``None``: the item states no such expectation, so the check is neither scored nor gated.
+    comparison_correct: bool | None = None
+    title_correct: bool | None = None
+    dashboard_correct: bool | None = None
+
+    def checks(self) -> dict[str, bool]:
+        """Every check that ran, by name; strict_pass requires all of them.
+
+        The conditional checks are left out when unasserted, so an item that names no title does
+        not report a ``title_correct`` it was never scored on, and a refusal on it is not counted
+        as a title failure.
+        """
+        ran: dict[str, bool] = {name: getattr(self, name) for name in _ALERT_CHECKS}
+        for name in _CONDITIONAL_ALERT_CHECKS:
+            value = getattr(self, name)
+            if value is not None:
+                ran[name] = value
+        return ran
 
     @property
     def strict_pass(self) -> bool:
-        return all(
-            [
-                self.alert_created,
-                self.operator_correct,
-                self.threshold_correct,
-                self.trigger_correct,
-                self.filters_correct,
-                self.metric_correct,
-                self.recipients_correct,
-                self.attributes_correct,
-                self.granularity_correct,
-            ]
-        )
+        return all(self.checks().values())
 
 
 @dataclass
@@ -666,6 +777,68 @@ def _normalize_expected_attributes(expected: dict) -> list | None:
     raise ValueError(f"Attributes expectation must be a list or a display string, got {type(attributes).__name__}")
 
 
+def _normalize_expected_comparison(expected: dict) -> dict[str, str | int | float | None] | None:
+    """
+    * ``Comparison`` object       -> its stated properties, keyed by tool-argument name
+    * "None" / "no comparison"    -> ``{"comparison_kind": None}`` (stated: no comparison)
+    * absent, empty, or prose     -> ``None`` (unstated; comparison not asserted)
+
+    Each property of the object is asserted only when it carries a value, so an item can pin
+    the comparison kind without also pinning the date dataset the agent may pick.
+    """
+    comparison = _case_insensitive_get(expected, "comparison")
+    if comparison is None:
+        return None
+    if isinstance(comparison, str):
+        return {"comparison_kind": None} if any(kw in comparison.lower() for kw in _NO_COMPARISON_MARKERS) else None
+    if not isinstance(comparison, dict):
+        raise ValueError(f"Comparison expectation must be an object or a display string, got {comparison!r}")
+    stated: dict[str, str | int | float | None] = {}
+    for key, value in comparison.items():
+        name = str(key).strip().lower()
+        arg = _COMPARISON_ARGS.get(name) or (name if name in _COMPARISON_ARGS.values() else None)
+        if arg is None:
+            # Validated here so a misspelled property fails before the run spends an API call.
+            raise ValueError(f"Unrecognised comparison property {key!r}; expected one of {sorted(_COMPARISON_ARGS)}")
+        canonical = _canonical_comparison_value(arg, value)
+        if canonical is None and arg == "comparison_periods_ago" and str(value or "").strip():
+            raise ValueError(f"Comparison periods_ago must be a number, got {value!r}")
+        if canonical is not None:
+            stated[arg] = canonical
+    return stated or None
+
+
+def _normalize_expected_text(expected: dict, *keys: str) -> str | None:
+    """A stated string expectation, or ``None`` when absent or blank (unasserted)."""
+    value = _case_insensitive_get(expected, *keys)
+    if value is None:
+        return None
+    if not isinstance(value, str):
+        raise ValueError(f"{keys[0]} expectation must be a string, got {value!r}")
+    return value.strip() or None
+
+
+def _normalize_expected_dashboard(expected: dict) -> str | None:
+    """
+    * ``dashboard_id``            -> that id
+    * ``Dashboard`` "Title (id)"  -> the id in the trailing parentheses, as ``Metric`` is read
+    * absent / empty              -> ``None`` (unstated; binding not asserted)
+
+    A ``Dashboard`` without a trailing ``(id)`` raises: a title alone can never equal the
+    ``dashboard_id`` the tool receives, so the item would fail on every run.
+    """
+    dashboard_id = _normalize_expected_text(expected, "dashboard_id")
+    if dashboard_id is not None:
+        return dashboard_id
+    display = _normalize_expected_text(expected, "dashboard")
+    if display is None:
+        return None
+    parsed = _parse_metric_id(display)
+    if parsed is None:
+        raise ValueError(f'Dashboard expectation must be "Title (id)", got {display!r}; use dashboard_id for a bare id')
+    return parsed
+
+
 def _normalize_expected_output(expected: dict) -> CatalogMetricAlert:
     """Parse expected_output dict into CatalogMetricAlert, accepting display-format or internal-format keys."""
     operator = _case_insensitive_get(expected, "operator") or "GREATER_THAN"
@@ -696,6 +869,9 @@ def _normalize_expected_output(expected: dict) -> CatalogMetricAlert:
     granularity = AnomalyDetectionGranularity.parse(
         _case_insensitive_get(expected, "granularity", "detection granularity")
     )
+    comparison = _normalize_expected_comparison(expected)
+    title = _normalize_expected_text(expected, "title")
+    dashboard_id = _normalize_expected_dashboard(expected)
 
     return CatalogMetricAlert(
         operator=operator,
@@ -708,6 +884,9 @@ def _normalize_expected_output(expected: dict) -> CatalogMetricAlert:
         filters=filters,
         attributes=attributes,
         granularity=granularity,
+        comparison=comparison,
+        title=title,
+        dashboard_id=dashboard_id,
     )
 
 
@@ -893,6 +1072,13 @@ def run_agentic_alert_skill(
                 and _check_recipients(expected, actual_args, sdk=sdk, current_user_id=current_user_id),
                 attributes_correct=tool_called and _check_attributes(expected, actual_args),
                 granularity_correct=tool_called and _check_granularity(expected, actual_args),
+                comparison_correct=None
+                if expected.comparison is None
+                else tool_called and _check_comparison(expected, actual_args),
+                title_correct=None if expected.title is None else tool_called and _check_title(expected, actual_args),
+                dashboard_correct=None
+                if expected.dashboard_id is None
+                else tool_called and _check_dashboard(expected, actual_args),
             )
             return AlertRunResult(
                 conversation_id=conv_id,
@@ -934,19 +1120,7 @@ def run_agentic_alert_skill(
     pass_power_k = all(r.eval.strict_pass for r in run_results)
     best = max(
         run_results,
-        key=lambda r: sum(
-            [
-                r.eval.alert_created,
-                r.eval.operator_correct,
-                r.eval.threshold_correct,
-                r.eval.trigger_correct,
-                r.eval.filters_correct,
-                r.eval.metric_correct,
-                r.eval.recipients_correct,
-                r.eval.attributes_correct,
-                r.eval.granularity_correct,
-            ]
-        ),
+        key=lambda r: sum(r.eval.checks().values()),
     )
     return AgenticAlertSummary(
         run_results=run_results,
@@ -966,17 +1140,8 @@ def _run_detail(run: AlertRunResult) -> dict:
     Extracted so a failing run is described by exactly the same keys as the winning run --
     for this kind that means the full per-field check breakdown and the arguments the run sent.
     """
-    ev = run.eval
     return {
-        "alert_created": ev.alert_created,
-        "operator_correct": ev.operator_correct,
-        "threshold_correct": ev.threshold_correct,
-        "trigger_correct": ev.trigger_correct,
-        "filters_correct": ev.filters_correct,
-        "metric_correct": ev.metric_correct,
-        "recipients_correct": ev.recipients_correct,
-        "attributes_correct": ev.attributes_correct,
-        "granularity_correct": ev.granularity_correct,
+        **run.eval.checks(),
         "actual_alert_arguments": run.actual_alert_arguments,
         # Why the loop stopped. alert_created=False alone cannot tell a refusal from a run
         # that hit max_iterations while still on track -- see LoopExit. Per-run rather than
@@ -1052,18 +1217,7 @@ def evaluate_agentic_alert_skill(
 
             for run_idx, run in enumerate(summary.run_results):
                 pt = ctx.trace(run.conversation_id)
-                ev = run.eval
-                strict_checks = {
-                    "alert_created": ev.alert_created,
-                    "operator_correct": ev.operator_correct,
-                    "threshold_correct": ev.threshold_correct,
-                    "trigger_correct": ev.trigger_correct,
-                    "filters_correct": ev.filters_correct,
-                    "metric_correct": ev.metric_correct,
-                    "recipients_correct": ev.recipients_correct,
-                    "attributes_correct": ev.attributes_correct,
-                    "granularity_correct": ev.granularity_correct,
-                }
+                strict_checks = run.eval.checks()
                 with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid:
                     for score_name, value in strict_checks.items():
                         ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN")
@@ -1112,13 +1266,8 @@ def evaluate_agentic_alert_skill(
         gate_note = gate_failure_note(gate, runs_passed, runs_effective)
         exc = AlertSkillAssertionError(
             f"Alert skill assertion failed. {gate_note} strict_pass={ev.strict_pass}. "
-            f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, "
-            f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, "
-            f"filters_correct={ev.filters_correct}, metric_correct={ev.metric_correct}, "
-            f"recipients_correct={ev.recipients_correct}, "
-            f"attributes_correct={ev.attributes_correct}, "
-            f"granularity_correct={ev.granularity_correct}. "
-            f"Actual args: {best.actual_alert_arguments}"
+            + ", ".join(f"{name}={value}" for name, value in ev.checks().items())
+            + f". Actual args: {best.actual_alert_arguments}"
         )
         exc.reasoning_steps = best.reasoning_steps
         exc.conversation_id = best.conversation_id

gooddata_eval/core/agentic/dashboard_skill.py

@@ -40,6 +40,7 @@ from gooddata_eval.core.models import (
     build_latency_breakdown,
     shift_and_index_events,
 )
+from gooddata_eval.core.scoring import AAC_VISUALIZATION_TYPES, same_viz_type
 from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings
 
 _DEFAULT_K = 1
@@ -582,6 +583,64 @@ def _check_date_bindings(widgets: list[dict], new_ids: set[str], expected: list[
     return failures
 
 
+def _has_chart_types(expected_output: dict) -> bool:
+    """Whether any expected chart states the type it must be placed as."""
+    return any(isinstance(v, dict) and "chart_type" in v for v in expected_output.get("visualizations") or [])
+
+
+def _bodies_by_id(*references: dict) -> dict[str, dict]:
+    """Every chart body the response's references carry, existing and authored, by chart id."""
+    bodies: dict[str, dict] = {}
+    for refs in references:
+        for key in ("visualizations", "new_visualizations"):
+            for body in refs.get(key) or []:
+                if isinstance(body, dict) and body.get("id"):
+                    bodies[body["id"]] = body
+    return bodies
+
+
+def _check_chart_types(
+    widgets: list[dict], new_ids: set[str], expected: list[dict], bodies: dict[str, dict]
+) -> tuple[list[str], list[str]]:
+    """Every widget of a chart that states a ``chart_type`` must show a chart of exactly that type.
+
+    A widget names only the chart id, so the type is read from the chart body the references
+    carry. The title alone cannot catch a chart authored as another type than the one asked
+    for, which is the silent swap GDAI-2054 rules out.
+
+    Returns the failures and the notes. A widget whose chart has no body in the references
+    cannot be checked: gen-ai carries a body only when it could resolve one, so a missing
+    body is a platform fault, noted as ``_check_references`` notes a missing id, and not
+    scored as the agent's.
+    """
+    failures: list[str] = []
+    notes: list[str] = []
+    for exp in expected:
+        if not isinstance(exp, dict) or "chart_type" not in exp:
+            continue
+        wanted = exp["chart_type"]
+        label = exp.get("title") if exp.get("id") is None else exp.get("id")
+        candidates = _matching_widgets(widgets, new_ids, exp)
+        if not candidates:
+            failures.append(f"cannot check the type of chart {label!r}: it is not on the dashboard")
+            continue
+        types: list[str] = []
+        for widget in candidates:
+            chart_type = (bodies.get(widget.get("visualization")) or {}).get("type")
+            if isinstance(chart_type, str):
+                types.append(chart_type)
+            else:
+                notes.append(
+                    f"cannot check the type of chart {label!r}: the response's references carry no body "
+                    f"for {widget.get('visualization')!r}"
+                )
+        # Every placement, as for date bindings: one right placement must not hide a wrong one.
+        if not all(same_viz_type(wanted, t) for t in types):
+            found = ", ".join(repr(t) for t in types)
+            failures.append(f"chart {label!r} is {found}, expected {wanted!r}")
+    return failures, notes
+
+
 def _tabs_of(dashboard: dict) -> list[dict]:
     """The dashboard's tabs, reading a document with root sections as its single tab."""
     tabs = dashboard.get("tabs")
@@ -936,6 +995,12 @@ def _validate_layout_expectations(expected_output: dict) -> None:
     for chart in expected_output.get("visualizations") or []:
         if isinstance(chart, dict) and "date_dataset" in chart and not _is_dataset_or_null(chart["date_dataset"]):
             raise ValueError(f"date_dataset must name a date dataset or be null, got {chart!r}")
+        if (
+            isinstance(chart, dict)
+            and "chart_type" in chart
+            and (not isinstance(chart["chart_type"], str) or chart["chart_type"] not in AAC_VISUALIZATION_TYPES)
+        ):
+            raise ValueError(f"chart_type must be an AAC visualization type, got {chart!r}")
 
     if "tabs" in expected_output:
         tabs = expected_output["tabs"]
@@ -1037,6 +1102,7 @@ class _Applies:
     tab_filters: bool = False
     date_filters: bool = False
     preserved_widgets: bool = False
+    chart_types: bool = False
 
     @classmethod
     def of(cls, expected_output: dict) -> _Applies:
@@ -1049,6 +1115,7 @@ class _Applies:
             tab_filters="tab_filters" in expected_output,
             date_filters="date_filters" in expected_output,
             preserved_widgets="preserved_widgets" in expected_output,
+            chart_types=_has_chart_types(expected_output),
         )
 
 
@@ -1089,6 +1156,7 @@ class DashboardEvaluation:
     tab_filters_correct: bool = False
     date_filters_correct: bool = False
     widgets_preserved: bool = False
+    chart_types_correct: bool = False
     failures: list[str] = field(default_factory=list)
     notes: list[str] = field(default_factory=list)
 
@@ -1129,6 +1197,8 @@ class DashboardEvaluation:
             checks["dashboard_date_filters_correct"] = self.date_filters_correct
         if self.applies.preserved_widgets:
             checks["dashboard_widgets_preserved"] = self.widgets_preserved
+        if self.applies.chart_types:
+            checks["dashboard_chart_types_correct"] = self.chart_types_correct
         return checks
 
     @property
@@ -1280,12 +1350,14 @@ def evaluate_dashboard_response(
             | {v.get("id") for v in patch_references.get("visualizations") or [] if v.get("id")}
             | new_ids
         )
+        bodies = _bodies_by_id(base_references, patch_references)
     else:
         if actual_saved_id is not None:
             saved_failures.append(f"a new draft must not be saved yet, but it reports id {actual_saved_id!r}")
         document = base
         new_ids = {v.get("id") for v in base_references.get("new_visualizations") or [] if v.get("id")}
         known_ids = {v.get("id") for v in base_references.get("visualizations") or [] if v.get("id")} | new_ids
+        bodies = _bodies_by_id(base_references)
 
     if document is None:
         return DashboardEvaluation(
@@ -1337,6 +1409,11 @@ def evaluate_dashboard_response(
     preserved_failures = (
         _check_preserved_widgets(widgets, expected_output["preserved_widgets"]) if applies.preserved_widgets else []
     )
+    chart_type_failures, chart_type_notes = (
+        _check_chart_types(widgets, new_ids, expected_output.get("visualizations") or [], bodies)
+        if applies.chart_types
+        else ([], [])
+    )
 
     return DashboardEvaluation(
         drafted=True,
@@ -1356,6 +1433,7 @@ def evaluate_dashboard_response(
         tab_filters_correct=not tab_filter_failures,
         date_filters_correct=not date_filter_failures,
         widgets_preserved=not preserved_failures,
+        chart_types_correct=not chart_type_failures,
         failures=[
             *chart_failures,
             *saved_failures,
@@ -1366,9 +1444,10 @@ def evaluate_dashboard_response(
             *tab_filter_failures,
             *date_filter_failures,
             *preserved_failures,
+            *chart_type_failures,
             *new_failures,
         ],
-        notes=[*title_notes, *reference_notes],
+        notes=[*title_notes, *reference_notes, *chart_type_notes],
     )