gooddata-eval 1.75.1.dev2__py3-none-any.whl → 1.75.1.dev3__py3-none-any.whl

gooddata_eval/core/agentic/dashboard_skill.py

@@ -7,6 +7,8 @@ import time
 from dataclasses import dataclass, field
 from typing import Any
 
+import jsonpatch
+
 from gooddata_eval.core.agentic._gate import (
     DEFAULT_GATE,
     EvalGate,
@@ -41,11 +43,14 @@ from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings
 _DEFAULT_K = 1
 # Matches kda_skill and visualization. The reply this loop sends is built from the expectation
 # and is byte-identical every turn, so the extra rounds are not there to say anything new --
-# they are slack for a turn that produced nothing. A turn can come back empty or answer without
-# drafting for reasons that are not the model's, and with a tight cap that one wasted turn
-# spends the only reply the case had; the run then fails as "no draft" when nothing was wrong
-# with the agent. Fewer than metric_skill's seven because those rounds do carry new content:
-# its reply is generated per turn by an LLM.
+# they are slack for a turn that answered without drafting, which can happen for reasons that
+# are not the model's. A turn that comes back with nothing at all does not consume the slack:
+# the loop ends there rather than replying into silence. Fewer than metric_skill's seven
+# because those rounds do carry new content, its reply being generated per turn by an LLM.
+#
+# Only creation ever reaches the later rounds. An edit stops after the first turn, since the
+# creation reply names charts and a date range and answers nothing a rename or a resize could
+# have asked.
 #
 # The cost of the slack is that four turns at ChatClient's 300s read timeout exceed the 720s
 # per-test timeout gdc-nas derives for a k=1 dataset, so a run that stalls on every turn is cut
@@ -55,8 +60,15 @@ _DEFAULT_K = 1
 _DEFAULT_MAX_ITERATIONS = 4
 
 _DRAFT_TOOL = "draft_dashboard"
+_PATCH_TOOL = "patch_dashboard"
 _SET_SKILLS_TOOL = "set_skills"
 _BUILDER_SKILL = "dashboard_builder"
+_EDITOR_SKILL = "dashboard_editor"
+# The expectation's `type` is the only switch between creating and editing: it selects the
+# response part to read, the tool that must have succeeded, and the skill that must have been
+# activated. There is no "either one" case -- an edit fixture that routes to the builder is a
+# failure, not an alternative route to the same answer.
+_PATCH_TYPE = "dashboardPatch"
 
 
 def _norm(text: str) -> str:
@@ -64,14 +76,14 @@ def _norm(text: str) -> str:
     return " ".join(text.split()).casefold()
 
 
-def _extract_draft_result(tool_call_events: list[ToolCallEvent]) -> dict | None:
-    """Result payload of the ``draft_dashboard`` call that produced a draft.
+def _extract_tool_result(tool_call_events: list[ToolCallEvent], tool_name: str) -> dict | None:
+    """Result payload of the ``tool_name`` call that produced the response.
 
     Takes the most recent *successful* call: when the agent retries after a rejected
     draft, the earlier failed attempt must not shadow the one that worked.
     """
     for tc in reversed(tool_call_events):
-        if tc.function_name != _DRAFT_TOOL or not tc.result:
+        if tc.function_name != tool_name or not tc.result:
             continue
         result_data = tc.parsed_result()
         if not isinstance(result_data, dict):
@@ -83,14 +95,22 @@ def _extract_draft_result(tool_call_events: list[ToolCallEvent]) -> dict | None:
     return None
 
 
-def _builder_skill_activated(tool_call_events: list[ToolCallEvent]) -> bool:
-    """Whether any ``set_skills`` result activated the dashboard-builder skill.
+def _skill_activated(tool_call_events: list[ToolCallEvent], skill: str) -> bool:
+    """Whether the run's skill routing landed on ``skill``.
 
-    Diagnostic only, deliberately outside ``strict_pass``: a mis-route already fails on the
-    missing draft, and gating on this would turn any future route that skips ``set_skills``
-    into a false failure. Reported so such a failure reads as a mis-route rather than as a
-    model that simply did not draft.
+    Gated, because the expectation's type names one skill and only one: a creation fixture
+    answered by the editor, or an edit fixture answered by the builder, is a routing failure
+    even when the dashboard that comes back looks plausible. It also names the failure -- a
+    mis-route reads as a mis-route instead of as a model that simply produced nothing.
+
+    A run that never routed passes. Absence of a ``set_skills`` call is absence of evidence,
+    not evidence of a mis-route, and there are two ways to reach it that say nothing about the
+    model: gen-ai omits the call for an agent whose skills are pinned
+    (``include_set_skills=not resolved_pinned_skills``), and a run continuing an existing
+    conversation through ``initial_conversation_id`` can have routed in a turn this run never
+    saw. Failing those would fail the case for the harness's configuration.
     """
+    routed = False
     for tc in tool_call_events:
         if tc.function_name != _SET_SKILLS_TOOL or not tc.result:
             continue
@@ -99,9 +119,27 @@ def _builder_skill_activated(tool_call_events: list[ToolCallEvent]) -> bool:
             continue
         payload = result_data.get("data", result_data)
         skills = payload.get("skills_to_activate") if isinstance(payload, dict) else None
-        if isinstance(skills, list) and _BUILDER_SKILL in skills:
+        if not isinstance(skills, list):
+            continue
+        routed = True
+        if skill in skills:
             return True
-    return False
+    return not routed
+
+
+def _is_edit(expected_output: dict) -> bool:
+    """Whether the expectation describes an edit rather than a fresh dashboard."""
+    return _expected_type(expected_output) == _PATCH_TYPE
+
+
+def _required_skill(expected_output: dict) -> str:
+    """The skill the run must have activated, decided by the expectation's type."""
+    return _EDITOR_SKILL if _is_edit(expected_output) else _BUILDER_SKILL
+
+
+def _producing_tool(expected_output: dict) -> str:
+    """The tool whose successful call the response must come from."""
+    return _PATCH_TOOL if _is_edit(expected_output) else _DRAFT_TOOL
 
 
 def _expected_type(expected_output: dict) -> str:
@@ -114,7 +152,7 @@ def _extract_dashboard_part(chat_result: ChatResult, part_type: str) -> dict | N
 
     That type is in the SSE client's known-part set but has no dedicated accumulator, so it
     arrives in ``unhandled_parts`` verbatim and is read back by type. Taken from the end for
-    the same reason ``_extract_draft_result`` does: a turn that drafts and then refines must
+    the same reason ``_extract_tool_result`` does: a turn that drafts and then refines must
     be read as the state it left behind, not the one it passed through.
     """
     for part in reversed(chat_result.unhandled_parts):
@@ -123,64 +161,113 @@ def _extract_dashboard_part(chat_result: ChatResult, part_type: str) -> dict | N
     return None
 
 
+def _sections_of(dashboard: dict) -> list[dict]:
+    """The dashboard's sections, whichever shape the document uses.
+
+    A freshly drafted dashboard is version 3 and nests its sections under ``tabs``. A saved
+    dashboard relayed for editing can be version 2, which carries ``sections`` at the root and
+    no ``tabs`` at all. ``version`` is not the discriminator -- the convertor computes it from
+    the declarative input and separately flattens a single untitled tab into root sections, so
+    a document can say ``version: "3"`` and still have no tabs. Read the presence of ``tabs``.
+    """
+    tabs = dashboard.get("tabs")
+    if tabs:
+        return [section for tab in tabs for section in tab.get("sections") or []]
+    return dashboard.get("sections") or []
+
+
 def _widgets_of(dashboard: dict) -> list[dict]:
-    """Every widget of the draft, flattened across tabs and sections.
+    """Every widget of the dashboard, flattened across sections.
 
     Charts are matched dashboard-wide on purpose: the expectation names charts, not a layout,
-    so which tab one lands on is the agent's call. The date filter is deliberately the
-    opposite -- ``_check_date_range`` requires *every* tab to match. That is defensive rather
-    than observed: ``draft_dashboard`` copies one filter onto every tab unconditionally, so a
-    draft cannot currently disagree with itself across tabs, and the per-tab check is there to
-    notice if that ever stops being true.
-
-    Only ``tabs`` is read. That is sound for a draft, which ``draft_dashboard`` always builds
-    tabbed, but an AAC v2 document carries its layout in root ``sections`` instead, so an
-    editing dataset cannot reuse this as it stands.
+    so which section or tab one lands on is the agent's call. The date filter is deliberately
+    the opposite -- ``_check_date_range`` requires *every* tab to match. That one is defensive
+    rather than observed: ``draft_dashboard`` copies a single filter onto every tab
+    unconditionally, so a draft cannot currently disagree with itself across tabs, and the
+    per-tab check is there to notice if that stops being true.
     """
-    return [
-        widget
-        for tab in dashboard.get("tabs") or []
-        for section in tab.get("sections") or []
-        for widget in section.get("widgets") or []
-    ]
+    return [widget for section in _sections_of(dashboard) for widget in section.get("widgets") or []]
+
+
+def _apply_patch(base: dict, operations: list[dict]) -> tuple[dict | None, str | None]:
+    """Apply RFC 6902 ``operations`` to a copy of ``base``; return ``(result, failure)``.
+
+    The agent emits ``test`` guards ahead of its real operations, pinning the widget it is
+    about to touch. Those are applied rather than skipped: a failing guard means the patch was
+    written against a document that is not the one it shipped, which is a failure of the edit
+    even though every later operation might apply cleanly. Scoring the base instead of the
+    result would pass a case whose change never happened.
+
+    A patch carrying no operations is that same failure in its purest form. ``jsonpatch``
+    applies an empty list happily and hands back the document untouched, so the run would go on
+    to score the saved dashboard and pass every case the saved dashboard already satisfied.
+    """
+    if not operations:
+        return None, "the patch carries no operations, so it proposes the dashboard it started from"
+    try:
+        # in_place=False is what makes the copy -- JsonPatch.apply deepcopies for us, so
+        # copying here as well would leave a second one behind for no reason.
+        return jsonpatch.JsonPatch(operations).apply(base, in_place=False), None
+    except Exception as exc:  # jsonpatch raises several unrelated types for a bad patch
+        return None, f"the patch does not apply to the dashboard it shipped with: {exc}"
 
 
 def _check_visualizations(widgets: list[dict], new_ids: set[str], expected: list[dict]) -> tuple[list[str], list[str]]:
-    """Match every expected chart against the draft; extra charts the agent added are allowed.
+    """Match every expected chart against the dashboard; extra widgets are allowed.
 
     Returns ``(failures, title_notes)``.
 
-    An entry with an ``id`` is an existing chart and is matched on that id alone: the id is
-    the chart's identity, while the widget title is ``title_override or fallback_title`` and
-    the override is the model's own choice, so requiring it adds flake without signal. A
-    title that differs is still reported, as a note rather than a failure.
+    An entry with an ``id`` is matched on that id. ``id: null`` marks a chart the agent had to
+    author, whose real id is generated at run time, so there the title *is* the match -- taken
+    among the authored charts only, since matching against every chart would let an existing
+    one of the same title satisfy it.
+
+    Title mismatches come back separately from the failures rather than mixed into them, so
+    the caller can gate on them or merely record them without the two decisions drifting: on
+    creation the title is ``title_override or fallback_title`` and the override is the model's
+    own choice, while on an edit the expectation states the title the change must leave behind
+    and a rename case that does not gate on it asserts nothing at all. Either way the
+    mismatches are returned, which is what keeps the reported score honest.
 
-    ``id: null`` marks a chart the agent had to author, whose real id is generated at run
-    time. It has no identity to match on, so the title *is* the match — taken among the
-    authored charts only, without which an existing chart of the same title would satisfy it.
+    ``columns`` is checked whenever the expectation carries it. A newly placed widget defaults
+    to half width, so a resize case is only meaningful against the number.
     """
     failures: list[str] = []
-    title_notes: list[str] = []
+    title_mismatches: list[str] = []
     for exp in expected:
         exp_id = exp.get("id")
         exp_title = str(exp.get("title") or "")
+        exp_columns = exp.get("columns")
+
         if exp_id is not None:
             matches = [w for w in widgets if w.get("visualization") == exp_id]
             if not matches:
                 failures.append(f"missing existing chart id={exp_id!r} title={exp_title!r}")
-            elif not any(_norm(str(w.get("title") or "")) == _norm(exp_title) for w in matches):
+                continue
+            titled = [w for w in matches if _norm(str(w.get("title") or "")) == _norm(exp_title)]
+            if not titled:
                 # Every title the id appears under: the same chart can be placed twice, and
                 # naming only the first would hide the rest.
                 titles = ", ".join(repr(w.get("title")) for w in matches)
-                title_notes.append(f"chart {exp_id!r} is titled {titles}, expected {exp_title!r}")
-        elif not any(
-            w.get("visualization") in new_ids and _norm(str(w.get("title") or "")) == _norm(exp_title) for w in widgets
-        ):
-            failures.append(
-                f"missing authored chart title={exp_title!r}"
-                + ("" if new_ids else " (the response listed no authored charts)")
-            )
-    return failures, title_notes
+                title_mismatches.append(f"chart {exp_id!r} is titled {titles}, expected {exp_title!r}")
+            candidates = titled or matches
+        else:
+            candidates = [
+                w
+                for w in widgets
+                if w.get("visualization") in new_ids and _norm(str(w.get("title") or "")) == _norm(exp_title)
+            ]
+            if not candidates:
+                failures.append(
+                    f"missing authored chart title={exp_title!r}"
+                    + ("" if new_ids else " (the response listed no authored charts)")
+                )
+                continue
+
+        if exp_columns is not None and not any(w.get("columns") == exp_columns for w in candidates):
+            widths = ", ".join(repr(w.get("columns")) for w in candidates)
+            failures.append(f"chart {exp_title!r} is {widths} column(s) wide, expected {exp_columns}")
+    return failures, title_mismatches
 
 
 def _check_references(widgets: list[dict], known_ids: set[str]) -> list[str]:
@@ -199,34 +286,148 @@ def _check_references(widgets: list[dict], known_ids: set[str]) -> list[str]:
     return notes
 
 
+def _filter_entries(dashboard: dict) -> list[tuple[str, dict]]:
+    """``(label, filter)`` for every filter the dashboard carries, whichever shape it uses.
+
+    A drafted dashboard hangs one filter map off each tab, keyed by role (``date``). A saved
+    dashboard relayed for editing carries a single map at the root, keyed by the filter's own
+    local identifier, so the role has to be read from each entry's ``type`` instead. The label
+    is only there to say *where* a failure was found.
+    """
+    tabs = dashboard.get("tabs")
+    if tabs:
+        return [
+            (f"tab {tab.get('id')!r}", value)
+            for tab in tabs
+            for value in (tab.get("filters") or {}).values()
+            if isinstance(value, dict)
+        ]
+    return [
+        (f"filter {key!r}", value) for key, value in (dashboard.get("filters") or {}).items() if isinstance(value, dict)
+    ]
+
+
+def _filters_of_type(dashboard: dict, filter_type: str) -> list[tuple[str, dict]]:
+    return [(label, f) for label, f in _filter_entries(dashboard) if f.get("type") == filter_type]
+
+
+def _date_filter_mismatch(label: str, date_filter: dict, expected: dict | None) -> list[str]:
+    """Why ``date_filter`` is not the expected range, or nothing.
+
+    ``expected is None`` means all time, which the document spells as a date filter with
+    neither bound — absent keys, not null ones.
+    """
+    if expected is None:
+        if "from" in date_filter or "to" in date_filter:
+            return [f"{label}: expected all time, got from={date_filter.get('from')!r} to={date_filter.get('to')!r}"]
+        return []
+    return [
+        f"{label}: date {key} expected {expected.get(key)!r}, got {date_filter.get(key)!r}"
+        for key in ("granularity", "from", "to")
+        if date_filter.get(key) != expected.get(key)
+    ]
+
+
 def _check_date_range(dashboard: dict, expected: dict | None) -> list[str]:
-    """The date filter of every tab must match the expectation.
+    """Check the dashboard's date range against the expectation.
+
+    What counts as "the" date filter differs by shape, so the two are checked differently.
+
+    A drafted dashboard carries one per tab, and the draft tool writes the same one onto every
+    tab, so every tab must have it and every one must match. A tab left without a date filter is
+    a failure: the expectation describes what the whole dashboard shows.
+
+    A saved dashboard relayed for editing carries a flat map that can legitimately hold more
+    than one date filter — a dashboard-wide one plus a dataset-scoped one — and nothing in the
+    document says which is which. Requiring every entry to match would fail a dashboard for
+    carrying a filter the fixture never described, so one matching entry satisfies the check and
+    the rest are left to the notes.
+    """
+    date_filters = _filters_of_type(dashboard, "date_filter")
+    if not date_filters:
+        return ["the dashboard carries no date filter to check the expected range against"]
+
+    tabs = dashboard.get("tabs")
+    if tabs:
+        failures: list[str] = [
+            f"tab {tab.get('id')!r} carries no date filter"
+            for tab in tabs
+            if not any(
+                f.get("type") == "date_filter" for f in (tab.get("filters") or {}).values() if isinstance(f, dict)
+            )
+        ]
+        for label, date_filter in date_filters:
+            failures.extend(_date_filter_mismatch(label, date_filter, expected))
+        return failures
+
+    mismatches = [_date_filter_mismatch(label, date_filter, expected) for label, date_filter in date_filters]
+    if any(not m for m in mismatches):
+        return []
+    return [reason for m in mismatches for reason in m]
 
-    A draft always carries a date filter, so ``expected is None`` (all time) means the tab
-    filter has neither bound — absent keys, not null ones.
+
+def _selection_of(attribute_filter: dict) -> tuple[str, list] | None:
+    """The elements an attribute filter restricts to, or ``None`` when it restricts nothing.
+
+    An unrestricted filter is spelled as the absence of a selection, and an empty list means
+    the same thing, so both read as "all".
     """
-    tabs = dashboard.get("tabs") or []
-    if not tabs:
-        return ["draft has no tabs to read a date filter from"]
+    state = attribute_filter.get("state") or {}
+    for kind in ("include", "exclude"):
+        values = state.get(kind)
+        if values:
+            return kind, list(values)
+    return None
+
+
+def _describe_selection(selection: tuple[str, list] | None) -> str:
+    return "all" if selection is None else f"{selection[0]} {selection[1]!r}"
+
+
+def _check_filters(dashboard: dict, expected: list[dict]) -> list[str]:
+    """Every expected attribute filter must be on the dashboard, with the stated selection.
+
+    This exists because an edit can silently drop or widen the filters the user already had:
+    the change asked for was a rename, and the filters coming back untouched is part of what
+    "untouched" means. Filters the dashboard carries beyond the expected ones are left alone,
+    the same way extra widgets are.
+    """
+    actual: dict[str | None, list[dict]] = {}
+    for _label, attribute_filter in _filters_of_type(dashboard, "attribute_filter"):
+        actual.setdefault(attribute_filter.get("using"), []).append(attribute_filter)
+
     failures: list[str] = []
-    for tab in tabs:
-        tab_id = tab.get("id")
-        date_filter = (tab.get("filters") or {}).get("date") or {}
-        if expected is None:
-            if "from" in date_filter or "to" in date_filter:
-                failures.append(
-                    f"tab {tab_id!r}: expected all time, got from={date_filter.get('from')!r} "
-                    f"to={date_filter.get('to')!r}"
-                )
+    for exp in expected:
+        using = exp.get("using")
+        matches = actual.get(using) or []
+        if not matches:
+            failures.append(f"the dashboard carries no attribute filter on {using!r}")
             continue
-        failures.extend(
-            f"tab {tab_id!r}: date {key} expected {expected.get(key)!r}, got {date_filter.get(key)!r}"
-            for key in ("granularity", "from", "to")
-            if date_filter.get(key) != expected.get(key)
-        )
+        wanted = _expected_selection(exp)
+        if not any(_selection_of(f) == wanted for f in matches):
+            found = ", ".join(_describe_selection(_selection_of(f)) for f in matches)
+            failures.append(f"filter on {using!r} selects {found}, expected {_describe_selection(wanted)}")
     return failures
 
 
+def _expected_selection(expected_filter: dict) -> tuple[str, list] | None:
+    """Read a fixture's filter entry into the same shape ``_selection_of`` produces.
+
+    Raises:
+        ValueError: the entry states no selection the evaluator knows how to check.
+            Deliberately loud -- a typo here would otherwise assert nothing and read green.
+    """
+    for kind in ("include", "exclude"):
+        if kind in expected_filter:
+            return kind, list(expected_filter[kind] or [])
+    if expected_filter.get("selection") == "all":
+        return None
+    raise ValueError(
+        f"filter expectation for {expected_filter.get('using')!r} needs 'selection': 'all', "
+        f"'include' or 'exclude', got {expected_filter!r}"
+    )
+
+
 def _date_range_text(date_range: dict | None) -> str:
     """Render a date range the way a user would say it, for the simulated reply.
 
@@ -290,6 +491,26 @@ def _min_new_visualizations(expected_output: dict) -> int:
     return value
 
 
+def _has_filters(expected_output: dict) -> bool:
+    """Whether the expectation says anything about the attribute filters.
+
+    Absent means the case does not look at them, exactly as for ``date_range``. Only the
+    cases that are about preserving filters carry the key, so the others stay unaffected.
+    """
+    return "filters" in expected_output
+
+
+def _has_date_range(expected_output: dict) -> bool:
+    """Whether the expectation says anything about the date filter at all.
+
+    Absence and ``null`` are different answers and must not be collapsed: ``null`` means the
+    dashboard has to be on all time, while a missing key means the case does not look at the
+    filter. Edit cases omit it because a dashboard the user has not opened carries no filter
+    context, so ``patch_dashboard`` refuses filter changes outright.
+    """
+    return "date_range" in expected_output
+
+
 def _validate_expectation(expected_output: dict) -> None:
     """Reject a fixture the run could not score meaningfully, before the first API call.
 
@@ -302,19 +523,67 @@ def _validate_expectation(expected_output: dict) -> None:
     """
     if not expected_output.get("visualizations"):
         raise ValueError("expected_output lists no visualizations; every chart check would pass vacuously")
-    _date_range_text(expected_output.get("date_range"))
+    if _has_date_range(expected_output):
+        date_range = expected_output.get("date_range")
+        # Shape first, and on both kinds of case. Only a creation case goes on to check that
+        # the reply can phrase the period -- an edit records whatever range the saved dashboard
+        # carries, which `_check_date_range` compares by number rather than reading aloud, so
+        # demanding a phrasable one there would cap what the preserved-filter cases can cover.
+        # Without the shape check a string slips through to scoring and dies there instead,
+        # after the run has already been spent.
+        if date_range is not None and not isinstance(date_range, dict):
+            raise ValueError(f"date_range must be an object or null, got {date_range!r}")
+        if not _is_edit(expected_output):
+            _date_range_text(date_range)
+
+    filters = expected_output.get("filters")
+    if filters is not None and not isinstance(filters, list):
+        raise ValueError(f"filters must be a list of filter expectations, got {filters!r}")
+    for entry in filters or []:
+        if not isinstance(entry, dict):
+            raise ValueError(f"a filter expectation must be an object, got {entry!r}")
+        if not entry.get("using"):
+            raise ValueError(f"a filter expectation needs the label it filters on, got {entry!r}")
+        _expected_selection(entry)
+    saved_id = expected_output.get("saved_dashboard_id")
+    if _is_edit(expected_output):
+        if not isinstance(saved_id, str) or not saved_id:
+            raise ValueError(f"an edit expectation needs the edited dashboard's saved_dashboard_id, got {saved_id!r}")
+    elif saved_id is not None:
+        raise ValueError(f"a creation expectation must not name a saved_dashboard_id, got {saved_id!r}")
     _min_new_visualizations(expected_output)
 
 
+@dataclass(frozen=True)
+class _Applies:
+    """Which of the conditional checks the case applies.
+
+    They are published only when they do. A check that could not fail is not evidence, and
+    publishing it as passed lifts `quality_score` -- the fraction of true booleans in the
+    detail -- above what the run earned, which would make a creation case read better than the
+    same case scored before editing existed.
+    """
+
+    patch: bool
+    date: bool
+    filters: bool
+
+
 @dataclass
 class DashboardEvaluation:
     """Per-run outcome of the dashboard-skill checks.
 
-    ``strict_checks`` is what the run is scored on, and every entry in it is something the
-    agent decided. ``skill_activated`` and ``references_carried`` are reported beside it but
-    never gate: the first can be false on a route that pins skills instead of calling
-    ``set_skills``, and the second reflects gen-ai's reference building rather than the
-    agent's choices. ``notes`` carries the same kind of observation for titles.
+    ``strict_checks`` is what the run is scored on. ``skill_activated`` is in it because the
+    expectation's type names one skill and only one: a creation fixture answered by the editor,
+    or an edit fixture answered by the builder, is a routing failure even if the dashboard that
+    came back looks plausible.
+
+    ``references_carried`` stays out of the gate: gen-ai rejects a response naming an
+    unresolvable chart before it can succeed, so a widget missing from the references means
+    reference building degraded on the way out -- a platform fault, not the model's.
+    ``titles_matched`` is also out for creation, where the title is the model's own wording;
+    on an edit the expectation states the title the change must leave behind, so there it is
+    scored through ``charts_matched`` instead.
     """
 
     drafted: bool
@@ -322,7 +591,11 @@ class DashboardEvaluation:
     charts_matched: bool
     date_range_correct: bool
     new_visualizations_met: bool
+    applies: _Applies
+    filters_correct: bool = True
     skill_activated: bool = False
+    saved_dashboard_id_correct: bool = True
+    patch_applies: bool = True
     references_carried: bool = True
     titles_matched: bool = True
     failures: list[str] = field(default_factory=list)
@@ -334,26 +607,42 @@ class DashboardEvaluation:
 
     @property
     def strict_checks(self) -> dict[str, bool]:
-        return {
+        checks = {
+            # Kept as-is through the editing work: every Langfuse view, saved filter and
+            # combo-report field list already refers to it, and a rename would break them for
+            # a word. It means "the tool that had to succeed did", patch or draft.
             "dashboard_drafted": self.drafted,
             "dashboard_part_present": self.part_present,
+            "dashboard_skill_activated": self.skill_activated,
+            "saved_dashboard_id_correct": self.saved_dashboard_id_correct,
             "charts_matched": self.charts_matched,
-            "date_range_correct": self.date_range_correct,
             "new_visualizations_met": self.new_visualizations_met,
         }
+        if self.applies.patch:
+            checks["patch_applies"] = self.patch_applies
+        if self.applies.date:
+            checks["date_range_correct"] = self.date_range_correct
+        if self.applies.filters:
+            # Prefixed: alert_skill already publishes a `filters_correct` score, and the combo
+            # report resolves a trace's skill by which score names it carries.
+            checks["dashboard_filters_correct"] = self.filters_correct
+        return checks
 
     @property
     def diagnostics(self) -> dict[str, bool]:
         """Observed but not scored — see the class docstring.
 
-        ``titles_matched`` is here rather than in the gate so the decision to stop failing on
-        a model-chosen title stays reviewable: without a score, nothing would record how often
-        it happens. The two chart-level observations are omitted when no dashboard part was
-        read, because reporting them as true would claim references and titles were fine on a
-        run that produced neither.
+        ``titles_matched`` reports the titles on both kinds of case, but only creation leaves
+        it out of the gate -- an edit scores the same mismatches through ``charts_matched``,
+        because there the expectation states the title the change must leave behind. Keeping
+        the score in both cases is what makes the creation decision reviewable later.
+
+        Both chart-level observations are omitted whenever no document was scored -- no part
+        read, or a patch that would not apply. Reporting them as true would claim references
+        and titles were fine on a run that never produced anything to look at.
         """
-        observed = {"dashboard_skill_activated": self.skill_activated}
-        if self.part_present:
+        observed: dict[str, bool] = {}
+        if self.part_present and self.patch_applies:
             observed["references_carried"] = self.references_carried
             observed["titles_matched"] = self.titles_matched
         return observed
@@ -361,12 +650,13 @@ class DashboardEvaluation:
 
 @dataclass
 class DashboardRunResult:
-    """Outcome of one K-run conversation for dashboard creation."""
+    """Outcome of one K-run conversation, creating or editing."""
 
     conversation_id: str
     evaluation: DashboardEvaluation
-    draft_result: dict | None = None
+    tool_result: dict | None = None
     dashboard_part: dict | None = None
+    patch_part: dict | None = None
     total_turns: int = 0
     total_steps: int = 0
     reasoning_steps: list[str] = field(default_factory=list)
@@ -378,7 +668,7 @@ class DashboardRunResult:
 
 @dataclass
 class AgenticDashboardSummary:
-    """Aggregated outcome of K runs for dashboard creation."""
+    """Aggregated outcome of K runs, creating or editing."""
 
     run_results: list[DashboardRunResult]
     pass_at_k: bool
@@ -386,30 +676,43 @@ class AgenticDashboardSummary:
     best: DashboardRunResult
 
 
-def evaluate_dashboard_draft(
-    draft_result: dict | None,
+def evaluate_dashboard_response(
+    tool_result: dict | None,
     dashboard_part: dict | None,
     expected_output: dict,
     skill_activated: bool,
+    patch_part: dict | None = None,
 ) -> DashboardEvaluation:
-    """Score one drafted dashboard against its expectation.
+    """Score one dashboard response against its expectation.
+
+    Creation reads the drafted dashboard straight off the ``dashboard`` part. Editing reads
+    two parts -- the saved dashboard as it stood, and the patch against it -- and scores the
+    document that results from applying one to the other, never the base.
 
-    Pure — no network, no conversation state — so the assertion logic is unit-testable
+    Pure: no network, no conversation state, so the whole assertion surface is unit-testable
     without an agent.
     """
-    if draft_result is None:
+    is_edit = _is_edit(expected_output)
+    applies = _Applies(patch=is_edit, date=_has_date_range(expected_output), filters=_has_filters(expected_output))
+    tool = _producing_tool(expected_output)
+
+    if tool_result is None:
         return DashboardEvaluation(
             drafted=False,
             part_present=False,
             charts_matched=False,
             date_range_correct=False,
+            filters_correct=False,
             new_visualizations_met=False,
+            applies=applies,
             skill_activated=skill_activated,
-            failures=["the agent never produced a successful draft_dashboard call"],
+            saved_dashboard_id_correct=False,
+            patch_applies=False,
+            failures=[f"the agent never produced a successful {tool} call"],
         )
 
     min_new = _min_new_visualizations(expected_output)
-    actual_new = draft_result.get("new_visualization_count")
+    actual_new = tool_result.get("new_visualization_count")
     if isinstance(actual_new, int):
         new_met = actual_new >= min_new
         # Naming the shortfall, never "expected at least 0" -- see the envelope branch below.
@@ -419,42 +722,116 @@ def evaluate_dashboard_draft(
         # "expected at least 0 authored chart(s), tool reported None" reads as a broken test
         # and would send whoever is on nightly duty looking at the model instead.
         new_met = False
-        new_failures = [f"the draft result carries no new_visualization_count (got {actual_new!r})"]
+        new_failures = [f"the {tool} result carries no new_visualization_count (got {actual_new!r})"]
 
+    missing_parts: list[str] = []
     if dashboard_part is None:
+        missing_parts.append("dashboard")
+    if is_edit and patch_part is None:
+        missing_parts.append(_PATCH_TYPE)
+    if missing_parts:
         return DashboardEvaluation(
             drafted=True,
             part_present=False,
             charts_matched=False,
             date_range_correct=False,
+            filters_correct=False,
             new_visualizations_met=new_met,
+            applies=applies,
             skill_activated=skill_activated,
+            saved_dashboard_id_correct=False,
+            patch_applies=False,
             failures=[
-                f"the response carries no {_expected_type(expected_output)!r} part",
+                f"the response carries no {' and no '.join(repr(p) for p in missing_parts)} part",
                 *new_failures,
             ],
         )
 
-    dashboard = dashboard_part.get("dashboard") or {}
-    references = dashboard_part.get("references") or {}
-    new_ids = {v.get("id") for v in references.get("new_visualizations") or [] if v.get("id")}
-    known_ids = {v.get("id") for v in references.get("visualizations") or [] if v.get("id")} | new_ids
-    widgets = _widgets_of(dashboard)
+    assert dashboard_part is not None  # noqa: S101  narrowed by missing_parts above
+    base = dashboard_part.get("dashboard") or {}
+    base_references = dashboard_part.get("references") or {}
+    expected_saved_id = expected_output.get("saved_dashboard_id")
+    actual_saved_id = dashboard_part.get("saved_dashboard_id")
+
+    saved_failures: list[str] = []
+    patch_failures: list[str] = []
+
+    if is_edit:
+        patch = (patch_part or {}).get("patch") or {}
+        patch_references = patch.get("references") or {}
+        # The patch names the dashboard it addresses; the part names the one it shipped with.
+        # Both have to be the dashboard the fixture asked to edit, or the change landed
+        # somewhere nobody looked.
+        for label, value in (("the response", actual_saved_id), ("the patch", patch.get("dashboard_id"))):
+            if value != expected_saved_id:
+                saved_failures.append(f"{label} addresses dashboard {value!r}, expected {expected_saved_id!r}")
+        document, patch_error = _apply_patch(base, patch.get("operations") or [])
+        if patch_error is not None:
+            patch_failures.append(patch_error)
+            document = None
+        new_ids = {v.get("id") for v in patch_references.get("new_visualizations") or [] if v.get("id")}
+        known_ids = (
+            {v.get("id") for v in base_references.get("visualizations") or [] if v.get("id")}
+            | {v.get("id") for v in patch_references.get("visualizations") or [] if v.get("id")}
+            | new_ids
+        )
+    else:
+        if actual_saved_id is not None:
+            saved_failures.append(f"a new draft must not be saved yet, but it reports id {actual_saved_id!r}")
+        document = base
+        new_ids = {v.get("id") for v in base_references.get("new_visualizations") or [] if v.get("id")}
+        known_ids = {v.get("id") for v in base_references.get("visualizations") or [] if v.get("id")} | new_ids
 
-    chart_failures, title_notes = _check_visualizations(widgets, new_ids, expected_output.get("visualizations") or [])
+    if document is None:
+        return DashboardEvaluation(
+            drafted=True,
+            part_present=True,
+            charts_matched=False,
+            date_range_correct=False,
+            filters_correct=False,
+            new_visualizations_met=new_met,
+            applies=applies,
+            skill_activated=skill_activated,
+            saved_dashboard_id_correct=not saved_failures,
+            patch_applies=False,
+            failures=[*patch_failures, *saved_failures, *new_failures],
+        )
+
+    widgets = _widgets_of(document)
+    chart_failures, title_mismatches = _check_visualizations(
+        widgets, new_ids, expected_output.get("visualizations") or []
+    )
+    # On an edit the title is part of what was asked for, so it fails the case; on creation it
+    # is only recorded. Either way `titles_matched` reads the mismatches themselves, never the
+    # list they were routed into -- a score that says "titles fine" beside a failed rename is
+    # worse than no score at all.
+    if is_edit:
+        chart_failures = [*chart_failures, *title_mismatches]
+        title_notes: list[str] = []
+    else:
+        title_notes = title_mismatches
     reference_notes = _check_references(widgets, known_ids)
-    date_failures = _check_date_range(dashboard, expected_output.get("date_range"))
+    date_failures = (
+        _check_date_range(document, expected_output.get("date_range")) if _has_date_range(expected_output) else []
+    )
+    filter_failures = (
+        _check_filters(document, expected_output.get("filters") or []) if _has_filters(expected_output) else []
+    )
 
     return DashboardEvaluation(
         drafted=True,
         part_present=True,
         charts_matched=not chart_failures,
         date_range_correct=not date_failures,
+        filters_correct=not filter_failures,
         new_visualizations_met=new_met,
+        applies=applies,
         skill_activated=skill_activated,
+        saved_dashboard_id_correct=not saved_failures,
+        patch_applies=True,
         references_carried=not reference_notes,
-        titles_matched=not title_notes,
-        failures=[*chart_failures, *date_failures, *new_failures],
+        titles_matched=not title_mismatches,
+        failures=[*chart_failures, *saved_failures, *date_failures, *filter_failures, *new_failures],
         notes=[*title_notes, *reference_notes],
     )
 
@@ -466,14 +843,16 @@ def _execute_single_dashboard_run(
     expected_output: dict,
     max_iterations: int,
 ) -> DashboardRunResult:
-    """Drive one conversation until the agent drafts a dashboard, then evaluate it.
+    """Drive one conversation until the agent produces a dashboard, then evaluate it.
 
     Nothing is cleaned up on the way out by design: gen-ai keeps the draft and any authored
     chart in conversation state and persists neither until a user saves from the UI.
     """
-    expected_type = str(expected_output.get("type") or "dashboard")
-    draft_result: dict | None = None
+    tool = _producing_tool(expected_output)
+    is_edit = _is_edit(expected_output)
+    tool_result: dict | None = None
     dashboard_part: dict | None = None
+    patch_part: dict | None = None
     turns = 0
     steps = 0
     current_question = question
@@ -504,14 +883,26 @@ def _execute_single_dashboard_run(
         all_reasoning_step_events.extend(chat_result.reasoning_step_events or [])
         steps += chat_result.reasoning_step_count
 
-        candidate = _extract_draft_result(chat_result.tool_call_events or [])
+        candidate = _extract_tool_result(chat_result.tool_call_events or [], tool)
         if candidate is not None:
             log_timer(
                 f"[timer] dashboard_skill {conversation_id} GoodData turn {turns} complete after "
-                f"{agent_elapsed:.2f}s; draft received"
+                f"{agent_elapsed:.2f}s; {tool} result received"
             )
-            draft_result = candidate
-            dashboard_part = _extract_dashboard_part(chat_result, expected_type)
+            tool_result = candidate
+            # An edit relays two parts: the dashboard as it stood, and the patch against it.
+            # Both are read from the turn that produced the patch -- the base is what the patch
+            # was written for, so pairing it with any other turn's would score a document the
+            # agent never proposed.
+            # A turn may carry several patches. They are alternatives rebased on the same base
+            # rather than steps to compose, and the client resolves the last one as the proposal
+            # that holds (gdc-ui's applyDashboardPatch; gdc-nas
+            # dashboard_edit_assertion.verify_last_proposal_holds does the same). Taking the last
+            # of each part type is therefore what the user would end up looking at, and the
+            # earlier patches are alternatives that lost, not changes that went missing.
+            dashboard_part = _extract_dashboard_part(chat_result, "dashboard")
+            if is_edit:
+                patch_part = _extract_dashboard_part(chat_result, _PATCH_TYPE)
             break
 
         response_text = (chat_result.text_response or "").strip() or render_answer_text(chat_result)
@@ -523,18 +914,26 @@ def _execute_single_dashboard_run(
             f"[timer] dashboard_skill {conversation_id} GoodData turn {turns} complete after "
             f"{agent_elapsed:.2f}s; answering with the expected charts and date range"
         )
+        if is_edit:
+            # The creation reply names charts and a date range, which answers nothing a rename
+            # or a resize could have asked. Rather than send something the question did not
+            # ask for, stop: the failure is the signal that an edit case needs a reply of its
+            # own, and inventing one here would hide which cases actually need it.
+            break
         current_question = build_simulated_reply(expected_output)
 
     return DashboardRunResult(
         conversation_id=conversation_id,
-        evaluation=evaluate_dashboard_draft(
-            draft_result,
+        evaluation=evaluate_dashboard_response(
+            tool_result,
             dashboard_part,
             expected_output,
-            _builder_skill_activated(all_tool_call_events),
+            _skill_activated(all_tool_call_events, _required_skill(expected_output)),
+            patch_part=patch_part,
         ),
-        draft_result=draft_result,
+        tool_result=tool_result,
         dashboard_part=dashboard_part,
+        patch_part=patch_part,
         total_turns=turns,
         total_steps=steps,
         reasoning_steps=reasoning_steps,
@@ -711,11 +1110,22 @@ def evaluate_agentic_dashboard_skill(
 
     if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k):
         gate_note = gate_failure_note(gate, runs_passed, runs_effective)
-        skill_note = (
-            ""
-            if best.evaluation.skill_activated or best.evaluation.drafted
-            else " No set_skills call activated dashboard_builder, so check the feature flag before the model."
-        )
+        # Two failures wear the same missing skill, and the advice differs. If the other
+        # dashboard skill was activated, the flag that registers both is plainly on and the
+        # model simply routed the wrong way. If neither was, the flag is off or the agent has
+        # its skills pinned. Sending a reader after the flag in the first case walks them past
+        # the real failure, which is why the run's own tool calls decide which line they get.
+        required_skill = _required_skill(expected_output)
+        other_skill = _BUILDER_SKILL if required_skill == _EDITOR_SKILL else _EDITOR_SKILL
+        if best.evaluation.skill_activated:
+            skill_note = ""
+        elif _skill_activated(best.tool_call_events, other_skill):
+            skill_note = f" It activated {other_skill} instead of {required_skill}: a routing failure, not the flag."
+        else:
+            skill_note = (
+                f" No set_skills call activated {required_skill} or {other_skill};"
+                " one feature flag registers both, so check that before the model."
+            )
         notes = "; ".join(best.evaluation.notes)
         exc = DashboardSkillAssertionError(
             f"Dashboard skill assertion failed. {gate_note}{skill_note} "

{gooddata_eval-1.75.1.dev2.dist-info → gooddata_eval-1.75.1.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.75.1.dev2
+Version: 1.75.1.dev3
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,8 +17,9 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.75.1.dev2
+Requires-Dist: gooddata-sdk~=1.75.1.dev3
 Requires-Dist: httpx<1.0,>=0.27
+Requires-Dist: jsonpatch<2.0,>=1.33
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6
 Requires-Dist: rich<15.0,>=13.0

{gooddata_eval-1.75.1.dev2.dist-info → gooddata_eval-1.75.1.dev3.dist-info}/RECORD

@@ -20,7 +20,7 @@ gooddata_eval/core/agentic/_langfuse.py,sha256=7V1PFQbWpRnFz0a3wklcI-ROYjGC1wGQv
 gooddata_eval/core/agentic/_trace_linker.py,sha256=iXMTxIXUsF5WNSen7v_K8JJ2j1_37lh-2VAGqQxxmkI,13538
 gooddata_eval/core/agentic/alert_skill.py,sha256=47V6LVQemAL0OmPeY9Yr1-lszkPu5bFkCgYgHSJJI_8,41712
 gooddata_eval/core/agentic/conversation.py,sha256=Y-SsGOth-TzXFlOKsElCigT05ZH6wNqIOOz1IUkzUo4,30044
-gooddata_eval/core/agentic/dashboard_skill.py,sha256=bzdGUsZ7PwqoTZHNYq_82Fj75aAkHBAXp8jIUAa2XsI,30905
+gooddata_eval/core/agentic/dashboard_skill.py,sha256=4lu7v6VeIR_h9FUFBrpD57upA0Rd2Kox1ZP_X-7jQaQ,51854
 gooddata_eval/core/agentic/general_question.py,sha256=Jp3FaS4Tvmfo0dx9iceiZ3Fr1QzJKDl7fVZZDetS22I,14684
 gooddata_eval/core/agentic/guardrail.py,sha256=rYpnsun9mLOSYucWfg1dbdrpbzQcz2V3eLwgII8dD0w,13145
 gooddata_eval/core/agentic/kda_skill.py,sha256=P_pR0iC12b_hIwnLzBmH0IBEjsO_CxZrlfkmu10cVWE,22721
@@ -63,8 +63,8 @@ gooddata_eval/core/reporting/json_report.py,sha256=q3Jirc_Zhvzg5lkG_kM6JdnViNLe6
 gooddata_eval/core/reporting/report_template.html,sha256=vLkl95OZsm3dMdijv5zda-15wYPE5UvTs4uKeWnjCiI,22332
 gooddata_eval/core/summary/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/summary/http_client.py,sha256=dPArJ6zoCu1W9xmYXFA1WMDNsSOkhn3gaCpYgCUp3gk,2174
-gooddata_eval-1.75.1.dev2.dist-info/METADATA,sha256=r44ORmpL9SsDmr7ceeoLp2wjIgL4Oqv4TPBnKszyS18,35033
-gooddata_eval-1.75.1.dev2.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-gooddata_eval-1.75.1.dev2.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
-gooddata_eval-1.75.1.dev2.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
-gooddata_eval-1.75.1.dev2.dist-info/RECORD,,
+gooddata_eval-1.75.1.dev3.dist-info/METADATA,sha256=hm6riuge5SGz7wMiNFSuWNZk39Qcx3ph7w8NDaMddGo,35069
+gooddata_eval-1.75.1.dev3.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+gooddata_eval-1.75.1.dev3.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
+gooddata_eval-1.75.1.dev3.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
+gooddata_eval-1.75.1.dev3.dist-info/RECORD,,