gooddata-eval 1.76.1.dev2__py3-none-any.whl → 1.76.1.dev4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,6 +9,7 @@ from typing import Any
9
9
 
10
10
  import jsonpatch
11
11
 
12
+ from gooddata_eval.core.agentic._failed_runs import build_failed_runs
12
13
  from gooddata_eval.core.agentic._gate import (
13
14
  DEFAULT_GATE,
14
15
  EvalGate,
@@ -294,17 +295,36 @@ def _filter_entries(dashboard: dict) -> list[tuple[str, dict]]:
294
295
  local identifier, so the role has to be read from each entry's ``type`` instead. The label
295
296
  is only there to say *where* a failure was found.
296
297
  """
298
+ return [(label, value) for label, value, _group in _grouped_filter_entries(dashboard)]
299
+
300
+
301
+ def _grouped_filter_entries(dashboard: dict) -> list[tuple[str, dict, str | None]]:
302
+ """``(label, filter, group title)`` for every filter, with a group's members read in its place.
303
+
304
+ A ``filter_group`` holds its members in a nested map, so reading only the outer map would
305
+ miss every filter the agent chose to group and report it missing. The group entry itself is
306
+ not a filter and is not returned.
307
+ """
297
308
  tabs = dashboard.get("tabs")
298
- if tabs:
299
- return [
300
- (f"tab {tab.get('id')!r}", value)
301
- for tab in tabs
302
- for value in (tab.get("filters") or {}).values()
303
- if isinstance(value, dict)
304
- ]
305
- return [
306
- (f"filter {key!r}", value) for key, value in (dashboard.get("filters") or {}).items() if isinstance(value, dict)
307
- ]
309
+ maps = (
310
+ [(f"tab {tab.get('id')!r}", tab.get("filters") or {}) for tab in tabs]
311
+ if tabs
312
+ else [("", dashboard.get("filters") or {})]
313
+ )
314
+ entries: list[tuple[str, dict, str | None]] = []
315
+ for where, filters in maps:
316
+ for key, value in filters.items():
317
+ if not isinstance(value, dict):
318
+ continue
319
+ label = where or f"filter {key!r}"
320
+ if value.get("type") != "filter_group":
321
+ entries.append((label, value, None))
322
+ continue
323
+ group = str(value.get("title") or "")
324
+ entries.extend(
325
+ (label, member, group) for member in (value.get("filters") or {}).values() if isinstance(member, dict)
326
+ )
327
+ return entries
308
328
 
309
329
 
310
330
  def _filters_of_type(dashboard: dict, filter_type: str) -> list[tuple[str, dict]]:
@@ -328,27 +348,29 @@ def _date_filter_mismatch(label: str, date_filter: dict, expected: dict | None)
328
348
  ]
329
349
 
330
350
 
331
- def _check_date_range(dashboard: dict, expected: dict | None) -> list[str]:
351
+ def _check_date_range(dashboard: dict, expected: dict | None, is_edit: bool) -> list[str]:
332
352
  """Check the dashboard's date range against the expectation.
333
353
 
334
- What counts as "the" date filter differs by shape, so the two are checked differently.
354
+ What counts as "the" date filter differs between a drafted and a saved dashboard, so the two
355
+ are checked differently. The case says which one it is: a saved dashboard can carry tabs as
356
+ well, so the document's shape cannot.
335
357
 
336
358
  A drafted dashboard carries one per tab, and the draft tool writes the same one onto every
337
359
  tab, so every tab must have it and every one must match. A tab left without a date filter is
338
360
  a failure: the expectation describes what the whole dashboard shows.
339
361
 
340
- A saved dashboard relayed for editing carries a flat map that can legitimately hold more
341
- than one date filter — a dashboard-wide one plus a dataset-scoped one — and nothing in the
342
- document says which is which. Requiring every entry to match would fail a dashboard for
343
- carrying a filter the fixture never described, so one matching entry satisfies the check and
344
- the rest are left to the notes.
362
+ A saved dashboard relayed for editing, tabbed or flat, can legitimately hold more than one
363
+ date filter — a dashboard-wide one plus a dataset-scoped one. Requiring every entry to match
364
+ would fail a dashboard for carrying a filter the fixture never described, so one matching
365
+ entry satisfies the check and the rest are left to the notes. A case that cares which filter
366
+ holds which range names them in ``date_filters``.
345
367
  """
346
368
  date_filters = _filters_of_type(dashboard, "date_filter")
347
369
  if not date_filters:
348
370
  return ["the dashboard carries no date filter to check the expected range against"]
349
371
 
350
372
  tabs = dashboard.get("tabs")
351
- if tabs:
373
+ if tabs and not is_edit:
352
374
  failures: list[str] = [
353
375
  f"tab {tab.get('id')!r} carries no date filter"
354
376
  for tab in tabs
@@ -385,24 +407,55 @@ def _describe_selection(selection: tuple[str, list] | None) -> str:
385
407
 
386
408
 
387
409
  def _check_filters(dashboard: dict, expected: list[dict]) -> list[str]:
388
- """Every expected attribute filter must be on the dashboard, with the stated selection.
410
+ """Every expected filter must be on the dashboard, with the stated selection or condition.
411
+
412
+ An entry is an attribute filter unless it names ``type`` as ``text_filter`` or
413
+ ``metric_value_filter``. ``group`` requires the filter to sit in the ``filter_group`` of that
414
+ title; without it, a grouped filter counts the same as one on its own.
389
415
 
390
416
  This exists because an edit can silently drop or widen the filters the user already had:
391
417
  the change asked for was a rename, and the filters coming back untouched is part of what
392
418
  "untouched" means. Filters the dashboard carries beyond the expected ones are left alone,
393
419
  the same way extra widgets are.
394
420
  """
395
- actual: dict[str | None, list[dict]] = {}
396
- for _label, attribute_filter in _filters_of_type(dashboard, "attribute_filter"):
397
- actual.setdefault(attribute_filter.get("using"), []).append(attribute_filter)
398
-
421
+ entries = _grouped_filter_entries(dashboard)
399
422
  failures: list[str] = []
400
423
  for exp in expected:
424
+ filter_type = _expected_filter_type(exp)
401
425
  using = exp.get("using")
402
- matches = actual.get(using) or []
403
- if not matches:
404
- failures.append(f"the dashboard carries no attribute filter on {using!r}")
426
+ candidates = [
427
+ (f, group)
428
+ for _label, f, group in entries
429
+ if f.get("type") == filter_type and _same_object_ref(f.get("using"), using)
430
+ ]
431
+ noun = filter_type.replace("_", " ")
432
+ if not candidates:
433
+ failures.append(f"the dashboard carries no {noun} on {using!r}")
434
+ continue
435
+ if "group" in exp:
436
+ grouped = [
437
+ (f, group) for f, group in candidates if group is not None and _norm(group) == _norm(exp["group"])
438
+ ]
439
+ if not grouped:
440
+ found = ", ".join(repr(group) for _f, group in candidates)
441
+ failures.append(f"{noun} on {using!r} sits in group {found}, expected {exp['group']!r}")
442
+ continue
443
+ candidates = grouped
444
+ matches = [f for f, _group in candidates]
445
+ if filter_type != "attribute_filter":
446
+ if not any(not _filter_value_mismatch(f, exp) for f in matches):
447
+ failures.append(f"{noun} on {using!r}: {'; '.join(_filter_value_mismatch(matches[0], exp))}")
405
448
  continue
449
+ if "display_as" in exp:
450
+ # GDAI-2488: a value change on a filter shown through a secondary label moves `using`
451
+ # to the primary label and keeps the secondary one as `display_as`. Losing it shows
452
+ # the user ids where they had names, while `using` and the selection stay right.
453
+ shown = [f for f in matches if _same_object_ref(f.get("display_as"), exp["display_as"])]
454
+ if not shown:
455
+ found = ", ".join(repr(f.get("display_as")) for f in matches)
456
+ failures.append(f"filter on {using!r} is displayed as {found}, expected {exp['display_as']!r}")
457
+ continue
458
+ matches = shown
406
459
  wanted = _expected_selection(exp)
407
460
  if not any(_selection_of(f) == wanted for f in matches):
408
461
  found = ", ".join(_describe_selection(_selection_of(f)) for f in matches)
@@ -410,6 +463,59 @@ def _check_filters(dashboard: dict, expected: list[dict]) -> list[str]:
410
463
  return failures
411
464
 
412
465
 
466
+ # The kinds a `filters` entry can name, and for each the keys compared beyond `using`. An entry
467
+ # without `type` is an attribute filter, which is every entry written before the others existed.
468
+ _FILTER_VALUE_KEYS: dict[str, tuple[str, ...]] = {
469
+ "attribute_filter": (),
470
+ "text_filter": ("condition", "value", "values", "case_sensitive"),
471
+ "metric_value_filter": ("conditions", "dimensionality", "null_values_as_zero"),
472
+ }
473
+ # What an attribute filter entry is compared on beyond `using`: its display label and selection.
474
+ _ATTRIBUTE_FILTER_KEYS = ("display_as", "include", "exclude", "selection")
475
+
476
+
477
+ def _expected_filter_type(expected_filter: dict) -> str:
478
+ return str(expected_filter.get("type") or "attribute_filter")
479
+
480
+
481
+ def _filter_value_mismatch(actual: dict, expected: dict) -> list[str]:
482
+ """Why a text or metric value filter differs from the expectation, on the keys it states.
483
+
484
+ ``values`` compares without order: the order of whole values for ``is``/``isNot`` carries no
485
+ meaning. A key the expectation leaves out is not compared, so a case states what it guards.
486
+ """
487
+ mismatches: list[str] = []
488
+ for key in _FILTER_VALUE_KEYS[_expected_filter_type(expected)]:
489
+ if key not in expected:
490
+ continue
491
+ want, got = expected[key], actual.get(key)
492
+ same = (
493
+ _same_unordered_values(want, got)
494
+ if key == "values" and isinstance(want, list) and isinstance(got, list)
495
+ else got == want
496
+ )
497
+ if not same:
498
+ mismatches.append(f"{key} expected {want!r}, got {got!r}")
499
+ return mismatches
500
+
501
+
502
+ def _same_unordered_values(want: list, got: list) -> bool:
503
+ """One-to-one match in any order, with each pair of the same type.
504
+
505
+ Not sorted: AAC allows `null` among the values, and Python cannot order it with strings.
506
+ Not stringified: `1` and `"1"` are different values.
507
+ """
508
+ remaining = list(got)
509
+ for expected in want:
510
+ index = next(
511
+ (i for i, actual in enumerate(remaining) if type(actual) is type(expected) and actual == expected), None
512
+ )
513
+ if index is None:
514
+ return False
515
+ del remaining[index]
516
+ return not remaining
517
+
518
+
413
519
  def _expected_selection(expected_filter: dict) -> tuple[str, list] | None:
414
520
  """Read a fixture's filter entry into the same shape ``_selection_of`` produces.
415
521
 
@@ -428,6 +534,231 @@ def _expected_selection(expected_filter: dict) -> tuple[str, list] | None:
428
534
  )
429
535
 
430
536
 
537
+ def _matching_widgets(widgets: list[dict], new_ids: set[str], expected: dict) -> list[dict]:
538
+ """The widgets an expected chart entry names: by ``id``, or by title among the authored charts.
539
+
540
+ Every widget the id appears under counts, whatever its title -- the checks that use this
541
+ look at what a chart is bound to, and a rename is ``charts_matched``'s business.
542
+ """
543
+ exp_id = expected.get("id")
544
+ if exp_id is not None:
545
+ return [w for w in widgets if w.get("visualization") == exp_id]
546
+ title = _norm(str(expected.get("title") or ""))
547
+ return [w for w in widgets if w.get("visualization") in new_ids and _norm(str(w.get("title") or "")) == title]
548
+
549
+
550
+ def _has_date_bindings(expected_output: dict) -> bool:
551
+ """Whether any expected chart states the date dataset its widget must follow.
552
+
553
+ Per chart, not per case: a chart that reaches no date dataset correctly ignores the date
554
+ filter, so "every widget has a date" is not an expectation that holds in general.
555
+ """
556
+ return any(isinstance(v, dict) and "date_dataset" in v for v in expected_output.get("visualizations") or [])
557
+
558
+
559
+ def _check_date_bindings(widgets: list[dict], new_ids: set[str], expected: list[dict]) -> list[str]:
560
+ """Every widget of a chart that states a ``date_dataset`` must be bound to exactly that one.
561
+
562
+ ``null`` means the widget must carry no date at all: the chart reaches no date dataset, and
563
+ a binding there would be one the dashboard cannot honour. The widget key is AAC's ``date``,
564
+ the declarative ``dateDataSet`` (GDAI-2391, GDAI-2462).
565
+ """
566
+ failures: list[str] = []
567
+ for exp in expected:
568
+ if not isinstance(exp, dict) or "date_dataset" not in exp:
569
+ continue
570
+ wanted = exp["date_dataset"]
571
+ label = exp.get("title") if exp.get("id") is None else exp.get("id")
572
+ candidates = _matching_widgets(widgets, new_ids, exp)
573
+ if not candidates:
574
+ # Not left to charts_matched alone: a check that could not look must not read as passed.
575
+ failures.append(f"cannot check the date binding of chart {label!r}: it is not on the dashboard")
576
+ continue
577
+ # Every placement: the expectation names the chart, so one right placement must not hide a wrong one.
578
+ if not all(w.get("date") == wanted for w in candidates):
579
+ found = ", ".join(repr(w.get("date")) for w in candidates)
580
+ failures.append(f"chart {label!r} follows date {found}, expected {wanted!r}")
581
+ return failures
582
+
583
+
584
+ def _tabs_of(dashboard: dict) -> list[dict]:
585
+ """The dashboard's tabs, reading a document with root sections as its single tab."""
586
+ tabs = dashboard.get("tabs")
587
+ if tabs:
588
+ return list(tabs)
589
+ return [{"sections": dashboard.get("sections") or [], "filters": dashboard.get("filters") or {}}]
590
+
591
+
592
+ def _check_tabs(dashboard: dict, new_ids: set[str], expected: list[dict]) -> list[str]:
593
+ """The dashboard must have exactly the expected tabs, in order (GDAI-2426).
594
+
595
+ A tab entry may name a ``title``, compared like every other title, and ``visualizations``
596
+ that must sit on that tab -- each an id, or ``{"title": ...}`` for an authored chart. An
597
+ empty entry asserts only that the tab exists, which is how a case says "one tab, any name".
598
+ """
599
+ tabs = _tabs_of(dashboard)
600
+ if len(tabs) != len(expected):
601
+ titles = ", ".join(repr(t.get("title")) for t in tabs)
602
+ return [f"the dashboard has {len(tabs)} tab(s) ({titles}), expected {len(expected)}"]
603
+ failures: list[str] = []
604
+ for index, (tab, exp) in enumerate(zip(tabs, expected)):
605
+ if "title" in exp and _norm(str(tab.get("title") or "")) != _norm(str(exp["title"] or "")):
606
+ failures.append(f"tab {index} is titled {tab.get('title')!r}, expected {exp['title']!r}")
607
+ tab_widgets = [w for section in tab.get("sections") or [] for w in section.get("widgets") or []]
608
+ for chart in exp.get("visualizations") or []:
609
+ entry = {"id": chart} if isinstance(chart, str) else {"id": None, **chart}
610
+ if not _matching_widgets(tab_widgets, new_ids, entry):
611
+ failures.append(f"tab {index} ({tab.get('title')!r}) does not hold chart {chart!r}")
612
+ return failures
613
+
614
+
615
+ # The keys a date filter is compared on. Absent on both sides is a match -- an all-time filter
616
+ # spells itself by leaving `from` and `to` out, and a dashboard-wide one by leaving `date` out.
617
+ _DATE_FILTER_KEYS = ("granularity", "from", "to", "date")
618
+
619
+
620
+ def _filter_mismatch(label: str, actual: dict, expected: dict) -> list[str]:
621
+ """Why ``actual`` is not the ``expected`` filter, or nothing.
622
+
623
+ A date filter is compared on its range and dataset, an attribute filter on its labels and
624
+ selection, both with absence significant. Any other kind is compared on the keys the
625
+ expectation names.
626
+ """
627
+ if actual.get("type") != expected.get("type"):
628
+ return [f"{label} is a {actual.get('type')!r}, expected a {expected.get('type')!r}"]
629
+ if expected.get("type") == "date_filter":
630
+ keys: tuple[str, ...] = _DATE_FILTER_KEYS
631
+ elif expected.get("type") == "attribute_filter":
632
+ if _selection_of(actual) != _selection_of(expected):
633
+ return [
634
+ f"{label} selects {_describe_selection(_selection_of(actual))}, "
635
+ f"expected {_describe_selection(_selection_of(expected))}"
636
+ ]
637
+ keys = ("using", "display_as")
638
+ else:
639
+ keys = tuple(k for k in expected if k != "type")
640
+ return [
641
+ f"{label}: {key} expected {expected.get(key)!r}, got {actual.get(key)!r}"
642
+ for key in keys
643
+ if not (
644
+ _same_object_ref(actual.get(key), expected.get(key))
645
+ if key in ("using", "display_as")
646
+ else actual.get(key) == expected.get(key)
647
+ )
648
+ ]
649
+
650
+
651
+ def _same_object_ref(actual: object, expected: object) -> bool:
652
+ """Whether two AAC refs name the same object, reading ``attribute/<id>`` as ``label/<id>``.
653
+
654
+ The draft and patch tools accept either prefix on a filter's ``using``, and the model picks
655
+ one per run. An attribute and its primary label share the id, so the two spell one filter;
656
+ matching the prefix verbatim failed a correct run on the model's wording.
657
+ """
658
+ if actual == expected:
659
+ return True
660
+ if not (isinstance(actual, str) and isinstance(expected, str)):
661
+ return False
662
+ return _ref_without_attribute_prefix(actual) == _ref_without_attribute_prefix(expected)
663
+
664
+
665
+ def _ref_without_attribute_prefix(ref: str) -> str:
666
+ return "label/" + ref.removeprefix("attribute/") if ref.startswith("attribute/") else ref
667
+
668
+
669
+ def _check_tab_filters(dashboard: dict, expected: dict[str, dict]) -> list[str]:
670
+ """Each named tab must carry exactly the expected filters, by the filter's own id.
671
+
672
+ Exact on purpose: GDAI-2540 is a filter that turns up on a tab it never belonged to, which a
673
+ check of only the expected entries would pass. Keyed by the saved dashboard's local filter
674
+ id rather than by label, so two filters on the same label stay distinguishable.
675
+ """
676
+ tabs = {tab.get("id"): tab for tab in dashboard.get("tabs") or []}
677
+ failures: list[str] = []
678
+ for tab_id, expected_filters in expected.items():
679
+ tab = tabs.get(tab_id)
680
+ if tab is None:
681
+ failures.append(f"the dashboard has no tab {tab_id!r}")
682
+ continue
683
+ actual = {k: v for k, v in (tab.get("filters") or {}).items() if isinstance(v, dict)}
684
+ failures.extend(
685
+ f"tab {tab_id!r} carries filter {filter_id!r}, which it should not"
686
+ for filter_id in sorted(actual.keys() - expected_filters.keys())
687
+ )
688
+ failures.extend(
689
+ f"tab {tab_id!r} lost filter {filter_id!r}" for filter_id in sorted(expected_filters.keys() - actual.keys())
690
+ )
691
+ for filter_id in sorted(expected_filters.keys() & actual.keys()):
692
+ failures.extend(
693
+ _filter_mismatch(f"tab {tab_id!r} filter {filter_id!r}", actual[filter_id], expected_filters[filter_id])
694
+ )
695
+ return failures
696
+
697
+
698
+ def _filters_by_id(dashboard: dict) -> dict[str, list[tuple[str, dict]]]:
699
+ """Every filter by its own id, with where it was found; one id can recur across tabs."""
700
+ found: dict[str, list[tuple[str, dict]]] = {}
701
+ for tab in dashboard.get("tabs") or [{"filters": dashboard.get("filters") or {}}]:
702
+ where = f"tab {tab['id']!r} " if "id" in tab else ""
703
+ for filter_id, value in (tab.get("filters") or {}).items():
704
+ if isinstance(value, dict):
705
+ found.setdefault(filter_id, []).append((f"{where}filter {filter_id!r}", value))
706
+ return found
707
+
708
+
709
+ def _check_date_filters(dashboard: dict, expected: dict[str, dict | None]) -> list[str]:
710
+ """Every named date filter must still be there, on the expected range (GDAI-2491).
711
+
712
+ Named by id because a dashboard can carry more than one date filter and the case states
713
+ which one holds which range. Filters the expectation does not name are left alone.
714
+
715
+ ``date`` is the dataset the filter itself is pinned to, not a widget's date binding, and its
716
+ absence is significant as in ``tab_filters``: an entry without it, or ``null`` for all time,
717
+ is the main filter, which follows each widget's own date dataset. A range kept on another
718
+ dataset, or a main filter pinned to one, is a changed filter.
719
+ """
720
+ found = _filters_by_id(dashboard)
721
+ failures: list[str] = []
722
+ for filter_id, wanted in expected.items():
723
+ entries = found.get(filter_id)
724
+ if not entries:
725
+ failures.append(f"the dashboard lost date filter {filter_id!r}")
726
+ continue
727
+ for label, date_filter in entries:
728
+ if date_filter.get("type") != "date_filter":
729
+ failures.append(f"{label} is a {date_filter.get('type')!r}, expected a date filter")
730
+ continue
731
+ failures.extend(_date_filter_mismatch(label, date_filter, wanted))
732
+ dataset = None if wanted is None else wanted.get("date")
733
+ if date_filter.get("date") != dataset:
734
+ failures.append(f"{label}: date dataset expected {dataset!r}, got {date_filter.get('date')!r}")
735
+ return failures
736
+
737
+
738
+ def _check_preserved_widgets(widgets: list[dict], expected: list[dict]) -> list[str]:
739
+ """A widget the change did not target must keep every key the expectation lists (GDAI-2541).
740
+
741
+ Compared on the keys listed and nothing else, so a case states what it guards -- AAC writes
742
+ a hidden title as ``title: false`` and drills as ``interactions``.
743
+ """
744
+ failures: list[str] = []
745
+ for exp in expected:
746
+ viz_id = exp.get("visualization")
747
+ keys = [k for k in exp if k != "visualization"]
748
+ candidates = [w for w in widgets if w.get("visualization") == viz_id]
749
+ if not candidates:
750
+ failures.append(f"the widget showing {viz_id!r} is gone")
751
+ continue
752
+ # Every placement keeps the keys: one intact copy must not hide one that lost its drill.
753
+ mismatched = [w for w in candidates if not all(w.get(k) == exp[k] for k in keys)]
754
+ if not mismatched:
755
+ continue
756
+ widget = mismatched[0]
757
+ changed = ", ".join(f"{k} {widget.get(k)!r} (expected {exp[k]!r})" for k in keys if widget.get(k) != exp[k])
758
+ failures.append(f"the widget showing {viz_id!r} changed: {changed}")
759
+ return failures
760
+
761
+
431
762
  def _date_range_text(date_range: dict | None) -> str:
432
763
  """Render a date range the way a user would say it, for the simulated reply.
433
764
 
@@ -463,9 +794,26 @@ def build_simulated_reply(expected_output: dict) -> str:
463
794
  if existing:
464
795
  segments.append(", ".join(existing))
465
796
  segments.extend(f'a new chart titled "{title}"' for title in authored)
797
+ # Only titled tabs are said: an untitled entry asserts a tab count, and asking for "one tab"
798
+ # out loud would be the reply leading the agent rather than the question. Each tab names its
799
+ # charts, since `_check_tabs` requires every chart on its own tab and the agent cannot guess.
800
+ titles_by_id = {v.get("id"): str(v.get("title", "")) for v in visualizations if v.get("id") is not None}
801
+ placements = []
802
+ for tab in expected_output.get("tabs") or []:
803
+ if not tab.get("title"):
804
+ continue
805
+ charts = [
806
+ titles_by_id.get(c, c) if isinstance(c, str) else str(c.get("title", ""))
807
+ for c in tab.get("visualizations") or []
808
+ ]
809
+ on_tab = f'the "{tab["title"]}" tab'
810
+ quoted = " and ".join(f'"{c}"' for c in charts)
811
+ placements.append(f"{quoted} on {on_tab}" if charts else on_tab)
812
+ tabs = f"Use these tabs: {'; '.join(placements)}. " if placements else ""
466
813
 
467
814
  return (
468
815
  f"Please use these charts: {', and '.join(segments)}. "
816
+ f"{tabs}"
469
817
  f"Date range: {_date_range_text(expected_output.get('date_range'))}. "
470
818
  f"Anything else is up to you. Please create the dashboard now."
471
819
  )
@@ -544,7 +892,30 @@ def _validate_expectation(expected_output: dict) -> None:
544
892
  raise ValueError(f"a filter expectation must be an object, got {entry!r}")
545
893
  if not entry.get("using"):
546
894
  raise ValueError(f"a filter expectation needs the label it filters on, got {entry!r}")
547
- _expected_selection(entry)
895
+ if "display_as" in entry and not (isinstance(entry["display_as"], str) and entry["display_as"]):
896
+ raise ValueError(f"a filter's display_as must name a label, got {entry!r}")
897
+ if "group" in entry and not (isinstance(entry["group"], str) and entry["group"]):
898
+ raise ValueError(f"a filter's group must name the group's title, got {entry!r}")
899
+ filter_type = _expected_filter_type(entry)
900
+ if filter_type not in _FILTER_VALUE_KEYS:
901
+ raise ValueError(f"a filter expectation's type must be one of {sorted(_FILTER_VALUE_KEYS)}, got {entry!r}")
902
+ # A key outside the compared ones, a misspelling included, would assert nothing.
903
+ compared = _ATTRIBUTE_FILTER_KEYS if filter_type == "attribute_filter" else _FILTER_VALUE_KEYS[filter_type]
904
+ unknown = sorted(entry.keys() - {"type", "using", "group", *compared})
905
+ if unknown:
906
+ raise ValueError(f"a {filter_type.replace('_', ' ')} expectation cannot state {unknown}, got {entry!r}")
907
+ if filter_type == "attribute_filter":
908
+ _expected_selection(entry)
909
+ elif filter_type == "text_filter":
910
+ if not entry.get("condition") or ("value" in entry) == ("values" in entry):
911
+ raise ValueError(f"a text filter expectation needs a condition and one of value/values, got {entry!r}")
912
+ if "values" in entry and not isinstance(entry["values"], list):
913
+ raise ValueError(f"a text filter's values must be a list, got {entry!r}")
914
+ if "value" in entry and not isinstance(entry["value"], str):
915
+ raise ValueError(f"a text filter's value must be a string, got {entry!r}")
916
+ elif not (isinstance(entry.get("conditions"), list) and entry["conditions"]):
917
+ raise ValueError(f"a metric value filter expectation needs its conditions, got {entry!r}")
918
+ _validate_layout_expectations(expected_output)
548
919
  saved_id = expected_output.get("saved_dashboard_id")
549
920
  if _is_edit(expected_output):
550
921
  if not isinstance(saved_id, str) or not saved_id:
@@ -554,6 +925,99 @@ def _validate_expectation(expected_output: dict) -> None:
554
925
  _min_new_visualizations(expected_output)
555
926
 
556
927
 
928
+ def _validate_layout_expectations(expected_output: dict) -> None:
929
+ """Shape checks for the opt-in keys on charts, tabs, per-tab filters and preserved widgets.
930
+
931
+ Raises:
932
+ ValueError: one of them is unusable. Loud for the same reason as the rest of the
933
+ fixture checks: a typo would otherwise assert nothing and read green.
934
+ """
935
+ for chart in expected_output.get("visualizations") or []:
936
+ if isinstance(chart, dict) and "date_dataset" in chart and not _is_dataset_or_null(chart["date_dataset"]):
937
+ raise ValueError(f"date_dataset must name a date dataset or be null, got {chart!r}")
938
+
939
+ if "tabs" in expected_output:
940
+ tabs = expected_output["tabs"]
941
+ if not isinstance(tabs, list) or not tabs or not all(isinstance(t, dict) for t in tabs):
942
+ raise ValueError(f"tabs must be a non-empty list of tab objects, got {tabs!r}")
943
+ for tab in tabs:
944
+ charts = tab.get("visualizations") or []
945
+ if not isinstance(charts, list) or not all(
946
+ (isinstance(c, str) and c) or (isinstance(c, dict) and c.get("title")) for c in charts
947
+ ):
948
+ raise ValueError(f"a tab's visualizations must be chart ids or {{'title': ...}}, got {tab!r}")
949
+
950
+ if "tab_filters" in expected_output:
951
+ tab_filters = expected_output["tab_filters"]
952
+ if not isinstance(tab_filters, dict) or not tab_filters:
953
+ raise ValueError(f"tab_filters must map tab ids to their filters, got {tab_filters!r}")
954
+ for tab_id, filters in tab_filters.items():
955
+ if not isinstance(filters, dict) or not all(
956
+ isinstance(f, dict) and f.get("type") for f in filters.values()
957
+ ):
958
+ raise ValueError(f"tab_filters[{tab_id!r}] must map filter ids to typed filters, got {filters!r}")
959
+ for filter_id, entry in filters.items():
960
+ if entry["type"] == "attribute_filter":
961
+ _validate_tab_attribute_filter(f"tab_filters[{tab_id!r}][{filter_id!r}]", entry)
962
+
963
+ if "date_filters" in expected_output:
964
+ date_filters = expected_output["date_filters"]
965
+ if not isinstance(date_filters, dict) or not date_filters:
966
+ raise ValueError(f"date_filters must map filter ids to a range or null, got {date_filters!r}")
967
+ for filter_id, date_range in date_filters.items():
968
+ if date_range is not None and not isinstance(date_range, dict):
969
+ raise ValueError(f"date_filters[{filter_id!r}] must be an object or null, got {date_range!r}")
970
+ if date_range == {}:
971
+ # Not all time: every bound would be compared against an absent one, and a real
972
+ # date filter always has a granularity, so the case could never pass.
973
+ raise ValueError(
974
+ f"date_filters[{filter_id!r}] is empty; use null for all time or state granularity/from/to"
975
+ )
976
+ unknown = sorted((date_range or {}).keys() - set(_DATE_FILTER_KEYS))
977
+ if unknown:
978
+ raise ValueError(
979
+ f"date_filters[{filter_id!r}] states {unknown}, which is not one of {list(_DATE_FILTER_KEYS)}"
980
+ )
981
+ if date_range and "date" in date_range and not _is_dataset_or_null(date_range["date"]):
982
+ raise ValueError(
983
+ f"date_filters[{filter_id!r}] date must name a date dataset or be null, got {date_range['date']!r}"
984
+ )
985
+
986
+ if "preserved_widgets" in expected_output:
987
+ preserved = expected_output["preserved_widgets"]
988
+ if not isinstance(preserved, list) or not preserved:
989
+ raise ValueError(f"preserved_widgets must be a non-empty list, got {preserved!r}")
990
+ for widget in preserved:
991
+ if not isinstance(widget, dict) or not widget.get("visualization") or len(widget) < 2:
992
+ raise ValueError(f"a preserved widget needs its visualization and a key to guard, got {widget!r}")
993
+
994
+
995
+ def _is_dataset_or_null(value: object) -> bool:
996
+ return value is None or (isinstance(value, str) and bool(value))
997
+
998
+
999
+ def _validate_tab_attribute_filter(where: str, entry: dict) -> None:
1000
+ """An attribute filter in ``tab_filters`` uses the AAC shape: the selection sits under ``state``.
1001
+
1002
+ The ``filters`` shorthand (``include``/``exclude``/``selection`` on the entry) does not apply
1003
+ here. Read as AAC, it would compare as "all" and pass a case that checks nothing.
1004
+
1005
+ Raises:
1006
+ ValueError: the entry is not in the AAC shape.
1007
+ """
1008
+ shorthand = sorted({"include", "exclude", "selection"} & entry.keys())
1009
+ if shorthand:
1010
+ raise ValueError(f"{where} puts {shorthand} on the filter; tab_filters takes the AAC `state` object")
1011
+ state = entry.get("state")
1012
+ if state is None:
1013
+ return
1014
+ if not isinstance(state, dict):
1015
+ raise ValueError(f"{where} state must be an object, got {state!r}")
1016
+ for kind in ("include", "exclude"):
1017
+ if kind in state and not isinstance(state[kind], list):
1018
+ raise ValueError(f"{where} state.{kind} must be a list, got {state[kind]!r}")
1019
+
1020
+
557
1021
  @dataclass(frozen=True)
558
1022
  class _Applies:
559
1023
  """Which of the conditional checks the case applies.
@@ -567,6 +1031,24 @@ class _Applies:
567
1031
  patch: bool
568
1032
  date: bool
569
1033
  filters: bool
1034
+ date_bindings: bool = False
1035
+ tabs: bool = False
1036
+ tab_filters: bool = False
1037
+ date_filters: bool = False
1038
+ preserved_widgets: bool = False
1039
+
1040
+ @classmethod
1041
+ def of(cls, expected_output: dict) -> _Applies:
1042
+ return cls(
1043
+ patch=_is_edit(expected_output),
1044
+ date=_has_date_range(expected_output),
1045
+ filters=_has_filters(expected_output),
1046
+ date_bindings=_has_date_bindings(expected_output),
1047
+ tabs="tabs" in expected_output,
1048
+ tab_filters="tab_filters" in expected_output,
1049
+ date_filters="date_filters" in expected_output,
1050
+ preserved_widgets="preserved_widgets" in expected_output,
1051
+ )
570
1052
 
571
1053
 
572
1054
  @dataclass
@@ -598,6 +1080,14 @@ class DashboardEvaluation:
598
1080
  patch_applies: bool = True
599
1081
  references_carried: bool = True
600
1082
  titles_matched: bool = True
1083
+ # False by default, unlike the older checks above: every early return then scores them as
1084
+ # failed without having to name them, and only the run that reached a document sets them.
1085
+ # They are published only where the case applies them, so a default never shows otherwise.
1086
+ date_bindings_correct: bool = False
1087
+ tabs_correct: bool = False
1088
+ tab_filters_correct: bool = False
1089
+ date_filters_correct: bool = False
1090
+ widgets_preserved: bool = False
601
1091
  failures: list[str] = field(default_factory=list)
602
1092
  notes: list[str] = field(default_factory=list)
603
1093
 
@@ -626,6 +1116,18 @@ class DashboardEvaluation:
626
1116
  # Prefixed: alert_skill already publishes a `filters_correct` score, and the combo
627
1117
  # report resolves a trace's skill by which score names it carries.
628
1118
  checks["dashboard_filters_correct"] = self.filters_correct
1119
+ # All prefixed, for the same reason as the filters check: the combo report tells skills
1120
+ # apart by the score names a trace carries.
1121
+ if self.applies.date_bindings:
1122
+ checks["dashboard_date_bindings_correct"] = self.date_bindings_correct
1123
+ if self.applies.tabs:
1124
+ checks["dashboard_tabs_correct"] = self.tabs_correct
1125
+ if self.applies.tab_filters:
1126
+ checks["dashboard_tab_filters_correct"] = self.tab_filters_correct
1127
+ if self.applies.date_filters:
1128
+ checks["dashboard_date_filters_correct"] = self.date_filters_correct
1129
+ if self.applies.preserved_widgets:
1130
+ checks["dashboard_widgets_preserved"] = self.widgets_preserved
629
1131
  return checks
630
1132
 
631
1133
  @property
@@ -693,7 +1195,7 @@ def evaluate_dashboard_response(
693
1195
  without an agent.
694
1196
  """
695
1197
  is_edit = _is_edit(expected_output)
696
- applies = _Applies(patch=is_edit, date=_has_date_range(expected_output), filters=_has_filters(expected_output))
1198
+ applies = _Applies.of(expected_output)
697
1199
  tool = _producing_tool(expected_output)
698
1200
 
699
1201
  if tool_result is None:
@@ -812,11 +1314,26 @@ def evaluate_dashboard_response(
812
1314
  title_notes = title_mismatches
813
1315
  reference_notes = _check_references(widgets, known_ids)
814
1316
  date_failures = (
815
- _check_date_range(document, expected_output.get("date_range")) if _has_date_range(expected_output) else []
1317
+ _check_date_range(document, expected_output.get("date_range"), applies.patch)
1318
+ if _has_date_range(expected_output)
1319
+ else []
816
1320
  )
817
1321
  filter_failures = (
818
1322
  _check_filters(document, expected_output.get("filters") or []) if _has_filters(expected_output) else []
819
1323
  )
1324
+ binding_failures = (
1325
+ _check_date_bindings(widgets, new_ids, expected_output.get("visualizations") or [])
1326
+ if applies.date_bindings
1327
+ else []
1328
+ )
1329
+ tab_failures = _check_tabs(document, new_ids, expected_output["tabs"]) if applies.tabs else []
1330
+ tab_filter_failures = _check_tab_filters(document, expected_output["tab_filters"]) if applies.tab_filters else []
1331
+ date_filter_failures = (
1332
+ _check_date_filters(document, expected_output["date_filters"]) if applies.date_filters else []
1333
+ )
1334
+ preserved_failures = (
1335
+ _check_preserved_widgets(widgets, expected_output["preserved_widgets"]) if applies.preserved_widgets else []
1336
+ )
820
1337
 
821
1338
  return DashboardEvaluation(
822
1339
  drafted=True,
@@ -831,7 +1348,23 @@ def evaluate_dashboard_response(
831
1348
  patch_applies=True,
832
1349
  references_carried=not reference_notes,
833
1350
  titles_matched=not title_mismatches,
834
- failures=[*chart_failures, *saved_failures, *date_failures, *filter_failures, *new_failures],
1351
+ date_bindings_correct=not binding_failures,
1352
+ tabs_correct=not tab_failures,
1353
+ tab_filters_correct=not tab_filter_failures,
1354
+ date_filters_correct=not date_filter_failures,
1355
+ widgets_preserved=not preserved_failures,
1356
+ failures=[
1357
+ *chart_failures,
1358
+ *saved_failures,
1359
+ *date_failures,
1360
+ *filter_failures,
1361
+ *binding_failures,
1362
+ *tab_failures,
1363
+ *tab_filter_failures,
1364
+ *date_filter_failures,
1365
+ *preserved_failures,
1366
+ *new_failures,
1367
+ ],
835
1368
  notes=[*title_notes, *reference_notes],
836
1369
  )
837
1370
 
@@ -944,6 +1477,22 @@ def _execute_single_dashboard_run(
944
1477
  )
945
1478
 
946
1479
 
1480
+ def _run_detail(run: DashboardRunResult) -> dict[str, Any]:
1481
+ """The diagnostic fields for ONE run, shared by the best run and every failing one.
1482
+
1483
+ Extracted so a failing run is described by exactly the same keys as the winning run --
1484
+ for this kind that means the per-check breakdown and the failures it reported, which is
1485
+ the whole diagnosis.
1486
+ """
1487
+ return {
1488
+ **run.evaluation.strict_checks,
1489
+ **run.evaluation.diagnostics,
1490
+ "failures": run.evaluation.failures,
1491
+ "notes": run.evaluation.notes,
1492
+ "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events),
1493
+ }
1494
+
1495
+
947
1496
  def run_agentic_dashboard_skill(
948
1497
  host: str,
949
1498
  token: str,
@@ -1108,13 +1657,14 @@ def evaluate_agentic_dashboard_skill(
1108
1657
  runs_effective = len(summary.run_results)
1109
1658
 
1110
1659
  best = summary.best
1111
- detail: dict[str, Any] = {
1112
- **best.evaluation.strict_checks,
1113
- **best.evaluation.diagnostics,
1114
- "failures": best.evaluation.failures,
1115
- "notes": best.evaluation.notes,
1116
- "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
1117
- }
1660
+ detail: dict[str, Any] = _run_detail(best)
1661
+ # Same predicate runs_passed is taken over, so an item's failed_runs and its counts
1662
+ # cannot disagree about which runs failed.
1663
+ failed_runs = build_failed_runs(
1664
+ summary.run_results,
1665
+ passed=lambda r: r.evaluation.strict_pass,
1666
+ detail=_run_detail,
1667
+ )
1118
1668
 
1119
1669
  if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k):
1120
1670
  gate_note = gate_failure_note(gate, runs_passed, runs_effective)
@@ -1148,6 +1698,7 @@ def evaluate_agentic_dashboard_skill(
1148
1698
  exc.detail = detail
1149
1699
  exc.runs_passed = runs_passed
1150
1700
  exc.runs_effective = runs_effective
1701
+ exc.failed_runs = failed_runs
1151
1702
  raise exc
1152
1703
  return AgenticEvalOutcome(
1153
1704
  runs_passed=runs_passed,
@@ -1157,4 +1708,5 @@ def evaluate_agentic_dashboard_skill(
1157
1708
  response_id=best.response_id,
1158
1709
  detail=detail,
1159
1710
  timings=item_timings,
1711
+ failed_runs=failed_runs,
1160
1712
  )