gooddata-eval 1.76.1.dev2__py3-none-any.whl → 1.76.1.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gooddata_eval/cli/agentic_runner.py +38 -0
- gooddata_eval/core/agentic/_failed_runs.py +62 -0
- gooddata_eval/core/agentic/alert_skill.py +42 -18
- gooddata_eval/core/agentic/anomaly_detection.py +10 -0
- gooddata_eval/core/agentic/dashboard_skill.py +589 -37
- gooddata_eval/core/agentic/general_question.py +70 -22
- gooddata_eval/core/agentic/guardrail.py +67 -18
- gooddata_eval/core/agentic/kda_skill.py +38 -14
- gooddata_eval/core/agentic/metric_skill.py +34 -12
- gooddata_eval/core/agentic/report_skill.py +22 -7
- gooddata_eval/core/agentic/search_tool.py +28 -6
- gooddata_eval/core/agentic/visualization.py +34 -9
- gooddata_eval/core/agentic/what_if.py +26 -2
- gooddata_eval/core/chat/sse_client.py +43 -1
- gooddata_eval/core/evaluators/_llm_judge.py +12 -0
- gooddata_eval/core/models.py +7 -0
- gooddata_eval/core/reporting/html_report.py +27 -1
- gooddata_eval/core/reporting/json_report.py +6 -0
- gooddata_eval/core/runner.py +40 -0
- {gooddata_eval-1.76.1.dev2.dist-info → gooddata_eval-1.76.1.dev4.dist-info}/METADATA +2 -2
- {gooddata_eval-1.76.1.dev2.dist-info → gooddata_eval-1.76.1.dev4.dist-info}/RECORD +24 -23
- {gooddata_eval-1.76.1.dev2.dist-info → gooddata_eval-1.76.1.dev4.dist-info}/WHEEL +0 -0
- {gooddata_eval-1.76.1.dev2.dist-info → gooddata_eval-1.76.1.dev4.dist-info}/entry_points.txt +0 -0
- {gooddata_eval-1.76.1.dev2.dist-info → gooddata_eval-1.76.1.dev4.dist-info}/licenses/LICENSE.txt +0 -0
|
@@ -9,6 +9,7 @@ from typing import Any
|
|
|
9
9
|
|
|
10
10
|
import jsonpatch
|
|
11
11
|
|
|
12
|
+
from gooddata_eval.core.agentic._failed_runs import build_failed_runs
|
|
12
13
|
from gooddata_eval.core.agentic._gate import (
|
|
13
14
|
DEFAULT_GATE,
|
|
14
15
|
EvalGate,
|
|
@@ -294,17 +295,36 @@ def _filter_entries(dashboard: dict) -> list[tuple[str, dict]]:
|
|
|
294
295
|
local identifier, so the role has to be read from each entry's ``type`` instead. The label
|
|
295
296
|
is only there to say *where* a failure was found.
|
|
296
297
|
"""
|
|
298
|
+
return [(label, value) for label, value, _group in _grouped_filter_entries(dashboard)]
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _grouped_filter_entries(dashboard: dict) -> list[tuple[str, dict, str | None]]:
|
|
302
|
+
"""``(label, filter, group title)`` for every filter, with a group's members read in its place.
|
|
303
|
+
|
|
304
|
+
A ``filter_group`` holds its members in a nested map, so reading only the outer map would
|
|
305
|
+
miss every filter the agent chose to group and report it missing. The group entry itself is
|
|
306
|
+
not a filter and is not returned.
|
|
307
|
+
"""
|
|
297
308
|
tabs = dashboard.get("tabs")
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
309
|
+
maps = (
|
|
310
|
+
[(f"tab {tab.get('id')!r}", tab.get("filters") or {}) for tab in tabs]
|
|
311
|
+
if tabs
|
|
312
|
+
else [("", dashboard.get("filters") or {})]
|
|
313
|
+
)
|
|
314
|
+
entries: list[tuple[str, dict, str | None]] = []
|
|
315
|
+
for where, filters in maps:
|
|
316
|
+
for key, value in filters.items():
|
|
317
|
+
if not isinstance(value, dict):
|
|
318
|
+
continue
|
|
319
|
+
label = where or f"filter {key!r}"
|
|
320
|
+
if value.get("type") != "filter_group":
|
|
321
|
+
entries.append((label, value, None))
|
|
322
|
+
continue
|
|
323
|
+
group = str(value.get("title") or "")
|
|
324
|
+
entries.extend(
|
|
325
|
+
(label, member, group) for member in (value.get("filters") or {}).values() if isinstance(member, dict)
|
|
326
|
+
)
|
|
327
|
+
return entries
|
|
308
328
|
|
|
309
329
|
|
|
310
330
|
def _filters_of_type(dashboard: dict, filter_type: str) -> list[tuple[str, dict]]:
|
|
@@ -328,27 +348,29 @@ def _date_filter_mismatch(label: str, date_filter: dict, expected: dict | None)
|
|
|
328
348
|
]
|
|
329
349
|
|
|
330
350
|
|
|
331
|
-
def _check_date_range(dashboard: dict, expected: dict | None) -> list[str]:
|
|
351
|
+
def _check_date_range(dashboard: dict, expected: dict | None, is_edit: bool) -> list[str]:
|
|
332
352
|
"""Check the dashboard's date range against the expectation.
|
|
333
353
|
|
|
334
|
-
What counts as "the" date filter differs
|
|
354
|
+
What counts as "the" date filter differs between a drafted and a saved dashboard, so the two
|
|
355
|
+
are checked differently. The case says which one it is: a saved dashboard can carry tabs as
|
|
356
|
+
well, so the document's shape cannot.
|
|
335
357
|
|
|
336
358
|
A drafted dashboard carries one per tab, and the draft tool writes the same one onto every
|
|
337
359
|
tab, so every tab must have it and every one must match. A tab left without a date filter is
|
|
338
360
|
a failure: the expectation describes what the whole dashboard shows.
|
|
339
361
|
|
|
340
|
-
A saved dashboard relayed for editing
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
362
|
+
A saved dashboard relayed for editing, tabbed or flat, can legitimately hold more than one
|
|
363
|
+
date filter — a dashboard-wide one plus a dataset-scoped one. Requiring every entry to match
|
|
364
|
+
would fail a dashboard for carrying a filter the fixture never described, so one matching
|
|
365
|
+
entry satisfies the check and the rest are left to the notes. A case that cares which filter
|
|
366
|
+
holds which range names them in ``date_filters``.
|
|
345
367
|
"""
|
|
346
368
|
date_filters = _filters_of_type(dashboard, "date_filter")
|
|
347
369
|
if not date_filters:
|
|
348
370
|
return ["the dashboard carries no date filter to check the expected range against"]
|
|
349
371
|
|
|
350
372
|
tabs = dashboard.get("tabs")
|
|
351
|
-
if tabs:
|
|
373
|
+
if tabs and not is_edit:
|
|
352
374
|
failures: list[str] = [
|
|
353
375
|
f"tab {tab.get('id')!r} carries no date filter"
|
|
354
376
|
for tab in tabs
|
|
@@ -385,24 +407,55 @@ def _describe_selection(selection: tuple[str, list] | None) -> str:
|
|
|
385
407
|
|
|
386
408
|
|
|
387
409
|
def _check_filters(dashboard: dict, expected: list[dict]) -> list[str]:
|
|
388
|
-
"""Every expected
|
|
410
|
+
"""Every expected filter must be on the dashboard, with the stated selection or condition.
|
|
411
|
+
|
|
412
|
+
An entry is an attribute filter unless it names ``type`` as ``text_filter`` or
|
|
413
|
+
``metric_value_filter``. ``group`` requires the filter to sit in the ``filter_group`` of that
|
|
414
|
+
title; without it, a grouped filter counts the same as one on its own.
|
|
389
415
|
|
|
390
416
|
This exists because an edit can silently drop or widen the filters the user already had:
|
|
391
417
|
the change asked for was a rename, and the filters coming back untouched is part of what
|
|
392
418
|
"untouched" means. Filters the dashboard carries beyond the expected ones are left alone,
|
|
393
419
|
the same way extra widgets are.
|
|
394
420
|
"""
|
|
395
|
-
|
|
396
|
-
for _label, attribute_filter in _filters_of_type(dashboard, "attribute_filter"):
|
|
397
|
-
actual.setdefault(attribute_filter.get("using"), []).append(attribute_filter)
|
|
398
|
-
|
|
421
|
+
entries = _grouped_filter_entries(dashboard)
|
|
399
422
|
failures: list[str] = []
|
|
400
423
|
for exp in expected:
|
|
424
|
+
filter_type = _expected_filter_type(exp)
|
|
401
425
|
using = exp.get("using")
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
426
|
+
candidates = [
|
|
427
|
+
(f, group)
|
|
428
|
+
for _label, f, group in entries
|
|
429
|
+
if f.get("type") == filter_type and _same_object_ref(f.get("using"), using)
|
|
430
|
+
]
|
|
431
|
+
noun = filter_type.replace("_", " ")
|
|
432
|
+
if not candidates:
|
|
433
|
+
failures.append(f"the dashboard carries no {noun} on {using!r}")
|
|
434
|
+
continue
|
|
435
|
+
if "group" in exp:
|
|
436
|
+
grouped = [
|
|
437
|
+
(f, group) for f, group in candidates if group is not None and _norm(group) == _norm(exp["group"])
|
|
438
|
+
]
|
|
439
|
+
if not grouped:
|
|
440
|
+
found = ", ".join(repr(group) for _f, group in candidates)
|
|
441
|
+
failures.append(f"{noun} on {using!r} sits in group {found}, expected {exp['group']!r}")
|
|
442
|
+
continue
|
|
443
|
+
candidates = grouped
|
|
444
|
+
matches = [f for f, _group in candidates]
|
|
445
|
+
if filter_type != "attribute_filter":
|
|
446
|
+
if not any(not _filter_value_mismatch(f, exp) for f in matches):
|
|
447
|
+
failures.append(f"{noun} on {using!r}: {'; '.join(_filter_value_mismatch(matches[0], exp))}")
|
|
405
448
|
continue
|
|
449
|
+
if "display_as" in exp:
|
|
450
|
+
# GDAI-2488: a value change on a filter shown through a secondary label moves `using`
|
|
451
|
+
# to the primary label and keeps the secondary one as `display_as`. Losing it shows
|
|
452
|
+
# the user ids where they had names, while `using` and the selection stay right.
|
|
453
|
+
shown = [f for f in matches if _same_object_ref(f.get("display_as"), exp["display_as"])]
|
|
454
|
+
if not shown:
|
|
455
|
+
found = ", ".join(repr(f.get("display_as")) for f in matches)
|
|
456
|
+
failures.append(f"filter on {using!r} is displayed as {found}, expected {exp['display_as']!r}")
|
|
457
|
+
continue
|
|
458
|
+
matches = shown
|
|
406
459
|
wanted = _expected_selection(exp)
|
|
407
460
|
if not any(_selection_of(f) == wanted for f in matches):
|
|
408
461
|
found = ", ".join(_describe_selection(_selection_of(f)) for f in matches)
|
|
@@ -410,6 +463,59 @@ def _check_filters(dashboard: dict, expected: list[dict]) -> list[str]:
|
|
|
410
463
|
return failures
|
|
411
464
|
|
|
412
465
|
|
|
466
|
+
# The kinds a `filters` entry can name, and for each the keys compared beyond `using`. An entry
|
|
467
|
+
# without `type` is an attribute filter, which is every entry written before the others existed.
|
|
468
|
+
_FILTER_VALUE_KEYS: dict[str, tuple[str, ...]] = {
|
|
469
|
+
"attribute_filter": (),
|
|
470
|
+
"text_filter": ("condition", "value", "values", "case_sensitive"),
|
|
471
|
+
"metric_value_filter": ("conditions", "dimensionality", "null_values_as_zero"),
|
|
472
|
+
}
|
|
473
|
+
# What an attribute filter entry is compared on beyond `using`: its display label and selection.
|
|
474
|
+
_ATTRIBUTE_FILTER_KEYS = ("display_as", "include", "exclude", "selection")
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def _expected_filter_type(expected_filter: dict) -> str:
|
|
478
|
+
return str(expected_filter.get("type") or "attribute_filter")
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _filter_value_mismatch(actual: dict, expected: dict) -> list[str]:
|
|
482
|
+
"""Why a text or metric value filter differs from the expectation, on the keys it states.
|
|
483
|
+
|
|
484
|
+
``values`` compares without order: the order of whole values for ``is``/``isNot`` carries no
|
|
485
|
+
meaning. A key the expectation leaves out is not compared, so a case states what it guards.
|
|
486
|
+
"""
|
|
487
|
+
mismatches: list[str] = []
|
|
488
|
+
for key in _FILTER_VALUE_KEYS[_expected_filter_type(expected)]:
|
|
489
|
+
if key not in expected:
|
|
490
|
+
continue
|
|
491
|
+
want, got = expected[key], actual.get(key)
|
|
492
|
+
same = (
|
|
493
|
+
_same_unordered_values(want, got)
|
|
494
|
+
if key == "values" and isinstance(want, list) and isinstance(got, list)
|
|
495
|
+
else got == want
|
|
496
|
+
)
|
|
497
|
+
if not same:
|
|
498
|
+
mismatches.append(f"{key} expected {want!r}, got {got!r}")
|
|
499
|
+
return mismatches
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def _same_unordered_values(want: list, got: list) -> bool:
|
|
503
|
+
"""One-to-one match in any order, with each pair of the same type.
|
|
504
|
+
|
|
505
|
+
Not sorted: AAC allows `null` among the values, and Python cannot order it with strings.
|
|
506
|
+
Not stringified: `1` and `"1"` are different values.
|
|
507
|
+
"""
|
|
508
|
+
remaining = list(got)
|
|
509
|
+
for expected in want:
|
|
510
|
+
index = next(
|
|
511
|
+
(i for i, actual in enumerate(remaining) if type(actual) is type(expected) and actual == expected), None
|
|
512
|
+
)
|
|
513
|
+
if index is None:
|
|
514
|
+
return False
|
|
515
|
+
del remaining[index]
|
|
516
|
+
return not remaining
|
|
517
|
+
|
|
518
|
+
|
|
413
519
|
def _expected_selection(expected_filter: dict) -> tuple[str, list] | None:
|
|
414
520
|
"""Read a fixture's filter entry into the same shape ``_selection_of`` produces.
|
|
415
521
|
|
|
@@ -428,6 +534,231 @@ def _expected_selection(expected_filter: dict) -> tuple[str, list] | None:
|
|
|
428
534
|
)
|
|
429
535
|
|
|
430
536
|
|
|
537
|
+
def _matching_widgets(widgets: list[dict], new_ids: set[str], expected: dict) -> list[dict]:
|
|
538
|
+
"""The widgets an expected chart entry names: by ``id``, or by title among the authored charts.
|
|
539
|
+
|
|
540
|
+
Every widget the id appears under counts, whatever its title -- the checks that use this
|
|
541
|
+
look at what a chart is bound to, and a rename is ``charts_matched``'s business.
|
|
542
|
+
"""
|
|
543
|
+
exp_id = expected.get("id")
|
|
544
|
+
if exp_id is not None:
|
|
545
|
+
return [w for w in widgets if w.get("visualization") == exp_id]
|
|
546
|
+
title = _norm(str(expected.get("title") or ""))
|
|
547
|
+
return [w for w in widgets if w.get("visualization") in new_ids and _norm(str(w.get("title") or "")) == title]
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def _has_date_bindings(expected_output: dict) -> bool:
|
|
551
|
+
"""Whether any expected chart states the date dataset its widget must follow.
|
|
552
|
+
|
|
553
|
+
Per chart, not per case: a chart that reaches no date dataset correctly ignores the date
|
|
554
|
+
filter, so "every widget has a date" is not an expectation that holds in general.
|
|
555
|
+
"""
|
|
556
|
+
return any(isinstance(v, dict) and "date_dataset" in v for v in expected_output.get("visualizations") or [])
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def _check_date_bindings(widgets: list[dict], new_ids: set[str], expected: list[dict]) -> list[str]:
|
|
560
|
+
"""Every widget of a chart that states a ``date_dataset`` must be bound to exactly that one.
|
|
561
|
+
|
|
562
|
+
``null`` means the widget must carry no date at all: the chart reaches no date dataset, and
|
|
563
|
+
a binding there would be one the dashboard cannot honour. The widget key is AAC's ``date``,
|
|
564
|
+
the declarative ``dateDataSet`` (GDAI-2391, GDAI-2462).
|
|
565
|
+
"""
|
|
566
|
+
failures: list[str] = []
|
|
567
|
+
for exp in expected:
|
|
568
|
+
if not isinstance(exp, dict) or "date_dataset" not in exp:
|
|
569
|
+
continue
|
|
570
|
+
wanted = exp["date_dataset"]
|
|
571
|
+
label = exp.get("title") if exp.get("id") is None else exp.get("id")
|
|
572
|
+
candidates = _matching_widgets(widgets, new_ids, exp)
|
|
573
|
+
if not candidates:
|
|
574
|
+
# Not left to charts_matched alone: a check that could not look must not read as passed.
|
|
575
|
+
failures.append(f"cannot check the date binding of chart {label!r}: it is not on the dashboard")
|
|
576
|
+
continue
|
|
577
|
+
# Every placement: the expectation names the chart, so one right placement must not hide a wrong one.
|
|
578
|
+
if not all(w.get("date") == wanted for w in candidates):
|
|
579
|
+
found = ", ".join(repr(w.get("date")) for w in candidates)
|
|
580
|
+
failures.append(f"chart {label!r} follows date {found}, expected {wanted!r}")
|
|
581
|
+
return failures
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def _tabs_of(dashboard: dict) -> list[dict]:
|
|
585
|
+
"""The dashboard's tabs, reading a document with root sections as its single tab."""
|
|
586
|
+
tabs = dashboard.get("tabs")
|
|
587
|
+
if tabs:
|
|
588
|
+
return list(tabs)
|
|
589
|
+
return [{"sections": dashboard.get("sections") or [], "filters": dashboard.get("filters") or {}}]
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def _check_tabs(dashboard: dict, new_ids: set[str], expected: list[dict]) -> list[str]:
|
|
593
|
+
"""The dashboard must have exactly the expected tabs, in order (GDAI-2426).
|
|
594
|
+
|
|
595
|
+
A tab entry may name a ``title``, compared like every other title, and ``visualizations``
|
|
596
|
+
that must sit on that tab -- each an id, or ``{"title": ...}`` for an authored chart. An
|
|
597
|
+
empty entry asserts only that the tab exists, which is how a case says "one tab, any name".
|
|
598
|
+
"""
|
|
599
|
+
tabs = _tabs_of(dashboard)
|
|
600
|
+
if len(tabs) != len(expected):
|
|
601
|
+
titles = ", ".join(repr(t.get("title")) for t in tabs)
|
|
602
|
+
return [f"the dashboard has {len(tabs)} tab(s) ({titles}), expected {len(expected)}"]
|
|
603
|
+
failures: list[str] = []
|
|
604
|
+
for index, (tab, exp) in enumerate(zip(tabs, expected)):
|
|
605
|
+
if "title" in exp and _norm(str(tab.get("title") or "")) != _norm(str(exp["title"] or "")):
|
|
606
|
+
failures.append(f"tab {index} is titled {tab.get('title')!r}, expected {exp['title']!r}")
|
|
607
|
+
tab_widgets = [w for section in tab.get("sections") or [] for w in section.get("widgets") or []]
|
|
608
|
+
for chart in exp.get("visualizations") or []:
|
|
609
|
+
entry = {"id": chart} if isinstance(chart, str) else {"id": None, **chart}
|
|
610
|
+
if not _matching_widgets(tab_widgets, new_ids, entry):
|
|
611
|
+
failures.append(f"tab {index} ({tab.get('title')!r}) does not hold chart {chart!r}")
|
|
612
|
+
return failures
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
# The keys a date filter is compared on. Absent on both sides is a match -- an all-time filter
|
|
616
|
+
# spells itself by leaving `from` and `to` out, and a dashboard-wide one by leaving `date` out.
|
|
617
|
+
_DATE_FILTER_KEYS = ("granularity", "from", "to", "date")
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def _filter_mismatch(label: str, actual: dict, expected: dict) -> list[str]:
|
|
621
|
+
"""Why ``actual`` is not the ``expected`` filter, or nothing.
|
|
622
|
+
|
|
623
|
+
A date filter is compared on its range and dataset, an attribute filter on its labels and
|
|
624
|
+
selection, both with absence significant. Any other kind is compared on the keys the
|
|
625
|
+
expectation names.
|
|
626
|
+
"""
|
|
627
|
+
if actual.get("type") != expected.get("type"):
|
|
628
|
+
return [f"{label} is a {actual.get('type')!r}, expected a {expected.get('type')!r}"]
|
|
629
|
+
if expected.get("type") == "date_filter":
|
|
630
|
+
keys: tuple[str, ...] = _DATE_FILTER_KEYS
|
|
631
|
+
elif expected.get("type") == "attribute_filter":
|
|
632
|
+
if _selection_of(actual) != _selection_of(expected):
|
|
633
|
+
return [
|
|
634
|
+
f"{label} selects {_describe_selection(_selection_of(actual))}, "
|
|
635
|
+
f"expected {_describe_selection(_selection_of(expected))}"
|
|
636
|
+
]
|
|
637
|
+
keys = ("using", "display_as")
|
|
638
|
+
else:
|
|
639
|
+
keys = tuple(k for k in expected if k != "type")
|
|
640
|
+
return [
|
|
641
|
+
f"{label}: {key} expected {expected.get(key)!r}, got {actual.get(key)!r}"
|
|
642
|
+
for key in keys
|
|
643
|
+
if not (
|
|
644
|
+
_same_object_ref(actual.get(key), expected.get(key))
|
|
645
|
+
if key in ("using", "display_as")
|
|
646
|
+
else actual.get(key) == expected.get(key)
|
|
647
|
+
)
|
|
648
|
+
]
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def _same_object_ref(actual: object, expected: object) -> bool:
|
|
652
|
+
"""Whether two AAC refs name the same object, reading ``attribute/<id>`` as ``label/<id>``.
|
|
653
|
+
|
|
654
|
+
The draft and patch tools accept either prefix on a filter's ``using``, and the model picks
|
|
655
|
+
one per run. An attribute and its primary label share the id, so the two spell one filter;
|
|
656
|
+
matching the prefix verbatim failed a correct run on the model's wording.
|
|
657
|
+
"""
|
|
658
|
+
if actual == expected:
|
|
659
|
+
return True
|
|
660
|
+
if not (isinstance(actual, str) and isinstance(expected, str)):
|
|
661
|
+
return False
|
|
662
|
+
return _ref_without_attribute_prefix(actual) == _ref_without_attribute_prefix(expected)
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
def _ref_without_attribute_prefix(ref: str) -> str:
|
|
666
|
+
return "label/" + ref.removeprefix("attribute/") if ref.startswith("attribute/") else ref
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def _check_tab_filters(dashboard: dict, expected: dict[str, dict]) -> list[str]:
|
|
670
|
+
"""Each named tab must carry exactly the expected filters, by the filter's own id.
|
|
671
|
+
|
|
672
|
+
Exact on purpose: GDAI-2540 is a filter that turns up on a tab it never belonged to, which a
|
|
673
|
+
check of only the expected entries would pass. Keyed by the saved dashboard's local filter
|
|
674
|
+
id rather than by label, so two filters on the same label stay distinguishable.
|
|
675
|
+
"""
|
|
676
|
+
tabs = {tab.get("id"): tab for tab in dashboard.get("tabs") or []}
|
|
677
|
+
failures: list[str] = []
|
|
678
|
+
for tab_id, expected_filters in expected.items():
|
|
679
|
+
tab = tabs.get(tab_id)
|
|
680
|
+
if tab is None:
|
|
681
|
+
failures.append(f"the dashboard has no tab {tab_id!r}")
|
|
682
|
+
continue
|
|
683
|
+
actual = {k: v for k, v in (tab.get("filters") or {}).items() if isinstance(v, dict)}
|
|
684
|
+
failures.extend(
|
|
685
|
+
f"tab {tab_id!r} carries filter {filter_id!r}, which it should not"
|
|
686
|
+
for filter_id in sorted(actual.keys() - expected_filters.keys())
|
|
687
|
+
)
|
|
688
|
+
failures.extend(
|
|
689
|
+
f"tab {tab_id!r} lost filter {filter_id!r}" for filter_id in sorted(expected_filters.keys() - actual.keys())
|
|
690
|
+
)
|
|
691
|
+
for filter_id in sorted(expected_filters.keys() & actual.keys()):
|
|
692
|
+
failures.extend(
|
|
693
|
+
_filter_mismatch(f"tab {tab_id!r} filter {filter_id!r}", actual[filter_id], expected_filters[filter_id])
|
|
694
|
+
)
|
|
695
|
+
return failures
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def _filters_by_id(dashboard: dict) -> dict[str, list[tuple[str, dict]]]:
|
|
699
|
+
"""Every filter by its own id, with where it was found; one id can recur across tabs."""
|
|
700
|
+
found: dict[str, list[tuple[str, dict]]] = {}
|
|
701
|
+
for tab in dashboard.get("tabs") or [{"filters": dashboard.get("filters") or {}}]:
|
|
702
|
+
where = f"tab {tab['id']!r} " if "id" in tab else ""
|
|
703
|
+
for filter_id, value in (tab.get("filters") or {}).items():
|
|
704
|
+
if isinstance(value, dict):
|
|
705
|
+
found.setdefault(filter_id, []).append((f"{where}filter {filter_id!r}", value))
|
|
706
|
+
return found
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
def _check_date_filters(dashboard: dict, expected: dict[str, dict | None]) -> list[str]:
|
|
710
|
+
"""Every named date filter must still be there, on the expected range (GDAI-2491).
|
|
711
|
+
|
|
712
|
+
Named by id because a dashboard can carry more than one date filter and the case states
|
|
713
|
+
which one holds which range. Filters the expectation does not name are left alone.
|
|
714
|
+
|
|
715
|
+
``date`` is the dataset the filter itself is pinned to, not a widget's date binding, and its
|
|
716
|
+
absence is significant as in ``tab_filters``: an entry without it, or ``null`` for all time,
|
|
717
|
+
is the main filter, which follows each widget's own date dataset. A range kept on another
|
|
718
|
+
dataset, or a main filter pinned to one, is a changed filter.
|
|
719
|
+
"""
|
|
720
|
+
found = _filters_by_id(dashboard)
|
|
721
|
+
failures: list[str] = []
|
|
722
|
+
for filter_id, wanted in expected.items():
|
|
723
|
+
entries = found.get(filter_id)
|
|
724
|
+
if not entries:
|
|
725
|
+
failures.append(f"the dashboard lost date filter {filter_id!r}")
|
|
726
|
+
continue
|
|
727
|
+
for label, date_filter in entries:
|
|
728
|
+
if date_filter.get("type") != "date_filter":
|
|
729
|
+
failures.append(f"{label} is a {date_filter.get('type')!r}, expected a date filter")
|
|
730
|
+
continue
|
|
731
|
+
failures.extend(_date_filter_mismatch(label, date_filter, wanted))
|
|
732
|
+
dataset = None if wanted is None else wanted.get("date")
|
|
733
|
+
if date_filter.get("date") != dataset:
|
|
734
|
+
failures.append(f"{label}: date dataset expected {dataset!r}, got {date_filter.get('date')!r}")
|
|
735
|
+
return failures
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def _check_preserved_widgets(widgets: list[dict], expected: list[dict]) -> list[str]:
|
|
739
|
+
"""A widget the change did not target must keep every key the expectation lists (GDAI-2541).
|
|
740
|
+
|
|
741
|
+
Compared on the keys listed and nothing else, so a case states what it guards -- AAC writes
|
|
742
|
+
a hidden title as ``title: false`` and drills as ``interactions``.
|
|
743
|
+
"""
|
|
744
|
+
failures: list[str] = []
|
|
745
|
+
for exp in expected:
|
|
746
|
+
viz_id = exp.get("visualization")
|
|
747
|
+
keys = [k for k in exp if k != "visualization"]
|
|
748
|
+
candidates = [w for w in widgets if w.get("visualization") == viz_id]
|
|
749
|
+
if not candidates:
|
|
750
|
+
failures.append(f"the widget showing {viz_id!r} is gone")
|
|
751
|
+
continue
|
|
752
|
+
# Every placement keeps the keys: one intact copy must not hide one that lost its drill.
|
|
753
|
+
mismatched = [w for w in candidates if not all(w.get(k) == exp[k] for k in keys)]
|
|
754
|
+
if not mismatched:
|
|
755
|
+
continue
|
|
756
|
+
widget = mismatched[0]
|
|
757
|
+
changed = ", ".join(f"{k} {widget.get(k)!r} (expected {exp[k]!r})" for k in keys if widget.get(k) != exp[k])
|
|
758
|
+
failures.append(f"the widget showing {viz_id!r} changed: {changed}")
|
|
759
|
+
return failures
|
|
760
|
+
|
|
761
|
+
|
|
431
762
|
def _date_range_text(date_range: dict | None) -> str:
|
|
432
763
|
"""Render a date range the way a user would say it, for the simulated reply.
|
|
433
764
|
|
|
@@ -463,9 +794,26 @@ def build_simulated_reply(expected_output: dict) -> str:
|
|
|
463
794
|
if existing:
|
|
464
795
|
segments.append(", ".join(existing))
|
|
465
796
|
segments.extend(f'a new chart titled "{title}"' for title in authored)
|
|
797
|
+
# Only titled tabs are said: an untitled entry asserts a tab count, and asking for "one tab"
|
|
798
|
+
# out loud would be the reply leading the agent rather than the question. Each tab names its
|
|
799
|
+
# charts, since `_check_tabs` requires every chart on its own tab and the agent cannot guess.
|
|
800
|
+
titles_by_id = {v.get("id"): str(v.get("title", "")) for v in visualizations if v.get("id") is not None}
|
|
801
|
+
placements = []
|
|
802
|
+
for tab in expected_output.get("tabs") or []:
|
|
803
|
+
if not tab.get("title"):
|
|
804
|
+
continue
|
|
805
|
+
charts = [
|
|
806
|
+
titles_by_id.get(c, c) if isinstance(c, str) else str(c.get("title", ""))
|
|
807
|
+
for c in tab.get("visualizations") or []
|
|
808
|
+
]
|
|
809
|
+
on_tab = f'the "{tab["title"]}" tab'
|
|
810
|
+
quoted = " and ".join(f'"{c}"' for c in charts)
|
|
811
|
+
placements.append(f"{quoted} on {on_tab}" if charts else on_tab)
|
|
812
|
+
tabs = f"Use these tabs: {'; '.join(placements)}. " if placements else ""
|
|
466
813
|
|
|
467
814
|
return (
|
|
468
815
|
f"Please use these charts: {', and '.join(segments)}. "
|
|
816
|
+
f"{tabs}"
|
|
469
817
|
f"Date range: {_date_range_text(expected_output.get('date_range'))}. "
|
|
470
818
|
f"Anything else is up to you. Please create the dashboard now."
|
|
471
819
|
)
|
|
@@ -544,7 +892,30 @@ def _validate_expectation(expected_output: dict) -> None:
|
|
|
544
892
|
raise ValueError(f"a filter expectation must be an object, got {entry!r}")
|
|
545
893
|
if not entry.get("using"):
|
|
546
894
|
raise ValueError(f"a filter expectation needs the label it filters on, got {entry!r}")
|
|
547
|
-
|
|
895
|
+
if "display_as" in entry and not (isinstance(entry["display_as"], str) and entry["display_as"]):
|
|
896
|
+
raise ValueError(f"a filter's display_as must name a label, got {entry!r}")
|
|
897
|
+
if "group" in entry and not (isinstance(entry["group"], str) and entry["group"]):
|
|
898
|
+
raise ValueError(f"a filter's group must name the group's title, got {entry!r}")
|
|
899
|
+
filter_type = _expected_filter_type(entry)
|
|
900
|
+
if filter_type not in _FILTER_VALUE_KEYS:
|
|
901
|
+
raise ValueError(f"a filter expectation's type must be one of {sorted(_FILTER_VALUE_KEYS)}, got {entry!r}")
|
|
902
|
+
# A key outside the compared ones, a misspelling included, would assert nothing.
|
|
903
|
+
compared = _ATTRIBUTE_FILTER_KEYS if filter_type == "attribute_filter" else _FILTER_VALUE_KEYS[filter_type]
|
|
904
|
+
unknown = sorted(entry.keys() - {"type", "using", "group", *compared})
|
|
905
|
+
if unknown:
|
|
906
|
+
raise ValueError(f"a {filter_type.replace('_', ' ')} expectation cannot state {unknown}, got {entry!r}")
|
|
907
|
+
if filter_type == "attribute_filter":
|
|
908
|
+
_expected_selection(entry)
|
|
909
|
+
elif filter_type == "text_filter":
|
|
910
|
+
if not entry.get("condition") or ("value" in entry) == ("values" in entry):
|
|
911
|
+
raise ValueError(f"a text filter expectation needs a condition and one of value/values, got {entry!r}")
|
|
912
|
+
if "values" in entry and not isinstance(entry["values"], list):
|
|
913
|
+
raise ValueError(f"a text filter's values must be a list, got {entry!r}")
|
|
914
|
+
if "value" in entry and not isinstance(entry["value"], str):
|
|
915
|
+
raise ValueError(f"a text filter's value must be a string, got {entry!r}")
|
|
916
|
+
elif not (isinstance(entry.get("conditions"), list) and entry["conditions"]):
|
|
917
|
+
raise ValueError(f"a metric value filter expectation needs its conditions, got {entry!r}")
|
|
918
|
+
_validate_layout_expectations(expected_output)
|
|
548
919
|
saved_id = expected_output.get("saved_dashboard_id")
|
|
549
920
|
if _is_edit(expected_output):
|
|
550
921
|
if not isinstance(saved_id, str) or not saved_id:
|
|
@@ -554,6 +925,99 @@ def _validate_expectation(expected_output: dict) -> None:
|
|
|
554
925
|
_min_new_visualizations(expected_output)
|
|
555
926
|
|
|
556
927
|
|
|
928
|
+
def _validate_layout_expectations(expected_output: dict) -> None:
|
|
929
|
+
"""Shape checks for the opt-in keys on charts, tabs, per-tab filters and preserved widgets.
|
|
930
|
+
|
|
931
|
+
Raises:
|
|
932
|
+
ValueError: one of them is unusable. Loud for the same reason as the rest of the
|
|
933
|
+
fixture checks: a typo would otherwise assert nothing and read green.
|
|
934
|
+
"""
|
|
935
|
+
for chart in expected_output.get("visualizations") or []:
|
|
936
|
+
if isinstance(chart, dict) and "date_dataset" in chart and not _is_dataset_or_null(chart["date_dataset"]):
|
|
937
|
+
raise ValueError(f"date_dataset must name a date dataset or be null, got {chart!r}")
|
|
938
|
+
|
|
939
|
+
if "tabs" in expected_output:
|
|
940
|
+
tabs = expected_output["tabs"]
|
|
941
|
+
if not isinstance(tabs, list) or not tabs or not all(isinstance(t, dict) for t in tabs):
|
|
942
|
+
raise ValueError(f"tabs must be a non-empty list of tab objects, got {tabs!r}")
|
|
943
|
+
for tab in tabs:
|
|
944
|
+
charts = tab.get("visualizations") or []
|
|
945
|
+
if not isinstance(charts, list) or not all(
|
|
946
|
+
(isinstance(c, str) and c) or (isinstance(c, dict) and c.get("title")) for c in charts
|
|
947
|
+
):
|
|
948
|
+
raise ValueError(f"a tab's visualizations must be chart ids or {{'title': ...}}, got {tab!r}")
|
|
949
|
+
|
|
950
|
+
if "tab_filters" in expected_output:
|
|
951
|
+
tab_filters = expected_output["tab_filters"]
|
|
952
|
+
if not isinstance(tab_filters, dict) or not tab_filters:
|
|
953
|
+
raise ValueError(f"tab_filters must map tab ids to their filters, got {tab_filters!r}")
|
|
954
|
+
for tab_id, filters in tab_filters.items():
|
|
955
|
+
if not isinstance(filters, dict) or not all(
|
|
956
|
+
isinstance(f, dict) and f.get("type") for f in filters.values()
|
|
957
|
+
):
|
|
958
|
+
raise ValueError(f"tab_filters[{tab_id!r}] must map filter ids to typed filters, got {filters!r}")
|
|
959
|
+
for filter_id, entry in filters.items():
|
|
960
|
+
if entry["type"] == "attribute_filter":
|
|
961
|
+
_validate_tab_attribute_filter(f"tab_filters[{tab_id!r}][{filter_id!r}]", entry)
|
|
962
|
+
|
|
963
|
+
if "date_filters" in expected_output:
|
|
964
|
+
date_filters = expected_output["date_filters"]
|
|
965
|
+
if not isinstance(date_filters, dict) or not date_filters:
|
|
966
|
+
raise ValueError(f"date_filters must map filter ids to a range or null, got {date_filters!r}")
|
|
967
|
+
for filter_id, date_range in date_filters.items():
|
|
968
|
+
if date_range is not None and not isinstance(date_range, dict):
|
|
969
|
+
raise ValueError(f"date_filters[{filter_id!r}] must be an object or null, got {date_range!r}")
|
|
970
|
+
if date_range == {}:
|
|
971
|
+
# Not all time: every bound would be compared against an absent one, and a real
|
|
972
|
+
# date filter always has a granularity, so the case could never pass.
|
|
973
|
+
raise ValueError(
|
|
974
|
+
f"date_filters[{filter_id!r}] is empty; use null for all time or state granularity/from/to"
|
|
975
|
+
)
|
|
976
|
+
unknown = sorted((date_range or {}).keys() - set(_DATE_FILTER_KEYS))
|
|
977
|
+
if unknown:
|
|
978
|
+
raise ValueError(
|
|
979
|
+
f"date_filters[{filter_id!r}] states {unknown}, which is not one of {list(_DATE_FILTER_KEYS)}"
|
|
980
|
+
)
|
|
981
|
+
if date_range and "date" in date_range and not _is_dataset_or_null(date_range["date"]):
|
|
982
|
+
raise ValueError(
|
|
983
|
+
f"date_filters[{filter_id!r}] date must name a date dataset or be null, got {date_range['date']!r}"
|
|
984
|
+
)
|
|
985
|
+
|
|
986
|
+
if "preserved_widgets" in expected_output:
|
|
987
|
+
preserved = expected_output["preserved_widgets"]
|
|
988
|
+
if not isinstance(preserved, list) or not preserved:
|
|
989
|
+
raise ValueError(f"preserved_widgets must be a non-empty list, got {preserved!r}")
|
|
990
|
+
for widget in preserved:
|
|
991
|
+
if not isinstance(widget, dict) or not widget.get("visualization") or len(widget) < 2:
|
|
992
|
+
raise ValueError(f"a preserved widget needs its visualization and a key to guard, got {widget!r}")
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
def _is_dataset_or_null(value: object) -> bool:
|
|
996
|
+
return value is None or (isinstance(value, str) and bool(value))
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def _validate_tab_attribute_filter(where: str, entry: dict) -> None:
|
|
1000
|
+
"""An attribute filter in ``tab_filters`` uses the AAC shape: the selection sits under ``state``.
|
|
1001
|
+
|
|
1002
|
+
The ``filters`` shorthand (``include``/``exclude``/``selection`` on the entry) does not apply
|
|
1003
|
+
here. Read as AAC, it would compare as "all" and pass a case that checks nothing.
|
|
1004
|
+
|
|
1005
|
+
Raises:
|
|
1006
|
+
ValueError: the entry is not in the AAC shape.
|
|
1007
|
+
"""
|
|
1008
|
+
shorthand = sorted({"include", "exclude", "selection"} & entry.keys())
|
|
1009
|
+
if shorthand:
|
|
1010
|
+
raise ValueError(f"{where} puts {shorthand} on the filter; tab_filters takes the AAC `state` object")
|
|
1011
|
+
state = entry.get("state")
|
|
1012
|
+
if state is None:
|
|
1013
|
+
return
|
|
1014
|
+
if not isinstance(state, dict):
|
|
1015
|
+
raise ValueError(f"{where} state must be an object, got {state!r}")
|
|
1016
|
+
for kind in ("include", "exclude"):
|
|
1017
|
+
if kind in state and not isinstance(state[kind], list):
|
|
1018
|
+
raise ValueError(f"{where} state.{kind} must be a list, got {state[kind]!r}")
|
|
1019
|
+
|
|
1020
|
+
|
|
557
1021
|
@dataclass(frozen=True)
|
|
558
1022
|
class _Applies:
|
|
559
1023
|
"""Which of the conditional checks the case applies.
|
|
@@ -567,6 +1031,24 @@ class _Applies:
|
|
|
567
1031
|
patch: bool
|
|
568
1032
|
date: bool
|
|
569
1033
|
filters: bool
|
|
1034
|
+
date_bindings: bool = False
|
|
1035
|
+
tabs: bool = False
|
|
1036
|
+
tab_filters: bool = False
|
|
1037
|
+
date_filters: bool = False
|
|
1038
|
+
preserved_widgets: bool = False
|
|
1039
|
+
|
|
1040
|
+
@classmethod
|
|
1041
|
+
def of(cls, expected_output: dict) -> _Applies:
|
|
1042
|
+
return cls(
|
|
1043
|
+
patch=_is_edit(expected_output),
|
|
1044
|
+
date=_has_date_range(expected_output),
|
|
1045
|
+
filters=_has_filters(expected_output),
|
|
1046
|
+
date_bindings=_has_date_bindings(expected_output),
|
|
1047
|
+
tabs="tabs" in expected_output,
|
|
1048
|
+
tab_filters="tab_filters" in expected_output,
|
|
1049
|
+
date_filters="date_filters" in expected_output,
|
|
1050
|
+
preserved_widgets="preserved_widgets" in expected_output,
|
|
1051
|
+
)
|
|
570
1052
|
|
|
571
1053
|
|
|
572
1054
|
@dataclass
|
|
@@ -598,6 +1080,14 @@ class DashboardEvaluation:
|
|
|
598
1080
|
patch_applies: bool = True
|
|
599
1081
|
references_carried: bool = True
|
|
600
1082
|
titles_matched: bool = True
|
|
1083
|
+
# False by default, unlike the older checks above: every early return then scores them as
|
|
1084
|
+
# failed without having to name them, and only the run that reached a document sets them.
|
|
1085
|
+
# They are published only where the case applies them, so a default never shows otherwise.
|
|
1086
|
+
date_bindings_correct: bool = False
|
|
1087
|
+
tabs_correct: bool = False
|
|
1088
|
+
tab_filters_correct: bool = False
|
|
1089
|
+
date_filters_correct: bool = False
|
|
1090
|
+
widgets_preserved: bool = False
|
|
601
1091
|
failures: list[str] = field(default_factory=list)
|
|
602
1092
|
notes: list[str] = field(default_factory=list)
|
|
603
1093
|
|
|
@@ -626,6 +1116,18 @@ class DashboardEvaluation:
|
|
|
626
1116
|
# Prefixed: alert_skill already publishes a `filters_correct` score, and the combo
|
|
627
1117
|
# report resolves a trace's skill by which score names it carries.
|
|
628
1118
|
checks["dashboard_filters_correct"] = self.filters_correct
|
|
1119
|
+
# All prefixed, for the same reason as the filters check: the combo report tells skills
|
|
1120
|
+
# apart by the score names a trace carries.
|
|
1121
|
+
if self.applies.date_bindings:
|
|
1122
|
+
checks["dashboard_date_bindings_correct"] = self.date_bindings_correct
|
|
1123
|
+
if self.applies.tabs:
|
|
1124
|
+
checks["dashboard_tabs_correct"] = self.tabs_correct
|
|
1125
|
+
if self.applies.tab_filters:
|
|
1126
|
+
checks["dashboard_tab_filters_correct"] = self.tab_filters_correct
|
|
1127
|
+
if self.applies.date_filters:
|
|
1128
|
+
checks["dashboard_date_filters_correct"] = self.date_filters_correct
|
|
1129
|
+
if self.applies.preserved_widgets:
|
|
1130
|
+
checks["dashboard_widgets_preserved"] = self.widgets_preserved
|
|
629
1131
|
return checks
|
|
630
1132
|
|
|
631
1133
|
@property
|
|
@@ -693,7 +1195,7 @@ def evaluate_dashboard_response(
|
|
|
693
1195
|
without an agent.
|
|
694
1196
|
"""
|
|
695
1197
|
is_edit = _is_edit(expected_output)
|
|
696
|
-
applies = _Applies(
|
|
1198
|
+
applies = _Applies.of(expected_output)
|
|
697
1199
|
tool = _producing_tool(expected_output)
|
|
698
1200
|
|
|
699
1201
|
if tool_result is None:
|
|
@@ -812,11 +1314,26 @@ def evaluate_dashboard_response(
|
|
|
812
1314
|
title_notes = title_mismatches
|
|
813
1315
|
reference_notes = _check_references(widgets, known_ids)
|
|
814
1316
|
date_failures = (
|
|
815
|
-
_check_date_range(document, expected_output.get("date_range")
|
|
1317
|
+
_check_date_range(document, expected_output.get("date_range"), applies.patch)
|
|
1318
|
+
if _has_date_range(expected_output)
|
|
1319
|
+
else []
|
|
816
1320
|
)
|
|
817
1321
|
filter_failures = (
|
|
818
1322
|
_check_filters(document, expected_output.get("filters") or []) if _has_filters(expected_output) else []
|
|
819
1323
|
)
|
|
1324
|
+
binding_failures = (
|
|
1325
|
+
_check_date_bindings(widgets, new_ids, expected_output.get("visualizations") or [])
|
|
1326
|
+
if applies.date_bindings
|
|
1327
|
+
else []
|
|
1328
|
+
)
|
|
1329
|
+
tab_failures = _check_tabs(document, new_ids, expected_output["tabs"]) if applies.tabs else []
|
|
1330
|
+
tab_filter_failures = _check_tab_filters(document, expected_output["tab_filters"]) if applies.tab_filters else []
|
|
1331
|
+
date_filter_failures = (
|
|
1332
|
+
_check_date_filters(document, expected_output["date_filters"]) if applies.date_filters else []
|
|
1333
|
+
)
|
|
1334
|
+
preserved_failures = (
|
|
1335
|
+
_check_preserved_widgets(widgets, expected_output["preserved_widgets"]) if applies.preserved_widgets else []
|
|
1336
|
+
)
|
|
820
1337
|
|
|
821
1338
|
return DashboardEvaluation(
|
|
822
1339
|
drafted=True,
|
|
@@ -831,7 +1348,23 @@ def evaluate_dashboard_response(
|
|
|
831
1348
|
patch_applies=True,
|
|
832
1349
|
references_carried=not reference_notes,
|
|
833
1350
|
titles_matched=not title_mismatches,
|
|
834
|
-
|
|
1351
|
+
date_bindings_correct=not binding_failures,
|
|
1352
|
+
tabs_correct=not tab_failures,
|
|
1353
|
+
tab_filters_correct=not tab_filter_failures,
|
|
1354
|
+
date_filters_correct=not date_filter_failures,
|
|
1355
|
+
widgets_preserved=not preserved_failures,
|
|
1356
|
+
failures=[
|
|
1357
|
+
*chart_failures,
|
|
1358
|
+
*saved_failures,
|
|
1359
|
+
*date_failures,
|
|
1360
|
+
*filter_failures,
|
|
1361
|
+
*binding_failures,
|
|
1362
|
+
*tab_failures,
|
|
1363
|
+
*tab_filter_failures,
|
|
1364
|
+
*date_filter_failures,
|
|
1365
|
+
*preserved_failures,
|
|
1366
|
+
*new_failures,
|
|
1367
|
+
],
|
|
835
1368
|
notes=[*title_notes, *reference_notes],
|
|
836
1369
|
)
|
|
837
1370
|
|
|
@@ -944,6 +1477,22 @@ def _execute_single_dashboard_run(
|
|
|
944
1477
|
)
|
|
945
1478
|
|
|
946
1479
|
|
|
1480
|
+
def _run_detail(run: DashboardRunResult) -> dict[str, Any]:
|
|
1481
|
+
"""The diagnostic fields for ONE run, shared by the best run and every failing one.
|
|
1482
|
+
|
|
1483
|
+
Extracted so a failing run is described by exactly the same keys as the winning run --
|
|
1484
|
+
for this kind that means the per-check breakdown and the failures it reported, which is
|
|
1485
|
+
the whole diagnosis.
|
|
1486
|
+
"""
|
|
1487
|
+
return {
|
|
1488
|
+
**run.evaluation.strict_checks,
|
|
1489
|
+
**run.evaluation.diagnostics,
|
|
1490
|
+
"failures": run.evaluation.failures,
|
|
1491
|
+
"notes": run.evaluation.notes,
|
|
1492
|
+
"latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events),
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
|
|
947
1496
|
def run_agentic_dashboard_skill(
|
|
948
1497
|
host: str,
|
|
949
1498
|
token: str,
|
|
@@ -1108,13 +1657,14 @@ def evaluate_agentic_dashboard_skill(
|
|
|
1108
1657
|
runs_effective = len(summary.run_results)
|
|
1109
1658
|
|
|
1110
1659
|
best = summary.best
|
|
1111
|
-
detail: dict[str, Any] =
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1660
|
+
detail: dict[str, Any] = _run_detail(best)
|
|
1661
|
+
# Same predicate runs_passed is taken over, so an item's failed_runs and its counts
|
|
1662
|
+
# cannot disagree about which runs failed.
|
|
1663
|
+
failed_runs = build_failed_runs(
|
|
1664
|
+
summary.run_results,
|
|
1665
|
+
passed=lambda r: r.evaluation.strict_pass,
|
|
1666
|
+
detail=_run_detail,
|
|
1667
|
+
)
|
|
1118
1668
|
|
|
1119
1669
|
if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k):
|
|
1120
1670
|
gate_note = gate_failure_note(gate, runs_passed, runs_effective)
|
|
@@ -1148,6 +1698,7 @@ def evaluate_agentic_dashboard_skill(
|
|
|
1148
1698
|
exc.detail = detail
|
|
1149
1699
|
exc.runs_passed = runs_passed
|
|
1150
1700
|
exc.runs_effective = runs_effective
|
|
1701
|
+
exc.failed_runs = failed_runs
|
|
1151
1702
|
raise exc
|
|
1152
1703
|
return AgenticEvalOutcome(
|
|
1153
1704
|
runs_passed=runs_passed,
|
|
@@ -1157,4 +1708,5 @@ def evaluate_agentic_dashboard_skill(
|
|
|
1157
1708
|
response_id=best.response_id,
|
|
1158
1709
|
detail=detail,
|
|
1159
1710
|
timings=item_timings,
|
|
1711
|
+
failed_runs=failed_runs,
|
|
1160
1712
|
)
|