gooddata-eval 1.68.0__py3-none-any.whl

gooddata_eval/core/reporting/console.py

@@ -0,0 +1,117 @@
+# (C) 2026 GoodData Corporation
+"""Render a human-readable console report using rich."""
+
+from rich.console import Console
+from rich.table import Table
+
+from gooddata_eval.core.runner import EvalReport
+
+
+def render_console(report: EvalReport, *, console: Console | None = None) -> str:
+    """Render the report to the console and return the rendered text.
+
+    Passing a Console lets callers print to stdout; the returned string aids testing.
+    """
+    out = console or Console(record=True, width=120)
+
+    table = Table(title=f"Evaluation — model={report.model} workspace={report.workspace_id}")
+    table.add_column("Item")
+    table.add_column("Kind")
+    table.add_column("Result")
+    table.add_column("Runs")
+    table.add_column("Latency")
+    table.add_column("Avg/run")
+    table.add_column("Quality")
+    table.add_column("Notes")
+
+    for item in report.items:
+        if item.skipped:
+            result, notes = "SKIPPED", f"test_kind '{item.test_kind}' not supported in this phase"
+        elif item.error:
+            result, notes = "ERROR", item.error
+        elif item.pass_at_k:
+            result, notes = "PASS", ""
+        else:
+            # Evaluator-agnostic: report whichever boolean checks came back False
+            # (visualization uses metrics_correct/…; dashboard_summary uses
+            # include_*/exclude_*/rubric_*). Falls back to a generic message.
+            failing = [k for k, v in item.best_detail.items() if v is False]
+            notes = "failed: " + ", ".join(failing) if failing else "did not pass strict checks"
+            result = "FAIL"
+        latency = "-" if item.runs == 0 else f"{item.latency_s:.2f}s"
+        avg = "-" if item.runs == 0 else f"{item.avg_latency_s:.2f}s"
+        quality = "-" if item.skipped else f"{item.quality_score:.0%}"
+        table.add_row(item.id, item.test_kind, result, str(item.runs), latency, avg, quality, notes)
+
+    out.print(table)
+    _wall = report.wall_clock_s
+    _agent = report.latency_s
+    if _wall > 0 and abs(_wall - _agent) > 1:  # concurrency > 1: show both
+        timing = f"{_wall:.2f}s wall-clock, {_agent:.2f}s agent time (avg {report.avg_latency_s:.2f}s/run)"
+    else:
+        timing = f"{_agent:.2f}s (avg {report.avg_latency_s:.2f}s/run)"
+    out.print(
+        f"\nSummary: {report.passed}/{report.total} passed "
+        f"({report.skipped} skipped, {report.errored} errored) "
+        f"avg quality {report.avg_quality_score:.0%} in {timing}"
+    )
+    return out.export_text() if out.record else ""
+
+
+def render_comparison(reports: list[EvalReport], *, console: Console | None = None) -> str:
+    """Render a side-by-side comparison for multiple model runs.
+
+    Winner is selected by (pass_rate, avg_quality_score, -avg_latency_s) —
+    higher pass rate first, then quality, then lower latency as final tiebreaker.
+    Returns an empty string when fewer than two reports are provided.
+    """
+    if len(reports) < 2:
+        return ""
+
+    out = console or Console(record=True, width=120)
+
+    table = Table(title="Model Comparison")
+    table.add_column("Model")
+    table.add_column("Passed")
+    table.add_column("Quality")
+    table.add_column("Avg/run")
+    table.add_column("Total time")
+
+    for r in reports:
+        evaluated = r.total - r.skipped
+        pass_pct = f"{r.passed / evaluated:.0%}" if evaluated else "—"
+        model_label = f"{r.provider_name}/{r.model}" if r.provider_name else r.model or "?"
+        table.add_row(
+            model_label,
+            f"{r.passed}/{r.total} ({pass_pct})",
+            f"{r.avg_quality_score:.0%}",
+            f"{r.avg_latency_s:.2f}s",
+            f"{r.latency_s:.0f}s",
+        )
+
+    out.print(table)
+
+    evaluated_reports = [r for r in reports if r.total > 0]
+    if evaluated_reports:
+        winner = max(
+            evaluated_reports,
+            key=lambda r: (
+                r.passed / r.total if r.total else 0,
+                r.avg_quality_score,
+                -r.avg_latency_s,  # lower latency wins when pass rate and quality tie
+            ),
+        )
+        runner_up = max(
+            (r for r in evaluated_reports if r is not winner),
+            key=lambda r: r.passed / r.total if r.total else 0,
+            default=None,
+        )
+        delta = ""
+        if runner_up and runner_up.total > 0:
+            delta_items = winner.passed - runner_up.passed
+            delta_quality = winner.avg_quality_score - runner_up.avg_quality_score
+            delta = f"  (+{delta_items} item(s) passed, +{delta_quality:.0%} quality)"
+        winner_label = f"{winner.provider_name}/{winner.model}" if winner.provider_name else winner.model or "?"
+        out.print(f"\n[bold]Winner: {winner_label}[/bold]{delta}")
+
+    return out.export_text() if out.record else ""

gooddata_eval/core/reporting/json_report.py

@@ -0,0 +1,81 @@
+# (C) 2026 GoodData Corporation
+"""Build and write machine-readable reports (single-model or multi-model)."""
+
+from pathlib import Path
+
+import orjson
+
+from gooddata_eval.core.runner import EvalReport
+
+
+def _build_run_dict(report: EvalReport) -> dict:
+    return {
+        "model": report.model,
+        "workspace_id": report.workspace_id,
+        "summary": {
+            "total": report.total,
+            "passed": report.passed,
+            "failed": report.total - report.passed - report.skipped,
+            "skipped": report.skipped,
+            "errored": report.errored,
+            "latency_s": round(report.latency_s, 3),
+            "avg_latency_s": round(report.avg_latency_s, 3),
+            "wall_clock_s": round(report.wall_clock_s, 3),
+        },
+        "items": {
+            item.id: {
+                "dataset_name": item.dataset_name,
+                "test_kind": item.test_kind,
+                "question": item.question,
+                "pass_at_k": item.pass_at_k,
+                "skipped": item.skipped,
+                "error": item.error,
+                "runs": item.runs,
+                "latency_s": round(item.latency_s, 3),
+                "avg_latency_s": round(item.avg_latency_s, 3),
+                "detail": item.best_detail,
+            }
+            for item in report.items
+        },
+    }
+
+
+def _build_comparison_entry(report: EvalReport) -> dict:
+    total = report.total
+    passed = report.passed
+    return {
+        "provider_name": report.provider_name,
+        "passed": passed,
+        "total": total,
+        "pass_rate": round(passed / total, 4) if total else 0.0,
+        "avg_quality_score": round(report.avg_quality_score, 4),
+        "avg_latency_s": round(report.avg_latency_s, 3),
+        "total_latency_s": round(report.latency_s, 3),
+    }
+
+
+def _run_key(report: EvalReport) -> str:
+    """Collision-free key matching the console comparison table label."""
+    return f"{report.provider_name}/{report.model}" if report.provider_name else report.model or "?"
+
+
+def build_multi_model_report(reports: list[EvalReport]) -> dict:
+    """Build the nested multi-model JSON report (used for single-model runs too)."""
+    return {
+        "models": [_run_key(r) for r in reports],
+        "runs": {_run_key(r): _build_run_dict(r) for r in reports},
+        "comparison": {_run_key(r): _build_comparison_entry(r) for r in reports},
+    }
+
+
+def write_multi_model_report(reports: list[EvalReport], path: Path) -> None:
+    Path(path).write_bytes(orjson.dumps(build_multi_model_report(reports), option=orjson.OPT_INDENT_2))
+
+
+# Backward-compatible aliases so existing callers keep working.
+def build_json_report(report: EvalReport) -> dict:
+    return _build_run_dict(report)
+
+
+def write_json_report(report: EvalReport, path: Path) -> None:
+    write_multi_model_report([report], path)

gooddata_eval/core/runner.py

@@ -0,0 +1,214 @@
+# (C) 2026 GoodData Corporation
+"""Dataset run orchestration: per item, K single-turn runs, route by test_kind, aggregate pass@K."""
+
+import time
+import traceback
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+from functools import partial
+from typing import Callable, Protocol
+
+from gooddata_eval.core.evaluators import get_evaluator, supported_test_kinds
+from gooddata_eval.core.evaluators.base import ItemEvaluation
+from gooddata_eval.core.models import ChatResult, DatasetItem
+
+
+class ChatBackend(Protocol):
+    # Receives the whole item so backends can use per-item context beyond the
+    # question text (e.g. dashboard_summary needs item.summary_input).
+    def ask(self, item: DatasetItem) -> ChatResult: ...
+
+
+@dataclass
+class ItemReport:
+    id: str
+    dataset_name: str
+    test_kind: str
+    question: str
+    pass_at_k: bool = False
+    skipped: bool = False
+    error: str | None = None
+    runs: int = 0
+    latency_s: float = 0.0  # total wall-clock across this item's runs
+    best_detail: dict = field(default_factory=dict)
+
+    @property
+    def avg_latency_s(self) -> float:
+        return self.latency_s / self.runs if self.runs else 0.0
+
+    @property
+    def quality_score(self) -> float:
+        """Fraction of bool-valued strict checks in best_detail that are True.
+
+        Falls back to 1.0 if pass_at_k else 0.0 when no bool checks exist
+        (e.g. text evaluators where best_detail has no bool flags).
+        """
+        checks = {k: v for k, v in self.best_detail.items() if isinstance(v, bool)}
+        if checks:
+            return sum(1 for v in checks.values() if v) / len(checks)
+        return 1.0 if self.pass_at_k else 0.0
+
+
+@dataclass
+class EvalReport:
+    model: str | None
+    provider_name: str = ""
+    provider_type: str = ""
+    workspace_id: str = ""
+    items: list[ItemReport] = field(default_factory=list)
+    wall_clock_s: float = 0.0  # actual elapsed time; differs from latency_s under concurrency
+
+    @property
+    def total(self) -> int:
+        return len(self.items)
+
+    @property
+    def passed(self) -> int:
+        return sum(1 for i in self.items if i.pass_at_k)
+
+    @property
+    def skipped(self) -> int:
+        return sum(1 for i in self.items if i.skipped)
+
+    @property
+    def errored(self) -> int:
+        return sum(1 for i in self.items if i.error is not None)
+
+    @property
+    def latency_s(self) -> float:
+        return sum(i.latency_s for i in self.items)
+
+    @property
+    def total_runs(self) -> int:
+        return sum(i.runs for i in self.items)
+
+    @property
+    def avg_latency_s(self) -> float:
+        total_runs = self.total_runs
+        return self.latency_s / total_runs if total_runs else 0.0
+
+    @property
+    def avg_quality_score(self) -> float:
+        """Mean quality_score across evaluated (non-skipped, non-errored) items."""
+        evaluated = [i.quality_score for i in self.items if not i.skipped and i.error is None]
+        return sum(evaluated) / len(evaluated) if evaluated else 0.0
+
+
+# A per-run progress callback: (run_index, runs_total, passed, latency_s) -> None
+RunCallback = Callable[[int, int, bool, float], None]
+
+
+def _run_one_item(
+    item: DatasetItem, backend: ChatBackend, runs: int, on_run_done: RunCallback | None = None
+) -> ItemReport:
+    report = ItemReport(id=item.id, dataset_name=item.dataset_name, test_kind=item.test_kind, question=item.question)
+    if item.test_kind not in supported_test_kinds():
+        report.skipped = True
+        return report
+
+    evaluator = get_evaluator(item.test_kind)
+    best: ItemEvaluation | None = None
+    try:
+        for run_index in range(1, runs + 1):
+            t0 = time.perf_counter()
+            chat_result = backend.ask(item)
+            evaluation = evaluator.evaluate(item, chat_result)
+            latency = time.perf_counter() - t0
+            report.runs += 1
+            report.latency_s += latency
+            if best is None or evaluation.rank_key > best.rank_key:
+                best = evaluation
+            if evaluation.passed:
+                report.pass_at_k = True
+            if on_run_done is not None:
+                on_run_done(run_index, runs, evaluation.passed, latency)
+    except Exception as e:  # agent/network/parse failure for this item
+        report.error = f"{type(e).__name__}: {e}"
+        if best is not None:
+            report.best_detail = best.detail
+        return report
+
+    if best is not None:
+        report.best_detail = best.detail
+    return report
+
+
+def _forward_run_event(
+    user_cb: "Callable[[int, int, int, int, bool, float], None]",
+    item_index: int,
+    total: int,
+    run_index: int,
+    runs_total: int,
+    passed: bool,
+    latency: float,
+) -> None:
+    user_cb(item_index, total, run_index, runs_total, passed, latency)
+
+
+def run_items(
+    items: list[DatasetItem],
+    backend: ChatBackend,
+    *,
+    runs: int = 2,
+    model: str | None = None,
+    provider_name: str = "",
+    provider_type: str = "",
+    workspace_id: str = "",
+    on_item_start: Callable[[int, int, DatasetItem], None] | None = None,
+    on_run_done: Callable[[int, int, int, int, bool, float], None] | None = None,
+    on_item_done: Callable[[int, int, ItemReport], None] | None = None,
+    on_langfuse_item_done: Callable[[int, int, ItemReport], None] | None = None,
+    concurrency: int = 1,
+) -> EvalReport:
+    """Run every item K times, routing by test_kind, and aggregate pass@K.
+
+    Optional callbacks stream progress without coupling core to any I/O library
+    (index/run_index are 1-based):
+      - on_item_start(index, total, item)                          before an item's runs begin
+      - on_run_done(index, total, run_index, runs, passed, latency) after each individual run
+      - on_item_done(index, total, report)                         after an item is fully evaluated
+      - on_langfuse_item_done(index, total, report)                after non-skipped, non-errored items only
+
+    concurrency > 1 dispatches items to a ThreadPoolExecutor so multiple
+    questions are sent to the agent simultaneously. Each item still runs
+    --runs times sequentially (pass@K). Results are collected in input order.
+    """
+    concurrency = max(1, concurrency)
+    report = EvalReport(
+        model=model, provider_name=provider_name, provider_type=provider_type, workspace_id=workspace_id
+    )
+    total = len(items)
+
+    def _process_item(index: int, item: DatasetItem) -> ItemReport:
+        try:
+            if on_item_start is not None:
+                on_item_start(index, total, item)
+        except Exception:  # non-fatal — callback must not abort a parallel run
+            traceback.print_exc()
+        run_cb = partial(_forward_run_event, on_run_done, index, total) if on_run_done is not None else None
+        item_report = _run_one_item(item, backend, runs, on_run_done=run_cb)
+        try:
+            if on_item_done is not None:
+                on_item_done(index, total, item_report)
+            if on_langfuse_item_done is not None and not item_report.skipped and item_report.error is None:
+                on_langfuse_item_done(index, total, item_report)
+        except Exception:  # non-fatal — log but don't abort
+            traceback.print_exc()
+        return item_report
+
+    _t0 = time.perf_counter()
+    if concurrency <= 1:
+        for index, item in enumerate(items, start=1):
+            report.items.append(_process_item(index, item))
+    else:
+        # Dispatch concurrently; collect in original order.
+        with ThreadPoolExecutor(max_workers=concurrency) as pool:
+            futures = {pool.submit(_process_item, index, item): index for index, item in enumerate(items, start=1)}
+            results: dict[int, ItemReport] = {}
+            for future in as_completed(futures):
+                idx = futures[future]
+                results[idx] = future.result()
+        for index in range(1, total + 1):
+            report.items.append(results[index])
+    report.wall_clock_s = time.perf_counter() - _t0
+    return report

gooddata_eval/core/scoring.py

@@ -0,0 +1,155 @@
+# (C) 2026 GoodData Corporation
+"""Visualization scoring — ported from gdc-nas tavern-e2e app/vis_assertions/metrics.py."""
+
+import json
+from dataclasses import dataclass
+
+from gooddata_eval.core.models import AacBucketRef, AacQueryField, CreatedVisualization
+
+# Maps dataset chart-type names (and agent enum values) to a canonical token.
+_AAC_TYPE_MAP = {
+    "line_chart": "LINE",
+    "bar_chart": "BAR",
+    "column_chart": "COLUMN",
+    "pie_chart": "PIE",
+    "table": "TABLE",
+    "headline": "HEADLINE",
+}
+
+
+@dataclass
+class FilterScores:
+    date_ok: bool
+    ranking_ok: bool
+    attribute_ok: bool
+
+    @property
+    def all_ok(self) -> bool:
+        return self.date_ok and self.ranking_ok and self.attribute_ok
+
+
+def _resolve_alias_to_uri(alias: str, fields: dict[str, AacQueryField | str]) -> str:
+    """Resolve a field alias to its `using` URI; return the alias unchanged if absent."""
+    field = fields.get(alias)
+    if field is None:
+        return alias
+    if isinstance(field, AacQueryField):
+        return field.using
+    return field
+
+
+def _resolve_bucket_to_uri_set(bucket: list[AacBucketRef | str], fields: dict[str, AacQueryField | str]) -> set[str]:
+    uris: set[str] = set()
+    for ref in bucket:
+        alias = ref.field if isinstance(ref, AacBucketRef) else ref
+        uris.add(_resolve_alias_to_uri(alias, fields))
+    return uris
+
+
+def get_metric_uri_set(viz: CreatedVisualization) -> set[str]:
+    return _resolve_bucket_to_uri_set(viz.metrics, viz.query.fields)
+
+
+def get_dimension_uri_set(viz: CreatedVisualization) -> set[str]:
+    all_dim_buckets = viz.view_by + viz.segment_by + viz.rows + viz.columns
+    return _resolve_bucket_to_uri_set(all_dim_buckets, viz.query.fields)
+
+
+def uri_to_display_name(uri: str) -> str:
+    """Convert 'metric/net_sales' -> 'net sales', 'label/date.month' -> 'date - month'."""
+    last = uri.split("/", 1)[-1]
+    return last.replace(".", " - ").replace("_", " ")
+
+
+def validate_cross_references(viz: CreatedVisualization) -> tuple[bool, list[str]]:
+    """Validate ranking-filter `using`/`attribute` resolve to correct URI prefixes."""
+    errors: list[str] = []
+    fields = viz.query.fields
+    for filter_key, filter_dict in viz.query.filter_by.items():
+        if filter_dict.get("type") != "ranking_filter":
+            continue
+        using_val = filter_dict.get("using", "")
+        using_uri = _resolve_alias_to_uri(using_val, fields)
+        if not using_uri.startswith(("metric/", "fact/")):
+            errors.append(
+                f"ranking filter '{filter_key}': using='{using_val}' "
+                f"resolves to '{using_uri}' — expected a metric/ or fact/ URI"
+            )
+        if "attribute" in filter_dict:
+            attr_val = filter_dict["attribute"]
+            attr_uri = _resolve_alias_to_uri(attr_val, fields)
+            if not attr_uri.startswith(("label/", "attribute/")):
+                errors.append(
+                    f"ranking filter '{filter_key}': attribute='{attr_val}' "
+                    f"resolves to '{attr_uri}' — expected a label/ or attribute/ URI"
+                )
+    return len(errors) == 0, errors
+
+
+def _normalize_date_filter(filter_dict: dict, _fields: dict) -> dict:
+    return {
+        "type": "date_filter",
+        "dataset_uri": filter_dict.get("using", ""),
+        "from": filter_dict.get("from"),
+        "to": filter_dict.get("to"),
+        "granularity": filter_dict.get("granularity"),
+    }
+
+
+def _normalize_ranking_filter(filter_dict: dict, fields: dict[str, AacQueryField | str]) -> dict:
+    entry: dict = {
+        "type": "ranking_filter",
+        "metric_uri": _resolve_alias_to_uri(filter_dict.get("using", ""), fields),
+        "dim_uri": _resolve_alias_to_uri(filter_dict.get("attribute", ""), fields),
+    }
+    if "top" in filter_dict:
+        entry["top"] = filter_dict["top"]
+    if "bottom" in filter_dict:
+        entry["bottom"] = filter_dict["bottom"]
+    return entry
+
+
+def _normalize_attribute_filter(filter_dict: dict, _fields: dict) -> dict:
+    raw_state = filter_dict.get("state") or {}
+    state = {k: v for k, v in raw_state.items() if v}
+    return {
+        "type": "attribute_filter",
+        "field_uri": filter_dict.get("using", ""),
+        "state": state,
+    }
+
+
+def _split_and_normalize_filters(viz: CreatedVisualization) -> tuple[set[str], set[str], set[str]]:
+    date_set: set[str] = set()
+    ranking_set: set[str] = set()
+    attr_set: set[str] = set()
+    fields = viz.query.fields
+    for filter_dict in viz.query.filter_by.values():
+        ft = filter_dict.get("type")
+        if ft == "date_filter":
+            date_set.add(json.dumps(_normalize_date_filter(filter_dict, fields), sort_keys=True))
+        elif ft == "ranking_filter":
+            ranking_set.add(json.dumps(_normalize_ranking_filter(filter_dict, fields), sort_keys=True))
+        elif ft == "attribute_filter":
+            attr_set.add(json.dumps(_normalize_attribute_filter(filter_dict, fields), sort_keys=True))
+    return date_set, ranking_set, attr_set
+
+
+def check_filters(expected: CreatedVisualization, actual: CreatedVisualization) -> FilterScores:
+    exp_date, exp_rank, exp_attr = _split_and_normalize_filters(expected)
+    act_date, act_rank, act_attr = _split_and_normalize_filters(actual)
+    return FilterScores(
+        date_ok=act_date == exp_date,
+        ranking_ok=act_rank == exp_rank,
+        attribute_ok=act_attr == exp_attr,
+    )
+
+
+def _normalize_viz_type(raw_type: str) -> str:
+    return _AAC_TYPE_MAP.get(raw_type, raw_type.replace("_chart", "").upper())
+
+
+def check_viz_type(expected: CreatedVisualization, actual: CreatedVisualization) -> bool:
+    if not expected.type:
+        return True
+    return _normalize_viz_type(expected.type) == _normalize_viz_type(actual.type)

gooddata_eval/core/summary/__init__.py

@@ -0,0 +1 @@
+# (C) 2026 GoodData Corporation

gooddata_eval/core/summary/http_client.py

@@ -0,0 +1,54 @@
+# (C) 2026 GoodData Corporation
+"""HTTP client for the dedicated dashboard-summary endpoint.
+
+Unlike the conversational chat skill, this endpoint executes the AFM for each
+visualization server-side and returns a plain synchronous JSON summary — no SSE
+stream and no client-side ``result_id`` wrangling. The response is adapted into
+a ``ChatResult`` (summary text -> ``text_response``) so the existing
+LLM-as-judge evaluators can score it unchanged.
+
+Endpoint (gen-ai service):
+    POST /api/v1/ai/workspaces/{workspace_id}/summary
+
+If the route is ever renamed (e.g. to ``/summarize``), change ``_PATH`` only.
+"""
+
+import httpx
+
+from gooddata_eval.core.models import ChatResult, DatasetItem
+
+_PATH = "/api/v1/ai/workspaces/{workspace_id}/summary"
+
+
+class SummaryClient:
+    """Single-shot client for the dashboard-summary endpoint."""
+
+    def __init__(self, host: str, token: str, workspace_id: str, *, timeout: float = 300.0):
+        self._url = f"{host.rstrip('/')}{_PATH.format(workspace_id=workspace_id)}"
+        self._auth = {"Authorization": f"Bearer {token}"}
+        self._client = httpx.Client(timeout=timeout)
+
+    def ask(self, item: DatasetItem) -> ChatResult:
+        """Request a summary for one dataset item and adapt it to a ChatResult."""
+        si = item.summary_input
+        if si is None:
+            raise ValueError(f"dashboard_summary item '{item.id}' is missing required 'summary_input'.")
+
+        body: dict = {"dashboardId": si.dashboard_id}
+        if si.visualizations is not None:
+            body["visualizations"] = si.visualizations
+        if si.filter_context is not None:
+            body["filterContext"] = si.filter_context
+        if si.tab_id is not None:
+            body["tabId"] = si.tab_id
+        if si.format_hint is not None:
+            body["formatHint"] = si.format_hint
+
+        resp = self._client.post(self._url, json=body, headers={**self._auth, "Content-Type": "application/json"})
+        resp.raise_for_status()
+        data = resp.json()
+        summary = data.get("summary") or ""
+        return ChatResult.model_validate({"textResponse": summary})
+
+    def close(self) -> None:
+        self._client.close()