gooddata-eval 1.68.0__py3-none-any.whl

gooddata_eval/core/evaluators/search_tool.py

@@ -0,0 +1,40 @@
+# (C) 2026 GoodData Corporation
+"""Evaluator for search_tool: agent must call the catalog search with expected parameters."""
+
+from gooddata_eval.core.evaluators.base import ItemEvaluation
+from gooddata_eval.core.models import ChatResult, DatasetItem
+
+
+def _args_match(actual_args: dict, expected_args: dict) -> bool:
+    if sorted(actual_args.get("keywords") or []) != sorted(expected_args.get("keywords") or []):
+        return False
+    if sorted(actual_args.get("object_types") or []) != sorted(expected_args.get("object_types") or []):
+        return False
+    if actual_args.get("limit") != expected_args.get("limit"):
+        return False
+    return actual_args.get("emit_widget") == expected_args.get("emit_widget")
+
+
+class SearchToolEvaluator:
+    test_kind = "search_tool"
+
+    def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation:
+        expected_call = (item.expected_output or {}).get("tool_call", {})
+        expected_fn = expected_call.get("function_name", "search_objects")
+        expected_args = expected_call.get("function_arguments", {})
+
+        matching_events = [ev for ev in chat_result.tool_call_events if ev.function_name == expected_fn]
+        tool_selection = len(matching_events) > 0
+        tool_correctness = any(_args_match(ev.parsed_arguments(), expected_args) for ev in matching_events)
+
+        # tool_selection is the hard gate; tool_correctness is scored but not blocking
+        return ItemEvaluation(
+            passed=tool_selection,
+            rank_key=(int(tool_selection), int(tool_correctness)),
+            detail={
+                "tool_selection": tool_selection,
+                "tool_correctness": tool_correctness,
+                "expected_function": expected_fn,
+                "calls_found": len(matching_events),
+            },
+        )

gooddata_eval/core/evaluators/summary.py

@@ -0,0 +1,96 @@
+# (C) 2026 GoodData Corporation
+"""Evaluator for dashboard_summary: rubric-based LLM-as-judge scoring.
+
+Summaries are free text, so we do not match strings. Instead, `expected_output`
+is a rubric of checkable criteria:
+
+    {
+      "must_include":     ["...facts a good summary must contain..."],
+      "must_not_include": ["...things a good summary must avoid (hallucinations)..."],
+      "rubric":           ["...soft quality dimensions..."]
+    }
+
+Each criterion is scored independently by the judge (True/False), so the
+runner's `quality_score` becomes the fraction of satisfied criteria. The item
+*passes* only when every `must_include` is satisfied and no `must_not_include`
+is violated; `rubric` items contribute to quality but do not gate pass/fail.
+
+As a fallback, a non-dict `expected_output` is treated as a single rubric
+criterion (same behaviour as `general_question`).
+"""
+
+from typing import Any
+
+from gooddata_eval.core.evaluators._llm_judge import LLMJudge
+from gooddata_eval.core.evaluators._text_utils import extract_text
+from gooddata_eval.core.evaluators.base import ItemEvaluation
+from gooddata_eval.core.models import ChatResult, DatasetItem
+
+_POSITIVE_STEPS = [
+    "Read the INPUT (the user's request) and the EXPECTED OUTPUT (one criterion the summary must satisfy).",
+    "Read the ACTUAL OUTPUT (the generated summary).",
+    "Score 1 if the actual output clearly satisfies the criterion (allow paraphrasing and reasonable numeric tolerance).",
+    "Score 0 if the criterion is missing, contradicted, or only partially addressed.",
+]
+
+# For must_not_include we ask the judge a plain presence question and invert the
+# result in code. Scoring "does the summary AVOID X?" via a field labelled
+# EXPECTED OUTPUT is unreliable: the model reads the forbidden behaviour as
+# desired and flips the verdict. Detecting presence (no negation, no
+# contradictory label) is far more robust.
+_VIOLATION_STEPS = [
+    "Read the CHARACTERISTIC described in EXPECTED OUTPUT.",
+    "Read the ACTUAL OUTPUT (the generated summary).",
+    "Score 1 if the actual output clearly exhibits the described characteristic.",
+    "Score 0 if it does not exhibit it.",
+]
+
+
+class DashboardSummaryEvaluator:
+    test_kind = "dashboard_summary"
+
+    def __init__(self):
+        self._positive_judge = LLMJudge(evaluation_steps=_POSITIVE_STEPS)
+        self._violation_judge = LLMJudge(evaluation_steps=_VIOLATION_STEPS)
+
+    @staticmethod
+    def _criteria(expected_output: Any) -> tuple[list[str], list[str], list[str]]:
+        if isinstance(expected_output, dict):
+            must_include = [str(c) for c in expected_output.get("must_include", [])]
+            must_not_include = [str(c) for c in expected_output.get("must_not_include", [])]
+            rubric = [str(c) for c in expected_output.get("rubric", [])]
+            if must_include or must_not_include or rubric:
+                return must_include, must_not_include, rubric
+        # Fallback: treat the whole expected_output as a single gating criterion
+        # (same pass/fail semantics as general_question).
+        return [str(expected_output)], [], []
+
+    def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation:
+        actual = extract_text(chat_result)
+        must_include, must_not_include, rubric = self._criteria(item.expected_output)
+
+        detail: dict[str, Any] = {"actual_output": actual}
+        passed = True
+
+        for i, criterion in enumerate(must_include):
+            ok, reason = self._positive_judge.score(item.question, criterion, actual)
+            detail[f"include_{i}"] = ok
+            detail[f"include_{i}_reason"] = reason
+            passed = passed and ok
+
+        for i, criterion in enumerate(must_not_include):
+            violated, reason = self._violation_judge.score(item.question, criterion, actual)
+            ok = not violated  # True == characteristic absent == correctly avoided
+            detail[f"exclude_{i}"] = ok
+            detail[f"exclude_{i}_reason"] = reason
+            passed = passed and ok
+
+        for i, criterion in enumerate(rubric):
+            ok, reason = self._positive_judge.score(item.question, criterion, actual)
+            detail[f"rubric_{i}"] = ok
+            detail[f"rubric_{i}_reason"] = reason
+
+        bool_checks = [v for v in detail.values() if isinstance(v, bool)]
+        quality = sum(1 for v in bool_checks if v) / len(bool_checks) if bool_checks else 0.0
+
+        return ItemEvaluation(passed=passed, rank_key=(int(passed), quality), detail=detail)

gooddata_eval/core/evaluators/visualization.py

@@ -0,0 +1,156 @@
+# (C) 2026 GoodData Corporation
+"""Agentic visualization evaluator — ported from gdc-nas tavern-e2e app/vis_agentic.py."""
+
+from dataclasses import dataclass
+
+from gooddata_eval.core.evaluators.base import ItemEvaluation
+from gooddata_eval.core.models import ChatResult, CreatedVisualization, DatasetItem
+from gooddata_eval.core.scoring import (
+    check_filters,
+    check_viz_type,
+    get_dimension_uri_set,
+    get_metric_uri_set,
+    validate_cross_references,
+)
+
+
+@dataclass
+class EvaluationResult:
+    visualization_created: bool
+    cross_ref_valid: bool
+    metrics_correct: bool
+    dimensions_correct: bool
+    filters_correct: bool
+    viz_type_hard: bool
+    filter_date_score: bool
+    filter_ranking_score: bool
+    filter_attribute_score: bool
+    cross_ref_errors: list[str]
+    expected_metric_uris: set[str]
+    actual_metric_uris: set[str]
+    expected_dim_uris: set[str]
+    actual_dim_uris: set[str]
+
+    @property
+    def strict_pass(self) -> bool:
+        return (
+            self.visualization_created
+            and self.cross_ref_valid
+            and self.metrics_correct
+            and self.dimensions_correct
+            and self.filters_correct
+            and self.viz_type_hard
+        )
+
+    @property
+    def strict_checks_passed_count(self) -> int:
+        return sum(
+            [
+                self.cross_ref_valid,
+                self.metrics_correct,
+                self.dimensions_correct,
+                self.filters_correct,
+                self.viz_type_hard,
+            ]
+        )
+
+
+def _evaluate_visualization(expected: CreatedVisualization, actual: CreatedVisualization | None) -> EvaluationResult:
+    exp_metric_uris = get_metric_uri_set(expected)
+    exp_dim_uris = get_dimension_uri_set(expected)
+    if actual is None:
+        return EvaluationResult(
+            visualization_created=False,
+            cross_ref_valid=False,
+            metrics_correct=False,
+            dimensions_correct=False,
+            filters_correct=False,
+            viz_type_hard=False,
+            filter_date_score=False,
+            filter_ranking_score=False,
+            filter_attribute_score=False,
+            cross_ref_errors=["No visualization was created"],
+            expected_metric_uris=exp_metric_uris,
+            actual_metric_uris=set(),
+            expected_dim_uris=exp_dim_uris,
+            actual_dim_uris=set(),
+        )
+    cross_ref_valid, cross_ref_errors = validate_cross_references(actual)
+    act_metric_uris = get_metric_uri_set(actual)
+    act_dim_uris = get_dimension_uri_set(actual)
+    filter_scores = check_filters(expected, actual)
+    return EvaluationResult(
+        visualization_created=True,
+        cross_ref_valid=cross_ref_valid,
+        metrics_correct=act_metric_uris == exp_metric_uris,
+        dimensions_correct=act_dim_uris == exp_dim_uris,
+        filters_correct=filter_scores.all_ok,
+        viz_type_hard=check_viz_type(expected, actual),
+        filter_date_score=filter_scores.date_ok,
+        filter_ranking_score=filter_scores.ranking_ok,
+        filter_attribute_score=filter_scores.attribute_ok,
+        cross_ref_errors=cross_ref_errors,
+        expected_metric_uris=exp_metric_uris,
+        actual_metric_uris=act_metric_uris,
+        expected_dim_uris=exp_dim_uris,
+        actual_dim_uris=act_dim_uris,
+    )
+
+
+def _evaluate_against_candidates(
+    expected_outputs: list[CreatedVisualization], actual: CreatedVisualization | None
+) -> tuple[EvaluationResult, CreatedVisualization]:
+    pairs = [(_evaluate_visualization(exp, actual), exp) for exp in expected_outputs]
+    best_result, best_expected = max(pairs, key=lambda p: (p[0].strict_pass, p[0].strict_checks_passed_count))
+    return best_result, best_expected
+
+
+def _parse_expected(expected_output: dict) -> list[CreatedVisualization]:
+    if not isinstance(expected_output, dict):
+        raise ValueError("'expected_output' must be a JSON object")
+    raw_viz = expected_output.get("visualization")
+    if raw_viz is None:
+        raise ValueError("'expected_output.visualization' is required")
+    if isinstance(raw_viz, list):
+        if not raw_viz:
+            raise ValueError("'expected_output.visualization' array must not be empty")
+        return [CreatedVisualization.model_validate(v) for v in raw_viz]
+    if isinstance(raw_viz, dict):
+        return [CreatedVisualization.model_validate(raw_viz)]
+    raise ValueError("'expected_output.visualization' must be a JSON object or non-empty array")
+
+
+def _extract_actual(chat_result: ChatResult) -> CreatedVisualization | None:
+    cv = chat_result.created_visualizations
+    if cv is None or not cv.objects:
+        return None
+    return cv.objects[0]
+
+
+class VisualizationEvaluator:
+    test_kind = "visualization"
+
+    def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation:
+        candidates = _parse_expected(item.expected_output)
+        actual = _extract_actual(chat_result)
+        ev, _best_expected = _evaluate_against_candidates(candidates, actual)
+        return ItemEvaluation(
+            passed=ev.strict_pass,
+            rank_key=(ev.strict_pass, ev.strict_checks_passed_count),
+            detail={
+                "visualization_created": ev.visualization_created,
+                "cross_ref_valid": ev.cross_ref_valid,
+                "cross_ref_errors": ev.cross_ref_errors,
+                "metrics_correct": ev.metrics_correct,
+                "dimensions_correct": ev.dimensions_correct,
+                "filters_correct": ev.filters_correct,
+                "filter_date_score": ev.filter_date_score,
+                "filter_ranking_score": ev.filter_ranking_score,
+                "filter_attribute_score": ev.filter_attribute_score,
+                "viz_type_hard": ev.viz_type_hard,
+                "expected_metric_uris": sorted(ev.expected_metric_uris),
+                "actual_metric_uris": sorted(ev.actual_metric_uris),
+                "expected_dim_uris": sorted(ev.expected_dim_uris),
+                "actual_dim_uris": sorted(ev.actual_dim_uris),
+            },
+        )

gooddata_eval/core/langfuse/__init__.py

@@ -0,0 +1 @@
+# (C) 2026 GoodData Corporation

gooddata_eval/core/langfuse/sink.py

@@ -0,0 +1,178 @@
+# (C) 2026 GoodData Corporation
+"""Langfuse scoring sink — posts evaluation results via the Langfuse REST API."""
+
+from __future__ import annotations
+
+import base64
+import os
+import sys
+import uuid
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any
+
+import httpx
+
+_MAX_LATENCY_S = 60.0
+_QUALITY_WEIGHT = 0.6
+_SPEED_WEIGHT = 0.2
+
+if TYPE_CHECKING:
+    from gooddata_eval.core.runner import ItemReport
+
+
+def compute_scores(
+    pass_at_k: bool,
+    avg_latency_s: float,
+    best_detail: dict[str, Any],
+) -> dict[str, float]:
+    """Compute Langfuse score values from an ItemReport.
+
+    Returns a dict with keys: pass_at_k, quality_score, value_score, latency_s.
+    quality_score = fraction of bool-valued keys in best_detail that are True.
+    Falls back to pass_at_k if no bool keys exist (text evaluators).
+    """
+    bool_checks = {k: v for k, v in best_detail.items() if isinstance(v, bool)}
+    quality = sum(1 for v in bool_checks.values() if v) / len(bool_checks) if bool_checks else 1.0 if pass_at_k else 0.0
+
+    speed = max(0.0, 1.0 - avg_latency_s / _MAX_LATENCY_S)
+    value = _QUALITY_WEIGHT * quality + _SPEED_WEIGHT * speed
+
+    return {
+        "pass_at_k": 1 if pass_at_k else 0,
+        "quality_score": round(quality, 4),
+        "value_score": round(value, 4),
+        "latency_s": round(avg_latency_s, 3),
+    }
+
+
+class LangfuseSink:
+    """Posts evaluation results to Langfuse via the ingestion REST API."""
+
+    def __init__(self, dataset_name: str, run_name: str, model_id: str = "", provider_type: str = ""):
+        self._dataset_name = dataset_name
+        self._run_name = run_name
+        self._model_id = model_id
+        self._provider_type = provider_type
+        host = os.environ.get("LANGFUSE_HOST", "https://cloud.langfuse.com").rstrip("/")
+        pub = os.environ.get("LANGFUSE_PUBLIC_KEY", "")
+        sec = os.environ.get("LANGFUSE_SECRET_KEY", "")
+        if not pub or not sec:
+            raise RuntimeError(
+                "Langfuse credentials not set. Export LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY to use --langfuse."
+            )
+        creds = base64.b64encode(f"{pub}:{sec}".encode()).decode()
+        self._host = host
+        self._auth_header = f"Basic {creds}"
+
+    def log_item(self, report: ItemReport, *, dataset_item_id: str) -> None:
+        """Send trace + dataset-run-item + scores for one evaluated item.
+
+        Swallows all errors — Langfuse failures never abort the eval run.
+        """
+        trace_id = str(uuid.uuid4())
+        now = datetime.now(timezone.utc).isoformat()
+        scores = compute_scores(
+            pass_at_k=report.pass_at_k,
+            avg_latency_s=report.avg_latency_s,
+            best_detail=report.best_detail,
+        )
+
+        # Each ingestion event needs a top-level id (dedup) and timestamp
+        # in addition to the body-level id/timestamp for the trace/score itself.
+        def _event(event_type: str, body: dict[str, Any]) -> dict[str, Any]:
+            return {"id": str(uuid.uuid4()), "timestamp": now, "type": event_type, "body": body}
+
+        batch: list[dict[str, Any]] = [
+            _event(
+                "trace-create",
+                {
+                    "id": trace_id,
+                    "timestamp": now,
+                    "name": f"gd-eval: {report.question[:80]}",
+                    # Expose the model on a first-class trace field so Langfuse
+                    # dashboards can filter / break down by it ("Version"); trace
+                    # metadata is not available as a breakdown dimension.
+                    "version": self._model_id or None,
+                    "input": {"question": report.question},
+                    "output": report.best_detail,
+                    "metadata": {
+                        "dataset_name": report.dataset_name,
+                        "test_kind": report.test_kind,
+                        "item_id": report.id,
+                        "model": self._model_id,
+                        "provider_type": self._provider_type,
+                    },
+                    "tags": [t for t in [report.test_kind, self._provider_type] if t],
+                },
+            ),
+        ]
+
+        score_defs = [
+            ("pass_at_k", scores["pass_at_k"], "BOOLEAN"),
+            ("quality_score", scores["quality_score"], "NUMERIC"),
+            ("value_score", scores["value_score"], "NUMERIC"),
+            ("latency_s", scores["latency_s"], "NUMERIC"),
+        ]
+        for name, value, data_type in score_defs:
+            batch.append(
+                _event(
+                    "score-create",
+                    {
+                        "id": str(uuid.uuid4()),
+                        "traceId": trace_id,
+                        "name": name,
+                        "value": value,
+                        "dataType": data_type,
+                    },
+                )
+            )
+
+        try:
+            with httpx.Client(
+                base_url=self._host,
+                headers={"Authorization": self._auth_header},
+                timeout=10,
+            ) as client:
+                resp = client.post("/api/public/ingestion", json={"batch": batch})
+                resp.raise_for_status()
+                # The ingestion endpoint returns HTTP 200 even when individual events
+                # fail — per-event errors are in the response body.
+                body = resp.json()
+                errors = body.get("errors") or []
+                for err in errors:
+                    print(
+                        f"warning: Langfuse event failed for item '{report.id}': "
+                        f"type={err.get('error')} status={err.get('status')} id={err.get('id')}",
+                        file=sys.stderr,
+                    )
+        except Exception as exc:
+            print(f"warning: Langfuse ingestion failed for item '{report.id}': {exc}", file=sys.stderr)
+
+        # Link trace to dataset run via the dedicated endpoint (simpler than ingestion —
+        # does not require datasetId/runId; creates the run by name if absent).
+        try:
+            with httpx.Client(
+                base_url=self._host,
+                headers={"Authorization": self._auth_header},
+                timeout=10,
+            ) as client:
+                r = client.post(
+                    "/api/public/dataset-run-items",
+                    json={
+                        "runName": self._run_name,
+                        "runDescription": (
+                            f"{self._provider_type}/{self._model_id}"
+                            if self._provider_type and self._model_id
+                            else self._model_id or ""
+                        ),
+                        "metadata": {
+                            "model": self._model_id,
+                            "provider_type": self._provider_type,
+                        },
+                        "datasetItemId": dataset_item_id,
+                        "traceId": trace_id,
+                    },
+                )
+                r.raise_for_status()
+        except Exception as exc:
+            print(f"warning: Langfuse dataset-run-item failed for item '{report.id}': {exc}", file=sys.stderr)

gooddata_eval/core/models.py

@@ -0,0 +1,116 @@
+# (C) 2026 GoodData Corporation
+"""Pydantic models for the eval dataset envelope and the agent's AAC output.
+
+Ported from gdc-nas tavern-e2e app/llm_as_judge/schemas/chat.py.
+"""
+
+import json
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class AacQueryField(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    using: str
+    title: str | None = None
+    aggregation: str | None = None
+
+
+class AacBucketRef(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    field: str
+
+
+class AacQuery(BaseModel):
+    fields: dict[str, AacQueryField | str]
+    filter_by: dict[str, dict] = Field(default_factory=dict)
+
+
+class CreatedVisualization(BaseModel):
+    """Visualization in the AAC format (agent output and dataset expected output)."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    id: str
+    title: str | None = None
+    type: str
+    query: AacQuery
+    metrics: list[AacBucketRef | str] = Field(default_factory=list)
+    view_by: list[AacBucketRef | str] = Field(default_factory=list)
+    segment_by: list[AacBucketRef | str] = Field(default_factory=list)
+    rows: list[AacBucketRef | str] = Field(default_factory=list)
+    columns: list[AacBucketRef | str] = Field(default_factory=list)
+    config: dict | None = None
+
+
+class CreatedVisualizations(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    objects: list[CreatedVisualization] = Field(default_factory=list)
+    reasoning: str = ""
+
+
+class ToolCallEvent(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    function_name: str = Field(alias="functionName")
+    function_arguments: str = Field(alias="functionArguments")
+    result: str | None = None
+
+    def parsed_arguments(self) -> dict[str, Any]:
+        try:
+            return json.loads(self.function_arguments) if self.function_arguments else {}
+        except json.JSONDecodeError:
+            return {}
+
+    def parsed_result(self) -> dict[str, Any] | None:
+        if not self.result:
+            return None
+        try:
+            return json.loads(self.result)
+        except json.JSONDecodeError:
+            return None
+
+
+class ChatResult(BaseModel):
+    """Subset of the agent chat response needed for Phase 1 evaluation."""
+
+    model_config = ConfigDict(extra="ignore", populate_by_name=True)
+
+    text_response: str | None = Field(default=None, alias="textResponse")
+    created_visualizations: CreatedVisualizations | None = Field(default=None, alias="createdVisualizations")
+    tool_call_events: list[ToolCallEvent] = Field(default_factory=list, alias="toolCallEvents")
+
+
+class SummaryInput(BaseModel):
+    """Structured input for the `dashboard_summary` test kind.
+
+    Maps onto the dedicated summary endpoint's request body
+    (`POST /api/v1/ai/workspaces/{ws}/summary`). Authored in snake_case in the
+    dataset; the SummaryClient maps it to the endpoint's camelCase fields.
+    """
+
+    model_config = ConfigDict(extra="ignore")
+
+    dashboard_id: str
+    visualizations: list[str] | None = None
+    filter_context: list[dict] | None = None
+    tab_id: str | None = None
+    format_hint: str | None = None
+
+
+class DatasetItem(BaseModel):
+    """Common dataset envelope. `expected_output` stays raw; each evaluator parses its own shape."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    id: str
+    dataset_name: str
+    test_kind: str
+    question: str
+    expected_output: Any
+    # Only used by the `dashboard_summary` test kind; ignored by all others.
+    summary_input: SummaryInput | None = None

gooddata_eval/core/reporting/__init__.py

@@ -0,0 +1 @@
+# (C) 2026 GoodData Corporation