gooddata-eval 1.76.1.dev3__tar.gz → 1.77.0__tar.gz

{gooddata_eval-1.76.1.dev3 → gooddata_eval-1.77.0}/AGENTS.md

@@ -29,7 +29,7 @@ experiment. The newest and most actively developed package in the repo.
 | `core/summary/` | HTTP client for the dedicated dashboard-summary endpoint — a single-shot chat backend, not reporting |
 | `core/dataset/` | dataset format, loading, and `from_insights.py` — dataset generation from a workspace's real insights |
 | `core/evaluators/` | single-shot evaluators and their registry |
-| `core/langfuse/` | the whole Langfuse v4 client: `_env` (base URL + credentials), `otlp` (OTLP/JSON encoding), `experiment` (root-span construction, score targets), `observations` (trace reads), `client` (httpx calls), `sink` (single-shot results as experiments) |
+| `core/langfuse/` | the whole Langfuse v4 client: `_env` (base URL + credentials), `otlp` (OTLP/JSON encoding), `experiment` (root-span construction, score targets), `item_scope` (an item's experiment scope sent to gen-ai as baggage, and which conversations joined), `observations` (trace reads), `client` (httpx calls), `sink` (single-shot results as experiments) |
 | `core/reporting/` | console and JSON output rendering |
 | `core/scoring.py`, `core/runner.py` | scoring and orchestration |
 | `core/models.py` | `DatasetItem`, `ChatResult`, `ItemReport` and friends |
@@ -157,6 +157,7 @@ Run this before guessing a `--model` string.
 | `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` | `--langfuse`, `--langfuse-dataset` |
 | `GOODDATA_EVAL_CHAT_*` | SSE retry, backoff and timeout knobs |
 | `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`, so set it only for a one-model run: every `--model` in one process gets the same labels. Keys must match `[A-Za-z0-9_-]+`; others are skipped with a warning. Dashboard-summary runs are not labelled |
+| `GOODDATA_EVAL_JOIN_GENAI_TRACE` | on unless set to `0`, `false`, `no`, `off` or empty. An agentic item's chat turns ask gen-ai to open them under gd-eval's experiment root, so each item is one Langfuse trace with its scores written once. A gen-ai that drops caller baggage keeps today's separate traces; see the gotcha below |
 | `GD_EVAL_TIMERS` | same as `--timers` |
 
 A gitignored `.env` at the repo root is the normal place for these; load it with
@@ -190,6 +191,13 @@ ingestion has no pass/fail signal and inflates or misattributes per-item latency
 (`run_trace_link_inline` is the synchronous alternative). Do not "fix" a slow item by
 making trace scoring synchronous again.
 
+**Turn `GOODDATA_EVAL_JOIN_GENAI_TRACE` off against a gen-ai that accepts caller baggage
+but cannot join.** The trace and root ids are derived from the conversation id, and a conversation
+counts as joined only when gen-ai reports that trace id on `response_started`. A gen-ai that
+reads the baggage without that support tags its own traces with experiment ids under a root
+gd-eval never exports. The same orphan is left when an item raises before its deferred
+scoring runs: its turns sit in environment `sdk-experiment` without a root or scores.
+
 **A generated item's `expected_output` is copied, never invented — keep it that way.**
 `core/dataset/from_insights.py` converts each insight with the platform's own
 `declarative_visualization_to_aac()` (from `gooddata-code-convertors`, via `gooddata-sdk`),

{gooddata_eval-1.76.1.dev3 → gooddata_eval-1.77.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.76.1.dev3
+Version: 1.77.0
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.76.1.dev3
+Requires-Dist: gooddata-sdk~=1.77.0
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: jsonpatch<2.0,>=1.33
 Requires-Dist: orjson<4.0.0,>=3.9.15

{gooddata_eval-1.76.1.dev3 → gooddata_eval-1.77.0}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.76.1.dev3"
+version = "1.77.0"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.76.1.dev3",
+    "gooddata-sdk~=1.77.0",
     "httpx>=0.27,<1.0",
     "jsonpatch>=1.33,<2.0",
     "orjson>=3.9.15,<4.0.0",
@@ -71,3 +71,6 @@ allowed-unresolved-imports = ["openai.**", "gooddata_api_client.**"]
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
+
+[tool.pytest]
+strict = true

{gooddata_eval-1.76.1.dev3 → gooddata_eval-1.77.0}/src/gooddata_eval/cli/agentic_runner.py

@@ -15,15 +15,16 @@ from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill
 from gooddata_eval.core.agentic.anomaly_detection import evaluate_agentic_anomaly_detection
 from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation
 from gooddata_eval.core.agentic.dashboard_skill import evaluate_agentic_dashboard_skill
+from gooddata_eval.core.agentic.document_skill import evaluate_agentic_document_skill
 from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question
 from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail
 from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill
 from gooddata_eval.core.agentic.metric_skill import evaluate_agentic_metric_skill
-from gooddata_eval.core.agentic.report_skill import evaluate_agentic_report_skill
 from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
 from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if
 from gooddata_eval.core.config import ReasoningEffort
+from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
 from gooddata_eval.core.evaluators.visualization import requires_execution_of
 from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
@@ -37,6 +38,7 @@ class _LfKw(TypedDict, total=False):
     model_version_override: str | None
     reasoning_effort: ReasoningEffort | None
     submit_trace_link: SubmitTraceLink
+    run_metadata_extra: dict[str, Any] | None
 
 
 AGENTIC_TEST_KINDS = frozenset(
@@ -46,7 +48,7 @@ AGENTIC_TEST_KINDS = frozenset(
         "agentic_metric_skill",
         "agentic_alert_skill",
         "agentic_dashboard_skill",
-        "agentic_report_skill",
+        "agentic_document_skill",
         "agentic_search",
         "agentic_general_question",
         "agentic_guardrail",
@@ -97,8 +99,8 @@ PARALLEL_SAFE_TEST_KINDS = frozenset(
 #
 # agentic_dashboard_skill is absent by default rather than by evidence: gen-ai holds the draft and
 # any chart it authors in conversation state and writes neither until a user saves from the UI, so
-# it is a candidate for the allowlist once the dataset has runs behind it. agentic_report_skill is
-# absent for the same reason: the report draft stays in conversation state until a user saves it.
+# it is a candidate for the allowlist once the dataset has runs behind it. agentic_document_skill is
+# absent for the same reason: the document draft stays in conversation state until a user saves it.
 WORKSPACE_MUTATING_TEST_KINDS = frozenset(AGENTIC_TEST_KINDS) - PARALLEL_SAFE_TEST_KINDS
 
 
@@ -151,11 +153,13 @@ def _dispatch_agentic(
     agent_id: str | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
     gate: EvalGate = DEFAULT_GATE,
+    run_metadata_extra: dict[str, Any] | None = None,
 ) -> AgenticEvalOutcome:
     """Call the appropriate evaluate_agentic_* function for the item's test_kind.
 
     `gate` reaches every kind except those in UNGATED_AGENTIC_TEST_KINDS, which have no K
     to gate over; the CLI refuses --gate power for a dataset containing one.
+    `run_metadata_extra` reaches every kind unchanged.
 
     Every evaluate_agentic_* function returns an AgenticEvalOutcome (reasoning_steps,
     conversation_id, response_id, detail) on success and attaches the same four attributes
@@ -171,6 +175,7 @@ def _dispatch_agentic(
         "model_version_override": model_version_override,
         "reasoning_effort": reasoning_effort,
         "submit_trace_link": submit_trace_link,
+        "run_metadata_extra": run_metadata_extra,
     }
 
     if kind in ("vis_agentic", "agentic_visualization"):
@@ -213,8 +218,8 @@ def _dispatch_agentic(
             user_context=item.user_context,
             **lf_kw,
         )
-    elif kind == "agentic_report_skill":
-        return evaluate_agentic_report_skill(
+    elif kind == "agentic_document_skill":
+        return evaluate_agentic_document_skill(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -302,6 +307,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, dict) else {},
             k=k,
+            gate=gate,
             agent_id=agent_id,
             user_context=item.user_context,
             **lf_kw,
@@ -355,6 +361,20 @@ def _apply_run_counts(item_report: ItemReport, source: Any) -> None:
         item_report.runs_ungraded = unscored
 
 
+def _apply_failed_runs(item_report: ItemReport, source: Any) -> None:
+    """Copy the per-run failure records off an outcome or a failure, if the kind built any.
+
+    Read the same way from both, like the run counts above: a partially passing item raises,
+    so an item's failing runs reach the report through the exception at least as often as
+    through the outcome. A kind that builds none keeps the empty default, which reads as
+    "not instrumented" rather than "nothing failed" -- ``runs_passed``/``runs`` already say
+    how many failed.
+    """
+    failed_runs = getattr(source, "failed_runs", None)
+    if failed_runs:
+        item_report.failed_runs = list(failed_runs)
+
+
 def _apply_timings(item_report: ItemReport, timings: Any) -> None:
     """Copy an outcome's phase breakdown onto the item report, if the kind recorded one.
 
@@ -384,6 +404,7 @@ def run_agentic_items(
     agent_id: str | None = None,
     concurrency: int = 1,
     gate: EvalGate = DEFAULT_GATE,
+    tags: tuple[str, ...] = (),
 ) -> EvalReport:
     """Run agentic items through evaluate_agentic_* and return an EvalReport.
 
@@ -392,9 +413,14 @@ def run_agentic_items(
     from the parallel ones -- a metric being created and dropped mid-run would otherwise be
     visible to a catalog-reading item running alongside it.
 
+    ``tags`` are added to every Langfuse trace gd-eval writes for these items, after the
+    automatic ones. They reach Langfuse whenever trace linking runs, with or without
+    ``use_langfuse``.
+
     Results are collected in dataset order regardless of completion order.
     """
     langfuse = make_langfuse_client() if use_langfuse else None
+    run_metadata_extra: dict[str, Any] | None = {"tags": list(tags)} if tags else None
 
     report = EvalReport(model=model_version, gate=normalize_gate(gate))
     total = len(items)
@@ -436,6 +462,7 @@ def run_agentic_items(
                 agent_id,
                 submit_trace_link=linker.submit,
                 gate=gate,
+                run_metadata_extra=run_metadata_extra,
             )
             if isinstance(outcome, AgenticEvalOutcome):
                 reasoning_steps = outcome.reasoning_steps
@@ -454,6 +481,7 @@ def run_agentic_items(
             item_report.best_detail = detail or {}
             _apply_timings(item_report, getattr(outcome, "timings", None))
             _apply_run_counts(item_report, outcome)
+            _apply_failed_runs(item_report, outcome)
         except AssertionError as exc:
             item_report.gate_passed = False if gated else None
             item_report.runs = k
@@ -463,11 +491,32 @@ def run_agentic_items(
             item_report.best_detail = getattr(exc, "detail", None) or {}
             _apply_timings(item_report, getattr(exc, "timings", None))
             _apply_run_counts(item_report, exc)
+            _apply_failed_runs(item_report, exc)
             # Read off the counts, not off the gate: pass^K fails items where runs did pass,
             # and reporting those as pass_at_k False would contradict the Langfuse score of
             # the same name. Kinds that report no count read as 0, i.e. a clean failure.
             item_report.pass_at_k = item_report.runs_passed > 0
             print(f"[agentic] {item.id} FAIL: {exc}", flush=True)
+        except JudgeResponseError as exc:
+            # Errored, like the branch below -- there is no verdict for any run, so this is
+            # not K failures -- but NOT diagnostically empty. This is the one failure mode
+            # where the per-run records are most worth having: the judge broke, so what the
+            # agent actually said is still there to read, and the conversation ids are how
+            # anyone gets to it. Reported with the runs it really drove rather than 0.
+            item_report.error = f"{type(exc).__name__}: {exc}"
+            item_report.runs = getattr(exc, "runs_effective", None) or k
+            item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or []
+            item_report.conversation_id = getattr(exc, "conversation_id", None)
+            item_report.response_id = getattr(exc, "response_id", None)
+            item_report.best_detail = getattr(exc, "detail", None) or {}
+            _apply_timings(item_report, getattr(exc, "timings", None))
+            _apply_run_counts(item_report, exc)
+            _apply_failed_runs(item_report, exc)
+            # Every run ungraded, by definition of this error -- said explicitly rather than
+            # left to _apply_run_counts, whose source is detail["unscored_runs"] and which a
+            # kind that attaches no diagnostics would leave at 0.
+            item_report.runs_ungraded = item_report.runs
+            print(f"[agentic] {item.id} UNGRADED: {exc}", flush=True)
         except Exception as exc:
             item_report.error = f"{type(exc).__name__}: {exc}"
             item_report.runs = 0

{gooddata_eval-1.76.1.dev3 → gooddata_eval-1.77.0}/src/gooddata_eval/cli/main.py

@@ -183,6 +183,17 @@ def _build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).",
     )
+    run.add_argument(
+        "--tag",
+        dest="tags",
+        action="append",
+        default=[],
+        type=_parse_tag,
+        metavar="TAG",
+        help="Custom tag for every Langfuse trace gd-eval writes, next to the automatic ones. "
+        "Repeat for several tags. Applied wherever traces are written: single-turn kinds only "
+        "with --langfuse, agentic kinds whenever LANGFUSE_* env vars are set.",
+    )
     run.add_argument(
         "--agent-id",
         dest="agent_id",
@@ -377,6 +388,14 @@ def _parse_model_arg(val: str) -> tuple[str | None, str]:
     return None, val
 
 
+def _parse_tag(val: str) -> str:
+    """Strip one --tag value and reject a blank one, which Langfuse would store as an empty tag."""
+    tag = val.strip()
+    if not tag:
+        raise argparse.ArgumentTypeError("tag must not be blank")
+    return tag
+
+
 def _make_progress_callbacks(console: Console):
     """Build (on_item_start, on_run_done, on_item_done) callbacks that stream progress.
 
@@ -549,6 +568,7 @@ def _run(config: RunConfig) -> int:
                     model_id=resolved.model_id,
                     provider_type=resolved.provider_type,
                     reasoning_effort=config.reasoning_effort,
+                    tags=config.tags,
                 )
 
                 def on_langfuse_item_done(
@@ -574,6 +594,7 @@ def _run(config: RunConfig) -> int:
                     model_version=resolved.model_id,
                     reasoning_effort=config.reasoning_effort,
                     use_langfuse=config.log_to_langfuse,
+                    tags=config.tags,
                     run_ts=run_ts,
                     on_item_start=on_item_start,
                     on_item_done=on_item_done,
@@ -728,6 +749,7 @@ def main(argv: list[str] | None = None) -> int:
             agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"),
             turn_timeout_s=args.turn_timeout,
             item_timeout_s=args.item_timeout,
+            tags=tuple(dict.fromkeys(args.tags)),
         )
         return _run(config)
     except (

{gooddata_eval-1.76.1.dev3 → gooddata_eval-1.77.0}/src/gooddata_eval/core/agentic/__init__.py

@@ -24,6 +24,14 @@ from gooddata_eval.core.agentic.dashboard_skill import (
     evaluate_agentic_dashboard_skill,
     run_agentic_dashboard_skill,
 )
+from gooddata_eval.core.agentic.document_skill import (
+    AgenticDocumentSummary,
+    DocumentEvaluation,
+    DocumentRunResult,
+    DocumentSkillAssertionError,
+    evaluate_agentic_document_skill,
+    run_agentic_document_skill,
+)
 from gooddata_eval.core.agentic.general_question import (
     AgenticGeneralQuestionSummary,
     GeneralQuestionAssertionError,
@@ -53,14 +61,6 @@ from gooddata_eval.core.agentic.metric_skill import (
     evaluate_agentic_metric_skill,
     run_agentic_metric_skill,
 )
-from gooddata_eval.core.agentic.report_skill import (
-    AgenticReportSummary,
-    ReportEvaluation,
-    ReportRunResult,
-    ReportSkillAssertionError,
-    evaluate_agentic_report_skill,
-    run_agentic_report_skill,
-)
 from gooddata_eval.core.agentic.search_tool import (
     AgenticSearchSummary,
     SearchResult,
@@ -76,6 +76,14 @@ from gooddata_eval.core.agentic.visualization import (
     run_agentic_visualization,
 )
 
+# Deprecated names of the document skill, kept for existing callers.
+AgenticReportSummary = AgenticDocumentSummary
+ReportEvaluation = DocumentEvaluation
+ReportRunResult = DocumentRunResult
+ReportSkillAssertionError = DocumentSkillAssertionError
+evaluate_agentic_report_skill = evaluate_agentic_document_skill
+run_agentic_report_skill = run_agentic_document_skill
+
 __all__ = [
     "AgenticAlertSummary",
     "AgenticDashboardSummary",
@@ -83,6 +91,7 @@ __all__ = [
     "AgenticGuardrailSummary",
     "AgenticKdaSummary",
     "AgenticMetricSummary",
+    "AgenticDocumentSummary",
     "AgenticReportSummary",
     "AgenticSearchSummary",
     "AgenticRunSummary",
@@ -95,6 +104,9 @@ __all__ = [
     "DashboardEvaluation",
     "DashboardRunResult",
     "DashboardSkillAssertionError",
+    "DocumentEvaluation",
+    "DocumentRunResult",
+    "DocumentSkillAssertionError",
     "GeneralQuestionAssertionError",
     "GeneralQuestionResult",
     "GuardrailAssertionError",
@@ -116,6 +128,7 @@ __all__ = [
     "evaluate_agentic_alert_skill",
     "evaluate_agentic_conversation",
     "evaluate_agentic_dashboard_skill",
+    "evaluate_agentic_document_skill",
     "evaluate_agentic_general_question",
     "evaluate_agentic_guardrail",
     "evaluate_agentic_kda_skill",
@@ -126,6 +139,7 @@ __all__ = [
     "run_agentic_alert_skill",
     "run_agentic_conversation",
     "run_agentic_dashboard_skill",
+    "run_agentic_document_skill",
     "run_agentic_general_question",
     "run_agentic_guardrail",
     "run_agentic_kda_skill",

gooddata_eval-1.77.0/src/gooddata_eval/core/agentic/_failed_runs.py

@@ -0,0 +1,62 @@
+# (C) 2026 GoodData Corporation. All rights reserved.
+"""Per-run failure records for the agentic evaluators.
+
+``core/runner.py`` records one of these for every failing run of a single-shot item. The
+agentic kinds cannot reuse it: each of them drives its own K-loop inside the evaluator and
+hands ``cli/agentic_runner`` a single aggregate per item, so by the time the runner sees
+the result the individual attempts are already collapsed into ``best``. The runs themselves
+are not lost -- every kind keeps its ``run_results`` list -- they simply never leave the
+evaluator. This turns that list into the same records the single-shot path writes.
+
+Keys mirror ``core.runner._failed_run_record`` so one consumer can read ``failed_runs`` from
+either path without branching on test kind. ``tool_call_count``/``tool_names`` are the
+addition: the agentic kinds capture tool calls per run, and an agent that produced a final
+answer having made no tool call at all answered from the model's own knowledge rather than
+from the workspace -- a distinction no other recorded field exposes.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from typing import Any
+
+
+def build_failed_runs(
+    run_results: Sequence[Any],
+    *,
+    passed: Callable[[Any], bool],
+    detail: Callable[[Any], dict],
+) -> list[dict]:
+    """One record per run that did not pass, in the order the runs happened.
+
+    ``passed`` and ``detail`` are supplied by the caller because neither is uniform across
+    kinds: guardrail reads ``run.passed`` while visualization reads
+    ``run.eval_result.strict_pass``, and each kind's diagnostic dict is its own. Everything
+    read here directly goes through ``getattr`` with a default, so a kind whose run result
+    lacks a field (no response id, no tool capture) records a null instead of raising.
+
+    A run the judge could not grade is recorded like any other non-passing run, with its
+    ``error`` set. That matches ``core/runner.py``, which also records ungraded runs; the
+    verdict-level accounting of ungraded runs stays with the caller's ``unscored_runs``.
+    """
+    records: list[dict] = []
+    for run_index, run in enumerate(run_results, start=1):
+        if passed(run):
+            continue
+        tool_calls = list(getattr(run, "tool_call_events", None) or [])
+        reasoning_steps = list(getattr(run, "reasoning_steps", None) or [])
+        records.append(
+            {
+                "run_index": run_index,
+                "passed": False,
+                "error": getattr(run, "judge_error", None),
+                "detail": detail(run),
+                "conversation_id": getattr(run, "conversation_id", None),
+                "response_id": getattr(run, "response_id", None),
+                "reasoning_step_count": len(reasoning_steps),
+                "reasoning_steps": reasoning_steps,
+                "tool_call_count": len(tool_calls),
+                "tool_names": [getattr(e, "function_name", None) for e in tool_calls],
+            }
+        )
+    return records