gooddata-eval 1.79.1.dev1__tar.gz → 1.80.0__tar.gz

{gooddata_eval-1.79.1.dev1 → gooddata_eval-1.80.0}/AGENTS.md

@@ -27,7 +27,7 @@ experiment. The newest and most actively developed package in the repo.
 | `core/agentic/` | multi-turn agentic evaluation per capability, **plus** all Langfuse trace polling and linking (`_langfuse.py`, `_trace_linker.py`) |
 | `core/chat/` | SSE client for the agent's streaming chat endpoint |
 | `core/summary/` | HTTP client for the dedicated dashboard-summary endpoint — a single-shot chat backend, not reporting |
-| `core/takeaway/` | HTTP client for the AI Publisher takeaway endpoint — a single-shot backend like `core/summary/`, keeping the whole response on `ChatResult.raw_response` — plus `evaluate_document_takeaway`, the K-run, gate and Langfuse entry point the Tavern shim calls |
+| `core/takeaway/` | HTTP client for the AI Publisher takeaway endpoint — a single-shot backend like `core/summary/`, keeping the whole response on `ChatResult.raw_response` — plus `evaluate_document_takeaway`, the K-run, gate and Langfuse entry point the Tavern shim calls. Also used by the agentic document skill to write the takeaways the copilot leaves to the editor, before the narrative is graded |
 | `core/dataset/` | dataset format, loading, and `from_insights.py` — dataset generation from a workspace's real insights |
 | `core/evaluators/` | single-shot evaluators and their registry |
 | `core/langfuse/` | the whole Langfuse v4 client: `_env` (base URL + credentials), `otlp` (OTLP/JSON encoding), `experiment` (root-span construction, score targets), `item_scope` (an item's experiment scope sent to gen-ai as baggage, and which conversations joined), `observations` (trace reads), `client` (httpx calls), `sink` (single-shot results as experiments) |

{gooddata_eval-1.79.1.dev1 → gooddata_eval-1.80.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gooddata-eval
-Version: 1.79.1.dev1
+Version: 1.80.0
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.79.1.dev1
+Requires-Dist: gooddata-sdk~=1.80.0
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: jsonpatch<2.0,>=1.33
 Requires-Dist: orjson<4.0.0,>=3.9.15
@@ -306,7 +306,8 @@ Each item additionally carries a per-phase breakdown:
 ```
 
 Every item reports `runs_ungraded` beside `runs_passed`: runs the agent answered but the LLM judge returned
-nothing readable for. Agentic items also list them as `unscored_runs` / `judge_errors` in their `detail`, and a
+nothing readable for. Agentic items also list them as `unscored_runs` / `judge_errors` in their `detail` (a document item whose takeaway-endpoint request failed
+lists that failure under `request_errors` instead), and a
 `dashboard_summary` item carries `ungraded_criteria`. Such a run — or, for `dashboard_summary`, such a
 criterion — is excluded from pass@K and from the quality score rather than counted as a failure: scoring it 0
 would be indistinguishable from the judge genuinely failing the answer, which is the confusion

{gooddata_eval-1.79.1.dev1 → gooddata_eval-1.80.0}/README.md

@@ -277,7 +277,8 @@ Each item additionally carries a per-phase breakdown:
 ```
 
 Every item reports `runs_ungraded` beside `runs_passed`: runs the agent answered but the LLM judge returned
-nothing readable for. Agentic items also list them as `unscored_runs` / `judge_errors` in their `detail`, and a
+nothing readable for. Agentic items also list them as `unscored_runs` / `judge_errors` in their `detail` (a document item whose takeaway-endpoint request failed
+lists that failure under `request_errors` instead), and a
 `dashboard_summary` item carries `ungraded_criteria`. Such a run — or, for `dashboard_summary`, such a
 criterion — is excluded from pass@K and from the quality score rather than counted as a failure: scoring it 0
 would be indistinguishable from the judge genuinely failing the answer, which is the confusion

{gooddata_eval-1.79.1.dev1 → gooddata_eval-1.80.0}/pyproject.toml

@@ -1,7 +1,7 @@
 # (C) 2026 GoodData Corporation
 [project]
 name = "gooddata-eval"
-version = "1.79.1.dev1"
+version = "1.80.0"
 description = "Evaluate the GoodData AI agent against your own questions and models."
 readme = "README.md"
 license = "MIT"
@@ -11,7 +11,7 @@ authors = [
 keywords = ["gooddata", "ai", "evaluation", "llm", "analytics", "cli"]
 requires-python = ">=3.10"
 dependencies = [
-    "gooddata-sdk~=1.79.1.dev1",
+    "gooddata-sdk~=1.80.0",
     "httpx>=0.27,<1.0",
     "jsonpatch>=1.33,<2.0",
     "orjson>=3.9.15,<4.0.0",

{gooddata_eval-1.79.1.dev1 → gooddata_eval-1.80.0}/src/gooddata_eval/cli/agentic_runner.py

@@ -16,7 +16,9 @@ from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill
 from gooddata_eval.core.agentic.anomaly_detection import evaluate_agentic_anomaly_detection
 from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation
 from gooddata_eval.core.agentic.dashboard_skill import evaluate_agentic_dashboard_skill
+from gooddata_eval.core.agentic.dashboard_summary import evaluate_agentic_dashboard_summary
 from gooddata_eval.core.agentic.document_skill import evaluate_agentic_document_skill
+from gooddata_eval.core.agentic.forecasting import evaluate_agentic_forecasting
 from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question
 from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail
 from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill
@@ -56,6 +58,8 @@ AGENTIC_TEST_KINDS = frozenset(
         "agentic_guardrail",
         "agentic_conversation",
         "agentic_kda_skill",
+        "agentic_forecasting",
+        "agentic_dashboard_summary",
         "agentic_anomaly_detection",
         "agentic_what_if",
     }
@@ -303,6 +307,41 @@ def _dispatch_agentic(
             user_context=item.user_context,
             **lf_kw,
         )
+    elif kind == "agentic_forecasting":
+        return evaluate_agentic_forecasting(
+            host=host,
+            token=token,
+            workspace_id=workspace_id,
+            question=item.question,
+            expected_output=eo if isinstance(eo, dict) else {},
+            k=k,
+            gate=gate,
+            agent_id=agent_id,
+            user_context=item.user_context,
+            **lf_kw,
+        )
+    elif kind == "agentic_dashboard_summary":
+        summary_input = item.summary_input
+        if summary_input is None:
+            raise ValueError(f"agentic_dashboard_summary item '{item.id}' is missing required 'summary_input'.")
+        return evaluate_agentic_dashboard_summary(
+            host=host,
+            token=token,
+            workspace_id=workspace_id,
+            dashboard_id=summary_input.dashboard_id,
+            expected_output=eo,
+            # The fixture's own wording is the prompt under test -- a localized fixture is
+            # only meaningful if its own phrasing is what reaches the agent.
+            question=item.question,
+            # Each widget costs an execution, so a fixture asserting on a handful of charts
+            # can name them instead of paying for the whole dashboard. Same field and meaning
+            # the headless /summary endpoint gives it; None summarizes everything.
+            only_visualizations=summary_input.visualizations,
+            k=k,
+            gate=gate,
+            agent_id=agent_id,
+            **lf_kw,
+        )
     elif kind == "agentic_what_if":
         return evaluate_agentic_what_if(
             host=host,
@@ -404,6 +443,18 @@ def _apply_timings(item_report: ItemReport, timings: Any) -> None:
     item_report.simulated_user_latency_s = timings.simulated_user_s
 
 
+def _apply_best_run_latency(item_report: ItemReport, source: Any) -> None:
+    """Copy the rank-selected run's own wall time, mirroring the single-shot path's
+    ``ItemReport.best_run_latency_s`` (see ``core/runner.py``'s ``_run_one_item``).
+
+    Kinds not yet wired to measure it pass None and keep the field at its own None
+    default rather than reporting an invented number.
+    """
+    best_run_latency_s = getattr(source, "best_run_latency_s", None)
+    if best_run_latency_s is not None:
+        item_report.best_run_latency_s = best_run_latency_s
+
+
 def run_agentic_items(
     items: list[DatasetItem],
     host: str,
@@ -501,6 +552,7 @@ def run_agentic_items(
             item_report.response_id = response_id
             item_report.best_detail = detail or {}
             _apply_timings(item_report, getattr(outcome, "timings", None))
+            _apply_best_run_latency(item_report, outcome)
             _apply_run_counts(item_report, outcome)
             _apply_failed_runs(item_report, outcome)
             _apply_run_latencies(item_report, outcome)
@@ -512,6 +564,7 @@ def run_agentic_items(
             item_report.response_id = getattr(exc, "response_id", None)
             item_report.best_detail = getattr(exc, "detail", None) or {}
             _apply_timings(item_report, getattr(exc, "timings", None))
+            _apply_best_run_latency(item_report, exc)
             _apply_run_counts(item_report, exc)
             _apply_failed_runs(item_report, exc)
             _apply_run_latencies(item_report, exc)
@@ -555,6 +608,7 @@ def run_agentic_items(
             # runs and then raised would otherwise show up in the per-response table and not in
             # the per-conversation one.
             traces.discard()
+            _apply_best_run_latency(item_report, exc)
         finally:
             item_report.latency_s = time.perf_counter() - t0
             item_report.trace_latencies_s = traces.stop()