gooddata-eval 1.74.1.dev2__py3-none-any.whl → 1.74.1.dev4__py3-none-any.whl

gooddata_eval/cli/agentic_runner.py

@@ -8,6 +8,7 @@ import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Any, TypedDict
 
+from gooddata_eval.core.agentic._gate import DEFAULT_GATE, EvalGate, normalize_gate
 from gooddata_eval.core.agentic._langfuse import make_langfuse_client
 from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline
 from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill
@@ -48,6 +49,13 @@ AGENTIC_TEST_KINDS = frozenset(
 )
 
 
+# Agentic kinds that no gate applies to: they drive their fixture exactly once, so there is
+# no K to take pass@K or pass^K over. Named here rather than inline in _dispatch_agentic so
+# the CLI can refuse --gate power for a dataset containing one instead of labelling the whole
+# report `power` when part of it was never gated.
+UNGATED_AGENTIC_TEST_KINDS = frozenset({"agentic_conversation"})
+
+
 # Kinds cleared to run several at a time. An EXPLICIT allowlist, not a subtraction: nothing
 # in this package can prove a kind is read-only, because the mutation happens server-side in
 # whichever tools the agent decides to call. So each entry here is a reviewed judgement, and
@@ -128,9 +136,13 @@ def _dispatch_agentic(
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
+    gate: EvalGate = DEFAULT_GATE,
 ) -> AgenticEvalOutcome:
     """Call the appropriate evaluate_agentic_* function for the item's test_kind.
 
+    `gate` reaches every kind except those in UNGATED_AGENTIC_TEST_KINDS, which have no K
+    to gate over; the CLI refuses --gate power for a dataset containing one.
+
     Every evaluate_agentic_* function returns an AgenticEvalOutcome (reasoning_steps,
     conversation_id, response_id, detail) on success and attaches the same four attributes
     to its raised *AssertionError on failure -- no kind is exempt.
@@ -155,6 +167,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_outputs=_parse_visualization_expected(eo),
             k=k,
+            gate=gate,
             agent_id=agent_id,
             **lf_kw,
         )
@@ -166,6 +179,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, (dict, list)) else {},
             k=k,
+            gate=gate,
             agent_id=agent_id,
             **lf_kw,
         )
@@ -177,6 +191,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, dict) else {},
             k=k,
+            gate=gate,
             agent_id=agent_id,
             **lf_kw,
         )
@@ -191,6 +206,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_tool_call=expected_args,
             k=k,
+            gate=gate,
             agent_id=agent_id,
             **lf_kw,
         )
@@ -202,6 +218,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, str) else str(eo),
             k=k,
+            gate=gate,
             agent_id=agent_id,
             user_context=item.user_context,
             **lf_kw,
@@ -214,6 +231,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, str) else str(eo),
             k=k,
+            gate=gate,
             agent_id=agent_id,
             **lf_kw,
         )
@@ -225,6 +243,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, dict) else {},
             k=k,
+            gate=gate,
             agent_id=agent_id,
             **lf_kw,
         )
@@ -291,6 +310,7 @@ def run_agentic_items(
     on_item_done: Any = None,
     agent_id: str | None = None,
     concurrency: int = 1,
+    gate: EvalGate = DEFAULT_GATE,
 ) -> EvalReport:
     """Run agentic items through evaluate_agentic_* and return an EvalReport.
 
@@ -303,7 +323,7 @@ def run_agentic_items(
     """
     langfuse = make_langfuse_client() if use_langfuse else None
 
-    report = EvalReport(model=model_version)
+    report = EvalReport(model=model_version, gate=normalize_gate(gate))
     total = len(items)
     # Trace linking runs here rather than inside each evaluate_agentic_*, so an item's
     # Langfuse poll overlaps the NEXT item's agent call instead of extending its own
@@ -325,6 +345,9 @@ def run_agentic_items(
             test_kind=item.test_kind,
             question=item.question,
         )
+        # None, not False, for the kinds _dispatch_agentic passes no gate to: ItemReport.passed
+        # then falls back to pass_at_k and gate_passed keeps meaning "a gate ran".
+        gated = item.test_kind not in UNGATED_AGENTIC_TEST_KINDS
         t0 = time.perf_counter()
         try:
             outcome = _dispatch_agentic(
@@ -339,6 +362,7 @@ def run_agentic_items(
                 reasoning_effort,
                 agent_id,
                 submit_trace_link=linker.submit,
+                gate=gate,
             )
             if isinstance(outcome, AgenticEvalOutcome):
                 reasoning_steps = outcome.reasoning_steps
@@ -347,6 +371,8 @@ def run_agentic_items(
                 detail = outcome.detail
             else:
                 reasoning_steps, conversation_id, response_id, detail = outcome, None, None, {}
+            item_report.gate_passed = True if gated else None
+            # Whichever gate decided the item, clearing it means at least one run passed.
             item_report.pass_at_k = True
             item_report.runs = k
             item_report.reasoning_steps = reasoning_steps or []
@@ -356,7 +382,7 @@ def run_agentic_items(
             _apply_timings(item_report, getattr(outcome, "timings", None))
             _apply_run_counts(item_report, outcome)
         except AssertionError as exc:
-            item_report.pass_at_k = False
+            item_report.gate_passed = False if gated else None
             item_report.runs = k
             item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or []
             item_report.conversation_id = getattr(exc, "conversation_id", None)
@@ -364,6 +390,10 @@ def run_agentic_items(
             item_report.best_detail = getattr(exc, "detail", None) or {}
             _apply_timings(item_report, getattr(exc, "timings", None))
             _apply_run_counts(item_report, exc)
+            # Read off the counts, not off the gate: pass^K fails items where runs did pass,
+            # and reporting those as pass_at_k False would contradict the Langfuse score of
+            # the same name. Kinds that report no count read as 0, i.e. a clean failure.
+            item_report.pass_at_k = item_report.runs_passed > 0
             print(f"[agentic] {item.id} FAIL: {exc}", flush=True)
         except Exception as exc:
             item_report.error = f"{type(exc).__name__}: {exc}"

gooddata_eval/cli/main.py

@@ -14,11 +14,20 @@ from gooddata_api_client.exceptions import ApiException
 from rich.console import Console
 from rich.table import Table
 
-from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items
+from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, UNGATED_AGENTIC_TEST_KINDS, run_agentic_items
 from gooddata_eval.core.chat.sse_client import ChatClient
-from gooddata_eval.core.config import DEFAULT_JUDGE_MODEL, JUDGE_MODEL_ENV_VAR, ReasoningEffort, RunConfig
+from gooddata_eval.core.config import (
+    DEFAULT_GATE,
+    DEFAULT_JUDGE_MODEL,
+    JUDGE_MODEL_ENV_VAR,
+    EvalGate,
+    ReasoningEffort,
+    RunConfig,
+    normalize_gate,
+)
 from gooddata_eval.core.connection import ConnectionError_, resolve_connection
 from gooddata_eval.core.dataset.local import load_local_dataset
+from gooddata_eval.core.evaluators import supported_test_kinds
 from gooddata_eval.core.langfuse.sink import LangfuseSink
 from gooddata_eval.core.models import ChatResult, DatasetItem
 from gooddata_eval.core.reporting.console import render_comparison, render_console
@@ -91,7 +100,15 @@ def _build_parser() -> argparse.ArgumentParser:
             "Default: workspace's current active model."
         ),
     )
-    run.add_argument("--runs", type=int, default=2, help="Independent runs per item (pass@K). Default 2.")
+    run.add_argument("--runs", type=int, default=2, help="Independent runs per item. Default 2.")
+    run.add_argument(
+        "--gate",
+        choices=get_args(EvalGate),
+        default=DEFAULT_GATE,
+        help="Which verdict decides an item: 'any' = pass@K (a run passing is enough, the "
+        "default and historic behaviour), 'power' = pass^K (every run must pass, so the verdict "
+        "measures stability). Identical at --runs 1. Agentic kinds only.",
+    )
     run.add_argument(
         "--concurrency",
         type=int,
@@ -180,16 +197,46 @@ def _apply_timer_flag(enabled: bool) -> None:
         os.environ[TIMERS_ENV_VAR] = "1"
 
 
+def _reject_power_gate_on_ungated_items(config: RunConfig, items: list) -> None:
+    """Refuse a pass^K request the run cannot honour for every item.
+
+    Two kinds of item are never gated: everything on the non-agentic path, because
+    `run_items` has no gate and always decides on pass@K, and agentic_conversation, which
+    drives its fixture once whatever --runs says and so has no K to gate over. Running a
+    mixed dataset anyway would decide part of it under each rule and label the whole report
+    `power`. test_kind is resolved per item, so a dataset does not have to be homogeneous.
+
+    Kinds no evaluator supports are not counted: those items are skipped rather than
+    decided, so refusing on them would make --gate power fail where --gate any runs.
+    """
+    if normalize_gate(config.gate) != "power":
+        return
+    supported = supported_test_kinds()
+    ungated = [
+        i
+        for i in items
+        if i.test_kind in UNGATED_AGENTIC_TEST_KINDS
+        or (i.test_kind not in AGENTIC_TEST_KINDS and i.test_kind in supported)
+    ]
+    if not ungated:
+        return
+    kinds = sorted({i.test_kind for i in ungated})
+    raise ValueError(
+        f"--gate power applies to kinds that repeat K runs, but this dataset has {len(ungated)} "
+        f"item(s) of kind {kinds}, which are always decided on pass@K. Run them separately, or "
+        f"use --gate any."
+    )
+
+
 def _warn_if_local_dataset_cannot_link(config: RunConfig, agentic_items: list) -> None:
-    """Say up front that dataset-run assembly will fail, rather than after the run.
+    """Say up front that experiment assembly will fail, rather than after the run.
 
     --langfuse is refused outright with a local dataset because local item ids cannot be
     linked. But every evaluate_agentic_* falls back to try_make_langfuse_client() when the
-    caller passes none, so with LANGFUSE_* exported the linking runs anyway and each
-    conversation earns a 404 from dataset-run-items -- arriving in a block at the very end
-    of the run, long after the flag that would have prevented it could be changed. The
-    fallback is deliberate (direct library and tavern callers rely on it), so this warns
-    instead of disabling it.
+    caller passes none, so with LANGFUSE_* exported the linking runs anyway and every
+    dataset-item lookup 404s -- arriving in a block at the very end of the run, long after
+    the flag that would have prevented it could be changed. The fallback is deliberate
+    (direct library and tavern callers rely on it), so this warns instead of disabling it.
     """
     from gooddata_eval.core.agentic._langfuse import SKIP_ENV_VAR, langfuse_credentials_present  # noqa: PLC0415
     from gooddata_eval.core.config import env_flag  # noqa: PLC0415
@@ -201,7 +248,7 @@ def _warn_if_local_dataset_cannot_link(config: RunConfig, agentic_items: list) -
     print(
         f"warning: --dataset is a local folder, so its item ids are not Langfuse dataset item ids. "
         f"Traces will be found and scored, but the per-run grouping that makes models comparable "
-        f"cannot be created and each conversation will report a 404 from dataset-run-items. "
+        f"cannot be created and each conversation will report that its item does not exist in Langfuse. "
         f"Use --langfuse-dataset for comparable runs, or set {SKIP_ENV_VAR}=1 to skip trace linking.",
         file=sys.stderr,
     )
@@ -253,7 +300,7 @@ def _make_progress_callbacks(console: Console):
             tag = "[yellow]SKIP[/yellow]"
         elif report.error:
             tag = "[red]ERR [/red]"
-        elif report.pass_at_k:
+        elif report.passed:
             tag = "[green]PASS[/green]"
         else:
             tag = "[red]FAIL[/red]"
@@ -343,6 +390,7 @@ def _run(config: RunConfig) -> int:
     items = _load_dataset(config)
     agentic_items = [i for i in items if i.test_kind in AGENTIC_TEST_KINDS]
     non_agentic_items = [i for i in items if i.test_kind not in AGENTIC_TEST_KINDS]
+    _reject_power_gate_on_ungated_items(config, items)
     _warn_if_local_dataset_cannot_link(config, agentic_items)
     models = config.models or []
     run_ts = datetime.now(timezone.utc).strftime("%Y-%m-%d-%H-%M")
@@ -417,6 +465,7 @@ def _run(config: RunConfig) -> int:
                     token=config.token,
                     workspace_id=config.workspace_id,
                     k=config.runs,
+                    gate=config.gate,
                     model_version=resolved.model_id,
                     reasoning_effort=config.reasoning_effort,
                     use_langfuse=config.log_to_langfuse,
@@ -466,6 +515,7 @@ def _run(config: RunConfig) -> int:
                 provider_name=resolved.provider_name or resolved.provider_id,
                 provider_type=resolved.provider_type,
                 workspace_id=config.workspace_id,
+                gate=config.gate,
             )
             if agentic_report is not None:
                 report.items.extend(agentic_report.items)
@@ -530,6 +580,7 @@ def main(argv: list[str] | None = None) -> int:
             kind=args.kind,
             preserve_failed=args.preserve_failed,
             reasoning_effort=args.reasoning_effort,
+            gate=normalize_gate(args.gate),
             agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"),
         )
         return _run(config)

gooddata_eval/core/agentic/_gate.py

@@ -0,0 +1,70 @@
+# (C) 2026 GoodData Corporation
+"""Which of pass@K / pass^K decides an item, and how both reach Langfuse."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from gooddata_eval.core.config import DEFAULT_GATE, EvalGate, normalize_gate
+
+__all__ = [
+    "DEFAULT_GATE",
+    "EvalGate",
+    "gate_failure_note",
+    "gate_label",
+    "gate_passed",
+    "log_gate_scores",
+    "normalize_gate",
+    "stamp_gate_metadata",
+]
+
+
+def gate_passed(gate: str | None, *, pass_at_k: bool, pass_power_k: bool) -> bool:
+    """Whether the item passes under ``gate``."""
+    return pass_power_k if normalize_gate(gate) == "power" else pass_at_k
+
+
+def gate_label(gate: str | None, k: int) -> str:
+    """Short name for the gate, e.g. ``pass^3`` or ``pass@2``."""
+    return f"pass^{k}" if normalize_gate(gate) == "power" else f"pass@{k}"
+
+
+def gate_failure_note(gate: str | None, runs_passed: int, runs_total: int, runs_ungraded: int = 0) -> str:
+    """Gate and how many runs met it, for the assertion message.
+
+    Needed because the message body describes the BEST run, which under pass^K can be a run
+    that passed — so the reported detail on its own looks like a pass.
+
+    An ungraded run counts in runs_total but can never count in runs_passed, so the remainder
+    is not evidence of instability — saying so would blame the agent for a judge outage.
+    """
+    label = gate_label(gate, runs_total)
+    note = f"Gate {label} failed: {runs_passed}/{runs_total} runs passed"
+    if runs_ungraded:
+        return f"{note}, {runs_ungraded} ungraded."
+    if normalize_gate(gate) == "power" and runs_passed:
+        return f"{note} — unstable, not a clean failure."
+    return f"{note}."
+
+
+def log_gate_scores(ctx: Any, trace_id: Any, *, gate: str | None, pass_at_k: bool, pass_power_k: bool) -> None:
+    """Log both candidate verdicts and the one that decided.
+
+    The names carry no K on purpose: `pass_at_2` becomes `pass_at_3` the moment K changes,
+    splitting every Langfuse view built on the old name.
+    """
+    ctx.score(trace_id, name="pass_at_k", value=pass_at_k, data_type="BOOLEAN")
+    ctx.score(trace_id, name="pass_power_k", value=pass_power_k, data_type="BOOLEAN")
+    ctx.score(
+        trace_id,
+        name="gate_passed",
+        value=gate_passed(gate, pass_at_k=pass_at_k, pass_power_k=pass_power_k),
+        data_type="BOOLEAN",
+    )
+
+
+def stamp_gate_metadata(metadata: dict, *, k: int, gate: str | None) -> dict:
+    """Record K and the gate on the dataset-run metadata (mutates and returns it)."""
+    metadata["eval_k"] = k
+    metadata["eval_gate"] = normalize_gate(gate)
+    return metadata