benchflow 0.6.7.dev1831__tar.gz → 0.6.7.dev1835__tar.gz

{benchflow-0.6.7.dev1831 → benchflow-0.6.7.dev1835}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1831
+Version: 0.6.7.dev1835
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1831 → benchflow-0.6.7.dev1835}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1831"
+version = "0.6.7.dev1835"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1831 → benchflow-0.6.7.dev1835}/src/benchflow/_utils/evaluation_results.py

@@ -73,6 +73,7 @@ def rollout_result_payload(
     )
     return {
         "task_name": result.task_name,
+        "rollout_name": result.rollout_name,
         "rewards": result.rewards,
         "error": result.error,
         "error_category": result.error_category,

{benchflow-0.6.7.dev1831 → benchflow-0.6.7.dev1835}/src/benchflow/_utils/scoring.py

@@ -1,5 +1,6 @@
 """Pure scoring and classification helpers — no external dependencies."""
 
+import math
 from collections.abc import Iterable, Mapping
 from typing import Any, Literal
 
@@ -343,6 +344,26 @@ def count_result_outcomes(results: Iterable[Mapping[str, Any]]) -> dict[str, int
     return count_audit_outcomes(results)
 
 
+def mean_scored_reward(results: Iterable[Mapping[str, Any]]) -> float | None:
+    """Mean reward over scored rollouts, or None when nothing scored.
+
+    A rollout is scored when ``rewards.reward`` is a finite numeric value.
+    Bools are excluded — ``classify_result`` treats a persisted ``true`` as a
+    pass (``True == 1``), but a bool is not a reward magnitude — and so are
+    non-finite values, which the resume path can feed in unvalidated (Python's
+    ``json`` round-trips NaN). Errored rollouts (reward None) are excluded,
+    not zeroed: a mean diluted by infra errors would misread as capability.
+    """
+    scored = [
+        rw
+        for r in results
+        if isinstance(rw := extract_reward(r), (int, float))
+        and not isinstance(rw, bool)
+        and math.isfinite(rw)
+    ]
+    return sum(scored) / len(scored) if scored else None
+
+
 def pass_rate(*, passed: int, total: int) -> float:
     """Pass rate over all tasks."""
     return passed / total if total > 0 else 0.0

benchflow-0.6.7.dev1835/src/benchflow/cli/_failure_evidence.py

@@ -0,0 +1,122 @@
+"""Verifier-artifact mining for the eval report's failure lines.
+
+The one file-touching corner of the CLI's final report block: when a failed
+task's reason would otherwise be the bare ``reward X`` fallback, mine the
+rollout's verifier artifacts for a one-line explanation. Lives in its own
+module so ``cli/_shared.py`` stays the side-effect-free display helpers it
+advertises.
+
+Contract (the engine stays file-free — these reads are CLI-side, report-time
+only):
+
+- The rollout dir is resolved exactly, from the ``rollout_name`` the engine
+  records on every result — never guessed from globs.
+- Evidence sources are tried in order (CTRF report, then test-stdout tail);
+  the first that yields a line wins.
+- Every read is bounded (``_ARTIFACT_READ_BYTES`` per file) and nothing here
+  raises — any surprise degrades to ``None``, i.e. the bare reward reason.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from pathlib import Path
+
+# Never read more than this many bytes per file, and evidence is only mined
+# for the <= _MAX_FAILURE_LINES tasks the report block displays.
+_ARTIFACT_READ_BYTES = 64 * 1024
+# Pytest final summary, decoration stripped: "1 failed, 2 passed in 40.86s".
+_PYTEST_SUMMARY_RE = re.compile(r"\d+ failed\b")
+
+
+def _ctrf_failure_line(ctrf_path: Path) -> str | None:
+    """``<test> failed[: <assertion>]`` from the first failed CTRF test.
+
+    The verifier's pytest run writes a CTRF report (``pytest --ctrf``, see the
+    standard path in ``task_authoring/structural_checks.py``). Per test the
+    useful failure evidence is the ``E …`` assertion line inside ``trace``;
+    ``message`` is only a generic phase description, so it is the fallback.
+    """
+    if ctrf_path.stat().st_size > _ARTIFACT_READ_BYTES:
+        # JSON only parses whole — an oversized report is skipped (not
+        # truncated) and the next evidence source gets its turn.
+        return None
+    data = json.loads(ctrf_path.read_text(encoding="utf-8", errors="replace"))
+    tests = (data.get("results") or {}).get("tests") or []
+    for test in tests:
+        if not isinstance(test, dict) or test.get("status") != "failed":
+            continue
+        # Full pytest node ids ("tests/test_x.py::test_build") waste the
+        # 100-char line budget; keep the test function name.
+        name = str(test.get("name") or "test").rsplit("::", 1)[-1]
+        trace = test.get("trace")
+        if isinstance(trace, str):
+            for line in trace.splitlines():
+                stripped = line.strip()
+                if stripped.startswith("E "):
+                    assertion = stripped[2:].strip()
+                    # "<test> failed:" already says it's an assertion — drop
+                    # the prefix to reclaim line budget for the message.
+                    assertion = assertion.removeprefix("AssertionError: ")
+                    return f"{name} failed: {assertion}"
+        message = test.get("message")
+        if isinstance(message, str) and message.strip():
+            return f"{name} failed: {' '.join(message.split())}"
+        return f"{name} failed"
+    return None
+
+
+def _stdout_tail_failure_line(stdout_path: Path) -> str | None:
+    """Last pytest summary ("N failed, M passed …") — else last ``FAILED …``
+    line — from a bounded tail of the verifier's test-stdout capture."""
+    with stdout_path.open("rb") as fh:
+        fh.seek(0, 2)  # SEEK_END
+        fh.seek(max(0, fh.tell() - _ARTIFACT_READ_BYTES))
+        tail = fh.read(_ARTIFACT_READ_BYTES).decode("utf-8", errors="replace")
+    last_failed: str | None = None
+    for line in reversed(tail.splitlines()):
+        bare = line.strip().strip("=").strip()
+        if _PYTEST_SUMMARY_RE.match(bare):
+            return bare
+        if last_failed is None and line.strip().startswith("FAILED "):
+            last_failed = line.strip()
+    return last_failed
+
+
+def artifact_failure_evidence(
+    job_dir: Path, rollout_name: str
+) -> tuple[str, Path] | None:
+    """One-liner mined from the rollout's verifier artifacts, or ``None``.
+
+    Returns the one-liner plus the verifier dir it came from (for the report
+    block's single ``(details: …)`` pointer line). Never raises.
+    """
+    # Local import: RolloutPaths pulls in the task package, which the CLI
+    # shouldn't pay for until a failure actually needs artifact evidence.
+    from benchflow.task.paths import RolloutPaths
+
+    rollout_paths = RolloutPaths(rollout_dir=job_dir / rollout_name)
+    verifier_dir = rollout_paths.verifier_dir
+    attempts: list[tuple[Path, Callable[[Path], str | None]]] = [
+        # ctrf.json has no RolloutPaths property — the verifier recovers it by
+        # literal name (verifier_core.py, `_recover_main_verifier_outputs`).
+        (verifier_dir / "ctrf.json", _ctrf_failure_line),
+        (rollout_paths.test_stdout_path, _stdout_tail_failure_line),
+    ]
+    for path, extract in attempts:
+        try:
+            if not path.is_file():
+                continue
+            detail = extract(path)
+        except Exception:
+            # Unreadable file, bad JSON, permissions … try the next source;
+            # the report must never crash over evidence mining.
+            continue
+        if detail is not None:
+            return detail, verifier_dir
+    return None

{benchflow-0.6.7.dev1831 → benchflow-0.6.7.dev1835}/src/benchflow/cli/_shared.py

@@ -3,7 +3,9 @@
 These are the cross-cutting, side-effect-free helpers that several CLI command
 groups (``cli/main.py`` and the ``cli/<group>.py`` modules) need in common: the
 shared Rich :data:`console`, the evaluation-result summary/exit helpers, and the
-agent ``Requires`` rendering used by ``agents``/``agent`` listings.
+agent ``Requires`` rendering used by ``agents``/``agent`` listings. The one
+file-reading step of the eval report (mining verifier artifacts for failure
+evidence) is delegated to :mod:`._failure_evidence` to keep that claim true.
 
 Keeping them here lets each command group import one stable surface instead of
 re-deriving the formatting, and lets ``cli/main.py`` stay a thin app + eval
@@ -19,6 +21,7 @@ from rich.console import Console
 from rich.markup import escape
 
 from benchflow._utils.text import truncate_end
+from benchflow.cli._failure_evidence import artifact_failure_evidence
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -109,17 +112,24 @@ _FAILURE_LINE_LIMIT = 100
 _FAILURE_REASON_METRICS = 3
 
 
-def _failure_reason(failure: TaskFailure) -> str:
+def _failure_reason(
+    failure: TaskFailure, job_dir: Path | None = None
+) -> tuple[str, Path | None]:
     """One cheap line explaining why a FAILED (scored, reward != 1) task
-    failed, from evidence already on the result — no file reads.
+    failed, plus the verifier dir when artifacts supplied the evidence.
 
     Priority: the verifier's own error if set; else the reward plus a compact
     breakdown of the named metrics in the reward dict (zero/failed metrics
-    first — they explain the miss); else just the reward.
+    first — they explain the miss); else the reward plus a one-liner mined
+    from the rollout's verifier artifacts (bounded CLI-side reads resolved via
+    the ``rollout_name`` the engine records — the engine stays file-free);
+    else just the reward. The returned path is non-``None`` only for the
+    artifact tier, so the caller can print one ``(details: …)`` pointer per
+    report block.
     """
     if failure.verifier_error:
         # Collapse whitespace: verifier errors are routinely multi-line.
-        return " ".join(failure.verifier_error.split())
+        return " ".join(failure.verifier_error.split()), None
     rewards = failure.rewards or {}
     reward = rewards.get("reward")
     metrics = [
@@ -127,15 +137,22 @@ def _failure_reason(failure: TaskFailure) -> str:
         for name, value in rewards.items()
         if name != "reward" and isinstance(value, (bool, int, float))
     ]
-    if not metrics:
-        return f"reward {reward}"
-    # Zero/failed metrics first (stable within each group), capped so one
-    # metric-happy verifier can't flood the line.
-    metrics.sort(key=lambda kv: kv[1] != 0)
-    shown = ", ".join(
-        f"{name} {value}" for name, value in metrics[:_FAILURE_REASON_METRICS]
-    )
-    return f"reward {reward} — {shown}"
+    if metrics:
+        # Zero/failed metrics first (stable within each group), capped so one
+        # metric-happy verifier can't flood the line.
+        metrics.sort(key=lambda kv: kv[1] != 0)
+        shown = ", ".join(
+            f"{name} {value}" for name, value in metrics[:_FAILURE_REASON_METRICS]
+        )
+        return f"reward {reward} — {shown}", None
+    # No rollout_name (pre-#957-follow-up result.json, sharded aggregation):
+    # nothing to resolve — the bare reward is as honest as it gets.
+    if job_dir is not None and failure.rollout_name:
+        evidence = artifact_failure_evidence(job_dir, failure.rollout_name)
+        if evidence is not None:
+            detail, verifier_dir = evidence
+            return f"reward {reward} — {detail}", verifier_dir
+    return f"reward {reward}", None
 
 
 def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -> None:
@@ -145,9 +162,13 @@ def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -
     now the line is green only on a full clean pass, red on a shutout, amber
     otherwise, and ``errors=N`` is red when non-zero. Each FAILED task gets one
     dim ``✗ task: reason`` line (capped at ``_MAX_FAILURE_LINES``) so the "why"
-    doesn't require opening summary.json. When ``job_dir`` is given, the
-    result/summary paths are printed so testers know where to look (the guide
-    repeatedly says "read summary.json" but the CLI never said where).
+    doesn't require opening summary.json; when ``job_dir`` is given, a reason
+    that would otherwise be a bare ``reward X`` is upgraded from the rollout's
+    verifier artifacts (bounded reads, displayed failures only), with a single
+    ``(details: …/verifier)`` pointer line for the block. When ``job_dir`` is
+    given, the result/summary paths are printed so testers know where to look
+    (the guide repeatedly says "read summary.json" but the CLI never said
+    where).
     """
     errors = int(getattr(result, "errored", 0) or 0)
     verifier_errors = int(getattr(result, "verifier_errored", 0) or 0)
@@ -168,23 +189,36 @@ def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -
         err_part = f", [red]{detail}[/red]"
     else:
         err_part = ", errors=0"
+    # Mean reward alongside the binarized counts: pass/fail thresholds at
+    # reward==1, so "0/1" alone can't distinguish 0.3 partial credit from a
+    # flat 0. getattr(): sharded aggregation and older callers don't carry it.
+    mean_reward = getattr(result, "mean_reward", None)
+    mean_part = f", mean reward {mean_reward:.2f}" if mean_reward is not None else ""
     console.print(
         f"\n[{style}]{mark} Score: {result.passed}/{result.total} "
-        f"({result.score:.1%})[/{style}]{err_part}"
+        f"({result.score:.1%})[/{style}]{mean_part}{err_part}"
     )
     # One dim reason line per FAILED task, so "0/1" doesn't force a dig into
     # summary.json to learn why. getattr(): sharded aggregation and older
     # SimpleNamespace-style callers don't carry task_failures.
     failures = getattr(result, "task_failures", None) or []
+    artifact_pointer: Path | None = None
     for failure in failures[:_MAX_FAILURE_LINES]:
+        reason, verifier_dir = _failure_reason(failure, job_dir)
+        if artifact_pointer is None and verifier_dir is not None:
+            artifact_pointer = verifier_dir
         line = truncate_end(
-            f"  ✗ {failure.task_name}: {_failure_reason(failure)}",
+            f"  ✗ {failure.task_name}: {reason}",
             _FAILURE_LINE_LIMIT,
         )
         console.print(f"[dim]{escape(line)}[/dim]")
     extra = len(failures) - _MAX_FAILURE_LINES
     if extra > 0:
         console.print(f"[dim]  … and {extra} more[/dim]")
+    # One pointer per block (not per line): the first artifact-backed reason's
+    # verifier dir, so "where do I look next" needs no summary.json dig either.
+    if artifact_pointer is not None:
+        console.print(f"[dim]  (details: {escape(str(artifact_pointer))})[/dim]")
     if job_dir is not None:
         console.print(f"[dim]Artifacts:[/dim] {escape(str(job_dir))}")
         console.print(f"[dim]Summary:  [/dim] {escape(str(job_dir))}/summary.json")

{benchflow-0.6.7.dev1831 → benchflow-0.6.7.dev1835}/src/benchflow/evaluation.py

@@ -64,6 +64,7 @@ from benchflow._utils.scoring import (
     classify_verifier_error,
     count_audit_outcomes,
     count_score_outcomes,
+    mean_scored_reward,
     pass_rate,
     pass_rate_excl_errors,
 )
@@ -588,6 +589,10 @@ class TaskFailure:
     task_name: str
     rewards: dict[str, Any] | None
     verifier_error: str | None
+    # The task's rollout dir name under the job dir (``<task>__<uuid8>``), so
+    # the CLI can find the rollout's verifier artifacts without guessing.
+    # None on results persisted before the key existed.
+    rollout_name: str | None = None
 
 
 @dataclass
@@ -605,6 +610,10 @@ class EvaluationResult:
     memory_score: float | None = None
     memory_scores: dict[str, float] = field(default_factory=dict)
     task_failures: list[TaskFailure] = field(default_factory=list)
+    # Mean of rewards over scored rollouts (None when nothing scored). The
+    # pass/fail counts binarize at reward==1, which erases partial credit —
+    # a 0.3 rubric score and a flat 0 both print as FAIL without this.
+    mean_reward: float | None = None
 
     @property
     def score(self) -> float:
@@ -1387,7 +1396,16 @@ class Evaluation:
         status = "PASS" if reward == 1 else ("FAIL" if reward is not None else "ERR")
         err_msg = result.error or result.verifier_error
         err = f" ({truncate_end(err_msg, 50)})" if err_msg else ""
-        logger.info(f"[{status}] {td.name} (tools={result.n_tool_calls}){err}")
+        # Show the fractional reward on scored lines: pass/fail binarizes at
+        # reward==1, so without it a 0.3 rubric score reads as a flat 0.
+        reward_part = (
+            f"reward={reward:.2f}, "
+            if isinstance(reward, (int, float)) and not isinstance(reward, bool)
+            else ""
+        )
+        logger.info(
+            f"[{status}] {td.name} ({reward_part}tools={result.n_tool_calls}){err}"
+        )
         self._fire_progress(self._on_result, td.name, result)
 
     async def _run_parallel_independent(
@@ -1801,6 +1819,9 @@ class Evaluation:
                 task_name=name,
                 rewards=r.get("rewards"),
                 verifier_error=r.get("verifier_error"),
+                # `or None`: RolloutResult defaults rollout_name to "" — don't
+                # let that masquerade as a resolvable rollout dir.
+                rollout_name=r.get("rollout_name") or None,
             )
             for name, r in sorted(all_results.items())
             if classify_score_outcome(r) == "failed"
@@ -1821,6 +1842,7 @@ class Evaluation:
             memory_score=memory["avg_score"],
             memory_scores=memory_scores,
             task_failures=task_failures,
+            mean_reward=mean_scored_reward(all_results.values()),
         )
 
         assert (
@@ -1878,6 +1900,7 @@ class Evaluation:
             "score_excl_errors_ratio": pass_rate_excl_errors(
                 passed=audit_counts["passed"], failed=audit_counts["failed"]
             ),
+            "mean_reward": job_result.mean_reward,
             "elapsed_sec": elapsed,
             "memory_score": job_result.memory_score,
             "memory_score_coverage": (
@@ -1977,9 +2000,14 @@ class Evaluation:
                     "This likely indicates a systemic verifier bug, not agent failure."
                 )
 
+        mean_part = (
+            f"mean_reward={job_result.mean_reward:.2f}, "
+            if job_result.mean_reward is not None
+            else ""
+        )
         logger.info(
             f"Job complete: {job_result.passed}/{job_result.total} "
-            f"({job_result.score:.1%}), errors={job_result.errored}, "
+            f"({job_result.score:.1%}), {mean_part}errors={job_result.errored}, "
             f"idle_timeouts={error_category_counts.get(IDLE_TIMEOUT, 0)}, "
             f"time={elapsed / 60:.1f}min"
         )