benchflow 0.6.7.dev1833__tar.gz → 0.6.7.dev1835__tar.gz

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1835}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1833
+Version: 0.6.7.dev1835
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1835}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1833"
+version = "0.6.7.dev1835"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1835}/src/benchflow/_utils/scoring.py

@@ -1,5 +1,6 @@
 """Pure scoring and classification helpers — no external dependencies."""
 
+import math
 from collections.abc import Iterable, Mapping
 from typing import Any, Literal
 
@@ -343,6 +344,26 @@ def count_result_outcomes(results: Iterable[Mapping[str, Any]]) -> dict[str, int
     return count_audit_outcomes(results)
 
 
+def mean_scored_reward(results: Iterable[Mapping[str, Any]]) -> float | None:
+    """Mean reward over scored rollouts, or None when nothing scored.
+
+    A rollout is scored when ``rewards.reward`` is a finite numeric value.
+    Bools are excluded — ``classify_result`` treats a persisted ``true`` as a
+    pass (``True == 1``), but a bool is not a reward magnitude — and so are
+    non-finite values, which the resume path can feed in unvalidated (Python's
+    ``json`` round-trips NaN). Errored rollouts (reward None) are excluded,
+    not zeroed: a mean diluted by infra errors would misread as capability.
+    """
+    scored = [
+        rw
+        for r in results
+        if isinstance(rw := extract_reward(r), (int, float))
+        and not isinstance(rw, bool)
+        and math.isfinite(rw)
+    ]
+    return sum(scored) / len(scored) if scored else None
+
+
 def pass_rate(*, passed: int, total: int) -> float:
     """Pass rate over all tasks."""
     return passed / total if total > 0 else 0.0

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1835}/src/benchflow/cli/_shared.py

@@ -189,9 +189,14 @@ def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -
         err_part = f", [red]{detail}[/red]"
     else:
         err_part = ", errors=0"
+    # Mean reward alongside the binarized counts: pass/fail thresholds at
+    # reward==1, so "0/1" alone can't distinguish 0.3 partial credit from a
+    # flat 0. getattr(): sharded aggregation and older callers don't carry it.
+    mean_reward = getattr(result, "mean_reward", None)
+    mean_part = f", mean reward {mean_reward:.2f}" if mean_reward is not None else ""
     console.print(
         f"\n[{style}]{mark} Score: {result.passed}/{result.total} "
-        f"({result.score:.1%})[/{style}]{err_part}"
+        f"({result.score:.1%})[/{style}]{mean_part}{err_part}"
     )
     # One dim reason line per FAILED task, so "0/1" doesn't force a dig into
     # summary.json to learn why. getattr(): sharded aggregation and older

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1835}/src/benchflow/evaluation.py

@@ -64,6 +64,7 @@ from benchflow._utils.scoring import (
     classify_verifier_error,
     count_audit_outcomes,
     count_score_outcomes,
+    mean_scored_reward,
     pass_rate,
     pass_rate_excl_errors,
 )
@@ -609,6 +610,10 @@ class EvaluationResult:
     memory_score: float | None = None
     memory_scores: dict[str, float] = field(default_factory=dict)
     task_failures: list[TaskFailure] = field(default_factory=list)
+    # Mean of rewards over scored rollouts (None when nothing scored). The
+    # pass/fail counts binarize at reward==1, which erases partial credit —
+    # a 0.3 rubric score and a flat 0 both print as FAIL without this.
+    mean_reward: float | None = None
 
     @property
     def score(self) -> float:
@@ -1391,7 +1396,16 @@ class Evaluation:
         status = "PASS" if reward == 1 else ("FAIL" if reward is not None else "ERR")
         err_msg = result.error or result.verifier_error
         err = f" ({truncate_end(err_msg, 50)})" if err_msg else ""
-        logger.info(f"[{status}] {td.name} (tools={result.n_tool_calls}){err}")
+        # Show the fractional reward on scored lines: pass/fail binarizes at
+        # reward==1, so without it a 0.3 rubric score reads as a flat 0.
+        reward_part = (
+            f"reward={reward:.2f}, "
+            if isinstance(reward, (int, float)) and not isinstance(reward, bool)
+            else ""
+        )
+        logger.info(
+            f"[{status}] {td.name} ({reward_part}tools={result.n_tool_calls}){err}"
+        )
         self._fire_progress(self._on_result, td.name, result)
 
     async def _run_parallel_independent(
@@ -1828,6 +1842,7 @@ class Evaluation:
             memory_score=memory["avg_score"],
             memory_scores=memory_scores,
             task_failures=task_failures,
+            mean_reward=mean_scored_reward(all_results.values()),
         )
 
         assert (
@@ -1885,6 +1900,7 @@ class Evaluation:
             "score_excl_errors_ratio": pass_rate_excl_errors(
                 passed=audit_counts["passed"], failed=audit_counts["failed"]
             ),
+            "mean_reward": job_result.mean_reward,
             "elapsed_sec": elapsed,
             "memory_score": job_result.memory_score,
             "memory_score_coverage": (
@@ -1984,9 +2000,14 @@ class Evaluation:
                     "This likely indicates a systemic verifier bug, not agent failure."
                 )
 
+        mean_part = (
+            f"mean_reward={job_result.mean_reward:.2f}, "
+            if job_result.mean_reward is not None
+            else ""
+        )
         logger.info(
             f"Job complete: {job_result.passed}/{job_result.total} "
-            f"({job_result.score:.1%}), errors={job_result.errored}, "
+            f"({job_result.score:.1%}), {mean_part}errors={job_result.errored}, "
             f"idle_timeouts={error_category_counts.get(IDLE_TIMEOUT, 0)}, "
             f"time={elapsed / 60:.1f}min"
         )

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1835}/tests/test_cli_live_progress.py

@@ -354,6 +354,42 @@ def _failed_result(task_failures):
     )
 
 
+def test_report_eval_result_shows_mean_reward():
+    # Partial credit must be visible next to the binarized counts: pass/fail
+    # thresholds at reward==1, so "0/1 (0.0%)" alone can't distinguish a 0.3
+    # rubric score from a flat 0.
+    out = _reported(
+        SimpleNamespace(
+            passed=0,
+            total=1,
+            errored=0,
+            verifier_errored=0,
+            score=0.0,
+            job_name="j",
+            mean_reward=0.3,
+        )
+    )
+    assert "Score: 0/1" in out
+    assert "mean reward 0.30" in out
+
+
+def test_report_eval_result_omits_mean_reward_when_unavailable():
+    # Sharded aggregation and older callers don't carry mean_reward — the line
+    # must render without it rather than showing a misleading 0.00.
+    out = _reported(
+        SimpleNamespace(
+            passed=0,
+            total=1,
+            errored=1,
+            verifier_errored=0,
+            score=0.0,
+            job_name="j",
+        )
+    )
+    assert "Score: 0/1" in out
+    assert "mean reward" not in out
+
+
 def test_report_eval_result_prints_failure_reason_lines():
     # Dogfood follow-up: "✗ Score: 0/1" alone forces a dig into summary.json.
     # Each FAILED task gets one dim reason line — verifier_error first, else a

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1835}/tests/test_job.py

@@ -656,6 +656,57 @@ class TestJobRunOrchestration:
             )
         ]
 
+    @pytest.mark.asyncio
+    async def test_run_computes_mean_reward_and_logs_fractional_rewards(
+        self, tmp_path, caplog
+    ):
+        """Partial credit must survive the binarized pass/fail view: run()
+        computes mean_reward over scored rollouts (errors excluded, not
+        zeroed), the per-task lines carry reward=, error lines don't, and the
+        job-complete line carries mean_reward=.
+        """
+        job = self._make_job(tmp_path, n_tasks=3, concurrency=1)
+        outcomes = {
+            "task-0": RunResult(task_name="task-0", rewards={"reward": 1.0}),
+            "task-1": RunResult(task_name="task-1", rewards={"reward": 0.3}),
+            "task-2": RunResult(task_name="task-2", error="agent crashed"),
+        }
+
+        async def fake_run(*, task_path, **kwargs):
+            return outcomes[task_path.name]
+
+        job._sdk = AsyncMock()
+        job._sdk.run = AsyncMock(side_effect=fake_run)
+
+        with caplog.at_level(logging.INFO, logger="benchflow.evaluation"):
+            result = await job.run()
+
+        assert result.mean_reward == pytest.approx(0.65)
+        assert "[PASS] task-0 (reward=1.00, tools=0)" in caplog.text
+        assert "[FAIL] task-1 (reward=0.30, tools=0)" in caplog.text
+        assert "[ERR] task-2 (tools=0)" in caplog.text
+        assert "mean_reward=0.65" in caplog.text
+        # The machine-readable surface must agree with the console.
+        summary = json.loads((job._jobs_dir / "summary.json").read_text())
+        assert summary["mean_reward"] == pytest.approx(0.65)
+
+    @pytest.mark.asyncio
+    async def test_run_mean_reward_none_when_nothing_scored(self, tmp_path, caplog):
+        """An all-error job has no scored rollouts: mean_reward must be None
+        and the job-complete line must omit the field (a fabricated 0.00 would
+        read as a real capability result)."""
+        job = self._make_job(tmp_path, n_tasks=1, concurrency=1)
+        job._sdk = AsyncMock()
+        job._sdk.run = AsyncMock(
+            return_value=RunResult(task_name="task-0", error="agent crashed")
+        )
+
+        with caplog.at_level(logging.INFO, logger="benchflow.evaluation"):
+            result = await job.run()
+
+        assert result.mean_reward is None
+        assert "mean_reward=" not in caplog.text
+
     @pytest.mark.asyncio
     async def test_error_and_verifier_error_does_not_crash_invariant(self, tmp_path):
         """A result with BOTH error and verifier_error (rewards=None) must not

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1835}/tests/test_scoring.py

@@ -7,6 +7,7 @@ from benchflow._utils.scoring import (
     classify_result_outcome,
     count_result_outcomes,
     extract_reward,
+    mean_scored_reward,
     pass_rate,
     pass_rate_excl_errors,
 )
@@ -280,3 +281,37 @@ class TestPassRateExclErrors:
     )
     def test_pass_rate_excl_errors(self, passed, failed, expected):
         assert pass_rate_excl_errors(passed=passed, failed=failed) == expected
+
+
+class TestMeanScoredReward:
+    """mean_scored_reward(results) -> float | None"""
+
+    def test_mean_over_scored_only(self):
+        results = [
+            {"rewards": {"reward": 1.0}},
+            {"rewards": {"reward": 0.3}},
+            {"error": "agent crashed"},
+        ]
+        assert mean_scored_reward(results) == pytest.approx(0.65)
+
+    def test_none_when_nothing_scored(self):
+        assert mean_scored_reward([{"error": "boom"}, {"rewards": None}]) is None
+        assert mean_scored_reward([]) is None
+
+    def test_bool_rewards_excluded(self):
+        # classify_result treats a persisted `true` as a pass (True == 1), but
+        # a bool is not a reward magnitude — it must not enter the mean.
+        results = [{"rewards": {"reward": True}}, {"rewards": {"reward": 0.5}}]
+        assert mean_scored_reward(results) == pytest.approx(0.5)
+
+    def test_non_finite_rewards_excluded(self):
+        # Python's json round-trips NaN; unvalidated resume data must not
+        # poison the mean.
+        results = [{"rewards": {"reward": float("nan")}}, {"rewards": {"reward": 1.0}}]
+        assert mean_scored_reward(results) == pytest.approx(1.0)
+
+    def test_malformed_rewards_shape_tolerated(self):
+        # The resume path feeds raw json.loads payloads with no shape
+        # validation — a non-dict rewards must not crash the aggregation.
+        results = [{"rewards": ["not", "a", "dict"]}, {"rewards": {"reward": 0.4}}]
+        assert mean_scored_reward(results) == pytest.approx(0.4)