benchflow 0.6.7.dev1833__tar.gz → 0.6.7.dev1839__tar.gz

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1839}/CHANGELOG.md

@@ -3,6 +3,14 @@
 ## [Unreleased]
 
 ### Added
+- **Built-in environment registry.** The committed env-axis pins
+  (`env0@prod`, `env0@outage`) moved from `benchmarks/_environments/` into
+  the package (`benchflow/environment/_registry/`) and ship inside the
+  wheel, so `--environment-manifest env0@prod` resolves on a bare
+  `pip install benchflow` with no checkout and no env vars.
+  `$BENCHFLOW_ENV_REGISTRY`, when set, still wins entirely; resolution
+  stays content-addressed (sha256 logged), and unknown names now error
+  listing the available specs.
 - **Console progress heartbeat.** Single-concurrency eval runs print a
   throttled progress line (`… 6.2min, 12 tool calls (last: …)`) about every
   45 seconds while the agent works, so a long prompt is distinguishable from

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1839}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1833
+Version: 0.6.7.dev1839
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1839}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1833"
+version = "0.6.7.dev1839"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1839}/src/benchflow/_utils/env_registry.py

@@ -8,11 +8,22 @@ Lee's ``S`` axis in ``I_eval = {T, A, M, S, C, R, τ}`` — from the task, so it
 swappable per run exactly like ``--agent`` / ``--model`` / ``--sandbox`` already
 are, instead of being pinned by a relative path baked into the task file.
 
-Registry layout (filesystem; ``$BENCHFLOW_ENV_REGISTRY``)::
+Registry layout (a directory of manifest files)::
 
     <registry>/<name>@<version>.toml   # a pinned environment version
     <registry>/<name>.toml             # optional default ("latest")
 
+Which directory is the registry:
+
+1. an explicit ``registry=`` argument (tests, embedders);
+2. else ``$BENCHFLOW_ENV_REGISTRY`` — when set it wins entirely (no fallback
+   into the built-in registry for names it does not contain); an empty value
+   counts as unset;
+3. else the **built-in registry** shipped inside the ``benchflow`` wheel
+   (``benchflow/environment/_registry/``), so a bare ``pip install benchflow``
+   resolves the committed pins (``env0@prod``, ``env0@outage``) with no
+   checkout and no env vars.
+
 Resolution::
 
     env0          -> <registry>/env0.toml          (default) else newest @version
@@ -80,21 +91,49 @@ def looks_like_env_spec(value: str) -> bool:
     return bool(_SPEC_RE.match(value))
 
 
+def _builtin_registry_dir() -> Path | None:
+    """Directory of the pinned manifests shipped inside the ``benchflow`` wheel.
+
+    ``benchflow/environment/_registry/`` is package data, so a bare
+    ``pip install benchflow`` carries the committed pins. Path(__file__)-relative
+    like every other package-data lookup in the repo (compose files, default
+    rubric, demo task) — zip imports are unsupported package-wide. Returns
+    ``None`` when the packaged registry is missing (a stripped install).
+    """
+    directory = Path(__file__).resolve().parents[1] / "environment" / "_registry"
+    return directory if directory.is_dir() else None
+
+
 def _registry_dir(registry: str | os.PathLike[str] | None) -> Path:
-    raw = registry if registry is not None else os.environ.get(ENV_REGISTRY_VAR)
-    if not raw:
+    if registry is not None:
+        raw: str | os.PathLike[str] | None = registry
+        source = "the registry= argument"
+    else:
+        raw = os.environ.get(ENV_REGISTRY_VAR)
+        source = f"${ENV_REGISTRY_VAR}"
+    if not raw:  # unset or empty — empty counts as unset
+        builtin = _builtin_registry_dir()
+        if builtin is not None:
+            return builtin
         raise EnvironmentRegistryError(
-            f"no environment registry configured; set ${ENV_REGISTRY_VAR} or pass "
-            "--environment-manifest a manifest file path instead of a name@version spec"
+            "no environment registry available: the built-in registry shipped "
+            f"with benchflow is missing and ${ENV_REGISTRY_VAR} is unset; set "
+            f"${ENV_REGISTRY_VAR} or pass --environment-manifest a manifest "
+            "file path instead of a name@version spec"
         )
     directory = Path(raw).expanduser()
     if not directory.is_dir():
         raise EnvironmentRegistryError(
-            f"environment registry {directory} is not a directory"
+            f"environment registry {directory} (from {source}) is not a directory"
         )
     return directory
 
 
+def _available_specs(directory: Path) -> list[str]:
+    """Sorted stems (``name`` / ``name@version``) present in the registry."""
+    return sorted({p.stem for ext in _MANIFEST_EXTS for p in directory.glob(f"*{ext}")})
+
+
 def _find_manifest(directory: Path, stem: str) -> Path | None:
     """Return ``directory/stem.<ext>`` for the first existing manifest extension."""
     for ext in _MANIFEST_EXTS:
@@ -124,9 +163,11 @@ def resolve_environment(
     if version is not None:
         path = _find_manifest(directory, f"{name}@{version}")
         if path is None:
+            available = ", ".join(_available_specs(directory)) or "none"
             raise EnvironmentRegistryError(
                 f"environment {spec!r} not found in {directory} "
-                f"(looked for {name}@{version}{{.toml,.yaml,.yml}})"
+                f"(looked for {name}@{version}{{.toml,.yaml,.yml}}; "
+                f"available: {available})"
             )
     else:
         path = _find_manifest(directory, name)
@@ -137,8 +178,10 @@ def resolve_environment(
                 p for ext in _MANIFEST_EXTS for p in directory.glob(f"{name}@*{ext}")
             )
             if not candidates:
+                available = ", ".join(_available_specs(directory)) or "none"
                 raise EnvironmentRegistryError(
-                    f"no versions of environment {name!r} found in {directory}"
+                    f"no versions of environment {name!r} found in {directory} "
+                    f"(available: {available})"
                 )
             path = candidates[-1]
             version = path.stem.split("@", 1)[1]

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1839}/src/benchflow/_utils/scoring.py

@@ -1,5 +1,6 @@
 """Pure scoring and classification helpers — no external dependencies."""
 
+import math
 from collections.abc import Iterable, Mapping
 from typing import Any, Literal
 
@@ -343,6 +344,26 @@ def count_result_outcomes(results: Iterable[Mapping[str, Any]]) -> dict[str, int
     return count_audit_outcomes(results)
 
 
+def mean_scored_reward(results: Iterable[Mapping[str, Any]]) -> float | None:
+    """Mean reward over scored rollouts, or None when nothing scored.
+
+    A rollout is scored when ``rewards.reward`` is a finite numeric value.
+    Bools are excluded — ``classify_result`` treats a persisted ``true`` as a
+    pass (``True == 1``), but a bool is not a reward magnitude — and so are
+    non-finite values, which the resume path can feed in unvalidated (Python's
+    ``json`` round-trips NaN). Errored rollouts (reward None) are excluded,
+    not zeroed: a mean diluted by infra errors would misread as capability.
+    """
+    scored = [
+        rw
+        for r in results
+        if isinstance(rw := extract_reward(r), (int, float))
+        and not isinstance(rw, bool)
+        and math.isfinite(rw)
+    ]
+    return sum(scored) / len(scored) if scored else None
+
+
 def pass_rate(*, passed: int, total: int) -> float:
     """Pass rate over all tasks."""
     return passed / total if total > 0 else 0.0

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1839}/src/benchflow/cli/_shared.py

@@ -189,9 +189,14 @@ def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -
         err_part = f", [red]{detail}[/red]"
     else:
         err_part = ", errors=0"
+    # Mean reward alongside the binarized counts: pass/fail thresholds at
+    # reward==1, so "0/1" alone can't distinguish 0.3 partial credit from a
+    # flat 0. getattr(): sharded aggregation and older callers don't carry it.
+    mean_reward = getattr(result, "mean_reward", None)
+    mean_part = f", mean reward {mean_reward:.2f}" if mean_reward is not None else ""
     console.print(
         f"\n[{style}]{mark} Score: {result.passed}/{result.total} "
-        f"({result.score:.1%})[/{style}]{err_part}"
+        f"({result.score:.1%})[/{style}]{mean_part}{err_part}"
     )
     # One dim reason line per FAILED task, so "0/1" doesn't force a dig into
     # summary.json to learn why. getattr(): sharded aggregation and older

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1839}/src/benchflow/cli/main.py

@@ -297,10 +297,11 @@ def eval_run(
             "--environment-manifest",
             help=(
                 "Environment-plane manifest applied to every rollout: a path to "
-                "an environment.toml, OR a 'name@version' registry spec resolved "
-                "via $BENCHFLOW_ENV_REGISTRY (the S axis). The manifest-declared "
-                "stateful environment is provisioned, gated on readiness, and "
-                "torn down."
+                "an environment.toml, OR a 'name@version' registry spec (the S "
+                "axis) resolved via $BENCHFLOW_ENV_REGISTRY when set, else the "
+                "built-in registry shipped with benchflow (env0@prod, "
+                "env0@outage). The manifest-declared stateful environment is "
+                "provisioned, gated on readiness, and torn down."
             ),
         ),
     ] = None,

benchflow-0.6.7.dev1839/src/benchflow/environment/_registry/env0@outage.toml

@@ -0,0 +1,63 @@
+# env0@outage — env-0 with the gmail + slack services removed.
+#
+# A perturbation variant for the "Same state, tool outage" experiment: bind the
+# SAME task to env0@prod vs env0@outage and attribute the reward delta to the
+# environment alone (the S axis). A task that needs slack (port 9005) scores 1.0
+# under env0@prod and fails under env0@outage because the service is absent.
+#
+# Shipped inside the benchflow wheel (benchflow/environment/_registry/), so it
+# resolves on a bare pip install with no $BENCHFLOW_ENV_REGISTRY set:
+#
+#   bench eval create --tasks-dir <task> --environment-manifest env0@outage ...
+
+[environment]
+name           = "env-0"
+base_image     = "xdotli/env0-base:latest"
+owns_lifecycle = false
+isolation      = "per_task"
+
+[environment.task_selection]
+mechanism = "image"
+
+[environment.readiness]
+timeout_sec = 60
+
+# Services below are copied VERBATIM from the upstream env-0.toml pin
+# (2026-08-09 sync). The previous pin had drifted: bare CLI names
+# (auth/gmail/...) that do not exist in ghcr.io/benchflow-ai/env0:0.2.0
+# images (which ship mock-auth/mock-gmail/... console scripts), plus a
+# phantom slack service and shifted ports — so ManifestEnvironment
+# probed, found ZERO startable services, and readiness passed vacuously
+# while every rollout died at the verifier with connection-refused.
+
+[[environment.services]]
+name    = "mock-auth"
+command = "mock-auth --db /data/auth.db serve --host 0.0.0.0 --port 9000 --no-mcp"
+port    = 9000
+
+
+[[environment.services]]
+name    = "mock-gcal"
+command = "mock-gcal --db /data/gcal.db serve --host 0.0.0.0 --port 9002 --no-mcp"
+port    = 9002
+
+[[environment.services]]
+name    = "mock-gdrive"
+command = "mock-gdrive --db /data/gdrive.db serve --host 0.0.0.0 --port 9003 --no-mcp"
+port    = 9003
+
+[[environment.services]]
+name    = "mock-gdoc"
+command = "mock-gdoc --db /data/gdoc.db serve --host 0.0.0.0 --port 9004 --no-mcp"
+port    = 9004
+
+
+[[environment.services]]
+name    = "mock-discord"
+command = "mock-discord --db /data/discord.db serve --host 0.0.0.0 --port 9006 --no-mcp"
+port    = 9006
+
+[[environment.services]]
+name    = "mock-stripe"
+command = "mock-stripe --db /data/stripe.db serve --host 0.0.0.0 --port 9007 --no-mcp"
+port    = 9007

benchflow-0.6.7.dev1839/src/benchflow/environment/_registry/env0@prod.toml

@@ -0,0 +1,77 @@
+# env0@prod — pinned, content-addressed Environment-plane manifest for env-0.
+#
+# This is a built-in registry entry, shipped inside the benchflow wheel
+# (benchflow/environment/_registry/) so PR #790's env-axis pin resolves on a
+# bare pip install — no checkout, no env vars:
+#
+#   bench eval create --tasks-dir <env0 tasks> --environment-manifest env0@prod ...
+#
+# Set $BENCHFLOW_ENV_REGISTRY to a local directory of name@version.toml files
+# to override the built-in registry entirely.
+#
+# Mirrors benchflow-ai/env0 tasks/_manifests/env-0.toml (the source of truth). env-0's per-task images build
+# FROM the shared base and bake seed data; benchflow's ManifestEnvironment
+# probes each service's CLI and starts only the ones installed in the per-task
+# image, gating the agent on each service's /health.
+#
+# NOTE: xdotli/env0-base:latest is amd64-only — run env0 on Daytona (x86_64).
+
+[environment]
+name           = "env-0"
+base_image     = "xdotli/env0-base:latest"
+owns_lifecycle = false
+isolation      = "per_task"
+
+[environment.task_selection]
+mechanism = "image"
+
+[environment.readiness]
+timeout_sec = 60
+
+# Services below are copied VERBATIM from the upstream env-0.toml pin
+# (2026-08-09 sync). The previous pin had drifted: bare CLI names
+# (auth/gmail/...) that do not exist in ghcr.io/benchflow-ai/env0:0.2.0
+# images (which ship mock-auth/mock-gmail/... console scripts), plus a
+# phantom slack service and shifted ports — so ManifestEnvironment
+# probed, found ZERO startable services, and readiness passed vacuously
+# while every rollout died at the verifier with connection-refused.
+
+[[environment.services]]
+name    = "mock-auth"
+command = "mock-auth --db /data/auth.db serve --host 0.0.0.0 --port 9000 --no-mcp"
+port    = 9000
+
+[[environment.services]]
+name    = "mock-gmail"
+command = "mock-gmail --db /data/gmail.db serve --host 0.0.0.0 --port 9001 --no-mcp"
+port    = 9001
+
+[[environment.services]]
+name    = "mock-gcal"
+command = "mock-gcal --db /data/gcal.db serve --host 0.0.0.0 --port 9002 --no-mcp"
+port    = 9002
+
+[[environment.services]]
+name    = "mock-gdrive"
+command = "mock-gdrive --db /data/gdrive.db serve --host 0.0.0.0 --port 9003 --no-mcp"
+port    = 9003
+
+[[environment.services]]
+name    = "mock-gdoc"
+command = "mock-gdoc --db /data/gdoc.db serve --host 0.0.0.0 --port 9004 --no-mcp"
+port    = 9004
+
+[[environment.services]]
+name    = "mock-slack"
+command = "mock-slack --db /data/slack.db serve --host 0.0.0.0 --port 9005 --no-mcp"
+port    = 9005
+
+[[environment.services]]
+name    = "mock-discord"
+command = "mock-discord --db /data/discord.db serve --host 0.0.0.0 --port 9006 --no-mcp"
+port    = 9006
+
+[[environment.services]]
+name    = "mock-stripe"
+command = "mock-stripe --db /data/stripe.db serve --host 0.0.0.0 --port 9007 --no-mcp"
+port    = 9007

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1839}/src/benchflow/environment/manifest.py

@@ -205,7 +205,9 @@ def load_manifest(path: str | Path) -> EnvironmentManifest:
     """Load and validate an environment manifest.
 
     ``path`` is either a TOML file path (the historical behavior) or a registry
-    spec ``name@version`` resolved via ``$BENCHFLOW_ENV_REGISTRY``. The spec form
+    spec ``name@version`` resolved via ``$BENCHFLOW_ENV_REGISTRY`` when set,
+    else the built-in registry shipped inside the wheel
+    (``benchflow/environment/_registry/``). The spec form
     lets a run bind its environment (the ``S`` axis) by name at the command line —
     decoupled from the task and swappable per run, like ``--agent`` / ``--model``
     / ``--sandbox``. Resolution is content-addressed so the bound world is

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1839}/src/benchflow/evaluation.py

@@ -64,6 +64,7 @@ from benchflow._utils.scoring import (
     classify_verifier_error,
     count_audit_outcomes,
     count_score_outcomes,
+    mean_scored_reward,
     pass_rate,
     pass_rate_excl_errors,
 )
@@ -609,6 +610,10 @@ class EvaluationResult:
     memory_score: float | None = None
     memory_scores: dict[str, float] = field(default_factory=dict)
     task_failures: list[TaskFailure] = field(default_factory=list)
+    # Mean of rewards over scored rollouts (None when nothing scored). The
+    # pass/fail counts binarize at reward==1, which erases partial credit —
+    # a 0.3 rubric score and a flat 0 both print as FAIL without this.
+    mean_reward: float | None = None
 
     @property
     def score(self) -> float:
@@ -1391,7 +1396,16 @@ class Evaluation:
         status = "PASS" if reward == 1 else ("FAIL" if reward is not None else "ERR")
         err_msg = result.error or result.verifier_error
         err = f" ({truncate_end(err_msg, 50)})" if err_msg else ""
-        logger.info(f"[{status}] {td.name} (tools={result.n_tool_calls}){err}")
+        # Show the fractional reward on scored lines: pass/fail binarizes at
+        # reward==1, so without it a 0.3 rubric score reads as a flat 0.
+        reward_part = (
+            f"reward={reward:.2f}, "
+            if isinstance(reward, (int, float)) and not isinstance(reward, bool)
+            else ""
+        )
+        logger.info(
+            f"[{status}] {td.name} ({reward_part}tools={result.n_tool_calls}){err}"
+        )
         self._fire_progress(self._on_result, td.name, result)
 
     async def _run_parallel_independent(
@@ -1828,6 +1842,7 @@ class Evaluation:
             memory_score=memory["avg_score"],
             memory_scores=memory_scores,
             task_failures=task_failures,
+            mean_reward=mean_scored_reward(all_results.values()),
         )
 
         assert (
@@ -1885,6 +1900,7 @@ class Evaluation:
             "score_excl_errors_ratio": pass_rate_excl_errors(
                 passed=audit_counts["passed"], failed=audit_counts["failed"]
             ),
+            "mean_reward": job_result.mean_reward,
             "elapsed_sec": elapsed,
             "memory_score": job_result.memory_score,
             "memory_score_coverage": (
@@ -1984,9 +2000,14 @@ class Evaluation:
                     "This likely indicates a systemic verifier bug, not agent failure."
                 )
 
+        mean_part = (
+            f"mean_reward={job_result.mean_reward:.2f}, "
+            if job_result.mean_reward is not None
+            else ""
+        )
         logger.info(
             f"Job complete: {job_result.passed}/{job_result.total} "
-            f"({job_result.score:.1%}), errors={job_result.errored}, "
+            f"({job_result.score:.1%}), {mean_part}errors={job_result.errored}, "
             f"idle_timeouts={error_category_counts.get(IDLE_TIMEOUT, 0)}, "
             f"time={elapsed / 60:.1f}min"
         )

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1839}/tests/test_cli_live_progress.py

@@ -354,6 +354,42 @@ def _failed_result(task_failures):
     )
 
 
+def test_report_eval_result_shows_mean_reward():
+    # Partial credit must be visible next to the binarized counts: pass/fail
+    # thresholds at reward==1, so "0/1 (0.0%)" alone can't distinguish a 0.3
+    # rubric score from a flat 0.
+    out = _reported(
+        SimpleNamespace(
+            passed=0,
+            total=1,
+            errored=0,
+            verifier_errored=0,
+            score=0.0,
+            job_name="j",
+            mean_reward=0.3,
+        )
+    )
+    assert "Score: 0/1" in out
+    assert "mean reward 0.30" in out
+
+
+def test_report_eval_result_omits_mean_reward_when_unavailable():
+    # Sharded aggregation and older callers don't carry mean_reward — the line
+    # must render without it rather than showing a misleading 0.00.
+    out = _reported(
+        SimpleNamespace(
+            passed=0,
+            total=1,
+            errored=1,
+            verifier_errored=0,
+            score=0.0,
+            job_name="j",
+        )
+    )
+    assert "Score: 0/1" in out
+    assert "mean reward" not in out
+
+
 def test_report_eval_result_prints_failure_reason_lines():
     # Dogfood follow-up: "✗ Score: 0/1" alone forces a dig into summary.json.
     # Each FAILED task gets one dim reason line — verifier_error first, else a

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1839}/tests/test_env_registry.py

@@ -80,12 +80,127 @@ def test_resolve_invalid_spec_errors(tmp_path):
         resolve_environment("bad/spec@x", registry=tmp_path)
 
 
-def test_resolve_no_registry_configured_errors(tmp_path, monkeypatch):
+def test_resolve_no_registry_available_errors(tmp_path, monkeypatch):
+    """Env var unset AND built-in registry missing → actionable error."""
+    import benchflow._utils.env_registry as env_registry
+
     monkeypatch.delenv("BENCHFLOW_ENV_REGISTRY", raising=False)
+    monkeypatch.setattr(env_registry, "_builtin_registry_dir", lambda: None)
     with pytest.raises(EnvironmentRegistryError, match="no environment registry"):
         resolve_environment("env0")
 
 
+# ---- built-in registry (shipped inside the wheel) ---------------------------
+
+
+def test_builtin_registry_resolves_env0_prod_when_env_unset(monkeypatch):
+    """The acceptance contract: a bare pip install resolves ``env0@prod`` with
+    no ``$BENCHFLOW_ENV_REGISTRY`` set — the pins ship inside the package."""
+    monkeypatch.delenv("BENCHFLOW_ENV_REGISTRY", raising=False)
+    r = resolve_environment("env0@prod")
+    assert (r.name, r.version) == ("env0", "prod")
+    assert r.manifest_path.name == "env0@prod.toml"
+    assert r.env_hash.startswith("sha256:")
+
+
+def test_builtin_registry_resolves_env0_outage_when_env_unset(monkeypatch):
+    monkeypatch.delenv("BENCHFLOW_ENV_REGISTRY", raising=False)
+    r = resolve_environment("env0@outage")
+    assert (r.name, r.version) == ("env0", "outage")
+
+
+def test_builtin_registry_manifests_are_importlib_resources(monkeypatch):
+    """The pins are package data: importlib.resources must find them (that is
+    what makes them ship in the wheel) and they must parse as manifests."""
+    import importlib.resources
+
+    root = importlib.resources.files("benchflow.environment") / "_registry"
+    names = {entry.name for entry in root.iterdir()}
+    assert {"env0@prod.toml", "env0@outage.toml"} <= names
+
+    monkeypatch.delenv("BENCHFLOW_ENV_REGISTRY", raising=False)
+    m = load_manifest("env0@prod")
+    assert {s.name for s in m.services} >= {"mock-gmail", "mock-slack"}
+    outage = load_manifest("env0@outage")
+    assert "mock-gmail" not in {s.name for s in outage.services}
+
+
+def test_env_var_registry_wins_over_builtin(tmp_path, monkeypatch):
+    """When set, ``$BENCHFLOW_ENV_REGISTRY`` is the registry — the same spec
+    resolves from it, not from the built-in pins."""
+    _write_env(tmp_path, "env0@prod", base_image="img:override")
+    monkeypatch.setenv("BENCHFLOW_ENV_REGISTRY", str(tmp_path))
+    r = resolve_environment("env0@prod")
+    assert r.manifest_path.parent == tmp_path
+    assert load_manifest("env0@prod").base_image == "img:override"
+
+
+def test_env_var_registry_wins_entirely_no_builtin_fallback(tmp_path, monkeypatch):
+    """A name the env-var registry lacks does NOT fall back to the built-in."""
+    monkeypatch.setenv("BENCHFLOW_ENV_REGISTRY", str(tmp_path))  # empty dir
+    with pytest.raises(
+        EnvironmentRegistryError, match=r"not found in .*available: none"
+    ):
+        resolve_environment("env0@prod")
+
+
+def test_explicit_registry_argument_beats_set_env_var(tmp_path, monkeypatch):
+    """Resolution order: an explicit ``registry=`` beats a SET env var."""
+    env_dir = tmp_path / "from-env"
+    arg_dir = tmp_path / "from-arg"
+    env_dir.mkdir()
+    arg_dir.mkdir()
+    _write_env(env_dir, "env0@v1", base_image="img:env")
+    _write_env(arg_dir, "env0@v1", base_image="img:arg")
+    monkeypatch.setenv("BENCHFLOW_ENV_REGISTRY", str(env_dir))
+    r = resolve_environment("env0@v1", registry=arg_dir)
+    assert r.manifest_path.parent == arg_dir
+
+
+def test_empty_env_var_counts_as_unset(monkeypatch):
+    """``BENCHFLOW_ENV_REGISTRY=""`` resolves the built-in registry, exactly
+    like an unset var (empty counts as unset, pinned by the docstring)."""
+    monkeypatch.setenv("BENCHFLOW_ENV_REGISTRY", "")
+    r = resolve_environment("env0@prod")
+    assert (r.name, r.version) == ("env0", "prod")
+
+
+def test_env_var_not_a_directory_error_names_the_env_var(tmp_path, monkeypatch):
+    monkeypatch.setenv("BENCHFLOW_ENV_REGISTRY", str(tmp_path / "nope"))
+    with pytest.raises(
+        EnvironmentRegistryError,
+        match=r"from \$BENCHFLOW_ENV_REGISTRY.*is not a directory",
+    ):
+        resolve_environment("env0@prod")
+
+
+def test_registry_argument_not_a_directory_error_names_the_argument(tmp_path):
+    with pytest.raises(
+        EnvironmentRegistryError,
+        match=r"from the registry= argument.*is not a directory",
+    ):
+        resolve_environment("env0@prod", registry=tmp_path / "nope")
+
+
+def test_builtin_unknown_name_error_lists_available(monkeypatch):
+    monkeypatch.delenv("BENCHFLOW_ENV_REGISTRY", raising=False)
+    with pytest.raises(
+        EnvironmentRegistryError, match=r"available:.*env0@outage.*env0@prod"
+    ):
+        resolve_environment("nope")
+
+
+def test_builtin_resolution_logs_sha256(monkeypatch, caplog):
+    """Content-addressed provenance is preserved: resolving a built-in pin
+    logs the manifest's sha256 exactly like a ``$BENCHFLOW_ENV_REGISTRY`` one."""
+    import logging
+
+    monkeypatch.delenv("BENCHFLOW_ENV_REGISTRY", raising=False)
+    with caplog.at_level(logging.INFO, logger="benchflow.environment.manifest"):
+        load_manifest("env0@prod")
+    assert any("sha256:" in rec.getMessage() for rec in caplog.records)
+
+
 # ---- load_manifest dispatch (spec vs file) --------------------------------
 
 

{benchflow-0.6.7.dev1833 → benchflow-0.6.7.dev1839}/tests/test_job.py

@@ -656,6 +656,57 @@ class TestJobRunOrchestration:
             )
         ]
 
+    @pytest.mark.asyncio
+    async def test_run_computes_mean_reward_and_logs_fractional_rewards(
+        self, tmp_path, caplog
+    ):
+        """Partial credit must survive the binarized pass/fail view: run()
+        computes mean_reward over scored rollouts (errors excluded, not
+        zeroed), the per-task lines carry reward=, error lines don't, and the
+        job-complete line carries mean_reward=.
+        """
+        job = self._make_job(tmp_path, n_tasks=3, concurrency=1)
+        outcomes = {
+            "task-0": RunResult(task_name="task-0", rewards={"reward": 1.0}),
+            "task-1": RunResult(task_name="task-1", rewards={"reward": 0.3}),
+            "task-2": RunResult(task_name="task-2", error="agent crashed"),
+        }
+
+        async def fake_run(*, task_path, **kwargs):
+            return outcomes[task_path.name]
+
+        job._sdk = AsyncMock()
+        job._sdk.run = AsyncMock(side_effect=fake_run)
+
+        with caplog.at_level(logging.INFO, logger="benchflow.evaluation"):
+            result = await job.run()
+
+        assert result.mean_reward == pytest.approx(0.65)
+        assert "[PASS] task-0 (reward=1.00, tools=0)" in caplog.text
+        assert "[FAIL] task-1 (reward=0.30, tools=0)" in caplog.text
+        assert "[ERR] task-2 (tools=0)" in caplog.text
+        assert "mean_reward=0.65" in caplog.text
+        # The machine-readable surface must agree with the console.
+        summary = json.loads((job._jobs_dir / "summary.json").read_text())
+        assert summary["mean_reward"] == pytest.approx(0.65)
+
+    @pytest.mark.asyncio
+    async def test_run_mean_reward_none_when_nothing_scored(self, tmp_path, caplog):
+        """An all-error job has no scored rollouts: mean_reward must be None
+        and the job-complete line must omit the field (a fabricated 0.00 would
+        read as a real capability result)."""
+        job = self._make_job(tmp_path, n_tasks=1, concurrency=1)
+        job._sdk = AsyncMock()
+        job._sdk.run = AsyncMock(
+            return_value=RunResult(task_name="task-0", error="agent crashed")
+        )
+
+        with caplog.at_level(logging.INFO, logger="benchflow.evaluation"):
+            result = await job.run()
+
+        assert result.mean_reward is None
+        assert "mean_reward=" not in caplog.text
+
     @pytest.mark.asyncio
     async def test_error_and_verifier_error_does_not_crash_invariant(self, tmp_path):
         """A result with BOTH error and verifier_error (rewards=None) must not