benchflow 0.6.7.dev1835__tar.gz → 0.6.7.dev1839__tar.gz

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1839}/CHANGELOG.md

@@ -3,6 +3,14 @@
 ## [Unreleased]
 
 ### Added
+- **Built-in environment registry.** The committed env-axis pins
+  (`env0@prod`, `env0@outage`) moved from `benchmarks/_environments/` into
+  the package (`benchflow/environment/_registry/`) and ship inside the
+  wheel, so `--environment-manifest env0@prod` resolves on a bare
+  `pip install benchflow` with no checkout and no env vars.
+  `$BENCHFLOW_ENV_REGISTRY`, when set, still wins entirely; resolution
+  stays content-addressed (sha256 logged), and unknown names now error
+  listing the available specs.
 - **Console progress heartbeat.** Single-concurrency eval runs print a
   throttled progress line (`… 6.2min, 12 tool calls (last: …)`) about every
   45 seconds while the agent works, so a long prompt is distinguishable from

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1839}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1835
+Version: 0.6.7.dev1839
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1839}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1835"
+version = "0.6.7.dev1839"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1839}/src/benchflow/_utils/env_registry.py

@@ -8,11 +8,22 @@ Lee's ``S`` axis in ``I_eval = {T, A, M, S, C, R, τ}`` — from the task, so it
 swappable per run exactly like ``--agent`` / ``--model`` / ``--sandbox`` already
 are, instead of being pinned by a relative path baked into the task file.
 
-Registry layout (filesystem; ``$BENCHFLOW_ENV_REGISTRY``)::
+Registry layout (a directory of manifest files)::
 
     <registry>/<name>@<version>.toml   # a pinned environment version
     <registry>/<name>.toml             # optional default ("latest")
 
+Which directory is the registry:
+
+1. an explicit ``registry=`` argument (tests, embedders);
+2. else ``$BENCHFLOW_ENV_REGISTRY`` — when set it wins entirely (no fallback
+   into the built-in registry for names it does not contain); an empty value
+   counts as unset;
+3. else the **built-in registry** shipped inside the ``benchflow`` wheel
+   (``benchflow/environment/_registry/``), so a bare ``pip install benchflow``
+   resolves the committed pins (``env0@prod``, ``env0@outage``) with no
+   checkout and no env vars.
+
 Resolution::
 
     env0          -> <registry>/env0.toml          (default) else newest @version
@@ -80,21 +91,49 @@ def looks_like_env_spec(value: str) -> bool:
     return bool(_SPEC_RE.match(value))
 
 
+def _builtin_registry_dir() -> Path | None:
+    """Directory of the pinned manifests shipped inside the ``benchflow`` wheel.
+
+    ``benchflow/environment/_registry/`` is package data, so a bare
+    ``pip install benchflow`` carries the committed pins. Path(__file__)-relative
+    like every other package-data lookup in the repo (compose files, default
+    rubric, demo task) — zip imports are unsupported package-wide. Returns
+    ``None`` when the packaged registry is missing (a stripped install).
+    """
+    directory = Path(__file__).resolve().parents[1] / "environment" / "_registry"
+    return directory if directory.is_dir() else None
+
+
 def _registry_dir(registry: str | os.PathLike[str] | None) -> Path:
-    raw = registry if registry is not None else os.environ.get(ENV_REGISTRY_VAR)
-    if not raw:
+    if registry is not None:
+        raw: str | os.PathLike[str] | None = registry
+        source = "the registry= argument"
+    else:
+        raw = os.environ.get(ENV_REGISTRY_VAR)
+        source = f"${ENV_REGISTRY_VAR}"
+    if not raw:  # unset or empty — empty counts as unset
+        builtin = _builtin_registry_dir()
+        if builtin is not None:
+            return builtin
         raise EnvironmentRegistryError(
-            f"no environment registry configured; set ${ENV_REGISTRY_VAR} or pass "
-            "--environment-manifest a manifest file path instead of a name@version spec"
+            "no environment registry available: the built-in registry shipped "
+            f"with benchflow is missing and ${ENV_REGISTRY_VAR} is unset; set "
+            f"${ENV_REGISTRY_VAR} or pass --environment-manifest a manifest "
+            "file path instead of a name@version spec"
         )
     directory = Path(raw).expanduser()
     if not directory.is_dir():
         raise EnvironmentRegistryError(
-            f"environment registry {directory} is not a directory"
+            f"environment registry {directory} (from {source}) is not a directory"
         )
     return directory
 
 
+def _available_specs(directory: Path) -> list[str]:
+    """Sorted stems (``name`` / ``name@version``) present in the registry."""
+    return sorted({p.stem for ext in _MANIFEST_EXTS for p in directory.glob(f"*{ext}")})
+
+
 def _find_manifest(directory: Path, stem: str) -> Path | None:
     """Return ``directory/stem.<ext>`` for the first existing manifest extension."""
     for ext in _MANIFEST_EXTS:
@@ -124,9 +163,11 @@ def resolve_environment(
     if version is not None:
         path = _find_manifest(directory, f"{name}@{version}")
         if path is None:
+            available = ", ".join(_available_specs(directory)) or "none"
             raise EnvironmentRegistryError(
                 f"environment {spec!r} not found in {directory} "
-                f"(looked for {name}@{version}{{.toml,.yaml,.yml}})"
+                f"(looked for {name}@{version}{{.toml,.yaml,.yml}}; "
+                f"available: {available})"
             )
     else:
         path = _find_manifest(directory, name)
@@ -137,8 +178,10 @@ def resolve_environment(
                 p for ext in _MANIFEST_EXTS for p in directory.glob(f"{name}@*{ext}")
             )
             if not candidates:
+                available = ", ".join(_available_specs(directory)) or "none"
                 raise EnvironmentRegistryError(
-                    f"no versions of environment {name!r} found in {directory}"
+                    f"no versions of environment {name!r} found in {directory} "
+                    f"(available: {available})"
                 )
             path = candidates[-1]
             version = path.stem.split("@", 1)[1]

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1839}/src/benchflow/cli/main.py

@@ -297,10 +297,11 @@ def eval_run(
             "--environment-manifest",
             help=(
                 "Environment-plane manifest applied to every rollout: a path to "
-                "an environment.toml, OR a 'name@version' registry spec resolved "
-                "via $BENCHFLOW_ENV_REGISTRY (the S axis). The manifest-declared "
-                "stateful environment is provisioned, gated on readiness, and "
-                "torn down."
+                "an environment.toml, OR a 'name@version' registry spec (the S "
+                "axis) resolved via $BENCHFLOW_ENV_REGISTRY when set, else the "
+                "built-in registry shipped with benchflow (env0@prod, "
+                "env0@outage). The manifest-declared stateful environment is "
+                "provisioned, gated on readiness, and torn down."
             ),
         ),
     ] = None,

benchflow-0.6.7.dev1839/src/benchflow/environment/_registry/env0@outage.toml

@@ -0,0 +1,63 @@
+# env0@outage — env-0 with the gmail + slack services removed.
+#
+# A perturbation variant for the "Same state, tool outage" experiment: bind the
+# SAME task to env0@prod vs env0@outage and attribute the reward delta to the
+# environment alone (the S axis). A task that needs slack (port 9005) scores 1.0
+# under env0@prod and fails under env0@outage because the service is absent.
+#
+# Shipped inside the benchflow wheel (benchflow/environment/_registry/), so it
+# resolves on a bare pip install with no $BENCHFLOW_ENV_REGISTRY set:
+#
+#   bench eval create --tasks-dir <task> --environment-manifest env0@outage ...
+
+[environment]
+name           = "env-0"
+base_image     = "xdotli/env0-base:latest"
+owns_lifecycle = false
+isolation      = "per_task"
+
+[environment.task_selection]
+mechanism = "image"
+
+[environment.readiness]
+timeout_sec = 60
+
+# Services below are copied VERBATIM from the upstream env-0.toml pin
+# (2026-08-09 sync). The previous pin had drifted: bare CLI names
+# (auth/gmail/...) that do not exist in ghcr.io/benchflow-ai/env0:0.2.0
+# images (which ship mock-auth/mock-gmail/... console scripts), plus a
+# phantom slack service and shifted ports — so ManifestEnvironment
+# probed, found ZERO startable services, and readiness passed vacuously
+# while every rollout died at the verifier with connection-refused.
+
+[[environment.services]]
+name    = "mock-auth"
+command = "mock-auth --db /data/auth.db serve --host 0.0.0.0 --port 9000 --no-mcp"
+port    = 9000
+
+
+[[environment.services]]
+name    = "mock-gcal"
+command = "mock-gcal --db /data/gcal.db serve --host 0.0.0.0 --port 9002 --no-mcp"
+port    = 9002
+
+[[environment.services]]
+name    = "mock-gdrive"
+command = "mock-gdrive --db /data/gdrive.db serve --host 0.0.0.0 --port 9003 --no-mcp"
+port    = 9003
+
+[[environment.services]]
+name    = "mock-gdoc"
+command = "mock-gdoc --db /data/gdoc.db serve --host 0.0.0.0 --port 9004 --no-mcp"
+port    = 9004
+
+
+[[environment.services]]
+name    = "mock-discord"
+command = "mock-discord --db /data/discord.db serve --host 0.0.0.0 --port 9006 --no-mcp"
+port    = 9006
+
+[[environment.services]]
+name    = "mock-stripe"
+command = "mock-stripe --db /data/stripe.db serve --host 0.0.0.0 --port 9007 --no-mcp"
+port    = 9007

benchflow-0.6.7.dev1839/src/benchflow/environment/_registry/env0@prod.toml

@@ -0,0 +1,77 @@
+# env0@prod — pinned, content-addressed Environment-plane manifest for env-0.
+#
+# This is a built-in registry entry, shipped inside the benchflow wheel
+# (benchflow/environment/_registry/) so PR #790's env-axis pin resolves on a
+# bare pip install — no checkout, no env vars:
+#
+#   bench eval create --tasks-dir <env0 tasks> --environment-manifest env0@prod ...
+#
+# Set $BENCHFLOW_ENV_REGISTRY to a local directory of name@version.toml files
+# to override the built-in registry entirely.
+#
+# Mirrors benchflow-ai/env0 tasks/_manifests/env-0.toml (the source of truth). env-0's per-task images build
+# FROM the shared base and bake seed data; benchflow's ManifestEnvironment
+# probes each service's CLI and starts only the ones installed in the per-task
+# image, gating the agent on each service's /health.
+#
+# NOTE: xdotli/env0-base:latest is amd64-only — run env0 on Daytona (x86_64).
+
+[environment]
+name           = "env-0"
+base_image     = "xdotli/env0-base:latest"
+owns_lifecycle = false
+isolation      = "per_task"
+
+[environment.task_selection]
+mechanism = "image"
+
+[environment.readiness]
+timeout_sec = 60
+
+# Services below are copied VERBATIM from the upstream env-0.toml pin
+# (2026-08-09 sync). The previous pin had drifted: bare CLI names
+# (auth/gmail/...) that do not exist in ghcr.io/benchflow-ai/env0:0.2.0
+# images (which ship mock-auth/mock-gmail/... console scripts), plus a
+# phantom slack service and shifted ports — so ManifestEnvironment
+# probed, found ZERO startable services, and readiness passed vacuously
+# while every rollout died at the verifier with connection-refused.
+
+[[environment.services]]
+name    = "mock-auth"
+command = "mock-auth --db /data/auth.db serve --host 0.0.0.0 --port 9000 --no-mcp"
+port    = 9000
+
+[[environment.services]]
+name    = "mock-gmail"
+command = "mock-gmail --db /data/gmail.db serve --host 0.0.0.0 --port 9001 --no-mcp"
+port    = 9001
+
+[[environment.services]]
+name    = "mock-gcal"
+command = "mock-gcal --db /data/gcal.db serve --host 0.0.0.0 --port 9002 --no-mcp"
+port    = 9002
+
+[[environment.services]]
+name    = "mock-gdrive"
+command = "mock-gdrive --db /data/gdrive.db serve --host 0.0.0.0 --port 9003 --no-mcp"
+port    = 9003
+
+[[environment.services]]
+name    = "mock-gdoc"
+command = "mock-gdoc --db /data/gdoc.db serve --host 0.0.0.0 --port 9004 --no-mcp"
+port    = 9004
+
+[[environment.services]]
+name    = "mock-slack"
+command = "mock-slack --db /data/slack.db serve --host 0.0.0.0 --port 9005 --no-mcp"
+port    = 9005
+
+[[environment.services]]
+name    = "mock-discord"
+command = "mock-discord --db /data/discord.db serve --host 0.0.0.0 --port 9006 --no-mcp"
+port    = 9006
+
+[[environment.services]]
+name    = "mock-stripe"
+command = "mock-stripe --db /data/stripe.db serve --host 0.0.0.0 --port 9007 --no-mcp"
+port    = 9007

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1839}/src/benchflow/environment/manifest.py

@@ -205,7 +205,9 @@ def load_manifest(path: str | Path) -> EnvironmentManifest:
     """Load and validate an environment manifest.
 
     ``path`` is either a TOML file path (the historical behavior) or a registry
-    spec ``name@version`` resolved via ``$BENCHFLOW_ENV_REGISTRY``. The spec form
+    spec ``name@version`` resolved via ``$BENCHFLOW_ENV_REGISTRY`` when set,
+    else the built-in registry shipped inside the wheel
+    (``benchflow/environment/_registry/``). The spec form
     lets a run bind its environment (the ``S`` axis) by name at the command line —
     decoupled from the task and swappable per run, like ``--agent`` / ``--model``
     / ``--sandbox``. Resolution is content-addressed so the bound world is

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1839}/tests/test_env_registry.py

@@ -80,12 +80,127 @@ def test_resolve_invalid_spec_errors(tmp_path):
         resolve_environment("bad/spec@x", registry=tmp_path)
 
 
-def test_resolve_no_registry_configured_errors(tmp_path, monkeypatch):
+def test_resolve_no_registry_available_errors(tmp_path, monkeypatch):
+    """Env var unset AND built-in registry missing → actionable error."""
+    import benchflow._utils.env_registry as env_registry
+
     monkeypatch.delenv("BENCHFLOW_ENV_REGISTRY", raising=False)
+    monkeypatch.setattr(env_registry, "_builtin_registry_dir", lambda: None)
     with pytest.raises(EnvironmentRegistryError, match="no environment registry"):
         resolve_environment("env0")
 
 
+# ---- built-in registry (shipped inside the wheel) ---------------------------
+
+
+def test_builtin_registry_resolves_env0_prod_when_env_unset(monkeypatch):
+    """The acceptance contract: a bare pip install resolves ``env0@prod`` with
+    no ``$BENCHFLOW_ENV_REGISTRY`` set — the pins ship inside the package."""
+    monkeypatch.delenv("BENCHFLOW_ENV_REGISTRY", raising=False)
+    r = resolve_environment("env0@prod")
+    assert (r.name, r.version) == ("env0", "prod")
+    assert r.manifest_path.name == "env0@prod.toml"
+    assert r.env_hash.startswith("sha256:")
+
+
+def test_builtin_registry_resolves_env0_outage_when_env_unset(monkeypatch):
+    monkeypatch.delenv("BENCHFLOW_ENV_REGISTRY", raising=False)
+    r = resolve_environment("env0@outage")
+    assert (r.name, r.version) == ("env0", "outage")
+
+
+def test_builtin_registry_manifests_are_importlib_resources(monkeypatch):
+    """The pins are package data: importlib.resources must find them (that is
+    what makes them ship in the wheel) and they must parse as manifests."""
+    import importlib.resources
+
+    root = importlib.resources.files("benchflow.environment") / "_registry"
+    names = {entry.name for entry in root.iterdir()}
+    assert {"env0@prod.toml", "env0@outage.toml"} <= names
+
+    monkeypatch.delenv("BENCHFLOW_ENV_REGISTRY", raising=False)
+    m = load_manifest("env0@prod")
+    assert {s.name for s in m.services} >= {"mock-gmail", "mock-slack"}
+    outage = load_manifest("env0@outage")
+    assert "mock-gmail" not in {s.name for s in outage.services}
+
+
+def test_env_var_registry_wins_over_builtin(tmp_path, monkeypatch):
+    """When set, ``$BENCHFLOW_ENV_REGISTRY`` is the registry — the same spec
+    resolves from it, not from the built-in pins."""
+    _write_env(tmp_path, "env0@prod", base_image="img:override")
+    monkeypatch.setenv("BENCHFLOW_ENV_REGISTRY", str(tmp_path))
+    r = resolve_environment("env0@prod")
+    assert r.manifest_path.parent == tmp_path
+    assert load_manifest("env0@prod").base_image == "img:override"
+
+
+def test_env_var_registry_wins_entirely_no_builtin_fallback(tmp_path, monkeypatch):
+    """A name the env-var registry lacks does NOT fall back to the built-in."""
+    monkeypatch.setenv("BENCHFLOW_ENV_REGISTRY", str(tmp_path))  # empty dir
+    with pytest.raises(
+        EnvironmentRegistryError, match=r"not found in .*available: none"
+    ):
+        resolve_environment("env0@prod")
+
+
+def test_explicit_registry_argument_beats_set_env_var(tmp_path, monkeypatch):
+    """Resolution order: an explicit ``registry=`` beats a SET env var."""
+    env_dir = tmp_path / "from-env"
+    arg_dir = tmp_path / "from-arg"
+    env_dir.mkdir()
+    arg_dir.mkdir()
+    _write_env(env_dir, "env0@v1", base_image="img:env")
+    _write_env(arg_dir, "env0@v1", base_image="img:arg")
+    monkeypatch.setenv("BENCHFLOW_ENV_REGISTRY", str(env_dir))
+    r = resolve_environment("env0@v1", registry=arg_dir)
+    assert r.manifest_path.parent == arg_dir
+
+
+def test_empty_env_var_counts_as_unset(monkeypatch):
+    """``BENCHFLOW_ENV_REGISTRY=""`` resolves the built-in registry, exactly
+    like an unset var (empty counts as unset, pinned by the docstring)."""
+    monkeypatch.setenv("BENCHFLOW_ENV_REGISTRY", "")
+    r = resolve_environment("env0@prod")
+    assert (r.name, r.version) == ("env0", "prod")
+
+
+def test_env_var_not_a_directory_error_names_the_env_var(tmp_path, monkeypatch):
+    monkeypatch.setenv("BENCHFLOW_ENV_REGISTRY", str(tmp_path / "nope"))
+    with pytest.raises(
+        EnvironmentRegistryError,
+        match=r"from \$BENCHFLOW_ENV_REGISTRY.*is not a directory",
+    ):
+        resolve_environment("env0@prod")
+
+
+def test_registry_argument_not_a_directory_error_names_the_argument(tmp_path):
+    with pytest.raises(
+        EnvironmentRegistryError,
+        match=r"from the registry= argument.*is not a directory",
+    ):
+        resolve_environment("env0@prod", registry=tmp_path / "nope")
+
+
+def test_builtin_unknown_name_error_lists_available(monkeypatch):
+    monkeypatch.delenv("BENCHFLOW_ENV_REGISTRY", raising=False)
+    with pytest.raises(
+        EnvironmentRegistryError, match=r"available:.*env0@outage.*env0@prod"
+    ):
+        resolve_environment("nope")
+
+
+def test_builtin_resolution_logs_sha256(monkeypatch, caplog):
+    """Content-addressed provenance is preserved: resolving a built-in pin
+    logs the manifest's sha256 exactly like a ``$BENCHFLOW_ENV_REGISTRY`` one."""
+    import logging
+
+    monkeypatch.delenv("BENCHFLOW_ENV_REGISTRY", raising=False)
+    with caplog.at_level(logging.INFO, logger="benchflow.environment.manifest"):
+        load_manifest("env0@prod")
+    assert any("sha256:" in rec.getMessage() for rec in caplog.records)
+
+
 # ---- load_manifest dispatch (spec vs file) --------------------------------