benchflow 0.6.7.dev1835__tar.gz → 0.6.7.dev1840__tar.gz

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1840}/CHANGELOG.md

@@ -3,6 +3,14 @@
 ## [Unreleased]
 
 ### Added
+- **Built-in environment registry.** The committed env-axis pins
+  (`env0@prod`, `env0@outage`) moved from `benchmarks/_environments/` into
+  the package (`benchflow/environment/_registry/`) and ship inside the
+  wheel, so `--environment-manifest env0@prod` resolves on a bare
+  `pip install benchflow` with no checkout and no env vars.
+  `$BENCHFLOW_ENV_REGISTRY`, when set, still wins entirely; resolution
+  stays content-addressed (sha256 logged), and unknown names now error
+  listing the available specs.
 - **Console progress heartbeat.** Single-concurrency eval runs print a
   throttled progress line (`… 6.2min, 12 tool calls (last: …)`) about every
   45 seconds while the agent works, so a long prompt is distinguishable from

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1840}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1835
+Version: 0.6.7.dev1840
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1840}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1835"
+version = "0.6.7.dev1840"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1840}/src/benchflow/_utils/env_registry.py

@@ -8,11 +8,22 @@ Lee's ``S`` axis in ``I_eval = {T, A, M, S, C, R, τ}`` — from the task, so it
 swappable per run exactly like ``--agent`` / ``--model`` / ``--sandbox`` already
 are, instead of being pinned by a relative path baked into the task file.
 
-Registry layout (filesystem; ``$BENCHFLOW_ENV_REGISTRY``)::
+Registry layout (a directory of manifest files)::
 
     <registry>/<name>@<version>.toml   # a pinned environment version
     <registry>/<name>.toml             # optional default ("latest")
 
+Which directory is the registry:
+
+1. an explicit ``registry=`` argument (tests, embedders);
+2. else ``$BENCHFLOW_ENV_REGISTRY`` — when set it wins entirely (no fallback
+   into the built-in registry for names it does not contain); an empty value
+   counts as unset;
+3. else the **built-in registry** shipped inside the ``benchflow`` wheel
+   (``benchflow/environment/_registry/``), so a bare ``pip install benchflow``
+   resolves the committed pins (``env0@prod``, ``env0@outage``) with no
+   checkout and no env vars.
+
 Resolution::
 
     env0          -> <registry>/env0.toml          (default) else newest @version
@@ -80,21 +91,49 @@ def looks_like_env_spec(value: str) -> bool:
     return bool(_SPEC_RE.match(value))
 
 
+def _builtin_registry_dir() -> Path | None:
+    """Directory of the pinned manifests shipped inside the ``benchflow`` wheel.
+
+    ``benchflow/environment/_registry/`` is package data, so a bare
+    ``pip install benchflow`` carries the committed pins. Path(__file__)-relative
+    like every other package-data lookup in the repo (compose files, default
+    rubric, demo task) — zip imports are unsupported package-wide. Returns
+    ``None`` when the packaged registry is missing (a stripped install).
+    """
+    directory = Path(__file__).resolve().parents[1] / "environment" / "_registry"
+    return directory if directory.is_dir() else None
+
+
 def _registry_dir(registry: str | os.PathLike[str] | None) -> Path:
-    raw = registry if registry is not None else os.environ.get(ENV_REGISTRY_VAR)
-    if not raw:
+    if registry is not None:
+        raw: str | os.PathLike[str] | None = registry
+        source = "the registry= argument"
+    else:
+        raw = os.environ.get(ENV_REGISTRY_VAR)
+        source = f"${ENV_REGISTRY_VAR}"
+    if not raw:  # unset or empty — empty counts as unset
+        builtin = _builtin_registry_dir()
+        if builtin is not None:
+            return builtin
         raise EnvironmentRegistryError(
-            f"no environment registry configured; set ${ENV_REGISTRY_VAR} or pass "
-            "--environment-manifest a manifest file path instead of a name@version spec"
+            "no environment registry available: the built-in registry shipped "
+            f"with benchflow is missing and ${ENV_REGISTRY_VAR} is unset; set "
+            f"${ENV_REGISTRY_VAR} or pass --environment-manifest a manifest "
+            "file path instead of a name@version spec"
         )
     directory = Path(raw).expanduser()
     if not directory.is_dir():
         raise EnvironmentRegistryError(
-            f"environment registry {directory} is not a directory"
+            f"environment registry {directory} (from {source}) is not a directory"
         )
     return directory
 
 
+def _available_specs(directory: Path) -> list[str]:
+    """Sorted stems (``name`` / ``name@version``) present in the registry."""
+    return sorted({p.stem for ext in _MANIFEST_EXTS for p in directory.glob(f"*{ext}")})
+
+
 def _find_manifest(directory: Path, stem: str) -> Path | None:
     """Return ``directory/stem.<ext>`` for the first existing manifest extension."""
     for ext in _MANIFEST_EXTS:
@@ -124,9 +163,11 @@ def resolve_environment(
     if version is not None:
         path = _find_manifest(directory, f"{name}@{version}")
         if path is None:
+            available = ", ".join(_available_specs(directory)) or "none"
             raise EnvironmentRegistryError(
                 f"environment {spec!r} not found in {directory} "
-                f"(looked for {name}@{version}{{.toml,.yaml,.yml}})"
+                f"(looked for {name}@{version}{{.toml,.yaml,.yml}}; "
+                f"available: {available})"
             )
     else:
         path = _find_manifest(directory, name)
@@ -137,8 +178,10 @@ def resolve_environment(
                 p for ext in _MANIFEST_EXTS for p in directory.glob(f"{name}@*{ext}")
             )
             if not candidates:
+                available = ", ".join(_available_specs(directory)) or "none"
                 raise EnvironmentRegistryError(
-                    f"no versions of environment {name!r} found in {directory}"
+                    f"no versions of environment {name!r} found in {directory} "
+                    f"(available: {available})"
                 )
             path = candidates[-1]
             version = path.stem.split("@", 1)[1]

benchflow-0.6.7.dev1840/src/benchflow/cli/_failure_evidence.py

@@ -0,0 +1,167 @@
+"""Verifier-artifact mining for the eval report's failure lines.
+
+The one file-touching corner of the CLI's final report block: when a failed
+task's reason would otherwise be the bare ``reward X`` fallback, mine the
+rollout's verifier artifacts for a one-line explanation. Lives in its own
+module so ``cli/_shared.py`` stays the side-effect-free display helpers it
+advertises.
+
+Contract (the engine stays file-free — these reads are CLI-side, report-time
+only):
+
+- The rollout dir is resolved exactly, from the ``rollout_name`` the engine
+  records on every result — never guessed from globs.
+- Evidence sources are tried in order (CTRF report, then test-stdout tail);
+  the first that yields a line wins.
+- Every read is bounded (``_ARTIFACT_READ_BYTES`` per file) and nothing here
+  raises — any surprise degrades to ``None``, i.e. the bare reward reason.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import TYPE_CHECKING, NamedTuple
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from pathlib import Path
+
+# Never read more than this many bytes per file, and evidence is only mined
+# for the <= _MAX_FAILURE_LINES tasks the report block displays.
+_ARTIFACT_READ_BYTES = 64 * 1024
+# Pytest final summary, decoration stripped: "1 failed, 2 passed in 40.86s".
+_PYTEST_SUMMARY_RE = re.compile(r"\d+ failed\b")
+
+
+class FailureLine(NamedTuple):
+    """A failure one-liner in two parts with distinct truncation contracts.
+
+    ``body`` is the free-text evidence (test name plus assertion) and competes
+    for the display line's char budget; ``suffix`` is the compact multi-failure
+    roll-up count (empty when the evidence covers everything) and must survive
+    display truncation. Renderers give ``body`` the full line budget and append
+    ``suffix`` whole past it — never truncating the concatenation, or a long
+    assertion would silently eat the "there is more broken than this" signal.
+    """
+
+    body: str
+    suffix: str = ""
+
+
+def _display_test_name(raw_name: str) -> str:
+    """Test name for display: node-id path segments dropped, param id kept.
+
+    Full pytest node ids ("tests/test_x.py::test_build[param]") waste the
+    100-char line budget; keep the function name plus any ``[param]`` id.
+    The split must not touch the bracket part — a param id may itself contain
+    ``::`` — so only the text before the first ``[`` is segment-split. (When
+    the report was written by pytest-json-ctrf — as of 0.5.x — the param id
+    is already gone — the plugin stores ``nodeid.split('[')[0]`` as ``name``
+    — but other CTRF producers keep the full name, and we must not re-trim
+    it.)
+    """
+    head, bracket, param = raw_name.partition("[")
+    return head.rsplit("::", 1)[-1] + bracket + param
+
+
+def _ctrf_failure_line(ctrf_path: Path) -> FailureLine | None:
+    """``<test> failed[: <assertion>]`` from the first failed CTRF test.
+
+    The verifier's pytest run writes a CTRF report (``pytest --ctrf``, see the
+    standard path in ``task_authoring/structural_checks.py``). Per test the
+    useful failure evidence is the ``E …`` assertion line inside ``trace``;
+    ``message`` is only a generic phase description, so it is the fallback.
+    When the report holds more than one failed test, the first failure's line
+    carries a count suffix (``(+N more failures; P/T checks passed)``) so the
+    console never under-reports how much is broken.
+    """
+    if ctrf_path.stat().st_size > _ARTIFACT_READ_BYTES:
+        # JSON only parses whole — an oversized report is skipped (not
+        # truncated) and the next evidence source gets its turn.
+        return None
+    data = json.loads(ctrf_path.read_text(encoding="utf-8", errors="replace"))
+    raw_tests = (data.get("results") or {}).get("tests") or []
+    tests = [test for test in raw_tests if isinstance(test, dict)]
+    failed = [test for test in tests if test.get("status") == "failed"]
+    if not failed:
+        return None
+    suffix = ""
+    if len(failed) > 1:
+        # Counts come from the same rolled-up test list the first-failure pick
+        # uses, so the suffix can never disagree with the shown evidence.
+        # T is all entries — skips count against the denominator, not as passes.
+        extra = len(failed) - 1
+        passed = sum(1 for test in tests if test.get("status") == "passed")
+        plural = "" if extra == 1 else "s"
+        suffix = (
+            f" (+{extra} more failure{plural}; {passed}/{len(tests)} checks passed)"
+        )
+    test = failed[0]
+    name = _display_test_name(str(test.get("name") or "test"))
+    trace = test.get("trace")
+    if isinstance(trace, str):
+        for line in trace.splitlines():
+            stripped = line.strip()
+            if stripped.startswith("E "):
+                assertion = stripped[2:].strip()
+                # "<test> failed:" already says it's an assertion — drop
+                # the prefix to reclaim line budget for the message.
+                assertion = assertion.removeprefix("AssertionError: ")
+                return FailureLine(f"{name} failed: {assertion}", suffix)
+    message = test.get("message")
+    if isinstance(message, str) and message.strip():
+        return FailureLine(f"{name} failed: {' '.join(message.split())}", suffix)
+    return FailureLine(f"{name} failed", suffix)
+
+
+def _stdout_tail_failure_line(stdout_path: Path) -> FailureLine | None:
+    """Last pytest summary ("N failed, M passed …") — else last ``FAILED …``
+    line — from a bounded tail of the verifier's test-stdout capture. No count
+    suffix: the summary line already carries the full counts itself."""
+    with stdout_path.open("rb") as fh:
+        fh.seek(0, 2)  # SEEK_END
+        fh.seek(max(0, fh.tell() - _ARTIFACT_READ_BYTES))
+        tail = fh.read(_ARTIFACT_READ_BYTES).decode("utf-8", errors="replace")
+    last_failed: str | None = None
+    for line in reversed(tail.splitlines()):
+        bare = line.strip().strip("=").strip()
+        if _PYTEST_SUMMARY_RE.match(bare):
+            return FailureLine(bare)
+        if last_failed is None and line.strip().startswith("FAILED "):
+            last_failed = line.strip()
+    return FailureLine(last_failed) if last_failed is not None else None
+
+
+def artifact_failure_evidence(
+    job_dir: Path, rollout_name: str
+) -> tuple[FailureLine, Path] | None:
+    """``FailureLine`` mined from the rollout's verifier artifacts, or ``None``.
+
+    Returns the structured one-liner plus the verifier dir it came from (for
+    the report block's single ``(details: …)`` pointer line). Never raises.
+    """
+    # Local import: RolloutPaths pulls in the task package, which the CLI
+    # shouldn't pay for until a failure actually needs artifact evidence.
+    from benchflow.task.paths import RolloutPaths
+
+    rollout_paths = RolloutPaths(rollout_dir=job_dir / rollout_name)
+    verifier_dir = rollout_paths.verifier_dir
+    attempts: list[tuple[Path, Callable[[Path], FailureLine | None]]] = [
+        # ctrf.json has no RolloutPaths property — the verifier recovers it by
+        # literal name (verifier_core.py, `_recover_main_verifier_outputs`).
+        (verifier_dir / "ctrf.json", _ctrf_failure_line),
+        (rollout_paths.test_stdout_path, _stdout_tail_failure_line),
+    ]
+    for path, extract in attempts:
+        try:
+            if not path.is_file():
+                continue
+            detail = extract(path)
+        except Exception:
+            # Unreadable file, bad JSON, permissions … try the next source;
+            # the report must never crash over evidence mining.
+            continue
+        if detail is not None:
+            return detail, verifier_dir
+    return None

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1840}/src/benchflow/cli/_shared.py

@@ -21,7 +21,7 @@ from rich.console import Console
 from rich.markup import escape
 
 from benchflow._utils.text import truncate_end
-from benchflow.cli._failure_evidence import artifact_failure_evidence
+from benchflow.cli._failure_evidence import FailureLine, artifact_failure_evidence
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -106,7 +106,8 @@ def _exit_if_evaluation_had_errors(result: object) -> None:
 
 
 # Final-block failure lines: keep the block skimmable on big jobs and each
-# line inside a typical terminal width.
+# line inside a typical terminal width — except lines carrying a multi-failure
+# count suffix, which deliberately run ~40 chars past the budget.
 _MAX_FAILURE_LINES = 5
 _FAILURE_LINE_LIMIT = 100
 _FAILURE_REASON_METRICS = 3
@@ -114,7 +115,7 @@ _FAILURE_REASON_METRICS = 3
 
 def _failure_reason(
     failure: TaskFailure, job_dir: Path | None = None
-) -> tuple[str, Path | None]:
+) -> tuple[FailureLine, Path | None]:
     """One cheap line explaining why a FAILED (scored, reward != 1) task
     failed, plus the verifier dir when artifacts supplied the evidence.
 
@@ -123,13 +124,15 @@ def _failure_reason(
     first — they explain the miss); else the reward plus a one-liner mined
     from the rollout's verifier artifacts (bounded CLI-side reads resolved via
     the ``rollout_name`` the engine records — the engine stays file-free);
-    else just the reward. The returned path is non-``None`` only for the
-    artifact tier, so the caller can print one ``(details: …)`` pointer per
-    report block.
+    else just the reward. The reason is a ``FailureLine`` so the artifact
+    tier's multi-failure count suffix stays separable from the truncatable
+    body (only the artifact tier ever sets it). The returned path is
+    non-``None`` only for the artifact tier, so the caller can print one
+    ``(details: …)`` pointer per report block.
     """
     if failure.verifier_error:
         # Collapse whitespace: verifier errors are routinely multi-line.
-        return " ".join(failure.verifier_error.split()), None
+        return FailureLine(" ".join(failure.verifier_error.split())), None
     rewards = failure.rewards or {}
     reward = rewards.get("reward")
     metrics = [
@@ -144,15 +147,18 @@ def _failure_reason(
         shown = ", ".join(
             f"{name} {value}" for name, value in metrics[:_FAILURE_REASON_METRICS]
         )
-        return f"reward {reward} — {shown}", None
+        return FailureLine(f"reward {reward} — {shown}"), None
     # No rollout_name (pre-#957-follow-up result.json, sharded aggregation):
     # nothing to resolve — the bare reward is as honest as it gets.
     if job_dir is not None and failure.rollout_name:
         evidence = artifact_failure_evidence(job_dir, failure.rollout_name)
         if evidence is not None:
             detail, verifier_dir = evidence
-            return f"reward {reward} — {detail}", verifier_dir
-    return f"reward {reward}", None
+            return (
+                FailureLine(f"reward {reward} — {detail.body}", detail.suffix),
+                verifier_dir,
+            )
+    return FailureLine(f"reward {reward}"), None
 
 
 def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -> None:
@@ -207,11 +213,16 @@ def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -
         reason, verifier_dir = _failure_reason(failure, job_dir)
         if artifact_pointer is None and verifier_dir is not None:
             artifact_pointer = verifier_dir
-        line = truncate_end(
-            f"  ✗ {failure.task_name}: {reason}",
+        # The char budget governs only the free-text body; the multi-failure
+        # count suffix is appended AFTER truncation so a long assertion can
+        # never swallow the "more than this is broken" signal. Worst case the
+        # line runs ~40 chars past the budget — completeness beats strict
+        # width for this one signal.
+        body = truncate_end(
+            f"  ✗ {failure.task_name}: {reason.body}",
             _FAILURE_LINE_LIMIT,
         )
-        console.print(f"[dim]{escape(line)}[/dim]")
+        console.print(f"[dim]{escape(body + reason.suffix)}[/dim]")
     extra = len(failures) - _MAX_FAILURE_LINES
     if extra > 0:
         console.print(f"[dim]  … and {extra} more[/dim]")

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1840}/src/benchflow/cli/main.py

@@ -297,10 +297,11 @@ def eval_run(
             "--environment-manifest",
             help=(
                 "Environment-plane manifest applied to every rollout: a path to "
-                "an environment.toml, OR a 'name@version' registry spec resolved "
-                "via $BENCHFLOW_ENV_REGISTRY (the S axis). The manifest-declared "
-                "stateful environment is provisioned, gated on readiness, and "
-                "torn down."
+                "an environment.toml, OR a 'name@version' registry spec (the S "
+                "axis) resolved via $BENCHFLOW_ENV_REGISTRY when set, else the "
+                "built-in registry shipped with benchflow (env0@prod, "
+                "env0@outage). The manifest-declared stateful environment is "
+                "provisioned, gated on readiness, and torn down."
             ),
         ),
     ] = None,

benchflow-0.6.7.dev1840/src/benchflow/environment/_registry/env0@outage.toml

@@ -0,0 +1,63 @@
+# env0@outage — env-0 with the gmail + slack services removed.
+#
+# A perturbation variant for the "Same state, tool outage" experiment: bind the
+# SAME task to env0@prod vs env0@outage and attribute the reward delta to the
+# environment alone (the S axis). A task that needs slack (port 9005) scores 1.0
+# under env0@prod and fails under env0@outage because the service is absent.
+#
+# Shipped inside the benchflow wheel (benchflow/environment/_registry/), so it
+# resolves on a bare pip install with no $BENCHFLOW_ENV_REGISTRY set:
+#
+#   bench eval create --tasks-dir <task> --environment-manifest env0@outage ...
+
+[environment]
+name           = "env-0"
+base_image     = "xdotli/env0-base:latest"
+owns_lifecycle = false
+isolation      = "per_task"
+
+[environment.task_selection]
+mechanism = "image"
+
+[environment.readiness]
+timeout_sec = 60
+
+# Services below are copied VERBATIM from the upstream env-0.toml pin
+# (2026-08-09 sync). The previous pin had drifted: bare CLI names
+# (auth/gmail/...) that do not exist in ghcr.io/benchflow-ai/env0:0.2.0
+# images (which ship mock-auth/mock-gmail/... console scripts), plus a
+# phantom slack service and shifted ports — so ManifestEnvironment
+# probed, found ZERO startable services, and readiness passed vacuously
+# while every rollout died at the verifier with connection-refused.
+
+[[environment.services]]
+name    = "mock-auth"
+command = "mock-auth --db /data/auth.db serve --host 0.0.0.0 --port 9000 --no-mcp"
+port    = 9000
+
+
+[[environment.services]]
+name    = "mock-gcal"
+command = "mock-gcal --db /data/gcal.db serve --host 0.0.0.0 --port 9002 --no-mcp"
+port    = 9002
+
+[[environment.services]]
+name    = "mock-gdrive"
+command = "mock-gdrive --db /data/gdrive.db serve --host 0.0.0.0 --port 9003 --no-mcp"
+port    = 9003
+
+[[environment.services]]
+name    = "mock-gdoc"
+command = "mock-gdoc --db /data/gdoc.db serve --host 0.0.0.0 --port 9004 --no-mcp"
+port    = 9004
+
+
+[[environment.services]]
+name    = "mock-discord"
+command = "mock-discord --db /data/discord.db serve --host 0.0.0.0 --port 9006 --no-mcp"
+port    = 9006
+
+[[environment.services]]
+name    = "mock-stripe"
+command = "mock-stripe --db /data/stripe.db serve --host 0.0.0.0 --port 9007 --no-mcp"
+port    = 9007

benchflow-0.6.7.dev1840/src/benchflow/environment/_registry/env0@prod.toml

@@ -0,0 +1,77 @@
+# env0@prod — pinned, content-addressed Environment-plane manifest for env-0.
+#
+# This is a built-in registry entry, shipped inside the benchflow wheel
+# (benchflow/environment/_registry/) so PR #790's env-axis pin resolves on a
+# bare pip install — no checkout, no env vars:
+#
+#   bench eval create --tasks-dir <env0 tasks> --environment-manifest env0@prod ...
+#
+# Set $BENCHFLOW_ENV_REGISTRY to a local directory of name@version.toml files
+# to override the built-in registry entirely.
+#
+# Mirrors benchflow-ai/env0 tasks/_manifests/env-0.toml (the source of truth). env-0's per-task images build
+# FROM the shared base and bake seed data; benchflow's ManifestEnvironment
+# probes each service's CLI and starts only the ones installed in the per-task
+# image, gating the agent on each service's /health.
+#
+# NOTE: xdotli/env0-base:latest is amd64-only — run env0 on Daytona (x86_64).
+
+[environment]
+name           = "env-0"
+base_image     = "xdotli/env0-base:latest"
+owns_lifecycle = false
+isolation      = "per_task"
+
+[environment.task_selection]
+mechanism = "image"
+
+[environment.readiness]
+timeout_sec = 60
+
+# Services below are copied VERBATIM from the upstream env-0.toml pin
+# (2026-08-09 sync). The previous pin had drifted: bare CLI names
+# (auth/gmail/...) that do not exist in ghcr.io/benchflow-ai/env0:0.2.0
+# images (which ship mock-auth/mock-gmail/... console scripts), plus a
+# phantom slack service and shifted ports — so ManifestEnvironment
+# probed, found ZERO startable services, and readiness passed vacuously
+# while every rollout died at the verifier with connection-refused.
+
+[[environment.services]]
+name    = "mock-auth"
+command = "mock-auth --db /data/auth.db serve --host 0.0.0.0 --port 9000 --no-mcp"
+port    = 9000
+
+[[environment.services]]
+name    = "mock-gmail"
+command = "mock-gmail --db /data/gmail.db serve --host 0.0.0.0 --port 9001 --no-mcp"
+port    = 9001
+
+[[environment.services]]
+name    = "mock-gcal"
+command = "mock-gcal --db /data/gcal.db serve --host 0.0.0.0 --port 9002 --no-mcp"
+port    = 9002
+
+[[environment.services]]
+name    = "mock-gdrive"
+command = "mock-gdrive --db /data/gdrive.db serve --host 0.0.0.0 --port 9003 --no-mcp"
+port    = 9003
+
+[[environment.services]]
+name    = "mock-gdoc"
+command = "mock-gdoc --db /data/gdoc.db serve --host 0.0.0.0 --port 9004 --no-mcp"
+port    = 9004
+
+[[environment.services]]
+name    = "mock-slack"
+command = "mock-slack --db /data/slack.db serve --host 0.0.0.0 --port 9005 --no-mcp"
+port    = 9005
+
+[[environment.services]]
+name    = "mock-discord"
+command = "mock-discord --db /data/discord.db serve --host 0.0.0.0 --port 9006 --no-mcp"
+port    = 9006
+
+[[environment.services]]
+name    = "mock-stripe"
+command = "mock-stripe --db /data/stripe.db serve --host 0.0.0.0 --port 9007 --no-mcp"
+port    = 9007

{benchflow-0.6.7.dev1835 → benchflow-0.6.7.dev1840}/src/benchflow/environment/manifest.py

@@ -205,7 +205,9 @@ def load_manifest(path: str | Path) -> EnvironmentManifest:
     """Load and validate an environment manifest.
 
     ``path`` is either a TOML file path (the historical behavior) or a registry
-    spec ``name@version`` resolved via ``$BENCHFLOW_ENV_REGISTRY``. The spec form
+    spec ``name@version`` resolved via ``$BENCHFLOW_ENV_REGISTRY`` when set,
+    else the built-in registry shipped inside the wheel
+    (``benchflow/environment/_registry/``). The spec form
     lets a run bind its environment (the ``S`` axis) by name at the command line —
     decoupled from the task and swappable per run, like ``--agent`` / ``--model``
     / ``--sandbox``. Resolution is content-addressed so the bound world is