benchflow 0.7.8.dev2339__tar.gz → 0.7.8.dev2342__tar.gz

{benchflow-0.7.8.dev2339 → benchflow-0.7.8.dev2342}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.8.dev2339
+Version: 0.7.8.dev2342
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.8.dev2339 → benchflow-0.7.8.dev2342}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.8.dev2339"
+version = "0.7.8.dev2342"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.8.dev2339 → benchflow-0.7.8.dev2342}/src/benchflow/__init__.py

@@ -62,6 +62,7 @@ from benchflow.monitor import (
 )
 from benchflow.review import (
     PublicationDecision,
+    ReviewerConfig,
     ReviewReport,
     ReviewRubricError,
     ReviewScoring,
@@ -172,6 +173,7 @@ __all__ = [
     "load_rubric_json",
     "load_rubric_toml",
     "ReviewReport",
+    "ReviewerConfig",
     "PublicationDecision",
     "ReviewRubric",
     "ReviewRubricCriterion",

benchflow-0.7.8.dev2342/src/benchflow/_utils/config_redaction.py

@@ -0,0 +1,55 @@
+"""Dependency-neutral redaction policy for persisted execution configuration.
+
+Both solver and reviewer artifacts keep nonsecret execution settings so that
+scoring-only resume preserves the inference protocol without persisting auth.
+"""
+
+from __future__ import annotations
+
+from benchflow.trajectories.types import redact_trajectory_text
+
+# Substrings that flag an env var name as secret-bearing for ``config.json``
+# redaction. Matching is case-insensitive (callers ``.upper()`` the key first)
+# and uses substring containment so derived names like ``MY_AUTH_HEADER``,
+# ``SESSION_COOKIE``, or ``GH_TOKEN`` are caught. This stays a denylist (rather
+# than an allowlist of safe keys) because agent env varies per agent — the
+# union of safe keys is not knowable here — but the list now covers the common
+# auth-bearing names that issue #410 called out (COOKIE, AUTHORIZATION, AUTH,
+# BEARER, SESSION) on top of the original KEY/TOKEN/SECRET/PASSWORD/CREDENTIALS.
+_SECRET_ENV_SUBSTRINGS: tuple[str, ...] = (
+    "KEY",
+    "TOKEN",
+    "SECRET",
+    "PASSWORD",
+    "CREDENTIALS",
+    "COOKIE",
+    "AUTHORIZATION",
+    "AUTH",
+    "BEARER",
+    "SESSION",
+)
+_SECRET_URL_PATH_MARKERS: tuple[str, ...] = ("/__benchflow/",)
+
+
+def _is_secret_env_key(name: str) -> bool:
+    """Return True if *name* looks like it carries a secret value.
+
+    Case-insensitive substring match against :data:`_SECRET_ENV_SUBSTRINGS`.
+    Used by :func:`_write_config` to drop secret-bearing entries before
+    persisting ``agent_env`` to the rollout's ``config.json``.
+    """
+    upper = name.upper()
+    return any(s in upper for s in _SECRET_ENV_SUBSTRINGS)
+
+
+def _is_secret_env_value(name: str, value: str) -> bool:
+    """Return True if a normally public env value embeds a runtime secret."""
+    # Inline harness JSON/TOML can contain credentials under a harmless env
+    # name. Drop the whole value; redacted placeholders cannot safely replay.
+    return any(marker in value for marker in _SECRET_URL_PATH_MARKERS) or (
+        redact_trajectory_text(value) != value
+    )
+
+
+def _should_record_env_entry(name: str, value: str) -> bool:
+    return not _is_secret_env_key(name) and not _is_secret_env_value(name, value)

{benchflow-0.7.8.dev2339 → benchflow-0.7.8.dev2342}/src/benchflow/_utils/evaluation_results.py

@@ -75,6 +75,12 @@ def rollout_result_payload(
         "task_name": result.task_name,
         "rollout_name": result.rollout_name,
         "rewards": result.rewards,
+        **({"scoring": result.scoring.to_dict()} if result.scoring is not None else {}),
+        **(
+            {"purpose": result.purpose, "parent_rollout": result.parent_rollout}
+            if result.purpose != "task"
+            else {}
+        ),
         "error": result.error,
         "error_category": result.error_category,
         "verifier_error": result.verifier_error,

benchflow-0.7.8.dev2342/src/benchflow/_utils/result_paths.py

@@ -0,0 +1,74 @@
+"""Discover task results without counting nested review/evidence runs."""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def iter_task_result_paths(root: Path) -> list[Path]:
+    """Return result files at trial boundaries, excluding reviewer children.
+
+    A result below a trial's result or solver checkpoint is an artifact of that
+    parent, not an additional trial. In particular, captured files must remain
+    excluded while the parent awaits review and has no final result yet. The
+    explicit role also covers orphaned reviewer runs. Corrupt task files are
+    left to callers' existing error handling.
+    """
+    paths = sorted(root.rglob("result.json"))
+    trial_dirs = {path.parent for path in paths}
+    trial_dirs.update(path.parent for path in root.rglob("solver.json"))
+    selected: list[Path] = []
+    for path in paths:
+        if any(parent in trial_dirs for parent in path.parent.parents):
+            continue
+        try:
+            result = json.loads(path.read_text())
+        except (OSError, ValueError):
+            result = None
+        if isinstance(result, dict) and result.get("purpose") == "reviewer":
+            continue
+        selected.append(path)
+    return selected
+
+
+def load_task_results(root: Path) -> dict[str, dict[str, Any]]:
+    """Load one durable result per task, preferring scored then newer trials.
+
+    Evaluation resume and scoring-only summary refresh must select identical
+    records. Malformed files remain logged and skipped as in evaluation resume;
+    a malformed reward envelope remains visible as an errored trial.
+    """
+    best: dict[str, tuple[tuple[bool, float, str], dict[str, Any]]] = {}
+    for path in iter_task_result_paths(root):
+        try:
+            result = json.loads(path.read_text())
+            task = result["task_name"]
+            rewards = result.get("rewards")
+            if (
+                rewards is None
+                and not result.get("verifier_error")
+                and result.get("scoring") is None
+            ):
+                continue
+            if rewards is not None and not isinstance(rewards, dict):
+                logger.warning(
+                    "Malformed rewards field in %s for task %r: "
+                    "expected dict or null, got %s %r — "
+                    "treating as no reward (task will count as errored)",
+                    path,
+                    task,
+                    type(rewards).__name__,
+                    rewards,
+                )
+            rank = (rewards is not None, path.stat().st_mtime, str(path))
+            previous = best.get(task)
+            if previous is None or rank >= previous[0]:
+                best[task] = (rank, result)
+        except (OSError, ValueError, TypeError, KeyError) as exc:
+            logger.debug("Skipping corrupt result file %s: %s", path, exc)
+    return {task: result for task, (_, result) in best.items()}

{benchflow-0.7.8.dev2339 → benchflow-0.7.8.dev2342}/src/benchflow/_utils/scoring.py

@@ -1,6 +1,7 @@
 """Pure scoring and classification helpers — no external dependencies."""
 
 import math
+from collections import Counter
 from collections.abc import Iterable, Mapping
 from typing import Any, Literal
 
@@ -110,6 +111,18 @@ ResultOutcome = Literal["passed", "failed", "errored", "verifier_errored", "unsc
 
 def extract_reward(result: Mapping[str, Any]) -> float | None:
     """Extract the reward value from a result dict, or None if absent."""
+    if result.get("scoring") is not None:
+        from benchflow.review.outcome import scoring_from_result
+
+        try:
+            scoring = scoring_from_result(result)
+        except ValueError:
+            return None
+        if scoring is None or scoring.status != "complete":
+            return None
+        rewards = scoring.numeric_rewards()
+        assert rewards is not None
+        return rewards["reward"]
     rewards = result.get("rewards")
     if not isinstance(rewards, dict):
         return None
@@ -283,10 +296,20 @@ def classify_result(
 def classify_score_outcome(result: Mapping[str, Any]) -> ScoreOutcome:
     """Classify a persisted result for score/invariant accounting.
 
-    This is the canonical terminal score view. It keeps the four score
-    buckets disjoint and gives an explicit reward or agent error precedence
-    over verifier evidence so ``EvaluationResult`` cannot double-count tasks.
+    Integrated review uses its explicit gate verdict. Legacy results keep
+    reward/agent-error precedence. Malformed or incomplete scoring is never a
+    capability failure or an accidental pass from a stale reward.
     """
+    if result.get("scoring") is not None:
+        from benchflow.review.outcome import scoring_from_result
+
+        try:
+            scoring = scoring_from_result(result)
+        except ValueError:
+            scoring = None
+        if scoring is not None and scoring.status == "complete":
+            return "passed" if scoring.passed else "failed"
+        return "errored" if result.get("error") else "verifier_errored"
     return classify_result(
         reward=extract_reward(result),
         error=result.get("error"),
@@ -309,6 +332,8 @@ def classify_audit_outcome(result: Mapping[str, Any]) -> ResultOutcome:
     """
     if result.get("verifier_error"):
         return "verifier_errored"
+    if result.get("scoring") is not None:
+        return classify_score_outcome(result)
     reward = extract_reward(result)
     if reward == 1.0:
         return "passed"
@@ -386,3 +411,46 @@ def pass_rate_excl_errors(*, passed: int, failed: int) -> float:
     """Pass rate excluding errored tasks."""
     completed = passed + failed
     return passed / completed if completed > 0 else 0.0
+
+
+def score_summary_fields(results: Iterable[Mapping[str, Any]]) -> dict[str, Any]:
+    """Scoring fields shared by initial evaluation and scoring-only resume.
+
+    Callers retain run configuration, solver usage, timing, and provenance; only
+    fields derived from the current terminal scoring evidence are rebuilt.
+    """
+    rows = list(results)
+    counts = count_audit_outcomes(rows)
+    error_categories: Counter[str] = Counter()
+    verifier_categories: Counter[str] = Counter()
+    for row in rows:
+        category = row.get("error_category") or classify_error(row.get("error"))
+        if category:
+            error_categories[category] += 1
+        verifier_category = row.get(
+            "verifier_error_category"
+        ) or classify_verifier_error(row.get("verifier_error"))
+        if verifier_category:
+            verifier_categories[verifier_category] += 1
+    ratio = pass_rate(passed=counts["passed"], total=len(rows))
+    scored_ratio = pass_rate_excl_errors(
+        passed=counts["passed"], failed=counts["failed"]
+    )
+    return {
+        "total": len(rows),
+        "passed": counts["passed"],
+        "failed": counts["failed"],
+        "errored": counts["errored"],
+        "verifier_errored": counts["verifier_errored"],
+        "pass": counts["passed"],
+        "fail": counts["failed"],
+        "error": counts["errored"],
+        "idle_timeout": error_categories.get(IDLE_TIMEOUT, 0),
+        "error_categories": dict(error_categories) or None,
+        "verifier_error_categories": dict(verifier_categories) or None,
+        "score": f"{ratio:.1%}",
+        "score_ratio": ratio,
+        "score_excl_errors": f"{scored_ratio:.1%}",
+        "score_excl_errors_ratio": scored_ratio,
+        "mean_reward": mean_scored_reward(rows),
+    }

{benchflow-0.7.8.dev2339 → benchflow-0.7.8.dev2342}/src/benchflow/_utils/yaml_loader.py

@@ -44,6 +44,7 @@ from typing import Any
 import yaml
 
 from benchflow._types import Role, Scene, Turn
+from benchflow.review.options import ReviewerConfig
 from benchflow.rollout import RolloutConfig
 from benchflow.skill_policy import SKILL_MODE_NO_SKILL
 from benchflow.usage_tracking import UsageTrackingConfig
@@ -109,6 +110,7 @@ def rollout_config_from_dict(
 
     return RolloutConfig(
         task_path=tp,
+        reviewer=ReviewerConfig.coerce(raw.get("reviewer")),
         scenes=scenes,
         environment=raw.get("environment", "docker"),
         sandbox_user=raw.get("sandbox_user", "agent"),

{benchflow-0.7.8.dev2339 → benchflow-0.7.8.dev2342}/src/benchflow/agents/credentials.py

@@ -19,14 +19,63 @@ Does not own:
 import json
 import logging
 import os
+import posixpath
 import shlex
 import tempfile
-from pathlib import Path
+from collections.abc import Mapping
+from pathlib import Path, PurePosixPath
 
 from benchflow.agents.registry import AGENTS
 
 logger = logging.getLogger(__name__)
 
+# These runtime files can contain API keys or OAuth tokens. Keep this inventory
+# beside the credential writers, including files written by harness launchers
+# rather than upload_credential. Evidence capture records every exclusion.
+CREDENTIAL_EVIDENCE_PATHS = (
+    ".codex/auth.json",
+    ".claude/.credentials.json",
+    ".config/opencode/auth.json",
+    ".config/opencode/opencode.json",
+    ".local/share/opencode/auth.json",
+    ".config/mimocode/mimocode.json",
+    "mimocode.json",
+    ".openhands/agent_settings.json",
+    ".pi/agent/models.json",
+    ".pi/agent/auth.json",
+    ".openclaw/openclaw.json",
+    ".openclaw/agents/main/agent/auth-profiles.json",
+    ".gemini/oauth_creds.json",
+    ".aws/credentials",
+    ".config/gcloud/application_default_credentials.json",
+    ".ssh",
+)
+
+
+def credential_evidence_overrides(
+    agent_env: Mapping[str, str], *, workspace: str, cred_home: str
+) -> tuple[str, ...]:
+    """Resolve supported credential/config file overrides in sandbox paths.
+
+    Values are filenames, not credential JSON. In particular, CODEX_CONFIG is
+    inline JSON forwarded to the adapter and must never be treated as a path.
+    """
+    paths: set[str] = set()
+    for variable in (
+        "MIMOCODE_CONFIG",
+        "OPENCODE_CONFIG",
+        "OPENCLAW_CONFIG_PATH",
+        "GOOGLE_APPLICATION_CREDENTIALS",
+        "AWS_SHARED_CREDENTIALS_FILE",
+    ):
+        value = agent_env.get(variable)
+        if not value:
+            continue
+        if value.startswith("~/"):
+            value = str(PurePosixPath(cred_home) / value[2:])
+        paths.add(posixpath.normpath(str(PurePosixPath(workspace) / value)))
+    return tuple(sorted(paths))
+
 
 def _owner_from_home(cred_home: str) -> str | None:
     """Return sandbox username for /home/<user> credential homes."""

{benchflow-0.7.8.dev2339 → benchflow-0.7.8.dev2342}/src/benchflow/cli/_live_progress.py

@@ -264,13 +264,10 @@ class LiveEvalProgress:
     def on_result(self, name: str, result: RunResult) -> None:
         with self._lock:
             self._running.pop(name, None)
-            # Mirror Evaluation._log_and_report exactly: reward==1 -> PASS,
-            # reward not None -> FAIL, else ERR (no reward reached).
-            rewards = getattr(result, "rewards", None)
-            reward = rewards.get("reward") if rewards else None
-            if reward == 1:
+            outcome = result.score_outcome
+            if outcome == "passed":
                 self._passed += 1
-            elif reward is not None:
+            elif outcome == "failed":
                 self._failed += 1
             else:
                 self._errored += 1

{benchflow-0.7.8.dev2339 → benchflow-0.7.8.dev2342}/src/benchflow/cli/eval_artifacts.py

@@ -35,6 +35,7 @@ def _redacted_eval_config(eval_config: EvaluationConfig) -> dict:
         "skills_dir": eval_config.skills_dir,
         "usage_tracking": eval_config.usage_tracking.to_config_artifact(),
         "agent_env_keys": sorted(eval_config.agent_env),
+        "reviewer": eval_config.reviewer.to_config_artifact(),
         "include_tasks": sorted(eval_config.include_tasks),
         "exclude_tasks": sorted(eval_config.exclude_tasks),
         "source_provenance": eval_config.source_provenance,

{benchflow-0.7.8.dev2339 → benchflow-0.7.8.dev2342}/src/benchflow/cli/main.py

@@ -52,7 +52,20 @@ from benchflow.cli.eval_artifacts import postprocess_eval_artifacts, run_matrix_
 from benchflow.cli.eval_lift import register_eval_lift
 from benchflow.cli.hub import register_hub
 from benchflow.cli.monitor import register_monitor
+from benchflow.cli.rescore import register_eval_score
 from benchflow.cli.review import register_review
+from benchflow.cli.reviewer_options import (
+    ReviewerAgentOption,
+    ReviewerConcurrencyOption,
+    ReviewerEffortOption,
+    ReviewerEnvOption,
+    ReviewerImageOption,
+    ReviewerModelOption,
+    ReviewerNetworkOption,
+    ReviewerSandboxOption,
+    ReviewerTimeoutOption,
+    reviewer_from_cli,
+)
 from benchflow.cli.sandbox import register_sandbox
 from benchflow.cli.skills import register_skills
 from benchflow.cli.tasks import register_tasks
@@ -189,6 +202,7 @@ app.add_typer(eval_app, name="eval", rich_help_panel="Core")
 # entry point; adopt makes a foreign benchmark runnable).
 register_eval_adopt(eval_app)
 register_eval_lift(eval_app)
+register_eval_score(eval_app)
 
 
 @eval_app.command("run")
@@ -465,6 +479,15 @@ def eval_run(
             ),
         ),
     ] = None,
+    reviewer_agent: ReviewerAgentOption = None,
+    reviewer_model: ReviewerModelOption = None,
+    reviewer_reasoning_effort: ReviewerEffortOption = None,
+    reviewer_sandbox: ReviewerSandboxOption = None,
+    reviewer_timeout_sec: ReviewerTimeoutOption = None,
+    reviewer_concurrency: ReviewerConcurrencyOption = None,
+    reviewer_image: ReviewerImageOption = None,
+    reviewer_agent_env: ReviewerEnvOption = None,
+    reviewer_open_network: ReviewerNetworkOption = None,
     agent_env: Annotated[
         list[str] | None,
         typer.Option("--agent-env", help="Agent env var (KEY=VALUE)"),
@@ -660,6 +683,21 @@ def eval_run(
         self_gen_no_internet=self_gen_no_internet,
         loop_strategy=loop_strategy,
         agent_env=_parse_agent_env(agent_env),
+        reviewer=reviewer_from_cli(
+            agent=reviewer_agent,
+            model=reviewer_model,
+            reasoning_effort=reviewer_reasoning_effort,
+            environment=reviewer_sandbox,
+            timeout_sec=reviewer_timeout_sec,
+            concurrency=reviewer_concurrency,
+            image=reviewer_image,
+            agent_env=(
+                _parse_agent_env(reviewer_agent_env)
+                if reviewer_agent_env is not None
+                else None
+            ),
+            open_network=reviewer_open_network,
+        ),
         include=include,
         exclude=exclude,
         dataset=dataset,
@@ -934,6 +972,21 @@ def _run_config_file_eval(plan: "EvalPlan") -> None:
         if req.environment is not None:
             j._config.environment = plan.eval_environment
         j._config.agent_env = {**j._config.agent_env, **plan.parsed_env}
+        if req.reviewer is not None:
+            from benchflow.review.options import ReviewerConfig
+
+            reviewer_fields = req.reviewer.model_dump(exclude_unset=True)
+            if "agent_env" in reviewer_fields:
+                reviewer_fields["agent_env"] = {
+                    **j._config.reviewer.agent_env,
+                    **req.reviewer.agent_env,
+                }
+            j._config.reviewer = ReviewerConfig.model_validate(
+                {
+                    **j._config.reviewer.to_dict(),
+                    **reviewer_fields,
+                }
+            )
         j._config.sandbox_user = normalize_sandbox_user(j._config.sandbox_user)
         if req.jobs_dir is not None:
             j._jobs_dir = Path(req.jobs_dir)

benchflow-0.7.8.dev2342/src/benchflow/cli/rescore.py

@@ -0,0 +1,116 @@
+"""``bench eval score`` — finish rubric scoring without rerunning the solver."""
+
+import asyncio
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.markup import escape
+
+from benchflow._utils.result_paths import load_task_results
+from benchflow._utils.scoring import score_summary_fields
+from benchflow.cli._shared import (
+    _apply_dotenv_to_process_env,
+    _parse_agent_env,
+    console,
+)
+from benchflow.cli.reviewer_options import (
+    ReviewerAgentOption,
+    ReviewerConcurrencyOption,
+    ReviewerEffortOption,
+    ReviewerEnvOption,
+    ReviewerImageOption,
+    ReviewerModelOption,
+    ReviewerNetworkOption,
+    ReviewerSandboxOption,
+    ReviewerTimeoutOption,
+    reviewer_from_cli,
+)
+from benchflow.review.outcome import scoring_from_result
+from benchflow.review.persistence import write_json_atomic
+from benchflow.review.resume import resume_review
+from benchflow.trajectories.results import write_job_results_jsonl
+
+
+def _refresh_job_artifacts(job_dir: Path) -> None:
+    """Rebuild derived exports while retaining job configuration and provenance."""
+    import json
+
+    from benchflow.trajectories.export import write_job_verifiers_jsonl
+    from benchflow.trajectories.export_adp import write_job_adp_jsonl
+
+    write_job_results_jsonl(job_dir)
+    write_job_verifiers_jsonl(job_dir)
+    write_job_adp_jsonl(job_dir)
+    summary_path = job_dir / "summary.json"
+    if not summary_path.is_file():
+        return
+    summary = json.loads(summary_path.read_text())
+    summary.update(score_summary_fields(load_task_results(job_dir).values()))
+    write_json_atomic(summary_path, summary)
+    root_summary_path = job_dir.parent / "summary.json"
+    if root_summary_path.is_file():
+        root_summary = json.loads(root_summary_path.read_text())
+        if root_summary.get("job_name") == summary.get("job_name"):
+            write_json_atomic(root_summary_path, summary)
+
+
+def eval_score(
+    path: Annotated[Path, typer.Argument(help="A saved task rollout directory")],
+    tasks_root: Annotated[
+        Path,
+        typer.Option(
+            "--tasks-root", help="Trusted original task directory or collection"
+        ),
+    ],
+    force: Annotated[
+        bool,
+        typer.Option(
+            "--force", help="Create a new revision even if scoring already completed"
+        ),
+    ] = False,
+    reviewer_agent: ReviewerAgentOption = None,
+    reviewer_model: ReviewerModelOption = None,
+    reviewer_reasoning_effort: ReviewerEffortOption = None,
+    reviewer_sandbox: ReviewerSandboxOption = None,
+    reviewer_timeout_sec: ReviewerTimeoutOption = None,
+    reviewer_concurrency: ReviewerConcurrencyOption = None,
+    reviewer_image: ReviewerImageOption = None,
+    reviewer_agent_env: ReviewerEnvOption = None,
+    reviewer_open_network: ReviewerNetworkOption = None,
+) -> None:
+    """Retry a failed reviewer using the saved workspace and trajectory."""
+    _apply_dotenv_to_process_env()
+    reviewer = reviewer_from_cli(
+        agent=reviewer_agent,
+        model=reviewer_model,
+        reasoning_effort=reviewer_reasoning_effort,
+        environment=reviewer_sandbox,
+        timeout_sec=reviewer_timeout_sec,
+        concurrency=reviewer_concurrency,
+        image=reviewer_image,
+        agent_env=_parse_agent_env(reviewer_agent_env) if reviewer_agent_env else None,
+        open_network=reviewer_open_network,
+    )
+    try:
+        result = asyncio.run(
+            resume_review(path, tasks_root=tasks_root, reviewer=reviewer, force=force)
+        )
+        _refresh_job_artifacts(path.resolve().parent)
+        scoring = scoring_from_result(result)
+    except (ValueError, RuntimeError, OSError) as exc:
+        console.print(f"[red]Scoring failed: {escape(str(exc))}[/red]")
+        raise typer.Exit(1) from exc
+    if scoring is None or scoring.status != "complete":
+        reason = scoring.error if scoring is not None else "No scoring verdict"
+        console.print(f"[red]Scoring incomplete: {escape(str(reason))}[/red]")
+        raise typer.Exit(1)
+    rewards = scoring.numeric_rewards()
+    assert rewards is not None
+    console.print(
+        f"Scoring complete: passed={scoring.passed}, reward={rewards['reward']:.3f}"
+    )
+
+
+def register_eval_score(eval_app: typer.Typer) -> None:
+    eval_app.command("score")(eval_score)

benchflow-0.7.8.dev2342/src/benchflow/cli/reviewer_options.py

@@ -0,0 +1,62 @@
+"""Reviewer flags for evaluation and scoring commands."""
+
+from typing import Annotated
+
+import typer
+
+from benchflow.review.options import ReviewerConfig
+
+ReviewerAgentOption = Annotated[
+    str | None, typer.Option("--reviewer-agent", help="Rubric reviewer harness")
+]
+ReviewerModelOption = Annotated[
+    str | None, typer.Option("--reviewer-model", help="Rubric reviewer model")
+]
+ReviewerEffortOption = Annotated[
+    str | None,
+    typer.Option(
+        "--reviewer-reasoning-effort", help="Rubric reviewer reasoning effort"
+    ),
+]
+ReviewerSandboxOption = Annotated[
+    str | None,
+    typer.Option("--reviewer-sandbox", help="Rubric reviewer sandbox backend"),
+]
+ReviewerTimeoutOption = Annotated[
+    int | None,
+    typer.Option(
+        "--reviewer-timeout-sec", min=1, help="Reviewer execution budget in seconds"
+    ),
+]
+ReviewerConcurrencyOption = Annotated[
+    int | None,
+    typer.Option(
+        "--reviewer-concurrency", min=1, help="Max concurrent rubric reviewers"
+    ),
+]
+ReviewerImageOption = Annotated[
+    str | None, typer.Option("--reviewer-image", help="Rubric reviewer container image")
+]
+ReviewerEnvOption = Annotated[
+    list[str] | None,
+    typer.Option(
+        "--reviewer-agent-env", help="Reviewer environment variable (KEY=VALUE)"
+    ),
+]
+ReviewerNetworkOption = Annotated[
+    bool | None,
+    typer.Option(
+        "--reviewer-open-network", help="Allow unrestricted reviewer network access"
+    ),
+]
+
+
+def reviewer_from_cli(**options: object) -> ReviewerConfig | None:
+    """Retain only explicit overrides, preserving a YAML reviewer's other fields."""
+    supplied = {key: value for key, value in options.items() if value is not None}
+    if not supplied:
+        return None
+    try:
+        return ReviewerConfig.model_validate(supplied)
+    except ValueError as exc:
+        raise typer.BadParameter(str(exc), param_hint="reviewer options") from exc

{benchflow-0.7.8.dev2339 → benchflow-0.7.8.dev2342}/src/benchflow/contracts/planes.py

@@ -127,6 +127,7 @@ class RolloutPlanes(Protocol):
     async def execute_prompts_session_factory(
         self, *args: Any, **kwargs: Any
     ) -> Any: ...
+    async def quiesce_agent(self, env: Any, sandbox_user: str) -> None: ...
     async def harden_before_verify(self, *args: Any, **kwargs: Any) -> None: ...
     def verifier(self, *args: Any, **kwargs: Any) -> Any: ...
     async def clear_verifier_output_dir(self, *args: Any, **kwargs: Any) -> None: ...