benchflow 0.7.7.dev2316__tar.gz → 0.7.8__tar.gz

{benchflow-0.7.7.dev2316 → benchflow-0.7.8}/CHANGELOG.md

@@ -2,6 +2,46 @@
 
 ## [Unreleased]
 
+## 0.7.8 — 2026-09-14
+
+### Added
+
+- **Automatic rubric review and scoring.** Tasks with a weighted `rubric.json`
+  run a separate reviewer after verification. Passing requires all required
+  tests and rubric blockers to pass; the final reward is the weighted quality
+  score. Solver evidence and reviewer runs are retained, and `bench eval score`
+  can retry an interrupted review without rerunning the solver. (#1126)
+- **Rubric results in the trajectory viewer.** The Rubric tab displays criterion
+  judgments, explanations, blocker outcomes, and weighted scores. (#1102)
+- **Viewer themes and syntax highlighting.** Trajectory pages support a dark
+  theme and highlighted tool content. (#1098)
+
+### Fixed
+
+- **Automatic rubric review works on Daytona when the task workspace is
+  `/root`.** Daytona's daemon keeps its session tree in `/root/.daytona` (the
+  entrypoint's FIFOs plus every session command's script, log and exit code),
+  and evidence capture aborted on the first FIFO, so the rollout ended in a
+  scoring error with `rewards: null` although its verifier had run. Capture now
+  leaves that tree out as `sandbox_runtime` and records any other socket, FIFO
+  or device node as a `special_file` exclusion instead of aborting. Capture
+  limits, escaping symlinks and concurrent changes still fail closed. (#1128)
+- **Automatic review supports shell-only task images.** Required Python
+  capture tools are provisioned before the solver starts, so a successful
+  shell task does not lose its review to a missing interpreter. (#1127)
+- **Denylist enforcement closes CONNECT and HTTP Host bypasses.** TLS identity
+  and HTTP authority are checked against the allowed destination on Docker and
+  Daytona. (#1122)
+- **IPv6 egress rules are installed when procfs reports zero-size files.**
+  Sandbox firewall setup reads the kernel data instead of relying on the
+  reported file size. (#1124)
+- **Buffered ACP prompts respect their deadlines.** Timeout handling no longer
+  repeats captured provider history. (#1125)
+- **Tool captures retain their content and raw input/output.** ACP trajectories
+  preserve observations needed for later review. (#1100)
+
+## 0.7.7 — 2026-09-09
+
 ### Added
 - **`network_mode: denylist` blocks a list of URLs and hosts for the agent on
   Docker and Daytona.** The task keeps internet access; `blocked_urls` and
@@ -10,6 +50,29 @@
   and every refused request lands in `trajectory/egress_denylist.jsonl`.
   Other backends refuse the mode at preflight. (#1113)
 
+### Fixed
+
+- **Denylist egress no longer cuts the agent off from its own model.** The
+  controller registers the running LiteLLM gateway's exact
+  `127.0.0.1:<port>` endpoint with the proxy, so clients that ignore
+  `NO_PROXY` (Gemini's Undici `ProxyAgent` tunnels even plain HTTP) still
+  reach it; the exception comes from the live gateway, never task metadata or
+  agent-supplied environment, and every other private destination stays
+  blocked. Reconnects re-register the current port. (#1118)
+- **The denylist is a shared sandbox policy, not a per-harness one.** Every
+  supported ACP harness — custom registrations included — gets the same proxy,
+  certificates, and UID firewall on primary connections and later roles alike,
+  independent of harness name, model id, or provider. (#1118)
+- **`codex-acp` sessions are configured through native ACP settings.** Hosted
+  search is switched off via `CODEX_CONFIG.web_search` (the CLI ignores `-c`
+  overrides), and Codex runs in `agent-full-access` session mode when
+  BenchFlow has already selected a non-root sandbox user, so its bubblewrap
+  sandbox is not nested inside Docker or Daytona — BenchFlow's own user,
+  filesystem restrictions, proxy, and firewall still apply. (#1118)
+- **`cryptography>=44` is a core dependency.** Denylist egress mints TLS
+  certificates on both Docker and Daytona, so the pin moved out of the
+  `sandbox-agentcore` extra. (#1118)
+
 ## 0.7.6 — 2026-09-04
 
 ### Added

{benchflow-0.7.7.dev2316 → benchflow-0.7.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.7.dev2316
+Version: 0.7.8
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.7.dev2316 → benchflow-0.7.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.7.dev2316"
+version = "0.7.8"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.7.dev2316 → benchflow-0.7.8}/src/benchflow/__init__.py

@@ -62,6 +62,7 @@ from benchflow.monitor import (
 )
 from benchflow.review import (
     PublicationDecision,
+    ReviewerConfig,
     ReviewReport,
     ReviewRubricError,
     ReviewScoring,
@@ -172,6 +173,7 @@ __all__ = [
     "load_rubric_json",
     "load_rubric_toml",
     "ReviewReport",
+    "ReviewerConfig",
     "PublicationDecision",
     "ReviewRubric",
     "ReviewRubricCriterion",

benchflow-0.7.8/src/benchflow/_utils/config_redaction.py

@@ -0,0 +1,55 @@
+"""Dependency-neutral redaction policy for persisted execution configuration.
+
+Both solver and reviewer artifacts keep nonsecret execution settings so that
+scoring-only resume preserves the inference protocol without persisting auth.
+"""
+
+from __future__ import annotations
+
+from benchflow.trajectories.types import redact_trajectory_text
+
+# Substrings that flag an env var name as secret-bearing for ``config.json``
+# redaction. Matching is case-insensitive (callers ``.upper()`` the key first)
+# and uses substring containment so derived names like ``MY_AUTH_HEADER``,
+# ``SESSION_COOKIE``, or ``GH_TOKEN`` are caught. This stays a denylist (rather
+# than an allowlist of safe keys) because agent env varies per agent — the
+# union of safe keys is not knowable here — but the list now covers the common
+# auth-bearing names that issue #410 called out (COOKIE, AUTHORIZATION, AUTH,
+# BEARER, SESSION) on top of the original KEY/TOKEN/SECRET/PASSWORD/CREDENTIALS.
+_SECRET_ENV_SUBSTRINGS: tuple[str, ...] = (
+    "KEY",
+    "TOKEN",
+    "SECRET",
+    "PASSWORD",
+    "CREDENTIALS",
+    "COOKIE",
+    "AUTHORIZATION",
+    "AUTH",
+    "BEARER",
+    "SESSION",
+)
+_SECRET_URL_PATH_MARKERS: tuple[str, ...] = ("/__benchflow/",)
+
+
+def _is_secret_env_key(name: str) -> bool:
+    """Return True if *name* looks like it carries a secret value.
+
+    Case-insensitive substring match against :data:`_SECRET_ENV_SUBSTRINGS`.
+    Used by :func:`_write_config` to drop secret-bearing entries before
+    persisting ``agent_env`` to the rollout's ``config.json``.
+    """
+    upper = name.upper()
+    return any(s in upper for s in _SECRET_ENV_SUBSTRINGS)
+
+
+def _is_secret_env_value(name: str, value: str) -> bool:
+    """Return True if a normally public env value embeds a runtime secret."""
+    # Inline harness JSON/TOML can contain credentials under a harmless env
+    # name. Drop the whole value; redacted placeholders cannot safely replay.
+    return any(marker in value for marker in _SECRET_URL_PATH_MARKERS) or (
+        redact_trajectory_text(value) != value
+    )
+
+
+def _should_record_env_entry(name: str, value: str) -> bool:
+    return not _is_secret_env_key(name) and not _is_secret_env_value(name, value)

{benchflow-0.7.7.dev2316 → benchflow-0.7.8}/src/benchflow/_utils/evaluation_results.py

@@ -75,6 +75,12 @@ def rollout_result_payload(
         "task_name": result.task_name,
         "rollout_name": result.rollout_name,
         "rewards": result.rewards,
+        **({"scoring": result.scoring.to_dict()} if result.scoring is not None else {}),
+        **(
+            {"purpose": result.purpose, "parent_rollout": result.parent_rollout}
+            if result.purpose != "task"
+            else {}
+        ),
         "error": result.error,
         "error_category": result.error_category,
         "verifier_error": result.verifier_error,

benchflow-0.7.8/src/benchflow/_utils/result_paths.py

@@ -0,0 +1,74 @@
+"""Discover task results without counting nested review/evidence runs."""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def iter_task_result_paths(root: Path) -> list[Path]:
+    """Return result files at trial boundaries, excluding reviewer children.
+
+    A result below a trial's result or solver checkpoint is an artifact of that
+    parent, not an additional trial. In particular, captured files must remain
+    excluded while the parent awaits review and has no final result yet. The
+    explicit role also covers orphaned reviewer runs. Corrupt task files are
+    left to callers' existing error handling.
+    """
+    paths = sorted(root.rglob("result.json"))
+    trial_dirs = {path.parent for path in paths}
+    trial_dirs.update(path.parent for path in root.rglob("solver.json"))
+    selected: list[Path] = []
+    for path in paths:
+        if any(parent in trial_dirs for parent in path.parent.parents):
+            continue
+        try:
+            result = json.loads(path.read_text())
+        except (OSError, ValueError):
+            result = None
+        if isinstance(result, dict) and result.get("purpose") == "reviewer":
+            continue
+        selected.append(path)
+    return selected
+
+
+def load_task_results(root: Path) -> dict[str, dict[str, Any]]:
+    """Load one durable result per task, preferring scored then newer trials.
+
+    Evaluation resume and scoring-only summary refresh must select identical
+    records. Malformed files remain logged and skipped as in evaluation resume;
+    a malformed reward envelope remains visible as an errored trial.
+    """
+    best: dict[str, tuple[tuple[bool, float, str], dict[str, Any]]] = {}
+    for path in iter_task_result_paths(root):
+        try:
+            result = json.loads(path.read_text())
+            task = result["task_name"]
+            rewards = result.get("rewards")
+            if (
+                rewards is None
+                and not result.get("verifier_error")
+                and result.get("scoring") is None
+            ):
+                continue
+            if rewards is not None and not isinstance(rewards, dict):
+                logger.warning(
+                    "Malformed rewards field in %s for task %r: "
+                    "expected dict or null, got %s %r — "
+                    "treating as no reward (task will count as errored)",
+                    path,
+                    task,
+                    type(rewards).__name__,
+                    rewards,
+                )
+            rank = (rewards is not None, path.stat().st_mtime, str(path))
+            previous = best.get(task)
+            if previous is None or rank >= previous[0]:
+                best[task] = (rank, result)
+        except (OSError, ValueError, TypeError, KeyError) as exc:
+            logger.debug("Skipping corrupt result file %s: %s", path, exc)
+    return {task: result for task, (_, result) in best.items()}

{benchflow-0.7.7.dev2316 → benchflow-0.7.8}/src/benchflow/_utils/scoring.py

@@ -1,6 +1,7 @@
 """Pure scoring and classification helpers — no external dependencies."""
 
 import math
+from collections import Counter
 from collections.abc import Iterable, Mapping
 from typing import Any, Literal
 
@@ -110,6 +111,18 @@ ResultOutcome = Literal["passed", "failed", "errored", "verifier_errored", "unsc
 
 def extract_reward(result: Mapping[str, Any]) -> float | None:
     """Extract the reward value from a result dict, or None if absent."""
+    if result.get("scoring") is not None:
+        from benchflow.review.outcome import scoring_from_result
+
+        try:
+            scoring = scoring_from_result(result)
+        except ValueError:
+            return None
+        if scoring is None or scoring.status != "complete":
+            return None
+        rewards = scoring.numeric_rewards()
+        assert rewards is not None
+        return rewards["reward"]
     rewards = result.get("rewards")
     if not isinstance(rewards, dict):
         return None
@@ -283,10 +296,20 @@ def classify_result(
 def classify_score_outcome(result: Mapping[str, Any]) -> ScoreOutcome:
     """Classify a persisted result for score/invariant accounting.
 
-    This is the canonical terminal score view. It keeps the four score
-    buckets disjoint and gives an explicit reward or agent error precedence
-    over verifier evidence so ``EvaluationResult`` cannot double-count tasks.
+    Integrated review uses its explicit gate verdict. Legacy results keep
+    reward/agent-error precedence. Malformed or incomplete scoring is never a
+    capability failure or an accidental pass from a stale reward.
     """
+    if result.get("scoring") is not None:
+        from benchflow.review.outcome import scoring_from_result
+
+        try:
+            scoring = scoring_from_result(result)
+        except ValueError:
+            scoring = None
+        if scoring is not None and scoring.status == "complete":
+            return "passed" if scoring.passed else "failed"
+        return "errored" if result.get("error") else "verifier_errored"
     return classify_result(
         reward=extract_reward(result),
         error=result.get("error"),
@@ -309,6 +332,8 @@ def classify_audit_outcome(result: Mapping[str, Any]) -> ResultOutcome:
     """
     if result.get("verifier_error"):
         return "verifier_errored"
+    if result.get("scoring") is not None:
+        return classify_score_outcome(result)
     reward = extract_reward(result)
     if reward == 1.0:
         return "passed"
@@ -386,3 +411,46 @@ def pass_rate_excl_errors(*, passed: int, failed: int) -> float:
     """Pass rate excluding errored tasks."""
     completed = passed + failed
     return passed / completed if completed > 0 else 0.0
+
+
+def score_summary_fields(results: Iterable[Mapping[str, Any]]) -> dict[str, Any]:
+    """Scoring fields shared by initial evaluation and scoring-only resume.
+
+    Callers retain run configuration, solver usage, timing, and provenance; only
+    fields derived from the current terminal scoring evidence are rebuilt.
+    """
+    rows = list(results)
+    counts = count_audit_outcomes(rows)
+    error_categories: Counter[str] = Counter()
+    verifier_categories: Counter[str] = Counter()
+    for row in rows:
+        category = row.get("error_category") or classify_error(row.get("error"))
+        if category:
+            error_categories[category] += 1
+        verifier_category = row.get(
+            "verifier_error_category"
+        ) or classify_verifier_error(row.get("verifier_error"))
+        if verifier_category:
+            verifier_categories[verifier_category] += 1
+    ratio = pass_rate(passed=counts["passed"], total=len(rows))
+    scored_ratio = pass_rate_excl_errors(
+        passed=counts["passed"], failed=counts["failed"]
+    )
+    return {
+        "total": len(rows),
+        "passed": counts["passed"],
+        "failed": counts["failed"],
+        "errored": counts["errored"],
+        "verifier_errored": counts["verifier_errored"],
+        "pass": counts["passed"],
+        "fail": counts["failed"],
+        "error": counts["errored"],
+        "idle_timeout": error_categories.get(IDLE_TIMEOUT, 0),
+        "error_categories": dict(error_categories) or None,
+        "verifier_error_categories": dict(verifier_categories) or None,
+        "score": f"{ratio:.1%}",
+        "score_ratio": ratio,
+        "score_excl_errors": f"{scored_ratio:.1%}",
+        "score_excl_errors_ratio": scored_ratio,
+        "mean_reward": mean_scored_reward(rows),
+    }

{benchflow-0.7.7.dev2316 → benchflow-0.7.8}/src/benchflow/_utils/yaml_loader.py

@@ -44,6 +44,7 @@ from typing import Any
 import yaml
 
 from benchflow._types import Role, Scene, Turn
+from benchflow.review.options import ReviewerConfig
 from benchflow.rollout import RolloutConfig
 from benchflow.skill_policy import SKILL_MODE_NO_SKILL
 from benchflow.usage_tracking import UsageTrackingConfig
@@ -109,6 +110,7 @@ def rollout_config_from_dict(
 
     return RolloutConfig(
         task_path=tp,
+        reviewer=ReviewerConfig.coerce(raw.get("reviewer")),
         scenes=scenes,
         environment=raw.get("environment", "docker"),
         sandbox_user=raw.get("sandbox_user", "agent"),

{benchflow-0.7.7.dev2316 → benchflow-0.7.8}/src/benchflow/acp/client.py

@@ -1,5 +1,6 @@
 """ACP client — benchflow acts as the client, agents are ACP servers."""
 
+import asyncio
 import logging
 from collections.abc import Awaitable, Callable
 from typing import Any
@@ -137,6 +138,10 @@ class ACPClient:
     async def _read_until_response(self, request_id: int) -> dict[str, Any]:
         """Read messages, handling notifications, until we get the response we want."""
         while True:
+            # Buffered transports can receive inline, and session capture is
+            # synchronous. Give cancellation and watchdog tasks a turn even
+            # while the peer's notification backlog remains non-empty.
+            await asyncio.sleep(0)
             msg = await self._transport.receive()
             logger.debug(
                 f"ACPClient recv: id={msg.get('id')} method={msg.get('method', '')} "

{benchflow-0.7.7.dev2316 → benchflow-0.7.8}/src/benchflow/acp/runtime.py

@@ -30,7 +30,7 @@ from benchflow.acp.timeout_cleanup import (
     cancel_and_drain_prompt_task,
     cancel_prompt_after_timeout,
 )
-from benchflow.acp.types import McpServerSpec
+from benchflow.acp.types import McpServerSpec, PromptResult
 from benchflow.acp.watchdog import IdleWatchdog
 from benchflow.agents.codex_config import apply_codex_launch_config
 from benchflow.agents.protocol import ACPSessionAdapter
@@ -800,6 +800,28 @@ def _agent_prompt_timeout_error(session, timeout: int) -> AgentPromptTimeoutErro
     )
 
 
+async def _timed_prompt(
+    acp_client: ACPClient, prompt: str
+) -> tuple[PromptResult, float]:
+    """Record completion in the prompt task, before a delayed observer runs."""
+    result = await acp_client.prompt(prompt)
+    return result, asyncio.get_running_loop().time()
+
+
+def _completed_prompt_result(
+    prompt_task: asyncio.Task[tuple[PromptResult, float]],
+    session,
+    *,
+    deadline: float,
+    timeout: int,
+) -> PromptResult:
+    """A delayed watchdog must distinguish late completion from late observation."""
+    result, completed_at = prompt_task.result()
+    if completed_at > deadline:
+        raise _agent_prompt_timeout_error(session, timeout)
+    return result
+
+
 async def _prompt_with_wall_clock_budget(
     acp_client: ACPClient,
     session,
@@ -807,12 +829,15 @@ async def _prompt_with_wall_clock_budget(
     timeout: int,
 ):
     """Run a prompt until either it finishes or BenchFlow's budget expires."""
-    prompt_task = asyncio.create_task(acp_client.prompt(prompt))
+    deadline = asyncio.get_running_loop().time() + timeout
+    prompt_task = asyncio.create_task(_timed_prompt(acp_client, prompt))
     cleanup_attempted = False
     try:
         done, _pending = await asyncio.wait({prompt_task}, timeout=timeout)
         if done:
-            return prompt_task.result()
+            return _completed_prompt_result(
+                prompt_task, session, deadline=deadline, timeout=timeout
+            )
         cleanup_attempted = True
         if await cancel_prompt_after_timeout(acp_client, prompt_task):
             raise _agent_prompt_timeout_error(session, timeout)
@@ -831,7 +856,7 @@ async def _prompt_with_idle_watchdog(
     idle_timeout: int,
 ):
     """Run one ACP prompt with wall-clock and idle-watchdog budgets."""
-    prompt_task = asyncio.create_task(acp_client.prompt(prompt))
+    prompt_task = asyncio.create_task(_timed_prompt(acp_client, prompt))
     cleanup_attempted = False
     loop = asyncio.get_running_loop()
     watchdog = IdleWatchdog.start(
@@ -864,7 +889,9 @@ async def _prompt_with_idle_watchdog(
                     f"Agent prompt exceeded wall-clock budget {timeout}s"
                 )
 
-        return prompt_task.result()
+        return _completed_prompt_result(
+            prompt_task, session, deadline=watchdog.deadline, timeout=timeout
+        )
     finally:
         # Always cancel + drain the prompt task on exit, including the
         # external-cancellation path (CancelledError from sleep). Bound the

{benchflow-0.7.7.dev2316 → benchflow-0.7.8}/src/benchflow/acp/session.py

@@ -162,6 +162,10 @@ class ToolCallRecord:
         self.kind = kind
         self.status = ToolCallStatus.PENDING
         self.content: list[dict] = []
+        # ACP ``rawInput`` / ``rawOutput``: the agent's own view of the call.
+        # codex-acp puts the command and its output here and nowhere else.
+        self.raw_input: object | None = None
+        self.raw_output: object | None = None
         self.started_at = datetime.now()
         self.finished_at: datetime | None = None
 
@@ -178,6 +182,13 @@ class ToolCallRecord:
         ):
             self.finished_at = datetime.now()
 
+    def absorb_raw_io(self, update: dict) -> None:
+        """Keep the latest ``rawInput`` / ``rawOutput`` an update carries."""
+        if update.get("rawInput") is not None:
+            self.raw_input = update["rawInput"]
+        if update.get("rawOutput") is not None:
+            self.raw_output = update["rawOutput"]
+
 
 class ACPSession:
     """Tracks mutable state for one ACP session.
@@ -417,6 +428,20 @@ class ACPSession:
                     update.get("kind", "other"), update.get("title", "")
                 ),
             )
+            # The opening notification may already carry content (codex-acp
+            # sends file-change diffs this way) and raw I/O.
+            initial_content = update.get("content")
+            if isinstance(initial_content, list) and initial_content:
+                record.content.extend(initial_content)
+            record.absorb_raw_io(update)
+            # An opening call may already be terminal (codex-acp file edits
+            # arrive completed with no later update); honor its status so it
+            # never lingers in pending_tool_call_ids().
+            if update.get("status") is not None:
+                try:
+                    record.update_status(ToolCallStatus(update["status"]))
+                except ValueError:
+                    logger.warning(f"Unknown tool call status: {update.get('status')}")
             self._record_tool_call(record)
 
         elif update_type == "tool_call_update":
@@ -440,6 +465,7 @@ class ACPSession:
                 status = ToolCallStatus.IN_PROGRESS
             content = update.get("content")
             record.update_status(status, content)
+            record.absorb_raw_io(update)
             if status in (ToolCallStatus.PENDING, ToolCallStatus.IN_PROGRESS):
                 self._pending_tool_call_update_counts[tc_id] = (
                     self._pending_tool_call_update_counts.get(tc_id, 0) + 1

{benchflow-0.7.7.dev2316 → benchflow-0.7.8}/src/benchflow/agents/credentials.py

@@ -19,14 +19,63 @@ Does not own:
 import json
 import logging
 import os
+import posixpath
 import shlex
 import tempfile
-from pathlib import Path
+from collections.abc import Mapping
+from pathlib import Path, PurePosixPath
 
 from benchflow.agents.registry import AGENTS
 
 logger = logging.getLogger(__name__)
 
+# These runtime files can contain API keys or OAuth tokens. Keep this inventory
+# beside the credential writers, including files written by harness launchers
+# rather than upload_credential. Evidence capture records every exclusion.
+CREDENTIAL_EVIDENCE_PATHS = (
+    ".codex/auth.json",
+    ".claude/.credentials.json",
+    ".config/opencode/auth.json",
+    ".config/opencode/opencode.json",
+    ".local/share/opencode/auth.json",
+    ".config/mimocode/mimocode.json",
+    "mimocode.json",
+    ".openhands/agent_settings.json",
+    ".pi/agent/models.json",
+    ".pi/agent/auth.json",
+    ".openclaw/openclaw.json",
+    ".openclaw/agents/main/agent/auth-profiles.json",
+    ".gemini/oauth_creds.json",
+    ".aws/credentials",
+    ".config/gcloud/application_default_credentials.json",
+    ".ssh",
+)
+
+
+def credential_evidence_overrides(
+    agent_env: Mapping[str, str], *, workspace: str, cred_home: str
+) -> tuple[str, ...]:
+    """Resolve supported credential/config file overrides in sandbox paths.
+
+    Values are filenames, not credential JSON. In particular, CODEX_CONFIG is
+    inline JSON forwarded to the adapter and must never be treated as a path.
+    """
+    paths: set[str] = set()
+    for variable in (
+        "MIMOCODE_CONFIG",
+        "OPENCODE_CONFIG",
+        "OPENCLAW_CONFIG_PATH",
+        "GOOGLE_APPLICATION_CREDENTIALS",
+        "AWS_SHARED_CREDENTIALS_FILE",
+    ):
+        value = agent_env.get(variable)
+        if not value:
+            continue
+        if value.startswith("~/"):
+            value = str(PurePosixPath(cred_home) / value[2:])
+        paths.add(posixpath.normpath(str(PurePosixPath(workspace) / value)))
+    return tuple(sorted(paths))
+
 
 def _owner_from_home(cred_home: str) -> str | None:
     """Return sandbox username for /home/<user> credential homes."""

{benchflow-0.7.7.dev2316 → benchflow-0.7.8}/src/benchflow/cli/_live_progress.py

@@ -264,13 +264,10 @@ class LiveEvalProgress:
     def on_result(self, name: str, result: RunResult) -> None:
         with self._lock:
             self._running.pop(name, None)
-            # Mirror Evaluation._log_and_report exactly: reward==1 -> PASS,
-            # reward not None -> FAIL, else ERR (no reward reached).
-            rewards = getattr(result, "rewards", None)
-            reward = rewards.get("reward") if rewards else None
-            if reward == 1:
+            outcome = result.score_outcome
+            if outcome == "passed":
                 self._passed += 1
-            elif reward is not None:
+            elif outcome == "failed":
                 self._failed += 1
             else:
                 self._errored += 1

{benchflow-0.7.7.dev2316 → benchflow-0.7.8}/src/benchflow/cli/eval_artifacts.py

@@ -35,6 +35,7 @@ def _redacted_eval_config(eval_config: EvaluationConfig) -> dict:
         "skills_dir": eval_config.skills_dir,
         "usage_tracking": eval_config.usage_tracking.to_config_artifact(),
         "agent_env_keys": sorted(eval_config.agent_env),
+        "reviewer": eval_config.reviewer.to_config_artifact(),
         "include_tasks": sorted(eval_config.include_tasks),
         "exclude_tasks": sorted(eval_config.exclude_tasks),
         "source_provenance": eval_config.source_provenance,