benchflow 0.6.7.dev1831__tar.gz → 0.6.7.dev1833__tar.gz

{benchflow-0.6.7.dev1831 → benchflow-0.6.7.dev1833}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1831
+Version: 0.6.7.dev1833
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1831 → benchflow-0.6.7.dev1833}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1831"
+version = "0.6.7.dev1833"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1831 → benchflow-0.6.7.dev1833}/src/benchflow/_utils/evaluation_results.py

@@ -73,6 +73,7 @@ def rollout_result_payload(
     )
     return {
         "task_name": result.task_name,
+        "rollout_name": result.rollout_name,
         "rewards": result.rewards,
         "error": result.error,
         "error_category": result.error_category,

benchflow-0.6.7.dev1833/src/benchflow/cli/_failure_evidence.py

@@ -0,0 +1,122 @@
+"""Verifier-artifact mining for the eval report's failure lines.
+
+The one file-touching corner of the CLI's final report block: when a failed
+task's reason would otherwise be the bare ``reward X`` fallback, mine the
+rollout's verifier artifacts for a one-line explanation. Lives in its own
+module so ``cli/_shared.py`` stays the side-effect-free display helpers it
+advertises.
+
+Contract (the engine stays file-free — these reads are CLI-side, report-time
+only):
+
+- The rollout dir is resolved exactly, from the ``rollout_name`` the engine
+  records on every result — never guessed from globs.
+- Evidence sources are tried in order (CTRF report, then test-stdout tail);
+  the first that yields a line wins.
+- Every read is bounded (``_ARTIFACT_READ_BYTES`` per file) and nothing here
+  raises — any surprise degrades to ``None``, i.e. the bare reward reason.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from pathlib import Path
+
+# Never read more than this many bytes per file, and evidence is only mined
+# for the <= _MAX_FAILURE_LINES tasks the report block displays.
+_ARTIFACT_READ_BYTES = 64 * 1024
+# Pytest final summary, decoration stripped: "1 failed, 2 passed in 40.86s".
+_PYTEST_SUMMARY_RE = re.compile(r"\d+ failed\b")
+
+
+def _ctrf_failure_line(ctrf_path: Path) -> str | None:
+    """``<test> failed[: <assertion>]`` from the first failed CTRF test.
+
+    The verifier's pytest run writes a CTRF report (``pytest --ctrf``, see the
+    standard path in ``task_authoring/structural_checks.py``). Per test the
+    useful failure evidence is the ``E …`` assertion line inside ``trace``;
+    ``message`` is only a generic phase description, so it is the fallback.
+    """
+    if ctrf_path.stat().st_size > _ARTIFACT_READ_BYTES:
+        # JSON only parses whole — an oversized report is skipped (not
+        # truncated) and the next evidence source gets its turn.
+        return None
+    data = json.loads(ctrf_path.read_text(encoding="utf-8", errors="replace"))
+    tests = (data.get("results") or {}).get("tests") or []
+    for test in tests:
+        if not isinstance(test, dict) or test.get("status") != "failed":
+            continue
+        # Full pytest node ids ("tests/test_x.py::test_build") waste the
+        # 100-char line budget; keep the test function name.
+        name = str(test.get("name") or "test").rsplit("::", 1)[-1]
+        trace = test.get("trace")
+        if isinstance(trace, str):
+            for line in trace.splitlines():
+                stripped = line.strip()
+                if stripped.startswith("E "):
+                    assertion = stripped[2:].strip()
+                    # "<test> failed:" already says it's an assertion — drop
+                    # the prefix to reclaim line budget for the message.
+                    assertion = assertion.removeprefix("AssertionError: ")
+                    return f"{name} failed: {assertion}"
+        message = test.get("message")
+        if isinstance(message, str) and message.strip():
+            return f"{name} failed: {' '.join(message.split())}"
+        return f"{name} failed"
+    return None
+
+
+def _stdout_tail_failure_line(stdout_path: Path) -> str | None:
+    """Last pytest summary ("N failed, M passed …") — else last ``FAILED …``
+    line — from a bounded tail of the verifier's test-stdout capture."""
+    with stdout_path.open("rb") as fh:
+        fh.seek(0, 2)  # SEEK_END
+        fh.seek(max(0, fh.tell() - _ARTIFACT_READ_BYTES))
+        tail = fh.read(_ARTIFACT_READ_BYTES).decode("utf-8", errors="replace")
+    last_failed: str | None = None
+    for line in reversed(tail.splitlines()):
+        bare = line.strip().strip("=").strip()
+        if _PYTEST_SUMMARY_RE.match(bare):
+            return bare
+        if last_failed is None and line.strip().startswith("FAILED "):
+            last_failed = line.strip()
+    return last_failed
+
+
+def artifact_failure_evidence(
+    job_dir: Path, rollout_name: str
+) -> tuple[str, Path] | None:
+    """One-liner mined from the rollout's verifier artifacts, or ``None``.
+
+    Returns the one-liner plus the verifier dir it came from (for the report
+    block's single ``(details: …)`` pointer line). Never raises.
+    """
+    # Local import: RolloutPaths pulls in the task package, which the CLI
+    # shouldn't pay for until a failure actually needs artifact evidence.
+    from benchflow.task.paths import RolloutPaths
+
+    rollout_paths = RolloutPaths(rollout_dir=job_dir / rollout_name)
+    verifier_dir = rollout_paths.verifier_dir
+    attempts: list[tuple[Path, Callable[[Path], str | None]]] = [
+        # ctrf.json has no RolloutPaths property — the verifier recovers it by
+        # literal name (verifier_core.py, `_recover_main_verifier_outputs`).
+        (verifier_dir / "ctrf.json", _ctrf_failure_line),
+        (rollout_paths.test_stdout_path, _stdout_tail_failure_line),
+    ]
+    for path, extract in attempts:
+        try:
+            if not path.is_file():
+                continue
+            detail = extract(path)
+        except Exception:
+            # Unreadable file, bad JSON, permissions … try the next source;
+            # the report must never crash over evidence mining.
+            continue
+        if detail is not None:
+            return detail, verifier_dir
+    return None

{benchflow-0.6.7.dev1831 → benchflow-0.6.7.dev1833}/src/benchflow/cli/_shared.py

@@ -3,7 +3,9 @@
 These are the cross-cutting, side-effect-free helpers that several CLI command
 groups (``cli/main.py`` and the ``cli/<group>.py`` modules) need in common: the
 shared Rich :data:`console`, the evaluation-result summary/exit helpers, and the
-agent ``Requires`` rendering used by ``agents``/``agent`` listings.
+agent ``Requires`` rendering used by ``agents``/``agent`` listings. The one
+file-reading step of the eval report (mining verifier artifacts for failure
+evidence) is delegated to :mod:`._failure_evidence` to keep that claim true.
 
 Keeping them here lets each command group import one stable surface instead of
 re-deriving the formatting, and lets ``cli/main.py`` stay a thin app + eval
@@ -19,6 +21,7 @@ from rich.console import Console
 from rich.markup import escape
 
 from benchflow._utils.text import truncate_end
+from benchflow.cli._failure_evidence import artifact_failure_evidence
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -109,17 +112,24 @@ _FAILURE_LINE_LIMIT = 100
 _FAILURE_REASON_METRICS = 3
 
 
-def _failure_reason(failure: TaskFailure) -> str:
+def _failure_reason(
+    failure: TaskFailure, job_dir: Path | None = None
+) -> tuple[str, Path | None]:
     """One cheap line explaining why a FAILED (scored, reward != 1) task
-    failed, from evidence already on the result — no file reads.
+    failed, plus the verifier dir when artifacts supplied the evidence.
 
     Priority: the verifier's own error if set; else the reward plus a compact
     breakdown of the named metrics in the reward dict (zero/failed metrics
-    first — they explain the miss); else just the reward.
+    first — they explain the miss); else the reward plus a one-liner mined
+    from the rollout's verifier artifacts (bounded CLI-side reads resolved via
+    the ``rollout_name`` the engine records — the engine stays file-free);
+    else just the reward. The returned path is non-``None`` only for the
+    artifact tier, so the caller can print one ``(details: …)`` pointer per
+    report block.
     """
     if failure.verifier_error:
         # Collapse whitespace: verifier errors are routinely multi-line.
-        return " ".join(failure.verifier_error.split())
+        return " ".join(failure.verifier_error.split()), None
     rewards = failure.rewards or {}
     reward = rewards.get("reward")
     metrics = [
@@ -127,15 +137,22 @@ def _failure_reason(failure: TaskFailure) -> str:
         for name, value in rewards.items()
         if name != "reward" and isinstance(value, (bool, int, float))
     ]
-    if not metrics:
-        return f"reward {reward}"
-    # Zero/failed metrics first (stable within each group), capped so one
-    # metric-happy verifier can't flood the line.
-    metrics.sort(key=lambda kv: kv[1] != 0)
-    shown = ", ".join(
-        f"{name} {value}" for name, value in metrics[:_FAILURE_REASON_METRICS]
-    )
-    return f"reward {reward} — {shown}"
+    if metrics:
+        # Zero/failed metrics first (stable within each group), capped so one
+        # metric-happy verifier can't flood the line.
+        metrics.sort(key=lambda kv: kv[1] != 0)
+        shown = ", ".join(
+            f"{name} {value}" for name, value in metrics[:_FAILURE_REASON_METRICS]
+        )
+        return f"reward {reward} — {shown}", None
+    # No rollout_name (pre-#957-follow-up result.json, sharded aggregation):
+    # nothing to resolve — the bare reward is as honest as it gets.
+    if job_dir is not None and failure.rollout_name:
+        evidence = artifact_failure_evidence(job_dir, failure.rollout_name)
+        if evidence is not None:
+            detail, verifier_dir = evidence
+            return f"reward {reward} — {detail}", verifier_dir
+    return f"reward {reward}", None
 
 
 def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -> None:
@@ -145,9 +162,13 @@ def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -
     now the line is green only on a full clean pass, red on a shutout, amber
     otherwise, and ``errors=N`` is red when non-zero. Each FAILED task gets one
     dim ``✗ task: reason`` line (capped at ``_MAX_FAILURE_LINES``) so the "why"
-    doesn't require opening summary.json. When ``job_dir`` is given, the
-    result/summary paths are printed so testers know where to look (the guide
-    repeatedly says "read summary.json" but the CLI never said where).
+    doesn't require opening summary.json; when ``job_dir`` is given, a reason
+    that would otherwise be a bare ``reward X`` is upgraded from the rollout's
+    verifier artifacts (bounded reads, displayed failures only), with a single
+    ``(details: …/verifier)`` pointer line for the block. When ``job_dir`` is
+    given, the result/summary paths are printed so testers know where to look
+    (the guide repeatedly says "read summary.json" but the CLI never said
+    where).
     """
     errors = int(getattr(result, "errored", 0) or 0)
     verifier_errors = int(getattr(result, "verifier_errored", 0) or 0)
@@ -176,15 +197,23 @@ def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -
     # summary.json to learn why. getattr(): sharded aggregation and older
     # SimpleNamespace-style callers don't carry task_failures.
     failures = getattr(result, "task_failures", None) or []
+    artifact_pointer: Path | None = None
     for failure in failures[:_MAX_FAILURE_LINES]:
+        reason, verifier_dir = _failure_reason(failure, job_dir)
+        if artifact_pointer is None and verifier_dir is not None:
+            artifact_pointer = verifier_dir
         line = truncate_end(
-            f"  ✗ {failure.task_name}: {_failure_reason(failure)}",
+            f"  ✗ {failure.task_name}: {reason}",
             _FAILURE_LINE_LIMIT,
         )
         console.print(f"[dim]{escape(line)}[/dim]")
     extra = len(failures) - _MAX_FAILURE_LINES
     if extra > 0:
         console.print(f"[dim]  … and {extra} more[/dim]")
+    # One pointer per block (not per line): the first artifact-backed reason's
+    # verifier dir, so "where do I look next" needs no summary.json dig either.
+    if artifact_pointer is not None:
+        console.print(f"[dim]  (details: {escape(str(artifact_pointer))})[/dim]")
     if job_dir is not None:
         console.print(f"[dim]Artifacts:[/dim] {escape(str(job_dir))}")
         console.print(f"[dim]Summary:  [/dim] {escape(str(job_dir))}/summary.json")

{benchflow-0.6.7.dev1831 → benchflow-0.6.7.dev1833}/src/benchflow/evaluation.py

@@ -588,6 +588,10 @@ class TaskFailure:
     task_name: str
     rewards: dict[str, Any] | None
     verifier_error: str | None
+    # The task's rollout dir name under the job dir (``<task>__<uuid8>``), so
+    # the CLI can find the rollout's verifier artifacts without guessing.
+    # None on results persisted before the key existed.
+    rollout_name: str | None = None
 
 
 @dataclass
@@ -1801,6 +1805,9 @@ class Evaluation:
                 task_name=name,
                 rewards=r.get("rewards"),
                 verifier_error=r.get("verifier_error"),
+                # `or None`: RolloutResult defaults rollout_name to "" — don't
+                # let that masquerade as a resolvable rollout dir.
+                rollout_name=r.get("rollout_name") or None,
             )
             for name, r in sorted(all_results.items())
             if classify_score_outcome(r) == "failed"

{benchflow-0.6.7.dev1831 → benchflow-0.6.7.dev1833}/tests/test_cli_live_progress.py

@@ -324,7 +324,7 @@ def test_report_eval_result_surfaces_verifier_errors(monkeypatch):
     assert "Score: 0/3" in out
 
 
-def _reported(result) -> str:
+def _reported(result, job_dir=None) -> str:
     """Render _report_eval_result through a captured Console, return the text."""
     import io
 
@@ -336,7 +336,7 @@ def _reported(result) -> str:
     original = shared.console
     shared.console = rec
     try:
-        shared._report_eval_result(result)
+        shared._report_eval_result(result, job_dir)
     finally:
         shared.console = original
     return rec.file.getvalue()
@@ -428,6 +428,210 @@ def test_report_eval_result_truncates_failure_lines():
     assert line.rstrip().endswith("…")
 
 
+def _bare_failure(task_name: str, rollout_name: str | None = None):
+    """A FAILED task whose reason would be the bare `reward 0.0` fallback."""
+    from benchflow.evaluation import TaskFailure
+
+    return TaskFailure(
+        task_name=task_name,
+        rewards={"reward": 0.0},
+        verifier_error=None,
+        rollout_name=rollout_name,
+    )
+
+
+def _write_ctrf(verifier_dir, *, name: str | None, trace: str | None = None) -> None:
+    """Write a minimal CTRF report (pytest-json-ctrf shape).
+
+    ``name`` is the failed test's node id; ``None`` writes an all-passed report.
+    """
+    import json
+
+    verifier_dir.mkdir(parents=True, exist_ok=True)
+    tests = [{"name": "tests/test_x.py::test_ok", "status": "passed"}]
+    if name is not None:
+        test: dict = {"name": name, "status": "failed"}
+        if trace is not None:
+            test["trace"] = trace
+        tests.append(test)
+    (verifier_dir / "ctrf.json").write_text(
+        json.dumps(
+            {
+                "reportFormat": "CTRF",
+                "results": {"tool": {"name": "pytest"}, "tests": tests},
+            }
+        )
+    )
+
+
+def _write_stdout(verifier_dir, text: str) -> None:
+    verifier_dir.mkdir(parents=True, exist_ok=True)
+    (verifier_dir / "test-stdout.txt").write_text(text)
+
+
+def test_failure_reason_artifact_tier_reads_ctrf(tmp_path):
+    # Dogfood follow-up: `✗ fix-build: reward 0.0` while verifier/ctrf.json held
+    # the real answer. A bare-reward reason is upgraded from the rollout dir the
+    # engine recorded (rollout_name — no globbing), plus one (details: …) line.
+    # The redundant "AssertionError: " prefix is stripped from the assertion.
+    _write_ctrf(
+        tmp_path / "fix-build__ab12cd34" / "verifier",
+        name="tests/test_build.py::test_build_success",
+        trace=(
+            "def test_build_success():\n"
+            ">       assert result.ok, 'Build failed for py312'\n"
+            "E       AssertionError: Build failed for py312\n"
+        ),
+    )
+    out = _reported(
+        _failed_result([_bare_failure("fix-build", "fix-build__ab12cd34")]), tmp_path
+    )
+    assert (
+        "✗ fix-build: reward 0.0 — test_build_success failed: "
+        "Build failed for py312" in out
+    )
+    assert f"(details: {tmp_path / 'fix-build__ab12cd34' / 'verifier'})" in out
+
+
+def test_failure_reason_artifact_tier_requires_rollout_name(tmp_path):
+    # No rollout_name (old result.json predating the key, sharded aggregation):
+    # nothing is resolved — even when a plausible dir exists on disk — and a
+    # stale rollout_name whose dir is gone degrades the same way.
+    _write_ctrf(tmp_path / "fix-build__ab12cd34" / "verifier", name="t::test_x")
+    out = _reported(
+        _failed_result(
+            [
+                _bare_failure("fix-build", None),
+                _bare_failure("other-task", "other-task__99999999"),
+            ]
+        ),
+        tmp_path,
+    )
+    assert "✗ fix-build: reward 0.0" in out
+    assert "✗ other-task: reward 0.0" in out
+    assert "test_x" not in out
+    assert "details:" not in out
+
+
+def test_failure_reason_artifact_tier_stdout_tail_summary(tmp_path):
+    # No CTRF report: fall back to the tail of test-stdout.txt — the pytest
+    # summary line wins over earlier FAILED lines, decoration stripped.
+    _write_stdout(
+        tmp_path / "fix-build__ab12cd34" / "verifier",
+        "collected 3 items\n"
+        "FAILED tests/test_build.py::test_build_success - AssertionError: boom\n"
+        "=========== 1 failed, 2 passed in 40.86s ===========\n",
+    )
+    out = _reported(
+        _failed_result([_bare_failure("fix-build", "fix-build__ab12cd34")]), tmp_path
+    )
+    assert "✗ fix-build: reward 0.0 — 1 failed, 2 passed in 40.86s" in out
+
+
+def test_failure_reason_artifact_tier_stdout_failed_line(tmp_path):
+    # No summary line in the tail: the last FAILED line is the evidence.
+    _write_stdout(
+        tmp_path / "fix-build__ab12cd34" / "verifier",
+        "collected 1 item\n"
+        "FAILED tests/test_build.py::test_build - AssertionError: boom\n",
+    )
+    out = _reported(
+        _failed_result([_bare_failure("fix-build", "fix-build__ab12cd34")]), tmp_path
+    )
+    assert (
+        "✗ fix-build: reward 0.0 — FAILED tests/test_build.py::test_build - "
+        "AssertionError: boom" in out
+    )
+
+
+def test_failure_reason_artifact_tier_tries_sources_in_order(tmp_path):
+    # Try-in-order, first non-None wins: a CTRF report that parses but carries
+    # no failed test (reward zeroed by something else) yields to the stdout
+    # tail, as does an oversized (> 64KB) report that can't be tail-parsed.
+    no_failed = tmp_path / "task-a__11111111" / "verifier"
+    _write_ctrf(no_failed, name=None)
+    _write_stdout(no_failed, "=== 1 failed, 2 passed in 1.00s ===\n")
+    oversized = tmp_path / "task-b__22222222" / "verifier"
+    oversized.mkdir(parents=True)
+    (oversized / "ctrf.json").write_text('{"pad": "%s"}' % ("x" * 70_000))
+    _write_stdout(oversized, "=== 2 failed, 1 passed in 2.00s ===\n")
+    out = _reported(
+        _failed_result(
+            [
+                _bare_failure("task-a", "task-a__11111111"),
+                _bare_failure("task-b", "task-b__22222222"),
+            ]
+        ),
+        tmp_path,
+    )
+    assert "✗ task-a: reward 0.0 — 1 failed, 2 passed in 1.00s" in out
+    assert "✗ task-b: reward 0.0 — 2 failed, 1 passed in 2.00s" in out
+
+
+def test_failure_reason_artifact_tier_never_raises(tmp_path):
+    # The tier's safety contract: a corrupt CTRF report falls through to the
+    # stdout tail; with no other evidence it degrades to the bare `reward 0.0`
+    # — no pointer, no exception.
+    corrupt = tmp_path / "corrupt-task__11111111" / "verifier"
+    corrupt.mkdir(parents=True)
+    (corrupt / "ctrf.json").write_text("{not json")
+    salvaged = tmp_path / "salvaged-task__33333333" / "verifier"
+    salvaged.mkdir(parents=True)
+    (salvaged / "ctrf.json").write_text("{not json")
+    _write_stdout(salvaged, "=== 1 failed in 3.00s ===\n")
+    (tmp_path / "empty-task__22222222" / "verifier").mkdir(parents=True)
+    out = _reported(
+        _failed_result(
+            [
+                _bare_failure("corrupt-task", "corrupt-task__11111111"),
+                _bare_failure("empty-task", "empty-task__22222222"),
+                _bare_failure("salvaged-task", "salvaged-task__33333333"),
+            ]
+        ),
+        tmp_path,
+    )
+    assert "✗ corrupt-task: reward 0.0" in out
+    assert "✗ empty-task: reward 0.0" in out
+    assert "✗ salvaged-task: reward 0.0 — 1 failed in 3.00s" in out
+    assert out.count("(details:") == 1
+
+
+def test_failure_reason_artifact_tier_yields_to_metric_breakdown(tmp_path):
+    # Named metrics already explain the miss — artifacts are only for reasons
+    # that would otherwise be a bare `reward X`.
+    from benchflow.evaluation import TaskFailure
+
+    _write_ctrf(tmp_path / "plan-meeting__ab12cd34" / "verifier", name="t::test_x")
+    out = _reported(
+        _failed_result(
+            [
+                TaskFailure(
+                    task_name="plan-meeting",
+                    rewards={"reward": 0.3, "sections": 0.0},
+                    verifier_error=None,
+                    rollout_name="plan-meeting__ab12cd34",
+                )
+            ]
+        ),
+        tmp_path,
+    )
+    assert "✗ plan-meeting: reward 0.3 — sections 0.0" in out
+    assert "details:" not in out
+
+
+def test_report_eval_result_artifact_reads_stay_capped(tmp_path):
+    # The display cap also bounds artifact reads: 7 bare failures with readable
+    # artifacts still render 5 lines + "… and 2 more", and the pointer prints
+    # once per block, not per line.
+    for i in range(7):
+        _write_ctrf(tmp_path / f"task-{i}__ab12cd34" / "verifier", name=f"t::test_{i}")
+    failures = [_bare_failure(f"task-{i}", f"task-{i}__ab12cd34") for i in range(7)]
+    out = _reported(_failed_result(failures), tmp_path)
+    assert out.count("✗ task-") == 5
+    assert "… and 2 more" in out
+    assert out.count("(details:") == 1
+
+
 def test_fire_progress_swallows_callback_errors():
     # The feature's core safety contract: a raising display hook must never
     # propagate out of the engine (a render bug can't abort a run).

{benchflow-0.6.7.dev1831 → benchflow-0.6.7.dev1833}/tests/test_job.py

@@ -630,6 +630,7 @@ class TestJobRunOrchestration:
             "task-0": RunResult(task_name="task-0", rewards={"reward": 1.0}),
             "task-1": RunResult(
                 task_name="task-1",
+                rollout_name="task-1__ab12cd34",
                 rewards={"reward": 0.5, "decisions_found": 0.0},
                 verifier_error=None,
             ),
@@ -644,11 +645,14 @@ class TestJobRunOrchestration:
 
         result = await job.run()
 
+        # rollout_name rides along (via rollout_result_payload) so the CLI can
+        # resolve the rollout's artifact dir exactly, without globbing.
         assert result.task_failures == [
             TaskFailure(
                 task_name="task-1",
                 rewards={"reward": 0.5, "decisions_found": 0.0},
                 verifier_error=None,
+                rollout_name="task-1__ab12cd34",
             )
         ]