benchflow 0.7.8.dev2327__tar.gz → 0.7.8.dev2329__tar.gz

{benchflow-0.7.8.dev2327 → benchflow-0.7.8.dev2329}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.8.dev2327
+Version: 0.7.8.dev2329
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.8.dev2327 → benchflow-0.7.8.dev2329}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.8.dev2327"
+version = "0.7.8.dev2329"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.8.dev2327 → benchflow-0.7.8.dev2329}/src/benchflow/acp/session.py

@@ -162,6 +162,10 @@ class ToolCallRecord:
         self.kind = kind
         self.status = ToolCallStatus.PENDING
         self.content: list[dict] = []
+        # ACP ``rawInput`` / ``rawOutput``: the agent's own view of the call.
+        # codex-acp puts the command and its output here and nowhere else.
+        self.raw_input: object | None = None
+        self.raw_output: object | None = None
         self.started_at = datetime.now()
         self.finished_at: datetime | None = None
 
@@ -178,6 +182,13 @@ class ToolCallRecord:
         ):
             self.finished_at = datetime.now()
 
+    def absorb_raw_io(self, update: dict) -> None:
+        """Keep the latest ``rawInput`` / ``rawOutput`` an update carries."""
+        if update.get("rawInput") is not None:
+            self.raw_input = update["rawInput"]
+        if update.get("rawOutput") is not None:
+            self.raw_output = update["rawOutput"]
+
 
 class ACPSession:
     """Tracks mutable state for one ACP session.
@@ -417,6 +428,20 @@ class ACPSession:
                     update.get("kind", "other"), update.get("title", "")
                 ),
             )
+            # The opening notification may already carry content (codex-acp
+            # sends file-change diffs this way) and raw I/O.
+            initial_content = update.get("content")
+            if isinstance(initial_content, list) and initial_content:
+                record.content.extend(initial_content)
+            record.absorb_raw_io(update)
+            # An opening call may already be terminal (codex-acp file edits
+            # arrive completed with no later update); honor its status so it
+            # never lingers in pending_tool_call_ids().
+            if update.get("status") is not None:
+                try:
+                    record.update_status(ToolCallStatus(update["status"]))
+                except ValueError:
+                    logger.warning(f"Unknown tool call status: {update.get('status')}")
             self._record_tool_call(record)
 
         elif update_type == "tool_call_update":
@@ -440,6 +465,7 @@ class ACPSession:
                 status = ToolCallStatus.IN_PROGRESS
             content = update.get("content")
             record.update_status(status, content)
+            record.absorb_raw_io(update)
             if status in (ToolCallStatus.PENDING, ToolCallStatus.IN_PROGRESS):
                 self._pending_tool_call_update_counts[tc_id] = (
                     self._pending_tool_call_update_counts.get(tc_id, 0) + 1

{benchflow-0.7.8.dev2327 → benchflow-0.7.8.dev2329}/src/benchflow/trajectories/_capture.py

@@ -70,16 +70,21 @@ def _events_to_trajectory(events: list[dict]) -> list[dict]:
     for event in events:
         if event["type"] == "tool_call":
             tc = event["record"]
-            out.append(
-                {
-                    "type": "tool_call",
-                    "tool_call_id": tc.tool_call_id,
-                    "kind": tc.kind,
-                    "title": tc.title,
-                    "status": tc.status.value,
-                    "content": tc.content,
-                }
-            )
+            record = {
+                "type": "tool_call",
+                "tool_call_id": tc.tool_call_id,
+                "kind": tc.kind,
+                "title": tc.title,
+                "status": tc.status.value,
+                "content": tc.content,
+            }
+            # Only present when the agent sent them, so trajectories of agents
+            # without raw I/O keep their exact shape.
+            if getattr(tc, "raw_input", None) is not None:
+                record["raw_input"] = tc.raw_input
+            if getattr(tc, "raw_output", None) is not None:
+                record["raw_output"] = tc.raw_output
+            out.append(record)
         elif event["type"] in ("user_message", "agent_message", "agent_thought"):
             out.append({"type": event["type"], "text": event["text"]})
         elif event["type"] == "agent_timeout":

{benchflow-0.7.8.dev2327 → benchflow-0.7.8.dev2329}/src/benchflow/trajectories/viewer/assets/detail.js

@@ -14,6 +14,7 @@ BF.detail = (() => {
     ["trace", "view-trace"],
     ["verifier", "view-verifier"],
     ["metrics", "view-metrics"],
+    ["rubric", "view-rubric"],
   ]);
   const STEP_KINDS = new Set(["prompt", "message", "thought", "tool", "timeout", "unknown"]);
   const TOOL_HUES = new Set(["read", "edit", "execute", "fetch", "search", "think", "skill", "other"]);
@@ -178,6 +179,9 @@ BF.detail = (() => {
     ) {
       available.push(["metrics", "Metrics"]);
     }
+    if (payload.rubric && typeof payload.rubric === "object") {
+      available.push(["rubric", "Rubric"]);
+    }
 
     const tabs = document.getElementById("tabs");
     tabs.textContent = "";
@@ -584,6 +588,72 @@ BF.detail = (() => {
     if (!main.children.length) main.appendChild(el("div", "panelblock", "No verifier artifacts in this rollout."));
   }
 
+  function renderRubric(payload) {
+    const rubric = payload.rubric && typeof payload.rubric === "object" ? payload.rubric : null;
+    const main = document.getElementById("view-rubric");
+    main.textContent = "";
+    if (!rubric) {
+      main.appendChild(el("div", "panelblock", "No review report found for this rollout."));
+      return;
+    }
+    const scoring = rubric.scoring && typeof rubric.scoring === "object" ? rubric.scoring : {};
+    const head = el("div", "panelblock");
+    head.appendChild(el("h2", null, "Review"));
+    const facts = el("table", "plain");
+    const rows = [
+      ["Reviewer", rubric.reviewer_model || ""],
+      ["Valid", rubric.review_valid ? "yes" : "no"],
+      ["Verifier pass", scoring.deterministic_pass === undefined ? "" : (scoring.deterministic_pass ? "yes" : "no")],
+      ["Blockers", scoring.all_blockers_pass === undefined ? ""
+        : (scoring.all_blockers_pass ? "all pass" : "failed: " + (scoring.failed_blockers || []).join(", "))],
+      ["Raw quality", scoring.raw_quality === undefined ? "" : String(scoring.raw_quality)],
+      ["Gated quality", scoring.gated_quality === undefined ? "" : String(scoring.gated_quality)],
+      ["Decision", scoring.decision || ""],
+    ];
+    rows.forEach(([label, value]) => {
+      const row = el("tr");
+      row.appendChild(el("th", null, label));
+      const cell = el("td", null, value);
+      if (label === "Blockers" || label === "Verifier pass") {
+        cell.style.color = /fail|^no$/.test(value) ? "var(--bad-ink)" : (value ? "var(--ok-ink)" : "");
+      }
+      row.appendChild(cell);
+      facts.appendChild(row);
+    });
+    head.appendChild(facts);
+    if (rubric.summary) collapsibleText(head, rubric.summary, "body");
+    main.appendChild(head);
+
+    const criteria = Array.isArray(rubric.criteria) ? rubric.criteria : [];
+    if (criteria.length) {
+      const block = el("div", "panelblock");
+      block.appendChild(el("h2", null, "Criteria"));
+      const table = el("table", "plain");
+      const heading = el("tr");
+      ["Criterion", "Kind", "Verdict", "Reviewer note"].forEach((label) => heading.appendChild(el("th", null, label)));
+      table.appendChild(heading);
+      criteria.forEach((criterion) => {
+        const record = criterion && typeof criterion === "object" ? criterion : {};
+        const row = el("tr");
+        row.appendChild(el("td", null, record.name));
+        // blocker true: v0.2 blocker; false: v0.2 scored; null: legacy v0.1 (outcome only)
+        const scored = record.blocker === false;
+        const kind = record.blocker === true ? "blocker" : (scored ? "weight " + (record.weight ?? "?") : "criterion");
+        row.appendChild(el("td", null, kind));
+        const verdict = el("td", null, scored ? ((record.score ?? "?") + " / 2") : (record.outcome || ""));
+        if (!scored && record.outcome) {
+          verdict.style.color = record.outcome === "pass" ? "var(--ok-ink)"
+            : (record.outcome === "fail" ? "var(--bad-ink)" : "var(--muted-foreground)");
+        }
+        row.appendChild(verdict);
+        row.appendChild(el("td", null, record.explanation || ""));
+        table.appendChild(row);
+      });
+      block.appendChild(table);
+      main.appendChild(block);
+    }
+  }
+
   function renderMetrics(payload) {
     const meta = payload.meta || {};
     const main = document.getElementById("view-metrics");
@@ -665,7 +735,7 @@ BF.detail = (() => {
   }
 
   function clearDetail() {
-    ["hdr", "tabs", "view-trace", "view-verifier", "view-metrics"].forEach((id) => {
+    ["hdr", "tabs", "view-trace", "view-verifier", "view-metrics", "view-rubric"].forEach((id) => {
       document.getElementById(id).textContent = "";
     });
     document.getElementById("toolbar").textContent = "";
@@ -705,6 +775,7 @@ BF.detail = (() => {
     renderTrace(payload);
     renderVerifier(payload);
     renderMetrics(payload);
+    renderRubric(payload);
     document.title = (payload.meta && payload.meta.task_name) || payload.rollout_name || "benchflow trajectory";
     if (options.focusHeading) requestAnimationFrame(() => heading.focus());
   }

{benchflow-0.7.8.dev2327 → benchflow-0.7.8.dev2329}/src/benchflow/trajectories/viewer/assets/template.html

@@ -36,6 +36,7 @@
     <main id="view-trace" role="tabpanel"></main>
     <main id="view-verifier" role="tabpanel" class="hidden"></main>
     <main id="view-metrics" role="tabpanel" class="hidden"></main>
+    <main id="view-rubric" role="tabpanel" class="hidden"></main>
   </div>
 </div>
 <script id="bf-payload" type="application/json">__BENCHFLOW_PAYLOAD__</script>

{benchflow-0.7.8.dev2327 → benchflow-0.7.8.dev2329}/src/benchflow/trajectories/viewer/models.py

@@ -351,12 +351,59 @@ class VerifierArtifacts:
         }
 
 
+@dataclass(frozen=True)
+class RubricCriterion:
+    """One rubric criterion as the reviewer judged it."""
+
+    name: str
+    # True for a v0.2 blocker, False for a v0.2 scored criterion, None for a
+    # legacy v0.1 criterion, which only carries an outcome.
+    blocker: bool | None
+    weight: int | None
+    outcome: str | None
+    score: int | None
+    explanation: str
+
+    def to_payload(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "blocker": self.blocker,
+            "weight": self.weight,
+            "outcome": self.outcome,
+            "score": self.score,
+            "explanation": self.explanation,
+        }
+
+
+@dataclass(frozen=True)
+class RubricReview:
+    """The ``bench review`` verdict for one rollout, from ``review_report.json``."""
+
+    reviewer_model: str | None
+    review_valid: bool
+    scoring: JsonObject
+    summary: str
+    criteria: list[RubricCriterion]
+    source: str
+
+    def to_payload(self) -> dict[str, Any]:
+        return {
+            "reviewer_model": self.reviewer_model,
+            "review_valid": self.review_valid,
+            "scoring": dict(self.scoring),
+            "summary": self.summary,
+            "criteria": [criterion.to_payload() for criterion in self.criteria],
+            "source": self.source,
+        }
+
+
 @dataclass(frozen=True)
 class ViewerPayload:
     rollout_name: str
     meta: Meta
     steps: list[Step]
     verifier: VerifierArtifacts
+    rubric: RubricReview | None = None
     schema_version: int = 1
 
     def to_payload(self) -> dict[str, Any]:
@@ -366,6 +413,7 @@ class ViewerPayload:
             "meta": self.meta.to_payload(),
             "steps": [step.to_payload() for step in self.steps],
             "verifier": self.verifier.to_payload(),
+            "rubric": self.rubric.to_payload() if self.rubric is not None else None,
         }
 
 

{benchflow-0.7.8.dev2327 → benchflow-0.7.8.dev2329}/src/benchflow/trajectories/viewer/payload.py

@@ -18,6 +18,8 @@ from .models import (
     Meta,
     PromptStep,
     RolloutMetadata,
+    RubricCriterion,
+    RubricReview,
     Step,
     StepCounts,
     ThoughtStep,
@@ -231,6 +233,34 @@ def _tool_content_texts(content: Any) -> list[str]:
     return texts
 
 
+def _raw_io_texts(kind: str, raw_input: Any, raw_output: Any) -> list[str]:
+    """Render ACP ``rawInput`` / ``rawOutput`` for a call without content blocks.
+
+    codex-acp reports a command as ``rawInput.command`` and its result as
+    ``rawOutput.formatted_output`` plus ``exit_code``; anything else is shown
+    as JSON so no recorded detail is hidden.
+    """
+    texts: list[str] = []
+    if isinstance(raw_input, dict) and kind == "execute" and "command" in raw_input:
+        command = raw_input["command"]
+        texts.append(
+            " ".join(str(part) for part in command)
+            if isinstance(command, list)
+            else _display_text(command)
+        )
+    elif raw_input is not None:
+        texts.append(dumps_finite(raw_input, ensure_ascii=False, default=str))
+    if isinstance(raw_output, dict) and "formatted_output" in raw_output:
+        output = _display_text(raw_output.get("formatted_output"))
+        exit_code = raw_output.get("exit_code")
+        if exit_code not in (None, 0):
+            output = f"{output.rstrip()}\n[exit code {exit_code}]"
+        texts.append(output)
+    elif raw_output is not None:
+        texts.append(dumps_finite(raw_output, ensure_ascii=False, default=str))
+    return [text for text in texts if text]
+
+
 def _parse_ts(value: Any) -> float | None:
     """Parse finite epoch or ISO-8601 timestamps into epoch seconds."""
     numeric = _finite_float(value)
@@ -319,7 +349,10 @@ def _normalize_steps(
                         kind=kind,
                         title=title,
                         status=normalize_tool_status(event.get("status")),
-                        content=_tool_content_texts(event.get("content")),
+                        content=_tool_content_texts(event.get("content"))
+                        or _raw_io_texts(
+                            kind, event.get("raw_input"), event.get("raw_output")
+                        ),
                         hue=tool_hue(kind, title),
                     ),
                     t=started,
@@ -556,6 +589,98 @@ def _load_verifier(rollout_dir: Path) -> VerifierArtifacts:
     )
 
 
+# How far above a rollout directory review reports are looked for. Covers
+# ``jobs/<run>/<rollout>`` next to ``jobs/review-<stamp>/`` (bench review's
+# default) and deeper per-trial layouts.
+_REVIEW_SEARCH_DEPTH = 4
+_REVIEW_REPORT = "review_report.json"
+
+
+def _criteria(trial: dict[str, Any]) -> list[RubricCriterion]:
+    checks = trial.get("checks")
+    checks = checks if isinstance(checks, dict) else {}
+    metadata = trial.get("criterion_metadata")
+    rows: list[RubricCriterion] = []
+    for meta in metadata if isinstance(metadata, list) else []:
+        if not isinstance(meta, dict):
+            continue
+        name = _display_text(meta.get("name"))
+        check = checks.get(name)
+        check = check if isinstance(check, dict) else {}
+        score = check.get("score")
+        weight = meta.get("weight")
+        blocker = meta.get("blocker")
+        rows.append(
+            RubricCriterion(
+                name=name,
+                blocker=None if blocker is None else bool(blocker),
+                weight=weight
+                if isinstance(weight, int) and not isinstance(weight, bool)
+                else None,
+                outcome=_optional_text(check.get("outcome")),
+                score=score
+                if isinstance(score, int) and not isinstance(score, bool)
+                else None,
+                explanation=_display_text(check.get("explanation")),
+            )
+        )
+    return rows
+
+
+def _rubric_from_report(path: Path, rollout_name: str) -> RubricReview | None:
+    report = _load_json(path)
+    if not isinstance(report, dict):
+        return None
+    reviewer = report.get("reviewer")
+    model = (
+        _optional_text(reviewer.get("model")) if isinstance(reviewer, dict) else None
+    )
+    found: RubricReview | None = None
+    trials = report.get("trials")
+    for trial in trials if isinstance(trials, list) else []:
+        if not isinstance(trial, dict) or trial.get("trial_name") != rollout_name:
+            continue
+        scoring = trial.get("scoring")
+        found = RubricReview(
+            reviewer_model=model,
+            review_valid=bool(trial.get("review_valid")),
+            scoring=dict(scoring) if isinstance(scoring, dict) else {},
+            summary=_display_text(trial.get("summary")),
+            criteria=_criteria(trial),
+            source=str(path),
+        )
+    return found
+
+
+def _load_rubric(rollout_dir: Path) -> RubricReview | None:
+    """Find the review of this rollout in the nearest ``review*`` directory.
+
+    The search walks up ``_REVIEW_SEARCH_DEPTH`` ancestors; at each level every
+    ``review*/**/review_report.json`` is read and the last valid entry for
+    this rollout in sorted order wins, the rule the leaderboard consumers use.
+    An invalid review (no scoring) never shadows a valid one.
+    """
+    name = rollout_dir.name
+    ancestor = rollout_dir
+    for _ in range(_REVIEW_SEARCH_DEPTH):
+        ancestor = ancestor.parent
+        found: RubricReview | None = None
+        for review_dir in sorted(ancestor.glob("review*")):
+            if not review_dir.is_dir():
+                continue
+            for path in sorted(review_dir.rglob(_REVIEW_REPORT)):
+                rubric = _rubric_from_report(path, name)
+                if rubric is None:
+                    continue
+                if rubric.scoring or found is None:
+                    found = rubric
+        if found is not None:
+            return found
+        if ancestor == ancestor.parent:
+            break
+    return None
+
+
 def _safe_json(obj: Any) -> str:
     """Emit strict, finite JSON and replace lone UTF-16 surrogates."""
     if not _within_json_depth(obj, max_depth=_MAX_JSON_OUTPUT_DEPTH):
@@ -581,6 +706,7 @@ def _build_acp_payload(rollout_dir: Path, prompts: list[str] | None) -> ViewerPa
         meta=_build_meta(_load_rollout_metadata(rollout_dir), steps),
         steps=steps,
         verifier=_load_verifier(rollout_dir),
+        rubric=_load_rubric(rollout_dir),
     )
 
 

{benchflow-0.7.8.dev2327 → benchflow-0.7.8.dev2329}/src/benchflow/trajectories/viewer/sources.py

@@ -26,6 +26,10 @@ _HF_VIEWER_FILES = (
     *(f"verifier/{name}" for name in VERIFIER_SIDECARS),
 )
 
+# bench review reports live beside the runs (``jobs/review-<stamp>/``), so a
+# scoped source also fetches its parent's review directories. Small JSON only.
+_HF_REVIEW_FILES = ("review_report.json",)
+
 _REPO_PART_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
 _GLOB_METACHARACTERS = frozenset("*?[]")
 
@@ -147,6 +151,11 @@ def resolve_hf_dataset(source: HfDatasetSource) -> Path:
     for name in _HF_VIEWER_FILES:
         patterns.append(f"{scope}{name}")
         patterns.append(f"{scope}**/{name}")
+    parent = f"{'/'.join(subpath_parts[:-1])}/" if len(subpath_parts) > 1 else ""
+    for name in _HF_REVIEW_FILES:
+        patterns.append(f"{scope}**/{name}")
+        patterns.append(f"{parent}review*/{name}")
+        patterns.append(f"{parent}review*/**/{name}")
     print(
         f"Fetching {source.repo_id}"
         + (f" @ {source.revision}" if source.revision else "")

benchflow-0.7.8.dev2329/tests/test_tool_call_raw_io.py

@@ -0,0 +1,175 @@
+"""Tool-call content and raw I/O capture (issue #1099).
+
+codex-acp reports a command as ``rawInput`` on the opening ``tool_call``, its
+output as ``rawOutput`` on the completing ``tool_call_update``, and file-change
+diffs as ``content`` on the opening ``tool_call``. Before the fix every one of
+those was dropped and the trajectory kept only the title.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from benchflow.acp.session import ACPSession
+from benchflow.trajectories._capture import _events_to_trajectory
+from benchflow.trajectories.viewer.payload import _build_acp_payload, _raw_io_texts
+
+
+def _codex_exec_session() -> ACPSession:
+    session = ACPSession("s")
+    session.handle_update(
+        {
+            "sessionUpdate": "tool_call",
+            "toolCallId": "call_1",
+            "title": "ls /root",
+            "kind": "execute",
+            "status": "in_progress",
+            "rawInput": {"command": "ls /root", "cwd": "/root"},
+        }
+    )
+    session.handle_update(
+        {
+            "sessionUpdate": "tool_call_update",
+            "toolCallId": "call_1",
+            "status": "completed",
+            "rawOutput": {"formatted_output": "a.py\nb.py\n", "exit_code": 0},
+        }
+    )
+    return session
+
+
+def test_session_keeps_raw_input_and_output() -> None:
+    """Guards the fix for issue #1099: rawInput from the opening tool_call and
+    rawOutput from the completing update both survive on the record."""
+    record = _codex_exec_session().tool_calls[0]
+    assert record.raw_input == {"command": "ls /root", "cwd": "/root"}
+    assert record.raw_output == {"formatted_output": "a.py\nb.py\n", "exit_code": 0}
+
+
+def test_session_keeps_content_sent_on_the_opening_tool_call() -> None:
+    """Guards the fix for issue #1099: codex-acp file-change diffs arrive as
+    content on the opening tool_call, not on an update."""
+    session = ACPSession("s")
+    diff = {"type": "diff", "path": "/root/a.py", "oldText": None, "newText": "x = 1\n"}
+    session.handle_update(
+        {
+            "sessionUpdate": "tool_call",
+            "toolCallId": "call_2",
+            "title": "Editing files",
+            "kind": "edit",
+            "status": "completed",
+            "content": [diff],
+        }
+    )
+    assert session.tool_calls[0].content == [diff]
+
+
+def test_opening_call_status_is_honored() -> None:
+    """Guards the follow-up on #1099 review: a file edit that codex-acp reports
+    completed in its opening tool_call (no later update) is terminal, so it
+    never lingers in pending_tool_call_ids(); an opening call without a status
+    stays pending as before."""
+    session = ACPSession("s")
+    session.handle_update(
+        {
+            "sessionUpdate": "tool_call",
+            "toolCallId": "done",
+            "title": "Editing files",
+            "kind": "edit",
+            "status": "completed",
+            "content": [
+                {"type": "diff", "path": "/a", "oldText": None, "newText": "x"}
+            ],
+        }
+    )
+    session.handle_update(
+        {
+            "sessionUpdate": "tool_call",
+            "toolCallId": "open",
+            "title": "ls",
+            "kind": "execute",
+        }
+    )
+    session.handle_update(
+        {
+            "sessionUpdate": "tool_call",
+            "toolCallId": "odd",
+            "title": "t",
+            "kind": "other",
+            "status": "not-a-status",
+        }
+    )
+    assert session.pending_tool_call_ids() == ["open", "odd"]
+    assert session.tool_calls[0].finished_at is not None
+
+
+def test_trajectory_emits_raw_fields_only_when_present() -> None:
+    """Guards the fix for issue #1099 without changing the shape agents that
+    send no raw I/O produce: the keys appear only when set."""
+    with_raw = _events_to_trajectory(_codex_exec_session().events)[0]
+    assert with_raw["raw_input"] == {"command": "ls /root", "cwd": "/root"}
+    assert with_raw["raw_output"]["exit_code"] == 0
+
+    bare = ACPSession("s")
+    bare.handle_update(
+        {"sessionUpdate": "tool_call", "toolCallId": "t", "title": "t", "kind": "read"}
+    )
+    assert set(_events_to_trajectory(bare.events)[0]) == {
+        "type",
+        "tool_call_id",
+        "kind",
+        "title",
+        "status",
+        "content",
+    }
+
+
+def test_raw_io_texts_render_commands_outputs_and_exit_codes() -> None:
+    """Execute calls read as command then output; a nonzero exit is stated;
+    other shapes fall back to JSON so nothing recorded is hidden."""
+    texts = _raw_io_texts(
+        "execute",
+        {"command": ["python", "-c", "print(1)"], "cwd": "/"},
+        {"formatted_output": "boom\n", "exit_code": 2},
+    )
+    assert texts == ["python -c print(1)", "boom\n[exit code 2]"]
+    assert _raw_io_texts("other", {"server": "mcp", "tool": "t"}, None) == [
+        '{"server": "mcp", "tool": "t"}'
+    ]
+    assert _raw_io_texts("execute", None, None) == []
+
+
+def test_payload_falls_back_to_raw_io_when_content_is_empty(tmp_path: Path) -> None:
+    """Guards the viewer side of issue #1099: a codex-style tool call with no
+    content blocks still renders its command and output; content wins when
+    both exist."""
+    traj = tmp_path / "trajectory"
+    traj.mkdir()
+    events = [
+        {
+            "type": "tool_call",
+            "tool_call_id": "c1",
+            "kind": "execute",
+            "title": "ls /root",
+            "status": "completed",
+            "content": [],
+            "raw_input": {"command": "ls /root", "cwd": "/root"},
+            "raw_output": {"formatted_output": "a.py\n", "exit_code": 0},
+        },
+        {
+            "type": "tool_call",
+            "tool_call_id": "c2",
+            "kind": "execute",
+            "title": "cat x",
+            "status": "completed",
+            "content": [{"type": "content", "content": {"type": "text", "text": "x"}}],
+            "raw_output": {"formatted_output": "ignored", "exit_code": 0},
+        },
+    ]
+    (traj / "acp_trajectory.jsonl").write_text("\n".join(json.dumps(e) for e in events))
+    (tmp_path / "result.json").write_text("{}")
+    steps = _build_acp_payload(tmp_path, None).to_payload()["steps"]
+    tools = [s["tool"] for s in steps if s["kind"] == "tool"]
+    assert tools[0]["content"] == ["ls /root", "a.py\n"]
+    assert tools[1]["content"] == ["x"]

{benchflow-0.7.8.dev2327 → benchflow-0.7.8.dev2329}/tests/trajectories/test_viewer_acp_renderer.py

@@ -640,8 +640,11 @@ class TestSources:
         assert root == tmp_path / "jobs" / "run-1"
         assert calls["repo_id"] == "org/name"
         assert calls["repo_type"] == "dataset"
-        assert all(p.startswith("jobs/run-1/") for p in calls["allow_patterns"])
+        rollout_patterns = [p for p in calls["allow_patterns"] if "review" not in p]
+        assert all(p.startswith("jobs/run-1/") for p in rollout_patterns)
         assert "jobs/run-1/trajectory/acp_trajectory.jsonl" in calls["allow_patterns"]
+        # review reports sit beside the run, so the parent's review-* dirs are in scope
+        assert "jobs/review*/**/review_report.json" in calls["allow_patterns"]
         assert not any("llm_trajectory" in p for p in calls["allow_patterns"])
 
     def test_resolve_raises_typed_error_for_missing_subpath(
@@ -888,6 +891,7 @@ class TestTypedContract:
             "meta",
             "steps",
             "verifier",
+            "rubric",
         }
         assert wire["steps"][0] == {
             "i": 1,