benchflow 0.7.6.dev2058__tar.gz → 0.7.6.dev2096__tar.gz

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2096}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.6.dev2058
+Version: 0.7.6.dev2096
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2096}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.6.dev2058"
+version = "0.7.6.dev2096"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2096}/src/benchflow/cli/eval_artifacts.py

@@ -66,7 +66,9 @@ def postprocess_eval_artifacts(
     eval_config = _apply_source_sidecar(eval_config, resolved_tasks_dir)
     from benchflow.eval_artifacts import (
         TaskManifestOptions,
+        canonical_task_rows,
         materialize_canonical_job,
+        render_run_summary_markdown,
         write_canonical_selection,
         write_health_summary,
         write_task_manifest,
@@ -138,6 +140,12 @@ def postprocess_eval_artifacts(
             console.print(
                 f"[green]Canonical jobs:[/green] {escape(str(req.canonical_jobs_dir))}"
             )
+    if req.publish_hf is not None or req.publish_bucket is not None:
+        (job_dir / "README.md").write_text(
+            render_run_summary_markdown(
+                job_dir, agent=eval_config.agent, model=eval_config.model
+            )
+        )
     if req.publish_hf is not None:
         from benchflow.publish.huggingface import publish_folder_to_hf
 
@@ -154,6 +162,42 @@ def postprocess_eval_artifacts(
             print_error(str(exc))
             raise typer.Exit(1) from None
         console.print(f"[green]Published HF artifacts:[/green] {escape(published.url)}")
+    if req.publish_bucket is not None:
+        from benchflow.publish.huggingface import publish_folder_to_bucket
+
+        prefix = req.hf_prefix or job_dir.name
+        try:
+            published = publish_folder_to_bucket(
+                job_dir, bucket_id=req.publish_bucket, path_in_repo=prefix
+            )
+        except ValueError as exc:
+            print_error(str(exc))
+            raise typer.Exit(1) from None
+        console.print(f"[green]Published HF bucket:[/green] {escape(published.url)}")
+    if req.eval_results_model is not None:
+        from benchflow.publish.huggingface import open_eval_results_pr
+
+        rewards = [
+            row["reward"] for row in canonical_task_rows(job_dir) if row["scored"]
+        ]
+        if rewards:
+            value = round(100.0 * sum(rewards) / len(rewards), 2)
+            try:
+                pr_url = open_eval_results_pr(
+                    model_repo=req.eval_results_model,
+                    dataset_id=cast(str, req.eval_results_dataset),
+                    task_id=cast(str, req.eval_results_task),
+                    value=value,
+                )
+            except ValueError as exc:
+                print_error(str(exc))
+                raise typer.Exit(1) from None
+            console.print(f"[green]Eval-results PR:[/green] {escape(pr_url or '')}")
+        else:
+            console.print(
+                "[yellow]--eval-results-model set but no scored rollouts were "
+                "found; skipping eval-results PR[/yellow]"
+            )
 
 
 def _load_eval_matrix(path: Path) -> dict[str, dict]:

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2096}/src/benchflow/cli/main.py

@@ -581,6 +581,31 @@ def eval_run(
             "--hf-public-read-check", help="Verify public HF reads after upload"
         ),
     ] = False,
+    publish_bucket: Annotated[
+        str | None,
+        typer.Option(
+            "--publish-bucket", help="Upload final eval artifacts to a HF bucket"
+        ),
+    ] = None,
+    eval_results_model: Annotated[
+        str | None,
+        typer.Option(
+            "--eval-results-model",
+            help="HF model repo to open a community eval-results PR on",
+        ),
+    ] = None,
+    eval_results_dataset: Annotated[
+        str | None,
+        typer.Option(
+            "--eval-results-dataset", help="HF benchmark dataset id, e.g. org/benchmark"
+        ),
+    ] = None,
+    eval_results_task: Annotated[
+        str | None,
+        typer.Option(
+            "--eval-results-task", help="Benchmark task_id from the dataset's eval.yaml"
+        ),
+    ] = None,
     matrix: Annotated[
         Path | None,
         typer.Option("--matrix", help="YAML model matrix for repeated evals"),
@@ -653,6 +678,10 @@ def eval_run(
         publish_hf=publish_hf,
         hf_prefix=hf_prefix,
         hf_public_read_check=hf_public_read_check,
+        publish_bucket=publish_bucket,
+        eval_results_model=eval_results_model,
+        eval_results_dataset=eval_results_dataset,
+        eval_results_task=eval_results_task,
         matrix=matrix,
         trials=trials,
     )

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2096}/src/benchflow/eval_artifacts.py

@@ -9,6 +9,7 @@ from pathlib import Path
 from typing import Any, Literal
 
 from benchflow._utils.task_authoring import task_digest
+from benchflow._utils.text import truncate_end
 from benchflow.task.discovery import is_task_dir, resolve_task_collection_root
 from benchflow.trajectories.export_prime_sft import (
     PrimeSftTrajectoryJsonlError,
@@ -268,6 +269,62 @@ def write_health_summary(path: Path, job_dir: Path) -> dict[str, Any]:
     return summary
 
 
+def _escape_md_cell(text: str) -> str:
+    return text.replace("|", "\\|").replace("\n", " ").strip()
+
+
+_ISSUE_CELL_LIMIT = 120
+
+
+def canonical_task_rows(job_dir: Path) -> list[dict[str, Any]]:
+    """One row per task_id — the best-scored/highest-reward rollout.
+
+    Reused by the README summary and ``--eval-results-model`` so retry
+    attempts (multiple rollout dirs for the same task) are never counted
+    or averaged as if they were distinct tasks.
+    """
+    return build_canonical_selection(job_dir, policy="one-healthy-per-task")["selected"]
+
+
+def render_run_summary_markdown(
+    job_dir: Path, *, agent: str | None = None, model: str | None = None
+) -> str:
+    """Render a Markdown run summary (results + issues) for publish folders."""
+    rows = canonical_task_rows(job_dir)
+    rewards = [row["reward"] for row in rows if row["scored"]]
+    mean_reward = sum(rewards) / len(rewards) if rewards else None
+    scored_count = sum(1 for row in rows if row["scored"])
+    tool_call_count = sum(1 for row in rows if row["tool_calls"] > 0)
+
+    lines = ["# BenchFlow run summary", ""]
+    if agent:
+        lines.append(f"- Agent: `{agent}`")
+    if model:
+        lines.append(f"- Model: `{model}`")
+    lines += [
+        f"- Tasks: {len(rows)}",
+        f"- Scored: {scored_count} / Unscored: {len(rows) - scored_count}",
+        f"- Mean reward: {mean_reward:.3f}"
+        if mean_reward is not None
+        else "- Mean reward: n/a",
+        f"- Tasks with tool calls: {tool_call_count}",
+        "",
+        "## Tasks",
+        "",
+        "| Task | Reward | Tool calls | Issue |",
+        "| --- | --- | --- | --- |",
+    ]
+    for row in rows:
+        reward = f"{row['reward']:.3f}" if isinstance(row["reward"], float) else "—"
+        issue = row.get("error") or row.get("verifier_error") or ""
+        issue = truncate_end(_escape_md_cell(str(issue)), _ISSUE_CELL_LIMIT)
+        lines.append(
+            f"| {_escape_md_cell(str(row['task_id']))} | {reward} | "
+            f"{row['tool_calls']} | {issue} |"
+        )
+    return "\n".join(lines) + "\n"
+
+
 def _canonical_score(row: dict[str, Any]) -> tuple[int, float, int]:
     scored = 1 if row["scored"] else 0
     reward = row["reward"] if isinstance(row["reward"], float) else float("-inf")

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2096}/src/benchflow/eval_plan.py

@@ -126,6 +126,10 @@ class EvalCreateRequest:
     publish_hf: str | None = None
     hf_prefix: str | None = None
     hf_public_read_check: bool = False
+    publish_bucket: str | None = None
+    eval_results_model: str | None = None
+    eval_results_dataset: str | None = None
+    eval_results_task: str | None = None
     matrix: Path | None = None
     trials: int = 1
 
@@ -274,8 +278,14 @@ def build_eval_plan(request: EvalCreateRequest) -> EvalPlan:
         raise EvalPlanError("--retry-attempts must be >= 0")
     if request.retry_concurrency is not None and request.retry_concurrency < 1:
         raise EvalPlanError("--retry-concurrency must be >= 1")
-    if request.hf_prefix and not request.publish_hf:
-        raise EvalPlanError("--hf-prefix requires --publish-hf")
+    if request.hf_prefix and not (request.publish_hf or request.publish_bucket):
+        raise EvalPlanError("--hf-prefix requires --publish-hf or --publish-bucket")
+    if request.eval_results_model and not (
+        request.eval_results_dataset and request.eval_results_task
+    ):
+        raise EvalPlanError(
+            "--eval-results-model requires --eval-results-dataset and --eval-results-task"
+        )
     if request.tasks_dir and not Path(request.tasks_dir).exists():
         raise EvalPlanError(f"--tasks-dir not found: {request.tasks_dir}")
     if request.matrix is not None and not Path(request.matrix).is_file():

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2096}/src/benchflow/publish/huggingface.py

@@ -38,6 +38,14 @@ def _resolve_url(repo_id: str, repo_type: str, path_in_repo: str) -> str:
     return f"https://huggingface.co/{kind}{repo_id}/resolve/main/{path_in_repo}"
 
 
+def _bucket_tree_url(bucket_id: str, path_in_repo: str) -> str:
+    # Buckets are non-versioned (no "main" branch), so the browsable listing
+    # is /tree/<path> directly, not /resolve/<path> (which 404s) or
+    # /tree/main/<path> (the git-repo convention _tree_url uses).
+    suffix = f"/tree/{path_in_repo.strip('/')}" if path_in_repo else ""
+    return f"https://huggingface.co/buckets/{bucket_id}{suffix}"
+
+
 def _check_public_files(repo_id: str, repo_type: str, path_in_repo: str) -> None:
     index_url = _tree_url(repo_id, repo_type, path_in_repo)
     response = httpx.get(index_url, follow_redirects=True, timeout=20)
@@ -79,6 +87,106 @@ def publish_folder_to_hf(
     )
 
 
+def publish_folder_to_bucket(
+    folder: Path,
+    *,
+    bucket_id: str,
+    path_in_repo: str = "",
+    private: bool = False,
+) -> HfPublishResult:
+    if not folder.is_dir():
+        raise ValueError(f"publish source folder not found: {folder}")
+    try:
+        from huggingface_hub import create_bucket, sync_bucket
+        from huggingface_hub.errors import HfHubHTTPError
+    except ImportError as exc:  # pragma: no cover - depends on env
+        raise ValueError(
+            "huggingface_hub with bucket support (create_bucket/sync_bucket) is "
+            "required for --publish-bucket; upgrade huggingface_hub"
+        ) from exc
+    try:
+        create_bucket(bucket_id, private=private)
+    except HfHubHTTPError as exc:
+        if exc.response is None or exc.response.status_code != 409:
+            raise
+    prefix = path_in_repo.strip("/")
+    remote = (
+        f"hf://buckets/{bucket_id}/{prefix}" if prefix else f"hf://buckets/{bucket_id}"
+    )
+    sync_bucket(str(folder), remote)
+    return HfPublishResult(
+        repo_id=bucket_id,
+        repo_type="bucket",
+        path_in_repo=prefix,
+        url=_bucket_tree_url(bucket_id, prefix),
+    )
+
+
+_EVAL_RESULTS_PATH = ".eval_results/benchflow.yaml"
+
+
+def _existing_eval_results(model_repo: str) -> list[dict]:
+    from huggingface_hub import hf_hub_download
+    from huggingface_hub.errors import EntryNotFoundError, RepositoryNotFoundError
+
+    try:
+        downloaded = hf_hub_download(
+            repo_id=model_repo, filename=_EVAL_RESULTS_PATH, repo_type="model"
+        )
+    except (EntryNotFoundError, RepositoryNotFoundError):
+        return []
+    import yaml
+
+    data = yaml.safe_load(Path(downloaded).read_text())
+    return data if isinstance(data, list) else []
+
+
+def _upsert_eval_result(existing: list[dict], entry: dict) -> list[dict]:
+    key = (entry["dataset"]["id"], entry["dataset"]["task_id"])
+    kept = [
+        row
+        for row in existing
+        if (row.get("dataset", {}).get("id"), row.get("dataset", {}).get("task_id"))
+        != key
+    ]
+    return [*kept, entry]
+
+
+def open_eval_results_pr(
+    *,
+    model_repo: str,
+    dataset_id: str,
+    task_id: str,
+    value: float,
+    source_url: str | None = None,
+    notes: str | None = None,
+) -> str | None:
+    try:
+        import yaml
+        from huggingface_hub import CommitOperationAdd, HfApi
+    except ImportError as exc:  # pragma: no cover - depends on env
+        raise ValueError(
+            "huggingface_hub is required for --eval-results-model"
+        ) from exc
+    entry: dict = {"dataset": {"id": dataset_id, "task_id": task_id}, "value": value}
+    if source_url:
+        entry["source"] = {"url": source_url}
+    if notes:
+        entry["notes"] = notes
+    merged = _upsert_eval_result(_existing_eval_results(model_repo), entry)
+    content = yaml.safe_dump(merged, sort_keys=False).encode("utf-8")
+    commit = HfApi().create_commit(
+        repo_id=model_repo,
+        repo_type="model",
+        operations=[
+            CommitOperationAdd(path_in_repo=_EVAL_RESULTS_PATH, path_or_fileobj=content)
+        ],
+        commit_message=f"Add {dataset_id} eval results",
+        create_pr=True,
+    )
+    return commit.pr_url
+
+
 def publish_file_to_hf(
     file_path: Path,
     *,

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2096}/src/benchflow/sandbox/setup.py

@@ -568,6 +568,37 @@ def _inject_skills_into_dockerfile(
     )
 
 
+def _decode_mountinfo_path(value: str) -> str:
+    """Decode the octal escapes used by ``/proc/self/mountinfo``."""
+    return (
+        value.replace("\\040", " ")
+        .replace("\\011", "\t")
+        .replace("\\012", "\n")
+        .replace("\\134", "\\")
+    )
+
+
+def _mountinfo_path_mapping(cwd: str, mountinfo: str) -> tuple[str, str] | None:
+    """Return host-source/container-destination for a containing bind mount."""
+    current = Path(cwd)
+    best: tuple[str, str] | None = None
+    for line in mountinfo.splitlines():
+        fields = line.split(" - ", 1)[0].split()
+        if len(fields) < 5:
+            continue
+        root = _decode_mountinfo_path(fields[3])
+        dest = _decode_mountinfo_path(fields[4])
+        try:
+            current.relative_to(dest)
+        except ValueError:
+            continue
+        if root == "/" or dest == "/":
+            continue
+        if best is None or len(dest) > len(best[1]):
+            best = (root, dest)
+    return best
+
+
 def _detect_dind_mount() -> tuple[str, str] | None:
     """Detect Docker-in-Docker host path translation.
 
@@ -576,30 +607,33 @@ def _detect_dind_mount() -> tuple[str, str] | None:
 
     Returns (host_source, container_dest) tuple, or None if not in DinD.
     """
-    if not Path("/.dockerenv").exists():
-        return None
     import subprocess as _sp
 
     try:
-        hostname = _sp.check_output(["hostname"], text=True).strip()
-        result = _sp.run(
-            ["docker", "inspect", hostname],
-            capture_output=True,
-            text=True,
-            timeout=5,
-        )
-        if result.returncode != 0:
-            return None
-        data = json.loads(result.stdout)
         cwd = str(Path.cwd())
-        best = None
-        for mount in data[0].get("Mounts", []):
-            if mount.get("Type") != "bind":
-                continue
-            dest = mount.get("Destination", "")
-            if cwd.startswith(dest) and (best is None or len(dest) > len(best[1])):
-                best = (mount["Source"], dest)
-        return best
+        if Path("/.dockerenv").exists():
+            hostname = _sp.check_output(["hostname"], text=True).strip()
+            result = _sp.run(
+                ["docker", "inspect", hostname],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            if result.returncode == 0:
+                data = json.loads(result.stdout)
+                best = None
+                for mount in data[0].get("Mounts", []):
+                    if mount.get("Type") != "bind":
+                        continue
+                    dest = mount.get("Destination", "")
+                    if cwd.startswith(dest) and (
+                        best is None or len(dest) > len(best[1])
+                    ):
+                        best = (mount["Source"], dest)
+                if best is not None:
+                    return best
+        mountinfo = Path("/proc/self/mountinfo").read_text(errors="replace")
+        return _mountinfo_path_mapping(cwd, mountinfo)
     except Exception:
         logger.debug("DinD mount detection failed", exc_info=True)
         return None

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2096}/src/benchflow/trajectories/results.py

@@ -30,6 +30,7 @@ from benchflow.trajectories.export_prime_sft import (
     validate_prime_sft_row,
 )
 from benchflow.trajectories.types import redact_trajectory_obj
+from benchflow.usage_tracking import USAGE_SOURCE_AGENT_NATIVE_ACP
 
 ROLLOUT_RESULTS_FILENAME = "results.jsonl"
 JOB_RESULTS_FILENAME = "results.jsonl"
@@ -465,6 +466,12 @@ def build_rollout_results_record(
         export_error=effective_export_error,
     )
     terminal_health_error = bool(error or verifier_error or partial_trajectory)
+    native_subscription_without_llm = bool(
+        (agent_result or {}).get("usage_source") == USAGE_SOURCE_AGENT_NATIVE_ACP
+        and not (rollout_path / "trajectory" / "llm_trajectory.jsonl").exists()
+        and effective_export_error is None
+        and not terminal_health_error
+    )
     training_ready = bool(
         steps
         and completion
@@ -493,7 +500,7 @@ def build_rollout_results_record(
             training_ready_reason = "verifier_error"
         else:
             training_ready_reason = "missing_healthy_structured_llm_trajectory"
-        if error_obj is None:
+        if error_obj is None and not native_subscription_without_llm:
             error_name = (
                 training_ready_reason
                 if training_ready_reason

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2096}/tests/test_benchflow_experiment_review_validator.py

@@ -133,6 +133,71 @@ def test_validator_accepts_training_ready_results_jsonl(tmp_path: Path) -> None:
     }
 
 
+def test_validator_oracle_mode_accepts_reward_only_rollout(tmp_path: Path) -> None:
+    """Guards BenchFlow PR #1049's explicit oracle artifact exception."""
+    validator = _load_validator()
+    rollout = _rollout(tmp_path)
+    result_path = rollout / "result.json"
+    result = json.loads(result_path.read_text())
+    result.update(
+        {
+            "agent": "oracle",
+            "agent_name": "oracle",
+            "model": None,
+            "n_tool_calls": 0,
+            "agent_result": {
+                "n_tool_calls": 0,
+                "n_input_tokens": 0,
+                "n_output_tokens": 0,
+                "total_tokens": 0,
+                "usage_source": "unavailable",
+            },
+        }
+    )
+    result_path.write_text(json.dumps(result))
+    (rollout / "trajectory" / "llm_trajectory.jsonl").unlink()
+    _write_jsonl(
+        rollout / "results.jsonl",
+        [
+            {
+                "example_id": 0,
+                "prompt": [{"role": "user", "content": "solve"}],
+                "completion": None,
+                "info": {
+                    "task_id": "task-a",
+                    "training_ready": False,
+                    "training_ready_reason": (
+                        "missing_healthy_structured_llm_trajectory"
+                    ),
+                },
+                "reward": 1.0,
+                "error": {
+                    "error": "missing_llm_trajectory",
+                    "error_chain_str": "oracle has no LLM trajectory",
+                },
+                "is_completed": False,
+                "is_truncated": False,
+                "stop_condition": "agent_completed",
+                "metrics": {"n_tool_calls": 0, "reward": 1.0},
+                "tool_defs": [],
+                "token_usage": {
+                    "final_input_tokens": 0,
+                    "final_output_tokens": 0,
+                    "total_tokens": 0,
+                },
+                "trajectory": [],
+            }
+        ],
+    )
+
+    strict = validator.validate_rollout(rollout)
+    oracle = validator.validate_rollout(rollout, allow_oracle_without_llm=True)
+
+    assert strict["healthy"] is False
+    assert oracle["healthy"] is True
+    assert oracle["issues"] == []
+
+
 def test_validator_accepts_provider_total_only_token_usage(tmp_path: Path) -> None:
     """Some providers expose only total tokens; Prime-RL can still render the row."""
     validator = _load_validator()
@@ -240,6 +305,92 @@ def test_validator_rejects_unready_results_for_healthy_rollout(tmp_path: Path) -
     assert any("training_ready=false" in issue for issue in report["issues"])
 
 
+def _native_subscription_rollout(tmp_path: Path) -> Path:
+    rollout = _rollout(tmp_path)
+    (rollout / "trajectory" / "llm_trajectory.jsonl").unlink()
+    result = json.loads((rollout / "result.json").read_text())
+    result["agent_result"]["usage_source"] = "agent_native_acp"
+    (rollout / "result.json").write_text(json.dumps(result))
+    row = json.loads((rollout / "results.jsonl").read_text())
+    row.update(
+        {
+            "completion": None,
+            "error": None,
+            "is_completed": True,
+            "trajectory": [],
+        }
+    )
+    row["info"].update(
+        {
+            "training_ready": False,
+            "training_ready_reason": "missing_healthy_structured_llm_trajectory",
+        }
+    )
+    _write_jsonl(rollout / "results.jsonl", [row])
+    return rollout
+
+
+def test_validator_accepts_explicit_native_subscription_mode(tmp_path: Path) -> None:
+    """Guards this PR's explicit native-subscription evidence mode."""
+    validator = _load_validator()
+    rollout = _native_subscription_rollout(tmp_path)
+
+    strict = validator.validate_rollout(rollout)
+    native = validator.validate_rollout(
+        rollout, allow_native_subscription_without_llm=True
+    )
+
+    assert strict["healthy"] is False
+    assert native["healthy"] is True
+    assert any("native subscription rollout" in item for item in native["warnings"])
+
+
+def test_validator_native_mode_rejects_wrong_usage_source(tmp_path: Path) -> None:
+    """Guards this PR against widening subscription mode to other runs."""
+    validator = _load_validator()
+    rollout = _native_subscription_rollout(tmp_path)
+    result = json.loads((rollout / "result.json").read_text())
+    result["agent_result"]["usage_source"] = "provider_response"
+    (rollout / "result.json").write_text(json.dumps(result))
+
+    report = validator.validate_rollout(
+        rollout, allow_native_subscription_without_llm=True
+    )
+
+    assert report["healthy"] is False
+    assert any("missing required artifact" in issue for issue in report["issues"])
+
+
+def test_validator_native_mode_requires_token_components(tmp_path: Path) -> None:
+    """Guards this PR against accepting usage-free subscription evidence."""
+    validator = _load_validator()
+    rollout = _native_subscription_rollout(tmp_path)
+    row = json.loads((rollout / "results.jsonl").read_text())
+    row["token_usage"] = {"total_tokens": 17}
+    _write_jsonl(rollout / "results.jsonl", [row])
+
+    report = validator.validate_rollout(
+        rollout, allow_native_subscription_without_llm=True
+    )
+
+    assert report["healthy"] is False
+    assert any("positive token components" in issue for issue in report["issues"])
+
+
+def test_validator_native_mode_rejects_malformed_present_llm(tmp_path: Path) -> None:
+    """Guards this PR against bypassing malformed provider evidence."""
+    validator = _load_validator()
+    rollout = _native_subscription_rollout(tmp_path)
+    (rollout / "trajectory" / "llm_trajectory.jsonl").write_text("{\n")
+
+    report = validator.validate_rollout(
+        rollout, allow_native_subscription_without_llm=True
+    )
+
+    assert report["healthy"] is False
+    assert any("JSON parse error" in issue for issue in report["issues"])
+
+
 def test_validator_rejects_results_with_dropped_successful_exchange(
     tmp_path: Path,
 ) -> None:

{benchflow-0.7.6.dev2058 → benchflow-0.7.6.dev2096}/tests/test_cli_arg_validation.py

@@ -393,6 +393,29 @@ def test_agent_timeout_sec_accepts_positive_or_none(ok):
 # --- v0.6 CLI-dogfood guards: clean fail-fast instead of raw tracebacks -------
 
 
+def test_hf_prefix_accepted_with_publish_bucket(tmp_path: Path):
+    # Guards against --hf-prefix rejecting --publish-bucket: the bucket
+    # postprocess path reads req.hf_prefix, so it must be a valid pairing.
+    plan = build_eval_plan(
+        EvalCreateRequest(
+            tasks_dir=_task_dir(tmp_path),
+            agent="oracle",
+            publish_bucket="org/bucket",
+            hf_prefix="runs/x",
+        )
+    )
+    assert plan.request.hf_prefix == "runs/x"
+
+
+def test_hf_prefix_without_publish_hf_or_bucket_rejected(tmp_path: Path):
+    with pytest.raises(EvalPlanError, match="--publish-hf or --publish-bucket"):
+        build_eval_plan(
+            EvalCreateRequest(
+                tasks_dir=_task_dir(tmp_path), agent="oracle", hf_prefix="runs/x"
+            )
+        )
+
+
 def test_config_missing_path_rejected_cleanly(tmp_path: Path):
     # Was: a raw FileNotFoundError traceback from Evaluation.from_yaml's open().
     with pytest.raises(EvalPlanError, match="--config not found"):