benchflow 0.7.6.dev2072__tar.gz → 0.7.6.dev2096__tar.gz

{benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2096}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.6.dev2072
+Version: 0.7.6.dev2096
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2096}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.6.dev2072"
+version = "0.7.6.dev2096"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2096}/src/benchflow/cli/eval_artifacts.py

@@ -66,7 +66,9 @@ def postprocess_eval_artifacts(
     eval_config = _apply_source_sidecar(eval_config, resolved_tasks_dir)
     from benchflow.eval_artifacts import (
         TaskManifestOptions,
+        canonical_task_rows,
         materialize_canonical_job,
+        render_run_summary_markdown,
         write_canonical_selection,
         write_health_summary,
         write_task_manifest,
@@ -138,6 +140,12 @@ def postprocess_eval_artifacts(
             console.print(
                 f"[green]Canonical jobs:[/green] {escape(str(req.canonical_jobs_dir))}"
             )
+    if req.publish_hf is not None or req.publish_bucket is not None:
+        (job_dir / "README.md").write_text(
+            render_run_summary_markdown(
+                job_dir, agent=eval_config.agent, model=eval_config.model
+            )
+        )
     if req.publish_hf is not None:
         from benchflow.publish.huggingface import publish_folder_to_hf
 
@@ -154,6 +162,42 @@ def postprocess_eval_artifacts(
             print_error(str(exc))
             raise typer.Exit(1) from None
         console.print(f"[green]Published HF artifacts:[/green] {escape(published.url)}")
+    if req.publish_bucket is not None:
+        from benchflow.publish.huggingface import publish_folder_to_bucket
+
+        prefix = req.hf_prefix or job_dir.name
+        try:
+            published = publish_folder_to_bucket(
+                job_dir, bucket_id=req.publish_bucket, path_in_repo=prefix
+            )
+        except ValueError as exc:
+            print_error(str(exc))
+            raise typer.Exit(1) from None
+        console.print(f"[green]Published HF bucket:[/green] {escape(published.url)}")
+    if req.eval_results_model is not None:
+        from benchflow.publish.huggingface import open_eval_results_pr
+
+        rewards = [
+            row["reward"] for row in canonical_task_rows(job_dir) if row["scored"]
+        ]
+        if rewards:
+            value = round(100.0 * sum(rewards) / len(rewards), 2)
+            try:
+                pr_url = open_eval_results_pr(
+                    model_repo=req.eval_results_model,
+                    dataset_id=cast(str, req.eval_results_dataset),
+                    task_id=cast(str, req.eval_results_task),
+                    value=value,
+                )
+            except ValueError as exc:
+                print_error(str(exc))
+                raise typer.Exit(1) from None
+            console.print(f"[green]Eval-results PR:[/green] {escape(pr_url or '')}")
+        else:
+            console.print(
+                "[yellow]--eval-results-model set but no scored rollouts were "
+                "found; skipping eval-results PR[/yellow]"
+            )
 
 
 def _load_eval_matrix(path: Path) -> dict[str, dict]:

{benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2096}/src/benchflow/cli/main.py

@@ -581,6 +581,31 @@ def eval_run(
             "--hf-public-read-check", help="Verify public HF reads after upload"
         ),
     ] = False,
+    publish_bucket: Annotated[
+        str | None,
+        typer.Option(
+            "--publish-bucket", help="Upload final eval artifacts to a HF bucket"
+        ),
+    ] = None,
+    eval_results_model: Annotated[
+        str | None,
+        typer.Option(
+            "--eval-results-model",
+            help="HF model repo to open a community eval-results PR on",
+        ),
+    ] = None,
+    eval_results_dataset: Annotated[
+        str | None,
+        typer.Option(
+            "--eval-results-dataset", help="HF benchmark dataset id, e.g. org/benchmark"
+        ),
+    ] = None,
+    eval_results_task: Annotated[
+        str | None,
+        typer.Option(
+            "--eval-results-task", help="Benchmark task_id from the dataset's eval.yaml"
+        ),
+    ] = None,
     matrix: Annotated[
         Path | None,
         typer.Option("--matrix", help="YAML model matrix for repeated evals"),
@@ -653,6 +678,10 @@ def eval_run(
         publish_hf=publish_hf,
         hf_prefix=hf_prefix,
         hf_public_read_check=hf_public_read_check,
+        publish_bucket=publish_bucket,
+        eval_results_model=eval_results_model,
+        eval_results_dataset=eval_results_dataset,
+        eval_results_task=eval_results_task,
         matrix=matrix,
         trials=trials,
     )

{benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2096}/src/benchflow/eval_artifacts.py

@@ -9,6 +9,7 @@ from pathlib import Path
 from typing import Any, Literal
 
 from benchflow._utils.task_authoring import task_digest
+from benchflow._utils.text import truncate_end
 from benchflow.task.discovery import is_task_dir, resolve_task_collection_root
 from benchflow.trajectories.export_prime_sft import (
     PrimeSftTrajectoryJsonlError,
@@ -268,6 +269,62 @@ def write_health_summary(path: Path, job_dir: Path) -> dict[str, Any]:
     return summary
 
 
+def _escape_md_cell(text: str) -> str:
+    return text.replace("|", "\\|").replace("\n", " ").strip()
+
+
+_ISSUE_CELL_LIMIT = 120
+
+
+def canonical_task_rows(job_dir: Path) -> list[dict[str, Any]]:
+    """One row per task_id — the best-scored/highest-reward rollout.
+
+    Reused by the README summary and ``--eval-results-model`` so retry
+    attempts (multiple rollout dirs for the same task) are never counted
+    or averaged as if they were distinct tasks.
+    """
+    return build_canonical_selection(job_dir, policy="one-healthy-per-task")["selected"]
+
+
+def render_run_summary_markdown(
+    job_dir: Path, *, agent: str | None = None, model: str | None = None
+) -> str:
+    """Render a Markdown run summary (results + issues) for publish folders."""
+    rows = canonical_task_rows(job_dir)
+    rewards = [row["reward"] for row in rows if row["scored"]]
+    mean_reward = sum(rewards) / len(rewards) if rewards else None
+    scored_count = sum(1 for row in rows if row["scored"])
+    tool_call_count = sum(1 for row in rows if row["tool_calls"] > 0)
+
+    lines = ["# BenchFlow run summary", ""]
+    if agent:
+        lines.append(f"- Agent: `{agent}`")
+    if model:
+        lines.append(f"- Model: `{model}`")
+    lines += [
+        f"- Tasks: {len(rows)}",
+        f"- Scored: {scored_count} / Unscored: {len(rows) - scored_count}",
+        f"- Mean reward: {mean_reward:.3f}"
+        if mean_reward is not None
+        else "- Mean reward: n/a",
+        f"- Tasks with tool calls: {tool_call_count}",
+        "",
+        "## Tasks",
+        "",
+        "| Task | Reward | Tool calls | Issue |",
+        "| --- | --- | --- | --- |",
+    ]
+    for row in rows:
+        reward = f"{row['reward']:.3f}" if isinstance(row["reward"], float) else "—"
+        issue = row.get("error") or row.get("verifier_error") or ""
+        issue = truncate_end(_escape_md_cell(str(issue)), _ISSUE_CELL_LIMIT)
+        lines.append(
+            f"| {_escape_md_cell(str(row['task_id']))} | {reward} | "
+            f"{row['tool_calls']} | {issue} |"
+        )
+    return "\n".join(lines) + "\n"
+
+
 def _canonical_score(row: dict[str, Any]) -> tuple[int, float, int]:
     scored = 1 if row["scored"] else 0
     reward = row["reward"] if isinstance(row["reward"], float) else float("-inf")

{benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2096}/src/benchflow/eval_plan.py

@@ -126,6 +126,10 @@ class EvalCreateRequest:
     publish_hf: str | None = None
     hf_prefix: str | None = None
     hf_public_read_check: bool = False
+    publish_bucket: str | None = None
+    eval_results_model: str | None = None
+    eval_results_dataset: str | None = None
+    eval_results_task: str | None = None
     matrix: Path | None = None
     trials: int = 1
 
@@ -274,8 +278,14 @@ def build_eval_plan(request: EvalCreateRequest) -> EvalPlan:
         raise EvalPlanError("--retry-attempts must be >= 0")
     if request.retry_concurrency is not None and request.retry_concurrency < 1:
         raise EvalPlanError("--retry-concurrency must be >= 1")
-    if request.hf_prefix and not request.publish_hf:
-        raise EvalPlanError("--hf-prefix requires --publish-hf")
+    if request.hf_prefix and not (request.publish_hf or request.publish_bucket):
+        raise EvalPlanError("--hf-prefix requires --publish-hf or --publish-bucket")
+    if request.eval_results_model and not (
+        request.eval_results_dataset and request.eval_results_task
+    ):
+        raise EvalPlanError(
+            "--eval-results-model requires --eval-results-dataset and --eval-results-task"
+        )
     if request.tasks_dir and not Path(request.tasks_dir).exists():
         raise EvalPlanError(f"--tasks-dir not found: {request.tasks_dir}")
     if request.matrix is not None and not Path(request.matrix).is_file():

{benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2096}/src/benchflow/publish/huggingface.py

@@ -38,6 +38,14 @@ def _resolve_url(repo_id: str, repo_type: str, path_in_repo: str) -> str:
     return f"https://huggingface.co/{kind}{repo_id}/resolve/main/{path_in_repo}"
 
 
+def _bucket_tree_url(bucket_id: str, path_in_repo: str) -> str:
+    # Buckets are non-versioned (no "main" branch), so the browsable listing
+    # is /tree/<path> directly, not /resolve/<path> (which 404s) or
+    # /tree/main/<path> (the git-repo convention _tree_url uses).
+    suffix = f"/tree/{path_in_repo.strip('/')}" if path_in_repo else ""
+    return f"https://huggingface.co/buckets/{bucket_id}{suffix}"
+
+
 def _check_public_files(repo_id: str, repo_type: str, path_in_repo: str) -> None:
     index_url = _tree_url(repo_id, repo_type, path_in_repo)
     response = httpx.get(index_url, follow_redirects=True, timeout=20)
@@ -79,6 +87,106 @@ def publish_folder_to_hf(
     )
 
 
+def publish_folder_to_bucket(
+    folder: Path,
+    *,
+    bucket_id: str,
+    path_in_repo: str = "",
+    private: bool = False,
+) -> HfPublishResult:
+    if not folder.is_dir():
+        raise ValueError(f"publish source folder not found: {folder}")
+    try:
+        from huggingface_hub import create_bucket, sync_bucket
+        from huggingface_hub.errors import HfHubHTTPError
+    except ImportError as exc:  # pragma: no cover - depends on env
+        raise ValueError(
+            "huggingface_hub with bucket support (create_bucket/sync_bucket) is "
+            "required for --publish-bucket; upgrade huggingface_hub"
+        ) from exc
+    try:
+        create_bucket(bucket_id, private=private)
+    except HfHubHTTPError as exc:
+        if exc.response is None or exc.response.status_code != 409:
+            raise
+    prefix = path_in_repo.strip("/")
+    remote = (
+        f"hf://buckets/{bucket_id}/{prefix}" if prefix else f"hf://buckets/{bucket_id}"
+    )
+    sync_bucket(str(folder), remote)
+    return HfPublishResult(
+        repo_id=bucket_id,
+        repo_type="bucket",
+        path_in_repo=prefix,
+        url=_bucket_tree_url(bucket_id, prefix),
+    )
+
+
+_EVAL_RESULTS_PATH = ".eval_results/benchflow.yaml"
+
+
+def _existing_eval_results(model_repo: str) -> list[dict]:
+    from huggingface_hub import hf_hub_download
+    from huggingface_hub.errors import EntryNotFoundError, RepositoryNotFoundError
+
+    try:
+        downloaded = hf_hub_download(
+            repo_id=model_repo, filename=_EVAL_RESULTS_PATH, repo_type="model"
+        )
+    except (EntryNotFoundError, RepositoryNotFoundError):
+        return []
+    import yaml
+
+    data = yaml.safe_load(Path(downloaded).read_text())
+    return data if isinstance(data, list) else []
+
+
+def _upsert_eval_result(existing: list[dict], entry: dict) -> list[dict]:
+    key = (entry["dataset"]["id"], entry["dataset"]["task_id"])
+    kept = [
+        row
+        for row in existing
+        if (row.get("dataset", {}).get("id"), row.get("dataset", {}).get("task_id"))
+        != key
+    ]
+    return [*kept, entry]
+
+
+def open_eval_results_pr(
+    *,
+    model_repo: str,
+    dataset_id: str,
+    task_id: str,
+    value: float,
+    source_url: str | None = None,
+    notes: str | None = None,
+) -> str | None:
+    try:
+        import yaml
+        from huggingface_hub import CommitOperationAdd, HfApi
+    except ImportError as exc:  # pragma: no cover - depends on env
+        raise ValueError(
+            "huggingface_hub is required for --eval-results-model"
+        ) from exc
+    entry: dict = {"dataset": {"id": dataset_id, "task_id": task_id}, "value": value}
+    if source_url:
+        entry["source"] = {"url": source_url}
+    if notes:
+        entry["notes"] = notes
+    merged = _upsert_eval_result(_existing_eval_results(model_repo), entry)
+    content = yaml.safe_dump(merged, sort_keys=False).encode("utf-8")
+    commit = HfApi().create_commit(
+        repo_id=model_repo,
+        repo_type="model",
+        operations=[
+            CommitOperationAdd(path_in_repo=_EVAL_RESULTS_PATH, path_or_fileobj=content)
+        ],
+        commit_message=f"Add {dataset_id} eval results",
+        create_pr=True,
+    )
+    return commit.pr_url
+
+
 def publish_file_to_hf(
     file_path: Path,
     *,

{benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2096}/tests/test_cli_arg_validation.py

@@ -393,6 +393,29 @@ def test_agent_timeout_sec_accepts_positive_or_none(ok):
 # --- v0.6 CLI-dogfood guards: clean fail-fast instead of raw tracebacks -------
 
 
+def test_hf_prefix_accepted_with_publish_bucket(tmp_path: Path):
+    # Guards against --hf-prefix rejecting --publish-bucket: the bucket
+    # postprocess path reads req.hf_prefix, so it must be a valid pairing.
+    plan = build_eval_plan(
+        EvalCreateRequest(
+            tasks_dir=_task_dir(tmp_path),
+            agent="oracle",
+            publish_bucket="org/bucket",
+            hf_prefix="runs/x",
+        )
+    )
+    assert plan.request.hf_prefix == "runs/x"
+
+
+def test_hf_prefix_without_publish_hf_or_bucket_rejected(tmp_path: Path):
+    with pytest.raises(EvalPlanError, match="--publish-hf or --publish-bucket"):
+        build_eval_plan(
+            EvalCreateRequest(
+                tasks_dir=_task_dir(tmp_path), agent="oracle", hf_prefix="runs/x"
+            )
+        )
+
+
 def test_config_missing_path_rejected_cleanly(tmp_path: Path):
     # Was: a raw FileNotFoundError traceback from Evaluation.from_yaml's open().
     with pytest.raises(EvalPlanError, match="--config not found"):

{benchflow-0.7.6.dev2072 → benchflow-0.7.6.dev2096}/tests/test_eval_artifact_cli.py

@@ -142,6 +142,254 @@ def test_eval_run_writes_manifest_health_and_canonical_artifacts(
     assert (canonical_jobs / "task-a__abc" / "result.json").is_file()
 
 
+def test_eval_run_publish_bucket_writes_readme_summary(
+    tmp_path: Path, monkeypatch
+) -> None:
+    """--publish-bucket writes a README.md run summary into job_dir before upload."""
+    tasks = tmp_path / "tasks"
+    _write_task(tasks / "task-a")
+
+    async def fake_run(self):
+        job_dir = self._jobs_dir / self._job_name
+        _write_rollout(job_dir / "task-a__abc", "task-a")
+        return SimpleNamespace(
+            job_name=self._job_name,
+            passed=1,
+            failed=0,
+            errored=0,
+            verifier_errored=0,
+            total=1,
+            score=1.0,
+            score_excl_errors=1.0,
+        )
+
+    monkeypatch.setattr(Evaluation, "run", fake_run)
+
+    captured: dict[str, Path] = {}
+
+    def fake_publish_folder_to_bucket(
+        folder, *, bucket_id, path_in_repo="", private=False
+    ):
+        captured["folder"] = folder
+        return SimpleNamespace(url=f"https://huggingface.co/buckets/{bucket_id}")
+
+    import benchflow.publish.huggingface as hf_publish
+
+    monkeypatch.setattr(
+        hf_publish, "publish_folder_to_bucket", fake_publish_folder_to_bucket
+    )
+
+    jobs_dir = tmp_path / "jobs"
+    result = runner.invoke(
+        app,
+        [
+            "eval",
+            "run",
+            "--tasks-dir",
+            str(tasks),
+            "--agent",
+            "oracle",
+            "--jobs-dir",
+            str(jobs_dir),
+            "--publish-bucket",
+            "org/some-bucket",
+        ],
+    )
+
+    assert result.exit_code == 0, result.output
+    readme = captured["folder"] / "README.md"
+    assert readme.is_file()
+    text = readme.read_text()
+    assert "task-a" in text
+    assert "Mean reward: 1.000" in text
+    assert "`oracle`" in text
+
+
+def test_eval_run_publish_bucket_readme_dedupes_retry_attempts(
+    tmp_path: Path, monkeypatch
+) -> None:
+    """Retry rollout dirs for the same task must count as one task, best-scored.
+
+    Guards against README.md (and the --eval-results-model reward mean)
+    treating each retry attempt as a distinct task.
+    """
+    tasks = tmp_path / "tasks"
+    _write_task(tasks / "task-a")
+
+    async def fake_run(self):
+        job_dir = self._jobs_dir / self._job_name
+        _write_rollout(job_dir / "task-a__attempt1", "task-a")
+        (job_dir / "task-a__attempt1" / "result.json").write_text(
+            json.dumps({"task_name": "task-a", "rewards": {"reward": 0.0}}),
+            encoding="utf-8",
+        )
+        _write_rollout(job_dir / "task-a__attempt2", "task-a")
+        return SimpleNamespace(
+            job_name=self._job_name,
+            passed=1,
+            failed=0,
+            errored=0,
+            verifier_errored=0,
+            total=1,
+            score=1.0,
+            score_excl_errors=1.0,
+        )
+
+    monkeypatch.setattr(Evaluation, "run", fake_run)
+
+    captured: dict[str, Path] = {}
+
+    def fake_publish_folder_to_bucket(
+        folder, *, bucket_id, path_in_repo="", private=False
+    ):
+        captured["folder"] = folder
+        return SimpleNamespace(url=f"https://huggingface.co/buckets/{bucket_id}")
+
+    import benchflow.publish.huggingface as hf_publish
+
+    monkeypatch.setattr(
+        hf_publish, "publish_folder_to_bucket", fake_publish_folder_to_bucket
+    )
+
+    result = runner.invoke(
+        app,
+        [
+            "eval",
+            "run",
+            "--tasks-dir",
+            str(tasks),
+            "--agent",
+            "oracle",
+            "--jobs-dir",
+            str(tmp_path / "jobs"),
+            "--publish-bucket",
+            "org/some-bucket",
+        ],
+    )
+
+    assert result.exit_code == 0, result.output
+    text = (captured["folder"] / "README.md").read_text()
+    assert "- Tasks: 1" in text
+    assert "- Mean reward: 1.000" in text
+    assert text.count("| task-a |") == 1
+
+
+def test_eval_run_eval_results_dedupes_retry_attempts(
+    tmp_path: Path, monkeypatch
+) -> None:
+    """The --eval-results-model reward mean must not double-count retries."""
+    tasks = tmp_path / "tasks"
+    _write_task(tasks / "task-a")
+
+    async def fake_run(self):
+        job_dir = self._jobs_dir / self._job_name
+        _write_rollout(job_dir / "task-a__attempt1", "task-a")
+        (job_dir / "task-a__attempt1" / "result.json").write_text(
+            json.dumps({"task_name": "task-a", "rewards": {"reward": 0.0}}),
+            encoding="utf-8",
+        )
+        _write_rollout(job_dir / "task-a__attempt2", "task-a")
+        return SimpleNamespace(
+            job_name=self._job_name,
+            passed=1,
+            failed=0,
+            errored=0,
+            verifier_errored=0,
+            total=1,
+            score=1.0,
+            score_excl_errors=1.0,
+        )
+
+    monkeypatch.setattr(Evaluation, "run", fake_run)
+
+    captured: dict[str, float] = {}
+
+    def fake_open_eval_results_pr(
+        *, model_repo, dataset_id, task_id, value, source_url=None, notes=None
+    ):
+        captured["value"] = value
+        return "https://huggingface.co/org/model/discussions/1"
+
+    import benchflow.publish.huggingface as hf_publish
+
+    monkeypatch.setattr(hf_publish, "open_eval_results_pr", fake_open_eval_results_pr)
+
+    result = runner.invoke(
+        app,
+        [
+            "eval",
+            "run",
+            "--tasks-dir",
+            str(tasks),
+            "--agent",
+            "oracle",
+            "--jobs-dir",
+            str(tmp_path / "jobs"),
+            "--eval-results-model",
+            "org/model",
+            "--eval-results-dataset",
+            "org/bench",
+            "--eval-results-task",
+            "t1",
+        ],
+    )
+
+    assert result.exit_code == 0, result.output
+    # Best-of-retries reward is 1.0 -> 100.0, not the naive two-attempt mean (50.0).
+    assert captured["value"] == 100.0
+
+
+def test_eval_run_eval_results_warns_when_nothing_scored(
+    tmp_path: Path, monkeypatch
+) -> None:
+    tasks = tmp_path / "tasks"
+    _write_task(tasks / "task-a")
+
+    async def fake_run(self):
+        job_dir = self._jobs_dir / self._job_name
+        rollout = job_dir / "task-a__abc"
+        rollout.mkdir(parents=True)
+        (rollout / "result.json").write_text(
+            json.dumps({"task_name": "task-a", "rewards": None}),
+            encoding="utf-8",
+        )
+        return SimpleNamespace(
+            job_name=self._job_name,
+            passed=0,
+            failed=1,
+            errored=0,
+            verifier_errored=0,
+            total=1,
+            score=0.0,
+            score_excl_errors=0.0,
+        )
+
+    monkeypatch.setattr(Evaluation, "run", fake_run)
+
+    result = runner.invoke(
+        app,
+        [
+            "eval",
+            "run",
+            "--tasks-dir",
+            str(tasks),
+            "--agent",
+            "oracle",
+            "--jobs-dir",
+            str(tmp_path / "jobs"),
+            "--eval-results-model",
+            "org/model",
+            "--eval-results-dataset",
+            "org/bench",
+            "--eval-results-task",
+            "t1",
+        ],
+    )
+
+    assert result.exit_code == 0, result.output
+    assert "no scored rollouts" in result.output
+
+
 def test_sharded_health_and_selection_discover_worker_shards(tmp_path: Path) -> None:
     jobs_dir = tmp_path / "jobs"
     advertised_job_dir = jobs_dir / "worker-sharded"