benchflow 0.5.3.dev983__tar.gz → 0.5.3.dev994__tar.gz

{benchflow-0.5.3.dev983 → benchflow-0.5.3.dev994}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.5.3.dev983
+Version: 0.5.3.dev994
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.5.3.dev983 → benchflow-0.5.3.dev994}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.5.3.dev983"
+version = "0.5.3.dev994"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.5.3.dev983 → benchflow-0.5.3.dev994}/src/benchflow/cli/main.py

@@ -1109,6 +1109,14 @@ def eval_create(
             "(default: the skillsbench registry). Only valid with --dataset.",
         ),
     ] = None,
+    ignore_bench_version: Annotated[
+        bool,
+        typer.Option(
+            "--ignore-bench-version",
+            help="Run a dataset even when this bench version is outside the "
+            "range it was validated against. Only valid with --dataset.",
+        ),
+    ] = False,
 ) -> None:
     """Run an evaluation — single task or batch.
 
@@ -1140,6 +1148,9 @@ def eval_create(
     if registry and not dataset:
         console.print("[red]--registry requires --dataset[/red]")
         raise typer.Exit(1)
+    if ignore_bench_version and not dataset:
+        console.print("[red]--ignore-bench-version requires --dataset[/red]")
+        raise typer.Exit(1)
     if worker_concurrency is not None and not (tasks_dir or source_repo):
         console.print(
             "[red]--worker-concurrency is supported for --tasks-dir and --source-repo batch runs[/red]"
@@ -1383,8 +1394,20 @@ def eval_create(
             console.print(f"[red]{e}[/red]")
             raise typer.Exit(1) from None
         version_issue = bench_version_issue(resolved_dataset.bench_version)
+        if version_issue and not ignore_bench_version:
+            # Hard gate: published results must come from a harness the
+            # dataset version was validated against. The escape hatch keeps
+            # local experimentation possible without weakening the default.
+            console.print(f"[red]{version_issue}[/red]")
+            console.print(
+                "Pick a dataset version validated for this harness, or re-run "
+                "with --ignore-bench-version to proceed anyway."
+            )
+            raise typer.Exit(1)
         if version_issue:
-            console.print(f"[yellow]Warning:[/yellow] {version_issue}")
+            console.print(
+                f"[yellow]Warning:[/yellow] {version_issue} (--ignore-bench-version)"
+            )
         console.print(
             f"[green]✓[/green] {resolved_dataset.spec}: "
             f"{len(resolved_dataset.task_names)} tasks, digests verified "

{benchflow-0.5.3.dev983 → benchflow-0.5.3.dev994}/tests/test_dataset_registry.py

@@ -39,7 +39,11 @@ def _make_task(parent: Path, name: str) -> Path:
 
 
 def _write_registry(
-    tmp_path: Path, tasks_parent: Path, *, digests: dict[str, str] | None = None
+    tmp_path: Path,
+    tasks_parent: Path,
+    *,
+    digests: dict[str, str] | None = None,
+    bench_version: str = ">=0.1,<999",
 ) -> Path:
     """Write a single-entry registry pinning the tasks under tasks_parent."""
     entry_tasks = []
@@ -61,7 +65,7 @@ def _write_registry(
                     "name": "skillsbench",
                     "version": "1.1",
                     "git_tag": "v1.1",
-                    "bench_version": ">=0.1,<999",
+                    "bench_version": bench_version,
                     "tasks": entry_tasks,
                 }
             ]
@@ -306,6 +310,80 @@ class TestEvalCreateDatasetCli:
         assert result.exit_code == 1
         assert "--registry requires --dataset" in result.stdout
 
+    def test_out_of_range_bench_version_blocks(self, tmp_path, snapshot, monkeypatch):
+        """The bench_version range is a hard gate for dataset runs; the
+        registry declares what the version was validated against."""
+        registry = _write_registry(
+            tmp_path, snapshot.tasks_parent, bench_version="<0.0.1"
+        )
+        ran = {}
+
+        async def fake_eval_run(self):
+            ran["ran"] = True
+            return SimpleNamespace(passed=2, total=2, score=1.0, errored=0)
+
+        monkeypatch.setattr("benchflow.evaluation.Evaluation.run", fake_eval_run)
+
+        result = CliRunner().invoke(
+            app,
+            [
+                "eval",
+                "create",
+                "--dataset",
+                "skillsbench@1.1",
+                "--registry",
+                str(registry),
+                "--agent",
+                "oracle",
+                "--jobs-dir",
+                str(tmp_path / "jobs"),
+            ],
+        )
+        assert result.exit_code == 1
+        assert "outside the range" in result.stdout
+        assert "--ignore-bench-version" in result.stdout
+        assert not ran
+
+    def test_ignore_bench_version_overrides_gate(self, tmp_path, snapshot, monkeypatch):
+        registry = _write_registry(
+            tmp_path, snapshot.tasks_parent, bench_version="<0.0.1"
+        )
+        ran = {}
+
+        async def fake_eval_run(self):
+            ran["ran"] = True
+            return SimpleNamespace(passed=2, total=2, score=1.0, errored=0)
+
+        monkeypatch.setattr("benchflow.evaluation.Evaluation.run", fake_eval_run)
+
+        result = CliRunner().invoke(
+            app,
+            [
+                "eval",
+                "create",
+                "--dataset",
+                "skillsbench@1.1",
+                "--registry",
+                str(registry),
+                "--ignore-bench-version",
+                "--agent",
+                "oracle",
+                "--jobs-dir",
+                str(tmp_path / "jobs"),
+            ],
+        )
+        assert result.exit_code == 0, result.stdout
+        assert ran == {"ran": True}
+        assert "Warning:" in result.stdout
+
+    def test_ignore_bench_version_requires_dataset(self, tmp_path):
+        result = CliRunner().invoke(
+            app,
+            ["eval", "create", "--ignore-bench-version", "--tasks-dir", str(tmp_path)],
+        )
+        assert result.exit_code == 1
+        assert "--ignore-bench-version requires --dataset" in result.stdout
+
     def test_resolution_error_exits_cleanly(self, tmp_path, snapshot):
         registry = _write_registry(tmp_path, snapshot.tasks_parent)
         result = CliRunner().invoke(