benchflow 0.5.3.dev978__tar.gz → 0.5.3.dev983__tar.gz

{benchflow-0.5.3.dev978 → benchflow-0.5.3.dev983}/CHANGELOG.md

@@ -4,6 +4,23 @@
 
 ### Added
 
+- **Registry-pinned dataset runs** — `bench eval create -d name@version`
+  (e.g. `-d skillsbench@1.1`) resolves a dataset from a git-backed
+  `registry.json` (see skillsbench `docs/dataset-versioning.md`): tasks are
+  cloned at their pinned `git_commit_id` into `.cache/datasets` and every
+  task directory is verified against its sha256 content digest before
+  anything runs; the entry's `bench_version` range is checked against the
+  installed benchflow. `--registry` overrides the default (skillsbench)
+  registry. `result.json`/`config.json` are stamped with `dataset_name`,
+  `dataset_version`, and a per-task `task_digest` (`summary.json` carries
+  the name/version); `--tasks-dir` dev runs carry no dataset fields but
+  still stamp a live-computed `task_digest`, so every trajectory stays
+  attributable to exact task content. `bench tasks digest <dir>` prints
+  the digest for task authoring, and `check_results.py` audits the stamps.
+  See [`docs/running-benchmarks.md`](docs/running-benchmarks.md). (#689,
+  #690, #691; `packaging` promoted to a core dependency for the
+  `bench_version` check.)
+
 - **`benchflow continue <run-folder>`** — resume a previous, unfinished
   (timed-out) `openhands` run to completion. A standalone tool (it does not
   touch the normal run path) that reconstructs the run's exact workspace and

{benchflow-0.5.3.dev978 → benchflow-0.5.3.dev983}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.5.3.dev978
+Version: 0.5.3.dev983
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow
@@ -22,6 +22,7 @@ Requires-Dist: agent-client-protocol>=0.10
 Requires-Dist: anyio>=4.0
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: litellm[proxy]==1.88.0rc1
+Requires-Dist: packaging>=24
 Requires-Dist: pydantic>=2.7
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: rich>=13.0
@@ -30,7 +31,6 @@ Requires-Dist: typer>=0.9
 Provides-Extra: bedrock
 Requires-Dist: boto3>=1.40; extra == 'bedrock'
 Provides-Extra: dev
-Requires-Dist: packaging>=24; extra == 'dev'
 Requires-Dist: pre-commit>=3.7; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
 Requires-Dist: pytest>=9.0.3; extra == 'dev'

{benchflow-0.5.3.dev978 → benchflow-0.5.3.dev983}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.5.3.dev978"
+version = "0.5.3.dev983"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -12,6 +12,8 @@ dependencies = [
     "pydantic>=2.7",
     "pyyaml>=6.0",
     "rich>=13.0",
+    # Dataset registry bench_version checks (PEP 440 specifier sets).
+    "packaging>=24",
     "litellm[proxy]==1.88.0rc1",
     "tomli-w>=1.0",
     "typer>=0.9",
@@ -38,7 +40,6 @@ classifiers = [
 
 [project.optional-dependencies]
 dev = [
-    "packaging>=24",
     "pre-commit>=3.7",
     "pytest>=9.0.3",
     "pytest-asyncio>=0.24.0",

{benchflow-0.5.3.dev978 → benchflow-0.5.3.dev983}/src/benchflow/evaluation.py

@@ -826,11 +826,20 @@ class Evaluation:
 
         dataset = None
         if cfg.dataset_name:
-            dataset = {
-                "name": cfg.dataset_name,
-                "version": cfg.dataset_version,
-                "task_digest": cfg.dataset_task_digests.get(task_dir.name),
-            }
+            dataset = {"name": cfg.dataset_name, "version": cfg.dataset_version}
+        task_digest_value = (
+            cfg.dataset_task_digests.get(task_dir.name) if cfg.dataset_name else None
+        )
+        if task_digest_value is None:
+            # Dev runs (--tasks-dir / --source-repo) stamp a live-computed
+            # digest so every trajectory stays attributable to the exact
+            # task content it ran, not just a directory name.
+            from benchflow._utils.task_authoring import task_digest
+
+            try:
+                task_digest_value = task_digest(task_dir)
+            except (OSError, ValueError, UnicodeError) as e:
+                logger.debug("Could not compute task digest for %s: %s", task_dir, e)
         skills_dir = (
             str(self._learner_skills_dir)
             if self._learner_skills_dir is not None
@@ -870,6 +879,7 @@ class Evaluation:
             export_generated_skills_to=export_to,
             source_provenance=task_source_provenance(cfg.source_provenance, task_dir),
             dataset=dataset,
+            task_digest=task_digest_value,
             usage_tracking=cfg.usage_tracking,
         )
         if skill_mode == SKILL_MODE_SELF_GEN:

{benchflow-0.5.3.dev978 → benchflow-0.5.3.dev983}/src/benchflow/rollout.py

@@ -526,6 +526,7 @@ def _write_config(
     scenes: list[Scene] | None = None,
     source_provenance: dict[str, Any] | None = None,
     dataset: dict[str, Any] | None = None,
+    task_digest: str | None = None,
 ) -> None:
     """Write config.json to rollout_dir with secrets filtered out."""
     recorded_env = {
@@ -556,7 +557,8 @@ def _write_config(
     if dataset is not None:
         config_data["dataset_name"] = dataset.get("name")
         config_data["dataset_version"] = dataset.get("version")
-        config_data["task_digest"] = dataset.get("task_digest")
+    if task_digest is not None:
+        config_data["task_digest"] = task_digest
     (rollout_dir / "config.json").write_text(json.dumps(config_data, indent=2))
 
 
@@ -622,6 +624,7 @@ def _build_rollout_result(
     evolved_skills: dict[str, str] | None = None,
     source_provenance: dict[str, Any] | None = None,
     dataset: dict[str, Any] | None = None,
+    task_digest: str | None = None,
     diagnostics: RolloutDiagnostics | None = None,
     skill_policy: TaskSkillPolicy | None = None,
     sandbox_id: str | None = None,
@@ -753,11 +756,11 @@ def _build_rollout_result(
                     {
                         "dataset_name": dataset.get("name"),
                         "dataset_version": dataset.get("version"),
-                        "task_digest": dataset.get("task_digest"),
                     }
                     if dataset is not None
                     else {}
                 ),
+                **({"task_digest": task_digest} if task_digest is not None else {}),
                 "sandbox_id": sandbox_id,
             },
             indent=2,
@@ -1107,10 +1110,14 @@ class RolloutConfig:
     skip_verify: bool = False
     export_generated_skills_to: str | Path | None = None
     source_provenance: dict[str, Any] | None = None
-    # Registry dataset identity for this task: {"name", "version",
-    # "task_digest"} — stamped into config.json/result.json so published
+    # Registry dataset identity: {"name", "version"} — stamped into
+    # config.json/result.json as dataset_name/dataset_version so published
     # runs declare the dataset version they ran (dataset-versioning plan).
     dataset: dict[str, Any] | None = None
+    # Content digest of the task directory ("sha256:<hex>", the skillsbench
+    # registry algorithm). Registry-pinned for dataset runs, computed live
+    # for dev runs — every result stays attributable to exact task content.
+    task_digest: str | None = None
     planes: RolloutPlanes | None = field(default=None, repr=False, compare=False)
 
     def __post_init__(self) -> None:
@@ -1560,6 +1567,7 @@ class Rollout:
             scenes=cfg.effective_scenes,
             source_provenance=cfg.source_provenance,
             dataset=cfg.dataset,
+            task_digest=cfg.task_digest,
         )
 
         self._phase = "setup"
@@ -2927,6 +2935,7 @@ class Rollout:
             evolved_skills=self._evolved_skills,
             source_provenance=self._config.source_provenance,
             dataset=self._config.dataset,
+            task_digest=self._config.task_digest,
             diagnostics=self._diagnostics,
             usage_tracking=self._usage_tracking_metadata(),
             skill_policy=getattr(self, "_task_skill_policy", None)

{benchflow-0.5.3.dev978 → benchflow-0.5.3.dev983}/tests/integration/check_results.py

@@ -32,7 +32,11 @@ from benchflow._utils.scoring import (
     classify_verifier_error,
     count_result_outcomes,
 )
-from benchflow._utils.source_provenance import source_issues, source_matches_parent
+from benchflow._utils.source_provenance import (
+    is_sha256_digest,
+    source_issues,
+    source_matches_parent,
+)
 from benchflow.trajectories.metrics import (
     count_skill_invocations,
     result_skill_invocations,
@@ -304,6 +308,73 @@ def _source_hash_truth_issues(source: object, label: str) -> list[str]:
     return _source_git_truth_issues(source_dict, local_path, label)
 
 
+# Dataset identity stamp (dataset-versioning plan: benchflow #690 + dev-run
+# digest stamping follow-up). dataset_name/dataset_version mark registry
+# dataset runs; task_digest pins the exact task content for every run.
+_DATASET_STAMP_KEYS = ("dataset_name", "dataset_version", "task_digest")
+
+
+def _dataset_stamp_issues(artifact: object, label: str) -> list[str]:
+    """Audit the dataset identity stamp on one result.json/config.json.
+
+    Rules: dataset_name and dataset_version travel together; a dataset run
+    must carry a task_digest; any task_digest must be ``sha256:<64 hex>``.
+    A bare task_digest without dataset fields is a dev run — allowed.
+    """
+    if not isinstance(artifact, dict):
+        return []
+    artifact_dict = cast(dict[str, Any], artifact)
+    name = artifact_dict.get("dataset_name")
+    version = artifact_dict.get("dataset_version")
+    digest = artifact_dict.get("task_digest")
+    if name is None and version is None and digest is None:
+        return []
+    issues: list[str] = []
+    if (name is None) != (version is None):
+        issues.append(f"{label}: dataset_name and dataset_version must travel together")
+    for field_name, value in (("dataset_name", name), ("dataset_version", version)):
+        if value is not None and (not isinstance(value, str) or not value):
+            issues.append(f"{label}: {field_name} must be a non-empty string")
+    if name is not None and digest is None:
+        issues.append(f"{label}: dataset-stamped artifact missing task_digest")
+    if digest is not None and not is_sha256_digest(digest):
+        issues.append(f"{label}: task_digest must be 'sha256:<64 hex>'")
+    return issues
+
+
+def _task_digest_truth_issues(result: object, label: str) -> list[str]:
+    """Recompute the task content digest when the task dir is still on disk.
+
+    Complements ``_source_hash_truth_issues``: ``source.file_hashes`` audits
+    per-file hashes, this audits the single registry-algorithm digest the
+    leaderboard pipeline keys on.
+    """
+    if not isinstance(result, dict):
+        return []
+    result_dict = cast(dict[str, Any], result)
+    digest = result_dict.get("task_digest")
+    if not is_sha256_digest(digest):
+        return []  # absent or malformed — format issues reported elsewhere
+    source = result_dict.get("source")
+    if not isinstance(source, dict):
+        return []
+    local_path_raw = source.get("local_path")
+    if not isinstance(local_path_raw, str):
+        return []
+    local_path = Path(local_path_raw)
+    if not (local_path / "task.toml").exists():
+        return []
+    try:
+        from benchflow._utils.task_authoring import task_digest
+
+        actual = task_digest(local_path)
+    except Exception as e:
+        return [f"{label}: task_digest could not be recomputed: {e}"]
+    if actual != digest:
+        return [f"{label}: task_digest does not match local_path content"]
+    return []
+
+
 def _git_stdout(cwd: Path, *args: str) -> str | None:
     completed = subprocess.run(
         ["git", "-C", str(cwd), *args],
@@ -535,6 +606,18 @@ def check_agent(agent_dir: Path) -> dict:
         if source_truth_issues:
             findings["issues"].extend(source_truth_issues)
             findings["ok"] = False
+        dataset_stamp_issues = _dataset_stamp_issues(
+            r, f"{r.get('task_name', '?')}: result.json"
+        )
+        if dataset_stamp_issues:
+            findings["issues"].extend(dataset_stamp_issues)
+            findings["ok"] = False
+        digest_truth_issues = _task_digest_truth_issues(
+            r, f"{r.get('task_name', '?')}: result.json"
+        )
+        if digest_truth_issues:
+            findings["issues"].extend(digest_truth_issues)
+            findings["ok"] = False
         _config_path, config, config_error = _load_config(result_file)
         if config_error:
             findings["issues"].append(f"{r.get('task_name', '?')}: {config_error}")
@@ -639,6 +722,18 @@ def check_agent(agent_dir: Path) -> dict:
                     f"{r.get('task_name', '?')}: config.json source does not match result.json"
                 )
                 findings["ok"] = False
+            config_dataset_issues = _dataset_stamp_issues(
+                config, f"{r.get('task_name', '?')}: config.json"
+            )
+            if config_dataset_issues:
+                findings["issues"].extend(config_dataset_issues)
+                findings["ok"] = False
+            if any(config.get(k) != r.get(k) for k in _DATASET_STAMP_KEYS):
+                findings["issues"].append(
+                    f"{r.get('task_name', '?')}: config.json dataset stamp "
+                    f"does not match result.json"
+                )
+                findings["ok"] = False
 
     # Infrastructure errors — driven from the central diagnostic registry
     # so adding a new diagnostic kind doesn't require editing this file
@@ -768,6 +863,17 @@ def check_agent(agent_dir: Path) -> dict:
                     "summary.json missing source provenance and not all results have source"
                 )
                 findings["ok"] = False
+            # Dataset identity must agree across summary.json and every
+            # result.json — one leaderboard namespace per dataset version.
+            for dataset_key in ("dataset_name", "dataset_version"):
+                dataset_values = {r.get(dataset_key) for r in results}
+                dataset_values.add(summary.get(dataset_key))
+                if len(dataset_values) > 1:
+                    findings["issues"].append(
+                        f"summary.json {dataset_key} does not agree across "
+                        f"summary and results: {sorted(map(repr, dataset_values))}"
+                    )
+                    findings["ok"] = False
             expected_model = _expected(agent, "model")
             expected_environment = _expected(agent, "environment")
             expected_concurrency = _expected(agent, "concurrency")

{benchflow-0.5.3.dev978 → benchflow-0.5.3.dev983}/tests/test_dataset_registry.py

@@ -324,19 +324,20 @@ class TestEvalCreateDatasetCli:
 
 
 class TestDatasetStamping:
-    DATASET: ClassVar[dict] = {
-        "name": "skillsbench",
-        "version": "1.1",
-        "task_digest": "sha256:" + "f" * 64,
-    }
+    DATASET: ClassVar[dict] = {"name": "skillsbench", "version": "1.1"}
+    TASK_DIGEST = "sha256:" + "f" * 64
 
     def test_rollout_config_threads_dataset_through_from_legacy(self, tmp_path):
         from benchflow.rollout import RolloutConfig
 
         cfg = RolloutConfig.from_legacy(
-            task_path=tmp_path, agent="oracle", dataset=self.DATASET
+            task_path=tmp_path,
+            agent="oracle",
+            dataset=self.DATASET,
+            task_digest=self.TASK_DIGEST,
         )
         assert cfg.dataset == self.DATASET
+        assert cfg.task_digest == self.TASK_DIGEST
 
     def test_config_json_stamped(self, tmp_path):
         from benchflow.rollout import _write_config
@@ -361,11 +362,12 @@ class TestDatasetStamping:
             started_at=datetime(2026, 6, 12),
             agent_env={},
             dataset=self.DATASET,
+            task_digest=self.TASK_DIGEST,
         )
         config = json.loads((tmp_path / "config.json").read_text())
         assert config["dataset_name"] == "skillsbench"
         assert config["dataset_version"] == "1.1"
-        assert config["task_digest"] == self.DATASET["task_digest"]
+        assert config["task_digest"] == self.TASK_DIGEST
 
     def test_config_json_unstamped_without_dataset(self, tmp_path):
         from benchflow.rollout import _write_config
@@ -392,6 +394,7 @@ class TestDatasetStamping:
         )
         config = json.loads((tmp_path / "config.json").read_text())
         assert "dataset_name" not in config
+        assert "task_digest" not in config
 
     def test_result_json_stamped(self, tmp_path):
         from benchflow.rollout import _build_rollout_result
@@ -413,13 +416,57 @@ class TestDatasetStamping:
             started_at=datetime(2026, 6, 12),
             timing={},
             dataset=self.DATASET,
+            task_digest=self.TASK_DIGEST,
         )
         result = json.loads((tmp_path / "result.json").read_text())
         assert result["dataset_name"] == "skillsbench"
         assert result["dataset_version"] == "1.1"
-        assert result["task_digest"] == self.DATASET["task_digest"]
+        assert result["task_digest"] == self.TASK_DIGEST
+
+    def test_result_json_dev_run_stamps_digest_without_dataset(self, tmp_path):
+        """Dev runs carry task_digest but no dataset identity fields."""
+        from benchflow.rollout import _build_rollout_result
+
+        _build_rollout_result(
+            tmp_path,
+            task_name="task-a",
+            rollout_name="task-a__r1",
+            agent="oracle",
+            agent_name="oracle",
+            model=None,
+            n_tool_calls=0,
+            prompts=[],
+            error=None,
+            verifier_error=None,
+            trajectory=[],
+            partial_trajectory=False,
+            rewards={"reward": 1.0},
+            started_at=datetime(2026, 6, 12),
+            timing={},
+            task_digest=self.TASK_DIGEST,
+        )
+        result = json.loads((tmp_path / "result.json").read_text())
+        assert result["task_digest"] == self.TASK_DIGEST
+        assert "dataset_name" not in result
+        assert "dataset_version" not in result
+
+    @staticmethod
+    def _capture_rollout_create(monkeypatch):
+        captured = {}
 
-    async def test_run_single_task_builds_per_task_dataset(self, tmp_path, monkeypatch):
+        async def _fake_run():
+            return SimpleNamespace(rewards={"reward": 1.0})
+
+        async def fake_create(rollout_config):
+            captured["config"] = rollout_config
+            return SimpleNamespace(run=_fake_run)
+
+        monkeypatch.setattr("benchflow.rollout.Rollout.create", fake_create)
+        return captured
+
+    async def test_run_single_task_uses_registry_digest_for_dataset_runs(
+        self, tmp_path, monkeypatch
+    ):
         from benchflow.evaluation import Evaluation, EvaluationConfig
 
         task_dir = _make_task(tmp_path / "tasks", "task-a")
@@ -430,24 +477,26 @@ class TestDatasetStamping:
             dataset_version="1.1",
             dataset_task_digests={"task-a": digest},
         )
-        captured = {}
-
-        class FakeRollout:
-            @staticmethod
-            async def create(rollout_config):
-                captured["config"] = rollout_config
-                return SimpleNamespace(run=_fake_run)
+        captured = self._capture_rollout_create(monkeypatch)
+        evaluation = Evaluation(
+            tasks_dir=tmp_path / "tasks", jobs_dir=tmp_path / "jobs", config=cfg
+        )
+        await evaluation._run_single_task(task_dir, cfg)
+        assert captured["config"].dataset == {"name": "skillsbench", "version": "1.1"}
+        assert captured["config"].task_digest == digest
 
-        async def _fake_run():
-            return SimpleNamespace(rewards={"reward": 1.0})
+    async def test_run_single_task_computes_digest_for_dev_runs(
+        self, tmp_path, monkeypatch
+    ):
+        """Dev runs (no --dataset) stamp a live-computed content digest."""
+        from benchflow.evaluation import Evaluation, EvaluationConfig
 
-        monkeypatch.setattr("benchflow.rollout.Rollout.create", FakeRollout.create)
+        task_dir = _make_task(tmp_path / "tasks", "task-a")
+        cfg = EvaluationConfig(agent="oracle")
+        captured = self._capture_rollout_create(monkeypatch)
         evaluation = Evaluation(
             tasks_dir=tmp_path / "tasks", jobs_dir=tmp_path / "jobs", config=cfg
         )
         await evaluation._run_single_task(task_dir, cfg)
-        assert captured["config"].dataset == {
-            "name": "skillsbench",
-            "version": "1.1",
-            "task_digest": digest,
-        }
+        assert captured["config"].dataset is None
+        assert captured["config"].task_digest == task_digest(task_dir)