benchflow 0.5.3.dev980__tar.gz → 0.5.3.dev983__tar.gz

{benchflow-0.5.3.dev980 → benchflow-0.5.3.dev983}/CHANGELOG.md

@@ -4,6 +4,23 @@
 
 ### Added
 
+- **Registry-pinned dataset runs** — `bench eval create -d name@version`
+  (e.g. `-d skillsbench@1.1`) resolves a dataset from a git-backed
+  `registry.json` (see skillsbench `docs/dataset-versioning.md`): tasks are
+  cloned at their pinned `git_commit_id` into `.cache/datasets` and every
+  task directory is verified against its sha256 content digest before
+  anything runs; the entry's `bench_version` range is checked against the
+  installed benchflow. `--registry` overrides the default (skillsbench)
+  registry. `result.json`/`config.json` are stamped with `dataset_name`,
+  `dataset_version`, and a per-task `task_digest` (`summary.json` carries
+  the name/version); `--tasks-dir` dev runs carry no dataset fields but
+  still stamp a live-computed `task_digest`, so every trajectory stays
+  attributable to exact task content. `bench tasks digest <dir>` prints
+  the digest for task authoring, and `check_results.py` audits the stamps.
+  See [`docs/running-benchmarks.md`](docs/running-benchmarks.md). (#689,
+  #690, #691; `packaging` promoted to a core dependency for the
+  `bench_version` check.)
+
 - **`benchflow continue <run-folder>`** — resume a previous, unfinished
   (timed-out) `openhands` run to completion. A standalone tool (it does not
   touch the normal run path) that reconstructs the run's exact workspace and

{benchflow-0.5.3.dev980 → benchflow-0.5.3.dev983}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.5.3.dev980
+Version: 0.5.3.dev983
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow
@@ -22,6 +22,7 @@ Requires-Dist: agent-client-protocol>=0.10
 Requires-Dist: anyio>=4.0
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: litellm[proxy]==1.88.0rc1
+Requires-Dist: packaging>=24
 Requires-Dist: pydantic>=2.7
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: rich>=13.0
@@ -30,7 +31,6 @@ Requires-Dist: typer>=0.9
 Provides-Extra: bedrock
 Requires-Dist: boto3>=1.40; extra == 'bedrock'
 Provides-Extra: dev
-Requires-Dist: packaging>=24; extra == 'dev'
 Requires-Dist: pre-commit>=3.7; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
 Requires-Dist: pytest>=9.0.3; extra == 'dev'

{benchflow-0.5.3.dev980 → benchflow-0.5.3.dev983}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.5.3.dev980"
+version = "0.5.3.dev983"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -12,6 +12,8 @@ dependencies = [
     "pydantic>=2.7",
     "pyyaml>=6.0",
     "rich>=13.0",
+    # Dataset registry bench_version checks (PEP 440 specifier sets).
+    "packaging>=24",
     "litellm[proxy]==1.88.0rc1",
     "tomli-w>=1.0",
     "typer>=0.9",
@@ -38,7 +40,6 @@ classifiers = [
 
 [project.optional-dependencies]
 dev = [
-    "packaging>=24",
     "pre-commit>=3.7",
     "pytest>=9.0.3",
     "pytest-asyncio>=0.24.0",