benchflow 0.5.3.dev957__tar.gz → 0.5.3.dev978__tar.gz

{benchflow-0.5.3.dev957 → benchflow-0.5.3.dev978}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.5.3.dev957
+Version: 0.5.3.dev978
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.5.3.dev957 → benchflow-0.5.3.dev978}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.5.3.dev957"
+version = "0.5.3.dev978"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

benchflow-0.5.3.dev978/src/benchflow/_utils/dataset_registry.py

@@ -0,0 +1,216 @@
+"""Dataset registry resolution — `bench eval create -d skillsbench@1.1`.
+
+Resolves a published dataset version from the skillsbench registry
+(``registry.json``, see ``docs/dataset-versioning.md`` in
+benchflow-ai/skillsbench): clones the pinned snapshot, verifies every
+task's content digest against the registry entry, and hands back the
+task set plus source provenance for result stamping.
+"""
+
+import json
+import posixpath
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+from urllib.request import urlopen
+
+from benchflow._utils import benchmark_repos
+from benchflow._utils.task_authoring import task_digest
+
+DEFAULT_REGISTRY_SOURCE = (
+    "https://raw.githubusercontent.com/benchflow-ai/skillsbench/main/registry.json"
+)
+
+_GITHUB_URL_RE = re.compile(
+    r"^(?:https://|git@)github\.com[:/]([^/]+)/([^/]+?)(?:\.git)?/?$"
+)
+
+
+class DatasetResolutionError(ValueError):
+    """A dataset spec could not be resolved against the registry."""
+
+
+class DatasetDigestMismatchError(DatasetResolutionError):
+    """Snapshot content does not match the registry's pinned digests."""
+
+
+@dataclass(frozen=True)
+class ResolvedDataset:
+    """A registry entry resolved to verified task directories on disk."""
+
+    name: str
+    version: str
+    bench_version: str | None
+    tasks_dir: Path
+    task_names: set[str]
+    task_digests: dict[str, str]
+    provenance: dict[str, Any]
+
+    @property
+    def spec(self) -> str:
+        return f"{self.name}@{self.version}"
+
+
+def parse_dataset_spec(spec: str) -> tuple[str, str]:
+    """Split ``<name>@<version>`` — the version is mandatory.
+
+    Published versions are immutable, so there is no "latest" default: a
+    run must name the exact version it pins.
+    """
+    name, sep, version = spec.partition("@")
+    if not sep or not name or not version:
+        raise DatasetResolutionError(
+            f"Invalid dataset spec {spec!r} — expected <name>@<version> "
+            f"(e.g. skillsbench@1.1)"
+        )
+    return name, version
+
+
+def load_registry(source: str) -> list[dict[str, Any]]:
+    """Load a dataset registry from an HTTP(S) URL or a local file path."""
+    if source.startswith(("http://", "https://")):
+        with urlopen(source, timeout=30) as response:
+            raw = response.read().decode("utf-8")
+    else:
+        raw = Path(source).read_text()
+    registry = json.loads(raw)
+    if not isinstance(registry, list):
+        raise DatasetResolutionError(
+            f"Registry at {source} is not a list of dataset entries"
+        )
+    return registry
+
+
+def bench_version_issue(declared_range: str | None) -> str | None:
+    """Warning text when the running bench falls outside the dataset's
+    validated ``bench_version`` range, else None. Advisory only — the
+    range records what the version was validated against, it is not a
+    hard gate."""
+    if not declared_range:
+        return None
+    from packaging.specifiers import InvalidSpecifier, SpecifierSet
+    from packaging.version import InvalidVersion, Version
+
+    from benchflow import __version__
+
+    try:
+        specifier = SpecifierSet(declared_range)
+    except InvalidSpecifier:
+        return f"registry declares an unparseable bench_version {declared_range!r}"
+    try:
+        current = Version(__version__)
+    except InvalidVersion:
+        return (
+            f"cannot compare bench version {__version__!r} against the "
+            f"dataset's validated range {declared_range!r}"
+        )
+    if specifier.contains(current, prereleases=True):
+        return None
+    return (
+        f"bench {__version__} is outside the range this dataset was "
+        f"validated against ({declared_range}) — results may not be "
+        f"comparable with published runs"
+    )
+
+
+def _github_org_repo(git_url: str) -> str:
+    match = _GITHUB_URL_RE.match(git_url)
+    if not match:
+        raise DatasetResolutionError(
+            f"Unsupported git_url {git_url!r} — expected a github.com repository"
+        )
+    return f"{match.group(1)}/{match.group(2)}"
+
+
+def resolve_dataset(
+    spec: str, registry: str = DEFAULT_REGISTRY_SOURCE
+) -> ResolvedDataset:
+    """Resolve ``<name>@<version>`` to digest-verified task dirs on disk.
+
+    Clones the snapshot pinned by the registry entry (by commit id, so a
+    moved git tag cannot change what runs), then recomputes every task's
+    content digest and fails hard on any mismatch — a dataset version is
+    an immutable artifact and must byte-match its registry entry.
+    """
+    name, version = parse_dataset_spec(spec)
+    entries = load_registry(registry)
+    entry = next(
+        (
+            e
+            for e in entries
+            if e.get("name") == name and str(e.get("version")) == version
+        ),
+        None,
+    )
+    if entry is None:
+        available = (
+            ", ".join(f"{e.get('name')}@{e.get('version')}" for e in entries) or "none"
+        )
+        raise DatasetResolutionError(
+            f"Dataset {spec!r} not found in registry (available: {available})"
+        )
+    tasks = entry.get("tasks") or []
+    if not tasks:
+        raise DatasetResolutionError(f"Dataset {spec!r} has no tasks in the registry")
+
+    snapshots = {(t["git_url"], t["git_commit_id"]) for t in tasks}
+    if len(snapshots) != 1:
+        raise DatasetResolutionError(
+            f"Dataset {spec!r} spans {len(snapshots)} git snapshots — "
+            f"only single-snapshot datasets are supported"
+        )
+    git_url, commit = next(iter(snapshots))
+
+    parents = {posixpath.dirname(t["path"].rstrip("/")) for t in tasks}
+    if len(parents) != 1:
+        raise DatasetResolutionError(
+            f"Dataset {spec!r} tasks live under {len(parents)} parent "
+            f"directories — only a single tasks directory is supported"
+        )
+    parent = next(iter(parents))
+
+    task_digests: dict[str, str] = {}
+    for t in tasks:
+        base = posixpath.basename(t["path"].rstrip("/"))
+        if base in task_digests:
+            raise DatasetResolutionError(
+                f"Dataset {spec!r} has two tasks named {base!r}"
+            )
+        task_digests[base] = t["digest"]
+
+    repo = _github_org_repo(git_url)
+    resolved = benchmark_repos.resolve_source_with_metadata(
+        repo, path=parent or None, ref=commit
+    )
+    resolved_sha = resolved.provenance.get("resolved_sha")
+    if resolved_sha != commit:
+        raise DatasetResolutionError(
+            f"Snapshot for {spec!r} resolved to {resolved_sha} but the "
+            f"registry pins {commit}"
+        )
+
+    mismatches = []
+    for base, expected in sorted(task_digests.items()):
+        candidate = resolved.path / base
+        if not candidate.is_dir():
+            mismatches.append(f"{base}: missing from snapshot")
+            continue
+        actual = task_digest(candidate)
+        if actual != expected:
+            mismatches.append(f"{base}: computed {actual}, registry pins {expected}")
+    if mismatches:
+        raise DatasetDigestMismatchError(
+            f"Dataset {spec!r} content does not match its registry digests:\n  "
+            + "\n  ".join(mismatches)
+        )
+
+    return ResolvedDataset(
+        name=name,
+        version=version,
+        bench_version=entry.get("bench_version"),
+        tasks_dir=resolved.path,
+        task_names=set(task_digests),
+        task_digests=task_digests,
+        provenance=resolved.provenance,
+    )

{benchflow-0.5.3.dev957 → benchflow-0.5.3.dev978}/src/benchflow/_utils/task_authoring.py

@@ -1,6 +1,8 @@
-"""Task authoring — init and check benchmark tasks."""
+"""Task authoring — init, check, and digest benchmark tasks."""
 
+import hashlib
 import logging
+import os
 import re
 import tomllib
 from pathlib import Path
@@ -94,6 +96,36 @@ def check_task(task_dir: Path) -> list[str]:
     return issues
 
 
+def task_digest(task_dir: Path) -> str:
+    """Content digest pinning a task's files, independent of git.
+
+    sha256 over every regular file under ``task_dir``, sorted by POSIX
+    relative path; each file contributes
+    ``path_utf8 + b"\\x00" + sha256(file_bytes).digest()``. Symlinks and
+    file modes are excluded, so the digest is reproducible from a plain
+    checkout. Must byte-match the reference digests in the skillsbench
+    dataset registry (``registry.json`` / ``docs/dataset-versioning.md``,
+    skillsbench PR #922).
+    """
+    if not task_dir.is_dir():
+        raise NotADirectoryError(f"Not a directory: {task_dir}")
+    files: list[tuple[str, Path]] = []
+    # os.walk never descends into symlinked directories (unlike pre-3.13
+    # Path.rglob), keeping "symlinks are excluded" true for whole subtrees.
+    for dirpath, _dirnames, filenames in os.walk(task_dir):
+        for filename in filenames:
+            path = Path(dirpath) / filename
+            if path.is_symlink() or not path.is_file():
+                continue
+            files.append((path.relative_to(task_dir).as_posix(), path))
+    digest = hashlib.sha256()
+    for rel, path in sorted(files):
+        digest.update(rel.encode("utf-8"))
+        digest.update(b"\x00")
+        digest.update(hashlib.sha256(path.read_bytes()).digest())
+    return f"sha256:{digest.hexdigest()}"
+
+
 _CTRF_STANDARD_PATH = "/logs/verifier/ctrf.json"
 
 

{benchflow-0.5.3.dev957 → benchflow-0.5.3.dev978}/src/benchflow/cli/main.py

@@ -674,6 +674,46 @@ def tasks_check(
         raise typer.Exit(1)
 
 
+@tasks_app.command("digest")
+def tasks_digest(
+    path: Annotated[
+        Path,
+        typer.Argument(
+            help="Path to a task directory, or a directory of task directories"
+        ),
+    ],
+) -> None:
+    """Compute the content digest that pins a task's files, independent of git.
+
+    Matches the digests in the skillsbench dataset registry (registry.json).
+    Given a single task directory (contains task.toml), prints the digest;
+    given a directory of tasks, prints one "<name> <digest>" line per task.
+    """
+    from benchflow._utils.task_authoring import task_digest
+
+    if not path.is_dir():
+        console.print(f"[red]Not a directory: {path}[/red]")
+        raise typer.Exit(1)
+
+    if (path / "task.toml").is_file():
+        # typer.echo, not console.print: Rich wraps lines at terminal width,
+        # which would corrupt piped machine-readable output.
+        typer.echo(task_digest(path))
+        return
+
+    task_dirs = sorted(
+        d for d in path.iterdir() if d.is_dir() and (d / "task.toml").is_file()
+    )
+    if not task_dirs:
+        console.print(
+            f"[red]No tasks under {path} — expected task.toml in it "
+            f"or in its immediate subdirectories[/red]"
+        )
+        raise typer.Exit(1)
+    for task_dir in task_dirs:
+        typer.echo(f"{task_dir.name} {task_digest(task_dir)}")
+
+
 compat_app = typer.Typer(help="Third-party framework compatibility checks.")
 app.add_typer(compat_app, name="compat")
 
@@ -1052,6 +1092,23 @@ def eval_create(
             help="Skip these task names; repeatable (e.g. --exclude quantum-numerical-simulation)",
         ),
     ] = None,
+    dataset: Annotated[
+        str | None,
+        typer.Option(
+            "--dataset",
+            "-d",
+            help="Registry dataset to run, as <name>@<version> (e.g. skillsbench@1.1). "
+            "Resolves the pinned snapshot and verifies task content digests.",
+        ),
+    ] = None,
+    registry: Annotated[
+        str | None,
+        typer.Option(
+            "--registry",
+            help="Dataset registry JSON URL or local file "
+            "(default: the skillsbench registry). Only valid with --dataset.",
+        ),
+    ] = None,
 ) -> None:
     """Run an evaluation — single task or batch.
 
@@ -1067,12 +1124,22 @@ def eval_create(
     parsed_env = _parse_agent_env(agent_env)
     include_tasks = set(include) if include else set()
     exclude_tasks = set(exclude) if exclude else set()
-    sources = [bool(config_file), bool(tasks_dir), bool(source_repo), bool(source_env)]
+    sources = [
+        bool(config_file),
+        bool(tasks_dir),
+        bool(source_repo),
+        bool(source_env),
+        bool(dataset),
+    ]
     if sum(sources) > 1:
         console.print(
-            "[red]Choose only one source: --config, --tasks-dir, --source-repo, or --source-env[/red]"
+            "[red]Choose only one source: --config, --tasks-dir, --source-repo, "
+            "--source-env, or --dataset[/red]"
         )
         raise typer.Exit(1)
+    if registry and not dataset:
+        console.print("[red]--registry requires --dataset[/red]")
+        raise typer.Exit(1)
     if worker_concurrency is not None and not (tasks_dir or source_repo):
         console.print(
             "[red]--worker-concurrency is supported for --tasks-dir and --source-repo batch runs[/red]"
@@ -1300,6 +1367,63 @@ def eval_create(
         if run_result.error:
             console.print(f"[red]Error:[/red] {run_result.error}")
             raise typer.Exit(1)
+    elif dataset:
+        from benchflow._utils.dataset_registry import (
+            DEFAULT_REGISTRY_SOURCE,
+            DatasetResolutionError,
+            bench_version_issue,
+            resolve_dataset,
+        )
+
+        registry_source = registry or DEFAULT_REGISTRY_SOURCE
+        try:
+            with console.status(f"Resolving dataset {dataset}…"):
+                resolved_dataset = resolve_dataset(dataset, registry=registry_source)
+        except DatasetResolutionError as e:
+            console.print(f"[red]{e}[/red]")
+            raise typer.Exit(1) from None
+        version_issue = bench_version_issue(resolved_dataset.bench_version)
+        if version_issue:
+            console.print(f"[yellow]Warning:[/yellow] {version_issue}")
+        console.print(
+            f"[green]✓[/green] {resolved_dataset.spec}: "
+            f"{len(resolved_dataset.task_names)} tasks, digests verified "
+            f"({str(resolved_dataset.provenance.get('resolved_sha', ''))[:12]})"
+        )
+        dataset_include = (
+            resolved_dataset.task_names & include_tasks
+            if include_tasks
+            else resolved_dataset.task_names
+        )
+        eff_model = effective_model(eval_agent, model)
+        _run_batch_eval(
+            resolved_dataset.tasks_dir,
+            EvaluationConfig(
+                agent=eval_agent,
+                model=eff_model,
+                reasoning_effort=eval_reasoning_effort,
+                environment=eval_environment,
+                concurrency=eval_concurrency,
+                build_concurrency=build_concurrency,
+                prompts=eval_prompts,
+                agent_idle_timeout=eval_agent_idle_timeout,
+                agent_env=parsed_env,
+                sandbox_user=sandbox_user,
+                sandbox_setup_timeout=sandbox_setup_timeout,
+                skills_dir=str(skills_dir) if skills_dir else None,
+                skill_mode=skill_mode,
+                skill_creator_dir=str(skill_creator_dir) if skill_creator_dir else None,
+                self_gen_no_internet=self_gen_no_internet,
+                source_provenance=resolved_dataset.provenance,
+                include_tasks=dataset_include,
+                exclude_tasks=exclude_tasks,
+                usage_tracking=eval_usage_tracking,
+                environment_manifest=eval_env_manifest,
+                dataset_name=resolved_dataset.name,
+                dataset_version=resolved_dataset.version,
+                dataset_task_digests=resolved_dataset.task_digests,
+            ),
+        )
     elif source_repo:
         from benchflow._utils.benchmark_repos import resolve_source_with_metadata
 
@@ -1368,7 +1492,8 @@ def eval_create(
         )
     else:
         console.print(
-            "[red]Provide --config, --tasks-dir, --source-repo, or --source-env[/red]"
+            "[red]Provide --config, --tasks-dir, --source-repo, --source-env, "
+            "or --dataset[/red]"
         )
         raise typer.Exit(1)
 

{benchflow-0.5.3.dev957 → benchflow-0.5.3.dev978}/src/benchflow/evaluation.py

@@ -267,6 +267,13 @@ class EvaluationConfig:
     self_gen_no_internet: bool = False
     job_mode: str = DEFAULT_JOB_MODE
     source_provenance: dict[str, Any] | None = None
+    # Registry dataset identity (`bench eval create -d name@version`). When
+    # set, every result.json/config.json is stamped with dataset_name,
+    # dataset_version, and the task's registry content digest — see
+    # docs/dataset-versioning.md in benchflow-ai/skillsbench.
+    dataset_name: str | None = None
+    dataset_version: str | None = None
+    dataset_task_digests: dict[str, str] = field(default_factory=dict)
     usage_tracking: UsageTrackingConfig = field(default_factory=UsageTrackingConfig)
     # Environment-plane manifest applied to every rollout in the batch.
     # When set, each task's RolloutConfig.environment_manifest is populated
@@ -817,6 +824,13 @@ class Evaluation:
         from benchflow._utils.benchmark_repos import task_source_provenance
         from benchflow.rollout import Rollout, RolloutConfig
 
+        dataset = None
+        if cfg.dataset_name:
+            dataset = {
+                "name": cfg.dataset_name,
+                "version": cfg.dataset_version,
+                "task_digest": cfg.dataset_task_digests.get(task_dir.name),
+            }
         skills_dir = (
             str(self._learner_skills_dir)
             if self._learner_skills_dir is not None
@@ -855,6 +869,7 @@ class Evaluation:
             self_gen_no_internet=cfg.self_gen_no_internet,
             export_generated_skills_to=export_to,
             source_provenance=task_source_provenance(cfg.source_provenance, task_dir),
+            dataset=dataset,
             usage_tracking=cfg.usage_tracking,
         )
         if skill_mode == SKILL_MODE_SELF_GEN:
@@ -1383,6 +1398,14 @@ class Evaluation:
             **trajectory_step_summary(all_results),
             **phase_timing_summary(all_results),
             **summary_source_fields(cfg.source_provenance, all_results),
+            **(
+                {
+                    "dataset_name": cfg.dataset_name,
+                    "dataset_version": cfg.dataset_version,
+                }
+                if cfg.dataset_name
+                else {}
+            ),
         }
         # Surface continual-learning provenance — generation, curve — so a
         # resumed run can be audited end-to-end (#394).

{benchflow-0.5.3.dev957 → benchflow-0.5.3.dev978}/src/benchflow/rollout.py

@@ -525,6 +525,7 @@ def _write_config(
     agent_idle_timeout: int | None = None,
     scenes: list[Scene] | None = None,
     source_provenance: dict[str, Any] | None = None,
+    dataset: dict[str, Any] | None = None,
 ) -> None:
     """Write config.json to rollout_dir with secrets filtered out."""
     recorded_env = {
@@ -552,6 +553,10 @@ def _write_config(
         config_data["usage_tracking"] = usage_tracking.to_config_artifact()
     if source_provenance is not None:
         config_data["source"] = source_provenance
+    if dataset is not None:
+        config_data["dataset_name"] = dataset.get("name")
+        config_data["dataset_version"] = dataset.get("version")
+        config_data["task_digest"] = dataset.get("task_digest")
     (rollout_dir / "config.json").write_text(json.dumps(config_data, indent=2))
 
 
@@ -616,6 +621,7 @@ def _build_rollout_result(
     usage_tracking: dict[str, Any] | None = None,
     evolved_skills: dict[str, str] | None = None,
     source_provenance: dict[str, Any] | None = None,
+    dataset: dict[str, Any] | None = None,
     diagnostics: RolloutDiagnostics | None = None,
     skill_policy: TaskSkillPolicy | None = None,
     sandbox_id: str | None = None,
@@ -743,6 +749,15 @@ def _build_rollout_result(
                     if source_provenance is not None
                     else {}
                 ),
+                **(
+                    {
+                        "dataset_name": dataset.get("name"),
+                        "dataset_version": dataset.get("version"),
+                        "task_digest": dataset.get("task_digest"),
+                    }
+                    if dataset is not None
+                    else {}
+                ),
                 "sandbox_id": sandbox_id,
             },
             indent=2,
@@ -1092,6 +1107,10 @@ class RolloutConfig:
     skip_verify: bool = False
     export_generated_skills_to: str | Path | None = None
     source_provenance: dict[str, Any] | None = None
+    # Registry dataset identity for this task: {"name", "version",
+    # "task_digest"} — stamped into config.json/result.json so published
+    # runs declare the dataset version they ran (dataset-versioning plan).
+    dataset: dict[str, Any] | None = None
     planes: RolloutPlanes | None = field(default=None, repr=False, compare=False)
 
     def __post_init__(self) -> None:
@@ -1540,6 +1559,7 @@ class Rollout:
             agent_idle_timeout=cfg.agent_idle_timeout,
             scenes=cfg.effective_scenes,
             source_provenance=cfg.source_provenance,
+            dataset=cfg.dataset,
         )
 
         self._phase = "setup"
@@ -2906,6 +2926,7 @@ class Rollout:
             scenes=self._config.effective_scenes,
             evolved_skills=self._evolved_skills,
             source_provenance=self._config.source_provenance,
+            dataset=self._config.dataset,
             diagnostics=self._diagnostics,
             usage_tracking=self._usage_tracking_metadata(),
             skill_policy=getattr(self, "_task_skill_policy", None)

benchflow-0.5.3.dev978/tests/fixtures/digest/README.md

@@ -0,0 +1,9 @@
+# Digest test fixtures
+
+Golden-test fixtures for `bench tasks digest`. The expected digests are
+hardcoded in `tests/test_task_digest.py` — do not modify any file under the
+task directories here without updating those golden values.
+
+Filenames are ASCII on purpose: macOS checks out unicode filenames in NFD,
+which would change the hashed path bytes across platforms. Unicode coverage
+lives in file *content* (`beta-task/data/unicode.txt`).

benchflow-0.5.3.dev978/tests/fixtures/digest/alpha-task/environment/Dockerfile

@@ -0,0 +1 @@
+FROM ubuntu:24.04

benchflow-0.5.3.dev978/tests/fixtures/digest/alpha-task/instruction.md

@@ -0,0 +1,3 @@
+# alpha-task
+
+Compute the answer.

benchflow-0.5.3.dev978/tests/fixtures/digest/alpha-task/task.toml

@@ -0,0 +1,4 @@
+version = "1.1"
+
+[metadata]
+difficulty = "easy"

benchflow-0.5.3.dev978/tests/fixtures/digest/alpha-task/tests/test.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+echo "1.0" > /logs/verifier/reward.txt

benchflow-0.5.3.dev978/tests/fixtures/digest/beta-task/data/unicode.txt

@@ -0,0 +1 @@
+naïve résumé — ünïcödé

benchflow-0.5.3.dev978/tests/fixtures/digest/beta-task/instruction.md

@@ -0,0 +1 @@
+# beta-task

benchflow-0.5.3.dev978/tests/fixtures/digest/beta-task/task.toml

@@ -0,0 +1 @@
+version = "1.1"

benchflow-0.5.3.dev978/tests/fixtures/digest/not-a-task/notes.txt

@@ -0,0 +1 @@
+Not a task — no task.toml. Multi-task digest mode must skip this directory.