benchflow 0.5.3.dev976__tar.gz → 0.5.3.dev980__tar.gz

{benchflow-0.5.3.dev976 → benchflow-0.5.3.dev980}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.5.3.dev976
+Version: 0.5.3.dev980
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.5.3.dev976 → benchflow-0.5.3.dev980}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.5.3.dev976"
+version = "0.5.3.dev980"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

benchflow-0.5.3.dev980/src/benchflow/_utils/dataset_registry.py

@@ -0,0 +1,216 @@
+"""Dataset registry resolution — `bench eval create -d skillsbench@1.1`.
+
+Resolves a published dataset version from the skillsbench registry
+(``registry.json``, see ``docs/dataset-versioning.md`` in
+benchflow-ai/skillsbench): clones the pinned snapshot, verifies every
+task's content digest against the registry entry, and hands back the
+task set plus source provenance for result stamping.
+"""
+
+import json
+import posixpath
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+from urllib.request import urlopen
+
+from benchflow._utils import benchmark_repos
+from benchflow._utils.task_authoring import task_digest
+
+DEFAULT_REGISTRY_SOURCE = (
+    "https://raw.githubusercontent.com/benchflow-ai/skillsbench/main/registry.json"
+)
+
+_GITHUB_URL_RE = re.compile(
+    r"^(?:https://|git@)github\.com[:/]([^/]+)/([^/]+?)(?:\.git)?/?$"
+)
+
+
+class DatasetResolutionError(ValueError):
+    """A dataset spec could not be resolved against the registry."""
+
+
+class DatasetDigestMismatchError(DatasetResolutionError):
+    """Snapshot content does not match the registry's pinned digests."""
+
+
+@dataclass(frozen=True)
+class ResolvedDataset:
+    """A registry entry resolved to verified task directories on disk."""
+
+    name: str
+    version: str
+    bench_version: str | None
+    tasks_dir: Path
+    task_names: set[str]
+    task_digests: dict[str, str]
+    provenance: dict[str, Any]
+
+    @property
+    def spec(self) -> str:
+        return f"{self.name}@{self.version}"
+
+
+def parse_dataset_spec(spec: str) -> tuple[str, str]:
+    """Split ``<name>@<version>`` — the version is mandatory.
+
+    Published versions are immutable, so there is no "latest" default: a
+    run must name the exact version it pins.
+    """
+    name, sep, version = spec.partition("@")
+    if not sep or not name or not version:
+        raise DatasetResolutionError(
+            f"Invalid dataset spec {spec!r} — expected <name>@<version> "
+            f"(e.g. skillsbench@1.1)"
+        )
+    return name, version
+
+
+def load_registry(source: str) -> list[dict[str, Any]]:
+    """Load a dataset registry from an HTTP(S) URL or a local file path."""
+    if source.startswith(("http://", "https://")):
+        with urlopen(source, timeout=30) as response:
+            raw = response.read().decode("utf-8")
+    else:
+        raw = Path(source).read_text()
+    registry = json.loads(raw)
+    if not isinstance(registry, list):
+        raise DatasetResolutionError(
+            f"Registry at {source} is not a list of dataset entries"
+        )
+    return registry
+
+
+def bench_version_issue(declared_range: str | None) -> str | None:
+    """Warning text when the running bench falls outside the dataset's
+    validated ``bench_version`` range, else None. Advisory only — the
+    range records what the version was validated against, it is not a
+    hard gate."""
+    if not declared_range:
+        return None
+    from packaging.specifiers import InvalidSpecifier, SpecifierSet
+    from packaging.version import InvalidVersion, Version
+
+    from benchflow import __version__
+
+    try:
+        specifier = SpecifierSet(declared_range)
+    except InvalidSpecifier:
+        return f"registry declares an unparseable bench_version {declared_range!r}"
+    try:
+        current = Version(__version__)
+    except InvalidVersion:
+        return (
+            f"cannot compare bench version {__version__!r} against the "
+            f"dataset's validated range {declared_range!r}"
+        )
+    if specifier.contains(current, prereleases=True):
+        return None
+    return (
+        f"bench {__version__} is outside the range this dataset was "
+        f"validated against ({declared_range}) — results may not be "
+        f"comparable with published runs"
+    )
+
+
+def _github_org_repo(git_url: str) -> str:
+    match = _GITHUB_URL_RE.match(git_url)
+    if not match:
+        raise DatasetResolutionError(
+            f"Unsupported git_url {git_url!r} — expected a github.com repository"
+        )
+    return f"{match.group(1)}/{match.group(2)}"
+
+
+def resolve_dataset(
+    spec: str, registry: str = DEFAULT_REGISTRY_SOURCE
+) -> ResolvedDataset:
+    """Resolve ``<name>@<version>`` to digest-verified task dirs on disk.
+
+    Clones the snapshot pinned by the registry entry (by commit id, so a
+    moved git tag cannot change what runs), then recomputes every task's
+    content digest and fails hard on any mismatch — a dataset version is
+    an immutable artifact and must byte-match its registry entry.
+    """
+    name, version = parse_dataset_spec(spec)
+    entries = load_registry(registry)
+    entry = next(
+        (
+            e
+            for e in entries
+            if e.get("name") == name and str(e.get("version")) == version
+        ),
+        None,
+    )
+    if entry is None:
+        available = (
+            ", ".join(f"{e.get('name')}@{e.get('version')}" for e in entries) or "none"
+        )
+        raise DatasetResolutionError(
+            f"Dataset {spec!r} not found in registry (available: {available})"
+        )
+    tasks = entry.get("tasks") or []
+    if not tasks:
+        raise DatasetResolutionError(f"Dataset {spec!r} has no tasks in the registry")
+
+    snapshots = {(t["git_url"], t["git_commit_id"]) for t in tasks}
+    if len(snapshots) != 1:
+        raise DatasetResolutionError(
+            f"Dataset {spec!r} spans {len(snapshots)} git snapshots — "
+            f"only single-snapshot datasets are supported"
+        )
+    git_url, commit = next(iter(snapshots))
+
+    parents = {posixpath.dirname(t["path"].rstrip("/")) for t in tasks}
+    if len(parents) != 1:
+        raise DatasetResolutionError(
+            f"Dataset {spec!r} tasks live under {len(parents)} parent "
+            f"directories — only a single tasks directory is supported"
+        )
+    parent = next(iter(parents))
+
+    task_digests: dict[str, str] = {}
+    for t in tasks:
+        base = posixpath.basename(t["path"].rstrip("/"))
+        if base in task_digests:
+            raise DatasetResolutionError(
+                f"Dataset {spec!r} has two tasks named {base!r}"
+            )
+        task_digests[base] = t["digest"]
+
+    repo = _github_org_repo(git_url)
+    resolved = benchmark_repos.resolve_source_with_metadata(
+        repo, path=parent or None, ref=commit
+    )
+    resolved_sha = resolved.provenance.get("resolved_sha")
+    if resolved_sha != commit:
+        raise DatasetResolutionError(
+            f"Snapshot for {spec!r} resolved to {resolved_sha} but the "
+            f"registry pins {commit}"
+        )
+
+    mismatches = []
+    for base, expected in sorted(task_digests.items()):
+        candidate = resolved.path / base
+        if not candidate.is_dir():
+            mismatches.append(f"{base}: missing from snapshot")
+            continue
+        actual = task_digest(candidate)
+        if actual != expected:
+            mismatches.append(f"{base}: computed {actual}, registry pins {expected}")
+    if mismatches:
+        raise DatasetDigestMismatchError(
+            f"Dataset {spec!r} content does not match its registry digests:\n  "
+            + "\n  ".join(mismatches)
+        )
+
+    return ResolvedDataset(
+        name=name,
+        version=version,
+        bench_version=entry.get("bench_version"),
+        tasks_dir=resolved.path,
+        task_names=set(task_digests),
+        task_digests=task_digests,
+        provenance=resolved.provenance,
+    )

{benchflow-0.5.3.dev976 → benchflow-0.5.3.dev980}/src/benchflow/cli/main.py

@@ -1092,6 +1092,23 @@ def eval_create(
             help="Skip these task names; repeatable (e.g. --exclude quantum-numerical-simulation)",
         ),
     ] = None,
+    dataset: Annotated[
+        str | None,
+        typer.Option(
+            "--dataset",
+            "-d",
+            help="Registry dataset to run, as <name>@<version> (e.g. skillsbench@1.1). "
+            "Resolves the pinned snapshot and verifies task content digests.",
+        ),
+    ] = None,
+    registry: Annotated[
+        str | None,
+        typer.Option(
+            "--registry",
+            help="Dataset registry JSON URL or local file "
+            "(default: the skillsbench registry). Only valid with --dataset.",
+        ),
+    ] = None,
 ) -> None:
     """Run an evaluation — single task or batch.
 
@@ -1107,12 +1124,22 @@ def eval_create(
     parsed_env = _parse_agent_env(agent_env)
     include_tasks = set(include) if include else set()
     exclude_tasks = set(exclude) if exclude else set()
-    sources = [bool(config_file), bool(tasks_dir), bool(source_repo), bool(source_env)]
+    sources = [
+        bool(config_file),
+        bool(tasks_dir),
+        bool(source_repo),
+        bool(source_env),
+        bool(dataset),
+    ]
     if sum(sources) > 1:
         console.print(
-            "[red]Choose only one source: --config, --tasks-dir, --source-repo, or --source-env[/red]"
+            "[red]Choose only one source: --config, --tasks-dir, --source-repo, "
+            "--source-env, or --dataset[/red]"
         )
         raise typer.Exit(1)
+    if registry and not dataset:
+        console.print("[red]--registry requires --dataset[/red]")
+        raise typer.Exit(1)
     if worker_concurrency is not None and not (tasks_dir or source_repo):
         console.print(
             "[red]--worker-concurrency is supported for --tasks-dir and --source-repo batch runs[/red]"
@@ -1340,6 +1367,63 @@ def eval_create(
         if run_result.error:
             console.print(f"[red]Error:[/red] {run_result.error}")
             raise typer.Exit(1)
+    elif dataset:
+        from benchflow._utils.dataset_registry import (
+            DEFAULT_REGISTRY_SOURCE,
+            DatasetResolutionError,
+            bench_version_issue,
+            resolve_dataset,
+        )
+
+        registry_source = registry or DEFAULT_REGISTRY_SOURCE
+        try:
+            with console.status(f"Resolving dataset {dataset}…"):
+                resolved_dataset = resolve_dataset(dataset, registry=registry_source)
+        except DatasetResolutionError as e:
+            console.print(f"[red]{e}[/red]")
+            raise typer.Exit(1) from None
+        version_issue = bench_version_issue(resolved_dataset.bench_version)
+        if version_issue:
+            console.print(f"[yellow]Warning:[/yellow] {version_issue}")
+        console.print(
+            f"[green]✓[/green] {resolved_dataset.spec}: "
+            f"{len(resolved_dataset.task_names)} tasks, digests verified "
+            f"({str(resolved_dataset.provenance.get('resolved_sha', ''))[:12]})"
+        )
+        dataset_include = (
+            resolved_dataset.task_names & include_tasks
+            if include_tasks
+            else resolved_dataset.task_names
+        )
+        eff_model = effective_model(eval_agent, model)
+        _run_batch_eval(
+            resolved_dataset.tasks_dir,
+            EvaluationConfig(
+                agent=eval_agent,
+                model=eff_model,
+                reasoning_effort=eval_reasoning_effort,
+                environment=eval_environment,
+                concurrency=eval_concurrency,
+                build_concurrency=build_concurrency,
+                prompts=eval_prompts,
+                agent_idle_timeout=eval_agent_idle_timeout,
+                agent_env=parsed_env,
+                sandbox_user=sandbox_user,
+                sandbox_setup_timeout=sandbox_setup_timeout,
+                skills_dir=str(skills_dir) if skills_dir else None,
+                skill_mode=skill_mode,
+                skill_creator_dir=str(skill_creator_dir) if skill_creator_dir else None,
+                self_gen_no_internet=self_gen_no_internet,
+                source_provenance=resolved_dataset.provenance,
+                include_tasks=dataset_include,
+                exclude_tasks=exclude_tasks,
+                usage_tracking=eval_usage_tracking,
+                environment_manifest=eval_env_manifest,
+                dataset_name=resolved_dataset.name,
+                dataset_version=resolved_dataset.version,
+                dataset_task_digests=resolved_dataset.task_digests,
+            ),
+        )
     elif source_repo:
         from benchflow._utils.benchmark_repos import resolve_source_with_metadata
 
@@ -1408,7 +1492,8 @@ def eval_create(
         )
     else:
         console.print(
-            "[red]Provide --config, --tasks-dir, --source-repo, or --source-env[/red]"
+            "[red]Provide --config, --tasks-dir, --source-repo, --source-env, "
+            "or --dataset[/red]"
         )
         raise typer.Exit(1)
 

{benchflow-0.5.3.dev976 → benchflow-0.5.3.dev980}/src/benchflow/evaluation.py

@@ -267,6 +267,13 @@ class EvaluationConfig:
     self_gen_no_internet: bool = False
     job_mode: str = DEFAULT_JOB_MODE
     source_provenance: dict[str, Any] | None = None
+    # Registry dataset identity (`bench eval create -d name@version`). When
+    # set, every result.json/config.json is stamped with dataset_name,
+    # dataset_version, and the task's registry content digest — see
+    # docs/dataset-versioning.md in benchflow-ai/skillsbench.
+    dataset_name: str | None = None
+    dataset_version: str | None = None
+    dataset_task_digests: dict[str, str] = field(default_factory=dict)
     usage_tracking: UsageTrackingConfig = field(default_factory=UsageTrackingConfig)
     # Environment-plane manifest applied to every rollout in the batch.
     # When set, each task's RolloutConfig.environment_manifest is populated
@@ -817,6 +824,22 @@ class Evaluation:
         from benchflow._utils.benchmark_repos import task_source_provenance
         from benchflow.rollout import Rollout, RolloutConfig
 
+        dataset = None
+        if cfg.dataset_name:
+            dataset = {"name": cfg.dataset_name, "version": cfg.dataset_version}
+        task_digest_value = (
+            cfg.dataset_task_digests.get(task_dir.name) if cfg.dataset_name else None
+        )
+        if task_digest_value is None:
+            # Dev runs (--tasks-dir / --source-repo) stamp a live-computed
+            # digest so every trajectory stays attributable to the exact
+            # task content it ran, not just a directory name.
+            from benchflow._utils.task_authoring import task_digest
+
+            try:
+                task_digest_value = task_digest(task_dir)
+            except (OSError, ValueError, UnicodeError) as e:
+                logger.debug("Could not compute task digest for %s: %s", task_dir, e)
         skills_dir = (
             str(self._learner_skills_dir)
             if self._learner_skills_dir is not None
@@ -855,6 +878,8 @@ class Evaluation:
             self_gen_no_internet=cfg.self_gen_no_internet,
             export_generated_skills_to=export_to,
             source_provenance=task_source_provenance(cfg.source_provenance, task_dir),
+            dataset=dataset,
+            task_digest=task_digest_value,
             usage_tracking=cfg.usage_tracking,
         )
         if skill_mode == SKILL_MODE_SELF_GEN:
@@ -1383,6 +1408,14 @@ class Evaluation:
             **trajectory_step_summary(all_results),
             **phase_timing_summary(all_results),
             **summary_source_fields(cfg.source_provenance, all_results),
+            **(
+                {
+                    "dataset_name": cfg.dataset_name,
+                    "dataset_version": cfg.dataset_version,
+                }
+                if cfg.dataset_name
+                else {}
+            ),
         }
         # Surface continual-learning provenance — generation, curve — so a
         # resumed run can be audited end-to-end (#394).

{benchflow-0.5.3.dev976 → benchflow-0.5.3.dev980}/src/benchflow/rollout.py

@@ -525,6 +525,8 @@ def _write_config(
     agent_idle_timeout: int | None = None,
     scenes: list[Scene] | None = None,
     source_provenance: dict[str, Any] | None = None,
+    dataset: dict[str, Any] | None = None,
+    task_digest: str | None = None,
 ) -> None:
     """Write config.json to rollout_dir with secrets filtered out."""
     recorded_env = {
@@ -552,6 +554,11 @@ def _write_config(
         config_data["usage_tracking"] = usage_tracking.to_config_artifact()
     if source_provenance is not None:
         config_data["source"] = source_provenance
+    if dataset is not None:
+        config_data["dataset_name"] = dataset.get("name")
+        config_data["dataset_version"] = dataset.get("version")
+    if task_digest is not None:
+        config_data["task_digest"] = task_digest
     (rollout_dir / "config.json").write_text(json.dumps(config_data, indent=2))
 
 
@@ -616,6 +623,8 @@ def _build_rollout_result(
     usage_tracking: dict[str, Any] | None = None,
     evolved_skills: dict[str, str] | None = None,
     source_provenance: dict[str, Any] | None = None,
+    dataset: dict[str, Any] | None = None,
+    task_digest: str | None = None,
     diagnostics: RolloutDiagnostics | None = None,
     skill_policy: TaskSkillPolicy | None = None,
     sandbox_id: str | None = None,
@@ -743,6 +752,15 @@ def _build_rollout_result(
                     if source_provenance is not None
                     else {}
                 ),
+                **(
+                    {
+                        "dataset_name": dataset.get("name"),
+                        "dataset_version": dataset.get("version"),
+                    }
+                    if dataset is not None
+                    else {}
+                ),
+                **({"task_digest": task_digest} if task_digest is not None else {}),
                 "sandbox_id": sandbox_id,
             },
             indent=2,
@@ -1092,6 +1110,14 @@ class RolloutConfig:
     skip_verify: bool = False
     export_generated_skills_to: str | Path | None = None
     source_provenance: dict[str, Any] | None = None
+    # Registry dataset identity: {"name", "version"} — stamped into
+    # config.json/result.json as dataset_name/dataset_version so published
+    # runs declare the dataset version they ran (dataset-versioning plan).
+    dataset: dict[str, Any] | None = None
+    # Content digest of the task directory ("sha256:<hex>", the skillsbench
+    # registry algorithm). Registry-pinned for dataset runs, computed live
+    # for dev runs — every result stays attributable to exact task content.
+    task_digest: str | None = None
     planes: RolloutPlanes | None = field(default=None, repr=False, compare=False)
 
     def __post_init__(self) -> None:
@@ -1540,6 +1566,8 @@ class Rollout:
             agent_idle_timeout=cfg.agent_idle_timeout,
             scenes=cfg.effective_scenes,
             source_provenance=cfg.source_provenance,
+            dataset=cfg.dataset,
+            task_digest=cfg.task_digest,
         )
 
         self._phase = "setup"
@@ -2906,6 +2934,8 @@ class Rollout:
             scenes=self._config.effective_scenes,
             evolved_skills=self._evolved_skills,
             source_provenance=self._config.source_provenance,
+            dataset=self._config.dataset,
+            task_digest=self._config.task_digest,
             diagnostics=self._diagnostics,
             usage_tracking=self._usage_tracking_metadata(),
             skill_policy=getattr(self, "_task_skill_policy", None)

{benchflow-0.5.3.dev976 → benchflow-0.5.3.dev980}/tests/integration/check_results.py

@@ -32,7 +32,11 @@ from benchflow._utils.scoring import (
     classify_verifier_error,
     count_result_outcomes,
 )
-from benchflow._utils.source_provenance import source_issues, source_matches_parent
+from benchflow._utils.source_provenance import (
+    is_sha256_digest,
+    source_issues,
+    source_matches_parent,
+)
 from benchflow.trajectories.metrics import (
     count_skill_invocations,
     result_skill_invocations,
@@ -304,6 +308,73 @@ def _source_hash_truth_issues(source: object, label: str) -> list[str]:
     return _source_git_truth_issues(source_dict, local_path, label)
 
 
+# Dataset identity stamp (dataset-versioning plan: benchflow #690 + dev-run
+# digest stamping follow-up). dataset_name/dataset_version mark registry
+# dataset runs; task_digest pins the exact task content for every run.
+_DATASET_STAMP_KEYS = ("dataset_name", "dataset_version", "task_digest")
+
+
+def _dataset_stamp_issues(artifact: object, label: str) -> list[str]:
+    """Audit the dataset identity stamp on one result.json/config.json.
+
+    Rules: dataset_name and dataset_version travel together; a dataset run
+    must carry a task_digest; any task_digest must be ``sha256:<64 hex>``.
+    A bare task_digest without dataset fields is a dev run — allowed.
+    """
+    if not isinstance(artifact, dict):
+        return []
+    artifact_dict = cast(dict[str, Any], artifact)
+    name = artifact_dict.get("dataset_name")
+    version = artifact_dict.get("dataset_version")
+    digest = artifact_dict.get("task_digest")
+    if name is None and version is None and digest is None:
+        return []
+    issues: list[str] = []
+    if (name is None) != (version is None):
+        issues.append(f"{label}: dataset_name and dataset_version must travel together")
+    for field_name, value in (("dataset_name", name), ("dataset_version", version)):
+        if value is not None and (not isinstance(value, str) or not value):
+            issues.append(f"{label}: {field_name} must be a non-empty string")
+    if name is not None and digest is None:
+        issues.append(f"{label}: dataset-stamped artifact missing task_digest")
+    if digest is not None and not is_sha256_digest(digest):
+        issues.append(f"{label}: task_digest must be 'sha256:<64 hex>'")
+    return issues
+
+
+def _task_digest_truth_issues(result: object, label: str) -> list[str]:
+    """Recompute the task content digest when the task dir is still on disk.
+
+    Complements ``_source_hash_truth_issues``: ``source.file_hashes`` audits
+    per-file hashes, this audits the single registry-algorithm digest the
+    leaderboard pipeline keys on.
+    """
+    if not isinstance(result, dict):
+        return []
+    result_dict = cast(dict[str, Any], result)
+    digest = result_dict.get("task_digest")
+    if not is_sha256_digest(digest):
+        return []  # absent or malformed — format issues reported elsewhere
+    source = result_dict.get("source")
+    if not isinstance(source, dict):
+        return []
+    local_path_raw = source.get("local_path")
+    if not isinstance(local_path_raw, str):
+        return []
+    local_path = Path(local_path_raw)
+    if not (local_path / "task.toml").exists():
+        return []
+    try:
+        from benchflow._utils.task_authoring import task_digest
+
+        actual = task_digest(local_path)
+    except Exception as e:
+        return [f"{label}: task_digest could not be recomputed: {e}"]
+    if actual != digest:
+        return [f"{label}: task_digest does not match local_path content"]
+    return []
+
+
 def _git_stdout(cwd: Path, *args: str) -> str | None:
     completed = subprocess.run(
         ["git", "-C", str(cwd), *args],
@@ -535,6 +606,18 @@ def check_agent(agent_dir: Path) -> dict:
         if source_truth_issues:
             findings["issues"].extend(source_truth_issues)
             findings["ok"] = False
+        dataset_stamp_issues = _dataset_stamp_issues(
+            r, f"{r.get('task_name', '?')}: result.json"
+        )
+        if dataset_stamp_issues:
+            findings["issues"].extend(dataset_stamp_issues)
+            findings["ok"] = False
+        digest_truth_issues = _task_digest_truth_issues(
+            r, f"{r.get('task_name', '?')}: result.json"
+        )
+        if digest_truth_issues:
+            findings["issues"].extend(digest_truth_issues)
+            findings["ok"] = False
         _config_path, config, config_error = _load_config(result_file)
         if config_error:
             findings["issues"].append(f"{r.get('task_name', '?')}: {config_error}")
@@ -639,6 +722,18 @@ def check_agent(agent_dir: Path) -> dict:
                     f"{r.get('task_name', '?')}: config.json source does not match result.json"
                 )
                 findings["ok"] = False
+            config_dataset_issues = _dataset_stamp_issues(
+                config, f"{r.get('task_name', '?')}: config.json"
+            )
+            if config_dataset_issues:
+                findings["issues"].extend(config_dataset_issues)
+                findings["ok"] = False
+            if any(config.get(k) != r.get(k) for k in _DATASET_STAMP_KEYS):
+                findings["issues"].append(
+                    f"{r.get('task_name', '?')}: config.json dataset stamp "
+                    f"does not match result.json"
+                )
+                findings["ok"] = False
 
     # Infrastructure errors — driven from the central diagnostic registry
     # so adding a new diagnostic kind doesn't require editing this file
@@ -768,6 +863,17 @@ def check_agent(agent_dir: Path) -> dict:
                     "summary.json missing source provenance and not all results have source"
                 )
                 findings["ok"] = False
+            # Dataset identity must agree across summary.json and every
+            # result.json — one leaderboard namespace per dataset version.
+            for dataset_key in ("dataset_name", "dataset_version"):
+                dataset_values = {r.get(dataset_key) for r in results}
+                dataset_values.add(summary.get(dataset_key))
+                if len(dataset_values) > 1:
+                    findings["issues"].append(
+                        f"summary.json {dataset_key} does not agree across "
+                        f"summary and results: {sorted(map(repr, dataset_values))}"
+                    )
+                    findings["ok"] = False
             expected_model = _expected(agent, "model")
             expected_environment = _expected(agent, "environment")
             expected_concurrency = _expected(agent, "concurrency")