benchflow 0.5.3.dev957__tar.gz → 0.5.3.dev976__tar.gz

{benchflow-0.5.3.dev957 → benchflow-0.5.3.dev976}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.5.3.dev957
+Version: 0.5.3.dev976
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.5.3.dev957 → benchflow-0.5.3.dev976}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.5.3.dev957"
+version = "0.5.3.dev976"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.5.3.dev957 → benchflow-0.5.3.dev976}/src/benchflow/_utils/task_authoring.py

@@ -1,6 +1,8 @@
-"""Task authoring — init and check benchmark tasks."""
+"""Task authoring — init, check, and digest benchmark tasks."""
 
+import hashlib
 import logging
+import os
 import re
 import tomllib
 from pathlib import Path
@@ -94,6 +96,36 @@ def check_task(task_dir: Path) -> list[str]:
     return issues
 
 
+def task_digest(task_dir: Path) -> str:
+    """Content digest pinning a task's files, independent of git.
+
+    sha256 over every regular file under ``task_dir``, sorted by POSIX
+    relative path; each file contributes
+    ``path_utf8 + b"\\x00" + sha256(file_bytes).digest()``. Symlinks and
+    file modes are excluded, so the digest is reproducible from a plain
+    checkout. Must byte-match the reference digests in the skillsbench
+    dataset registry (``registry.json`` / ``docs/dataset-versioning.md``,
+    skillsbench PR #922).
+    """
+    if not task_dir.is_dir():
+        raise NotADirectoryError(f"Not a directory: {task_dir}")
+    files: list[tuple[str, Path]] = []
+    # os.walk never descends into symlinked directories (unlike pre-3.13
+    # Path.rglob), keeping "symlinks are excluded" true for whole subtrees.
+    for dirpath, _dirnames, filenames in os.walk(task_dir):
+        for filename in filenames:
+            path = Path(dirpath) / filename
+            if path.is_symlink() or not path.is_file():
+                continue
+            files.append((path.relative_to(task_dir).as_posix(), path))
+    digest = hashlib.sha256()
+    for rel, path in sorted(files):
+        digest.update(rel.encode("utf-8"))
+        digest.update(b"\x00")
+        digest.update(hashlib.sha256(path.read_bytes()).digest())
+    return f"sha256:{digest.hexdigest()}"
+
+
 _CTRF_STANDARD_PATH = "/logs/verifier/ctrf.json"
 
 

{benchflow-0.5.3.dev957 → benchflow-0.5.3.dev976}/src/benchflow/cli/main.py

@@ -674,6 +674,46 @@ def tasks_check(
         raise typer.Exit(1)
 
 
+@tasks_app.command("digest")
+def tasks_digest(
+    path: Annotated[
+        Path,
+        typer.Argument(
+            help="Path to a task directory, or a directory of task directories"
+        ),
+    ],
+) -> None:
+    """Compute the content digest that pins a task's files, independent of git.
+
+    Matches the digests in the skillsbench dataset registry (registry.json).
+    Given a single task directory (contains task.toml), prints the digest;
+    given a directory of tasks, prints one "<name> <digest>" line per task.
+    """
+    from benchflow._utils.task_authoring import task_digest
+
+    if not path.is_dir():
+        console.print(f"[red]Not a directory: {path}[/red]")
+        raise typer.Exit(1)
+
+    if (path / "task.toml").is_file():
+        # typer.echo, not console.print: Rich wraps lines at terminal width,
+        # which would corrupt piped machine-readable output.
+        typer.echo(task_digest(path))
+        return
+
+    task_dirs = sorted(
+        d for d in path.iterdir() if d.is_dir() and (d / "task.toml").is_file()
+    )
+    if not task_dirs:
+        console.print(
+            f"[red]No tasks under {path} — expected task.toml in it "
+            f"or in its immediate subdirectories[/red]"
+        )
+        raise typer.Exit(1)
+    for task_dir in task_dirs:
+        typer.echo(f"{task_dir.name} {task_digest(task_dir)}")
+
+
 compat_app = typer.Typer(help="Third-party framework compatibility checks.")
 app.add_typer(compat_app, name="compat")
 

benchflow-0.5.3.dev976/tests/fixtures/digest/README.md

@@ -0,0 +1,9 @@
+# Digest test fixtures
+
+Golden-test fixtures for `bench tasks digest`. The expected digests are
+hardcoded in `tests/test_task_digest.py` — do not modify any file under the
+task directories here without updating those golden values.
+
+Filenames are ASCII on purpose: macOS checks out unicode filenames in NFD,
+which would change the hashed path bytes across platforms. Unicode coverage
+lives in file *content* (`beta-task/data/unicode.txt`).

benchflow-0.5.3.dev976/tests/fixtures/digest/alpha-task/environment/Dockerfile

@@ -0,0 +1 @@
+FROM ubuntu:24.04

benchflow-0.5.3.dev976/tests/fixtures/digest/alpha-task/instruction.md

@@ -0,0 +1,3 @@
+# alpha-task
+
+Compute the answer.

benchflow-0.5.3.dev976/tests/fixtures/digest/alpha-task/task.toml

@@ -0,0 +1,4 @@
+version = "1.1"
+
+[metadata]
+difficulty = "easy"

benchflow-0.5.3.dev976/tests/fixtures/digest/alpha-task/tests/test.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+echo "1.0" > /logs/verifier/reward.txt

benchflow-0.5.3.dev976/tests/fixtures/digest/beta-task/data/unicode.txt

@@ -0,0 +1 @@
+naïve résumé — ünïcödé

benchflow-0.5.3.dev976/tests/fixtures/digest/beta-task/instruction.md

@@ -0,0 +1 @@
+# beta-task

benchflow-0.5.3.dev976/tests/fixtures/digest/beta-task/task.toml

@@ -0,0 +1 @@
+version = "1.1"

benchflow-0.5.3.dev976/tests/fixtures/digest/not-a-task/notes.txt

@@ -0,0 +1 @@
+Not a task — no task.toml. Multi-task digest mode must skip this directory.

benchflow-0.5.3.dev976/tests/test_task_digest.py

@@ -0,0 +1,161 @@
+"""Tests for `bench tasks digest` and task_digest().
+
+The digest pins a task's content independent of git and must byte-match the
+reference digests published in the skillsbench dataset registry
+(registry.json / docs/dataset-versioning.md, skillsbench PR #922):
+
+    task_digest = sha256( for each regular file under the task directory,
+                          sorted by POSIX relative path:
+                            update( path_utf8 + b"\\x00"
+                                    + sha256(file_bytes).digest() ) )
+
+Symlinks and file modes are excluded; the hex digest is prefixed "sha256:".
+
+The implementation was cross-validated against all 87 published digests of
+the skillsbench v1.1 registry entry (git tag v1.1, commit 14f33967) — e.g.
+tasks/3d-scan-calc reproduces
+sha256:587a79f13a31098b5f7b4d1f99582ac6d4184e2cca1f02e2679fc98b81861220.
+The golden values below freeze the same algorithm onto an offline fixture.
+"""
+
+import hashlib
+from pathlib import Path
+
+import pytest
+from typer.testing import CliRunner
+
+from benchflow._utils.task_authoring import task_digest
+from benchflow.cli.main import app
+
+FIXTURES = Path(__file__).parent / "fixtures" / "digest"
+
+# Frozen against the checked-in fixture tasks. If these fail without a
+# deliberate fixture change, the algorithm drifted from the published
+# skillsbench registry digests — fix the code, not the constants.
+ALPHA_DIGEST = "sha256:6a0c023a27c1dd1b68b16ed23a41cdd9b3ea3401599bc6abbb95ed77d41e08f9"
+BETA_DIGEST = "sha256:8f5a4f868cef8c5c50b700183d0b9bd7405d92245eed79a0d50494b7daa0de16"
+
+
+def _spec_digest(file_entries: list[tuple[str, bytes]]) -> str:
+    """The registry construction, written out literally, in the given order."""
+    h = hashlib.sha256()
+    for rel_posix, content in file_entries:
+        h.update(rel_posix.encode("utf-8"))
+        h.update(b"\x00")
+        h.update(hashlib.sha256(content).digest())
+    return f"sha256:{h.hexdigest()}"
+
+
+class TestTaskDigest:
+    def test_golden_fixture_digests(self):
+        assert task_digest(FIXTURES / "alpha-task") == ALPHA_DIGEST
+        assert task_digest(FIXTURES / "beta-task") == BETA_DIGEST
+
+    def test_matches_spec_construction(self, tmp_path):
+        task = tmp_path / "task"
+        (task / "environment").mkdir(parents=True)
+        (task / "task.toml").write_bytes(b'version = "1.1"\n')
+        (task / "instruction.md").write_bytes(b"do the thing\n")
+        (task / "environment" / "Dockerfile").write_bytes(b"FROM scratch\n")
+        expected = _spec_digest(
+            [
+                ("environment/Dockerfile", b"FROM scratch\n"),
+                ("instruction.md", b"do the thing\n"),
+                ("task.toml", b'version = "1.1"\n'),
+            ]
+        )
+        assert task_digest(task) == expected
+
+    def test_sorts_by_posix_path_string_not_path_parts(self, tmp_path):
+        """'a.b/c' sorts before 'a/c' as a string ('.' < '/'), after it as
+        Path parts — the registry orders by the POSIX path string."""
+        task = tmp_path / "task"
+        (task / "a").mkdir(parents=True)
+        (task / "a.b").mkdir()
+        (task / "a" / "c").write_bytes(b"slash")
+        (task / "a.b" / "c").write_bytes(b"dot")
+        string_order = _spec_digest([("a.b/c", b"dot"), ("a/c", b"slash")])
+        parts_order = _spec_digest([("a/c", b"slash"), ("a.b/c", b"dot")])
+        assert string_order != parts_order
+        assert task_digest(task) == string_order
+
+    def test_symlinks_excluded(self, tmp_path):
+        task = tmp_path / "task"
+        task.mkdir()
+        (task / "task.toml").write_bytes(b'version = "1.1"\n')
+        before = task_digest(task)
+
+        outside = tmp_path / "outside"
+        outside.mkdir()
+        (outside / "payload.txt").write_bytes(b"must not be hashed")
+        (task / "file-link.toml").symlink_to(task / "task.toml")
+        (task / "dir-link").symlink_to(outside, target_is_directory=True)
+        (task / "broken-link").symlink_to(task / "does-not-exist")
+
+        assert task_digest(task) == before
+
+    def test_file_modes_excluded(self, tmp_path):
+        task = tmp_path / "task"
+        task.mkdir()
+        script = task / "solve.sh"
+        script.write_bytes(b"#!/bin/bash\n")
+        before = task_digest(task)
+        script.chmod(0o755)
+        assert task_digest(task) == before
+
+    def test_empty_directories_excluded(self, tmp_path):
+        task = tmp_path / "task"
+        task.mkdir()
+        (task / "task.toml").write_bytes(b'version = "1.1"\n')
+        before = task_digest(task)
+        (task / "empty" / "nested").mkdir(parents=True)
+        assert task_digest(task) == before
+
+    def test_sensitive_to_content_and_path(self, tmp_path):
+        task = tmp_path / "task"
+        task.mkdir()
+        f = task / "task.toml"
+        f.write_bytes(b'version = "1.1"\n')
+        original = task_digest(task)
+
+        f.write_bytes(b'version = "1.2"\n')
+        changed_content = task_digest(task)
+        assert changed_content != original
+
+        f.write_bytes(b'version = "1.1"\n')
+        f.rename(task / "renamed.toml")
+        assert task_digest(task) not in (original, changed_content)
+
+    def test_not_a_directory_raises(self, tmp_path):
+        with pytest.raises(NotADirectoryError):
+            task_digest(tmp_path / "missing")
+
+
+class TestTasksDigestCli:
+    def test_single_task_prints_digest(self):
+        result = CliRunner().invoke(
+            app, ["tasks", "digest", str(FIXTURES / "alpha-task")]
+        )
+        assert result.exit_code == 0
+        assert result.output == f"{ALPHA_DIGEST}\n"
+
+    def test_directory_of_tasks_prints_name_digest_lines(self):
+        """One '<name> <digest>' line per task, sorted; non-task entries
+        (not-a-task/, README.md) are skipped."""
+        result = CliRunner().invoke(app, ["tasks", "digest", str(FIXTURES)])
+        assert result.exit_code == 0
+        assert result.output.splitlines() == [
+            f"alpha-task {ALPHA_DIGEST}",
+            f"beta-task {BETA_DIGEST}",
+        ]
+
+    def test_missing_path_fails(self, tmp_path):
+        result = CliRunner().invoke(app, ["tasks", "digest", str(tmp_path / "nope")])
+        assert result.exit_code == 1
+
+    def test_directory_without_tasks_fails(self):
+        result = CliRunner().invoke(
+            app, ["tasks", "digest", str(FIXTURES / "not-a-task")]
+        )
+        assert result.exit_code == 1
+        assert "No tasks under" in result.output