benchflow 0.5.3.dev956__tar.gz → 0.5.3.dev976__tar.gz

{benchflow-0.5.3.dev956 → benchflow-0.5.3.dev976}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.5.3.dev956
+Version: 0.5.3.dev976
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.5.3.dev956 → benchflow-0.5.3.dev976}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.5.3.dev956"
+version = "0.5.3.dev976"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.5.3.dev956 → benchflow-0.5.3.dev976}/src/benchflow/_utils/task_authoring.py

@@ -1,6 +1,8 @@
-"""Task authoring — init and check benchmark tasks."""
+"""Task authoring — init, check, and digest benchmark tasks."""
 
+import hashlib
 import logging
+import os
 import re
 import tomllib
 from pathlib import Path
@@ -94,6 +96,36 @@ def check_task(task_dir: Path) -> list[str]:
     return issues
 
 
+def task_digest(task_dir: Path) -> str:
+    """Content digest pinning a task's files, independent of git.
+
+    sha256 over every regular file under ``task_dir``, sorted by POSIX
+    relative path; each file contributes
+    ``path_utf8 + b"\\x00" + sha256(file_bytes).digest()``. Symlinks and
+    file modes are excluded, so the digest is reproducible from a plain
+    checkout. Must byte-match the reference digests in the skillsbench
+    dataset registry (``registry.json`` / ``docs/dataset-versioning.md``,
+    skillsbench PR #922).
+    """
+    if not task_dir.is_dir():
+        raise NotADirectoryError(f"Not a directory: {task_dir}")
+    files: list[tuple[str, Path]] = []
+    # os.walk never descends into symlinked directories (unlike pre-3.13
+    # Path.rglob), keeping "symlinks are excluded" true for whole subtrees.
+    for dirpath, _dirnames, filenames in os.walk(task_dir):
+        for filename in filenames:
+            path = Path(dirpath) / filename
+            if path.is_symlink() or not path.is_file():
+                continue
+            files.append((path.relative_to(task_dir).as_posix(), path))
+    digest = hashlib.sha256()
+    for rel, path in sorted(files):
+        digest.update(rel.encode("utf-8"))
+        digest.update(b"\x00")
+        digest.update(hashlib.sha256(path.read_bytes()).digest())
+    return f"sha256:{digest.hexdigest()}"
+
+
 _CTRF_STANDARD_PATH = "/logs/verifier/ctrf.json"
 
 

{benchflow-0.5.3.dev956 → benchflow-0.5.3.dev976}/src/benchflow/cli/main.py

@@ -674,6 +674,46 @@ def tasks_check(
         raise typer.Exit(1)
 
 
+@tasks_app.command("digest")
+def tasks_digest(
+    path: Annotated[
+        Path,
+        typer.Argument(
+            help="Path to a task directory, or a directory of task directories"
+        ),
+    ],
+) -> None:
+    """Compute the content digest that pins a task's files, independent of git.
+
+    Matches the digests in the skillsbench dataset registry (registry.json).
+    Given a single task directory (contains task.toml), prints the digest;
+    given a directory of tasks, prints one "<name> <digest>" line per task.
+    """
+    from benchflow._utils.task_authoring import task_digest
+
+    if not path.is_dir():
+        console.print(f"[red]Not a directory: {path}[/red]")
+        raise typer.Exit(1)
+
+    if (path / "task.toml").is_file():
+        # typer.echo, not console.print: Rich wraps lines at terminal width,
+        # which would corrupt piped machine-readable output.
+        typer.echo(task_digest(path))
+        return
+
+    task_dirs = sorted(
+        d for d in path.iterdir() if d.is_dir() and (d / "task.toml").is_file()
+    )
+    if not task_dirs:
+        console.print(
+            f"[red]No tasks under {path} — expected task.toml in it "
+            f"or in its immediate subdirectories[/red]"
+        )
+        raise typer.Exit(1)
+    for task_dir in task_dirs:
+        typer.echo(f"{task_dir.name} {task_digest(task_dir)}")
+
+
 compat_app = typer.Typer(help="Third-party framework compatibility checks.")
 app.add_typer(compat_app, name="compat")
 

benchflow-0.5.3.dev976/src/benchflow/sandbox/_cache_reclaim.py

@@ -0,0 +1,97 @@
+"""Pre-verifier cache reclaim command construction.
+
+The sandbox cannot import benchflow internals, so the reclaim implementation is
+an embedded Python snippet passed to ``python3 -c``. Keep it here instead of in
+``lockdown.py`` so the hardening orchestrator stays readable.
+"""
+
+import shlex
+from textwrap import dedent
+
+# Symlink- and workspace-safe cache reclaim for #601.
+#
+# The previous shell form (`rm -rf "$u/.cache/uv" ...; rm -rf /tmp/uv-*`) could
+# delete agent-visible state before scoring: `rm -rf` traverses intermediate
+# symlinks (an agent-planted `~/.cache -> /app` turned the cache delete into
+# `rm -rf /app/uv`), and the `/tmp/uv-*` glob matched a legitimate
+# `/tmp/uv-workspace`. The textual `$WS`/`$u` guard saw none of that. Since
+# ``restore_workspace`` defaults to False, nothing undid the damage.
+#
+# This snippet hardens every deletion candidate:
+#  - rejects any candidate with a symlink path component, including the final
+#    one, so agent-controlled parents can no longer redirect the delete;
+#  - resolves protected roots (workspace + /logs, covering /logs/artifacts and
+#    /logs/verifier) via realpath and skips candidates overlapping in either
+#    direction, applied uniformly to per-user caches, /tmp globs, and apt;
+#  - deletes with shutil.rmtree, which removes child symlinks without following
+#    them.
+#
+# argv[1] = active workspace (or /nonexistent). argv[2] = optional filesystem
+# prefix, "" in production; tests pass a temp root so the exact production code
+# runs hermetically against a fake filesystem.
+#
+# Fail-safe by construction: if python3 is unavailable the reclaim simply does
+# not run (the trailing `true` swallows it). Losing best-effort ENOSPC mitigation
+# is strictly better than deleting the wrong state.
+RECLAIM_CACHES_PY = dedent(
+    """
+    import glob, os, shutil, sys
+
+    ws = sys.argv[1]
+    px = sys.argv[2] if len(sys.argv) > 2 else ''
+
+    def linked(path):
+        p = px or ''
+        for part in path[len(px):].strip('/').split('/'):
+            p = p + '/' + part
+            if os.path.islink(p):
+                return True
+        return False
+
+    protected = []
+    for root in (ws, px + '/logs'):
+        try:
+            if root and os.path.exists(root):
+                protected.append(os.path.realpath(root))
+        except OSError:
+            pass
+
+    def overlaps(path):
+        return any(
+            path == r or path.startswith(r + '/') or r.startswith(path + '/')
+            for r in protected
+        )
+
+    cands = [
+        u + '/.cache/' + name
+        for u in [px + '/root'] + sorted(glob.glob(px + '/home/*'))
+        for name in ('uv', 'pip', 'uv_build')
+    ]
+    cands += sorted(glob.glob(px + '/tmp/uv-*'))
+    cands += sorted(glob.glob(px + '/tmp/.uv-*'))
+    cands += sorted(glob.glob(px + '/var/cache/apt/archives/*.deb'))
+    for c in cands:
+        try:
+            if not os.path.lexists(c) or linked(c):
+                continue
+            # linked() proved no symlink components below the prefix, so
+            # realpath only normalizes the prefix itself - consistent with
+            # the realpath'd protected roots.
+            if overlaps(os.path.realpath(c)):
+                continue
+            if os.path.isdir(c):
+                shutil.rmtree(c, ignore_errors=True)
+            else:
+                os.remove(c)
+        except OSError:
+            pass
+    """
+).strip()
+
+
+def build_reclaim_caches_cmd(workspace: str | None) -> str:
+    """Build the best-effort cache reclaim shell command."""
+    workspace_arg = shlex.quote(workspace) if workspace else "/nonexistent"
+    return (
+        f"python3 -c {shlex.quote(RECLAIM_CACHES_PY)} {workspace_arg} 2>/dev/null; true"
+    )

{benchflow-0.5.3.dev956 → benchflow-0.5.3.dev976}/src/benchflow/sandbox/lockdown.py

@@ -20,6 +20,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 from benchflow.agents.registry import get_sandbox_home_dirs
+from benchflow.sandbox._cache_reclaim import build_reclaim_caches_cmd
 
 if TYPE_CHECKING:
     from benchflow.task import Task
@@ -877,22 +878,15 @@ async def harden_before_verify(
     # installed tools, or task assets — and a failure here never blocks the
     # verifier. Runs on `main` only, consistent with the hardening policy above.
     #
-    # Workspace-aware: a task can legitimately use /root or /home/<user> as its
-    # workspace (so "$u/.cache" would be workspace state the verifier must see
-    # untouched). Skip any per-user cache that overlaps the active workspace in
-    # either direction — this matters because restore_workspace defaults to False,
-    # so the later full restore does NOT run to undo a stray deletion.
-    workspace_guard = shlex.quote(workspace) if workspace else "/nonexistent"
+    # Workspace-aware AND symlink-safe (#601): a task can legitimately use
+    # /root, /home/<user>, or /tmp/uv-* as its workspace, and an agent can
+    # plant `~/.cache -> /app` so a naive `rm -rf "$u/.cache/uv"` traverses
+    # into workspace/output state. The reclaim command rejects symlinked
+    # candidates and realpath-guards every deletion against the workspace and
+    # /logs — this matters because restore_workspace defaults to False.
     try:
         await env.exec(
-            f"WS={workspace_guard}; "
-            "for u in /root /home/*; do "
-            '  case "$WS" in "$u" | "$u"/*) continue ;; esac; '
-            '  case "$u" in "$WS"/*) continue ;; esac; '
-            '  rm -rf "$u/.cache/uv" "$u/.cache/pip" "$u/.cache/uv_build" 2>/dev/null; '
-            "done; "
-            "rm -rf /tmp/uv-* /tmp/.uv-* /var/cache/apt/archives/*.deb 2>/dev/null; "
-            "true",
+            build_reclaim_caches_cmd(workspace),
             user="root",
             timeout_sec=30,
         )

benchflow-0.5.3.dev976/tests/fixtures/digest/README.md

@@ -0,0 +1,9 @@
+# Digest test fixtures
+
+Golden-test fixtures for `bench tasks digest`. The expected digests are
+hardcoded in `tests/test_task_digest.py` — do not modify any file under the
+task directories here without updating those golden values.
+
+Filenames are ASCII on purpose: macOS checks out unicode filenames in NFD,
+which would change the hashed path bytes across platforms. Unicode coverage
+lives in file *content* (`beta-task/data/unicode.txt`).

benchflow-0.5.3.dev976/tests/fixtures/digest/alpha-task/environment/Dockerfile

@@ -0,0 +1 @@
+FROM ubuntu:24.04

benchflow-0.5.3.dev976/tests/fixtures/digest/alpha-task/instruction.md

@@ -0,0 +1,3 @@
+# alpha-task
+
+Compute the answer.

benchflow-0.5.3.dev976/tests/fixtures/digest/alpha-task/task.toml

@@ -0,0 +1,4 @@
+version = "1.1"
+
+[metadata]
+difficulty = "easy"

benchflow-0.5.3.dev976/tests/fixtures/digest/alpha-task/tests/test.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+echo "1.0" > /logs/verifier/reward.txt

benchflow-0.5.3.dev976/tests/fixtures/digest/beta-task/data/unicode.txt

@@ -0,0 +1 @@
+naïve résumé — ünïcödé

benchflow-0.5.3.dev976/tests/fixtures/digest/beta-task/instruction.md

@@ -0,0 +1 @@
+# beta-task

benchflow-0.5.3.dev976/tests/fixtures/digest/beta-task/task.toml

@@ -0,0 +1 @@
+version = "1.1"

benchflow-0.5.3.dev976/tests/fixtures/digest/not-a-task/notes.txt

@@ -0,0 +1 @@
+Not a task — no task.toml. Multi-task digest mode must skip this directory.

{benchflow-0.5.3.dev956 → benchflow-0.5.3.dev976}/tests/test_sandbox_hardening.py

@@ -280,37 +280,36 @@ class TestVerifierDirWipe:
     async def test_reclaims_redownloadable_caches_before_verify(self):
         """Frees uv/pip/apt download caches so the verifier's own deps fit on
         disk-constrained sandboxes (e.g. Daytona's 10GB cap), without ever
-        touching the workspace, agent outputs, installed tools, or task assets."""
+        touching the workspace, agent outputs, installed tools, or task assets.
+        Guards PR #669."""
         from benchflow.sandbox.lockdown import harden_before_verify
 
         env = _make_env()
         await harden_before_verify(env, _make_task(), sandbox_user=None)
 
         reclaim = next(
-            (c for c in env.exec.call_args_list if ".cache/uv" in c.args[0]),
+            (c for c in env.exec.call_args_list if "/.cache/" in c.args[0]),
             None,
         )
         assert reclaim is not None, "expected a pre-verifier disk reclaim exec"
         cmd = reclaim.args[0]
-        # clears only re-downloadable caches ...
-        assert ".cache/pip" in cmd and "apt/archives" in cmd
-        # ... and never the workspace, agent outputs, installed tools, or assets
-        for forbidden in ("/app", "/workspace", "/tests", "site-packages", ".venv"):
-            assert forbidden not in cmd, f"reclaim must not touch {forbidden}"
+        # clears only re-downloadable caches (uv/pip/apt) ...
+        assert "uv_build" in cmd and "pip" in cmd and "apt/archives" in cmd
+        # ... guarded against symlinks and the workspace/output roots (#601)
+        assert "islink" in cmd and "realpath" in cmd and "/logs" in cmd
+        # no active workspace -> the guard placeholder is plumbed through
+        assert "/nonexistent" in cmd
         # best-effort: swallows errors and never aborts hardening
         assert "2>/dev/null" in cmd and cmd.rstrip().endswith("true")
         # runs as root so it can clear every user's cache
         assert reclaim.kwargs.get("user") == "root"
 
     @pytest.mark.asyncio
-    @pytest.mark.parametrize("workspace", ["/root", "/home/agent"])
-    async def test_reclaim_skips_caches_inside_the_workspace(self, workspace):
-        """When a task uses /root or /home/<user> as its workspace, the reclaim
-        must NOT delete "$workspace/.cache" — that is workspace state the verifier
-        must see untouched. restore_workspace defaults False, so nothing would
-        restore a stray deletion. Guards the workspace-fidelity review blocker."""
+    @pytest.mark.parametrize("workspace", ["/root", "/home/agent", "/app"])
+    async def test_reclaim_plumbs_workspace_into_guard(self, workspace):
+        """The active workspace is passed to the reclaim snippet as argv[1] so
+        the realpath overlap guard can protect it. Guards PR #669."""
         import shlex
-        import subprocess
 
         from benchflow.sandbox.lockdown import harden_before_verify
 
@@ -320,47 +319,104 @@ class TestVerifierDirWipe:
         )
 
         reclaim = next(
-            (c for c in env.exec.call_args_list if ".cache/uv" in c.args[0]), None
+            (c for c in env.exec.call_args_list if "/.cache/" in c.args[0]), None
         )
         assert reclaim is not None
-        cmd = reclaim.args[0]
-        # the active workspace is plumbed into the guard ...
-        assert f"WS={shlex.quote(workspace)}" in cmd
-        # ... and overlap is guarded in both directions
-        assert 'case "$WS" in "$u" | "$u"/*) continue' in cmd
-        assert 'case "$u" in "$WS"/*) continue' in cmd
-
-        # behaviorally: the guard skips the workspace dir itself (hermetic check)
-        decide = subprocess.run(
-            [
-                "sh",
-                "-c",
-                f"WS={shlex.quote(workspace)}; u={shlex.quote(workspace)}; "
-                'case "$WS" in "$u" | "$u"/*) echo skip; exit ;; esac; '
-                'case "$u" in "$WS"/*) echo skip; exit ;; esac; echo clear',
-            ],
-            capture_output=True,
-            text=True,
-        ).stdout.strip()
-        assert decide == "skip", (
-            f"workspace {workspace} should be skipped, got {decide}"
+        assert (
+            reclaim.args[0]
+            .rstrip()
+            .endswith(f"{shlex.quote(workspace)} 2>/dev/null; true")
         )
 
-    @pytest.mark.asyncio
-    async def test_reclaim_still_clears_caches_outside_the_workspace(self):
-        """A workspace like /app (not under /root or /home) leaves the per-user
-        cache reclaim fully active."""
-        from benchflow.sandbox.lockdown import harden_before_verify
+    # ── Behavior tests: run the REAL reclaim snippet against a temp root ──────
+    # (#601 regression suite — the old string-shape tests passed while the
+    # shell command deleted agent state through symlinks and /tmp globs.)
 
-        env = _make_env()
-        await harden_before_verify(
-            env, _make_task(), sandbox_user=None, workspace="/app"
-        )
-        reclaim = next(
-            (c for c in env.exec.call_args_list if ".cache/uv" in c.args[0]), None
-        )
-        assert reclaim is not None
-        assert "WS=/app" in reclaim.args[0]
+    @staticmethod
+    def _run_reclaim(workspace: str, prefix) -> None:
+        """Execute the production reclaim snippet hermetically under
+        ``prefix`` - the same code, same interpreter contract as the sandbox."""
+        import subprocess
+        import sys
+
+        from benchflow.sandbox._cache_reclaim import RECLAIM_CACHES_PY
+
+        result = subprocess.run(
+            [sys.executable, "-c", RECLAIM_CACHES_PY, workspace, str(prefix)],
+            capture_output=True,
+            text=True,
+            timeout=60,
+        )
+        assert result.returncode == 0, result.stderr
+
+    def test_reclaim_does_not_traverse_cache_symlink_into_workspace(self, tmp_path):
+        """Guards PR #669 for issue #601 (repro 1): an agent-planted
+        ``~/.cache -> /app`` must not turn the cache delete into
+        ``rm -rf /app/uv`` — the old shell form did exactly that."""
+        (tmp_path / "app" / "uv").mkdir(parents=True)
+        (tmp_path / "app" / "uv" / "state.txt").write_text("agent state")
+        (tmp_path / "root").mkdir()
+        (tmp_path / "root" / ".cache").symlink_to(tmp_path / "app")
+
+        self._run_reclaim(str(tmp_path / "app"), tmp_path)
+
+        assert (tmp_path / "app" / "uv" / "state.txt").exists()
+
+    def test_reclaim_does_not_traverse_symlink_into_artifacts(self, tmp_path):
+        """Guards PR #669 for issue #601 (repro 2): ``.cache -> /logs/artifacts``
+        must not delete artifact state — /logs is a protected root even when it
+        is not the workspace."""
+        (tmp_path / "logs" / "artifacts" / "uv").mkdir(parents=True)
+        (tmp_path / "logs" / "artifacts" / "uv" / "state.txt").write_text("x")
+        (tmp_path / "home" / "agent").mkdir(parents=True)
+        (tmp_path / "home" / "agent" / ".cache").symlink_to(
+            tmp_path / "logs" / "artifacts"
+        )
+
+        self._run_reclaim("/nonexistent", tmp_path)
+
+        assert (tmp_path / "logs" / "artifacts" / "uv" / "state.txt").exists()
+
+    def test_reclaim_spares_tmp_workspace_matching_glob(self, tmp_path):
+        """Guards PR #669 for issue #601 (repro 3): a legitimate
+        ``/tmp/uv-workspace`` workspace must survive the ``/tmp/uv-*`` glob —
+        the old form deleted it unconditionally."""
+        ws = tmp_path / "tmp" / "uv-workspace"
+        ws.mkdir(parents=True)
+        (ws / "state.txt").write_text("answer")
+
+        self._run_reclaim(str(ws), tmp_path)
+
+        assert (ws / "state.txt").exists()
+
+    def test_reclaim_still_clears_real_caches_outside_workspace(self, tmp_path):
+        """Guards PR #669 for issue #601 (repro 4): real, non-symlinked caches
+        outside the workspace are still reclaimed, so the ENOSPC mitigation
+        the reclaim exists for stays effective."""
+        (tmp_path / "root" / ".cache" / "uv" / "blob").mkdir(parents=True)
+        (tmp_path / "home" / "u1" / ".cache" / "pip").mkdir(parents=True)
+        (tmp_path / "tmp" / "uv-build123").mkdir(parents=True)
+        (tmp_path / "var" / "cache" / "apt" / "archives").mkdir(parents=True)
+        (tmp_path / "var" / "cache" / "apt" / "archives" / "x.deb").write_text("d")
+
+        self._run_reclaim(str(tmp_path / "app"), tmp_path)
+
+        assert not (tmp_path / "root" / ".cache" / "uv").exists()
+        assert not (tmp_path / "home" / "u1" / ".cache" / "pip").exists()
+        assert not (tmp_path / "tmp" / "uv-build123").exists()
+        assert not (tmp_path / "var" / "cache" / "apt" / "archives" / "x.deb").exists()
+
+    def test_reclaim_skips_caches_inside_the_workspace(self, tmp_path):
+        """When a task uses /root as its workspace, "$ws/.cache" is workspace
+        state the verifier must see untouched (the pre-#601 guarantee, now
+        enforced by realpath overlap instead of string comparison). Guards PR #669."""
+        cache = tmp_path / "root" / ".cache" / "uv"
+        cache.mkdir(parents=True)
+        (cache / "state.txt").write_text("workspace state")
+
+        self._run_reclaim(str(tmp_path / "root"), tmp_path)
+
+        assert (cache / "state.txt").exists()
 
 
 # ── TestBuildConfigSnapshot ───────────────────────────────────────────────────

benchflow-0.5.3.dev976/tests/test_task_digest.py

@@ -0,0 +1,161 @@
+"""Tests for `bench tasks digest` and task_digest().
+
+The digest pins a task's content independent of git and must byte-match the
+reference digests published in the skillsbench dataset registry
+(registry.json / docs/dataset-versioning.md, skillsbench PR #922):
+
+    task_digest = sha256( for each regular file under the task directory,
+                          sorted by POSIX relative path:
+                            update( path_utf8 + b"\\x00"
+                                    + sha256(file_bytes).digest() ) )
+
+Symlinks and file modes are excluded; the hex digest is prefixed "sha256:".
+
+The implementation was cross-validated against all 87 published digests of
+the skillsbench v1.1 registry entry (git tag v1.1, commit 14f33967) — e.g.
+tasks/3d-scan-calc reproduces
+sha256:587a79f13a31098b5f7b4d1f99582ac6d4184e2cca1f02e2679fc98b81861220.
+The golden values below freeze the same algorithm onto an offline fixture.
+"""
+
+import hashlib
+from pathlib import Path
+
+import pytest
+from typer.testing import CliRunner
+
+from benchflow._utils.task_authoring import task_digest
+from benchflow.cli.main import app
+
+FIXTURES = Path(__file__).parent / "fixtures" / "digest"
+
+# Frozen against the checked-in fixture tasks. If these fail without a
+# deliberate fixture change, the algorithm drifted from the published
+# skillsbench registry digests — fix the code, not the constants.
+ALPHA_DIGEST = "sha256:6a0c023a27c1dd1b68b16ed23a41cdd9b3ea3401599bc6abbb95ed77d41e08f9"
+BETA_DIGEST = "sha256:8f5a4f868cef8c5c50b700183d0b9bd7405d92245eed79a0d50494b7daa0de16"
+
+
+def _spec_digest(file_entries: list[tuple[str, bytes]]) -> str:
+    """The registry construction, written out literally, in the given order."""
+    h = hashlib.sha256()
+    for rel_posix, content in file_entries:
+        h.update(rel_posix.encode("utf-8"))
+        h.update(b"\x00")
+        h.update(hashlib.sha256(content).digest())
+    return f"sha256:{h.hexdigest()}"
+
+
+class TestTaskDigest:
+    def test_golden_fixture_digests(self):
+        assert task_digest(FIXTURES / "alpha-task") == ALPHA_DIGEST
+        assert task_digest(FIXTURES / "beta-task") == BETA_DIGEST
+
+    def test_matches_spec_construction(self, tmp_path):
+        task = tmp_path / "task"
+        (task / "environment").mkdir(parents=True)
+        (task / "task.toml").write_bytes(b'version = "1.1"\n')
+        (task / "instruction.md").write_bytes(b"do the thing\n")
+        (task / "environment" / "Dockerfile").write_bytes(b"FROM scratch\n")
+        expected = _spec_digest(
+            [
+                ("environment/Dockerfile", b"FROM scratch\n"),
+                ("instruction.md", b"do the thing\n"),
+                ("task.toml", b'version = "1.1"\n'),
+            ]
+        )
+        assert task_digest(task) == expected
+
+    def test_sorts_by_posix_path_string_not_path_parts(self, tmp_path):
+        """'a.b/c' sorts before 'a/c' as a string ('.' < '/'), after it as
+        Path parts — the registry orders by the POSIX path string."""
+        task = tmp_path / "task"
+        (task / "a").mkdir(parents=True)
+        (task / "a.b").mkdir()
+        (task / "a" / "c").write_bytes(b"slash")
+        (task / "a.b" / "c").write_bytes(b"dot")
+        string_order = _spec_digest([("a.b/c", b"dot"), ("a/c", b"slash")])
+        parts_order = _spec_digest([("a/c", b"slash"), ("a.b/c", b"dot")])
+        assert string_order != parts_order
+        assert task_digest(task) == string_order
+
+    def test_symlinks_excluded(self, tmp_path):
+        task = tmp_path / "task"
+        task.mkdir()
+        (task / "task.toml").write_bytes(b'version = "1.1"\n')
+        before = task_digest(task)
+
+        outside = tmp_path / "outside"
+        outside.mkdir()
+        (outside / "payload.txt").write_bytes(b"must not be hashed")
+        (task / "file-link.toml").symlink_to(task / "task.toml")
+        (task / "dir-link").symlink_to(outside, target_is_directory=True)
+        (task / "broken-link").symlink_to(task / "does-not-exist")
+
+        assert task_digest(task) == before
+
+    def test_file_modes_excluded(self, tmp_path):
+        task = tmp_path / "task"
+        task.mkdir()
+        script = task / "solve.sh"
+        script.write_bytes(b"#!/bin/bash\n")
+        before = task_digest(task)
+        script.chmod(0o755)
+        assert task_digest(task) == before
+
+    def test_empty_directories_excluded(self, tmp_path):
+        task = tmp_path / "task"
+        task.mkdir()
+        (task / "task.toml").write_bytes(b'version = "1.1"\n')
+        before = task_digest(task)
+        (task / "empty" / "nested").mkdir(parents=True)
+        assert task_digest(task) == before
+
+    def test_sensitive_to_content_and_path(self, tmp_path):
+        task = tmp_path / "task"
+        task.mkdir()
+        f = task / "task.toml"
+        f.write_bytes(b'version = "1.1"\n')
+        original = task_digest(task)
+
+        f.write_bytes(b'version = "1.2"\n')
+        changed_content = task_digest(task)
+        assert changed_content != original
+
+        f.write_bytes(b'version = "1.1"\n')
+        f.rename(task / "renamed.toml")
+        assert task_digest(task) not in (original, changed_content)
+
+    def test_not_a_directory_raises(self, tmp_path):
+        with pytest.raises(NotADirectoryError):
+            task_digest(tmp_path / "missing")
+
+
+class TestTasksDigestCli:
+    def test_single_task_prints_digest(self):
+        result = CliRunner().invoke(
+            app, ["tasks", "digest", str(FIXTURES / "alpha-task")]
+        )
+        assert result.exit_code == 0
+        assert result.output == f"{ALPHA_DIGEST}\n"
+
+    def test_directory_of_tasks_prints_name_digest_lines(self):
+        """One '<name> <digest>' line per task, sorted; non-task entries
+        (not-a-task/, README.md) are skipped."""
+        result = CliRunner().invoke(app, ["tasks", "digest", str(FIXTURES)])
+        assert result.exit_code == 0
+        assert result.output.splitlines() == [
+            f"alpha-task {ALPHA_DIGEST}",
+            f"beta-task {BETA_DIGEST}",
+        ]
+
+    def test_missing_path_fails(self, tmp_path):
+        result = CliRunner().invoke(app, ["tasks", "digest", str(tmp_path / "nope")])
+        assert result.exit_code == 1
+
+    def test_directory_without_tasks_fails(self):
+        result = CliRunner().invoke(
+            app, ["tasks", "digest", str(FIXTURES / "not-a-task")]
+        )
+        assert result.exit_code == 1
+        assert "No tasks under" in result.output