dblock 0.1.0__py3-none-any.whl

dblock/lockfile.py

@@ -0,0 +1,132 @@
+"""Read and write dblock.lock (TOML)."""
+
+from __future__ import annotations
+
+import os
+import tomllib
+from dataclasses import asdict, dataclass, field, fields
+from pathlib import Path
+
+from dblock.recipes import Recipe, Release
+
+LOCK_VERSION = 1
+HEADER = (
+    "# dblock lockfile: exact reference database releases used by this analysis.\n"
+    "# Generated by dblock. Commit it with your code; edit with `dblock add/remove/fetch`.\n"
+)
+
+
+class LockError(Exception):
+    pass
+
+
+@dataclass
+class Entry:
+    name: str
+    release: str
+    recipe_digest: str
+    filename: str
+    format: str
+    urls: list[str] = field(default_factory=list)
+    upstream_checksum: str = ""
+    tool: str = ""
+    tool_versions: str = ""
+    # Filled by `dblock fetch`:
+    archive_sha256: str = ""
+    checksum_origin: str = ""  # "upstream-verified" or "first-download"
+    tree_digest: str = ""
+    files: int = 0
+    bytes: int = 0
+    fetched_at: str = ""
+
+    @property
+    def fetched(self) -> bool:
+        return bool(self.tree_digest)
+
+    @classmethod
+    def from_recipe(cls, recipe: Recipe, release: Release) -> "Entry":
+        return cls(
+            name=recipe.name,
+            release=release.version,
+            recipe_digest=recipe.digest,
+            filename=release.filename,
+            format=release.format,
+            urls=list(release.urls),
+            upstream_checksum=release.checksum,
+            tool=recipe.tool,
+            tool_versions=release.tool_versions,
+        )
+
+
+@dataclass
+class Lockfile:
+    path: Path
+    entries: dict[str, Entry] = field(default_factory=dict)
+
+    @classmethod
+    def load(cls, path: str | Path, must_exist: bool = True) -> "Lockfile":
+        path = Path(path)
+        if not path.exists():
+            if must_exist:
+                raise LockError(f"lockfile not found: {path} (create one with `dblock add NAME@RELEASE`)")
+            return cls(path)
+        try:
+            data = tomllib.loads(path.read_text(encoding="utf-8"))
+        except tomllib.TOMLDecodeError as exc:
+            raise LockError(f"{path}: invalid TOML: {exc}") from exc
+        if data.get("lock_version") != LOCK_VERSION:
+            raise LockError(f"{path}: unsupported lock_version {data.get('lock_version')!r}")
+        known = {f.name for f in fields(Entry)}
+        entries = {}
+        for name, raw in data.get("databases", {}).items():
+            unknown = set(raw) - known
+            if unknown:
+                raise LockError(f"{path}: unknown keys for '{name}': {sorted(unknown)}")
+            entries[name] = Entry(name=name, **{k: v for k, v in raw.items() if k != "name"})
+        return cls(path, entries)
+
+    def save(self) -> None:
+        lines = [HEADER, f"lock_version = {LOCK_VERSION}\n"]
+        for name in sorted(self.entries):
+            entry = asdict(self.entries[name])
+            entry.pop("name")
+            lines.append(f"\n[databases.{_key(name)}]\n")
+            for key, value in entry.items():
+                lines.append(f"{key} = {_value(value)}\n")
+        tmp = self.path.with_name(self.path.name + ".tmp")
+        tmp.write_text("".join(lines), encoding="utf-8")
+        os.replace(tmp, self.path)
+
+    def get(self, name: str) -> Entry:
+        try:
+            return self.entries[name]
+        except KeyError:
+            raise LockError(f"'{name}' is not in {self.path}") from None
+
+    def select(self, names: list[str] | None) -> list[Entry]:
+        if not names:
+            return [self.entries[n] for n in sorted(self.entries)]
+        return [self.get(n) for n in names]
+
+
+def _key(name: str) -> str:
+    if name.replace("-", "").replace("_", "").isalnum():
+        return name
+    return _string(name)
+
+
+def _string(text: str) -> str:
+    escaped = text.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
+    return f'"{escaped}"'
+
+
+def _value(value: object) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, str):
+        return _string(value)
+    if isinstance(value, list):
+        return "[" + ", ".join(_value(v) for v in value) + "]"
+    raise TypeError(f"cannot write {type(value).__name__} to TOML")

dblock/recipes/bakta-galaxy-test.toml

@@ -0,0 +1,17 @@
+name = "bakta-galaxy-test"
+description = "Tiny Bakta test database used by the Galaxy wrapper (390 KB); handy for trying dblock"
+homepage = "https://zenodo.org/records/22298301"
+notes = [
+  "Test data only, not for annotation.",
+  "The Zenodo record is labelled 6.0, but the archive contains a folder named V5.1_light_2024-01-19.",
+]
+
+[[releases]]
+version = "6.0"
+date = "2026-09-04"
+filename = "bakta-galaxy-test-db-light.tar.gz"
+format = "tar.gz"
+size = 390214
+checksum = "md5:29ac0addf985d7255c0928326a67178c"
+layout = ["*/version.json", "*/bakta.db"]
+urls = ["https://zenodo.org/api/records/22298301/files/db-light.tar.gz/content"]

dblock/recipes/bakta.toml

@@ -0,0 +1,34 @@
+name = "bakta"
+description = "Bakta annotation database (full or light)"
+homepage = "https://doi.org/10.5281/zenodo.4247253"
+tool = "bakta"
+version_command = ["bakta", "--version"]
+citation = "Schwengers O et al. (2021) Bakta: rapid and standardized annotation of bacterial genomes via alignment-free sequence identification. Microbial Genomics 7(11):000685"
+notes = [
+  "Minimum Bakta versions are from the database's db-versions.json on Zenodo.",
+  "Point Bakta at the unpacked folder: bakta --db $(dblock path bakta)/db  (or /db-light)",
+]
+
+[[releases]]
+version = "6.0-light"
+date = "2025-02-24"
+filename = "db-light.tar.xz"
+format = "tar.xz"
+size = 1344199496
+checksum = "md5:4a6e059ded39e9c5537ef4137d2f5648"
+tool_versions = ">=1.11"
+layout = ["db-light/version.json"]
+urls = ["https://zenodo.org/api/records/14916843/files/db-light.tar.xz/content"]
+note = "Zenodo record 14916843"
+
+[[releases]]
+version = "6.0"
+date = "2025-02-24"
+filename = "db.tar.xz"
+format = "tar.xz"
+size = 31921769288
+checksum = "md5:4c1115e40abfa2b464ae5dd988bdd88e"
+tool_versions = ">=1.11"
+layout = ["db/version.json"]
+urls = ["https://zenodo.org/api/records/14916843/files/db.tar.xz/content"]
+note = "Zenodo record 14916843"

dblock/recipes/checkm2.toml

@@ -0,0 +1,18 @@
+name = "checkm2"
+description = "CheckM2 reference database (UniRef100/KO DIAMOND database)"
+homepage = "https://github.com/chklovski/CheckM2"
+tool = "checkm2"
+version_command = ["checkm2", "--version"]
+citation = "Chklovski A et al. (2023) CheckM2: a rapid, scalable and accurate tool for assessing microbial genome quality using machine learning. Nature Methods 20:1203-1212"
+notes = ["Point CheckM2 at the .dmnd file: checkm2 predict --database_path $(dblock path checkm2)/CheckM2_database/uniref100.KO.1.dmnd"]
+
+[[releases]]
+version = "3"
+date = "2025-02-20"
+filename = "checkm2_database.tar.gz"
+format = "tar.gz"
+size = 1735095710
+checksum = "md5:07c10655620843b517d0df0c160d911f"
+layout = ["CheckM2_database/uniref100.KO.1.dmnd"]
+urls = ["https://zenodo.org/api/records/14897628/files/checkm2_database.tar.gz/content"]
+note = "Zenodo record 14897628"

dblock/recipes/gtdbtk.toml

@@ -0,0 +1,50 @@
+name = "gtdbtk"
+description = "GTDB-Tk reference data package (GTDB release)"
+homepage = "https://ecogenomics.github.io/GTDBTk/installing/index.html"
+tool = "gtdbtk"
+version_command = ["gtdbtk", "--version"]
+citation = "Chaumeil P-A et al. (2022) GTDB-Tk v2. Bioinformatics 38(23):5315-5316"
+notes = [
+  "GTDB does not publish checksums for these packages; dblock records the SHA-256 at first download.",
+  "Compatibility ranges are from the GTDB-Tk installation docs.",
+  "Point GTDB-Tk at the unpacked folder: export GTDBTK_DATA_PATH=$(dblock path gtdbtk)/releaseNNN",
+]
+
+[[releases]]
+version = "r232"
+date = "2026-04-13"
+filename = "gtdbtk_r232_data.tar.gz"
+format = "tar.gz"
+size = 60806405195
+tool_versions = ">=2.7.0"
+layout = ["release232/", "release232/skani/database/sketches.db"]
+urls = [
+  "https://data.gtdb.ecogenomic.org/releases/release232/232.0/auxillary_files/gtdbtk_package/full_package/gtdbtk_r232_data.tar.gz",
+  "https://data.ace.uq.edu.au/public/gtdb/data/releases/release232/232.0/auxillary_files/gtdbtk_package/full_package/gtdbtk_r232_data.tar.gz",
+]
+
+[[releases]]
+version = "r226"
+date = "2025-04-11"
+filename = "gtdbtk_r226_data.tar.gz"
+format = "tar.gz"
+size = 141442235198
+tool_versions = ">=2.4.1,<=2.6.1"
+layout = ["release226/", "release226/masks/"]
+urls = [
+  "https://data.gtdb.ecogenomic.org/releases/release226/226.0/auxillary_files/gtdbtk_package/full_package/gtdbtk_r226_data.tar.gz",
+  "https://data.ace.uq.edu.au/public/gtdb/data/releases/release226/226.0/auxillary_files/gtdbtk_package/full_package/gtdbtk_r226_data.tar.gz",
+]
+
+[[releases]]
+version = "r220"
+date = "2024-04-16"
+filename = "gtdbtk_r220_data.tar.gz"
+format = "tar.gz"
+size = 108491169765
+tool_versions = ">=2.4.0,<=2.6.1"
+layout = ["release220/", "release220/skani/"]
+urls = [
+  "https://data.gtdb.ecogenomic.org/releases/release220/220.0/auxillary_files/gtdbtk_package/full_package/gtdbtk_r220_data.tar.gz",
+  "https://data.ace.uq.edu.au/public/gtdb/data/releases/release220/220.0/auxillary_files/gtdbtk_package/full_package/gtdbtk_r220_data.tar.gz",
+]

dblock/recipes/kraken2.toml

@@ -0,0 +1,73 @@
+name = "kraken2"
+description = "Prebuilt Kraken 2 / Bracken indexes (Langmead lab index collection)"
+homepage = "https://benlangmead.github.io/aws-indexes/k2"
+tool = "kraken2"
+version_command = ["kraken2", "--version"]
+citation = "Wood DE, Lu J, Langmead B (2019) Improved metagenomic analysis with Kraken 2. Genome Biology 20:257"
+notes = [
+  "Archive MD5s come from the collection's published .md5 files (https://genome-idx.s3.amazonaws.com/kraken/<collection>_<date>/<collection>.md5), checked 2026-11-10.",
+  "standard-8gb-20240904 has no checksum: its .md5 file names k2_standard08gb_20240904.tar.gz, not the served k2_standard_08gb_20240904.tar.gz, so it is not used. dblock records the SHA-256 at first download.",
+  "From 2025-07 upstream names the 8 GB indexes k2_standard_08_GB_<date> instead of k2_standard_08gb_<date>.",
+  "No tool-version rule is stated upstream for these indexes.",
+]
+
+[[releases]]
+version = "standard-8gb-20261022"
+date = "2026-10-22"
+filename = "k2_standard_08_GB_20261022.tar.gz"
+format = "tar.gz"
+size = 5936595768
+checksum = "md5:7447d230ea402d0b523c14d217b5c20b"
+layout = ["hash.k2d", "opts.k2d", "taxo.k2d"]
+urls = ["https://genome-idx.s3.amazonaws.com/kraken/k2_standard_08_GB_20261022.tar.gz"]
+note = "Layout taken from the published .md5 file list; archive not downloaded for this check."
+
+[[releases]]
+version = "pluspf-20261022"
+date = "2026-10-22"
+filename = "k2_pluspf_20261022.tar.gz"
+format = "tar.gz"
+size = 96358495912
+checksum = "md5:7d28dbee49fe7db4809d6110d7c23689"
+layout = ["hash.k2d", "opts.k2d", "taxo.k2d"]
+urls = ["https://genome-idx.s3.amazonaws.com/kraken/k2_pluspf_20261022.tar.gz"]
+note = "Layout taken from the published .md5 file list; archive not downloaded for this check."
+
+[[releases]]
+version = "viral-20261022"
+date = "2026-10-22"
+filename = "k2_viral_20261022.tar.gz"
+format = "tar.gz"
+size = 589941425
+checksum = "md5:2c8c97686c45474d1d82cf940b4d9893"
+layout = ["hash.k2d", "opts.k2d", "taxo.k2d"]
+urls = ["https://genome-idx.s3.amazonaws.com/kraken/k2_viral_20261022.tar.gz"]
+
+[[releases]]
+version = "standard-8gb-20240904"
+date = "2024-09-04"
+filename = "k2_standard_08gb_20240904.tar.gz"
+format = "tar.gz"
+size = 5948839817
+layout = ["hash.k2d", "opts.k2d", "taxo.k2d"]
+urls = ["https://genome-idx.s3.amazonaws.com/kraken/k2_standard_08gb_20240904.tar.gz"]
+
+[[releases]]
+version = "pluspf-20240904"
+date = "2024-09-04"
+filename = "k2_pluspf_20240904.tar.gz"
+format = "tar.gz"
+size = 71110483952
+checksum = "md5:81012c99f5f1c32db3704dbba1b93428"
+layout = ["hash.k2d", "opts.k2d", "taxo.k2d"]
+urls = ["https://genome-idx.s3.amazonaws.com/kraken/k2_pluspf_20240904.tar.gz"]
+
+[[releases]]
+version = "viral-20240904"
+date = "2024-09-04"
+filename = "k2_viral_20240904.tar.gz"
+format = "tar.gz"
+size = 564123047
+checksum = "md5:37fc72e5ef54018641b9900421eaa678"
+layout = ["hash.k2d", "opts.k2d", "taxo.k2d"]
+urls = ["https://genome-idx.s3.amazonaws.com/kraken/k2_viral_20240904.tar.gz"]

dblock/recipes.py

@@ -0,0 +1,126 @@
+"""Load database recipes: one TOML file per database, listing its releases."""
+
+from __future__ import annotations
+
+import hashlib
+import tomllib
+from dataclasses import dataclass, field
+from importlib import resources
+from pathlib import Path
+
+FORMATS = {"tar.gz", "tar.xz", "tar.bz2", "tar", "zip", "gz", "file"}
+
+
+class RecipeError(Exception):
+    pass
+
+
+@dataclass(frozen=True)
+class Release:
+    version: str
+    urls: tuple[str, ...]
+    filename: str
+    format: str
+    checksum: str = ""  # upstream checksum as "md5:<hex>" or "sha256:<hex>"; empty if none published
+    size: int = 0
+    date: str = ""
+    tool_versions: str = ""  # e.g. ">=2.4.1,<=2.6.1"
+    layout: tuple[str, ...] = ()  # paths (or glob patterns) that must exist after unpacking
+    note: str = ""
+
+
+@dataclass(frozen=True)
+class Recipe:
+    name: str
+    description: str
+    homepage: str
+    releases: tuple[Release, ...]
+    digest: str  # sha256 of the recipe file, recorded in the lockfile
+    source: str
+    tool: str = ""
+    version_command: tuple[str, ...] = ()
+    version_regex: str = r"(\d+(?:\.\d+)+)"
+    citation: str = ""
+    notes: tuple[str, ...] = field(default_factory=tuple)
+
+    def release(self, version: str | None) -> Release:
+        if version is None:
+            return self.releases[0]
+        for rel in self.releases:
+            if rel.version == version:
+                return rel
+        known = ", ".join(r.version for r in self.releases)
+        raise RecipeError(f"{self.name} has no release '{version}'. Known releases: {known}")
+
+
+def _parse(text: str, source: str) -> Recipe:
+    try:
+        data = tomllib.loads(text)
+    except tomllib.TOMLDecodeError as exc:
+        raise RecipeError(f"{source}: invalid TOML: {exc}") from exc
+    for key in ("name", "description", "homepage", "releases"):
+        if key not in data:
+            raise RecipeError(f"{source}: missing '{key}'")
+    releases = []
+    for raw in data["releases"]:
+        for key in ("version", "urls", "filename", "format"):
+            if key not in raw:
+                raise RecipeError(f"{source}: release missing '{key}'")
+        if raw["format"] not in FORMATS:
+            raise RecipeError(f"{source}: unknown format '{raw['format']}' (use one of {sorted(FORMATS)})")
+        checksum = raw.get("checksum", "")
+        if checksum and checksum.split(":", 1)[0] not in ("md5", "sha256"):
+            raise RecipeError(f"{source}: checksum must start with 'md5:' or 'sha256:'")
+        releases.append(Release(
+            version=str(raw["version"]),
+            urls=tuple(raw["urls"]),
+            filename=raw["filename"],
+            format=raw["format"],
+            checksum=checksum,
+            size=int(raw.get("size", 0)),
+            date=str(raw.get("date", "")),
+            tool_versions=raw.get("tool_versions", ""),
+            layout=tuple(raw.get("layout", ())),
+            note=raw.get("note", ""),
+        ))
+    if not releases:
+        raise RecipeError(f"{source}: no releases")
+    return Recipe(
+        name=data["name"],
+        description=data["description"],
+        homepage=data["homepage"],
+        releases=tuple(releases),
+        digest="sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest(),
+        source=source,
+        tool=data.get("tool", ""),
+        version_command=tuple(data.get("version_command", ())),
+        version_regex=data.get("version_regex", r"(\d+(?:\.\d+)+)"),
+        citation=data.get("citation", ""),
+        notes=tuple(data.get("notes", ())),
+    )
+
+
+def load_recipes(extra_dir: str | Path | None = None) -> dict[str, Recipe]:
+    """Bundled recipes, overridden by recipes in extra_dir with the same name."""
+    recipes: dict[str, Recipe] = {}
+    bundled = resources.files("dblock") / "recipes"
+    for entry in sorted(bundled.iterdir(), key=lambda e: e.name):
+        if entry.name.endswith(".toml"):
+            recipe = _parse(entry.read_text(encoding="utf-8"), f"bundled:{entry.name}")
+            recipes[recipe.name] = recipe
+    if extra_dir:
+        directory = Path(extra_dir)
+        if not directory.is_dir():
+            raise RecipeError(f"recipe directory not found: {directory}")
+        for path in sorted(directory.glob("*.toml")):
+            recipe = _parse(path.read_text(encoding="utf-8"), str(path))
+            recipes[recipe.name] = recipe
+    return recipes
+
+
+def parse_spec(spec: str) -> tuple[str, str | None]:
+    """'gtdbtk@r232' -> ('gtdbtk', 'r232'); 'gtdbtk' -> ('gtdbtk', None)."""
+    name, _, version = spec.partition("@")
+    if not name:
+        raise RecipeError(f"invalid database spec '{spec}' (expected NAME or NAME@RELEASE)")
+    return name, (version or None)

dblock/store.py

@@ -0,0 +1,231 @@
+"""The local store: unpacked databases plus a per-file manifest.
+
+Layout:  <store>/<name>/<release>/data/...         unpacked database
+         <store>/<name>/<release>/manifest.json    per-file size and sha256
+         <store>/.downloads/<filename>[.part]      archives (removed after unpacking)
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import gzip
+import hashlib
+import json
+import os
+import shutil
+import tarfile
+import zipfile
+from dataclasses import dataclass, field
+from pathlib import Path, PurePosixPath
+
+from dblock.download import hash_file
+
+MANIFEST_VERSION = 1
+
+
+class StoreError(Exception):
+    pass
+
+
+def default_store() -> Path:
+    env = os.environ.get("DBLOCK_STORE")
+    return Path(env) if env else Path.home() / ".dblock" / "store"
+
+
+def install_dir(store: Path, name: str, release: str) -> Path:
+    return store / name / release
+
+
+def data_dir(store: Path, name: str, release: str) -> Path:
+    return install_dir(store, name, release) / "data"
+
+
+# ---------------------------------------------------------------- unpacking
+
+def unpack(archive: Path, fmt: str, target: Path) -> None:
+    """Unpack into target (which must not exist yet), rejecting unsafe paths."""
+    tmp = target.with_name(target.name + ".unpacking")
+    if tmp.exists():
+        shutil.rmtree(tmp)
+    tmp.mkdir(parents=True)
+    try:
+        if fmt.startswith("tar"):
+            _untar(archive, tmp)
+        elif fmt == "zip":
+            _unzip(archive, tmp)
+        elif fmt == "gz":
+            name = archive.name[:-3] if archive.name.endswith(".gz") else archive.name + ".out"
+            with gzip.open(archive, "rb") as src, (tmp / name).open("wb") as dst:
+                shutil.copyfileobj(src, dst, 1024 * 1024)
+        elif fmt == "file":
+            shutil.copy2(archive, tmp / archive.name)
+        else:
+            raise StoreError(f"unsupported format '{fmt}'")
+    except BaseException:
+        shutil.rmtree(tmp, ignore_errors=True)
+        raise
+    if target.exists():
+        shutil.rmtree(target)
+    tmp.rename(target)
+
+
+def _untar(archive: Path, dest: Path) -> None:
+    with tarfile.open(archive, "r:*") as tar:
+        if hasattr(tarfile, "data_filter"):
+            try:
+                tar.extractall(dest, filter="data")
+            except tarfile.FilterError as exc:
+                raise StoreError(f"refusing unsafe archive member: {exc}") from exc
+            return
+        for member in tar.getmembers():  # Python < 3.12 fallback
+            _check_member(member.name)
+            if member.issym() or member.islnk():
+                _check_member(str(PurePosixPath(member.name).parent / member.linkname))
+            if member.isdev():
+                raise StoreError(f"refusing device file in archive: {member.name}")
+        tar.extractall(dest)
+
+
+def _unzip(archive: Path, dest: Path) -> None:
+    with zipfile.ZipFile(archive) as zf:
+        for name in zf.namelist():
+            _check_member(name)
+        zf.extractall(dest)
+
+
+def _check_member(name: str) -> None:
+    path = PurePosixPath(name)
+    if path.is_absolute() or ".." in path.parts:
+        raise StoreError(f"refusing unsafe path in archive: {name}")
+
+
+# ---------------------------------------------------------------- manifests
+
+@dataclass
+class Manifest:
+    name: str
+    release: str
+    archive_sha256: str
+    files: dict[str, tuple[int, str]] = field(default_factory=dict)  # relpath -> (size, sha256)
+
+    @property
+    def digest(self) -> str:
+        return tree_digest(self.files)
+
+    @property
+    def total_bytes(self) -> int:
+        return sum(size for size, _ in self.files.values())
+
+    def save(self, path: Path) -> None:
+        payload = {
+            "manifest_version": MANIFEST_VERSION,
+            "name": self.name,
+            "release": self.release,
+            "archive_sha256": self.archive_sha256,
+            "tree_digest": self.digest,
+            "files": {k: {"size": s, "sha256": h} for k, (s, h) in sorted(self.files.items())},
+        }
+        path.write_text(json.dumps(payload, indent=1) + "\n", encoding="utf-8")
+
+    @classmethod
+    def load(cls, path: Path) -> "Manifest":
+        if not path.exists():
+            raise StoreError(f"no manifest at {path}; run `dblock fetch`")
+        raw = json.loads(path.read_text(encoding="utf-8"))
+        files = {k: (v["size"], v["sha256"]) for k, v in raw["files"].items()}
+        return cls(raw["name"], raw["release"], raw["archive_sha256"], files)
+
+
+def tree_digest(files: dict[str, tuple[int, str]]) -> str:
+    h = hashlib.sha256()
+    for rel in sorted(files):
+        size, digest = files[rel]
+        h.update(f"{rel}\t{size}\t{digest}\n".encode("utf-8"))
+    return "sha256:" + h.hexdigest()
+
+
+def walk(root: Path) -> dict[str, int]:
+    """Regular files under root as posix relpath -> size."""
+    out = {}
+    for dirpath, _, filenames in os.walk(root):
+        for fn in filenames:
+            p = Path(dirpath) / fn
+            out[p.relative_to(root).as_posix()] = p.stat().st_size
+    return out
+
+
+def build_manifest(name: str, release: str, archive_sha256: str, root: Path, quiet: bool = False) -> Manifest:
+    files = {}
+    for rel, size in sorted(walk(root).items()):
+        label = "" if quiet or size < 256 * 1024 * 1024 else f"  hashing {rel}"
+        files[rel] = (size, hash_file(root / rel, progress=label)["sha256"])
+    return Manifest(name, release, archive_sha256, files)
+
+
+# ---------------------------------------------------------------- checks
+
+@dataclass
+class VerifyResult:
+    missing: list[str] = field(default_factory=list)
+    extra: list[str] = field(default_factory=list)
+    size_changed: list[str] = field(default_factory=list)
+    content_changed: list[str] = field(default_factory=list)
+
+    @property
+    def ok(self) -> bool:
+        return not (self.missing or self.extra or self.size_changed or self.content_changed)
+
+    def summary(self, limit: int = 5) -> list[str]:
+        lines = []
+        for label, items in (("missing", self.missing), ("unexpected", self.extra),
+                             ("size changed", self.size_changed), ("content changed", self.content_changed)):
+            if items:
+                shown = ", ".join(items[:limit]) + (f" (+{len(items) - limit} more)" if len(items) > limit else "")
+                lines.append(f"{len(items)} {label}: {shown}")
+        return lines
+
+
+def verify(manifest: Manifest, root: Path, full: bool = False, quiet: bool = False) -> VerifyResult:
+    result = VerifyResult()
+    if not root.is_dir():
+        result.missing = sorted(manifest.files)
+        return result
+    present = walk(root)
+    result.missing = sorted(set(manifest.files) - set(present))
+    result.extra = sorted(set(present) - set(manifest.files))
+    for rel in sorted(set(manifest.files) & set(present)):
+        size, digest = manifest.files[rel]
+        if present[rel] != size:
+            result.size_changed.append(rel)
+        elif full:
+            label = "" if quiet or size < 256 * 1024 * 1024 else f"  hashing {rel}"
+            if hash_file(root / rel, progress=label)["sha256"] != digest:
+                result.content_changed.append(rel)
+    return result
+
+
+def check_layout(root: Path, patterns: tuple[str, ...]) -> list[str]:
+    """Return the layout patterns that match nothing under root."""
+    if not patterns:
+        return []
+    paths = list(walk(root))
+    dirs = {str(PurePosixPath(p).parent) for p in paths}
+    dirs |= {str(parent) for p in paths for parent in PurePosixPath(p).parents}
+    candidates = paths + [d + "/" for d in dirs if d != "."]
+    return [pat for pat in patterns if not any(fnmatch.fnmatchcase(c, pat) for c in candidates)]
+
+
+@dataclass
+class TreeDiff:
+    added: list[str]
+    removed: list[str]
+    changed: list[str]
+    unchanged: int
+
+
+def diff(a: Manifest, b: Manifest) -> TreeDiff:
+    added = sorted(set(b.files) - set(a.files))
+    removed = sorted(set(a.files) - set(b.files))
+    common = set(a.files) & set(b.files)
+    changed = sorted(p for p in common if a.files[p] != b.files[p])
+    return TreeDiff(added, removed, changed, len(common) - len(changed))