modelferry 0.1.0__py3-none-any.whl

modelferry/__init__.py

@@ -0,0 +1,3 @@
+"""modelferry: pack Hugging Face models into chunked, self-verifying bundles."""
+
+__version__ = "0.1.0"

modelferry/cli.py

@@ -0,0 +1,139 @@
+"""Typer command-line surface for modelferry (SPEC.md section 3).
+
+`pack` runs the connected-side pack orchestration. `verify`, `unpack`, and
+`inspect` are thin wrappers over offline.main, the exact code that ships inside
+every bundle, so the installed CLI and the bundled tool behave identically.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+import typer
+
+from . import __version__, offline
+from . import pack as pack_mod
+from .errors import PackError
+
+app = typer.Typer(
+    add_completion=False,
+    help="Pack Hugging Face models into chunked, self-verifying bundles for air-gapped transfer.",
+    no_args_is_help=True,
+)
+
+
+@app.command()
+def pack(
+    repo_id: Annotated[
+        str, typer.Argument(help="Hugging Face repo id, e.g. Qwen/Qwen2.5-7B-Instruct.")
+    ],
+    dest: Annotated[str, typer.Option("--dest", help="Directory to write the bundle into.")],
+    revision: Annotated[
+        str, typer.Option("--revision", help="Git revision to pin. Resolved to a commit SHA.")
+    ] = "main",
+    chunk_size: Annotated[
+        str, typer.Option("--chunk-size", help="Max part size (e.g. 3900M, 16G) or 'none'.")
+    ] = "3900M",
+    include: Annotated[
+        list[str] | None, typer.Option("--include", help="fnmatch include pattern (repeatable).")
+    ] = None,
+    exclude: Annotated[
+        list[str] | None, typer.Option("--exclude", help="fnmatch exclude pattern (repeatable).")
+    ] = None,
+    staging: Annotated[
+        str | None,
+        typer.Option(
+            "--staging", help="Local download directory. Defaults to ~/.cache/modelferry/."
+        ),
+    ] = None,
+) -> None:
+    """Pack a Hugging Face model repo into a bundle (connected side)."""
+    try:
+        bundle_dir = pack_mod.pack(
+            repo_id,
+            dest,
+            revision=revision,
+            chunk_size=chunk_size,
+            include=include,
+            exclude=exclude,
+            staging=staging,
+        )
+    except PackError as e:
+        typer.echo(f"error: {e}", err=True)
+        raise typer.Exit(e.exit_code) from None
+    except OSError as e:
+        typer.echo(
+            f"error: local filesystem error while packing: {e}. "
+            "Check permissions and free disk space, then re-run.",
+            err=True,
+        )
+        raise typer.Exit(4) from None
+    typer.echo(f"wrote bundle: {bundle_dir}")
+
+
+@app.command()
+def verify(
+    bundle_dir: Annotated[str, typer.Argument(help="Bundle directory to verify.")],
+    quiet: Annotated[
+        bool, typer.Option("--quiet", help="Print only the summary line and failures.")
+    ] = False,
+) -> None:
+    """Verify a bundle offline against its manifest."""
+    argv = ["verify", bundle_dir] + (["--quiet"] if quiet else [])
+    raise typer.Exit(offline.main(argv))
+
+
+@app.command()
+def unpack(
+    bundle_dir: Annotated[str, typer.Argument(help="Bundle directory to unpack.")],
+    dest_dir: Annotated[
+        str, typer.Argument(help="Destination directory for the reconstructed tree.")
+    ],
+    no_verify: Annotated[
+        bool, typer.Option("--no-verify", help="Skip the verify pass before unpacking.")
+    ] = False,
+    force: Annotated[
+        bool, typer.Option("--force", help="Allow unpacking into a non-empty destination.")
+    ] = False,
+) -> None:
+    """Verify and unpack a bundle offline into a loadable model tree."""
+    argv = ["unpack", bundle_dir, dest_dir]
+    if no_verify:
+        argv.append("--no-verify")
+    if force:
+        argv.append("--force")
+    raise typer.Exit(offline.main(argv))
+
+
+@app.command()
+def inspect(
+    bundle_dir: Annotated[str, typer.Argument(help="Bundle directory to inspect.")],
+) -> None:
+    """Print a bundle summary offline. No hashing."""
+    raise typer.Exit(offline.main(["inspect", bundle_dir]))
+
+
+def version_callback(value: bool) -> None:
+    if value:
+        typer.echo(__version__)
+        raise typer.Exit()
+
+
+@app.callback()
+def _root(
+    version: Annotated[
+        bool,
+        typer.Option(
+            "--version", callback=version_callback, is_eager=True, help="Show version and exit."
+        ),
+    ] = False,
+) -> None:
+    pass
+
+
+def main() -> None:
+    app()
+
+
+if __name__ == "__main__":
+    main()

modelferry/errors.py

@@ -0,0 +1,25 @@
+"""Pack-side error types carrying SPEC section 10 exit codes.
+
+offline.py has its own independent hierarchy (it imports nothing from this
+package); these are for the connected-side pack path and the CLI wrappers.
+"""
+
+from __future__ import annotations
+
+
+class PackError(Exception):
+    """Base for anticipated pack-side failures. Prints one line, no traceback."""
+
+    exit_code = 1
+
+
+class UsageError(PackError):
+    exit_code = 2  # bad arguments, pre-flight payload violations
+
+
+class SourceError(PackError):
+    exit_code = 3  # network failure, HF auth / 404 / gated-without-token
+
+
+class LocalFsError(PackError):
+    exit_code = 4  # permissions, disk full, destination already exists

modelferry/hf.py

@@ -0,0 +1,156 @@
+"""Hugging Face hub access: resolve a revision to a commit, download at that
+commit, and report source metadata (license, gated, endpoint) for the manifest.
+
+HF_TOKEN is read from the environment only and is never returned, logged, or
+written anywhere (§9). Hub failures map to SPEC exit code 3 (source error).
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import os
+
+from .errors import SourceError
+
+DEFAULT_ENDPOINT = "https://huggingface.co"
+
+
+def _select(candidates, include, exclude):
+    """Apply fnmatch include/exclude against repo-relative paths. Exclude wins."""
+    selected = []
+    for path in candidates:
+        if include and not any(fnmatch.fnmatch(path, pat) for pat in include):
+            continue
+        if exclude and any(fnmatch.fnmatch(path, pat) for pat in exclude):
+            continue
+        selected.append(path)
+    return selected
+
+
+def _extract_license(info):
+    card = getattr(info, "card_data", None)
+    if card is not None:
+        value = card.get("license") if hasattr(card, "get") else getattr(card, "license", None)
+        if value:
+            return str(value)
+    for tag in getattr(info, "tags", None) or []:
+        if isinstance(tag, str) and tag.startswith("license:"):
+            return tag.split(":", 1)[1]
+    return "UNKNOWN"
+
+
+def _staging_dir_for(staging, repo_id):
+    root = staging or os.path.join(os.path.expanduser("~"), ".cache", "modelferry")
+    return os.path.join(root, repo_id.replace("/", "__"))
+
+
+def resolve(repo_id, revision, staging, include, exclude):
+    """Resolve a repo to a commit and select files, without downloading anything.
+
+    Returns a dict: commit_sha, source (the §5 source block), files (list of
+    (repo_rel_path, size_bytes) for the selected files, sizes from hub metadata),
+    total_bytes, local_dir (the staging path the download will land in), endpoint.
+    Raises SourceError (exit 3). Separated from download() so pack can validate
+    --dest and free space against total_bytes before any bytes move.
+    """
+    from huggingface_hub import HfApi
+    from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError
+
+    token = os.environ.get("HF_TOKEN")
+    endpoint = os.environ.get("HF_ENDPOINT", DEFAULT_ENDPOINT)
+    local_dir = _staging_dir_for(staging, repo_id)
+    api = HfApi(endpoint=endpoint, token=token)
+
+    try:
+        info = api.repo_info(repo_id, revision=revision, repo_type="model", files_metadata=True)
+    except GatedRepoError as e:
+        raise SourceError(
+            f"repo {repo_id!r} is gated. Request access and set HF_TOKEN to an account "
+            f"that has it, then re-run pack ({e})."
+        ) from None
+    except RepositoryNotFoundError as e:
+        raise SourceError(
+            f"repo {repo_id!r} not found. Check the repo id, or set HF_TOKEN if it is "
+            f"private ({e})."
+        ) from None
+    except Exception as e:
+        raise SourceError(
+            f"could not read repo {repo_id!r} from {endpoint}: {e}. Check your network "
+            f"connection and the repo id."
+        ) from None
+
+    commit_sha = getattr(info, "sha", None)
+    if not commit_sha:
+        raise SourceError(
+            f"the hub returned no commit sha for {repo_id!r} at revision {revision!r}; "
+            f"try a different --revision."
+        )
+
+    size_by_path = {}
+    for s in getattr(info, "siblings", None) or []:
+        name = getattr(s, "rfilename", None)
+        if name is not None:
+            size_by_path[name] = getattr(s, "size", None) or 0
+    wanted = _select(list(size_by_path), include, exclude)
+    if not wanted:
+        raise SourceError(
+            f"no files in {repo_id!r} matched the --include/--exclude patterns; widen them."
+        )
+    files = [(rel, size_by_path.get(rel, 0)) for rel in wanted]
+
+    gated_raw = getattr(info, "gated", False)
+    source = {
+        "type": "huggingface",
+        "endpoint": endpoint,
+        "repo_id": repo_id,
+        "repo_type": "model",
+        "revision_requested": revision,
+        "commit_sha": commit_sha,
+        "license": _extract_license(info),
+        "gated": bool(gated_raw) and gated_raw is not False,
+    }
+    return {
+        "commit_sha": commit_sha,
+        "source": source,
+        "files": files,
+        "total_bytes": sum(size for _, size in files),
+        "local_dir": local_dir,
+        "endpoint": endpoint,
+    }
+
+
+def download(repo_id, commit_sha, local_dir, endpoint, include, exclude, wanted):
+    """Download the selected files at commit_sha into local_dir.
+
+    Returns (snapshot_dir, rel_files). Downloads in local_dir mode (§3): real
+    files under --staging, no symlinks, resumable. snapshot_dir also holds a
+    .cache/huggingface metadata dir that is never part of rel_files. Raises
+    SourceError (exit 3).
+    """
+    from huggingface_hub import snapshot_download
+
+    token = os.environ.get("HF_TOKEN")
+    print(f"downloading {len(wanted)} file(s) from {repo_id} at {commit_sha[:7]} ...")
+    try:
+        snapshot_dir = snapshot_download(
+            repo_id,
+            repo_type="model",
+            revision=commit_sha,
+            local_dir=local_dir,
+            allow_patterns=include or None,
+            ignore_patterns=exclude or None,
+            token=token,
+            endpoint=endpoint,
+        )
+    except Exception as e:
+        raise SourceError(
+            f"download of {repo_id!r} into {local_dir} failed: {e}. Check your network "
+            f"connection and free disk space, then re-run pack to resume."
+        ) from None
+
+    rel_files = [
+        rel for rel in wanted if os.path.isfile(os.path.join(snapshot_dir, *rel.split("/")))
+    ]
+    if not rel_files:
+        raise SourceError(f"nothing was downloaded for {repo_id!r} into {local_dir}.")
+    return snapshot_dir, rel_files

modelferry/manifest.py

@@ -0,0 +1,190 @@
+"""Manifest construction and MANIFEST.md rendering (pack side only).
+
+The reader in offline.py has its own independent manifest parser; the §11
+writer/reader round-trip test keeps the two honest. Serialization here is
+deterministic: json.dumps(indent=2, sort_keys=True) plus a trailing newline, so
+the same inputs always produce byte-identical manifest.json.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+
+MANIFEST_VERSION = 1
+
+
+def build_manifest(bundle_name, created_at, tool, source, chunk_size_bytes, files, verifier):
+    """Assemble the manifest dict. Pure function of its inputs (see §5).
+
+    files is a list of file entries already carrying path/bytes/sha256 (and parts
+    for chunked files). total_bytes and file_count are derived here.
+    """
+    ordered = sorted(files, key=lambda f: f["path"])
+    total_bytes = sum(f["bytes"] for f in ordered)
+    return {
+        "manifest_version": MANIFEST_VERSION,
+        "bundle_name": bundle_name,
+        "created_at": created_at,
+        "tool": tool,
+        "source": source,
+        "payload": {
+            "hash_algorithm": "sha256",
+            "chunk_size_bytes": int(chunk_size_bytes or 0),
+            "file_count": len(ordered),
+            "total_bytes": total_bytes,
+            "files": ordered,
+        },
+        "verifier": verifier,
+    }
+
+
+def serialize(manifest):
+    """Return the canonical manifest.json bytes."""
+    return (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode("utf-8")
+
+
+def sidecar_text(manifest_bytes):
+    """Return the manifest.sha256 sidecar line, sha256sum format (two spaces)."""
+    return f"{hashlib.sha256(manifest_bytes).hexdigest()}  manifest.json\n"
+
+
+def human_bytes(n):
+    """Human-readable size with the exact byte count, e.g. '3.8 GiB (4089446400 bytes)'."""
+    if not isinstance(n, int):
+        return str(n)
+    size = float(n)
+    for unit in ("B", "KiB", "MiB", "GiB", "TiB", "PiB"):
+        if size < 1024.0 or unit == "PiB":
+            if unit == "B":
+                return f"{n} B"
+            return f"{size:.1f} {unit} ({n} bytes)"
+        size /= 1024.0
+    return f"{n} B"
+
+
+def _md_cell(text):
+    """Escape a value for a Markdown table cell so a '|' in it can't add columns."""
+    return str(text).replace("\\", "\\\\").replace("|", "\\|")
+
+
+def _object_count(entry):
+    """Number of payload objects on media for one file: 1 whole, else one per part.
+
+    Matches how offline.py verify counts objects, so the MANIFEST.md Parts column
+    sums to what verify reports.
+    """
+    parts = entry.get("parts")
+    return len(parts) if parts else 1
+
+
+def render_manifest_md(manifest, manifest_sha256):
+    """Render the officer-facing MANIFEST.md per §6. Plain prose, no marketing."""
+    src = manifest["source"]
+    payload = manifest["payload"]
+    tool = manifest["tool"]
+    verifier = manifest.get("verifier") or {}
+    chunk_bytes = payload.get("chunk_size_bytes") or 0
+    chunk_display = "none" if chunk_bytes == 0 else human_bytes(chunk_bytes)
+
+    files = payload["files"]
+    file_count = payload.get("file_count")
+    if file_count is None:
+        file_count = len(files)
+    object_count = sum(_object_count(e) for e in files)
+
+    gated = "yes" if src.get("gated") else "no"
+    lines = [
+        f"# modelferry bundle: {manifest.get('bundle_name')}",
+        "",
+    ]
+    if src.get("license") == "UNKNOWN":
+        lines += [
+            "> WARNING: the license for this model could not be determined from repo",
+            "> metadata. Confirm the license terms before using or redistributing this",
+            "> model.",
+            "",
+        ]
+    lines += [
+        "This document is the approval record for this bundle. It gets used twice.",
+        "",
+        "Before transfer: review the details below and approve them, then keep a copy of",
+        "this file. The manifest checksum in your copy is what the receiving side checks",
+        "against.",
+        "",
+        "On arrival: run the commands in the Verify section. If the checksum inspect",
+        "prints differs from the one in your approved copy, or if verify reports anything",
+        "other than OK, this is not the bundle that was approved. Do not use it.",
+        "",
+        "## Source",
+        "",
+        f"- Repo: {src.get('repo_id')}",
+        f"- Commit: {src.get('commit_sha')}",
+        f"- Revision requested: {src.get('revision_requested')}",
+        f"- License: {src.get('license')}",
+        f"- Gated: {gated}",
+        f"- Endpoint: {src.get('endpoint')}",
+        f"- Created: {manifest.get('created_at')}",
+        f"- Tool: {tool.get('name')} {tool.get('version')}",
+        "",
+        "## Totals",
+        "",
+        f"- Files: {file_count}",
+        f"- Payload objects on media: {object_count}",
+        f"- Total size: {human_bytes(payload.get('total_bytes'))}",
+        f"- Chunk size: {chunk_display}",
+        "",
+    ]
+    if object_count > file_count:
+        lines += [
+            "Files larger than the chunk size are split into parts, so the media holds more",
+            "objects than the model has files. The Parts column below shows the split.",
+            "",
+        ]
+    lines += [
+        "## Manifest checksum",
+        "",
+        "The sha256 of manifest.json is:",
+        "",
+        f"    {manifest_sha256}",
+        "",
+        "This is the checksum your approved copy carries. The Verify section explains how",
+        "the receiving side checks the arrived bundle against it.",
+        "",
+        "## Verifier",
+        "",
+        f"The bundled verifier is {verifier.get('path')}. Its sha256 is:",
+        "",
+        f"    {verifier.get('sha256')}",
+        "",
+        'This hash is also recorded in manifest.json under "verifier". Compare it to the',
+        "canonical hash published with the modelferry release, or bring your own copy of",
+        "the verifier, to check the verifier out-of-band (see the trust model in the",
+        "README).",
+        "",
+        "## Verify",
+        "",
+        "Run these on the receiving side. They need Python 3.9 or newer, no network and",
+        "no packages.",
+        "",
+        "    cd <bundle directory>",
+        "    python3 tools/modelferry_offline.py inspect .",
+        "    python3 tools/modelferry_offline.py verify .",
+        "",
+        "inspect recomputes the manifest checksum from the file on disk and prints it as",
+        "manifest_sha256. Compare that to the checksum in the approved copy of this",
+        "document. verify then checks every object on the media against the manifest and",
+        'prints "verify OK" only if all of them match.',
+        "",
+        "## Files",
+        "",
+        "| Path | Size (bytes) | Parts | sha256 |",
+        "| --- | --- | --- | --- |",
+    ]
+    for entry in files:
+        lines.append(
+            f"| {_md_cell(entry['path'])} | {entry['bytes']} | "
+            f"{_object_count(entry)} | {entry['sha256']} |"
+        )
+    lines.append("")
+    return "\n".join(lines)