chutes 0.5.3rc2__py3-none-any.whl → 0.5.3rc4__py3-none-any.whl

chutes/_version.py

@@ -1 +1 @@
-version = "0.5.3.rc2"
+version = "0.5.3.rc4"

chutes/entrypoint/run.py

@@ -34,6 +34,7 @@ from fastapi import Request, Response, status, HTTPException
 from fastapi.responses import ORJSONResponse
 from starlette.middleware.base import BaseHTTPMiddleware
 from chutes.entrypoint.verify import GpuVerifier
+from chutes.util.hf import verify_cache, CacheVerificationError
 from prometheus_client import generate_latest, CONTENT_TYPE_LATEST
 from substrateinterface import Keypair, KeypairType
 from chutes.entrypoint._shared import (
@@ -1411,6 +1412,37 @@ def run_chute(
         chute.add_api_route("/_toca", envdump.handle_toca, methods=["POST"])
         chute.add_api_route("/_eslurp", envdump.handle_slurp, methods=["POST"])
 
+        async def _handle_hf_check(request: Request):
+            """
+            Verify HuggingFace cache integrity.
+            """
+            data = request.state.decrypted
+            repo_id = data.get("repo_id")
+            revision = data.get("revision")
+            full_hash_check = data.get("full_hash_check", False)
+
+            if not repo_id or not revision:
+                return {
+                    "error": True,
+                    "reason": "bad_request",
+                    "message": "repo_id and revision are required",
+                    "repo_id": repo_id,
+                    "revision": revision,
+                }
+
+            try:
+                result = await verify_cache(
+                    repo_id=repo_id,
+                    revision=revision,
+                    full_hash_check=full_hash_check,
+                )
+                result["error"] = False
+                return result
+            except CacheVerificationError as e:
+                return e.to_dict()
+
+        chute.add_api_route("/_hf_check", _handle_hf_check, methods=["POST"])
+
         logger.success("Added all chutes internal endpoints.")
 
         # Job shutdown/kill endpoint.

chutes/util/hf.py

@@ -6,8 +6,10 @@ import os
 import shutil
 import asyncio
 import aiohttp
+import hashlib
 from pathlib import Path
 from loguru import logger
+from concurrent.futures import ThreadPoolExecutor
 
 PROXY_URL = "https://api.chutes.ai/misc/hf_repo_info"
 
@@ -31,7 +33,38 @@ def purge_model_cache(repo_id: str, cache_dir: str = "/cache") -> bool:
 class CacheVerificationError(Exception):
     """Raised when cache verification fails."""
 
-    pass
+    def __init__(
+        self,
+        message: str,
+        reason: str = "verification_failed",
+        repo_id: str | None = None,
+        revision: str | None = None,
+        mismatches: list[str] | None = None,
+        missing: list[str] | None = None,
+        extra: list[str] | None = None,
+        errors: list[str] | None = None,
+    ):
+        super().__init__(message)
+        self.reason = reason
+        self.repo_id = repo_id
+        self.revision = revision
+        self.mismatches = mismatches or []
+        self.missing = missing or []
+        self.extra = extra or []
+        self.errors = errors or []
+
+    def to_dict(self) -> dict:
+        return {
+            "error": True,
+            "reason": self.reason,
+            "message": str(self),
+            "repo_id": self.repo_id,
+            "revision": self.revision,
+            "mismatches": self.mismatches,
+            "missing": self.missing,
+            "extra": self.extra,
+            "errors": self.errors,
+        }
 
 
 def _get_hf_token() -> str | None:
@@ -40,23 +73,59 @@ def _get_hf_token() -> str | None:
 
 def _get_symlink_hash(file_path: Path) -> str | None:
     """
-    Extract SHA256 from symlink target (blob filename).
+    Extract hash from symlink target (blob filename).
     """
     if file_path.is_symlink():
         target = os.readlink(file_path)
         blob_name = Path(target).name
-        if len(blob_name) == 64:
+        # 64 chars = SHA256 (LFS), 40 chars = SHA1 (git blob)
+        if len(blob_name) in (40, 64):
             return blob_name
     return None
 
 
+def git_blob_hash(filepath: Path, chunk_size: int = 8 * 1024 * 1024) -> str:
+    """
+    Compute git blob SHA-1 for a file using streaming (memory efficient).
+    Git blob format: "blob {size}\0{content}"
+    """
+    size = filepath.stat().st_size
+    sha1 = hashlib.sha1()
+    sha1.update(f"blob {size}\0".encode())
+    with open(filepath, "rb") as f:
+        while chunk := f.read(chunk_size):
+            sha1.update(chunk)
+    return sha1.hexdigest()
+
+
+def compute_sha256(filepath: Path, chunk_size: int = 8 * 1024 * 1024) -> str:
+    """
+    Compute SHA256 hash of a file using streaming (memory efficient).
+    """
+    sha256 = hashlib.sha256()
+    with open(filepath, "rb") as f:
+        while chunk := f.read(chunk_size):
+            sha256.update(chunk)
+    return sha256.hexdigest()
+
+
 async def verify_cache(
     repo_id: str,
     revision: str,
     cache_dir: str = "/cache",
+    full_hash_check: bool = False,
+    max_workers: int = 4,
 ) -> dict:
     """
     Verify cached HuggingFace model files match checksums on the Hub.
+
+    Args:
+        repo_id: HuggingFace repository ID
+        revision: Git revision (commit hash, branch, or tag)
+        cache_dir: Cache directory path
+        full_hash_check: If True, compute full file hashes instead of just
+                        checking symlink names. Slower but more thorough.
+        max_workers: Number of parallel workers for hash computation
     """
     cache_dir = Path(cache_dir)
     params = {
@@ -73,6 +142,22 @@ async def verify_cache(
             async with session.get(
                 PROXY_URL, params=params, timeout=aiohttp.ClientTimeout(total=30)
             ) as resp:
+                if resp.status == 404:
+                    text = await resp.text()
+                    raise CacheVerificationError(
+                        f"Repository or revision not found: {repo_id}@{revision} - {text}",
+                        reason="not_found",
+                        repo_id=repo_id,
+                        revision=revision,
+                    )
+                if resp.status in (401, 403):
+                    text = await resp.text()
+                    raise CacheVerificationError(
+                        f"Access denied to {repo_id}: {text}",
+                        reason="access_denied",
+                        repo_id=repo_id,
+                        revision=revision,
+                    )
                 if resp.status != 200:
                     text = await resp.text()
                     logger.warning(
@@ -80,34 +165,53 @@ async def verify_cache(
                     )
                     return {"verified": 0, "skipped": 0, "total": 0, "skipped_api_error": True}
                 repo_info = await resp.json()
+    except CacheVerificationError:
+        raise
     except (aiohttp.ClientError, asyncio.TimeoutError) as e:
         logger.warning(f"Cache verification skipped - proxy request failed: {e}")
         return {"verified": 0, "skipped": 0, "total": 0, "skipped_api_error": True}
 
-    # Build remote files dict: {path: (sha256, size)}
+    # Build remote files dict: {path: (hash, size, is_lfs)}
     remote_files = {}
     for item in repo_info["files"]:
-        if item["path"].startswith("_"):
-            continue
-        if item.get("is_lfs"):
-            remote_files[item["path"]] = (item.get("sha256"), item.get("size"))
+        is_lfs = item.get("is_lfs", False)
+        if is_lfs:
+            remote_files[item["path"]] = (item.get("sha256"), item.get("size"), True)
         else:
-            remote_files[item["path"]] = (item.get("blob_id"), item.get("size"))
+            remote_files[item["path"]] = (item.get("blob_id"), item.get("size"), False)
+
+    # Directories.
+    directories = repo_info.get("directories")
+    if directories is not None:
+        for dir_path in directories:
+            if dir_path not in remote_files:
+                remote_files[dir_path] = (None, None, False)
+    else:
+        for item in repo_info["files"]:
+            parts = item["path"].split("/")
+            for i in range(1, len(parts)):
+                dir_path = "/".join(parts[:i])
+                if dir_path not in remote_files:
+                    remote_files[dir_path] = (None, None, False)
 
     # Find local cache
     repo_folder_name = f"models--{repo_id.replace('/', '--')}"
     snapshot_dir = cache_dir / "hub" / repo_folder_name / "snapshots" / revision
 
     if not snapshot_dir.exists():
-        raise CacheVerificationError(f"Cache directory not found: {snapshot_dir}")
-
-    # Get local files (ignore _ prefixed)
+        raise CacheVerificationError(
+            f"Cache directory not found: {snapshot_dir}",
+            reason="cache_not_found",
+            repo_id=repo_id,
+            revision=revision,
+        )
+
+    # Get local files and directories
     local_files = {}
     for path in snapshot_dir.rglob("*"):
-        if path.is_file() or path.is_symlink():
+        if path.is_file() or path.is_symlink() or path.is_dir():
             rel_path = str(path.relative_to(snapshot_dir))
-            if not any(part.startswith("_") for part in Path(rel_path).parts):
-                local_files[rel_path] = path
+            local_files[rel_path] = path
 
     verified = 0
     skipped = 0
@@ -115,21 +219,23 @@ async def verify_cache(
     missing = []
     errors = []
 
-    for remote_path, (remote_hash, remote_size) in remote_files.items():
+    # Files needing hash computation: (remote_path, resolved_path, expected_hash, hash_type)
+    files_to_hash = []
+
+    for remote_path, (remote_hash, remote_size, is_lfs) in remote_files.items():
         local_path = local_files.get(remote_path)
 
         if not local_path or (not local_path.exists() and not local_path.is_symlink()):
             missing.append(remote_path)
             continue
 
-        # Skip non-LFS files (sha1 blob id = 40 chars)
-        if remote_hash is None or len(remote_hash) == 40:
+        if remote_hash is None:
             skipped += 1
             continue
 
         resolved_path = local_path.resolve()
 
-        # Check size
+        # Check size first (quick sanity check)
         if remote_size is not None:
             try:
                 actual_size = resolved_path.stat().st_size
@@ -142,21 +248,76 @@ async def verify_cache(
                 errors.append(f"{remote_path}: cannot stat: {e}")
                 continue
 
-        # Check symlink hash
-        symlink_hash = _get_symlink_hash(local_path)
-        if symlink_hash:
-            if symlink_hash != remote_hash:
-                mismatches.append(f"{remote_path}: hash {symlink_hash} != expected {remote_hash}")
-                continue
+        if is_lfs:
+            if full_hash_check:
+                # Queue for full SHA256 computation
+                files_to_hash.append((remote_path, resolved_path, remote_hash, "sha256"))
+            else:
+                # Fast check via symlink name
+                symlink_hash = _get_symlink_hash(local_path)
+                if symlink_hash:
+                    if symlink_hash != remote_hash:
+                        mismatches.append(
+                            f"{remote_path}: hash {symlink_hash} != expected {remote_hash}"
+                        )
+                    else:
+                        verified += 1
+                else:
+                    errors.append(f"{remote_path}: LFS file not a symlink, cannot fast-verify")
         else:
-            # Not a symlink - can't fast-verify, treat as error
-            errors.append(f"{remote_path}: not a symlink, cannot fast-verify")
-            continue
-
-        verified += 1
-
-    # Check for extra local files
-    extra = [p for p in local_files if p not in remote_files]
+            # Non-LFS file: verify via git blob hash
+            if full_hash_check:
+                # Queue for git blob hash computation
+                files_to_hash.append((remote_path, resolved_path, remote_hash, "git_blob"))
+            else:
+                # Fast check via symlink name (if available)
+                symlink_hash = _get_symlink_hash(local_path)
+                if symlink_hash:
+                    if symlink_hash != remote_hash:
+                        mismatches.append(
+                            f"{remote_path}: hash {symlink_hash} != expected {remote_hash}"
+                        )
+                    else:
+                        verified += 1
+                else:
+                    # Not a symlink, must compute hash
+                    files_to_hash.append((remote_path, resolved_path, remote_hash, "git_blob"))
+
+    # Compute hashes in parallel using thread pool
+    if files_to_hash:
+        loop = asyncio.get_running_loop()
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+
+            def compute_hash_sync(item):
+                remote_path, resolved_path, expected_hash, hash_type = item
+                try:
+                    if hash_type == "sha256":
+                        computed = compute_sha256(resolved_path)
+                    else:  # git_blob
+                        computed = git_blob_hash(resolved_path)
+                    return (remote_path, computed, expected_hash, None)
+                except Exception as e:
+                    return (remote_path, None, expected_hash, str(e))
+
+            futures = [
+                loop.run_in_executor(executor, compute_hash_sync, item) for item in files_to_hash
+            ]
+            results = await asyncio.gather(*futures)
+
+            for remote_path, computed, expected, error in results:
+                if error:
+                    errors.append(f"{remote_path}: hash computation failed: {error}")
+                elif computed != expected:
+                    mismatches.append(f"{remote_path}: hash {computed} != expected {expected}")
+                else:
+                    verified += 1
+
+    # Check for extra local files (ignore _ prefixed paths not in remote)
+    extra = [
+        p
+        for p in local_files
+        if p not in remote_files and not any(part.startswith("_") for part in Path(p).parts)
+    ]
 
     # Build error message if needed
     if mismatches or missing or extra or errors:
@@ -169,7 +330,16 @@ async def verify_cache(
             msg_parts.append(f"Extra ({len(extra)}): " + ", ".join(extra))
         if errors:
             msg_parts.append(f"Errors ({len(errors)}): " + "; ".join(errors))
-        raise CacheVerificationError("\n".join(msg_parts))
+        raise CacheVerificationError(
+            "\n".join(msg_parts),
+            reason="integrity_mismatch",
+            repo_id=repo_id,
+            revision=revision,
+            mismatches=mismatches,
+            missing=missing,
+            extra=extra,
+            errors=errors,
+        )
 
     logger.success(f"Successfully verified HF cache for {repo_id=} {revision=}")
 
@@ -194,17 +364,39 @@ if __name__ == "__main__":
         "--revision", required=True, help="Git revision (commit hash, branch, or tag)"
     )
     parser.add_argument(
-        "--cache-dir", default=HF_HUB_CACHE, help="Cache directory (default: HF_HUB_CACHE)"
+        "--cache-dir",
+        default=str(Path(HF_HUB_CACHE).parent),
+        help="Cache directory (default: ~/.cache/huggingface)",
+    )
+    parser.add_argument(
+        "--full-hash-check",
+        action="store_true",
+        help="Compute full file hashes (slower but verifies actual content integrity)",
+    )
+    parser.add_argument(
+        "--max-workers",
+        type=int,
+        default=4,
+        help="Number of parallel workers for hash computation (default: 4)",
     )
     args = parser.parse_args()
 
     try:
-        result = asyncio.run(verify_cache(args.repo_id, args.revision, args.cache_dir))
+        result = asyncio.run(
+            verify_cache(
+                args.repo_id,
+                args.revision,
+                args.cache_dir,
+                full_hash_check=args.full_hash_check,
+                max_workers=args.max_workers,
+            )
+        )
         if result["skipped_api_error"]:
             print("⚠️  Verification skipped (API unavailable)")
         else:
             print(
-                f"✅ Verified {result['verified']}/{result['total']} files (skipped {result['skipped']} non-LFS)"
+                f"✅ Verified {result['verified']}/{result['total']} files "
+                f"(skipped {result['skipped']} without hash)"
             )
     except CacheVerificationError as e:
         print(f"❌ {e}")

{chutes-0.5.3rc2.dist-info → chutes-0.5.3rc4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chutes
-Version: 0.5.3rc2
+Version: 0.5.3rc4
 Summary: Chutes development kit and CLI.
 Home-page: https://github.com/rayonlabs/chutes
 Author: Jon Durbin
@@ -34,7 +34,7 @@ Requires-Dist: aiofiles>=23
 Requires-Dist: semver
 Requires-Dist: huggingface-hub
 Requires-Dist: setproctitle
-Requires-Dist: cllmv==0.1.0
+Requires-Dist: cllmv==0.1.1
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: flake8; extra == "dev"

{chutes-0.5.3rc2.dist-info → chutes-0.5.3rc4.dist-info}/RECORD

@@ -1,5 +1,5 @@
 chutes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-chutes/_version.py,sha256=vswF5HS-xiZ3bpgPznEf2gV9Qt5mo98B8axZBHS8gPc,22
+chutes/_version.py,sha256=cKnghV_3APd96pp_jy-68hcCFvmouXrmvLbwmMnTTB8,22
 chutes/cfsv,sha256=vkR_q3qB0hkD8Knv8rqqrb4A29y1muHW2-eNYZiuI-g,892932
 chutes/cfsv_v2,sha256=fa8-WFqaNeD-BXcCayUjCvR0NNizcc6Ewd6vNmFjPc0,893296
 chutes/cfsv_v3,sha256=aIe3rnM3-g416Ry7eMyDXDOJkLZPxW_EjKJDwh7BMGc,1198496
@@ -37,7 +37,7 @@ chutes/entrypoint/logger.py,sha256=MGfKmxni_4daAwFWb7JcSBDXTpmBJE8guACz2uYY6fE,9
 chutes/entrypoint/login.py,sha256=9d_MzS3ZQ884d4NAc8JGwvyNR6SaUazrpseOTAIhWNU,6076
 chutes/entrypoint/register.py,sha256=jHwZphEZCLpF4BRXT7iaieBZOvmpByCz25UQd804Kp0,9080
 chutes/entrypoint/report.py,sha256=-AeUFF8DFoaL14C8lAfVqd97aDH7fLDCzxZo-COYINg,1949
-chutes/entrypoint/run.py,sha256=Z6YWjQ470xtfQAJByqqQL1rvL-0D4byWe4TRzZTjUJo,59458
+chutes/entrypoint/run.py,sha256=yf1_3VHW0Xka8kHB5-dqVY0XoltarniFHfdjYLsRDx8,60611
 chutes/entrypoint/secret.py,sha256=_mtBXm_YL-nDg9XWmg2S4gGU_OdNBub_zb6jpLV7458,1581
 chutes/entrypoint/share.py,sha256=_1yIdIMwK5khxNZ8WApTRXx1htMi_dg1_VV7j1IVUjs,1532
 chutes/entrypoint/ssh.py,sha256=ryBRL_-bREyYpV_cZAhhZBFyIgvdWsNIIRwnM_sX41g,2285
@@ -70,12 +70,12 @@ chutes/pyarmor_runtime_006563/py313/pyarmor_runtime.so,sha256=1Fm2wCKz5770HEKbkv
 chutes/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chutes/util/auth.py,sha256=V9BShLNym6bPmp44d0FUg79e8N9oe446UqwoYqo5sPk,2238
 chutes/util/context.py,sha256=_dHVEM6S6CYwgaxrA0yZCplSDA2HuiyNOmedSsyiDWA,355
-chutes/util/hf.py,sha256=2XKSqvy0j0QROZ5NagXf3JHyb1uk-NEL34E44ZhjF14,7083
+chutes/util/hf.py,sha256=C76T-k3uYTnCJ1I3jgJf62Bkv66ltZYLN-fnIaP-zXE,14260
 chutes/util/schema.py,sha256=b0NX-hWhc0FKnWD6FObBmQxINrJDFNocKsbGJn79D-8,6699
 chutes/util/user.py,sha256=WBx6vyw0P32wJ2GFYMjO5oQJPA3jn_XsyRglNJx0PhY,423
-chutes-0.5.3rc2.dist-info/LICENSE,sha256=9qFhoY0O1XdKOczAAc7vcveZzk32-a0Wq2_diH_hAD8,1067
-chutes-0.5.3rc2.dist-info/METADATA,sha256=1EgvV8qzzNy7RgWyZSsMh7KbzQOc5daaFsafz3f7LNA,18789
-chutes-0.5.3rc2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-chutes-0.5.3rc2.dist-info/entry_points.txt,sha256=93PU_dKxEGA1hCGaabvRXO9WpyFgon6eqT4wIN91YAU,42
-chutes-0.5.3rc2.dist-info/top_level.txt,sha256=oRxU-Kvd5BhaNbBQtqZCp9uzY0FrMBocoL0Q6kokxzA,7
-chutes-0.5.3rc2.dist-info/RECORD,,
+chutes-0.5.3rc4.dist-info/LICENSE,sha256=9qFhoY0O1XdKOczAAc7vcveZzk32-a0Wq2_diH_hAD8,1067
+chutes-0.5.3rc4.dist-info/METADATA,sha256=gIMUE4xU0F12_XS5W_Ra6nqsQxBK_RlhoAAodyZhkX8,18789
+chutes-0.5.3rc4.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+chutes-0.5.3rc4.dist-info/entry_points.txt,sha256=93PU_dKxEGA1hCGaabvRXO9WpyFgon6eqT4wIN91YAU,42
+chutes-0.5.3rc4.dist-info/top_level.txt,sha256=oRxU-Kvd5BhaNbBQtqZCp9uzY0FrMBocoL0Q6kokxzA,7
+chutes-0.5.3rc4.dist-info/RECORD,,