modal-benchmark 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2 @@
1
+ """Shared capabilities for Modal applications."""
2
+ from .hf import create_downloader
@@ -0,0 +1,13 @@
1
+ """Reusable Hugging Face model download capability.
2
+
3
+ Importing this module does not configure Modal resources or load any model data.
4
+ """
5
+ from .config import DownloadConfig
6
+ from .manifest import discover_manifest
7
+ from .download import download_selected, validate_manifest, verify_persisted
8
+
9
+
10
+ def create_downloader(volume_name="modal-model-downloads", **options):
11
+ """Create an independent downloader, optionally registered on a caller's App."""
12
+ from .modal import create_downloader as create
13
+ return create(volume_name, **options)
@@ -0,0 +1,57 @@
1
+ """Immutable settings for one downloader; importing this module has no side effects."""
2
+ from dataclasses import dataclass
3
+ import math
4
+ from pathlib import PurePosixPath
5
+
6
+
7
+ @dataclass(frozen=True)
8
+ class DownloadConfig:
9
+ volume_name: str = "modal-model-downloads"
10
+ volume_version: int | None = 2
11
+ region: str = "ap-southeast"
12
+ cpu: float = 4
13
+ memory_mib: int = 8192
14
+ xet_concurrency: int = 8
15
+ secret_name: str = "huggingface-secret"
16
+ mount_path: str = "/mnt/models"
17
+ max_workers: int = 3
18
+ verify_workers: int = 5
19
+ verification_mode: str = "xet"
20
+ discover_workers: int = 3
21
+ commit_batch_files: int = 3
22
+ commit_batch_bytes: int = 16 * 1024**3
23
+
24
+ def __post_init__(self):
25
+ if self.volume_version is not None and (
26
+ isinstance(self.volume_version, bool)
27
+ or not isinstance(self.volume_version, int)
28
+ or self.volume_version not in (1, 2)):
29
+ raise ValueError("volume_version must be 1, 2, or None")
30
+ for name in ("volume_name", "region", "secret_name"):
31
+ value = getattr(self, name)
32
+ if not isinstance(value, str) or not value.strip() or value != value.strip():
33
+ raise ValueError(f"{name} must be a non-empty name without surrounding whitespace")
34
+ if (isinstance(self.cpu, bool) or not isinstance(self.cpu, (int, float))
35
+ or not math.isfinite(self.cpu) or self.cpu <= 0):
36
+ raise ValueError("cpu must be positive and finite")
37
+ for name in ("memory_mib", "xet_concurrency", "max_workers", "verify_workers",
38
+ "discover_workers", "commit_batch_files", "commit_batch_bytes"):
39
+ value = getattr(self, name)
40
+ if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
41
+ raise ValueError(f"{name} must be a positive integer")
42
+ if self.max_workers > 5:
43
+ raise ValueError("max_workers must be 1..5")
44
+ for name in ("verify_workers", "discover_workers"):
45
+ if getattr(self, name) > 10:
46
+ raise ValueError(f"{name} must be 1..10")
47
+ if self.commit_batch_files > 16:
48
+ raise ValueError("commit_batch_files must be 1..16")
49
+ if self.verification_mode not in ("xet", "fast", "strict"):
50
+ raise ValueError("verification_mode must be xet, fast or strict")
51
+ if not isinstance(self.mount_path, str):
52
+ raise ValueError("mount_path must be a canonical absolute POSIX directory")
53
+ path = PurePosixPath(self.mount_path)
54
+ if (not path.is_absolute() or str(path) != self.mount_path or str(path) == "/"
55
+ or ".." in path.parts or "\\" in self.mount_path
56
+ or ":" in self.mount_path or self.mount_path.startswith("//")):
57
+ raise ValueError("mount_path must be a canonical absolute POSIX directory")
@@ -0,0 +1,285 @@
1
+ """Model-independent Hugging Face file download and verification."""
2
+ import hashlib
3
+ import json
4
+ import os
5
+ import re
6
+ import shutil
7
+ import threading
8
+ import time
9
+ from concurrent.futures import ThreadPoolExecutor, as_completed
10
+ from pathlib import Path
11
+
12
+ from .evidence import publish_receipt, receipt_path, receipt_valid
13
+ from .transport import XetDownload, fetch_xet_file
14
+
15
+
16
+ def validate_manifest(manifest, *, expected_models=None, allow_zero=True):
17
+ """固定 revision、路径、大小和文件总量,防止错误或恶意文件清单。"""
18
+ if (manifest.get("version") != 1 or not manifest.get("models") or
19
+ (expected_models is not None and len(manifest["models"]) != expected_models)):
20
+ raise ValueError("模型清单版本或仓库数量不合法")
21
+ for model in manifest["models"]:
22
+ if (not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", model["repo_id"])
23
+ or any(part in (".", "..") for part in model["repo_id"].split("/"))):
24
+ raise ValueError("非法 repo_id")
25
+ if not re.fullmatch(r"[a-f0-9]{40}", model["revision"]):
26
+ raise ValueError("必须固定 40 字符 commit SHA")
27
+ if not model["files"]:
28
+ raise ValueError("权重清单为空")
29
+ for rel, entry in model["files"].items():
30
+ path = Path(rel)
31
+ if (path.is_absolute() or "\\" in rel or ":" in rel or
32
+ any(x in ("", ".", "..") for x in rel.split("/"))):
33
+ raise ValueError(f"非法文件路径:{rel}")
34
+ size = entry.get("size")
35
+ if not isinstance(size, int) or isinstance(size, bool) or size < (0 if allow_zero else 1):
36
+ raise ValueError(f"非法文件大小:{rel}")
37
+ checksum = entry.get("sha256")
38
+ if checksum is not None and not re.fullmatch(r"[a-f0-9]{64}", checksum):
39
+ raise ValueError(f"非法文件 sha256:{rel}")
40
+ return manifest
41
+
42
+
43
+ def selected_entries(manifest, smoke=False):
44
+ """Smoke mode selects one file <= 1 MiB per repository, independent of names."""
45
+ validate_manifest(manifest)
46
+ if not smoke:
47
+ return manifest["models"]
48
+ selected = []
49
+ for model in manifest["models"]:
50
+ small = [(entry["size"], name) for name, entry in model["files"].items()
51
+ if entry["size"] <= 1024 * 1024]
52
+ if not small:
53
+ raise ValueError(f"No smoke-test file <= 1 MiB in {model['repo_id']}; pass a smaller manifest")
54
+ _, name = min(small)
55
+ selected.append({**model, "files": {name: model["files"][name]}})
56
+ return selected
57
+
58
+
59
+ def repo_root(base, model):
60
+ return Path(base) / model["repo_id"] / model["revision"]
61
+
62
+
63
+ def shard_staging_dir(folder, rel):
64
+ identity = hashlib.sha256(rel.encode("utf-8")).hexdigest()[:20]
65
+ return folder / "_shards" / identity
66
+
67
+
68
+ def checksum_file(path):
69
+ digest = hashlib.sha256()
70
+ with Path(path).open("rb") as reader:
71
+ for chunk in iter(lambda: reader.read(4 * 1024 * 1024), b""):
72
+ digest.update(chunk)
73
+ return digest.hexdigest()
74
+
75
+
76
+ def inspect_files(folder, model, verify_hash=True, require_receipt=False, *, allow_xet=False):
77
+ """完整且大小一致才认为该分片完成;损坏直接报错,不静默覆盖。"""
78
+ results = {"complete": [], "missing": [], "invalid": []}
79
+ for rel, entry in model["files"].items():
80
+ file = folder / "model" / rel
81
+ if not file.exists():
82
+ results["missing"].append(rel)
83
+ elif not file.is_file() or file.stat().st_size != entry["size"]:
84
+ results["invalid"].append(rel)
85
+ elif require_receipt and not receipt_valid(folder, model, rel, file.stat(), allow_xet=allow_xet):
86
+ # Legacy files can be revalidated in place instead of redownloaded.
87
+ results["missing"].append(rel)
88
+ elif verify_hash and entry.get("sha256") and checksum_file(file) != entry["sha256"]:
89
+ results["invalid"].append(rel)
90
+ else:
91
+ results["complete"].append(rel)
92
+ return results
93
+
94
+
95
+ def is_transient(exc):
96
+ """Retry known transport failures and temporary HTTP responses only."""
97
+ status = getattr(getattr(exc, "response", None), "status_code", None)
98
+ if status is not None:
99
+ return status in (408, 429, 500, 502, 503, 504)
100
+ return isinstance(exc, (TimeoutError, ConnectionError)) or any(
101
+ cls.__name__ in ("TimeoutException", "TransportError")
102
+ for cls in type(exc).__mro__
103
+ )
104
+
105
+
106
+ def retry_operation(operation, phase, metrics, *, attempts=3, sleep=time.sleep):
107
+ for attempt in range(attempts):
108
+ started = time.perf_counter()
109
+ try:
110
+ return operation()
111
+ except Exception as exc:
112
+ if attempt + 1 == attempts or not is_transient(exc):
113
+ raise
114
+ finally:
115
+ metrics[phase + "_seconds"] += time.perf_counter() - started
116
+ metrics["retries"] += 1
117
+ sleep(2 ** attempt)
118
+
119
+
120
+ def download_selected(manifest, smoke=False, *, base=None, fetch_file=None, commit=None,
121
+ workers=8, work=None, write_marker=True,
122
+ metrics=None, attempts=3, sleep=time.sleep, defer_commit=False,
123
+ attest=False, trust_xet=False):
124
+ """按传入清单恢复并校验文件,由调用方提供持久化提交回调。"""
125
+ if defer_commit and write_marker:
126
+ raise ValueError("Deferred commits cannot publish a completion marker")
127
+ if not isinstance(attempts, int) or isinstance(attempts, bool) or not 1 <= attempts <= 5:
128
+ raise ValueError("attempts must be 1..5")
129
+ validate_manifest(manifest)
130
+ if work is not None:
131
+ validate_manifest({"version": 1, "models": work})
132
+ if not smoke and write_marker and work != manifest["models"]:
133
+ raise ValueError("Partial work cannot publish a full completion marker")
134
+ attest = attest or trust_xet
135
+ if fetch_file is None and trust_xet:
136
+ fetch_file = fetch_xet_file
137
+ if fetch_file is None:
138
+ from huggingface_hub import hf_hub_download
139
+ fetch_file = hf_hub_download
140
+ if commit is None:
141
+ raise ValueError("An explicit commit callback is required")
142
+
143
+ work = selected_entries(manifest, smoke) if work is None else work
144
+ root = Path(base) if base is not None else Path("/mnt/models")
145
+ results = []
146
+ for model in work:
147
+ folder = repo_root(root, model)
148
+ before = inspect_files(folder, model, verify_hash=not attest, require_receipt=attest,
149
+ allow_xet=trust_xet)
150
+ if before["invalid"]:
151
+ raise RuntimeError(f"持久权重与预期大小不符:{before['invalid'][:3]}")
152
+ downloaded = 0
153
+ reused = len(before["complete"])
154
+ commit_lock = threading.Lock()
155
+ # The caller configures HF cache paths before importing huggingface_hub.
156
+
157
+ def download_one(rel):
158
+ row = {"repo_id": model["repo_id"], "revision": model["revision"],
159
+ "filename": rel, "bytes": model["files"][rel]["size"],
160
+ "download_seconds": 0.0, "verify_seconds": 0.0,
161
+ "commit_seconds": 0.0, "retries": 0, "status": "FAILED"}
162
+ started = time.perf_counter()
163
+ try:
164
+ save_one(rel, row)
165
+ row["status"] = "STAGED" if defer_commit else "COMPLETE"
166
+ except Exception as exc:
167
+ row["error"] = str(exc)[:800]
168
+ raise
169
+ finally:
170
+ row["seconds"] = time.perf_counter() - started
171
+ if metrics is not None:
172
+ metrics.append(row)
173
+
174
+ def save_one(rel, row):
175
+ target = folder / "model" / rel
176
+ entry = model["files"][rel]
177
+ if attest and target.is_file() and target.stat().st_size == entry["size"]:
178
+ # Old committed files have no receipt: hash once, without fetching
179
+ # again. A digest mismatch is a hard error; never attest bad bytes.
180
+ started = time.perf_counter()
181
+ try:
182
+ if entry.get("sha256") and checksum_file(target) != entry["sha256"]:
183
+ raise RuntimeError(f"Existing file SHA256 mismatch: {rel}")
184
+ finally:
185
+ row["verify_seconds"] += time.perf_counter() - started
186
+ publish_receipt(folder, model, rel, target.stat())
187
+ row["verification_method"] = "sha256_match" if entry.get("sha256") else "size_only"
188
+ row["reused_existing"] = True
189
+ if not defer_commit:
190
+ retry_operation(commit, "commit", row, attempts=attempts, sleep=sleep)
191
+ return
192
+ stage = shard_staging_dir(folder, rel)
193
+ stage.mkdir(parents=True, exist_ok=True)
194
+ # 分片暂存目录独立;模型级 Xet cache 允许同一容器多个下载线程共用。
195
+ fetched = retry_operation(lambda: fetch_file(
196
+ repo_id=model["repo_id"], revision=model["revision"],
197
+ filename=rel, local_dir=str(stage),
198
+ ), "download", row, attempts=attempts, sleep=sleep)
199
+ transport = None
200
+ if isinstance(fetched, XetDownload):
201
+ local = Path(fetched.path)
202
+ if trust_xet:
203
+ transport = fetched.evidence()
204
+ if entry.get("sha256") and fetched.source_sha256 != entry["sha256"]:
205
+ raise RuntimeError(f"Xet source metadata SHA256 mismatch: {rel}")
206
+ else:
207
+ local = Path(fetched)
208
+ started = time.perf_counter()
209
+ try:
210
+ if not local.is_file() or local.stat().st_size != entry["size"]:
211
+ raise RuntimeError(f"权重文件不完整:{rel}")
212
+ if transport is None and entry.get("sha256") and checksum_file(local) != entry["sha256"]:
213
+ # Never reuse invalid local_dir metadata on a later retry.
214
+ shutil.rmtree(stage, ignore_errors=True)
215
+ raise RuntimeError(f"权重 SHA256 与 HF 不符:{rel}")
216
+ finally:
217
+ row["verify_seconds"] += time.perf_counter() - started
218
+ row["verification_method"] = ("trusted_xet_download" if transport is not None else
219
+ "sha256_match" if entry.get("sha256") else "size_only")
220
+ # 提交互斥,以免并发 commit 导致不确定的持久化时序。
221
+ with commit_lock:
222
+ target.parent.mkdir(parents=True, exist_ok=True)
223
+ if attest:
224
+ # Invalidate any stale receipt before replacing its target.
225
+ receipt_path(folder, rel).unlink(missing_ok=True)
226
+ os.replace(local, target)
227
+ if attest:
228
+ publish_receipt(folder, model, rel, target.stat(), transport=transport)
229
+ # Remove temporary HF metadata before committing the Volume.
230
+ # Keep incomplete staging on download/verification failures.
231
+ shutil.rmtree(stage, ignore_errors=True)
232
+ if not defer_commit:
233
+ retry_operation(commit, "commit", row, attempts=attempts, sleep=sleep)
234
+ print(("SHARD_STAGED " if defer_commit else "SHARD_OK ") + json.dumps({
235
+ "repo": model["repo_id"], "filename": rel,
236
+ "bytes": entry["size"], "reused": reused,
237
+ }, ensure_ascii=False), flush=True)
238
+
239
+ failures = []
240
+ with ThreadPoolExecutor(max_workers=workers) as pool:
241
+ futures = {pool.submit(download_one, rel): rel for rel in before["missing"]}
242
+ for future in as_completed(futures):
243
+ try:
244
+ future.result()
245
+ downloaded += 1
246
+ except Exception as exc:
247
+ failures.append(f"{futures[future]}: {exc}")
248
+ if failures:
249
+ raise RuntimeError(f"部分分片下载失败(已完成分片可恢复):{failures[:3]}")
250
+ # Contents were checked by this writer or downloaded via trusted Xet.
251
+ after = inspect_files(folder, model, verify_hash=False)
252
+ if after["missing"] or after["invalid"]:
253
+ raise RuntimeError(f"完成校验失败:{model['repo_id']}")
254
+ results.append({
255
+ "repo_id": model["repo_id"], "revision": model["revision"],
256
+ "files": len(model["files"]), "reused": reused,
257
+ "downloaded": downloaded,
258
+ "bytes": sum(x["size"] for x in model["files"].values()),
259
+ "path": str(folder / "model"),
260
+ })
261
+
262
+ if not smoke and write_marker:
263
+ # Publish a completion marker only after all selected repositories pass.
264
+ report = {"status": "COMPLETE", "models": results}
265
+ completion = repo_root(root, manifest["models"][0]) / "all-components-complete.json"
266
+ completion.parent.mkdir(parents=True, exist_ok=True)
267
+ completion.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
268
+ commit()
269
+ return results
270
+
271
+
272
+ def verify_persisted(manifest, smoke=False, *, base=None):
273
+ """独立读取端核查已提交的每个目标文件,防止仅下载端可见。"""
274
+ root = Path(base) if base is not None else Path("/mnt/models")
275
+ results = []
276
+ for model in selected_entries(manifest, smoke):
277
+ check = inspect_files(repo_root(root, model), model)
278
+ if check["missing"] or check["invalid"]:
279
+ raise RuntimeError(f"Volume 持久化校验失败:{model['repo_id']}: {check}")
280
+ results.append({"repo": model["repo_id"], "verified_files": len(check["complete"])})
281
+ if not smoke:
282
+ marker = repo_root(root, manifest["models"][0]) / "all-components-complete.json"
283
+ if not marker.is_file() or json.loads(marker.read_text(encoding="utf-8")).get("status") != "COMPLETE":
284
+ raise RuntimeError("缺失最终完成标记")
285
+ return results
@@ -0,0 +1,74 @@
1
+ """Small, untrusted-by-itself attestations for files validated before Volume commit.
2
+
3
+ A receipt only states what our writer checked. Its independent visibility after
4
+ Volume.reload proves a commit was published, not post-commit byte integrity.
5
+ """
6
+ import hashlib
7
+ import json
8
+ import os
9
+ import re
10
+ from pathlib import Path
11
+ import uuid
12
+
13
+ from .transport import HUB_VERSION, XET_VERSION
14
+
15
+
16
+ def receipt_path(folder, filename):
17
+ name = hashlib.sha256(filename.encode("utf-8")).hexdigest() + ".json"
18
+ return Path(folder) / "_verification" / name
19
+
20
+
21
+ def receipt_record(model, filename, stat, *, transport=None):
22
+ entry = model["files"][filename]
23
+ expected_sha = entry.get("sha256")
24
+ record = {
25
+ "version": 1,
26
+ "repo_id": model["repo_id"],
27
+ "revision": model["revision"],
28
+ "filename": filename,
29
+ "size": entry["size"],
30
+ "source_sha256": expected_sha,
31
+ "method": "sha256_match" if expected_sha is not None else "size_only",
32
+ "mtime_ns": stat.st_mtime_ns,
33
+ }
34
+ if transport is not None:
35
+ if (set(transport) != {"method", "xet_hash", "huggingface_hub", "hf_xet"}
36
+ or transport["method"] != "trusted_xet_download"
37
+ or not isinstance(transport["xet_hash"], str)
38
+ or not re.fullmatch(r"[a-f0-9]{64}", transport["xet_hash"])
39
+ or transport["huggingface_hub"] != HUB_VERSION
40
+ or transport["hf_xet"] != XET_VERSION):
41
+ raise ValueError("Invalid Xet transport evidence")
42
+ record.update(transport)
43
+ return record
44
+
45
+
46
+ def receipt_valid(folder, model, filename, stat, *, allow_xet=False):
47
+ """Metadata check only; never reads model-file bytes or claims storage integrity."""
48
+ path = receipt_path(folder, filename)
49
+ try:
50
+ if not path.is_file() or path.is_symlink():
51
+ return False
52
+ saved = json.loads(path.read_text(encoding="utf-8"))
53
+ transport = None
54
+ if allow_xet and isinstance(saved, dict) and saved.get("method") == "trusted_xet_download":
55
+ transport = {key: saved.get(key) for key in
56
+ ("method", "xet_hash", "huggingface_hub", "hf_xet")}
57
+ return saved == receipt_record(model, filename, stat, transport=transport)
58
+ except (OSError, ValueError, TypeError):
59
+ return False
60
+
61
+
62
+ def publish_receipt(folder, model, filename, stat, *, transport=None):
63
+ """Atomically stage evidence beside the validated file; caller must commit both."""
64
+ path = receipt_path(folder, filename)
65
+ path.parent.mkdir(parents=True, exist_ok=True)
66
+ temporary = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp"
67
+ try:
68
+ temporary.write_text(
69
+ json.dumps(receipt_record(model, filename, stat, transport=transport), sort_keys=True),
70
+ encoding="utf-8",
71
+ )
72
+ os.replace(temporary, path)
73
+ finally:
74
+ temporary.unlink(missing_ok=True)
@@ -0,0 +1,29 @@
1
+ """Discover immutable HF model snapshots without model presets."""
2
+ from fnmatch import fnmatchcase
3
+ from .download import validate_manifest
4
+
5
+
6
+ def discover_manifest(repo_id, revision="main", include="", exclude="", *, api=None):
7
+ """Fetch metadata only; all file metadata is read at one immutable commit."""
8
+ if api is None:
9
+ from huggingface_hub import HfApi
10
+ api = HfApi()
11
+ commit = api.repo_info(repo_id=repo_id, revision=revision, repo_type="model").sha
12
+ info = api.repo_info(repo_id=repo_id, revision=commit, repo_type="model", files_metadata=True)
13
+ allows = [s.strip() for s in include.split(",") if s.strip()]
14
+ denies = [s.strip() for s in exclude.split(",") if s.strip()]
15
+ files = {}
16
+ for item in info.siblings:
17
+ name = item.rfilename
18
+ if allows and not any(fnmatchcase(name, p) for p in allows):
19
+ continue
20
+ if any(fnmatchcase(name, p) for p in denies):
21
+ continue
22
+ lfs = item.lfs
23
+ sha = lfs.get("sha256") if isinstance(lfs, dict) else getattr(lfs, "sha256", None)
24
+ files[name] = {"size": item.size, "sha256": sha}
25
+ if not files:
26
+ raise ValueError("No files matched the requested repository/filter")
27
+ manifest = {"version": 1, "models": [{"repo_id": repo_id, "revision": commit, "files": files}]}
28
+ validate_manifest(manifest, expected_models=None, allow_zero=True)
29
+ return manifest