model-mirror-cli 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- model_mirror/__init__.py +7 -0
- model_mirror/audit.py +214 -0
- model_mirror/checksums.py +371 -0
- model_mirror/cli.py +2436 -0
- model_mirror/config.py +300 -0
- model_mirror/hub.py +901 -0
- model_mirror/lock.py +119 -0
- model_mirror/mirror.py +201 -0
- model_mirror/progress.py +260 -0
- model_mirror/removal.py +164 -0
- model_mirror/repair.py +399 -0
- model_mirror/state.py +150 -0
- model_mirror/torrent.py +453 -0
- model_mirror/torrent_coverage.py +400 -0
- model_mirror/torrent_hashes.py +168 -0
- model_mirror/torrent_import.py +620 -0
- model_mirror/torrent_publication.py +582 -0
- model_mirror/torrent_seed.py +228 -0
- model_mirror/verify.py +153 -0
- model_mirror_cli-0.2.0.dist-info/METADATA +550 -0
- model_mirror_cli-0.2.0.dist-info/RECORD +24 -0
- model_mirror_cli-0.2.0.dist-info/WHEEL +4 -0
- model_mirror_cli-0.2.0.dist-info/entry_points.txt +2 -0
- model_mirror_cli-0.2.0.dist-info/licenses/LICENSE +21 -0
model_mirror/cli.py
ADDED
|
@@ -0,0 +1,2436 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import time
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from datetime import datetime, timezone
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import yaml
|
|
15
|
+
|
|
16
|
+
from . import __version__
|
|
17
|
+
from .audit import audit_model
|
|
18
|
+
from .checksums import MANIFEST, iter_payload_files, load_manifest, write_checksums, verify_checksums
|
|
19
|
+
from .config import (
|
|
20
|
+
Config,
|
|
21
|
+
REPO_TYPE_DIRS,
|
|
22
|
+
archive_path,
|
|
23
|
+
archive_runtime_cache_path,
|
|
24
|
+
archive_runtime_tmp_path,
|
|
25
|
+
load_config,
|
|
26
|
+
parse_bool,
|
|
27
|
+
parse_nonnegative_int,
|
|
28
|
+
parse_positive_int,
|
|
29
|
+
save_config,
|
|
30
|
+
)
|
|
31
|
+
from .hub import HuggingFaceHub, get_snapshot, read_snapshot_plan
|
|
32
|
+
from .lock import ModelBusyError, ModelLock, lock_label, read_active_lock
|
|
33
|
+
from .mirror import mirror
|
|
34
|
+
from .progress import ProgressEntry, ProgressSnapshot, progress_snapshot
|
|
35
|
+
from .repair import repair
|
|
36
|
+
from .removal import (
|
|
37
|
+
REMOVALS_DIR,
|
|
38
|
+
RemovalRecord,
|
|
39
|
+
complete_removal,
|
|
40
|
+
prune_empty_removal_parents,
|
|
41
|
+
read_removal_record,
|
|
42
|
+
removal_path,
|
|
43
|
+
removal_record_path,
|
|
44
|
+
stage_removal,
|
|
45
|
+
write_removal_record,
|
|
46
|
+
)
|
|
47
|
+
from .state import (
|
|
48
|
+
VerificationState,
|
|
49
|
+
read_verification_state,
|
|
50
|
+
state_from_results,
|
|
51
|
+
verification_state_path,
|
|
52
|
+
write_verification_state,
|
|
53
|
+
)
|
|
54
|
+
from .verify import RemoteVerifyResult, merge_checksum_result, verify_remote
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
TORRENT_EXPERIMENTAL_NOTICE = (
|
|
58
|
+
"EXPERIMENTAL: torrent interfaces and metadata formats may change before stabilization."
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
CONFIG_OPTIONS = [
|
|
63
|
+
(
|
|
64
|
+
"directory",
|
|
65
|
+
"MODEL_MIRROR_DIRECTORY",
|
|
66
|
+
"Archive root. Repos are stored below models/, datasets/, or spaces/ under this directory.",
|
|
67
|
+
),
|
|
68
|
+
("repo_type", "MODEL_MIRROR_REPO_TYPE", "Default Hugging Face repo type: model, dataset, or space."),
|
|
69
|
+
("revision", "MODEL_MIRROR_REVISION", "Default revision to mirror or verify, usually main."),
|
|
70
|
+
("checksum", None, "Whether mirror/repair writes local hash manifest records."),
|
|
71
|
+
(
|
|
72
|
+
"checksum_workers",
|
|
73
|
+
"MODEL_MIRROR_CHECKSUM_WORKERS",
|
|
74
|
+
"Number of files to hash concurrently. Use 1 for HDD-friendly sequential reads.",
|
|
75
|
+
),
|
|
76
|
+
(
|
|
77
|
+
"download_workers",
|
|
78
|
+
"MODEL_MIRROR_DOWNLOAD_WORKERS",
|
|
79
|
+
"Number of files to download concurrently. Default 1 is HDD-friendly; increase for SSD/NVMe.",
|
|
80
|
+
),
|
|
81
|
+
(
|
|
82
|
+
"stall_timeout_seconds",
|
|
83
|
+
"MODEL_MIRROR_STALL_TIMEOUT",
|
|
84
|
+
"Abort and retry a file when no local byte progress occurs for this many seconds. "
|
|
85
|
+
"Default 600; set 0 to disable.",
|
|
86
|
+
),
|
|
87
|
+
(
|
|
88
|
+
"stall_retries",
|
|
89
|
+
"MODEL_MIRROR_STALL_RETRIES",
|
|
90
|
+
"Number of stall-triggered resume retries per file before failing the mirror.",
|
|
91
|
+
),
|
|
92
|
+
("verify_after_mirror", None, "Whether mirror runs verification after download unless --no-verify is passed."),
|
|
93
|
+
(
|
|
94
|
+
"hf_xet_high_performance",
|
|
95
|
+
"MODEL_MIRROR_HF_XET_HIGH_PERFORMANCE",
|
|
96
|
+
"Sets HF_XET_HIGH_PERFORMANCE=1. Off by default; use only on high-bandwidth machines "
|
|
97
|
+
"with fast disks and ample memory, typically 64 GB RAM or more.",
|
|
98
|
+
),
|
|
99
|
+
(
|
|
100
|
+
"hf_xet_reconstruct_write_sequentially",
|
|
101
|
+
"MODEL_MIRROR_HF_XET_RECONSTRUCT_WRITE_SEQUENTIALLY",
|
|
102
|
+
"Uses HDD-friendly Xet reconstruction writes by disabling vectored reconstruction writes where supported.",
|
|
103
|
+
),
|
|
104
|
+
(
|
|
105
|
+
"hf_xet_num_concurrent_range_gets",
|
|
106
|
+
"MODEL_MIRROR_HF_XET_NUM_CONCURRENT_RANGE_GETS",
|
|
107
|
+
"Limits Xet internal download concurrency. Default 1 is HDD-friendly; increase for SSD/NVMe.",
|
|
108
|
+
),
|
|
109
|
+
(
|
|
110
|
+
"token_path",
|
|
111
|
+
"MODEL_MIRROR_TOKEN_PATH",
|
|
112
|
+
"Path to a Hugging Face token file. If unset, model-mirror checks HF_TOKEN_PATH, "
|
|
113
|
+
"HF_HOME/token, ~/.cache/huggingface/token, and ~/.huggingface/token. Token contents are never printed.",
|
|
114
|
+
),
|
|
115
|
+
(
|
|
116
|
+
"cache_dir",
|
|
117
|
+
None,
|
|
118
|
+
"Overrides the Hugging Face cache root; defaults to DIRECTORY/.model-mirror/cache.",
|
|
119
|
+
),
|
|
120
|
+
(
|
|
121
|
+
"tmp_dir",
|
|
122
|
+
None,
|
|
123
|
+
"Overrides temporary file directory; defaults to DIRECTORY/.model-mirror/tmp.",
|
|
124
|
+
),
|
|
125
|
+
]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@dataclass(slots=True)
|
|
129
|
+
class CacheUsage:
|
|
130
|
+
archive_cache: int
|
|
131
|
+
archive_tmp: int
|
|
132
|
+
mirror_cache: int
|
|
133
|
+
|
|
134
|
+
@property
|
|
135
|
+
def total(self) -> int:
|
|
136
|
+
return self.archive_cache + self.archive_tmp + self.mirror_cache
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclass(slots=True)
|
|
140
|
+
class MirrorStatusEntry:
|
|
141
|
+
root: Path
|
|
142
|
+
repo_id: str
|
|
143
|
+
repo_type: str
|
|
144
|
+
state: VerificationState | None
|
|
145
|
+
active_lock: dict | None
|
|
146
|
+
payload_files: int | None
|
|
147
|
+
payload_size: int | None
|
|
148
|
+
payload_source: str
|
|
149
|
+
expected_files: int | None
|
|
150
|
+
snapshot_commit: str
|
|
151
|
+
snapshot_requested_revision: str
|
|
152
|
+
progress: ProgressSnapshot
|
|
153
|
+
torrent_status: str | None
|
|
154
|
+
upstream_observation: UpstreamObservation | None = None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@dataclass(frozen=True, slots=True)
|
|
158
|
+
class UpstreamObservation:
|
|
159
|
+
status: str
|
|
160
|
+
commit: str | None
|
|
161
|
+
observed_at_utc: str
|
|
162
|
+
error: str | None = None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
166
|
+
parser = argparse.ArgumentParser(
|
|
167
|
+
prog="model-mirror",
|
|
168
|
+
description="Mirror Hugging Face repositories into local bulk storage and verify their integrity.",
|
|
169
|
+
epilog="Run 'model-mirror COMMAND --help' for command-specific options.",
|
|
170
|
+
)
|
|
171
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
172
|
+
parser.add_argument("--config", help="path to config file; defaults to ~/.model-mirror.yaml")
|
|
173
|
+
subparsers = parser.add_subparsers(dest="command")
|
|
174
|
+
command_parsers: dict[str, argparse.ArgumentParser] = {}
|
|
175
|
+
|
|
176
|
+
def add_command_parser(name: str, **kwargs) -> argparse.ArgumentParser:
|
|
177
|
+
command_parser = subparsers.add_parser(name, **kwargs)
|
|
178
|
+
command_parsers[name] = command_parser
|
|
179
|
+
return command_parser
|
|
180
|
+
|
|
181
|
+
mirror_parser = add_command_parser(
|
|
182
|
+
"mirror",
|
|
183
|
+
help="mirror a Hugging Face repo",
|
|
184
|
+
description="Download a repo at a resolved Hub commit and verify it unless --no-verify is used.",
|
|
185
|
+
epilog="Exit status: 0 when complete or downloaded cleanly; 1 when final verification is not clean.",
|
|
186
|
+
)
|
|
187
|
+
mirror_parser.add_argument("model", metavar="repo", help="Hugging Face repo id, e.g. org/model")
|
|
188
|
+
mirror_parser.add_argument("--repo-type", choices=["model", "dataset", "space"], help="repo kind to mirror")
|
|
189
|
+
add_revision_options(mirror_parser)
|
|
190
|
+
mirror_parser.add_argument("--force", action="store_true", help="download even if the local copy looks complete")
|
|
191
|
+
mirror_parser.add_argument("--no-verify", action="store_true", help="skip verification after download")
|
|
192
|
+
mirror_parser.add_argument(
|
|
193
|
+
"--stall-timeout",
|
|
194
|
+
type=parse_nonnegative_int_arg,
|
|
195
|
+
metavar="SECONDS",
|
|
196
|
+
help="override stall timeout for this mirror; default 600 seconds, 0 disables timeout",
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
remove_parser = add_command_parser(
|
|
200
|
+
"remove",
|
|
201
|
+
help="permanently remove a mirror",
|
|
202
|
+
description=(
|
|
203
|
+
"Show the selected mirror's identity, commit, verification age, file count, and size, "
|
|
204
|
+
"then require confirmation before permanent removal. Interrupted removals are resumable."
|
|
205
|
+
),
|
|
206
|
+
epilog=(
|
|
207
|
+
"Exit status: 0 when removed or cancelled; 1 when the mirror is missing, busy, unsafe, "
|
|
208
|
+
"has an active torrent publication, or removal is interrupted."
|
|
209
|
+
),
|
|
210
|
+
)
|
|
211
|
+
remove_parser.add_argument("model", metavar="repo", help="repo id to remove, e.g. org/model")
|
|
212
|
+
remove_parser.add_argument(
|
|
213
|
+
"--repo-type", choices=["model", "dataset", "space"], default="model", help="repo kind to remove"
|
|
214
|
+
)
|
|
215
|
+
remove_parser.add_argument(
|
|
216
|
+
"-y",
|
|
217
|
+
"--yes",
|
|
218
|
+
action="store_true",
|
|
219
|
+
help="skip the interactive repository-name confirmation",
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
verify_parser = add_command_parser(
|
|
223
|
+
"verify",
|
|
224
|
+
help="verify mirrored archives",
|
|
225
|
+
description="Check local files against Hub metadata, local checksums, and model metadata.",
|
|
226
|
+
epilog=(
|
|
227
|
+
"Exit status: 0 when verification is clean; 1 when files are missing, corrupt, extra, busy, "
|
|
228
|
+
"the upstream repository is unavailable, or cached verification data is missing/stale; "
|
|
229
|
+
"2 for command-line errors."
|
|
230
|
+
),
|
|
231
|
+
)
|
|
232
|
+
verify_parser.add_argument("model", metavar="repo", nargs="?", help="repo id to verify unless --all is used")
|
|
233
|
+
verify_parser.add_argument("--cached", action="store_true", help="verify Hub hashes from cached .manifest rows")
|
|
234
|
+
verify_parser.add_argument("--all", action="store_true", help="verify every mirrored model")
|
|
235
|
+
verify_parser.add_argument(
|
|
236
|
+
"--repo-type", choices=["model", "dataset", "space"], default="model", help="repo kind to verify"
|
|
237
|
+
)
|
|
238
|
+
add_revision_options(verify_parser)
|
|
239
|
+
verify_parser.add_argument("--strict", action="store_true", help="fail on extra local files")
|
|
240
|
+
verify_parser.add_argument("--max-age", help="with --all, skip clean archives verified within this age, e.g. 7d")
|
|
241
|
+
verify_parser.add_argument("--offline", action="store_true", help="verify only against local checksum state")
|
|
242
|
+
|
|
243
|
+
repair_parser = add_command_parser(
|
|
244
|
+
"repair",
|
|
245
|
+
help="repair a mirrored archive",
|
|
246
|
+
description=(
|
|
247
|
+
"Redownload files listed in existing .verification repair paths, "
|
|
248
|
+
"reconcile a stale pinned snapshot plan, then run a final verification. Run verify first."
|
|
249
|
+
),
|
|
250
|
+
epilog=(
|
|
251
|
+
"Exit status: 0 when complete, repaired, or updated cleanly; 1 when verification state is missing, "
|
|
252
|
+
"repair is incomplete, cached verification data is incomplete, or a model is busy; "
|
|
253
|
+
"2 for command-line errors."
|
|
254
|
+
),
|
|
255
|
+
)
|
|
256
|
+
repair_parser.add_argument("model", metavar="repo", nargs="?", help="repo id to repair unless --all is used")
|
|
257
|
+
repair_parser.add_argument("--all", action="store_true", help="repair every mirrored model with verification state")
|
|
258
|
+
repair_parser.add_argument(
|
|
259
|
+
"--update",
|
|
260
|
+
action="store_true",
|
|
261
|
+
help="apply upstream commit changes recorded by verify before repairing",
|
|
262
|
+
)
|
|
263
|
+
repair_parser.add_argument(
|
|
264
|
+
"--force-partial",
|
|
265
|
+
action="store_true",
|
|
266
|
+
help=(
|
|
267
|
+
"attempt repair even when cached verification data for untouched files is incomplete; "
|
|
268
|
+
"may leave the repository inconsistent"
|
|
269
|
+
),
|
|
270
|
+
)
|
|
271
|
+
repair_parser.add_argument(
|
|
272
|
+
"--repo-type", choices=["model", "dataset", "space"], default="model", help="repo kind to repair"
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
upgrade_parser = add_command_parser(
|
|
276
|
+
"upgrade",
|
|
277
|
+
help="EXPERIMENTAL: add complete torrent hash coverage to existing archives",
|
|
278
|
+
description=(
|
|
279
|
+
f"{TORRENT_EXPERIMENTAL_NOTICE} "
|
|
280
|
+
"Read only payload files missing reusable torrent hash coverage. "
|
|
281
|
+
"Payload bytes and the pinned resolved commit are not changed."
|
|
282
|
+
),
|
|
283
|
+
epilog="Exit status: 0 when coverage is complete or the dry-run succeeds; 1 on an unreadable archive or busy model.",
|
|
284
|
+
)
|
|
285
|
+
upgrade_parser.add_argument("model", metavar="repo", nargs="?", help="repo id to upgrade unless --all is used")
|
|
286
|
+
upgrade_parser.add_argument("--all", action="store_true", help="upgrade every mirrored repository of this type")
|
|
287
|
+
upgrade_parser.add_argument("--dry-run", action="store_true", help="report required reads without changing metadata")
|
|
288
|
+
upgrade_parser.add_argument(
|
|
289
|
+
"--repo-type", choices=["model", "dataset", "space"], default="model", help="repo kind to upgrade"
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
torrent_parser = add_command_parser(
|
|
293
|
+
"torrent",
|
|
294
|
+
help="EXPERIMENTAL: publish, seed, inspect, or retire commit-pinned torrents",
|
|
295
|
+
description=(
|
|
296
|
+
f"{TORRENT_EXPERIMENTAL_NOTICE} "
|
|
297
|
+
"Manage client-independent torrent artifacts and model-mirror's durable seed intent."
|
|
298
|
+
),
|
|
299
|
+
)
|
|
300
|
+
torrent_subparsers = torrent_parser.add_subparsers(dest="torrent_command")
|
|
301
|
+
|
|
302
|
+
def add_torrent_repo_command(name: str, help_text: str) -> argparse.ArgumentParser:
|
|
303
|
+
command = torrent_subparsers.add_parser(
|
|
304
|
+
name,
|
|
305
|
+
help=help_text,
|
|
306
|
+
description=f"{TORRENT_EXPERIMENTAL_NOTICE} {help_text.capitalize()}.",
|
|
307
|
+
)
|
|
308
|
+
command.add_argument("model", metavar="repo", help="repo id, e.g. org/model")
|
|
309
|
+
command.add_argument(
|
|
310
|
+
"--repo-type",
|
|
311
|
+
choices=["model", "dataset", "space"],
|
|
312
|
+
default="model",
|
|
313
|
+
help="repo kind",
|
|
314
|
+
)
|
|
315
|
+
return command
|
|
316
|
+
|
|
317
|
+
create_torrent_parser = add_torrent_repo_command(
|
|
318
|
+
"create",
|
|
319
|
+
"create an ordinary .torrent and recovery record without requesting managed seeding",
|
|
320
|
+
)
|
|
321
|
+
create_torrent_parser.add_argument(
|
|
322
|
+
"--external",
|
|
323
|
+
action="store_true",
|
|
324
|
+
help="record that runtime seeding is owned by an external client",
|
|
325
|
+
)
|
|
326
|
+
publish_torrent_parser = add_torrent_repo_command(
|
|
327
|
+
"publish",
|
|
328
|
+
"create or reuse a publication and request durable seeding",
|
|
329
|
+
)
|
|
330
|
+
publish_torrent_parser.add_argument(
|
|
331
|
+
"--external",
|
|
332
|
+
action="store_true",
|
|
333
|
+
help="emit external-client handoff instead of managed seed intent",
|
|
334
|
+
)
|
|
335
|
+
add_torrent_repo_command("show", "show publication, trust, fence, and backend state")
|
|
336
|
+
add_torrent_repo_command("stop", "stop desired managed seeding without releasing the update fence")
|
|
337
|
+
add_torrent_repo_command("retire", "remove desired seeding and release the update fence")
|
|
338
|
+
handoff_torrent_parser = torrent_subparsers.add_parser(
|
|
339
|
+
"handoff",
|
|
340
|
+
help="print a standard-client download destination and exact import command",
|
|
341
|
+
description=(
|
|
342
|
+
f"{TORRENT_EXPERIMENTAL_NOTICE} "
|
|
343
|
+
"Print a standard-client download destination and exact import command."
|
|
344
|
+
),
|
|
345
|
+
)
|
|
346
|
+
handoff_torrent_parser.add_argument("torrent_file", type=Path, help="model-mirror .torrent file")
|
|
347
|
+
import_torrent_parser = torrent_subparsers.add_parser(
|
|
348
|
+
"import",
|
|
349
|
+
help="validate and atomically finalize payload downloaded by an external client",
|
|
350
|
+
description=(
|
|
351
|
+
f"{TORRENT_EXPERIMENTAL_NOTICE} "
|
|
352
|
+
"Validate and atomically finalize payload downloaded by an external client."
|
|
353
|
+
),
|
|
354
|
+
)
|
|
355
|
+
import_torrent_parser.add_argument("torrent_file", type=Path, help="model-mirror .torrent file")
|
|
356
|
+
import_torrent_parser.add_argument("payload_root", type=Path, help="downloaded torrent root directory")
|
|
357
|
+
import_torrent_parser.add_argument(
|
|
358
|
+
"--seed",
|
|
359
|
+
action="store_true",
|
|
360
|
+
help="request managed seeding after successful import",
|
|
361
|
+
)
|
|
362
|
+
join_torrent_parser = torrent_subparsers.add_parser(
|
|
363
|
+
"join",
|
|
364
|
+
help="download a model-mirror torrent or magnet and finalize it as a local archive",
|
|
365
|
+
description=(
|
|
366
|
+
f"{TORRENT_EXPERIMENTAL_NOTICE} "
|
|
367
|
+
"Download a model-mirror torrent or magnet and finalize it as a local archive."
|
|
368
|
+
),
|
|
369
|
+
)
|
|
370
|
+
join_torrent_parser.add_argument("source", help="path to a .torrent file or a magnet URI")
|
|
371
|
+
join_torrent_parser.add_argument(
|
|
372
|
+
"--seed",
|
|
373
|
+
action="store_true",
|
|
374
|
+
help="request managed seeding after successful join",
|
|
375
|
+
)
|
|
376
|
+
join_torrent_parser.add_argument(
|
|
377
|
+
"--metadata-timeout",
|
|
378
|
+
type=float,
|
|
379
|
+
default=120.0,
|
|
380
|
+
help="seconds to wait for magnet metadata; default 120",
|
|
381
|
+
)
|
|
382
|
+
serve_torrent_parser = torrent_subparsers.add_parser(
|
|
383
|
+
"serve",
|
|
384
|
+
help="run the managed libtorrent backend and reconcile durable seed intent",
|
|
385
|
+
description=(
|
|
386
|
+
f"{TORRENT_EXPERIMENTAL_NOTICE} "
|
|
387
|
+
"Run the managed libtorrent backend and reconcile durable seed intent."
|
|
388
|
+
),
|
|
389
|
+
)
|
|
390
|
+
serve_torrent_parser.add_argument(
|
|
391
|
+
"--poll-seconds",
|
|
392
|
+
type=float,
|
|
393
|
+
default=2.0,
|
|
394
|
+
help="desired-state reconciliation interval; default 2 seconds",
|
|
395
|
+
)
|
|
396
|
+
serve_torrent_parser.add_argument(
|
|
397
|
+
"--once",
|
|
398
|
+
action="store_true",
|
|
399
|
+
help="perform one reconciliation pass and stop (diagnostic)",
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
offline_parser = add_command_parser(
|
|
403
|
+
"offline",
|
|
404
|
+
help="mark a mirror offline-only",
|
|
405
|
+
description=(
|
|
406
|
+
"Disable Hub checks for one mirrored repo. Use this when the upstream repo is gone "
|
|
407
|
+
"and you want verification to use local state only."
|
|
408
|
+
),
|
|
409
|
+
epilog="Exit status: 0 when the mirror is marked offline-only; 1 when local verification state is missing or busy.",
|
|
410
|
+
)
|
|
411
|
+
offline_parser.add_argument("model", metavar="repo", help="repo id to mark offline-only")
|
|
412
|
+
offline_parser.add_argument(
|
|
413
|
+
"--repo-type", choices=["model", "dataset", "space"], default="model", help="repo kind to update"
|
|
414
|
+
)
|
|
415
|
+
|
|
416
|
+
online_parser = add_command_parser(
|
|
417
|
+
"online",
|
|
418
|
+
help="re-enable Hub checks for a mirror",
|
|
419
|
+
description="Clear offline-only mode so verify and repair contact the Hub again.",
|
|
420
|
+
epilog="Exit status: 0 when the mirror is marked online; 1 when local verification state is missing or busy.",
|
|
421
|
+
)
|
|
422
|
+
online_parser.add_argument("model", metavar="repo", help="repo id to mark online")
|
|
423
|
+
online_parser.add_argument(
|
|
424
|
+
"--repo-type", choices=["model", "dataset", "space"], default="model", help="repo kind to update"
|
|
425
|
+
)
|
|
426
|
+
|
|
427
|
+
list_parser = add_command_parser(
|
|
428
|
+
"list",
|
|
429
|
+
help="list mirrors in columns or show one mirror in detail",
|
|
430
|
+
description=(
|
|
431
|
+
"Show a compact archive-wide table, or detailed last-known local state when REPO is supplied. "
|
|
432
|
+
"This command is metadata-only and does not contact upstream unless --check-upstream is supplied."
|
|
433
|
+
),
|
|
434
|
+
)
|
|
435
|
+
status_parser = add_command_parser(
|
|
436
|
+
"status",
|
|
437
|
+
help="show archive status or detailed local state for one mirror",
|
|
438
|
+
description=(
|
|
439
|
+
"Show a compact archive-wide table, or detailed last-known local state when REPO is supplied. "
|
|
440
|
+
"This command is metadata-only and does not contact upstream unless --check-upstream is supplied."
|
|
441
|
+
),
|
|
442
|
+
)
|
|
443
|
+
for status_like_parser in (list_parser, status_parser):
|
|
444
|
+
status_like_parser.add_argument("model", metavar="repo", nargs="?", help="optional repo id, e.g. org/model")
|
|
445
|
+
status_like_parser.add_argument(
|
|
446
|
+
"--repo-type",
|
|
447
|
+
choices=["model", "dataset", "space"],
|
|
448
|
+
help="repo kind; filters the table or selects the detailed repo",
|
|
449
|
+
)
|
|
450
|
+
status_like_parser.add_argument(
|
|
451
|
+
"--check-upstream",
|
|
452
|
+
action="store_true",
|
|
453
|
+
help="advisory live comparison with upstream; does not update local metadata",
|
|
454
|
+
)
|
|
455
|
+
status_like_parser.add_argument(
|
|
456
|
+
"--verbose",
|
|
457
|
+
action="store_true",
|
|
458
|
+
help="show all recorded metadata fields in the human-readable detail view",
|
|
459
|
+
)
|
|
460
|
+
status_like_parser.add_argument(
|
|
461
|
+
"--json",
|
|
462
|
+
action="store_true",
|
|
463
|
+
dest="json_output",
|
|
464
|
+
help="emit stable structured JSON; --verbose has no effect",
|
|
465
|
+
)
|
|
466
|
+
|
|
467
|
+
clean_cache_parser = add_command_parser(
|
|
468
|
+
"clean-cache",
|
|
469
|
+
help="remove Hugging Face cache and temporary files",
|
|
470
|
+
description=(
|
|
471
|
+
"Remove archive cache, temporary files, and per-mirror Hugging Face metadata caches "
|
|
472
|
+
"without touching mirrored payload files. Without --force, only reports reclaimable space."
|
|
473
|
+
),
|
|
474
|
+
epilog="Exit status: 0 when cleanup succeeds; 1 when a configured cleanup target is unsafe.",
|
|
475
|
+
)
|
|
476
|
+
clean_cache_parser.add_argument("--force", action="store_true", help="delete cache and temporary files")
|
|
477
|
+
|
|
478
|
+
config_parser = add_command_parser(
|
|
479
|
+
"config",
|
|
480
|
+
help="show or change configuration",
|
|
481
|
+
description="Print configuration, describe supported keys, or persist configuration changes.",
|
|
482
|
+
)
|
|
483
|
+
config_subparsers = config_parser.add_subparsers(dest="config_command")
|
|
484
|
+
config_subparsers.add_parser("show", help="print the resolved configuration")
|
|
485
|
+
config_subparsers.add_parser("options", help="print supported configuration keys with descriptions")
|
|
486
|
+
directory_parser = config_subparsers.add_parser("directory", help="get or set archive directory")
|
|
487
|
+
directory_parser.add_argument("path", nargs="?", help="new archive directory; omit to print current value")
|
|
488
|
+
set_parser = config_subparsers.add_parser("set", help="set a supported configuration key")
|
|
489
|
+
set_parser.add_argument("key", help="configuration key; see 'model-mirror config options'")
|
|
490
|
+
set_parser.add_argument("value", help="new value")
|
|
491
|
+
|
|
492
|
+
help_parser = add_command_parser(
|
|
493
|
+
"help",
|
|
494
|
+
help="show help",
|
|
495
|
+
description="Show full help or command-specific help.",
|
|
496
|
+
)
|
|
497
|
+
help_parser.add_argument("topic", nargs="?", help="optional command to show help for")
|
|
498
|
+
parser.command_parsers = command_parsers
|
|
499
|
+
|
|
500
|
+
return parser
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def add_revision_options(parser: argparse.ArgumentParser) -> None:
|
|
504
|
+
revision_group = parser.add_mutually_exclusive_group()
|
|
505
|
+
revision_group.add_argument("--revision", help="branch, tag, or commit to use; defaults to config revision")
|
|
506
|
+
revision_group.add_argument("--commit", help="commit SHA to use")
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def parse_nonnegative_int_arg(value: str) -> int:
|
|
510
|
+
try:
|
|
511
|
+
return parse_nonnegative_int(value, default=0)
|
|
512
|
+
except ValueError as exc:
|
|
513
|
+
raise argparse.ArgumentTypeError(str(exc)) from exc
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def selected_revision_arg(args) -> str | None:
|
|
517
|
+
return args.commit or args.revision
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def main(argv: list[str] | None = None, *, hub=None) -> int:
|
|
521
|
+
raw_argv = list(sys.argv[1:] if argv is None else argv)
|
|
522
|
+
parser = build_parser()
|
|
523
|
+
args = parser.parse_args(raw_argv)
|
|
524
|
+
if args.command is None:
|
|
525
|
+
parser.print_help()
|
|
526
|
+
return 0
|
|
527
|
+
if args.command == "help":
|
|
528
|
+
return handle_help(parser, args.topic)
|
|
529
|
+
config_path = Path(args.config).expanduser() if args.config else None
|
|
530
|
+
config = load_config(config_path)
|
|
531
|
+
|
|
532
|
+
if should_supervise_mirror(args, config, hub):
|
|
533
|
+
return run_supervised_mirror(raw_argv, args, config)
|
|
534
|
+
|
|
535
|
+
try:
|
|
536
|
+
if args.command == "config":
|
|
537
|
+
return handle_config(args, config, config_path)
|
|
538
|
+
if args.command in {"list", "status"}:
|
|
539
|
+
return handle_list(
|
|
540
|
+
config,
|
|
541
|
+
repo_id=args.model,
|
|
542
|
+
repo_type=args.repo_type,
|
|
543
|
+
check_upstream=args.check_upstream,
|
|
544
|
+
verbose=args.verbose,
|
|
545
|
+
json_output=args.json_output,
|
|
546
|
+
hub=hub,
|
|
547
|
+
)
|
|
548
|
+
if args.command == "clean-cache":
|
|
549
|
+
return handle_clean_cache(config, force=args.force)
|
|
550
|
+
if args.command == "mirror":
|
|
551
|
+
selected_hub = hub or HuggingFaceHub(config)
|
|
552
|
+
result = mirror(
|
|
553
|
+
config,
|
|
554
|
+
args.model,
|
|
555
|
+
hub=selected_hub,
|
|
556
|
+
repo_type=args.repo_type,
|
|
557
|
+
revision=selected_revision_arg(args),
|
|
558
|
+
force=args.force,
|
|
559
|
+
verify_after=config.verify_after_mirror and not args.no_verify,
|
|
560
|
+
stall_timeout_seconds=args.stall_timeout,
|
|
561
|
+
)
|
|
562
|
+
print(f"{result.status}: {args.model} -> {result.path}")
|
|
563
|
+
return mirror_exit_code(result.status)
|
|
564
|
+
if args.command == "remove":
|
|
565
|
+
return handle_remove(args, config)
|
|
566
|
+
if args.command == "verify":
|
|
567
|
+
return handle_verify(args, config, hub=hub)
|
|
568
|
+
if args.command == "repair":
|
|
569
|
+
return handle_repair(args, config, hub=hub)
|
|
570
|
+
if args.command == "upgrade":
|
|
571
|
+
return handle_upgrade(args, config)
|
|
572
|
+
if args.command == "torrent":
|
|
573
|
+
return handle_torrent(args, config)
|
|
574
|
+
if args.command == "offline":
|
|
575
|
+
return handle_offline_mode(args, config, offline_only=True)
|
|
576
|
+
if args.command == "online":
|
|
577
|
+
return handle_offline_mode(args, config, offline_only=False)
|
|
578
|
+
except ModelBusyError as exc:
|
|
579
|
+
print(str(exc))
|
|
580
|
+
return 1
|
|
581
|
+
|
|
582
|
+
parser.error(f"Unhandled command: {args.command}")
|
|
583
|
+
return 2
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def should_supervise_mirror(args, config: Config, hub) -> bool:
|
|
587
|
+
if args.command != "mirror":
|
|
588
|
+
return False
|
|
589
|
+
if hub is not None:
|
|
590
|
+
return False
|
|
591
|
+
if os.environ.get("MODEL_MIRROR_SUPERVISED_CHILD") == "1":
|
|
592
|
+
return False
|
|
593
|
+
return effective_stall_timeout(args, config) > 0
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def effective_stall_timeout(args, config: Config) -> int:
|
|
597
|
+
selected = getattr(args, "stall_timeout", None)
|
|
598
|
+
return config.stall_timeout_seconds if selected is None else selected
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def run_supervised_mirror(raw_argv: list[str], args, config: Config) -> int:
|
|
602
|
+
repo_type = args.repo_type or config.repo_type
|
|
603
|
+
root = archive_path(config, args.model, repo_type)
|
|
604
|
+
timeout = effective_stall_timeout(args, config)
|
|
605
|
+
retry_limit = config.stall_retries
|
|
606
|
+
retry_counts: dict[str, int] = {}
|
|
607
|
+
command = [sys.executable, "-m", "model_mirror.cli", *raw_argv]
|
|
608
|
+
env = dict(os.environ)
|
|
609
|
+
env["MODEL_MIRROR_SUPERVISED_CHILD"] = "1"
|
|
610
|
+
|
|
611
|
+
while True:
|
|
612
|
+
child = subprocess.Popen(command, env=env)
|
|
613
|
+
restart, returncode = supervise_child(child, root, timeout, retry_limit, retry_counts)
|
|
614
|
+
if restart:
|
|
615
|
+
continue
|
|
616
|
+
return returncode
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def supervise_child(
|
|
620
|
+
child,
|
|
621
|
+
root: Path,
|
|
622
|
+
timeout: int,
|
|
623
|
+
retry_limit: int,
|
|
624
|
+
retry_counts: dict[str, int],
|
|
625
|
+
) -> tuple[bool, int]:
|
|
626
|
+
started = time.monotonic()
|
|
627
|
+
started_at_utc = datetime.now(timezone.utc)
|
|
628
|
+
poll_interval = min(30.0, max(1.0, timeout / 10))
|
|
629
|
+
while True:
|
|
630
|
+
returncode = child.poll()
|
|
631
|
+
if returncode is not None:
|
|
632
|
+
return False, returncode
|
|
633
|
+
|
|
634
|
+
snapshot = progress_snapshot(root, stall_timeout_seconds=timeout)
|
|
635
|
+
fresh_entries = fresh_progress_entries(snapshot.entries, started_at_utc)
|
|
636
|
+
stalled_entries = [entry for entry in fresh_entries if entry.stalled]
|
|
637
|
+
if stalled_entries:
|
|
638
|
+
entry = selected_progress_entry(stalled_entries)
|
|
639
|
+
path = entry.path
|
|
640
|
+
retry_counts[path] = retry_counts.get(path, 0) + 1
|
|
641
|
+
if retry_counts[path] > retry_limit:
|
|
642
|
+
print(
|
|
643
|
+
f"stall retry limit exceeded for {path}: "
|
|
644
|
+
f"{retry_counts[path] - 1}/{retry_limit}; terminating mirror child"
|
|
645
|
+
)
|
|
646
|
+
terminate_child(child)
|
|
647
|
+
return False, 1
|
|
648
|
+
print(
|
|
649
|
+
f"stall detected for {path}: idle={format_age_seconds(entry.idle_seconds)}; "
|
|
650
|
+
f"restarting mirror child ({retry_counts[path]}/{retry_limit})"
|
|
651
|
+
)
|
|
652
|
+
terminate_child(child)
|
|
653
|
+
return True, 0
|
|
654
|
+
|
|
655
|
+
if not fresh_entries and time.monotonic() - started >= timeout:
|
|
656
|
+
retry_counts["<no-progress>"] = retry_counts.get("<no-progress>", 0) + 1
|
|
657
|
+
if retry_counts["<no-progress>"] > retry_limit:
|
|
658
|
+
print("stall retry limit exceeded before progress was reported; terminating mirror child")
|
|
659
|
+
terminate_child(child)
|
|
660
|
+
return False, 1
|
|
661
|
+
print(
|
|
662
|
+
"stall detected before progress was reported; "
|
|
663
|
+
f"restarting mirror child ({retry_counts['<no-progress>']}/{retry_limit})"
|
|
664
|
+
)
|
|
665
|
+
terminate_child(child)
|
|
666
|
+
return True, 0
|
|
667
|
+
|
|
668
|
+
time.sleep(poll_interval)
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def fresh_progress_entries(entries: list[ProgressEntry], started_at_utc: datetime) -> list[ProgressEntry]:
|
|
672
|
+
return [entry for entry in entries if progress_entry_updated_after(entry, started_at_utc)]
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def progress_entry_updated_after(entry: ProgressEntry, cutoff: datetime) -> bool:
|
|
676
|
+
try:
|
|
677
|
+
updated = datetime.fromisoformat(entry.updated_at_utc)
|
|
678
|
+
except ValueError:
|
|
679
|
+
return False
|
|
680
|
+
if updated.tzinfo is None:
|
|
681
|
+
updated = updated.replace(tzinfo=timezone.utc)
|
|
682
|
+
return updated >= cutoff
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
def terminate_child(child) -> None:
|
|
686
|
+
child.terminate()
|
|
687
|
+
try:
|
|
688
|
+
child.wait(timeout=30)
|
|
689
|
+
except subprocess.TimeoutExpired:
|
|
690
|
+
child.kill()
|
|
691
|
+
child.wait(timeout=30)
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def handle_help(parser: argparse.ArgumentParser, topic: str | None) -> int:
|
|
695
|
+
if topic is None:
|
|
696
|
+
parser.print_help()
|
|
697
|
+
return 0
|
|
698
|
+
command_parser = parser.command_parsers.get(topic)
|
|
699
|
+
if command_parser is None:
|
|
700
|
+
parser.error(f"Unknown help topic: {topic}")
|
|
701
|
+
command_parser.print_help()
|
|
702
|
+
return 0
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
def mirror_exit_code(status: str) -> int:
|
|
706
|
+
return 0 if status in {"complete", "downloaded"} else 1
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
def handle_config(args, config: Config, config_path: Path | None) -> int:
|
|
710
|
+
if args.config_command in {None, "options"}:
|
|
711
|
+
return handle_config_options(config)
|
|
712
|
+
|
|
713
|
+
if args.config_command == "show":
|
|
714
|
+
print(f"directory: {config.directory}")
|
|
715
|
+
print(f"repo_type: {config.repo_type}")
|
|
716
|
+
print(f"revision: {config.revision}")
|
|
717
|
+
print(f"checksum: {config.checksum}")
|
|
718
|
+
print(f"checksum_workers: {config.checksum_workers}")
|
|
719
|
+
print(f"download_workers: {config.download_workers}")
|
|
720
|
+
print(f"stall_timeout_seconds: {config.stall_timeout_seconds}")
|
|
721
|
+
print(f"stall_retries: {config.stall_retries}")
|
|
722
|
+
print(f"verify_after_mirror: {config.verify_after_mirror}")
|
|
723
|
+
print(f"hf_xet_high_performance: {config.hf_xet_high_performance}")
|
|
724
|
+
print(f"hf_xet_reconstruct_write_sequentially: {config.hf_xet_reconstruct_write_sequentially}")
|
|
725
|
+
if config.hf_xet_num_concurrent_range_gets is not None:
|
|
726
|
+
print(f"hf_xet_num_concurrent_range_gets: {config.hf_xet_num_concurrent_range_gets}")
|
|
727
|
+
if config.token_path:
|
|
728
|
+
print(f"token_path: {config.token_path}")
|
|
729
|
+
return 0
|
|
730
|
+
|
|
731
|
+
if args.config_command == "directory":
|
|
732
|
+
if args.path is None:
|
|
733
|
+
print(config.directory)
|
|
734
|
+
return 0
|
|
735
|
+
config.directory = Path(args.path).expanduser()
|
|
736
|
+
config.directory.mkdir(parents=True, exist_ok=True)
|
|
737
|
+
save_config(config, config_path)
|
|
738
|
+
print(f"directory: {config.directory}")
|
|
739
|
+
return 0
|
|
740
|
+
|
|
741
|
+
if args.config_command == "set":
|
|
742
|
+
set_config_value(config, args.key, args.value)
|
|
743
|
+
save_config(config, config_path)
|
|
744
|
+
print(f"{args.key}: {args.value}")
|
|
745
|
+
return 0
|
|
746
|
+
|
|
747
|
+
return 2
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
def handle_config_options(config: Config) -> int:
|
|
751
|
+
values = {
|
|
752
|
+
"directory": config.directory,
|
|
753
|
+
"repo_type": config.repo_type,
|
|
754
|
+
"revision": config.revision,
|
|
755
|
+
"checksum": config.checksum,
|
|
756
|
+
"checksum_workers": config.checksum_workers,
|
|
757
|
+
"download_workers": config.download_workers,
|
|
758
|
+
"stall_timeout_seconds": config.stall_timeout_seconds,
|
|
759
|
+
"stall_retries": config.stall_retries,
|
|
760
|
+
"verify_after_mirror": config.verify_after_mirror,
|
|
761
|
+
"hf_xet_high_performance": config.hf_xet_high_performance,
|
|
762
|
+
"hf_xet_reconstruct_write_sequentially": config.hf_xet_reconstruct_write_sequentially,
|
|
763
|
+
"hf_xet_num_concurrent_range_gets": config.hf_xet_num_concurrent_range_gets,
|
|
764
|
+
"token_path": config.token_path,
|
|
765
|
+
"cache_dir": config.cache_dir,
|
|
766
|
+
"tmp_dir": config.tmp_dir,
|
|
767
|
+
}
|
|
768
|
+
for key, env_var, description in CONFIG_OPTIONS:
|
|
769
|
+
print(f"{key}: {values[key]}")
|
|
770
|
+
if env_var:
|
|
771
|
+
print(f" env: {env_var}")
|
|
772
|
+
print(f" {description}")
|
|
773
|
+
return 0
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
def set_config_value(config: Config, key: str, value: str) -> None:
|
|
777
|
+
normalized = key.replace("-", "_")
|
|
778
|
+
if normalized == "directory":
|
|
779
|
+
config.directory = Path(value).expanduser()
|
|
780
|
+
elif normalized == "repo_type":
|
|
781
|
+
config.repo_type = value
|
|
782
|
+
elif normalized == "revision":
|
|
783
|
+
config.revision = value
|
|
784
|
+
elif normalized == "checksum":
|
|
785
|
+
config.checksum = parse_bool(value)
|
|
786
|
+
elif normalized == "checksum_workers":
|
|
787
|
+
config.checksum_workers = parse_positive_int(value, default=1)
|
|
788
|
+
elif normalized == "download_workers":
|
|
789
|
+
config.download_workers = parse_positive_int(value, default=1)
|
|
790
|
+
elif normalized in {"stall_timeout", "stall_timeout_seconds"}:
|
|
791
|
+
config.stall_timeout_seconds = parse_nonnegative_int(value, default=600)
|
|
792
|
+
elif normalized == "stall_retries":
|
|
793
|
+
config.stall_retries = parse_nonnegative_int(value, default=3)
|
|
794
|
+
elif normalized in {"verify_after_mirror", "audit_after_mirror"}:
|
|
795
|
+
config.verify_after_mirror = parse_bool(value)
|
|
796
|
+
elif normalized == "hf_xet_high_performance":
|
|
797
|
+
config.hf_xet_high_performance = parse_bool(value)
|
|
798
|
+
elif normalized == "hf_xet_reconstruct_write_sequentially":
|
|
799
|
+
config.hf_xet_reconstruct_write_sequentially = parse_bool(value)
|
|
800
|
+
elif normalized == "hf_xet_num_concurrent_range_gets":
|
|
801
|
+
config.hf_xet_num_concurrent_range_gets = parse_positive_int(value, default=16)
|
|
802
|
+
elif normalized == "token_path":
|
|
803
|
+
config.token_path = Path(value).expanduser()
|
|
804
|
+
elif normalized == "cache_dir":
|
|
805
|
+
config.cache_dir = Path(value).expanduser()
|
|
806
|
+
elif normalized == "tmp_dir":
|
|
807
|
+
config.tmp_dir = Path(value).expanduser()
|
|
808
|
+
else:
|
|
809
|
+
raise SystemExit(f"Unsupported config key: {key}")
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
def handle_list(
|
|
813
|
+
config: Config,
|
|
814
|
+
*,
|
|
815
|
+
repo_id: str | None = None,
|
|
816
|
+
repo_type: str | None = None,
|
|
817
|
+
check_upstream: bool = False,
|
|
818
|
+
verbose: bool = False,
|
|
819
|
+
json_output: bool = False,
|
|
820
|
+
hub=None,
|
|
821
|
+
) -> int:
|
|
822
|
+
errors: list[dict[str, str]] = []
|
|
823
|
+
if repo_id is not None:
|
|
824
|
+
selected_type = repo_type or config.repo_type
|
|
825
|
+
root = archive_path(config, repo_id, selected_type)
|
|
826
|
+
if not root.is_dir():
|
|
827
|
+
message = f"mirror not found: {selected_type}:{repo_id} -> {root}"
|
|
828
|
+
if json_output:
|
|
829
|
+
errors.append(
|
|
830
|
+
{
|
|
831
|
+
"code": "mirror-not-found",
|
|
832
|
+
"repo_id": repo_id,
|
|
833
|
+
"repo_type": selected_type,
|
|
834
|
+
"message": message,
|
|
835
|
+
}
|
|
836
|
+
)
|
|
837
|
+
print_status_json(config, [], errors)
|
|
838
|
+
else:
|
|
839
|
+
print(message)
|
|
840
|
+
return 1
|
|
841
|
+
entries = [build_mirror_status(config, root, repo_id, selected_type)]
|
|
842
|
+
if check_upstream:
|
|
843
|
+
observe_upstreams(config, entries, hub=hub)
|
|
844
|
+
if json_output:
|
|
845
|
+
print_status_json(config, entries, errors)
|
|
846
|
+
elif verbose:
|
|
847
|
+
print_mirror_detail_verbose(entries[0])
|
|
848
|
+
else:
|
|
849
|
+
print_mirror_detail(entries[0])
|
|
850
|
+
return 0
|
|
851
|
+
|
|
852
|
+
entries = collect_mirror_status(config, repo_type=repo_type)
|
|
853
|
+
if check_upstream:
|
|
854
|
+
observe_upstreams(config, entries, hub=hub)
|
|
855
|
+
if json_output:
|
|
856
|
+
print_status_json(config, entries, errors)
|
|
857
|
+
return 0
|
|
858
|
+
|
|
859
|
+
archive_root = Path(config.directory)
|
|
860
|
+
known_sizes = [entry.payload_size for entry in entries if entry.payload_size is not None]
|
|
861
|
+
unknown_sizes = len(entries) - len(known_sizes)
|
|
862
|
+
summary = f"archive: {archive_root} mirrors={len(entries)} recorded_payload={format_bytes(sum(known_sizes))}"
|
|
863
|
+
if unknown_sizes:
|
|
864
|
+
summary += f" unknown_payload={unknown_sizes}"
|
|
865
|
+
print(summary)
|
|
866
|
+
print_mirror_table(entries)
|
|
867
|
+
return 0
|
|
868
|
+
|
|
869
|
+
|
|
870
|
+
def handle_remove(args, config: Config) -> int:
|
|
871
|
+
root = archive_path(config, args.model, args.repo_type)
|
|
872
|
+
staged = removal_path(config, args.model, args.repo_type)
|
|
873
|
+
removal_control = Path(config.directory) / REMOVALS_DIR
|
|
874
|
+
try:
|
|
875
|
+
with ModelLock(removal_control, "remove", args.model, args.repo_type):
|
|
876
|
+
if staged.exists() or staged.is_symlink():
|
|
877
|
+
return resume_remove(args, config, root, staged)
|
|
878
|
+
if root.is_symlink():
|
|
879
|
+
print(f"remove refused unsafe mirror path: {root}")
|
|
880
|
+
return 1
|
|
881
|
+
if not root.is_dir():
|
|
882
|
+
print(f"mirror not found: {args.repo_type}:{args.model} -> {root}")
|
|
883
|
+
return 1
|
|
884
|
+
|
|
885
|
+
with ModelLock(root, "remove", args.model, args.repo_type):
|
|
886
|
+
marker = removal_record_path(root)
|
|
887
|
+
pending = marker.exists() or marker.is_symlink()
|
|
888
|
+
try:
|
|
889
|
+
record = read_removal_record(root) if pending else None
|
|
890
|
+
except ValueError:
|
|
891
|
+
record = None
|
|
892
|
+
record = record or inspect_removal(root, args.model, args.repo_type)
|
|
893
|
+
if pending:
|
|
894
|
+
print(f"prepared interrupted removal found: {root}")
|
|
895
|
+
print_removal_summary(record)
|
|
896
|
+
publication_error = active_publication_removal_error(root)
|
|
897
|
+
if publication_error is not None:
|
|
898
|
+
marker.unlink(missing_ok=True)
|
|
899
|
+
print(publication_error)
|
|
900
|
+
return 1
|
|
901
|
+
if not args.yes and not confirm_removal(args.model):
|
|
902
|
+
marker.unlink(missing_ok=True)
|
|
903
|
+
print(f"cancelled: {args.repo_type}:{args.model} was not removed")
|
|
904
|
+
return 0
|
|
905
|
+
write_removal_record(root, record)
|
|
906
|
+
|
|
907
|
+
try:
|
|
908
|
+
stage_removal(root, staged, record)
|
|
909
|
+
complete_removal(staged)
|
|
910
|
+
except (OSError, ValueError) as exc:
|
|
911
|
+
print(
|
|
912
|
+
f"removal interrupted: {args.repo_type}:{args.model} -> "
|
|
913
|
+
f"{staged if staged.exists() else root} ({exc}); "
|
|
914
|
+
"rerun the same remove command to resume"
|
|
915
|
+
)
|
|
916
|
+
return 1
|
|
917
|
+
finish_removal_cleanup(config, staged)
|
|
918
|
+
except (OSError, ValueError) as exc:
|
|
919
|
+
print(f"remove failed: {args.repo_type}:{args.model} -> {exc}")
|
|
920
|
+
return 1
|
|
921
|
+
|
|
922
|
+
print(
|
|
923
|
+
f"removed: {args.repo_type}:{args.model} "
|
|
924
|
+
f"({record.payload_files} files, {format_bytes(record.payload_size)})"
|
|
925
|
+
)
|
|
926
|
+
return 0
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
def resume_remove(args, config: Config, root: Path, staged: Path) -> int:
|
|
930
|
+
if staged.is_symlink() or not staged.is_dir():
|
|
931
|
+
print(f"remove refused unsafe interrupted-removal path: {staged}")
|
|
932
|
+
return 1
|
|
933
|
+
try:
|
|
934
|
+
record = read_removal_record(staged)
|
|
935
|
+
except ValueError:
|
|
936
|
+
record = None
|
|
937
|
+
if record is not None and (
|
|
938
|
+
record.repo_id != args.model or record.repo_type != args.repo_type
|
|
939
|
+
):
|
|
940
|
+
print(
|
|
941
|
+
f"remove resume refused: removal record identity is "
|
|
942
|
+
f"{record.repo_type}:{record.repo_id}, expected {args.repo_type}:{args.model}"
|
|
943
|
+
)
|
|
944
|
+
return 1
|
|
945
|
+
record = record or inspect_removal(staged, args.model, args.repo_type, original_path=root)
|
|
946
|
+
print(f"interrupted removal found: {staged}")
|
|
947
|
+
print_removal_summary(record)
|
|
948
|
+
if not args.yes and not confirm_removal(args.model):
|
|
949
|
+
print(f"cancelled: interrupted removal remains at {staged}")
|
|
950
|
+
return 0
|
|
951
|
+
try:
|
|
952
|
+
complete_removal(staged)
|
|
953
|
+
except OSError as exc:
|
|
954
|
+
print(
|
|
955
|
+
f"removal still incomplete: {args.repo_type}:{args.model} -> {staged} ({exc}); "
|
|
956
|
+
"rerun the same remove command to resume"
|
|
957
|
+
)
|
|
958
|
+
return 1
|
|
959
|
+
|
|
960
|
+
finish_removal_cleanup(config, staged)
|
|
961
|
+
print(
|
|
962
|
+
f"removed interrupted mirror: {args.repo_type}:{args.model} "
|
|
963
|
+
f"({record.payload_files} original files, {format_bytes(record.payload_size)})"
|
|
964
|
+
)
|
|
965
|
+
if root.exists() or root.is_symlink():
|
|
966
|
+
print(f"note: a newer mirror still exists and was not removed: {root}")
|
|
967
|
+
return 0
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def inspect_removal(
|
|
971
|
+
root: Path,
|
|
972
|
+
repo_id: str,
|
|
973
|
+
repo_type: str,
|
|
974
|
+
*,
|
|
975
|
+
original_path: Path | None = None,
|
|
976
|
+
) -> RemovalRecord:
|
|
977
|
+
payload_files, payload_size = mirror_payload_stats(root)
|
|
978
|
+
state = None
|
|
979
|
+
snapshot = None
|
|
980
|
+
exceptions: list[str] = []
|
|
981
|
+
try:
|
|
982
|
+
state = read_verification_state(root)
|
|
983
|
+
except (OSError, ValueError, yaml.YAMLError):
|
|
984
|
+
exceptions.append("verification-metadata-error")
|
|
985
|
+
try:
|
|
986
|
+
snapshot = read_snapshot_plan(root)
|
|
987
|
+
except (OSError, ValueError):
|
|
988
|
+
exceptions.append("snapshot-metadata-error")
|
|
989
|
+
if state is not None:
|
|
990
|
+
for tag in list_exception_tags(state, None):
|
|
991
|
+
append_unique(exceptions, tag)
|
|
992
|
+
if (
|
|
993
|
+
state is not None
|
|
994
|
+
and state.resolved_commit
|
|
995
|
+
and snapshot is not None
|
|
996
|
+
and snapshot.resolved_commit != state.resolved_commit
|
|
997
|
+
):
|
|
998
|
+
append_unique(exceptions, "snapshot-stale")
|
|
999
|
+
resolved_commit = (
|
|
1000
|
+
state.resolved_commit
|
|
1001
|
+
if state is not None and state.resolved_commit
|
|
1002
|
+
else snapshot.resolved_commit if snapshot is not None else ""
|
|
1003
|
+
)
|
|
1004
|
+
checked_at = state.checked_at_utc if state is not None else ""
|
|
1005
|
+
return RemovalRecord(
|
|
1006
|
+
repo_id=repo_id,
|
|
1007
|
+
repo_type=repo_type,
|
|
1008
|
+
original_path=str(original_path or root),
|
|
1009
|
+
status=state.status if state is not None else "unverified",
|
|
1010
|
+
exceptions=",".join(exceptions) or "none",
|
|
1011
|
+
resolved_commit=resolved_commit,
|
|
1012
|
+
checked_at_utc=checked_at or "unknown",
|
|
1013
|
+
check_age=verification_age_label(checked_at) if checked_at else "unknown",
|
|
1014
|
+
payload_files=payload_files,
|
|
1015
|
+
payload_size=payload_size,
|
|
1016
|
+
)
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def print_removal_summary(record: RemovalRecord) -> None:
|
|
1020
|
+
print("mirror selected for permanent removal:")
|
|
1021
|
+
print(f" repository: {record.repo_id}")
|
|
1022
|
+
print(f" repo_type: {record.repo_type}")
|
|
1023
|
+
print(f" path: {record.original_path}")
|
|
1024
|
+
print(f" status: {record.status}")
|
|
1025
|
+
print(f" exceptions: {record.exceptions}")
|
|
1026
|
+
print(f" resolved_commit: {record.resolved_commit or 'unknown'}")
|
|
1027
|
+
print(f" last_checked: {record.checked_at_utc}")
|
|
1028
|
+
print(f" verification_age: {record.check_age}")
|
|
1029
|
+
print(f" payload_files: {record.payload_files}")
|
|
1030
|
+
print(f" payload_size: {format_bytes(record.payload_size)} ({record.payload_size} bytes)")
|
|
1031
|
+
|
|
1032
|
+
|
|
1033
|
+
def active_publication_removal_error(root: Path) -> str | None:
|
|
1034
|
+
from .torrent_publication import load_fenced_publication
|
|
1035
|
+
|
|
1036
|
+
try:
|
|
1037
|
+
fenced = load_fenced_publication(root)
|
|
1038
|
+
except (OSError, ValueError) as exc:
|
|
1039
|
+
return f"remove blocked: torrent publication state is unreadable: {exc}"
|
|
1040
|
+
if fenced is None:
|
|
1041
|
+
return None
|
|
1042
|
+
record = fenced[0]
|
|
1043
|
+
return (
|
|
1044
|
+
f"remove blocked by active torrent publication {record.publication_id}. "
|
|
1045
|
+
f"Stop any external client if applicable, then run "
|
|
1046
|
+
f"'model-mirror torrent stop --repo-type {record.repo_type} {record.repo_id}' and "
|
|
1047
|
+
f"'model-mirror torrent retire --repo-type {record.repo_type} {record.repo_id}'."
|
|
1048
|
+
)
|
|
1049
|
+
|
|
1050
|
+
|
|
1051
|
+
def confirm_removal(repo_id: str) -> bool:
|
|
1052
|
+
try:
|
|
1053
|
+
response = input(f"Type '{repo_id}' to confirm permanent removal: ")
|
|
1054
|
+
except (EOFError, KeyboardInterrupt):
|
|
1055
|
+
print()
|
|
1056
|
+
return False
|
|
1057
|
+
return response.strip() == repo_id
|
|
1058
|
+
|
|
1059
|
+
|
|
1060
|
+
def finish_removal_cleanup(config: Config, staged: Path) -> None:
|
|
1061
|
+
prune_empty_removal_parents(
|
|
1062
|
+
staged.parent,
|
|
1063
|
+
stop=Path(config.directory) / REMOVALS_DIR,
|
|
1064
|
+
)
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
def collect_mirror_status(config: Config, *, repo_type: str | None = None) -> list[MirrorStatusEntry]:
|
|
1068
|
+
archive_root = Path(config.directory)
|
|
1069
|
+
entries: list[MirrorStatusEntry] = []
|
|
1070
|
+
selected_types = [repo_type] if repo_type is not None else list(REPO_TYPE_DIRS)
|
|
1071
|
+
for selected_type in selected_types:
|
|
1072
|
+
type_dir = REPO_TYPE_DIRS[selected_type]
|
|
1073
|
+
repo_root = archive_root / type_dir
|
|
1074
|
+
if not repo_root.exists():
|
|
1075
|
+
continue
|
|
1076
|
+
for owner in sorted(path for path in repo_root.iterdir() if path.is_dir()):
|
|
1077
|
+
for repo in sorted(path for path in owner.iterdir() if path.is_dir()):
|
|
1078
|
+
repo_id = f"{owner.name}/{repo.name}"
|
|
1079
|
+
entries.append(build_mirror_status(config, repo, repo_id, selected_type))
|
|
1080
|
+
entries.sort(key=lambda entry: (entry.repo_type, entry.repo_id))
|
|
1081
|
+
return entries
|
|
1082
|
+
|
|
1083
|
+
|
|
1084
|
+
def build_mirror_status(config: Config, root: Path, repo_id: str, repo_type: str) -> MirrorStatusEntry:
|
|
1085
|
+
state = read_verification_state(root)
|
|
1086
|
+
snapshot = read_snapshot_plan(root)
|
|
1087
|
+
payload_files, payload_size, payload_source = recorded_payload_stats(root, snapshot)
|
|
1088
|
+
if (
|
|
1089
|
+
payload_source == "snapshot"
|
|
1090
|
+
and state is not None
|
|
1091
|
+
and state.resolved_commit
|
|
1092
|
+
and snapshot is not None
|
|
1093
|
+
and snapshot.resolved_commit != state.resolved_commit
|
|
1094
|
+
):
|
|
1095
|
+
payload_files, payload_size, payload_source = None, None, "unknown"
|
|
1096
|
+
expected_files = len(snapshot.files) if snapshot is not None else None
|
|
1097
|
+
return MirrorStatusEntry(
|
|
1098
|
+
root=root,
|
|
1099
|
+
repo_id=repo_id,
|
|
1100
|
+
repo_type=repo_type,
|
|
1101
|
+
state=state,
|
|
1102
|
+
active_lock=read_active_lock(root),
|
|
1103
|
+
payload_files=payload_files,
|
|
1104
|
+
payload_size=payload_size,
|
|
1105
|
+
payload_source=payload_source,
|
|
1106
|
+
expected_files=expected_files,
|
|
1107
|
+
snapshot_commit=snapshot.resolved_commit if snapshot is not None else "",
|
|
1108
|
+
snapshot_requested_revision=snapshot.requested_revision if snapshot is not None else "",
|
|
1109
|
+
progress=progress_snapshot(
|
|
1110
|
+
root,
|
|
1111
|
+
stall_timeout_seconds=config.stall_timeout_seconds,
|
|
1112
|
+
scan_incomplete=False,
|
|
1113
|
+
),
|
|
1114
|
+
torrent_status=list_torrent_status(root),
|
|
1115
|
+
)
|
|
1116
|
+
|
|
1117
|
+
|
|
1118
|
+
def recorded_payload_stats(root: Path, snapshot) -> tuple[int | None, int | None, str]:
|
|
1119
|
+
manifest_path = root / MANIFEST
|
|
1120
|
+
if manifest_path.exists():
|
|
1121
|
+
manifest = load_manifest(root)
|
|
1122
|
+
sizes = [row.get("size") for row in manifest.values()]
|
|
1123
|
+
total_size = (
|
|
1124
|
+
sum(sizes)
|
|
1125
|
+
if all(isinstance(size, int) and not isinstance(size, bool) and size >= 0 for size in sizes)
|
|
1126
|
+
else None
|
|
1127
|
+
)
|
|
1128
|
+
return len(manifest), total_size, "manifest"
|
|
1129
|
+
if snapshot is not None:
|
|
1130
|
+
sizes = [item.size for item in snapshot.files]
|
|
1131
|
+
total_size = sum(sizes) if all(size is not None and size >= 0 for size in sizes) else None
|
|
1132
|
+
return len(snapshot.files), total_size, "snapshot"
|
|
1133
|
+
return None, None, "unknown"
|
|
1134
|
+
|
|
1135
|
+
|
|
1136
|
+
def observe_upstreams(config: Config, entries: list[MirrorStatusEntry], *, hub=None) -> None:
|
|
1137
|
+
selected_hub = hub or HuggingFaceHub(config)
|
|
1138
|
+
for entry in entries:
|
|
1139
|
+
state = entry.state
|
|
1140
|
+
requested_revision = (
|
|
1141
|
+
state.requested_revision
|
|
1142
|
+
if state is not None and state.requested_revision
|
|
1143
|
+
else entry.snapshot_requested_revision or config.revision
|
|
1144
|
+
)
|
|
1145
|
+
observed_at = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
1146
|
+
try:
|
|
1147
|
+
snapshot = get_snapshot(selected_hub, entry.repo_id, entry.repo_type, requested_revision)
|
|
1148
|
+
except Exception as exc:
|
|
1149
|
+
entry.upstream_observation = UpstreamObservation(
|
|
1150
|
+
status="unavailable",
|
|
1151
|
+
commit=None,
|
|
1152
|
+
observed_at_utc=observed_at,
|
|
1153
|
+
error=str(exc),
|
|
1154
|
+
)
|
|
1155
|
+
continue
|
|
1156
|
+
local_commit = resolved_commit_for_status(entry)
|
|
1157
|
+
entry.upstream_observation = UpstreamObservation(
|
|
1158
|
+
status=(
|
|
1159
|
+
"available"
|
|
1160
|
+
if not local_commit
|
|
1161
|
+
else "current"
|
|
1162
|
+
if local_commit == snapshot.resolved_commit
|
|
1163
|
+
else "changed"
|
|
1164
|
+
),
|
|
1165
|
+
commit=snapshot.resolved_commit,
|
|
1166
|
+
observed_at_utc=observed_at,
|
|
1167
|
+
)
|
|
1168
|
+
|
|
1169
|
+
|
|
1170
|
+
def print_mirror_table(entries: list[MirrorStatusEntry]) -> None:
|
|
1171
|
+
headers = ["TYPE", "REPOSITORY", "FILES", "SIZE", "COMMIT", "CHECKED", "EXCEPTIONS"]
|
|
1172
|
+
include_torrent = any(entry.torrent_status is not None for entry in entries)
|
|
1173
|
+
include_activity = any(entry.progress.active for entry in entries)
|
|
1174
|
+
include_live_upstream = any(entry.upstream_observation is not None for entry in entries)
|
|
1175
|
+
if include_live_upstream:
|
|
1176
|
+
headers.append("UPSTREAM NOW")
|
|
1177
|
+
if include_torrent:
|
|
1178
|
+
headers.append("TORRENT")
|
|
1179
|
+
if include_activity:
|
|
1180
|
+
headers.append("ACTIVITY")
|
|
1181
|
+
|
|
1182
|
+
rows = []
|
|
1183
|
+
for entry in entries:
|
|
1184
|
+
state = entry.state
|
|
1185
|
+
commit = resolved_commit_for_status(entry)
|
|
1186
|
+
row = [
|
|
1187
|
+
entry.repo_type,
|
|
1188
|
+
entry.repo_id,
|
|
1189
|
+
file_count_label(entry),
|
|
1190
|
+
format_optional_bytes(entry.payload_size),
|
|
1191
|
+
shorten_commit(commit),
|
|
1192
|
+
verification_age_label(state.checked_at_utc) if state is not None else "unknown",
|
|
1193
|
+
",".join(mirror_exception_tags(entry)) or "-",
|
|
1194
|
+
]
|
|
1195
|
+
if include_live_upstream:
|
|
1196
|
+
row.append(format_upstream_observation(entry.upstream_observation))
|
|
1197
|
+
if include_torrent:
|
|
1198
|
+
row.append(entry.torrent_status or "-")
|
|
1199
|
+
if include_activity:
|
|
1200
|
+
row.append(format_progress_detail(entry.progress) or "-")
|
|
1201
|
+
rows.append(row)
|
|
1202
|
+
print_table(headers, rows, right_aligned={"FILES", "SIZE"})
|
|
1203
|
+
|
|
1204
|
+
|
|
1205
|
+
def print_table(headers: list[str], rows: list[list[str]], *, right_aligned: set[str]) -> None:
|
|
1206
|
+
widths = [
|
|
1207
|
+
max([len(header), *(len(row[index]) for row in rows)])
|
|
1208
|
+
for index, header in enumerate(headers)
|
|
1209
|
+
]
|
|
1210
|
+
|
|
1211
|
+
def formatted(row: list[str]) -> str:
|
|
1212
|
+
cells = []
|
|
1213
|
+
for index, value in enumerate(row):
|
|
1214
|
+
if headers[index] in right_aligned:
|
|
1215
|
+
cells.append(value.rjust(widths[index]))
|
|
1216
|
+
else:
|
|
1217
|
+
cells.append(value.ljust(widths[index]))
|
|
1218
|
+
return " ".join(cells).rstrip()
|
|
1219
|
+
|
|
1220
|
+
print(formatted(headers))
|
|
1221
|
+
print(formatted(["-" * width for width in widths]))
|
|
1222
|
+
for row in rows:
|
|
1223
|
+
print(formatted(row))
|
|
1224
|
+
|
|
1225
|
+
|
|
1226
|
+
def print_mirror_detail(entry: MirrorStatusEntry) -> None:
|
|
1227
|
+
state = entry.state
|
|
1228
|
+
exceptions = mirror_exception_tags(entry)
|
|
1229
|
+
print(entry.repo_id)
|
|
1230
|
+
print()
|
|
1231
|
+
status = state.status if state is not None else "unverified"
|
|
1232
|
+
checked = verification_age_label(state.checked_at_utc) if state is not None else "unknown"
|
|
1233
|
+
print_status_field("Verification", f"{status} · {checked} ago" if checked != "unknown" else status)
|
|
1234
|
+
commit = resolved_commit_for_status(entry)
|
|
1235
|
+
revision = (
|
|
1236
|
+
state.requested_revision
|
|
1237
|
+
if state is not None and state.requested_revision
|
|
1238
|
+
else entry.snapshot_requested_revision or "unknown"
|
|
1239
|
+
)
|
|
1240
|
+
snapshot = shorten_commit(commit) if commit else "unknown"
|
|
1241
|
+
print_status_field("Commit", f"{snapshot} · {revision}")
|
|
1242
|
+
payload = (
|
|
1243
|
+
f"{entry.payload_files} {'file' if entry.payload_files == 1 else 'files'} · "
|
|
1244
|
+
f"{format_optional_bytes(entry.payload_size)}"
|
|
1245
|
+
if entry.payload_files is not None
|
|
1246
|
+
else "unknown"
|
|
1247
|
+
)
|
|
1248
|
+
print_status_field("Payload", payload)
|
|
1249
|
+
print_status_field("Upstream", human_upstream_status(entry))
|
|
1250
|
+
print_status_field("Torrent", entry.torrent_status or "not published")
|
|
1251
|
+
|
|
1252
|
+
if exceptions:
|
|
1253
|
+
print_status_field("Attention", ", ".join(exceptions))
|
|
1254
|
+
if entry.active_lock is not None:
|
|
1255
|
+
print_status_field("Lock", format_lock_detail(entry.active_lock))
|
|
1256
|
+
progress = format_progress_detail(entry.progress)
|
|
1257
|
+
if progress is not None:
|
|
1258
|
+
print_status_field("Activity", progress)
|
|
1259
|
+
|
|
1260
|
+
next_steps = status_next_steps(entry)
|
|
1261
|
+
for index, step in enumerate(next_steps):
|
|
1262
|
+
print_status_field("Next" if index == 0 else "", step)
|
|
1263
|
+
|
|
1264
|
+
if state is not None and state.repair_paths:
|
|
1265
|
+
print()
|
|
1266
|
+
print("Repair paths")
|
|
1267
|
+
for value in state.repair_paths:
|
|
1268
|
+
print(f" {value}")
|
|
1269
|
+
if state is not None and state.issues:
|
|
1270
|
+
print()
|
|
1271
|
+
print("Issues")
|
|
1272
|
+
for value in state.issues:
|
|
1273
|
+
print(f" {value}")
|
|
1274
|
+
|
|
1275
|
+
|
|
1276
|
+
def print_status_field(label: str, value: str) -> None:
|
|
1277
|
+
print(f" {label:<14}{value}")
|
|
1278
|
+
|
|
1279
|
+
|
|
1280
|
+
def human_upstream_status(entry: MirrorStatusEntry) -> str:
|
|
1281
|
+
observation = entry.upstream_observation
|
|
1282
|
+
if observation is not None:
|
|
1283
|
+
value = f"{observation.status} now"
|
|
1284
|
+
if observation.commit:
|
|
1285
|
+
value += f" · {shorten_commit(observation.commit)}"
|
|
1286
|
+
if observation.error:
|
|
1287
|
+
value += f" · {observation.error}"
|
|
1288
|
+
return value
|
|
1289
|
+
state = entry.state
|
|
1290
|
+
if state is None or not state.upstream_status:
|
|
1291
|
+
return "unknown"
|
|
1292
|
+
suffix = " when last checked" if state.upstream_status != "unknown" else ""
|
|
1293
|
+
return f"{state.upstream_status}{suffix}"
|
|
1294
|
+
|
|
1295
|
+
|
|
1296
|
+
def format_upstream_observation(observation: UpstreamObservation | None) -> str:
|
|
1297
|
+
if observation is None:
|
|
1298
|
+
return "-"
|
|
1299
|
+
if observation.commit:
|
|
1300
|
+
return f"{observation.status}:{shorten_commit(observation.commit)}"
|
|
1301
|
+
return observation.status
|
|
1302
|
+
|
|
1303
|
+
|
|
1304
|
+
def status_next_steps(entry: MirrorStatusEntry) -> list[str]:
|
|
1305
|
+
state = entry.state
|
|
1306
|
+
if state is None:
|
|
1307
|
+
return [f"model-mirror verify {entry.repo_id}"]
|
|
1308
|
+
if primary_state_tag(state) == "unverified":
|
|
1309
|
+
return [f"model-mirror verify {entry.repo_id}"]
|
|
1310
|
+
steps = []
|
|
1311
|
+
if state.repair_paths:
|
|
1312
|
+
steps.append(f"model-mirror repair {entry.repo_id}")
|
|
1313
|
+
if state.upstream_status == "changed":
|
|
1314
|
+
steps.append(f"model-mirror repair --update {entry.repo_id}")
|
|
1315
|
+
return steps
|
|
1316
|
+
|
|
1317
|
+
|
|
1318
|
+
def print_mirror_detail_verbose(entry: MirrorStatusEntry) -> None:
|
|
1319
|
+
state = entry.state
|
|
1320
|
+
exceptions = mirror_exception_tags(entry)
|
|
1321
|
+
print(f"repository: {entry.repo_id}")
|
|
1322
|
+
print(f"repo_type: {entry.repo_type}")
|
|
1323
|
+
print(f"path: {entry.root}")
|
|
1324
|
+
print(f"status: {state.status if state is not None else 'unverified'}")
|
|
1325
|
+
print(f"exceptions: {','.join(exceptions) if exceptions else 'none'}")
|
|
1326
|
+
print(f"last_checked: {state.checked_at_utc if state is not None and state.checked_at_utc else 'unknown'}")
|
|
1327
|
+
print(f"last_check_age: {verification_age_label(state.checked_at_utc) if state is not None else 'unknown'}")
|
|
1328
|
+
print(f"requested_revision: {state.requested_revision if state is not None else 'unknown'}")
|
|
1329
|
+
print(f"resolved_commit: {resolved_commit_for_status(entry) or 'unknown'}")
|
|
1330
|
+
print(f"snapshot_commit: {entry.snapshot_commit or 'unknown'}")
|
|
1331
|
+
print(f"upstream_commit: {state.upstream_commit if state is not None and state.upstream_commit else 'unknown'}")
|
|
1332
|
+
print(f"upstream_status: {state.upstream_status if state is not None else 'unknown'}")
|
|
1333
|
+
print(f"offline_only: {str(state.offline_only).lower() if state is not None else 'unknown'}")
|
|
1334
|
+
print(f"payload_files: {entry.payload_files if entry.payload_files is not None else 'unknown'}")
|
|
1335
|
+
if entry.payload_size is None:
|
|
1336
|
+
print("payload_size: unknown")
|
|
1337
|
+
else:
|
|
1338
|
+
print(f"payload_size: {format_bytes(entry.payload_size)} ({entry.payload_size} bytes)")
|
|
1339
|
+
print(f"payload_source: {entry.payload_source}")
|
|
1340
|
+
if entry.expected_files is None:
|
|
1341
|
+
print("expected_files: unknown")
|
|
1342
|
+
else:
|
|
1343
|
+
stale_suffix = " (snapshot stale)" if snapshot_is_stale(entry) else ""
|
|
1344
|
+
print(f"expected_files: {entry.expected_files} recorded{stale_suffix}")
|
|
1345
|
+
print(f"lock: {format_lock_detail(entry.active_lock) if entry.active_lock is not None else 'none'}")
|
|
1346
|
+
print(f"progress: {format_progress_detail(entry.progress) or 'none'}")
|
|
1347
|
+
print(f"torrent: {entry.torrent_status or 'none'}")
|
|
1348
|
+
observation = entry.upstream_observation
|
|
1349
|
+
if observation is None:
|
|
1350
|
+
print("live_upstream: not checked")
|
|
1351
|
+
else:
|
|
1352
|
+
detail = f"{observation.status} commit={observation.commit or 'unknown'} observed_at={observation.observed_at_utc}"
|
|
1353
|
+
if observation.error:
|
|
1354
|
+
detail += f" error={observation.error}"
|
|
1355
|
+
print(f"live_upstream: {detail}")
|
|
1356
|
+
print_detail_values("repair_paths", state.repair_paths if state is not None else [])
|
|
1357
|
+
print_detail_values("issues", state.issues if state is not None else [])
|
|
1358
|
+
|
|
1359
|
+
|
|
1360
|
+
def print_status_json(
|
|
1361
|
+
config: Config,
|
|
1362
|
+
entries: list[MirrorStatusEntry],
|
|
1363
|
+
errors: list[dict[str, str]],
|
|
1364
|
+
) -> None:
|
|
1365
|
+
document = {
|
|
1366
|
+
"schema": "model-mirror-status",
|
|
1367
|
+
"version": 1,
|
|
1368
|
+
"archive": str(Path(config.directory)),
|
|
1369
|
+
"repositories": [mirror_status_json(entry) for entry in entries],
|
|
1370
|
+
"errors": errors,
|
|
1371
|
+
}
|
|
1372
|
+
print(json.dumps(document, indent=2, sort_keys=True))
|
|
1373
|
+
|
|
1374
|
+
|
|
1375
|
+
def mirror_status_json(entry: MirrorStatusEntry) -> dict:
|
|
1376
|
+
state = entry.state
|
|
1377
|
+
observation = entry.upstream_observation
|
|
1378
|
+
return {
|
|
1379
|
+
"repo_id": entry.repo_id,
|
|
1380
|
+
"repo_type": entry.repo_type,
|
|
1381
|
+
"path": str(entry.root),
|
|
1382
|
+
"requested_revision": (
|
|
1383
|
+
state.requested_revision
|
|
1384
|
+
if state is not None and state.requested_revision
|
|
1385
|
+
else entry.snapshot_requested_revision or None
|
|
1386
|
+
),
|
|
1387
|
+
"resolved_commit": resolved_commit_for_status(entry) or None,
|
|
1388
|
+
"last_verification": {
|
|
1389
|
+
"status": state.status if state is not None else "unverified",
|
|
1390
|
+
"checked_at": state.checked_at_utc if state is not None and state.checked_at_utc else None,
|
|
1391
|
+
"offline_only": state.offline_only if state is not None else None,
|
|
1392
|
+
"issues": list(state.issues) if state is not None else [],
|
|
1393
|
+
"repair_paths": list(state.repair_paths) if state is not None else [],
|
|
1394
|
+
},
|
|
1395
|
+
"payload": {
|
|
1396
|
+
"files": entry.payload_files,
|
|
1397
|
+
"bytes": entry.payload_size,
|
|
1398
|
+
"source": entry.payload_source,
|
|
1399
|
+
},
|
|
1400
|
+
"snapshot": {
|
|
1401
|
+
"commit": entry.snapshot_commit or None,
|
|
1402
|
+
"expected_files": entry.expected_files,
|
|
1403
|
+
"stale": snapshot_is_stale(entry),
|
|
1404
|
+
},
|
|
1405
|
+
"upstream": {
|
|
1406
|
+
"recorded": {
|
|
1407
|
+
"status": state.upstream_status if state is not None else "unknown",
|
|
1408
|
+
"commit": state.upstream_commit if state is not None and state.upstream_commit else None,
|
|
1409
|
+
},
|
|
1410
|
+
"live": (
|
|
1411
|
+
{
|
|
1412
|
+
"status": observation.status,
|
|
1413
|
+
"commit": observation.commit,
|
|
1414
|
+
"observed_at": observation.observed_at_utc,
|
|
1415
|
+
"error": observation.error,
|
|
1416
|
+
}
|
|
1417
|
+
if observation is not None
|
|
1418
|
+
else None
|
|
1419
|
+
),
|
|
1420
|
+
},
|
|
1421
|
+
"exceptions": mirror_exception_tags(entry),
|
|
1422
|
+
"lock": entry.active_lock,
|
|
1423
|
+
"activity": [progress_entry_json(item) for item in entry.progress.entries],
|
|
1424
|
+
"torrent": torrent_status_json(entry),
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1427
|
+
|
|
1428
|
+
def progress_entry_json(entry: ProgressEntry) -> dict:
|
|
1429
|
+
return {
|
|
1430
|
+
"path": entry.path,
|
|
1431
|
+
"stage": entry.stage,
|
|
1432
|
+
"bytes_done": entry.bytes_done,
|
|
1433
|
+
"bytes_total": entry.bytes_total,
|
|
1434
|
+
"updated_at": entry.updated_at_utc or None,
|
|
1435
|
+
"idle_seconds": entry.idle_seconds,
|
|
1436
|
+
"stalled": entry.stalled,
|
|
1437
|
+
"source": entry.source,
|
|
1438
|
+
"rate_bytes_per_second": entry.rate_bytes_per_second,
|
|
1439
|
+
}
|
|
1440
|
+
|
|
1441
|
+
|
|
1442
|
+
def torrent_status_json(entry: MirrorStatusEntry) -> dict | None:
|
|
1443
|
+
if entry.torrent_status is None:
|
|
1444
|
+
return None
|
|
1445
|
+
if entry.torrent_status == "metadata-error":
|
|
1446
|
+
return {"status": "metadata-error"}
|
|
1447
|
+
from .torrent_publication import load_fenced_publication
|
|
1448
|
+
|
|
1449
|
+
try:
|
|
1450
|
+
fenced = load_fenced_publication(entry.root)
|
|
1451
|
+
except (OSError, ValueError):
|
|
1452
|
+
return {"status": "metadata-error"}
|
|
1453
|
+
if fenced is None:
|
|
1454
|
+
return None
|
|
1455
|
+
record = fenced[0]
|
|
1456
|
+
try:
|
|
1457
|
+
coverage = torrent_coverage_status(entry.root)
|
|
1458
|
+
except (OSError, ValueError, RuntimeError):
|
|
1459
|
+
coverage = "metadata-error"
|
|
1460
|
+
return {
|
|
1461
|
+
"publication_id": record.publication_id,
|
|
1462
|
+
"resolved_commit": record.resolved_commit,
|
|
1463
|
+
"profile": record.profile,
|
|
1464
|
+
"lifecycle": record.lifecycle,
|
|
1465
|
+
"client_mode": record.client_mode,
|
|
1466
|
+
"desired_seed": record.desired_seed,
|
|
1467
|
+
"observed_backend": record.observed_backend,
|
|
1468
|
+
"coverage": coverage,
|
|
1469
|
+
"content_verification": record.content_verification,
|
|
1470
|
+
"publication_trust": record.publication_trust,
|
|
1471
|
+
"upstream_provenance": record.upstream_provenance,
|
|
1472
|
+
"upstream_availability": record.upstream_availability,
|
|
1473
|
+
"infohash_v1": record.infohash_v1,
|
|
1474
|
+
"infohash_v2": record.infohash_v2,
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1477
|
+
|
|
1478
|
+
def print_detail_values(label: str, values: list[str]) -> None:
|
|
1479
|
+
if not values:
|
|
1480
|
+
print(f"{label}: none")
|
|
1481
|
+
return
|
|
1482
|
+
print(f"{label}:")
|
|
1483
|
+
for value in values:
|
|
1484
|
+
print(f" - {value}")
|
|
1485
|
+
|
|
1486
|
+
|
|
1487
|
+
def resolved_commit_for_status(entry: MirrorStatusEntry) -> str:
|
|
1488
|
+
if entry.state is not None and entry.state.resolved_commit:
|
|
1489
|
+
return entry.state.resolved_commit
|
|
1490
|
+
return entry.snapshot_commit
|
|
1491
|
+
|
|
1492
|
+
|
|
1493
|
+
def shorten_commit(commit: str) -> str:
|
|
1494
|
+
return commit[:12] if commit else "-"
|
|
1495
|
+
|
|
1496
|
+
|
|
1497
|
+
def file_count_label(entry: MirrorStatusEntry) -> str:
|
|
1498
|
+
return str(entry.payload_files) if entry.payload_files is not None else "?"
|
|
1499
|
+
|
|
1500
|
+
|
|
1501
|
+
def mirror_exception_tags(entry: MirrorStatusEntry) -> list[str]:
|
|
1502
|
+
tags = list_exception_tags(entry.state, entry.active_lock, entry.progress)
|
|
1503
|
+
if snapshot_is_stale(entry):
|
|
1504
|
+
append_unique(tags, "snapshot-stale")
|
|
1505
|
+
marker = removal_record_path(entry.root)
|
|
1506
|
+
if marker.exists() or marker.is_symlink():
|
|
1507
|
+
append_unique(tags, "removal-pending")
|
|
1508
|
+
return tags
|
|
1509
|
+
|
|
1510
|
+
|
|
1511
|
+
def snapshot_is_stale(entry: MirrorStatusEntry) -> bool:
|
|
1512
|
+
return bool(
|
|
1513
|
+
entry.snapshot_commit
|
|
1514
|
+
and entry.state is not None
|
|
1515
|
+
and entry.state.resolved_commit
|
|
1516
|
+
and entry.snapshot_commit != entry.state.resolved_commit
|
|
1517
|
+
)
|
|
1518
|
+
|
|
1519
|
+
|
|
1520
|
+
def list_torrent_status(root: Path) -> str | None:
|
|
1521
|
+
from .torrent_publication import load_fenced_publication
|
|
1522
|
+
|
|
1523
|
+
try:
|
|
1524
|
+
fenced = load_fenced_publication(root)
|
|
1525
|
+
except (OSError, ValueError):
|
|
1526
|
+
return "metadata-error"
|
|
1527
|
+
if fenced is None:
|
|
1528
|
+
return None
|
|
1529
|
+
record = fenced[0]
|
|
1530
|
+
parts = [record.lifecycle, record.client_mode]
|
|
1531
|
+
parts.append(f"desired={'seeding' if record.desired_seed else 'stopped'}")
|
|
1532
|
+
parts.append(f"observed={record.observed_backend}")
|
|
1533
|
+
state = read_verification_state(root)
|
|
1534
|
+
if state is not None and state.upstream_status == "changed":
|
|
1535
|
+
parts.append("update-available")
|
|
1536
|
+
try:
|
|
1537
|
+
coverage_status = torrent_coverage_status(root)
|
|
1538
|
+
except (OSError, ValueError, RuntimeError):
|
|
1539
|
+
coverage_status = "metadata-error"
|
|
1540
|
+
parts.append(f"coverage={coverage_status}")
|
|
1541
|
+
return ",".join(parts)
|
|
1542
|
+
|
|
1543
|
+
|
|
1544
|
+
def torrent_coverage_status(root: Path) -> str:
|
|
1545
|
+
snapshot = read_snapshot_plan(root)
|
|
1546
|
+
if snapshot is None:
|
|
1547
|
+
return "unavailable"
|
|
1548
|
+
from .torrent_coverage import TorrentCoverageRecorder
|
|
1549
|
+
|
|
1550
|
+
recorder = TorrentCoverageRecorder(root, snapshot)
|
|
1551
|
+
return "complete" if recorder.complete else "partial"
|
|
1552
|
+
|
|
1553
|
+
|
|
1554
|
+
def archive_cache_usage(config: Config) -> CacheUsage:
|
|
1555
|
+
archive_root = Path(config.directory)
|
|
1556
|
+
cache_root = archive_runtime_cache_path(config)
|
|
1557
|
+
tmp_root = archive_runtime_tmp_path(config)
|
|
1558
|
+
return CacheUsage(
|
|
1559
|
+
archive_cache=directory_size(cache_root),
|
|
1560
|
+
archive_tmp=directory_size(tmp_root),
|
|
1561
|
+
mirror_cache=sum(directory_size(path) for path in mirror_cache_dirs(archive_root)),
|
|
1562
|
+
)
|
|
1563
|
+
|
|
1564
|
+
|
|
1565
|
+
def handle_clean_cache(config: Config, *, force: bool) -> int:
|
|
1566
|
+
archive_root = Path(config.directory)
|
|
1567
|
+
targets = cleanup_targets(config)
|
|
1568
|
+
unsafe = [path for path, label in targets if not is_safe_cleanup_target(path, archive_root, label)]
|
|
1569
|
+
if unsafe:
|
|
1570
|
+
for path in unsafe:
|
|
1571
|
+
print(f"refusing unsafe cleanup target: {path}")
|
|
1572
|
+
return 1
|
|
1573
|
+
|
|
1574
|
+
sized_targets = [(path, label, directory_size(path)) for path, label in targets if path.exists()]
|
|
1575
|
+
total_size = sum(size for _path, _label, size in sized_targets)
|
|
1576
|
+
print(f"cleanup mode: {'force' if force else 'dry-run'}")
|
|
1577
|
+
print(f"reclaimable={format_bytes(total_size)}")
|
|
1578
|
+
for path, label, size in sized_targets:
|
|
1579
|
+
if force:
|
|
1580
|
+
shutil.rmtree(path)
|
|
1581
|
+
action = "removed"
|
|
1582
|
+
else:
|
|
1583
|
+
action = "would-remove"
|
|
1584
|
+
print(f"{action}: {label} {path} size={format_bytes(size)}")
|
|
1585
|
+
return 0
|
|
1586
|
+
|
|
1587
|
+
|
|
1588
|
+
def cleanup_targets(config: Config) -> list[tuple[Path, str]]:
|
|
1589
|
+
archive_root = Path(config.directory)
|
|
1590
|
+
cache_root = archive_runtime_cache_path(config)
|
|
1591
|
+
tmp_root = archive_runtime_tmp_path(config)
|
|
1592
|
+
targets = [(cache_root, "archive-cache"), (tmp_root, "archive-tmp")]
|
|
1593
|
+
selected = {path.resolve(strict=False) for path, _label in targets}
|
|
1594
|
+
for path, label in (
|
|
1595
|
+
(archive_root / ".cache", "legacy-archive-cache"),
|
|
1596
|
+
(archive_root / ".tmp", "legacy-archive-tmp"),
|
|
1597
|
+
):
|
|
1598
|
+
if path.resolve(strict=False) not in selected:
|
|
1599
|
+
targets.append((path, label))
|
|
1600
|
+
targets.extend((path, "mirror-cache") for path in mirror_cache_dirs(archive_root))
|
|
1601
|
+
return targets
|
|
1602
|
+
|
|
1603
|
+
|
|
1604
|
+
def mirror_cache_dirs(archive_root: Path) -> list[Path]:
|
|
1605
|
+
targets: list[Path] = []
|
|
1606
|
+
for type_dir in REPO_TYPE_DIRS.values():
|
|
1607
|
+
for repo in sorted((archive_root / type_dir).glob("*/*")):
|
|
1608
|
+
targets.append(repo / ".cache")
|
|
1609
|
+
return targets
|
|
1610
|
+
|
|
1611
|
+
|
|
1612
|
+
def is_safe_cleanup_target(path: Path, archive_root: Path, label: str) -> bool:
|
|
1613
|
+
resolved_path = path.resolve(strict=False)
|
|
1614
|
+
resolved_archive = archive_root.resolve(strict=False)
|
|
1615
|
+
protected = {resolved_archive, *(resolved_archive / type_dir for type_dir in REPO_TYPE_DIRS.values())}
|
|
1616
|
+
expected_names = {
|
|
1617
|
+
"archive-cache": {"cache", ".cache"},
|
|
1618
|
+
"archive-tmp": {"tmp", ".tmp"},
|
|
1619
|
+
"legacy-archive-cache": {".cache"},
|
|
1620
|
+
"legacy-archive-tmp": {".tmp"},
|
|
1621
|
+
"mirror-cache": {".cache"},
|
|
1622
|
+
}
|
|
1623
|
+
return (
|
|
1624
|
+
resolved_path.name in expected_names.get(label, set())
|
|
1625
|
+
and resolved_path.is_relative_to(resolved_archive)
|
|
1626
|
+
and resolved_path not in protected
|
|
1627
|
+
)
|
|
1628
|
+
|
|
1629
|
+
|
|
1630
|
+
def directory_size(path: Path) -> int:
|
|
1631
|
+
if not path.exists():
|
|
1632
|
+
return 0
|
|
1633
|
+
total_size = 0
|
|
1634
|
+
for child in path.rglob("*"):
|
|
1635
|
+
if child.is_file():
|
|
1636
|
+
total_size += child.stat().st_size
|
|
1637
|
+
return total_size
|
|
1638
|
+
|
|
1639
|
+
|
|
1640
|
+
def mirror_payload_stats(root: Path) -> tuple[int, int]:
|
|
1641
|
+
files = 0
|
|
1642
|
+
total_size = 0
|
|
1643
|
+
for path in iter_payload_files(root):
|
|
1644
|
+
files += 1
|
|
1645
|
+
total_size += path.stat().st_size
|
|
1646
|
+
return files, total_size
|
|
1647
|
+
|
|
1648
|
+
|
|
1649
|
+
def format_bytes(size: int) -> str:
|
|
1650
|
+
value = float(size)
|
|
1651
|
+
for unit in ("B", "KiB", "MiB", "GiB", "TiB", "PiB"): # pragma: no branch
|
|
1652
|
+
if abs(value) < 1024 or unit == "PiB":
|
|
1653
|
+
if unit == "B":
|
|
1654
|
+
return f"{int(value)} B"
|
|
1655
|
+
return f"{value:.1f} {unit}"
|
|
1656
|
+
value /= 1024
|
|
1657
|
+
|
|
1658
|
+
|
|
1659
|
+
def format_optional_bytes(size: int | None) -> str:
|
|
1660
|
+
return format_bytes(size) if size is not None else "unknown"
|
|
1661
|
+
|
|
1662
|
+
|
|
1663
|
+
def format_lock_detail(info: dict | None) -> str:
|
|
1664
|
+
if not info:
|
|
1665
|
+
return "lock held"
|
|
1666
|
+
parts = []
|
|
1667
|
+
for key in ("command", "pid", "host", "started_at_utc"):
|
|
1668
|
+
value = info.get(key)
|
|
1669
|
+
if value:
|
|
1670
|
+
parts.append(f"{key}={value}")
|
|
1671
|
+
return " ".join(parts) if parts else "lock held"
|
|
1672
|
+
|
|
1673
|
+
|
|
1674
|
+
def format_progress_detail(progress: ProgressSnapshot) -> str | None:
|
|
1675
|
+
if not progress.active:
|
|
1676
|
+
return None
|
|
1677
|
+
entry = selected_progress_entry(progress.entries)
|
|
1678
|
+
parts = [
|
|
1679
|
+
f"active={len(progress.entries)}",
|
|
1680
|
+
f"path={shorten_path(entry.path)}",
|
|
1681
|
+
f"stage={entry.stage}",
|
|
1682
|
+
f"bytes={format_progress_bytes(entry)}",
|
|
1683
|
+
]
|
|
1684
|
+
rate = format_rate(entry.rate_bytes_per_second)
|
|
1685
|
+
if rate is not None:
|
|
1686
|
+
parts.append(f"rate={rate}")
|
|
1687
|
+
if entry.idle_seconds is not None:
|
|
1688
|
+
parts.append(f"idle={format_age_seconds(entry.idle_seconds)}")
|
|
1689
|
+
if progress.stalled_count:
|
|
1690
|
+
parts.append(f"stalled={progress.stalled_count}")
|
|
1691
|
+
if entry.source != "heartbeat":
|
|
1692
|
+
parts.append(f"source={entry.source}")
|
|
1693
|
+
return " ".join(parts)
|
|
1694
|
+
|
|
1695
|
+
|
|
1696
|
+
def selected_progress_entry(entries: list[ProgressEntry]) -> ProgressEntry:
|
|
1697
|
+
stalled = [entry for entry in entries if entry.stalled]
|
|
1698
|
+
if stalled:
|
|
1699
|
+
return max(stalled, key=lambda entry: entry.idle_seconds or 0)
|
|
1700
|
+
return sorted(entries, key=lambda entry: entry.path)[0]
|
|
1701
|
+
|
|
1702
|
+
|
|
1703
|
+
def format_progress_bytes(entry: ProgressEntry) -> str:
|
|
1704
|
+
if entry.bytes_total:
|
|
1705
|
+
percent = (entry.bytes_done / entry.bytes_total) * 100
|
|
1706
|
+
return f"{format_bytes(entry.bytes_done)}/{format_bytes(entry.bytes_total)}({percent:.1f}%)"
|
|
1707
|
+
return format_bytes(entry.bytes_done)
|
|
1708
|
+
|
|
1709
|
+
|
|
1710
|
+
def format_rate(rate: float | None) -> str | None:
|
|
1711
|
+
if rate is None:
|
|
1712
|
+
return None
|
|
1713
|
+
return f"{format_bytes(int(rate))}/s"
|
|
1714
|
+
|
|
1715
|
+
|
|
1716
|
+
def shorten_path(path: str, *, max_length: int = 48) -> str:
|
|
1717
|
+
if len(path) <= max_length:
|
|
1718
|
+
return path
|
|
1719
|
+
return f"...{path[-(max_length - 3):]}"
|
|
1720
|
+
|
|
1721
|
+
|
|
1722
|
+
def list_state_tags(state, active_lock: dict | None, progress: ProgressSnapshot | None = None) -> list[str]:
|
|
1723
|
+
tags = [primary_state_tag(state)]
|
|
1724
|
+
if state.offline_only:
|
|
1725
|
+
append_unique(tags, "offline")
|
|
1726
|
+
if state.upstream_status == "changed":
|
|
1727
|
+
append_unique(tags, "upstream-changed")
|
|
1728
|
+
if state_has_upstream_unavailable(state):
|
|
1729
|
+
append_unique(tags, "upstream-unavailable")
|
|
1730
|
+
if active_lock is not None:
|
|
1731
|
+
append_unique(tags, "busy")
|
|
1732
|
+
if progress is not None and progress.any_stalled:
|
|
1733
|
+
append_unique(tags, "stalled")
|
|
1734
|
+
return tags
|
|
1735
|
+
|
|
1736
|
+
|
|
1737
|
+
def list_exception_tags(
|
|
1738
|
+
state: VerificationState | None,
|
|
1739
|
+
active_lock: dict | None,
|
|
1740
|
+
progress: ProgressSnapshot | None = None,
|
|
1741
|
+
) -> list[str]:
|
|
1742
|
+
if state is None:
|
|
1743
|
+
tags = ["unverified"]
|
|
1744
|
+
if active_lock is not None:
|
|
1745
|
+
tags.append("busy")
|
|
1746
|
+
if progress is not None and progress.any_stalled:
|
|
1747
|
+
tags.append("stalled")
|
|
1748
|
+
return tags
|
|
1749
|
+
tags = list_state_tags(state, active_lock, progress)
|
|
1750
|
+
return [tag for tag in tags if tag != "clean"]
|
|
1751
|
+
|
|
1752
|
+
|
|
1753
|
+
def primary_state_tag(state) -> str:
|
|
1754
|
+
if state.status == "unavailable":
|
|
1755
|
+
return "upstream-unavailable"
|
|
1756
|
+
if state.status == "incomplete":
|
|
1757
|
+
if state_has_manifest_incomplete(state):
|
|
1758
|
+
return "manifest-incomplete"
|
|
1759
|
+
return "incomplete"
|
|
1760
|
+
if state.status == "dirty":
|
|
1761
|
+
if state.repair_paths:
|
|
1762
|
+
return "needs-repair"
|
|
1763
|
+
if any(str(issue) == "verification skipped" for issue in state.issues):
|
|
1764
|
+
return "unverified"
|
|
1765
|
+
return "dirty"
|
|
1766
|
+
if state.status == "in_progress":
|
|
1767
|
+
return "in-progress"
|
|
1768
|
+
return state.status or "unknown"
|
|
1769
|
+
|
|
1770
|
+
|
|
1771
|
+
def append_unique(values: list[str], value: str) -> None:
|
|
1772
|
+
if value not in values:
|
|
1773
|
+
values.append(value)
|
|
1774
|
+
|
|
1775
|
+
|
|
1776
|
+
def state_has_manifest_incomplete(state) -> bool:
|
|
1777
|
+
return state_has_cached_hash_missing(state) or any(".manifest missing" in str(issue) for issue in state.issues)
|
|
1778
|
+
|
|
1779
|
+
|
|
1780
|
+
def handle_verify(args, config: Config, *, hub=None) -> int:
|
|
1781
|
+
if args.all:
|
|
1782
|
+
failures = 0
|
|
1783
|
+
changed = 0
|
|
1784
|
+
repair_needed = 0
|
|
1785
|
+
cache_incomplete = 0
|
|
1786
|
+
for repo_id in list_model_ids(config):
|
|
1787
|
+
if args.max_age and should_skip_recent_clean(config, repo_id, args.repo_type, args.max_age):
|
|
1788
|
+
print(f"skipped recent clean verification: {repo_id}")
|
|
1789
|
+
continue
|
|
1790
|
+
try:
|
|
1791
|
+
rc = verify_one(config, repo_id, args, hub=hub)
|
|
1792
|
+
except ModelBusyError as exc:
|
|
1793
|
+
print(f"skipped busy: {repo_id} -> {exc.root} ({lock_label(exc.info)})")
|
|
1794
|
+
failures += 1
|
|
1795
|
+
continue
|
|
1796
|
+
state = read_verification_state(archive_path(config, repo_id, args.repo_type))
|
|
1797
|
+
if state is not None and state.upstream_status == "changed":
|
|
1798
|
+
changed += 1
|
|
1799
|
+
if rc != 0:
|
|
1800
|
+
failures += 1
|
|
1801
|
+
if state is not None and state.repair_paths:
|
|
1802
|
+
repair_needed += 1
|
|
1803
|
+
if state is not None and state_has_cached_hash_missing(state):
|
|
1804
|
+
cache_incomplete += 1
|
|
1805
|
+
if repair_needed:
|
|
1806
|
+
print("next: model-mirror repair --all")
|
|
1807
|
+
if cache_incomplete:
|
|
1808
|
+
print("cached verification incomplete: run full verification with model-mirror verify --all")
|
|
1809
|
+
if changed:
|
|
1810
|
+
print("update changed upstreams: model-mirror repair --all --update")
|
|
1811
|
+
return 1 if failures else 0
|
|
1812
|
+
|
|
1813
|
+
if not args.model:
|
|
1814
|
+
raise SystemExit("verify requires a model id unless --all is used")
|
|
1815
|
+
return verify_one(config, args.model, args, hub=hub)
|
|
1816
|
+
|
|
1817
|
+
|
|
1818
|
+
def handle_repair(args, config: Config, *, hub=None) -> int:
|
|
1819
|
+
if args.all and args.model:
|
|
1820
|
+
raise SystemExit("repair accepts a model id or --all, not both")
|
|
1821
|
+
if args.all:
|
|
1822
|
+
failures = 0
|
|
1823
|
+
for repo_id in list_model_ids(config):
|
|
1824
|
+
state = read_verification_state(archive_path(config, repo_id, args.repo_type))
|
|
1825
|
+
if state is not None and state.offline_only:
|
|
1826
|
+
print(f"skipped offline-only: {repo_id}; repair requires an upstream repository")
|
|
1827
|
+
continue
|
|
1828
|
+
try:
|
|
1829
|
+
rc = repair_one(config, repo_id, args, hub=hub)
|
|
1830
|
+
except ModelBusyError as exc:
|
|
1831
|
+
print(f"skipped busy: {repo_id} -> {exc.root} ({lock_label(exc.info)})")
|
|
1832
|
+
failures += 1
|
|
1833
|
+
continue
|
|
1834
|
+
if rc != 0:
|
|
1835
|
+
failures += 1
|
|
1836
|
+
return 1 if failures else 0
|
|
1837
|
+
|
|
1838
|
+
if not args.model:
|
|
1839
|
+
raise SystemExit("repair requires a model id unless --all is used")
|
|
1840
|
+
return repair_one(config, args.model, args, hub=hub)
|
|
1841
|
+
|
|
1842
|
+
|
|
1843
|
+
def handle_upgrade(args, config: Config) -> int:
|
|
1844
|
+
if args.all and args.model:
|
|
1845
|
+
raise SystemExit("upgrade accepts a model id or --all, not both")
|
|
1846
|
+
if not args.all and not args.model:
|
|
1847
|
+
raise SystemExit("upgrade requires a model id unless --all is used")
|
|
1848
|
+
repo_ids = list_repo_ids(config, args.repo_type) if args.all else [args.model]
|
|
1849
|
+
failures = 0
|
|
1850
|
+
for repo_id in repo_ids:
|
|
1851
|
+
root = archive_path(config, repo_id, args.repo_type)
|
|
1852
|
+
try:
|
|
1853
|
+
with ModelLock(root, "upgrade", repo_id, args.repo_type):
|
|
1854
|
+
snapshot = read_snapshot_plan(root)
|
|
1855
|
+
if snapshot is None:
|
|
1856
|
+
print(
|
|
1857
|
+
f"upgrade unavailable: {repo_id} has no pinned snapshot description; "
|
|
1858
|
+
f"run model-mirror verify {repo_id}"
|
|
1859
|
+
)
|
|
1860
|
+
failures += 1
|
|
1861
|
+
continue
|
|
1862
|
+
from .torrent_coverage import upgrade_coverage
|
|
1863
|
+
|
|
1864
|
+
result = upgrade_coverage(
|
|
1865
|
+
root,
|
|
1866
|
+
snapshot,
|
|
1867
|
+
dry_run=args.dry_run,
|
|
1868
|
+
on_progress=lambda rel, done, total: print_upgrade_progress(
|
|
1869
|
+
repo_id,
|
|
1870
|
+
rel,
|
|
1871
|
+
done,
|
|
1872
|
+
total,
|
|
1873
|
+
),
|
|
1874
|
+
)
|
|
1875
|
+
except (OSError, ValueError, RuntimeError) as exc:
|
|
1876
|
+
print(f"upgrade failed: {repo_id} -> {exc}")
|
|
1877
|
+
failures += 1
|
|
1878
|
+
continue
|
|
1879
|
+
action = "would-hash" if args.dry_run else "hashed"
|
|
1880
|
+
print(
|
|
1881
|
+
f"{'upgrade dry-run' if args.dry_run else 'upgraded'}: {repo_id} "
|
|
1882
|
+
f"coverage={result.covered_files}/{result.total_files} "
|
|
1883
|
+
f"{action}_files={result.hashed_files} {action}_bytes={format_bytes(result.hashed_bytes)} "
|
|
1884
|
+
f"path={result.path}"
|
|
1885
|
+
)
|
|
1886
|
+
if not result.complete and not args.dry_run:
|
|
1887
|
+
failures += 1
|
|
1888
|
+
return 1 if failures else 0
|
|
1889
|
+
|
|
1890
|
+
|
|
1891
|
+
def print_upgrade_progress(repo_id: str, rel: str, done: int, total: int) -> None:
|
|
1892
|
+
if done == total:
|
|
1893
|
+
print(f"hashed: {repo_id}:{rel} {format_bytes(total)}")
|
|
1894
|
+
|
|
1895
|
+
|
|
1896
|
+
def handle_torrent(args, config: Config) -> int:
|
|
1897
|
+
if args.torrent_command is None:
|
|
1898
|
+
print("torrent requires a subcommand; run model-mirror help torrent")
|
|
1899
|
+
return 2
|
|
1900
|
+
if args.torrent_command == "serve":
|
|
1901
|
+
from .torrent_seed import serve
|
|
1902
|
+
|
|
1903
|
+
print(
|
|
1904
|
+
f"managed torrent backend: {'single reconciliation' if args.once else 'serving'} "
|
|
1905
|
+
f"archive={config.directory}"
|
|
1906
|
+
)
|
|
1907
|
+
try:
|
|
1908
|
+
serve(config, poll_seconds=args.poll_seconds, once=args.once)
|
|
1909
|
+
except (OSError, ValueError, RuntimeError) as exc:
|
|
1910
|
+
print(f"managed torrent backend failed: {exc}")
|
|
1911
|
+
return 1
|
|
1912
|
+
return 0
|
|
1913
|
+
if args.torrent_command in {"handoff", "import", "join"}:
|
|
1914
|
+
return handle_torrent_receive(args, config)
|
|
1915
|
+
|
|
1916
|
+
root = archive_path(config, args.model, args.repo_type)
|
|
1917
|
+
if args.torrent_command == "retire":
|
|
1918
|
+
return handle_torrent_retire(args, root)
|
|
1919
|
+
try:
|
|
1920
|
+
with ModelLock(root, f"torrent-{args.torrent_command}", args.model, args.repo_type):
|
|
1921
|
+
return handle_torrent_locked(args, root)
|
|
1922
|
+
except (OSError, ValueError, RuntimeError) as exc:
|
|
1923
|
+
print(f"torrent {args.torrent_command} failed: {args.model} -> {exc}")
|
|
1924
|
+
return 1
|
|
1925
|
+
|
|
1926
|
+
|
|
1927
|
+
def handle_torrent_locked(args, root: Path) -> int:
|
|
1928
|
+
from .torrent_publication import (
|
|
1929
|
+
create_publication,
|
|
1930
|
+
load_fenced_publication,
|
|
1931
|
+
set_seed_desired,
|
|
1932
|
+
)
|
|
1933
|
+
|
|
1934
|
+
if args.torrent_command in {"create", "publish"}:
|
|
1935
|
+
external = bool(args.external)
|
|
1936
|
+
result = create_publication(
|
|
1937
|
+
root,
|
|
1938
|
+
repo_id=args.model,
|
|
1939
|
+
repo_type=args.repo_type,
|
|
1940
|
+
desired_seed=args.torrent_command == "publish" and not external,
|
|
1941
|
+
client_mode="external" if external else "managed",
|
|
1942
|
+
)
|
|
1943
|
+
action = "created" if result.created else "reused"
|
|
1944
|
+
print(
|
|
1945
|
+
f"torrent {action}: {result.record.publication_id} "
|
|
1946
|
+
f"coverage_hashed_files={result.coverage_hashed_files} "
|
|
1947
|
+
f"coverage_hashed_bytes={format_bytes(result.coverage_hashed_bytes)}"
|
|
1948
|
+
)
|
|
1949
|
+
print_torrent_handoff(result.record, root)
|
|
1950
|
+
if args.torrent_command == "publish" and not external:
|
|
1951
|
+
print("desired seed: enabled; model-mirror torrent serve reconciles it now and after restart")
|
|
1952
|
+
elif external:
|
|
1953
|
+
print("client mode: external; model-mirror records the fence but does not manage client runtime")
|
|
1954
|
+
return 0
|
|
1955
|
+
if args.torrent_command == "show":
|
|
1956
|
+
fenced = load_fenced_publication(root)
|
|
1957
|
+
if fenced is None:
|
|
1958
|
+
print(f"unpublished: {args.model}")
|
|
1959
|
+
return 0
|
|
1960
|
+
print_torrent_status(fenced[0], root)
|
|
1961
|
+
return 0
|
|
1962
|
+
if args.torrent_command == "stop":
|
|
1963
|
+
record = set_seed_desired(root, desired=False)
|
|
1964
|
+
print(f"torrent stopping: {record.publication_id}; publication fence retained")
|
|
1965
|
+
return 0
|
|
1966
|
+
raise ValueError(f"unsupported torrent command: {args.torrent_command}")
|
|
1967
|
+
|
|
1968
|
+
|
|
1969
|
+
def handle_torrent_retire(args, root: Path) -> int:
|
|
1970
|
+
from .torrent_publication import (
|
|
1971
|
+
retire_publication,
|
|
1972
|
+
set_seed_desired,
|
|
1973
|
+
wait_for_maintenance_detach,
|
|
1974
|
+
)
|
|
1975
|
+
|
|
1976
|
+
try:
|
|
1977
|
+
with ModelLock(root, "torrent-retire-stop", args.model, args.repo_type):
|
|
1978
|
+
set_seed_desired(root, desired=False)
|
|
1979
|
+
wait_for_maintenance_detach(root)
|
|
1980
|
+
with ModelLock(root, "torrent-retire", args.model, args.repo_type):
|
|
1981
|
+
record = retire_publication(root)
|
|
1982
|
+
except (OSError, ValueError, RuntimeError) as exc:
|
|
1983
|
+
print(f"torrent retire failed: {args.model} -> {exc}")
|
|
1984
|
+
return 1
|
|
1985
|
+
print(
|
|
1986
|
+
f"torrent retired: {record.publication_id}; local update fence released. "
|
|
1987
|
+
"Already distributed torrent metadata cannot be revoked."
|
|
1988
|
+
)
|
|
1989
|
+
return 0
|
|
1990
|
+
|
|
1991
|
+
|
|
1992
|
+
def print_torrent_handoff(record, root: Path) -> None:
|
|
1993
|
+
print(f"torrent: {root / record.torrent_path}")
|
|
1994
|
+
print(f"recovery: {root / record.recovery_path}")
|
|
1995
|
+
print(f"magnet: {record.magnet_uri}")
|
|
1996
|
+
print(f"external client data location: {root.parent}")
|
|
1997
|
+
|
|
1998
|
+
|
|
1999
|
+
def handle_torrent_receive(args, config: Config) -> int:
|
|
2000
|
+
from .torrent_import import (
|
|
2001
|
+
external_handoff,
|
|
2002
|
+
import_external_payload,
|
|
2003
|
+
join_torrent,
|
|
2004
|
+
parse_publication_metainfo,
|
|
2005
|
+
)
|
|
2006
|
+
|
|
2007
|
+
try:
|
|
2008
|
+
if args.torrent_command == "handoff":
|
|
2009
|
+
metainfo = args.torrent_file.read_bytes()
|
|
2010
|
+
parsed = parse_publication_metainfo(metainfo)
|
|
2011
|
+
destination, command = external_handoff(config, parsed, args.torrent_file)
|
|
2012
|
+
print(f"publication: huggingface:{parsed.descriptor.repo_type}:{parsed.descriptor.repo_id}@{parsed.descriptor.resolved_commit}")
|
|
2013
|
+
print(f"external client data location: {destination}")
|
|
2014
|
+
print(f"after download: {command}")
|
|
2015
|
+
return 0
|
|
2016
|
+
if args.torrent_command == "import":
|
|
2017
|
+
result = import_external_payload(
|
|
2018
|
+
config,
|
|
2019
|
+
metainfo=args.torrent_file.read_bytes(),
|
|
2020
|
+
payload_root=args.payload_root,
|
|
2021
|
+
seed=args.seed,
|
|
2022
|
+
)
|
|
2023
|
+
else:
|
|
2024
|
+
result = join_torrent(
|
|
2025
|
+
config,
|
|
2026
|
+
args.source,
|
|
2027
|
+
seed=args.seed,
|
|
2028
|
+
metadata_timeout_seconds=args.metadata_timeout,
|
|
2029
|
+
on_progress=print_join_progress,
|
|
2030
|
+
)
|
|
2031
|
+
except (OSError, ValueError, RuntimeError) as exc:
|
|
2032
|
+
print(f"torrent {args.torrent_command} failed: {exc}")
|
|
2033
|
+
return 1
|
|
2034
|
+
print(
|
|
2035
|
+
f"torrent imported: {result.publication.publication_id} -> {result.path} "
|
|
2036
|
+
f"reread_files={result.reread_files} reread_bytes={format_bytes(result.reread_bytes)}"
|
|
2037
|
+
)
|
|
2038
|
+
print(
|
|
2039
|
+
"content=torrent-verified publication_trust=trusted-infohash "
|
|
2040
|
+
"upstream_provenance=not-upstream-verified upstream_availability=unknown"
|
|
2041
|
+
)
|
|
2042
|
+
print_torrent_handoff(result.publication, result.path)
|
|
2043
|
+
if result.publication.desired_seed:
|
|
2044
|
+
print("desired seed: enabled; model-mirror torrent serve will reconcile it")
|
|
2045
|
+
return 0
|
|
2046
|
+
|
|
2047
|
+
|
|
2048
|
+
def print_join_progress(status) -> None:
|
|
2049
|
+
print(
|
|
2050
|
+
f"torrent download: {status.name} {status.progress * 100:.1f}% "
|
|
2051
|
+
f"rate={format_bytes(int(status.download_payload_rate))}/s peers={status.num_peers}"
|
|
2052
|
+
)
|
|
2053
|
+
|
|
2054
|
+
|
|
2055
|
+
def print_torrent_status(record, root: Path) -> None:
|
|
2056
|
+
print("feature_stability: experimental")
|
|
2057
|
+
print(f"publication: {record.publication_id}")
|
|
2058
|
+
print(f"lifecycle: {record.lifecycle}")
|
|
2059
|
+
print(f"resolved_commit: {record.resolved_commit}")
|
|
2060
|
+
print(f"profile: {record.profile}")
|
|
2061
|
+
print(f"content_verification: {record.content_verification}")
|
|
2062
|
+
print(f"publication_trust: {record.publication_trust}")
|
|
2063
|
+
print(f"upstream_provenance: {record.upstream_provenance}")
|
|
2064
|
+
print(f"upstream_availability: {record.upstream_availability}")
|
|
2065
|
+
state = read_verification_state(root)
|
|
2066
|
+
print(
|
|
2067
|
+
f"update_available: "
|
|
2068
|
+
f"{str(state is not None and state.upstream_status == 'changed').lower()}"
|
|
2069
|
+
)
|
|
2070
|
+
print("archive_schema: model-mirror-snapshot/1")
|
|
2071
|
+
print(f"coverage_profile: {record.profile}")
|
|
2072
|
+
print(f"coverage_state: {torrent_coverage_status(root)}")
|
|
2073
|
+
print(f"client_mode: {record.client_mode}")
|
|
2074
|
+
print(f"desired_seed: {str(record.desired_seed).lower()}")
|
|
2075
|
+
print(f"observed_backend: {record.observed_backend}")
|
|
2076
|
+
if record.observed_detail:
|
|
2077
|
+
print(f"observed_detail: {record.observed_detail}")
|
|
2078
|
+
print(f"infohash_v1: {record.infohash_v1}")
|
|
2079
|
+
print(f"infohash_v2: {record.infohash_v2}")
|
|
2080
|
+
print_torrent_handoff(record, root)
|
|
2081
|
+
|
|
2082
|
+
|
|
2083
|
+
def state_has_cached_hash_missing(state) -> bool:
|
|
2084
|
+
return any(str(issue).startswith("cached_hash_missing:") for issue in state.issues)
|
|
2085
|
+
|
|
2086
|
+
|
|
2087
|
+
def state_has_upstream_unavailable(state) -> bool:
|
|
2088
|
+
return state.status == "unavailable" or any(
|
|
2089
|
+
str(issue).startswith("upstream unavailable:") for issue in state.issues
|
|
2090
|
+
)
|
|
2091
|
+
|
|
2092
|
+
|
|
2093
|
+
def repair_one(config: Config, repo_id: str, args, *, hub=None) -> int:
|
|
2094
|
+
selected_hub = hub or HuggingFaceHub(config)
|
|
2095
|
+
print_verification_age(config, repo_id, args.repo_type)
|
|
2096
|
+
if args.force_partial:
|
|
2097
|
+
print(
|
|
2098
|
+
"warning: --force-partial can leave the repository inconsistent when verification data is incomplete"
|
|
2099
|
+
)
|
|
2100
|
+
result = repair(
|
|
2101
|
+
config,
|
|
2102
|
+
repo_id,
|
|
2103
|
+
hub=selected_hub,
|
|
2104
|
+
repo_type=args.repo_type,
|
|
2105
|
+
update=args.update,
|
|
2106
|
+
force_partial=args.force_partial,
|
|
2107
|
+
)
|
|
2108
|
+
if result.status == "verify-required":
|
|
2109
|
+
print(f"run verify first: model-mirror verify {repo_id}")
|
|
2110
|
+
if result.status == "verification-incomplete":
|
|
2111
|
+
print(
|
|
2112
|
+
f"could not fully repair {repo_id}; missing verification data for some files. "
|
|
2113
|
+
f"Run full verify and repair again: model-mirror verify {repo_id} && model-mirror repair {repo_id}"
|
|
2114
|
+
)
|
|
2115
|
+
if result.status == "offline-only":
|
|
2116
|
+
print(
|
|
2117
|
+
f"cannot repair offline-only model {repo_id}; upstream link is disabled. "
|
|
2118
|
+
f"Run model-mirror online {repo_id} to re-enable Hub-backed repair."
|
|
2119
|
+
)
|
|
2120
|
+
print_repair_commit_notice(repo_id, result)
|
|
2121
|
+
print(f"{result.status}: {repo_id} -> {result.path}")
|
|
2122
|
+
return 0 if result.status in {"complete", "repaired", "updated"} else 1
|
|
2123
|
+
|
|
2124
|
+
|
|
2125
|
+
def verify_one(config: Config, repo_id: str, args, *, hub=None) -> int:
|
|
2126
|
+
selected_hub = hub or HuggingFaceHub(config)
|
|
2127
|
+
root = archive_path(config, repo_id, args.repo_type)
|
|
2128
|
+
with ModelLock(root, "verify", repo_id, args.repo_type):
|
|
2129
|
+
return verify_one_locked(config, repo_id, args, selected_hub, root)
|
|
2130
|
+
|
|
2131
|
+
|
|
2132
|
+
def verify_one_locked(config: Config, repo_id: str, args, selected_hub, root: Path) -> int:
|
|
2133
|
+
existing_state = read_verification_state(root)
|
|
2134
|
+
requested_revision = selected_revision_arg(args) or (
|
|
2135
|
+
existing_state.requested_revision if existing_state else config.revision
|
|
2136
|
+
)
|
|
2137
|
+
|
|
2138
|
+
if args.offline:
|
|
2139
|
+
return verify_one_offline(config, root, repo_id, args, existing_state, mode="offline")
|
|
2140
|
+
if existing_state is not None and existing_state.offline_only:
|
|
2141
|
+
return verify_one_offline(config, root, repo_id, args, existing_state, mode="offline-only")
|
|
2142
|
+
|
|
2143
|
+
try:
|
|
2144
|
+
upstream_snapshot = get_snapshot(selected_hub, repo_id, args.repo_type, requested_revision)
|
|
2145
|
+
except Exception as exc:
|
|
2146
|
+
return handle_upstream_unavailable(root, repo_id, args, existing_state, requested_revision, exc)
|
|
2147
|
+
resolved_commit = existing_state.resolved_commit if existing_state and existing_state.resolved_commit else upstream_snapshot.resolved_commit
|
|
2148
|
+
try:
|
|
2149
|
+
snapshot = upstream_snapshot if resolved_commit == upstream_snapshot.resolved_commit else get_snapshot(
|
|
2150
|
+
selected_hub, repo_id, args.repo_type, resolved_commit
|
|
2151
|
+
)
|
|
2152
|
+
except Exception as exc:
|
|
2153
|
+
return handle_upstream_unavailable(root, repo_id, args, existing_state, requested_revision, exc)
|
|
2154
|
+
metadata = snapshot.files
|
|
2155
|
+
checksum_result = None
|
|
2156
|
+
manifest_verified = False
|
|
2157
|
+
if not args.cached and config.checksum:
|
|
2158
|
+
if (root / MANIFEST).exists():
|
|
2159
|
+
checksum_result = verify_checksums(root, strict=args.strict)
|
|
2160
|
+
if checksum_result.ok:
|
|
2161
|
+
write_checksums(root, max_workers=config.checksum_workers)
|
|
2162
|
+
manifest_verified = True
|
|
2163
|
+
else:
|
|
2164
|
+
write_checksums(root, max_workers=config.checksum_workers)
|
|
2165
|
+
manifest_verified = True
|
|
2166
|
+
from_manifest = args.cached or manifest_verified
|
|
2167
|
+
result = verify_remote(
|
|
2168
|
+
root,
|
|
2169
|
+
metadata,
|
|
2170
|
+
cached=args.cached,
|
|
2171
|
+
from_manifest=from_manifest,
|
|
2172
|
+
strict=args.strict,
|
|
2173
|
+
)
|
|
2174
|
+
if checksum_result is not None:
|
|
2175
|
+
merge_checksum_result(result, checksum_result)
|
|
2176
|
+
if args.repo_type == "model":
|
|
2177
|
+
audit = audit_model(root, skip_transformers=True)
|
|
2178
|
+
else:
|
|
2179
|
+
audit = None
|
|
2180
|
+
state = state_from_results(
|
|
2181
|
+
repo_id,
|
|
2182
|
+
args.repo_type,
|
|
2183
|
+
requested_revision,
|
|
2184
|
+
result,
|
|
2185
|
+
audit,
|
|
2186
|
+
resolved_commit=resolved_commit,
|
|
2187
|
+
upstream_commit=upstream_snapshot.resolved_commit,
|
|
2188
|
+
)
|
|
2189
|
+
write_verification_state(root, state)
|
|
2190
|
+
|
|
2191
|
+
if state.status == "incomplete":
|
|
2192
|
+
print(f"cached verification incomplete: {repo_id}{upstream_change_suffix(state)}")
|
|
2193
|
+
print(f"run full verification: model-mirror verify {repo_id}")
|
|
2194
|
+
if state.upstream_status == "changed":
|
|
2195
|
+
print_update_next_step(repo_id)
|
|
2196
|
+
return 1
|
|
2197
|
+
if audit is not None and not audit.ok:
|
|
2198
|
+
print(f"verification failed: {repo_id}{upstream_change_suffix(state)}")
|
|
2199
|
+
print_verification_next_steps(repo_id, state)
|
|
2200
|
+
return 1
|
|
2201
|
+
if result.ok:
|
|
2202
|
+
mode = "cached" if args.cached else "full"
|
|
2203
|
+
print(f"verified ({mode}): {repo_id}{upstream_change_suffix(state)}")
|
|
2204
|
+
if state.upstream_status == "changed":
|
|
2205
|
+
print_update_next_step(repo_id)
|
|
2206
|
+
return 0
|
|
2207
|
+
print(f"verification failed: {repo_id}{upstream_change_suffix(state)}")
|
|
2208
|
+
print_verification_next_steps(repo_id, state)
|
|
2209
|
+
return 1
|
|
2210
|
+
|
|
2211
|
+
|
|
2212
|
+
def upstream_change_suffix(state) -> str:
|
|
2213
|
+
return " upstream=changed" if state.upstream_status == "changed" else ""
|
|
2214
|
+
|
|
2215
|
+
|
|
2216
|
+
def print_verification_next_steps(repo_id: str, state) -> None:
|
|
2217
|
+
if state.repair_paths:
|
|
2218
|
+
print(f"next: model-mirror repair {repo_id}")
|
|
2219
|
+
if state.upstream_status == "changed":
|
|
2220
|
+
print_update_next_step(repo_id)
|
|
2221
|
+
|
|
2222
|
+
|
|
2223
|
+
def print_update_next_step(repo_id: str) -> None:
|
|
2224
|
+
print(f"update changed upstream: model-mirror repair --update {repo_id}")
|
|
2225
|
+
|
|
2226
|
+
|
|
2227
|
+
def print_repair_commit_notice(repo_id: str, result) -> None:
|
|
2228
|
+
if result.upstream_status == "changed":
|
|
2229
|
+
print(
|
|
2230
|
+
f"upstream changed: {repo_id} local={result.resolved_commit} "
|
|
2231
|
+
f"upstream={result.upstream_commit} not_applied"
|
|
2232
|
+
)
|
|
2233
|
+
|
|
2234
|
+
|
|
2235
|
+
def verify_one_offline(config: Config, root: Path, repo_id: str, args, existing_state, *, mode: str) -> int:
|
|
2236
|
+
if existing_state is None:
|
|
2237
|
+
print(f"{mode} verification unavailable: {repo_id}")
|
|
2238
|
+
return 1
|
|
2239
|
+
if args.cached:
|
|
2240
|
+
print(f"verified ({mode} cached): {repo_id} state={existing_state.status}")
|
|
2241
|
+
return 0 if existing_state.clean else 1
|
|
2242
|
+
if not (root / MANIFEST).exists():
|
|
2243
|
+
state = local_incomplete_state(repo_id, args.repo_type, existing_state, f"{MANIFEST} missing")
|
|
2244
|
+
write_verification_state(root, state)
|
|
2245
|
+
print(f"{mode} verification incomplete: {repo_id} missing {MANIFEST}")
|
|
2246
|
+
return 1
|
|
2247
|
+
result = verify_checksums(root, strict=args.strict)
|
|
2248
|
+
remote_result = RemoteVerifyResult(
|
|
2249
|
+
missing=result.missing,
|
|
2250
|
+
hash_mismatches=result.failures,
|
|
2251
|
+
extras=result.extras,
|
|
2252
|
+
)
|
|
2253
|
+
state = state_from_results(
|
|
2254
|
+
repo_id,
|
|
2255
|
+
args.repo_type,
|
|
2256
|
+
existing_state.requested_revision,
|
|
2257
|
+
remote_result,
|
|
2258
|
+
resolved_commit=existing_state.resolved_commit,
|
|
2259
|
+
upstream_commit=existing_state.upstream_commit,
|
|
2260
|
+
offline_only=existing_state.offline_only,
|
|
2261
|
+
)
|
|
2262
|
+
write_verification_state(root, state)
|
|
2263
|
+
if result.ok:
|
|
2264
|
+
print(f"verified ({mode} full): {repo_id}")
|
|
2265
|
+
return 0
|
|
2266
|
+
print(f"{mode} verification failed: {repo_id}")
|
|
2267
|
+
if existing_state.offline_only:
|
|
2268
|
+
print(f"repair unavailable for offline-only model: {repo_id}")
|
|
2269
|
+
else:
|
|
2270
|
+
print_verification_next_steps(repo_id, state)
|
|
2271
|
+
return 1
|
|
2272
|
+
|
|
2273
|
+
|
|
2274
|
+
def local_incomplete_state(repo_id: str, repo_type: str, existing_state, issue: str) -> VerificationState:
|
|
2275
|
+
return VerificationState(
|
|
2276
|
+
status="incomplete",
|
|
2277
|
+
repo_id=repo_id,
|
|
2278
|
+
repo_type=repo_type,
|
|
2279
|
+
requested_revision=existing_state.requested_revision,
|
|
2280
|
+
resolved_commit=existing_state.resolved_commit,
|
|
2281
|
+
upstream_commit=existing_state.upstream_commit,
|
|
2282
|
+
upstream_status=existing_state.upstream_status,
|
|
2283
|
+
offline_only=existing_state.offline_only,
|
|
2284
|
+
repair_paths=[],
|
|
2285
|
+
issues=[issue],
|
|
2286
|
+
)
|
|
2287
|
+
|
|
2288
|
+
|
|
2289
|
+
def handle_upstream_unavailable(
|
|
2290
|
+
root: Path,
|
|
2291
|
+
repo_id: str,
|
|
2292
|
+
args,
|
|
2293
|
+
existing_state,
|
|
2294
|
+
requested_revision: str,
|
|
2295
|
+
exc: Exception,
|
|
2296
|
+
) -> int:
|
|
2297
|
+
issue = f"upstream unavailable: {exc}"
|
|
2298
|
+
if existing_state is not None:
|
|
2299
|
+
issues = [item for item in existing_state.issues if not str(item).startswith("upstream unavailable:")]
|
|
2300
|
+
issues.append(issue)
|
|
2301
|
+
state = VerificationState(
|
|
2302
|
+
status=existing_state.status,
|
|
2303
|
+
repo_id=repo_id,
|
|
2304
|
+
repo_type=args.repo_type,
|
|
2305
|
+
requested_revision=requested_revision,
|
|
2306
|
+
resolved_commit=existing_state.resolved_commit,
|
|
2307
|
+
upstream_commit=existing_state.upstream_commit,
|
|
2308
|
+
upstream_status=existing_state.upstream_status,
|
|
2309
|
+
offline_only=existing_state.offline_only,
|
|
2310
|
+
repair_paths=existing_state.repair_paths,
|
|
2311
|
+
issues=issues,
|
|
2312
|
+
checked_at_utc=existing_state.checked_at_utc,
|
|
2313
|
+
)
|
|
2314
|
+
else:
|
|
2315
|
+
state = VerificationState(
|
|
2316
|
+
status="unavailable",
|
|
2317
|
+
repo_id=repo_id,
|
|
2318
|
+
repo_type=args.repo_type,
|
|
2319
|
+
requested_revision=requested_revision,
|
|
2320
|
+
issues=[issue],
|
|
2321
|
+
)
|
|
2322
|
+
write_verification_state(root, state)
|
|
2323
|
+
print(f"verification failed: {repo_id} upstream repository unavailable: {exc}")
|
|
2324
|
+
print(
|
|
2325
|
+
f"if the source repository is no longer available and you want to keep this local mirror, "
|
|
2326
|
+
f"run: model-mirror offline {repo_id}"
|
|
2327
|
+
)
|
|
2328
|
+
return 1
|
|
2329
|
+
|
|
2330
|
+
|
|
2331
|
+
def handle_offline_mode(args, config: Config, *, offline_only: bool) -> int:
|
|
2332
|
+
command = "offline" if offline_only else "online"
|
|
2333
|
+
root = archive_path(config, args.model, args.repo_type)
|
|
2334
|
+
with ModelLock(root, command, args.model, args.repo_type):
|
|
2335
|
+
state = read_verification_state(root)
|
|
2336
|
+
if state is None:
|
|
2337
|
+
print(f"verification state unavailable: {args.model}")
|
|
2338
|
+
print(f"run verify first: model-mirror verify {args.model}")
|
|
2339
|
+
return 1
|
|
2340
|
+
state.offline_only = offline_only
|
|
2341
|
+
if offline_only:
|
|
2342
|
+
state.issues = [item for item in state.issues if not str(item).startswith("upstream unavailable:")]
|
|
2343
|
+
if state.status == "unavailable":
|
|
2344
|
+
state.status = "incomplete"
|
|
2345
|
+
state.issues.append("local verification required")
|
|
2346
|
+
write_verification_state(root, state)
|
|
2347
|
+
if offline_only:
|
|
2348
|
+
print(f"offline-only enabled: {args.model}")
|
|
2349
|
+
else:
|
|
2350
|
+
print(f"offline-only disabled: {args.model}")
|
|
2351
|
+
return 0
|
|
2352
|
+
|
|
2353
|
+
|
|
2354
|
+
def list_model_ids(config: Config) -> list[str]:
|
|
2355
|
+
return list_repo_ids(config, "model")
|
|
2356
|
+
|
|
2357
|
+
|
|
2358
|
+
def list_repo_ids(config: Config, repo_type: str) -> list[str]:
|
|
2359
|
+
repos_root = Path(config.directory) / REPO_TYPE_DIRS[repo_type]
|
|
2360
|
+
if not repos_root.exists():
|
|
2361
|
+
return []
|
|
2362
|
+
result = []
|
|
2363
|
+
for owner in sorted(path for path in repos_root.iterdir() if path.is_dir()):
|
|
2364
|
+
for repo in sorted(path for path in owner.iterdir() if path.is_dir()):
|
|
2365
|
+
result.append(f"{owner.name}/{repo.name}")
|
|
2366
|
+
return result
|
|
2367
|
+
|
|
2368
|
+
|
|
2369
|
+
def parse_age(value: str) -> int:
|
|
2370
|
+
units = {"s": 1, "m": 60, "h": 3600, "d": 86400}
|
|
2371
|
+
value = value.strip().lower()
|
|
2372
|
+
if not value:
|
|
2373
|
+
raise ValueError("empty age")
|
|
2374
|
+
unit = value[-1]
|
|
2375
|
+
if unit in units:
|
|
2376
|
+
return int(value[:-1]) * units[unit]
|
|
2377
|
+
return int(value)
|
|
2378
|
+
|
|
2379
|
+
|
|
2380
|
+
def verification_age_seconds(checked_at_utc: str) -> int | None:
|
|
2381
|
+
if not checked_at_utc:
|
|
2382
|
+
return None
|
|
2383
|
+
try:
|
|
2384
|
+
checked = datetime.fromisoformat(checked_at_utc)
|
|
2385
|
+
except ValueError:
|
|
2386
|
+
return None
|
|
2387
|
+
if checked.tzinfo is None:
|
|
2388
|
+
checked = checked.replace(tzinfo=timezone.utc)
|
|
2389
|
+
return max(0, int((datetime.now(timezone.utc) - checked).total_seconds()))
|
|
2390
|
+
|
|
2391
|
+
|
|
2392
|
+
def verification_age_label(checked_at_utc: str) -> str:
|
|
2393
|
+
seconds = verification_age_seconds(checked_at_utc)
|
|
2394
|
+
return format_age_seconds(seconds)
|
|
2395
|
+
|
|
2396
|
+
|
|
2397
|
+
def format_age_seconds(seconds: int | None) -> str:
|
|
2398
|
+
if seconds is None:
|
|
2399
|
+
return "unknown"
|
|
2400
|
+
if seconds < 120:
|
|
2401
|
+
return f"{seconds}s"
|
|
2402
|
+
minutes = seconds // 60
|
|
2403
|
+
if minutes < 120:
|
|
2404
|
+
return f"{minutes}m"
|
|
2405
|
+
hours = minutes // 60
|
|
2406
|
+
if hours < 48:
|
|
2407
|
+
return f"{hours}h"
|
|
2408
|
+
return f"{hours // 24}d"
|
|
2409
|
+
|
|
2410
|
+
|
|
2411
|
+
def print_verification_age(config: Config, repo_id: str, repo_type: str) -> None:
|
|
2412
|
+
root = archive_path(config, repo_id, repo_type)
|
|
2413
|
+
state = read_verification_state(root)
|
|
2414
|
+
if state is None:
|
|
2415
|
+
print("verification age: unavailable")
|
|
2416
|
+
return
|
|
2417
|
+
age = verification_age_seconds(state.checked_at_utc)
|
|
2418
|
+
if age is None:
|
|
2419
|
+
path = verification_state_path(root)
|
|
2420
|
+
modified = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc)
|
|
2421
|
+
age = max(0, int((datetime.now(timezone.utc) - modified).total_seconds()))
|
|
2422
|
+
print(f"verification age: {format_age_seconds(age)}")
|
|
2423
|
+
if age is not None and age > 24 * 60 * 60:
|
|
2424
|
+
print("warning: verification is older than 24h; run verify again for fresh repair paths")
|
|
2425
|
+
|
|
2426
|
+
|
|
2427
|
+
def should_skip_recent_clean(config: Config, repo_id: str, repo_type: str, max_age: str) -> bool:
|
|
2428
|
+
state = read_verification_state(archive_path(config, repo_id, repo_type))
|
|
2429
|
+
if state is None or not state.clean or state_has_upstream_unavailable(state):
|
|
2430
|
+
return False
|
|
2431
|
+
age = verification_age_seconds(state.checked_at_utc)
|
|
2432
|
+
return age is not None and age <= parse_age(max_age)
|
|
2433
|
+
|
|
2434
|
+
|
|
2435
|
+
if __name__ == "__main__":
|
|
2436
|
+
raise SystemExit(main())
|