model-mirror-cli 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
model_mirror/cli.py ADDED
@@ -0,0 +1,2436 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import json
5
+ import os
6
+ import shutil
7
+ import subprocess
8
+ import sys
9
+ import time
10
+ from dataclasses import dataclass
11
+ from datetime import datetime, timezone
12
+ from pathlib import Path
13
+
14
+ import yaml
15
+
16
+ from . import __version__
17
+ from .audit import audit_model
18
+ from .checksums import MANIFEST, iter_payload_files, load_manifest, write_checksums, verify_checksums
19
+ from .config import (
20
+ Config,
21
+ REPO_TYPE_DIRS,
22
+ archive_path,
23
+ archive_runtime_cache_path,
24
+ archive_runtime_tmp_path,
25
+ load_config,
26
+ parse_bool,
27
+ parse_nonnegative_int,
28
+ parse_positive_int,
29
+ save_config,
30
+ )
31
+ from .hub import HuggingFaceHub, get_snapshot, read_snapshot_plan
32
+ from .lock import ModelBusyError, ModelLock, lock_label, read_active_lock
33
+ from .mirror import mirror
34
+ from .progress import ProgressEntry, ProgressSnapshot, progress_snapshot
35
+ from .repair import repair
36
+ from .removal import (
37
+ REMOVALS_DIR,
38
+ RemovalRecord,
39
+ complete_removal,
40
+ prune_empty_removal_parents,
41
+ read_removal_record,
42
+ removal_path,
43
+ removal_record_path,
44
+ stage_removal,
45
+ write_removal_record,
46
+ )
47
+ from .state import (
48
+ VerificationState,
49
+ read_verification_state,
50
+ state_from_results,
51
+ verification_state_path,
52
+ write_verification_state,
53
+ )
54
+ from .verify import RemoteVerifyResult, merge_checksum_result, verify_remote
55
+
56
+
57
+ TORRENT_EXPERIMENTAL_NOTICE = (
58
+ "EXPERIMENTAL: torrent interfaces and metadata formats may change before stabilization."
59
+ )
60
+
61
+
62
+ CONFIG_OPTIONS = [
63
+ (
64
+ "directory",
65
+ "MODEL_MIRROR_DIRECTORY",
66
+ "Archive root. Repos are stored below models/, datasets/, or spaces/ under this directory.",
67
+ ),
68
+ ("repo_type", "MODEL_MIRROR_REPO_TYPE", "Default Hugging Face repo type: model, dataset, or space."),
69
+ ("revision", "MODEL_MIRROR_REVISION", "Default revision to mirror or verify, usually main."),
70
+ ("checksum", None, "Whether mirror/repair writes local hash manifest records."),
71
+ (
72
+ "checksum_workers",
73
+ "MODEL_MIRROR_CHECKSUM_WORKERS",
74
+ "Number of files to hash concurrently. Use 1 for HDD-friendly sequential reads.",
75
+ ),
76
+ (
77
+ "download_workers",
78
+ "MODEL_MIRROR_DOWNLOAD_WORKERS",
79
+ "Number of files to download concurrently. Default 1 is HDD-friendly; increase for SSD/NVMe.",
80
+ ),
81
+ (
82
+ "stall_timeout_seconds",
83
+ "MODEL_MIRROR_STALL_TIMEOUT",
84
+ "Abort and retry a file when no local byte progress occurs for this many seconds. "
85
+ "Default 600; set 0 to disable.",
86
+ ),
87
+ (
88
+ "stall_retries",
89
+ "MODEL_MIRROR_STALL_RETRIES",
90
+ "Number of stall-triggered resume retries per file before failing the mirror.",
91
+ ),
92
+ ("verify_after_mirror", None, "Whether mirror runs verification after download unless --no-verify is passed."),
93
+ (
94
+ "hf_xet_high_performance",
95
+ "MODEL_MIRROR_HF_XET_HIGH_PERFORMANCE",
96
+ "Sets HF_XET_HIGH_PERFORMANCE=1. Off by default; use only on high-bandwidth machines "
97
+ "with fast disks and ample memory, typically 64 GB RAM or more.",
98
+ ),
99
+ (
100
+ "hf_xet_reconstruct_write_sequentially",
101
+ "MODEL_MIRROR_HF_XET_RECONSTRUCT_WRITE_SEQUENTIALLY",
102
+ "Uses HDD-friendly Xet reconstruction writes by disabling vectored reconstruction writes where supported.",
103
+ ),
104
+ (
105
+ "hf_xet_num_concurrent_range_gets",
106
+ "MODEL_MIRROR_HF_XET_NUM_CONCURRENT_RANGE_GETS",
107
+ "Limits Xet internal download concurrency. Default 1 is HDD-friendly; increase for SSD/NVMe.",
108
+ ),
109
+ (
110
+ "token_path",
111
+ "MODEL_MIRROR_TOKEN_PATH",
112
+ "Path to a Hugging Face token file. If unset, model-mirror checks HF_TOKEN_PATH, "
113
+ "HF_HOME/token, ~/.cache/huggingface/token, and ~/.huggingface/token. Token contents are never printed.",
114
+ ),
115
+ (
116
+ "cache_dir",
117
+ None,
118
+ "Overrides the Hugging Face cache root; defaults to DIRECTORY/.model-mirror/cache.",
119
+ ),
120
+ (
121
+ "tmp_dir",
122
+ None,
123
+ "Overrides temporary file directory; defaults to DIRECTORY/.model-mirror/tmp.",
124
+ ),
125
+ ]
126
+
127
+
128
+ @dataclass(slots=True)
129
+ class CacheUsage:
130
+ archive_cache: int
131
+ archive_tmp: int
132
+ mirror_cache: int
133
+
134
+ @property
135
+ def total(self) -> int:
136
+ return self.archive_cache + self.archive_tmp + self.mirror_cache
137
+
138
+
139
+ @dataclass(slots=True)
140
+ class MirrorStatusEntry:
141
+ root: Path
142
+ repo_id: str
143
+ repo_type: str
144
+ state: VerificationState | None
145
+ active_lock: dict | None
146
+ payload_files: int | None
147
+ payload_size: int | None
148
+ payload_source: str
149
+ expected_files: int | None
150
+ snapshot_commit: str
151
+ snapshot_requested_revision: str
152
+ progress: ProgressSnapshot
153
+ torrent_status: str | None
154
+ upstream_observation: UpstreamObservation | None = None
155
+
156
+
157
+ @dataclass(frozen=True, slots=True)
158
+ class UpstreamObservation:
159
+ status: str
160
+ commit: str | None
161
+ observed_at_utc: str
162
+ error: str | None = None
163
+
164
+
165
+ def build_parser() -> argparse.ArgumentParser:
166
+ parser = argparse.ArgumentParser(
167
+ prog="model-mirror",
168
+ description="Mirror Hugging Face repositories into local bulk storage and verify their integrity.",
169
+ epilog="Run 'model-mirror COMMAND --help' for command-specific options.",
170
+ )
171
+ parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
172
+ parser.add_argument("--config", help="path to config file; defaults to ~/.model-mirror.yaml")
173
+ subparsers = parser.add_subparsers(dest="command")
174
+ command_parsers: dict[str, argparse.ArgumentParser] = {}
175
+
176
+ def add_command_parser(name: str, **kwargs) -> argparse.ArgumentParser:
177
+ command_parser = subparsers.add_parser(name, **kwargs)
178
+ command_parsers[name] = command_parser
179
+ return command_parser
180
+
181
+ mirror_parser = add_command_parser(
182
+ "mirror",
183
+ help="mirror a Hugging Face repo",
184
+ description="Download a repo at a resolved Hub commit and verify it unless --no-verify is used.",
185
+ epilog="Exit status: 0 when complete or downloaded cleanly; 1 when final verification is not clean.",
186
+ )
187
+ mirror_parser.add_argument("model", metavar="repo", help="Hugging Face repo id, e.g. org/model")
188
+ mirror_parser.add_argument("--repo-type", choices=["model", "dataset", "space"], help="repo kind to mirror")
189
+ add_revision_options(mirror_parser)
190
+ mirror_parser.add_argument("--force", action="store_true", help="download even if the local copy looks complete")
191
+ mirror_parser.add_argument("--no-verify", action="store_true", help="skip verification after download")
192
+ mirror_parser.add_argument(
193
+ "--stall-timeout",
194
+ type=parse_nonnegative_int_arg,
195
+ metavar="SECONDS",
196
+ help="override stall timeout for this mirror; default 600 seconds, 0 disables timeout",
197
+ )
198
+
199
+ remove_parser = add_command_parser(
200
+ "remove",
201
+ help="permanently remove a mirror",
202
+ description=(
203
+ "Show the selected mirror's identity, commit, verification age, file count, and size, "
204
+ "then require confirmation before permanent removal. Interrupted removals are resumable."
205
+ ),
206
+ epilog=(
207
+ "Exit status: 0 when removed or cancelled; 1 when the mirror is missing, busy, unsafe, "
208
+ "has an active torrent publication, or removal is interrupted."
209
+ ),
210
+ )
211
+ remove_parser.add_argument("model", metavar="repo", help="repo id to remove, e.g. org/model")
212
+ remove_parser.add_argument(
213
+ "--repo-type", choices=["model", "dataset", "space"], default="model", help="repo kind to remove"
214
+ )
215
+ remove_parser.add_argument(
216
+ "-y",
217
+ "--yes",
218
+ action="store_true",
219
+ help="skip the interactive repository-name confirmation",
220
+ )
221
+
222
+ verify_parser = add_command_parser(
223
+ "verify",
224
+ help="verify mirrored archives",
225
+ description="Check local files against Hub metadata, local checksums, and model metadata.",
226
+ epilog=(
227
+ "Exit status: 0 when verification is clean; 1 when files are missing, corrupt, extra, busy, "
228
+ "the upstream repository is unavailable, or cached verification data is missing/stale; "
229
+ "2 for command-line errors."
230
+ ),
231
+ )
232
+ verify_parser.add_argument("model", metavar="repo", nargs="?", help="repo id to verify unless --all is used")
233
+ verify_parser.add_argument("--cached", action="store_true", help="verify Hub hashes from cached .manifest rows")
234
+ verify_parser.add_argument("--all", action="store_true", help="verify every mirrored model")
235
+ verify_parser.add_argument(
236
+ "--repo-type", choices=["model", "dataset", "space"], default="model", help="repo kind to verify"
237
+ )
238
+ add_revision_options(verify_parser)
239
+ verify_parser.add_argument("--strict", action="store_true", help="fail on extra local files")
240
+ verify_parser.add_argument("--max-age", help="with --all, skip clean archives verified within this age, e.g. 7d")
241
+ verify_parser.add_argument("--offline", action="store_true", help="verify only against local checksum state")
242
+
243
+ repair_parser = add_command_parser(
244
+ "repair",
245
+ help="repair a mirrored archive",
246
+ description=(
247
+ "Redownload files listed in existing .verification repair paths, "
248
+ "reconcile a stale pinned snapshot plan, then run a final verification. Run verify first."
249
+ ),
250
+ epilog=(
251
+ "Exit status: 0 when complete, repaired, or updated cleanly; 1 when verification state is missing, "
252
+ "repair is incomplete, cached verification data is incomplete, or a model is busy; "
253
+ "2 for command-line errors."
254
+ ),
255
+ )
256
+ repair_parser.add_argument("model", metavar="repo", nargs="?", help="repo id to repair unless --all is used")
257
+ repair_parser.add_argument("--all", action="store_true", help="repair every mirrored model with verification state")
258
+ repair_parser.add_argument(
259
+ "--update",
260
+ action="store_true",
261
+ help="apply upstream commit changes recorded by verify before repairing",
262
+ )
263
+ repair_parser.add_argument(
264
+ "--force-partial",
265
+ action="store_true",
266
+ help=(
267
+ "attempt repair even when cached verification data for untouched files is incomplete; "
268
+ "may leave the repository inconsistent"
269
+ ),
270
+ )
271
+ repair_parser.add_argument(
272
+ "--repo-type", choices=["model", "dataset", "space"], default="model", help="repo kind to repair"
273
+ )
274
+
275
+ upgrade_parser = add_command_parser(
276
+ "upgrade",
277
+ help="EXPERIMENTAL: add complete torrent hash coverage to existing archives",
278
+ description=(
279
+ f"{TORRENT_EXPERIMENTAL_NOTICE} "
280
+ "Read only payload files missing reusable torrent hash coverage. "
281
+ "Payload bytes and the pinned resolved commit are not changed."
282
+ ),
283
+ epilog="Exit status: 0 when coverage is complete or the dry-run succeeds; 1 on an unreadable archive or busy model.",
284
+ )
285
+ upgrade_parser.add_argument("model", metavar="repo", nargs="?", help="repo id to upgrade unless --all is used")
286
+ upgrade_parser.add_argument("--all", action="store_true", help="upgrade every mirrored repository of this type")
287
+ upgrade_parser.add_argument("--dry-run", action="store_true", help="report required reads without changing metadata")
288
+ upgrade_parser.add_argument(
289
+ "--repo-type", choices=["model", "dataset", "space"], default="model", help="repo kind to upgrade"
290
+ )
291
+
292
+ torrent_parser = add_command_parser(
293
+ "torrent",
294
+ help="EXPERIMENTAL: publish, seed, inspect, or retire commit-pinned torrents",
295
+ description=(
296
+ f"{TORRENT_EXPERIMENTAL_NOTICE} "
297
+ "Manage client-independent torrent artifacts and model-mirror's durable seed intent."
298
+ ),
299
+ )
300
+ torrent_subparsers = torrent_parser.add_subparsers(dest="torrent_command")
301
+
302
+ def add_torrent_repo_command(name: str, help_text: str) -> argparse.ArgumentParser:
303
+ command = torrent_subparsers.add_parser(
304
+ name,
305
+ help=help_text,
306
+ description=f"{TORRENT_EXPERIMENTAL_NOTICE} {help_text.capitalize()}.",
307
+ )
308
+ command.add_argument("model", metavar="repo", help="repo id, e.g. org/model")
309
+ command.add_argument(
310
+ "--repo-type",
311
+ choices=["model", "dataset", "space"],
312
+ default="model",
313
+ help="repo kind",
314
+ )
315
+ return command
316
+
317
+ create_torrent_parser = add_torrent_repo_command(
318
+ "create",
319
+ "create an ordinary .torrent and recovery record without requesting managed seeding",
320
+ )
321
+ create_torrent_parser.add_argument(
322
+ "--external",
323
+ action="store_true",
324
+ help="record that runtime seeding is owned by an external client",
325
+ )
326
+ publish_torrent_parser = add_torrent_repo_command(
327
+ "publish",
328
+ "create or reuse a publication and request durable seeding",
329
+ )
330
+ publish_torrent_parser.add_argument(
331
+ "--external",
332
+ action="store_true",
333
+ help="emit external-client handoff instead of managed seed intent",
334
+ )
335
+ add_torrent_repo_command("show", "show publication, trust, fence, and backend state")
336
+ add_torrent_repo_command("stop", "stop desired managed seeding without releasing the update fence")
337
+ add_torrent_repo_command("retire", "remove desired seeding and release the update fence")
338
+ handoff_torrent_parser = torrent_subparsers.add_parser(
339
+ "handoff",
340
+ help="print a standard-client download destination and exact import command",
341
+ description=(
342
+ f"{TORRENT_EXPERIMENTAL_NOTICE} "
343
+ "Print a standard-client download destination and exact import command."
344
+ ),
345
+ )
346
+ handoff_torrent_parser.add_argument("torrent_file", type=Path, help="model-mirror .torrent file")
347
+ import_torrent_parser = torrent_subparsers.add_parser(
348
+ "import",
349
+ help="validate and atomically finalize payload downloaded by an external client",
350
+ description=(
351
+ f"{TORRENT_EXPERIMENTAL_NOTICE} "
352
+ "Validate and atomically finalize payload downloaded by an external client."
353
+ ),
354
+ )
355
+ import_torrent_parser.add_argument("torrent_file", type=Path, help="model-mirror .torrent file")
356
+ import_torrent_parser.add_argument("payload_root", type=Path, help="downloaded torrent root directory")
357
+ import_torrent_parser.add_argument(
358
+ "--seed",
359
+ action="store_true",
360
+ help="request managed seeding after successful import",
361
+ )
362
+ join_torrent_parser = torrent_subparsers.add_parser(
363
+ "join",
364
+ help="download a model-mirror torrent or magnet and finalize it as a local archive",
365
+ description=(
366
+ f"{TORRENT_EXPERIMENTAL_NOTICE} "
367
+ "Download a model-mirror torrent or magnet and finalize it as a local archive."
368
+ ),
369
+ )
370
+ join_torrent_parser.add_argument("source", help="path to a .torrent file or a magnet URI")
371
+ join_torrent_parser.add_argument(
372
+ "--seed",
373
+ action="store_true",
374
+ help="request managed seeding after successful join",
375
+ )
376
+ join_torrent_parser.add_argument(
377
+ "--metadata-timeout",
378
+ type=float,
379
+ default=120.0,
380
+ help="seconds to wait for magnet metadata; default 120",
381
+ )
382
+ serve_torrent_parser = torrent_subparsers.add_parser(
383
+ "serve",
384
+ help="run the managed libtorrent backend and reconcile durable seed intent",
385
+ description=(
386
+ f"{TORRENT_EXPERIMENTAL_NOTICE} "
387
+ "Run the managed libtorrent backend and reconcile durable seed intent."
388
+ ),
389
+ )
390
+ serve_torrent_parser.add_argument(
391
+ "--poll-seconds",
392
+ type=float,
393
+ default=2.0,
394
+ help="desired-state reconciliation interval; default 2 seconds",
395
+ )
396
+ serve_torrent_parser.add_argument(
397
+ "--once",
398
+ action="store_true",
399
+ help="perform one reconciliation pass and stop (diagnostic)",
400
+ )
401
+
402
+ offline_parser = add_command_parser(
403
+ "offline",
404
+ help="mark a mirror offline-only",
405
+ description=(
406
+ "Disable Hub checks for one mirrored repo. Use this when the upstream repo is gone "
407
+ "and you want verification to use local state only."
408
+ ),
409
+ epilog="Exit status: 0 when the mirror is marked offline-only; 1 when local verification state is missing or busy.",
410
+ )
411
+ offline_parser.add_argument("model", metavar="repo", help="repo id to mark offline-only")
412
+ offline_parser.add_argument(
413
+ "--repo-type", choices=["model", "dataset", "space"], default="model", help="repo kind to update"
414
+ )
415
+
416
+ online_parser = add_command_parser(
417
+ "online",
418
+ help="re-enable Hub checks for a mirror",
419
+ description="Clear offline-only mode so verify and repair contact the Hub again.",
420
+ epilog="Exit status: 0 when the mirror is marked online; 1 when local verification state is missing or busy.",
421
+ )
422
+ online_parser.add_argument("model", metavar="repo", help="repo id to mark online")
423
+ online_parser.add_argument(
424
+ "--repo-type", choices=["model", "dataset", "space"], default="model", help="repo kind to update"
425
+ )
426
+
427
+ list_parser = add_command_parser(
428
+ "list",
429
+ help="list mirrors in columns or show one mirror in detail",
430
+ description=(
431
+ "Show a compact archive-wide table, or detailed last-known local state when REPO is supplied. "
432
+ "This command is metadata-only and does not contact upstream unless --check-upstream is supplied."
433
+ ),
434
+ )
435
+ status_parser = add_command_parser(
436
+ "status",
437
+ help="show archive status or detailed local state for one mirror",
438
+ description=(
439
+ "Show a compact archive-wide table, or detailed last-known local state when REPO is supplied. "
440
+ "This command is metadata-only and does not contact upstream unless --check-upstream is supplied."
441
+ ),
442
+ )
443
+ for status_like_parser in (list_parser, status_parser):
444
+ status_like_parser.add_argument("model", metavar="repo", nargs="?", help="optional repo id, e.g. org/model")
445
+ status_like_parser.add_argument(
446
+ "--repo-type",
447
+ choices=["model", "dataset", "space"],
448
+ help="repo kind; filters the table or selects the detailed repo",
449
+ )
450
+ status_like_parser.add_argument(
451
+ "--check-upstream",
452
+ action="store_true",
453
+ help="advisory live comparison with upstream; does not update local metadata",
454
+ )
455
+ status_like_parser.add_argument(
456
+ "--verbose",
457
+ action="store_true",
458
+ help="show all recorded metadata fields in the human-readable detail view",
459
+ )
460
+ status_like_parser.add_argument(
461
+ "--json",
462
+ action="store_true",
463
+ dest="json_output",
464
+ help="emit stable structured JSON; --verbose has no effect",
465
+ )
466
+
467
+ clean_cache_parser = add_command_parser(
468
+ "clean-cache",
469
+ help="remove Hugging Face cache and temporary files",
470
+ description=(
471
+ "Remove archive cache, temporary files, and per-mirror Hugging Face metadata caches "
472
+ "without touching mirrored payload files. Without --force, only reports reclaimable space."
473
+ ),
474
+ epilog="Exit status: 0 when cleanup succeeds; 1 when a configured cleanup target is unsafe.",
475
+ )
476
+ clean_cache_parser.add_argument("--force", action="store_true", help="delete cache and temporary files")
477
+
478
+ config_parser = add_command_parser(
479
+ "config",
480
+ help="show or change configuration",
481
+ description="Print configuration, describe supported keys, or persist configuration changes.",
482
+ )
483
+ config_subparsers = config_parser.add_subparsers(dest="config_command")
484
+ config_subparsers.add_parser("show", help="print the resolved configuration")
485
+ config_subparsers.add_parser("options", help="print supported configuration keys with descriptions")
486
+ directory_parser = config_subparsers.add_parser("directory", help="get or set archive directory")
487
+ directory_parser.add_argument("path", nargs="?", help="new archive directory; omit to print current value")
488
+ set_parser = config_subparsers.add_parser("set", help="set a supported configuration key")
489
+ set_parser.add_argument("key", help="configuration key; see 'model-mirror config options'")
490
+ set_parser.add_argument("value", help="new value")
491
+
492
+ help_parser = add_command_parser(
493
+ "help",
494
+ help="show help",
495
+ description="Show full help or command-specific help.",
496
+ )
497
+ help_parser.add_argument("topic", nargs="?", help="optional command to show help for")
498
+ parser.command_parsers = command_parsers
499
+
500
+ return parser
501
+
502
+
503
+ def add_revision_options(parser: argparse.ArgumentParser) -> None:
504
+ revision_group = parser.add_mutually_exclusive_group()
505
+ revision_group.add_argument("--revision", help="branch, tag, or commit to use; defaults to config revision")
506
+ revision_group.add_argument("--commit", help="commit SHA to use")
507
+
508
+
509
+ def parse_nonnegative_int_arg(value: str) -> int:
510
+ try:
511
+ return parse_nonnegative_int(value, default=0)
512
+ except ValueError as exc:
513
+ raise argparse.ArgumentTypeError(str(exc)) from exc
514
+
515
+
516
+ def selected_revision_arg(args) -> str | None:
517
+ return args.commit or args.revision
518
+
519
+
520
+ def main(argv: list[str] | None = None, *, hub=None) -> int:
521
+ raw_argv = list(sys.argv[1:] if argv is None else argv)
522
+ parser = build_parser()
523
+ args = parser.parse_args(raw_argv)
524
+ if args.command is None:
525
+ parser.print_help()
526
+ return 0
527
+ if args.command == "help":
528
+ return handle_help(parser, args.topic)
529
+ config_path = Path(args.config).expanduser() if args.config else None
530
+ config = load_config(config_path)
531
+
532
+ if should_supervise_mirror(args, config, hub):
533
+ return run_supervised_mirror(raw_argv, args, config)
534
+
535
+ try:
536
+ if args.command == "config":
537
+ return handle_config(args, config, config_path)
538
+ if args.command in {"list", "status"}:
539
+ return handle_list(
540
+ config,
541
+ repo_id=args.model,
542
+ repo_type=args.repo_type,
543
+ check_upstream=args.check_upstream,
544
+ verbose=args.verbose,
545
+ json_output=args.json_output,
546
+ hub=hub,
547
+ )
548
+ if args.command == "clean-cache":
549
+ return handle_clean_cache(config, force=args.force)
550
+ if args.command == "mirror":
551
+ selected_hub = hub or HuggingFaceHub(config)
552
+ result = mirror(
553
+ config,
554
+ args.model,
555
+ hub=selected_hub,
556
+ repo_type=args.repo_type,
557
+ revision=selected_revision_arg(args),
558
+ force=args.force,
559
+ verify_after=config.verify_after_mirror and not args.no_verify,
560
+ stall_timeout_seconds=args.stall_timeout,
561
+ )
562
+ print(f"{result.status}: {args.model} -> {result.path}")
563
+ return mirror_exit_code(result.status)
564
+ if args.command == "remove":
565
+ return handle_remove(args, config)
566
+ if args.command == "verify":
567
+ return handle_verify(args, config, hub=hub)
568
+ if args.command == "repair":
569
+ return handle_repair(args, config, hub=hub)
570
+ if args.command == "upgrade":
571
+ return handle_upgrade(args, config)
572
+ if args.command == "torrent":
573
+ return handle_torrent(args, config)
574
+ if args.command == "offline":
575
+ return handle_offline_mode(args, config, offline_only=True)
576
+ if args.command == "online":
577
+ return handle_offline_mode(args, config, offline_only=False)
578
+ except ModelBusyError as exc:
579
+ print(str(exc))
580
+ return 1
581
+
582
+ parser.error(f"Unhandled command: {args.command}")
583
+ return 2
584
+
585
+
586
+ def should_supervise_mirror(args, config: Config, hub) -> bool:
587
+ if args.command != "mirror":
588
+ return False
589
+ if hub is not None:
590
+ return False
591
+ if os.environ.get("MODEL_MIRROR_SUPERVISED_CHILD") == "1":
592
+ return False
593
+ return effective_stall_timeout(args, config) > 0
594
+
595
+
596
+ def effective_stall_timeout(args, config: Config) -> int:
597
+ selected = getattr(args, "stall_timeout", None)
598
+ return config.stall_timeout_seconds if selected is None else selected
599
+
600
+
601
+ def run_supervised_mirror(raw_argv: list[str], args, config: Config) -> int:
602
+ repo_type = args.repo_type or config.repo_type
603
+ root = archive_path(config, args.model, repo_type)
604
+ timeout = effective_stall_timeout(args, config)
605
+ retry_limit = config.stall_retries
606
+ retry_counts: dict[str, int] = {}
607
+ command = [sys.executable, "-m", "model_mirror.cli", *raw_argv]
608
+ env = dict(os.environ)
609
+ env["MODEL_MIRROR_SUPERVISED_CHILD"] = "1"
610
+
611
+ while True:
612
+ child = subprocess.Popen(command, env=env)
613
+ restart, returncode = supervise_child(child, root, timeout, retry_limit, retry_counts)
614
+ if restart:
615
+ continue
616
+ return returncode
617
+
618
+
619
+ def supervise_child(
620
+ child,
621
+ root: Path,
622
+ timeout: int,
623
+ retry_limit: int,
624
+ retry_counts: dict[str, int],
625
+ ) -> tuple[bool, int]:
626
+ started = time.monotonic()
627
+ started_at_utc = datetime.now(timezone.utc)
628
+ poll_interval = min(30.0, max(1.0, timeout / 10))
629
+ while True:
630
+ returncode = child.poll()
631
+ if returncode is not None:
632
+ return False, returncode
633
+
634
+ snapshot = progress_snapshot(root, stall_timeout_seconds=timeout)
635
+ fresh_entries = fresh_progress_entries(snapshot.entries, started_at_utc)
636
+ stalled_entries = [entry for entry in fresh_entries if entry.stalled]
637
+ if stalled_entries:
638
+ entry = selected_progress_entry(stalled_entries)
639
+ path = entry.path
640
+ retry_counts[path] = retry_counts.get(path, 0) + 1
641
+ if retry_counts[path] > retry_limit:
642
+ print(
643
+ f"stall retry limit exceeded for {path}: "
644
+ f"{retry_counts[path] - 1}/{retry_limit}; terminating mirror child"
645
+ )
646
+ terminate_child(child)
647
+ return False, 1
648
+ print(
649
+ f"stall detected for {path}: idle={format_age_seconds(entry.idle_seconds)}; "
650
+ f"restarting mirror child ({retry_counts[path]}/{retry_limit})"
651
+ )
652
+ terminate_child(child)
653
+ return True, 0
654
+
655
+ if not fresh_entries and time.monotonic() - started >= timeout:
656
+ retry_counts["<no-progress>"] = retry_counts.get("<no-progress>", 0) + 1
657
+ if retry_counts["<no-progress>"] > retry_limit:
658
+ print("stall retry limit exceeded before progress was reported; terminating mirror child")
659
+ terminate_child(child)
660
+ return False, 1
661
+ print(
662
+ "stall detected before progress was reported; "
663
+ f"restarting mirror child ({retry_counts['<no-progress>']}/{retry_limit})"
664
+ )
665
+ terminate_child(child)
666
+ return True, 0
667
+
668
+ time.sleep(poll_interval)
669
+
670
+
671
+ def fresh_progress_entries(entries: list[ProgressEntry], started_at_utc: datetime) -> list[ProgressEntry]:
672
+ return [entry for entry in entries if progress_entry_updated_after(entry, started_at_utc)]
673
+
674
+
675
+ def progress_entry_updated_after(entry: ProgressEntry, cutoff: datetime) -> bool:
676
+ try:
677
+ updated = datetime.fromisoformat(entry.updated_at_utc)
678
+ except ValueError:
679
+ return False
680
+ if updated.tzinfo is None:
681
+ updated = updated.replace(tzinfo=timezone.utc)
682
+ return updated >= cutoff
683
+
684
+
685
+ def terminate_child(child) -> None:
686
+ child.terminate()
687
+ try:
688
+ child.wait(timeout=30)
689
+ except subprocess.TimeoutExpired:
690
+ child.kill()
691
+ child.wait(timeout=30)
692
+
693
+
694
+ def handle_help(parser: argparse.ArgumentParser, topic: str | None) -> int:
695
+ if topic is None:
696
+ parser.print_help()
697
+ return 0
698
+ command_parser = parser.command_parsers.get(topic)
699
+ if command_parser is None:
700
+ parser.error(f"Unknown help topic: {topic}")
701
+ command_parser.print_help()
702
+ return 0
703
+
704
+
705
+ def mirror_exit_code(status: str) -> int:
706
+ return 0 if status in {"complete", "downloaded"} else 1
707
+
708
+
709
+ def handle_config(args, config: Config, config_path: Path | None) -> int:
710
+ if args.config_command in {None, "options"}:
711
+ return handle_config_options(config)
712
+
713
+ if args.config_command == "show":
714
+ print(f"directory: {config.directory}")
715
+ print(f"repo_type: {config.repo_type}")
716
+ print(f"revision: {config.revision}")
717
+ print(f"checksum: {config.checksum}")
718
+ print(f"checksum_workers: {config.checksum_workers}")
719
+ print(f"download_workers: {config.download_workers}")
720
+ print(f"stall_timeout_seconds: {config.stall_timeout_seconds}")
721
+ print(f"stall_retries: {config.stall_retries}")
722
+ print(f"verify_after_mirror: {config.verify_after_mirror}")
723
+ print(f"hf_xet_high_performance: {config.hf_xet_high_performance}")
724
+ print(f"hf_xet_reconstruct_write_sequentially: {config.hf_xet_reconstruct_write_sequentially}")
725
+ if config.hf_xet_num_concurrent_range_gets is not None:
726
+ print(f"hf_xet_num_concurrent_range_gets: {config.hf_xet_num_concurrent_range_gets}")
727
+ if config.token_path:
728
+ print(f"token_path: {config.token_path}")
729
+ return 0
730
+
731
+ if args.config_command == "directory":
732
+ if args.path is None:
733
+ print(config.directory)
734
+ return 0
735
+ config.directory = Path(args.path).expanduser()
736
+ config.directory.mkdir(parents=True, exist_ok=True)
737
+ save_config(config, config_path)
738
+ print(f"directory: {config.directory}")
739
+ return 0
740
+
741
+ if args.config_command == "set":
742
+ set_config_value(config, args.key, args.value)
743
+ save_config(config, config_path)
744
+ print(f"{args.key}: {args.value}")
745
+ return 0
746
+
747
+ return 2
748
+
749
+
750
+ def handle_config_options(config: Config) -> int:
751
+ values = {
752
+ "directory": config.directory,
753
+ "repo_type": config.repo_type,
754
+ "revision": config.revision,
755
+ "checksum": config.checksum,
756
+ "checksum_workers": config.checksum_workers,
757
+ "download_workers": config.download_workers,
758
+ "stall_timeout_seconds": config.stall_timeout_seconds,
759
+ "stall_retries": config.stall_retries,
760
+ "verify_after_mirror": config.verify_after_mirror,
761
+ "hf_xet_high_performance": config.hf_xet_high_performance,
762
+ "hf_xet_reconstruct_write_sequentially": config.hf_xet_reconstruct_write_sequentially,
763
+ "hf_xet_num_concurrent_range_gets": config.hf_xet_num_concurrent_range_gets,
764
+ "token_path": config.token_path,
765
+ "cache_dir": config.cache_dir,
766
+ "tmp_dir": config.tmp_dir,
767
+ }
768
+ for key, env_var, description in CONFIG_OPTIONS:
769
+ print(f"{key}: {values[key]}")
770
+ if env_var:
771
+ print(f" env: {env_var}")
772
+ print(f" {description}")
773
+ return 0
774
+
775
+
776
+ def set_config_value(config: Config, key: str, value: str) -> None:
777
+ normalized = key.replace("-", "_")
778
+ if normalized == "directory":
779
+ config.directory = Path(value).expanduser()
780
+ elif normalized == "repo_type":
781
+ config.repo_type = value
782
+ elif normalized == "revision":
783
+ config.revision = value
784
+ elif normalized == "checksum":
785
+ config.checksum = parse_bool(value)
786
+ elif normalized == "checksum_workers":
787
+ config.checksum_workers = parse_positive_int(value, default=1)
788
+ elif normalized == "download_workers":
789
+ config.download_workers = parse_positive_int(value, default=1)
790
+ elif normalized in {"stall_timeout", "stall_timeout_seconds"}:
791
+ config.stall_timeout_seconds = parse_nonnegative_int(value, default=600)
792
+ elif normalized == "stall_retries":
793
+ config.stall_retries = parse_nonnegative_int(value, default=3)
794
+ elif normalized in {"verify_after_mirror", "audit_after_mirror"}:
795
+ config.verify_after_mirror = parse_bool(value)
796
+ elif normalized == "hf_xet_high_performance":
797
+ config.hf_xet_high_performance = parse_bool(value)
798
+ elif normalized == "hf_xet_reconstruct_write_sequentially":
799
+ config.hf_xet_reconstruct_write_sequentially = parse_bool(value)
800
+ elif normalized == "hf_xet_num_concurrent_range_gets":
801
+ config.hf_xet_num_concurrent_range_gets = parse_positive_int(value, default=16)
802
+ elif normalized == "token_path":
803
+ config.token_path = Path(value).expanduser()
804
+ elif normalized == "cache_dir":
805
+ config.cache_dir = Path(value).expanduser()
806
+ elif normalized == "tmp_dir":
807
+ config.tmp_dir = Path(value).expanduser()
808
+ else:
809
+ raise SystemExit(f"Unsupported config key: {key}")
810
+
811
+
812
+ def handle_list(
813
+ config: Config,
814
+ *,
815
+ repo_id: str | None = None,
816
+ repo_type: str | None = None,
817
+ check_upstream: bool = False,
818
+ verbose: bool = False,
819
+ json_output: bool = False,
820
+ hub=None,
821
+ ) -> int:
822
+ errors: list[dict[str, str]] = []
823
+ if repo_id is not None:
824
+ selected_type = repo_type or config.repo_type
825
+ root = archive_path(config, repo_id, selected_type)
826
+ if not root.is_dir():
827
+ message = f"mirror not found: {selected_type}:{repo_id} -> {root}"
828
+ if json_output:
829
+ errors.append(
830
+ {
831
+ "code": "mirror-not-found",
832
+ "repo_id": repo_id,
833
+ "repo_type": selected_type,
834
+ "message": message,
835
+ }
836
+ )
837
+ print_status_json(config, [], errors)
838
+ else:
839
+ print(message)
840
+ return 1
841
+ entries = [build_mirror_status(config, root, repo_id, selected_type)]
842
+ if check_upstream:
843
+ observe_upstreams(config, entries, hub=hub)
844
+ if json_output:
845
+ print_status_json(config, entries, errors)
846
+ elif verbose:
847
+ print_mirror_detail_verbose(entries[0])
848
+ else:
849
+ print_mirror_detail(entries[0])
850
+ return 0
851
+
852
+ entries = collect_mirror_status(config, repo_type=repo_type)
853
+ if check_upstream:
854
+ observe_upstreams(config, entries, hub=hub)
855
+ if json_output:
856
+ print_status_json(config, entries, errors)
857
+ return 0
858
+
859
+ archive_root = Path(config.directory)
860
+ known_sizes = [entry.payload_size for entry in entries if entry.payload_size is not None]
861
+ unknown_sizes = len(entries) - len(known_sizes)
862
+ summary = f"archive: {archive_root} mirrors={len(entries)} recorded_payload={format_bytes(sum(known_sizes))}"
863
+ if unknown_sizes:
864
+ summary += f" unknown_payload={unknown_sizes}"
865
+ print(summary)
866
+ print_mirror_table(entries)
867
+ return 0
868
+
869
+
870
+ def handle_remove(args, config: Config) -> int:
871
+ root = archive_path(config, args.model, args.repo_type)
872
+ staged = removal_path(config, args.model, args.repo_type)
873
+ removal_control = Path(config.directory) / REMOVALS_DIR
874
+ try:
875
+ with ModelLock(removal_control, "remove", args.model, args.repo_type):
876
+ if staged.exists() or staged.is_symlink():
877
+ return resume_remove(args, config, root, staged)
878
+ if root.is_symlink():
879
+ print(f"remove refused unsafe mirror path: {root}")
880
+ return 1
881
+ if not root.is_dir():
882
+ print(f"mirror not found: {args.repo_type}:{args.model} -> {root}")
883
+ return 1
884
+
885
+ with ModelLock(root, "remove", args.model, args.repo_type):
886
+ marker = removal_record_path(root)
887
+ pending = marker.exists() or marker.is_symlink()
888
+ try:
889
+ record = read_removal_record(root) if pending else None
890
+ except ValueError:
891
+ record = None
892
+ record = record or inspect_removal(root, args.model, args.repo_type)
893
+ if pending:
894
+ print(f"prepared interrupted removal found: {root}")
895
+ print_removal_summary(record)
896
+ publication_error = active_publication_removal_error(root)
897
+ if publication_error is not None:
898
+ marker.unlink(missing_ok=True)
899
+ print(publication_error)
900
+ return 1
901
+ if not args.yes and not confirm_removal(args.model):
902
+ marker.unlink(missing_ok=True)
903
+ print(f"cancelled: {args.repo_type}:{args.model} was not removed")
904
+ return 0
905
+ write_removal_record(root, record)
906
+
907
+ try:
908
+ stage_removal(root, staged, record)
909
+ complete_removal(staged)
910
+ except (OSError, ValueError) as exc:
911
+ print(
912
+ f"removal interrupted: {args.repo_type}:{args.model} -> "
913
+ f"{staged if staged.exists() else root} ({exc}); "
914
+ "rerun the same remove command to resume"
915
+ )
916
+ return 1
917
+ finish_removal_cleanup(config, staged)
918
+ except (OSError, ValueError) as exc:
919
+ print(f"remove failed: {args.repo_type}:{args.model} -> {exc}")
920
+ return 1
921
+
922
+ print(
923
+ f"removed: {args.repo_type}:{args.model} "
924
+ f"({record.payload_files} files, {format_bytes(record.payload_size)})"
925
+ )
926
+ return 0
927
+
928
+
929
+ def resume_remove(args, config: Config, root: Path, staged: Path) -> int:
930
+ if staged.is_symlink() or not staged.is_dir():
931
+ print(f"remove refused unsafe interrupted-removal path: {staged}")
932
+ return 1
933
+ try:
934
+ record = read_removal_record(staged)
935
+ except ValueError:
936
+ record = None
937
+ if record is not None and (
938
+ record.repo_id != args.model or record.repo_type != args.repo_type
939
+ ):
940
+ print(
941
+ f"remove resume refused: removal record identity is "
942
+ f"{record.repo_type}:{record.repo_id}, expected {args.repo_type}:{args.model}"
943
+ )
944
+ return 1
945
+ record = record or inspect_removal(staged, args.model, args.repo_type, original_path=root)
946
+ print(f"interrupted removal found: {staged}")
947
+ print_removal_summary(record)
948
+ if not args.yes and not confirm_removal(args.model):
949
+ print(f"cancelled: interrupted removal remains at {staged}")
950
+ return 0
951
+ try:
952
+ complete_removal(staged)
953
+ except OSError as exc:
954
+ print(
955
+ f"removal still incomplete: {args.repo_type}:{args.model} -> {staged} ({exc}); "
956
+ "rerun the same remove command to resume"
957
+ )
958
+ return 1
959
+
960
+ finish_removal_cleanup(config, staged)
961
+ print(
962
+ f"removed interrupted mirror: {args.repo_type}:{args.model} "
963
+ f"({record.payload_files} original files, {format_bytes(record.payload_size)})"
964
+ )
965
+ if root.exists() or root.is_symlink():
966
+ print(f"note: a newer mirror still exists and was not removed: {root}")
967
+ return 0
968
+
969
+
970
+ def inspect_removal(
971
+ root: Path,
972
+ repo_id: str,
973
+ repo_type: str,
974
+ *,
975
+ original_path: Path | None = None,
976
+ ) -> RemovalRecord:
977
+ payload_files, payload_size = mirror_payload_stats(root)
978
+ state = None
979
+ snapshot = None
980
+ exceptions: list[str] = []
981
+ try:
982
+ state = read_verification_state(root)
983
+ except (OSError, ValueError, yaml.YAMLError):
984
+ exceptions.append("verification-metadata-error")
985
+ try:
986
+ snapshot = read_snapshot_plan(root)
987
+ except (OSError, ValueError):
988
+ exceptions.append("snapshot-metadata-error")
989
+ if state is not None:
990
+ for tag in list_exception_tags(state, None):
991
+ append_unique(exceptions, tag)
992
+ if (
993
+ state is not None
994
+ and state.resolved_commit
995
+ and snapshot is not None
996
+ and snapshot.resolved_commit != state.resolved_commit
997
+ ):
998
+ append_unique(exceptions, "snapshot-stale")
999
+ resolved_commit = (
1000
+ state.resolved_commit
1001
+ if state is not None and state.resolved_commit
1002
+ else snapshot.resolved_commit if snapshot is not None else ""
1003
+ )
1004
+ checked_at = state.checked_at_utc if state is not None else ""
1005
+ return RemovalRecord(
1006
+ repo_id=repo_id,
1007
+ repo_type=repo_type,
1008
+ original_path=str(original_path or root),
1009
+ status=state.status if state is not None else "unverified",
1010
+ exceptions=",".join(exceptions) or "none",
1011
+ resolved_commit=resolved_commit,
1012
+ checked_at_utc=checked_at or "unknown",
1013
+ check_age=verification_age_label(checked_at) if checked_at else "unknown",
1014
+ payload_files=payload_files,
1015
+ payload_size=payload_size,
1016
+ )
1017
+
1018
+
1019
+ def print_removal_summary(record: RemovalRecord) -> None:
1020
+ print("mirror selected for permanent removal:")
1021
+ print(f" repository: {record.repo_id}")
1022
+ print(f" repo_type: {record.repo_type}")
1023
+ print(f" path: {record.original_path}")
1024
+ print(f" status: {record.status}")
1025
+ print(f" exceptions: {record.exceptions}")
1026
+ print(f" resolved_commit: {record.resolved_commit or 'unknown'}")
1027
+ print(f" last_checked: {record.checked_at_utc}")
1028
+ print(f" verification_age: {record.check_age}")
1029
+ print(f" payload_files: {record.payload_files}")
1030
+ print(f" payload_size: {format_bytes(record.payload_size)} ({record.payload_size} bytes)")
1031
+
1032
+
1033
+ def active_publication_removal_error(root: Path) -> str | None:
1034
+ from .torrent_publication import load_fenced_publication
1035
+
1036
+ try:
1037
+ fenced = load_fenced_publication(root)
1038
+ except (OSError, ValueError) as exc:
1039
+ return f"remove blocked: torrent publication state is unreadable: {exc}"
1040
+ if fenced is None:
1041
+ return None
1042
+ record = fenced[0]
1043
+ return (
1044
+ f"remove blocked by active torrent publication {record.publication_id}. "
1045
+ f"Stop any external client if applicable, then run "
1046
+ f"'model-mirror torrent stop --repo-type {record.repo_type} {record.repo_id}' and "
1047
+ f"'model-mirror torrent retire --repo-type {record.repo_type} {record.repo_id}'."
1048
+ )
1049
+
1050
+
1051
+ def confirm_removal(repo_id: str) -> bool:
1052
+ try:
1053
+ response = input(f"Type '{repo_id}' to confirm permanent removal: ")
1054
+ except (EOFError, KeyboardInterrupt):
1055
+ print()
1056
+ return False
1057
+ return response.strip() == repo_id
1058
+
1059
+
1060
+ def finish_removal_cleanup(config: Config, staged: Path) -> None:
1061
+ prune_empty_removal_parents(
1062
+ staged.parent,
1063
+ stop=Path(config.directory) / REMOVALS_DIR,
1064
+ )
1065
+
1066
+
1067
+ def collect_mirror_status(config: Config, *, repo_type: str | None = None) -> list[MirrorStatusEntry]:
1068
+ archive_root = Path(config.directory)
1069
+ entries: list[MirrorStatusEntry] = []
1070
+ selected_types = [repo_type] if repo_type is not None else list(REPO_TYPE_DIRS)
1071
+ for selected_type in selected_types:
1072
+ type_dir = REPO_TYPE_DIRS[selected_type]
1073
+ repo_root = archive_root / type_dir
1074
+ if not repo_root.exists():
1075
+ continue
1076
+ for owner in sorted(path for path in repo_root.iterdir() if path.is_dir()):
1077
+ for repo in sorted(path for path in owner.iterdir() if path.is_dir()):
1078
+ repo_id = f"{owner.name}/{repo.name}"
1079
+ entries.append(build_mirror_status(config, repo, repo_id, selected_type))
1080
+ entries.sort(key=lambda entry: (entry.repo_type, entry.repo_id))
1081
+ return entries
1082
+
1083
+
1084
+ def build_mirror_status(config: Config, root: Path, repo_id: str, repo_type: str) -> MirrorStatusEntry:
1085
+ state = read_verification_state(root)
1086
+ snapshot = read_snapshot_plan(root)
1087
+ payload_files, payload_size, payload_source = recorded_payload_stats(root, snapshot)
1088
+ if (
1089
+ payload_source == "snapshot"
1090
+ and state is not None
1091
+ and state.resolved_commit
1092
+ and snapshot is not None
1093
+ and snapshot.resolved_commit != state.resolved_commit
1094
+ ):
1095
+ payload_files, payload_size, payload_source = None, None, "unknown"
1096
+ expected_files = len(snapshot.files) if snapshot is not None else None
1097
+ return MirrorStatusEntry(
1098
+ root=root,
1099
+ repo_id=repo_id,
1100
+ repo_type=repo_type,
1101
+ state=state,
1102
+ active_lock=read_active_lock(root),
1103
+ payload_files=payload_files,
1104
+ payload_size=payload_size,
1105
+ payload_source=payload_source,
1106
+ expected_files=expected_files,
1107
+ snapshot_commit=snapshot.resolved_commit if snapshot is not None else "",
1108
+ snapshot_requested_revision=snapshot.requested_revision if snapshot is not None else "",
1109
+ progress=progress_snapshot(
1110
+ root,
1111
+ stall_timeout_seconds=config.stall_timeout_seconds,
1112
+ scan_incomplete=False,
1113
+ ),
1114
+ torrent_status=list_torrent_status(root),
1115
+ )
1116
+
1117
+
1118
+ def recorded_payload_stats(root: Path, snapshot) -> tuple[int | None, int | None, str]:
1119
+ manifest_path = root / MANIFEST
1120
+ if manifest_path.exists():
1121
+ manifest = load_manifest(root)
1122
+ sizes = [row.get("size") for row in manifest.values()]
1123
+ total_size = (
1124
+ sum(sizes)
1125
+ if all(isinstance(size, int) and not isinstance(size, bool) and size >= 0 for size in sizes)
1126
+ else None
1127
+ )
1128
+ return len(manifest), total_size, "manifest"
1129
+ if snapshot is not None:
1130
+ sizes = [item.size for item in snapshot.files]
1131
+ total_size = sum(sizes) if all(size is not None and size >= 0 for size in sizes) else None
1132
+ return len(snapshot.files), total_size, "snapshot"
1133
+ return None, None, "unknown"
1134
+
1135
+
1136
+ def observe_upstreams(config: Config, entries: list[MirrorStatusEntry], *, hub=None) -> None:
1137
+ selected_hub = hub or HuggingFaceHub(config)
1138
+ for entry in entries:
1139
+ state = entry.state
1140
+ requested_revision = (
1141
+ state.requested_revision
1142
+ if state is not None and state.requested_revision
1143
+ else entry.snapshot_requested_revision or config.revision
1144
+ )
1145
+ observed_at = datetime.now(timezone.utc).isoformat(timespec="seconds")
1146
+ try:
1147
+ snapshot = get_snapshot(selected_hub, entry.repo_id, entry.repo_type, requested_revision)
1148
+ except Exception as exc:
1149
+ entry.upstream_observation = UpstreamObservation(
1150
+ status="unavailable",
1151
+ commit=None,
1152
+ observed_at_utc=observed_at,
1153
+ error=str(exc),
1154
+ )
1155
+ continue
1156
+ local_commit = resolved_commit_for_status(entry)
1157
+ entry.upstream_observation = UpstreamObservation(
1158
+ status=(
1159
+ "available"
1160
+ if not local_commit
1161
+ else "current"
1162
+ if local_commit == snapshot.resolved_commit
1163
+ else "changed"
1164
+ ),
1165
+ commit=snapshot.resolved_commit,
1166
+ observed_at_utc=observed_at,
1167
+ )
1168
+
1169
+
1170
+ def print_mirror_table(entries: list[MirrorStatusEntry]) -> None:
1171
+ headers = ["TYPE", "REPOSITORY", "FILES", "SIZE", "COMMIT", "CHECKED", "EXCEPTIONS"]
1172
+ include_torrent = any(entry.torrent_status is not None for entry in entries)
1173
+ include_activity = any(entry.progress.active for entry in entries)
1174
+ include_live_upstream = any(entry.upstream_observation is not None for entry in entries)
1175
+ if include_live_upstream:
1176
+ headers.append("UPSTREAM NOW")
1177
+ if include_torrent:
1178
+ headers.append("TORRENT")
1179
+ if include_activity:
1180
+ headers.append("ACTIVITY")
1181
+
1182
+ rows = []
1183
+ for entry in entries:
1184
+ state = entry.state
1185
+ commit = resolved_commit_for_status(entry)
1186
+ row = [
1187
+ entry.repo_type,
1188
+ entry.repo_id,
1189
+ file_count_label(entry),
1190
+ format_optional_bytes(entry.payload_size),
1191
+ shorten_commit(commit),
1192
+ verification_age_label(state.checked_at_utc) if state is not None else "unknown",
1193
+ ",".join(mirror_exception_tags(entry)) or "-",
1194
+ ]
1195
+ if include_live_upstream:
1196
+ row.append(format_upstream_observation(entry.upstream_observation))
1197
+ if include_torrent:
1198
+ row.append(entry.torrent_status or "-")
1199
+ if include_activity:
1200
+ row.append(format_progress_detail(entry.progress) or "-")
1201
+ rows.append(row)
1202
+ print_table(headers, rows, right_aligned={"FILES", "SIZE"})
1203
+
1204
+
1205
+ def print_table(headers: list[str], rows: list[list[str]], *, right_aligned: set[str]) -> None:
1206
+ widths = [
1207
+ max([len(header), *(len(row[index]) for row in rows)])
1208
+ for index, header in enumerate(headers)
1209
+ ]
1210
+
1211
+ def formatted(row: list[str]) -> str:
1212
+ cells = []
1213
+ for index, value in enumerate(row):
1214
+ if headers[index] in right_aligned:
1215
+ cells.append(value.rjust(widths[index]))
1216
+ else:
1217
+ cells.append(value.ljust(widths[index]))
1218
+ return " ".join(cells).rstrip()
1219
+
1220
+ print(formatted(headers))
1221
+ print(formatted(["-" * width for width in widths]))
1222
+ for row in rows:
1223
+ print(formatted(row))
1224
+
1225
+
1226
+ def print_mirror_detail(entry: MirrorStatusEntry) -> None:
1227
+ state = entry.state
1228
+ exceptions = mirror_exception_tags(entry)
1229
+ print(entry.repo_id)
1230
+ print()
1231
+ status = state.status if state is not None else "unverified"
1232
+ checked = verification_age_label(state.checked_at_utc) if state is not None else "unknown"
1233
+ print_status_field("Verification", f"{status} · {checked} ago" if checked != "unknown" else status)
1234
+ commit = resolved_commit_for_status(entry)
1235
+ revision = (
1236
+ state.requested_revision
1237
+ if state is not None and state.requested_revision
1238
+ else entry.snapshot_requested_revision or "unknown"
1239
+ )
1240
+ snapshot = shorten_commit(commit) if commit else "unknown"
1241
+ print_status_field("Commit", f"{snapshot} · {revision}")
1242
+ payload = (
1243
+ f"{entry.payload_files} {'file' if entry.payload_files == 1 else 'files'} · "
1244
+ f"{format_optional_bytes(entry.payload_size)}"
1245
+ if entry.payload_files is not None
1246
+ else "unknown"
1247
+ )
1248
+ print_status_field("Payload", payload)
1249
+ print_status_field("Upstream", human_upstream_status(entry))
1250
+ print_status_field("Torrent", entry.torrent_status or "not published")
1251
+
1252
+ if exceptions:
1253
+ print_status_field("Attention", ", ".join(exceptions))
1254
+ if entry.active_lock is not None:
1255
+ print_status_field("Lock", format_lock_detail(entry.active_lock))
1256
+ progress = format_progress_detail(entry.progress)
1257
+ if progress is not None:
1258
+ print_status_field("Activity", progress)
1259
+
1260
+ next_steps = status_next_steps(entry)
1261
+ for index, step in enumerate(next_steps):
1262
+ print_status_field("Next" if index == 0 else "", step)
1263
+
1264
+ if state is not None and state.repair_paths:
1265
+ print()
1266
+ print("Repair paths")
1267
+ for value in state.repair_paths:
1268
+ print(f" {value}")
1269
+ if state is not None and state.issues:
1270
+ print()
1271
+ print("Issues")
1272
+ for value in state.issues:
1273
+ print(f" {value}")
1274
+
1275
+
1276
+ def print_status_field(label: str, value: str) -> None:
1277
+ print(f" {label:<14}{value}")
1278
+
1279
+
1280
+ def human_upstream_status(entry: MirrorStatusEntry) -> str:
1281
+ observation = entry.upstream_observation
1282
+ if observation is not None:
1283
+ value = f"{observation.status} now"
1284
+ if observation.commit:
1285
+ value += f" · {shorten_commit(observation.commit)}"
1286
+ if observation.error:
1287
+ value += f" · {observation.error}"
1288
+ return value
1289
+ state = entry.state
1290
+ if state is None or not state.upstream_status:
1291
+ return "unknown"
1292
+ suffix = " when last checked" if state.upstream_status != "unknown" else ""
1293
+ return f"{state.upstream_status}{suffix}"
1294
+
1295
+
1296
+ def format_upstream_observation(observation: UpstreamObservation | None) -> str:
1297
+ if observation is None:
1298
+ return "-"
1299
+ if observation.commit:
1300
+ return f"{observation.status}:{shorten_commit(observation.commit)}"
1301
+ return observation.status
1302
+
1303
+
1304
+ def status_next_steps(entry: MirrorStatusEntry) -> list[str]:
1305
+ state = entry.state
1306
+ if state is None:
1307
+ return [f"model-mirror verify {entry.repo_id}"]
1308
+ if primary_state_tag(state) == "unverified":
1309
+ return [f"model-mirror verify {entry.repo_id}"]
1310
+ steps = []
1311
+ if state.repair_paths:
1312
+ steps.append(f"model-mirror repair {entry.repo_id}")
1313
+ if state.upstream_status == "changed":
1314
+ steps.append(f"model-mirror repair --update {entry.repo_id}")
1315
+ return steps
1316
+
1317
+
1318
+ def print_mirror_detail_verbose(entry: MirrorStatusEntry) -> None:
1319
+ state = entry.state
1320
+ exceptions = mirror_exception_tags(entry)
1321
+ print(f"repository: {entry.repo_id}")
1322
+ print(f"repo_type: {entry.repo_type}")
1323
+ print(f"path: {entry.root}")
1324
+ print(f"status: {state.status if state is not None else 'unverified'}")
1325
+ print(f"exceptions: {','.join(exceptions) if exceptions else 'none'}")
1326
+ print(f"last_checked: {state.checked_at_utc if state is not None and state.checked_at_utc else 'unknown'}")
1327
+ print(f"last_check_age: {verification_age_label(state.checked_at_utc) if state is not None else 'unknown'}")
1328
+ print(f"requested_revision: {state.requested_revision if state is not None else 'unknown'}")
1329
+ print(f"resolved_commit: {resolved_commit_for_status(entry) or 'unknown'}")
1330
+ print(f"snapshot_commit: {entry.snapshot_commit or 'unknown'}")
1331
+ print(f"upstream_commit: {state.upstream_commit if state is not None and state.upstream_commit else 'unknown'}")
1332
+ print(f"upstream_status: {state.upstream_status if state is not None else 'unknown'}")
1333
+ print(f"offline_only: {str(state.offline_only).lower() if state is not None else 'unknown'}")
1334
+ print(f"payload_files: {entry.payload_files if entry.payload_files is not None else 'unknown'}")
1335
+ if entry.payload_size is None:
1336
+ print("payload_size: unknown")
1337
+ else:
1338
+ print(f"payload_size: {format_bytes(entry.payload_size)} ({entry.payload_size} bytes)")
1339
+ print(f"payload_source: {entry.payload_source}")
1340
+ if entry.expected_files is None:
1341
+ print("expected_files: unknown")
1342
+ else:
1343
+ stale_suffix = " (snapshot stale)" if snapshot_is_stale(entry) else ""
1344
+ print(f"expected_files: {entry.expected_files} recorded{stale_suffix}")
1345
+ print(f"lock: {format_lock_detail(entry.active_lock) if entry.active_lock is not None else 'none'}")
1346
+ print(f"progress: {format_progress_detail(entry.progress) or 'none'}")
1347
+ print(f"torrent: {entry.torrent_status or 'none'}")
1348
+ observation = entry.upstream_observation
1349
+ if observation is None:
1350
+ print("live_upstream: not checked")
1351
+ else:
1352
+ detail = f"{observation.status} commit={observation.commit or 'unknown'} observed_at={observation.observed_at_utc}"
1353
+ if observation.error:
1354
+ detail += f" error={observation.error}"
1355
+ print(f"live_upstream: {detail}")
1356
+ print_detail_values("repair_paths", state.repair_paths if state is not None else [])
1357
+ print_detail_values("issues", state.issues if state is not None else [])
1358
+
1359
+
1360
+ def print_status_json(
1361
+ config: Config,
1362
+ entries: list[MirrorStatusEntry],
1363
+ errors: list[dict[str, str]],
1364
+ ) -> None:
1365
+ document = {
1366
+ "schema": "model-mirror-status",
1367
+ "version": 1,
1368
+ "archive": str(Path(config.directory)),
1369
+ "repositories": [mirror_status_json(entry) for entry in entries],
1370
+ "errors": errors,
1371
+ }
1372
+ print(json.dumps(document, indent=2, sort_keys=True))
1373
+
1374
+
1375
+ def mirror_status_json(entry: MirrorStatusEntry) -> dict:
1376
+ state = entry.state
1377
+ observation = entry.upstream_observation
1378
+ return {
1379
+ "repo_id": entry.repo_id,
1380
+ "repo_type": entry.repo_type,
1381
+ "path": str(entry.root),
1382
+ "requested_revision": (
1383
+ state.requested_revision
1384
+ if state is not None and state.requested_revision
1385
+ else entry.snapshot_requested_revision or None
1386
+ ),
1387
+ "resolved_commit": resolved_commit_for_status(entry) or None,
1388
+ "last_verification": {
1389
+ "status": state.status if state is not None else "unverified",
1390
+ "checked_at": state.checked_at_utc if state is not None and state.checked_at_utc else None,
1391
+ "offline_only": state.offline_only if state is not None else None,
1392
+ "issues": list(state.issues) if state is not None else [],
1393
+ "repair_paths": list(state.repair_paths) if state is not None else [],
1394
+ },
1395
+ "payload": {
1396
+ "files": entry.payload_files,
1397
+ "bytes": entry.payload_size,
1398
+ "source": entry.payload_source,
1399
+ },
1400
+ "snapshot": {
1401
+ "commit": entry.snapshot_commit or None,
1402
+ "expected_files": entry.expected_files,
1403
+ "stale": snapshot_is_stale(entry),
1404
+ },
1405
+ "upstream": {
1406
+ "recorded": {
1407
+ "status": state.upstream_status if state is not None else "unknown",
1408
+ "commit": state.upstream_commit if state is not None and state.upstream_commit else None,
1409
+ },
1410
+ "live": (
1411
+ {
1412
+ "status": observation.status,
1413
+ "commit": observation.commit,
1414
+ "observed_at": observation.observed_at_utc,
1415
+ "error": observation.error,
1416
+ }
1417
+ if observation is not None
1418
+ else None
1419
+ ),
1420
+ },
1421
+ "exceptions": mirror_exception_tags(entry),
1422
+ "lock": entry.active_lock,
1423
+ "activity": [progress_entry_json(item) for item in entry.progress.entries],
1424
+ "torrent": torrent_status_json(entry),
1425
+ }
1426
+
1427
+
1428
+ def progress_entry_json(entry: ProgressEntry) -> dict:
1429
+ return {
1430
+ "path": entry.path,
1431
+ "stage": entry.stage,
1432
+ "bytes_done": entry.bytes_done,
1433
+ "bytes_total": entry.bytes_total,
1434
+ "updated_at": entry.updated_at_utc or None,
1435
+ "idle_seconds": entry.idle_seconds,
1436
+ "stalled": entry.stalled,
1437
+ "source": entry.source,
1438
+ "rate_bytes_per_second": entry.rate_bytes_per_second,
1439
+ }
1440
+
1441
+
1442
+ def torrent_status_json(entry: MirrorStatusEntry) -> dict | None:
1443
+ if entry.torrent_status is None:
1444
+ return None
1445
+ if entry.torrent_status == "metadata-error":
1446
+ return {"status": "metadata-error"}
1447
+ from .torrent_publication import load_fenced_publication
1448
+
1449
+ try:
1450
+ fenced = load_fenced_publication(entry.root)
1451
+ except (OSError, ValueError):
1452
+ return {"status": "metadata-error"}
1453
+ if fenced is None:
1454
+ return None
1455
+ record = fenced[0]
1456
+ try:
1457
+ coverage = torrent_coverage_status(entry.root)
1458
+ except (OSError, ValueError, RuntimeError):
1459
+ coverage = "metadata-error"
1460
+ return {
1461
+ "publication_id": record.publication_id,
1462
+ "resolved_commit": record.resolved_commit,
1463
+ "profile": record.profile,
1464
+ "lifecycle": record.lifecycle,
1465
+ "client_mode": record.client_mode,
1466
+ "desired_seed": record.desired_seed,
1467
+ "observed_backend": record.observed_backend,
1468
+ "coverage": coverage,
1469
+ "content_verification": record.content_verification,
1470
+ "publication_trust": record.publication_trust,
1471
+ "upstream_provenance": record.upstream_provenance,
1472
+ "upstream_availability": record.upstream_availability,
1473
+ "infohash_v1": record.infohash_v1,
1474
+ "infohash_v2": record.infohash_v2,
1475
+ }
1476
+
1477
+
1478
+ def print_detail_values(label: str, values: list[str]) -> None:
1479
+ if not values:
1480
+ print(f"{label}: none")
1481
+ return
1482
+ print(f"{label}:")
1483
+ for value in values:
1484
+ print(f" - {value}")
1485
+
1486
+
1487
+ def resolved_commit_for_status(entry: MirrorStatusEntry) -> str:
1488
+ if entry.state is not None and entry.state.resolved_commit:
1489
+ return entry.state.resolved_commit
1490
+ return entry.snapshot_commit
1491
+
1492
+
1493
+ def shorten_commit(commit: str) -> str:
1494
+ return commit[:12] if commit else "-"
1495
+
1496
+
1497
+ def file_count_label(entry: MirrorStatusEntry) -> str:
1498
+ return str(entry.payload_files) if entry.payload_files is not None else "?"
1499
+
1500
+
1501
+ def mirror_exception_tags(entry: MirrorStatusEntry) -> list[str]:
1502
+ tags = list_exception_tags(entry.state, entry.active_lock, entry.progress)
1503
+ if snapshot_is_stale(entry):
1504
+ append_unique(tags, "snapshot-stale")
1505
+ marker = removal_record_path(entry.root)
1506
+ if marker.exists() or marker.is_symlink():
1507
+ append_unique(tags, "removal-pending")
1508
+ return tags
1509
+
1510
+
1511
+ def snapshot_is_stale(entry: MirrorStatusEntry) -> bool:
1512
+ return bool(
1513
+ entry.snapshot_commit
1514
+ and entry.state is not None
1515
+ and entry.state.resolved_commit
1516
+ and entry.snapshot_commit != entry.state.resolved_commit
1517
+ )
1518
+
1519
+
1520
+ def list_torrent_status(root: Path) -> str | None:
1521
+ from .torrent_publication import load_fenced_publication
1522
+
1523
+ try:
1524
+ fenced = load_fenced_publication(root)
1525
+ except (OSError, ValueError):
1526
+ return "metadata-error"
1527
+ if fenced is None:
1528
+ return None
1529
+ record = fenced[0]
1530
+ parts = [record.lifecycle, record.client_mode]
1531
+ parts.append(f"desired={'seeding' if record.desired_seed else 'stopped'}")
1532
+ parts.append(f"observed={record.observed_backend}")
1533
+ state = read_verification_state(root)
1534
+ if state is not None and state.upstream_status == "changed":
1535
+ parts.append("update-available")
1536
+ try:
1537
+ coverage_status = torrent_coverage_status(root)
1538
+ except (OSError, ValueError, RuntimeError):
1539
+ coverage_status = "metadata-error"
1540
+ parts.append(f"coverage={coverage_status}")
1541
+ return ",".join(parts)
1542
+
1543
+
1544
+ def torrent_coverage_status(root: Path) -> str:
1545
+ snapshot = read_snapshot_plan(root)
1546
+ if snapshot is None:
1547
+ return "unavailable"
1548
+ from .torrent_coverage import TorrentCoverageRecorder
1549
+
1550
+ recorder = TorrentCoverageRecorder(root, snapshot)
1551
+ return "complete" if recorder.complete else "partial"
1552
+
1553
+
1554
+ def archive_cache_usage(config: Config) -> CacheUsage:
1555
+ archive_root = Path(config.directory)
1556
+ cache_root = archive_runtime_cache_path(config)
1557
+ tmp_root = archive_runtime_tmp_path(config)
1558
+ return CacheUsage(
1559
+ archive_cache=directory_size(cache_root),
1560
+ archive_tmp=directory_size(tmp_root),
1561
+ mirror_cache=sum(directory_size(path) for path in mirror_cache_dirs(archive_root)),
1562
+ )
1563
+
1564
+
1565
+ def handle_clean_cache(config: Config, *, force: bool) -> int:
1566
+ archive_root = Path(config.directory)
1567
+ targets = cleanup_targets(config)
1568
+ unsafe = [path for path, label in targets if not is_safe_cleanup_target(path, archive_root, label)]
1569
+ if unsafe:
1570
+ for path in unsafe:
1571
+ print(f"refusing unsafe cleanup target: {path}")
1572
+ return 1
1573
+
1574
+ sized_targets = [(path, label, directory_size(path)) for path, label in targets if path.exists()]
1575
+ total_size = sum(size for _path, _label, size in sized_targets)
1576
+ print(f"cleanup mode: {'force' if force else 'dry-run'}")
1577
+ print(f"reclaimable={format_bytes(total_size)}")
1578
+ for path, label, size in sized_targets:
1579
+ if force:
1580
+ shutil.rmtree(path)
1581
+ action = "removed"
1582
+ else:
1583
+ action = "would-remove"
1584
+ print(f"{action}: {label} {path} size={format_bytes(size)}")
1585
+ return 0
1586
+
1587
+
1588
+ def cleanup_targets(config: Config) -> list[tuple[Path, str]]:
1589
+ archive_root = Path(config.directory)
1590
+ cache_root = archive_runtime_cache_path(config)
1591
+ tmp_root = archive_runtime_tmp_path(config)
1592
+ targets = [(cache_root, "archive-cache"), (tmp_root, "archive-tmp")]
1593
+ selected = {path.resolve(strict=False) for path, _label in targets}
1594
+ for path, label in (
1595
+ (archive_root / ".cache", "legacy-archive-cache"),
1596
+ (archive_root / ".tmp", "legacy-archive-tmp"),
1597
+ ):
1598
+ if path.resolve(strict=False) not in selected:
1599
+ targets.append((path, label))
1600
+ targets.extend((path, "mirror-cache") for path in mirror_cache_dirs(archive_root))
1601
+ return targets
1602
+
1603
+
1604
+ def mirror_cache_dirs(archive_root: Path) -> list[Path]:
1605
+ targets: list[Path] = []
1606
+ for type_dir in REPO_TYPE_DIRS.values():
1607
+ for repo in sorted((archive_root / type_dir).glob("*/*")):
1608
+ targets.append(repo / ".cache")
1609
+ return targets
1610
+
1611
+
1612
+ def is_safe_cleanup_target(path: Path, archive_root: Path, label: str) -> bool:
1613
+ resolved_path = path.resolve(strict=False)
1614
+ resolved_archive = archive_root.resolve(strict=False)
1615
+ protected = {resolved_archive, *(resolved_archive / type_dir for type_dir in REPO_TYPE_DIRS.values())}
1616
+ expected_names = {
1617
+ "archive-cache": {"cache", ".cache"},
1618
+ "archive-tmp": {"tmp", ".tmp"},
1619
+ "legacy-archive-cache": {".cache"},
1620
+ "legacy-archive-tmp": {".tmp"},
1621
+ "mirror-cache": {".cache"},
1622
+ }
1623
+ return (
1624
+ resolved_path.name in expected_names.get(label, set())
1625
+ and resolved_path.is_relative_to(resolved_archive)
1626
+ and resolved_path not in protected
1627
+ )
1628
+
1629
+
1630
+ def directory_size(path: Path) -> int:
1631
+ if not path.exists():
1632
+ return 0
1633
+ total_size = 0
1634
+ for child in path.rglob("*"):
1635
+ if child.is_file():
1636
+ total_size += child.stat().st_size
1637
+ return total_size
1638
+
1639
+
1640
+ def mirror_payload_stats(root: Path) -> tuple[int, int]:
1641
+ files = 0
1642
+ total_size = 0
1643
+ for path in iter_payload_files(root):
1644
+ files += 1
1645
+ total_size += path.stat().st_size
1646
+ return files, total_size
1647
+
1648
+
1649
+ def format_bytes(size: int) -> str:
1650
+ value = float(size)
1651
+ for unit in ("B", "KiB", "MiB", "GiB", "TiB", "PiB"): # pragma: no branch
1652
+ if abs(value) < 1024 or unit == "PiB":
1653
+ if unit == "B":
1654
+ return f"{int(value)} B"
1655
+ return f"{value:.1f} {unit}"
1656
+ value /= 1024
1657
+
1658
+
1659
+ def format_optional_bytes(size: int | None) -> str:
1660
+ return format_bytes(size) if size is not None else "unknown"
1661
+
1662
+
1663
+ def format_lock_detail(info: dict | None) -> str:
1664
+ if not info:
1665
+ return "lock held"
1666
+ parts = []
1667
+ for key in ("command", "pid", "host", "started_at_utc"):
1668
+ value = info.get(key)
1669
+ if value:
1670
+ parts.append(f"{key}={value}")
1671
+ return " ".join(parts) if parts else "lock held"
1672
+
1673
+
1674
+ def format_progress_detail(progress: ProgressSnapshot) -> str | None:
1675
+ if not progress.active:
1676
+ return None
1677
+ entry = selected_progress_entry(progress.entries)
1678
+ parts = [
1679
+ f"active={len(progress.entries)}",
1680
+ f"path={shorten_path(entry.path)}",
1681
+ f"stage={entry.stage}",
1682
+ f"bytes={format_progress_bytes(entry)}",
1683
+ ]
1684
+ rate = format_rate(entry.rate_bytes_per_second)
1685
+ if rate is not None:
1686
+ parts.append(f"rate={rate}")
1687
+ if entry.idle_seconds is not None:
1688
+ parts.append(f"idle={format_age_seconds(entry.idle_seconds)}")
1689
+ if progress.stalled_count:
1690
+ parts.append(f"stalled={progress.stalled_count}")
1691
+ if entry.source != "heartbeat":
1692
+ parts.append(f"source={entry.source}")
1693
+ return " ".join(parts)
1694
+
1695
+
1696
+ def selected_progress_entry(entries: list[ProgressEntry]) -> ProgressEntry:
1697
+ stalled = [entry for entry in entries if entry.stalled]
1698
+ if stalled:
1699
+ return max(stalled, key=lambda entry: entry.idle_seconds or 0)
1700
+ return sorted(entries, key=lambda entry: entry.path)[0]
1701
+
1702
+
1703
+ def format_progress_bytes(entry: ProgressEntry) -> str:
1704
+ if entry.bytes_total:
1705
+ percent = (entry.bytes_done / entry.bytes_total) * 100
1706
+ return f"{format_bytes(entry.bytes_done)}/{format_bytes(entry.bytes_total)}({percent:.1f}%)"
1707
+ return format_bytes(entry.bytes_done)
1708
+
1709
+
1710
+ def format_rate(rate: float | None) -> str | None:
1711
+ if rate is None:
1712
+ return None
1713
+ return f"{format_bytes(int(rate))}/s"
1714
+
1715
+
1716
+ def shorten_path(path: str, *, max_length: int = 48) -> str:
1717
+ if len(path) <= max_length:
1718
+ return path
1719
+ return f"...{path[-(max_length - 3):]}"
1720
+
1721
+
1722
+ def list_state_tags(state, active_lock: dict | None, progress: ProgressSnapshot | None = None) -> list[str]:
1723
+ tags = [primary_state_tag(state)]
1724
+ if state.offline_only:
1725
+ append_unique(tags, "offline")
1726
+ if state.upstream_status == "changed":
1727
+ append_unique(tags, "upstream-changed")
1728
+ if state_has_upstream_unavailable(state):
1729
+ append_unique(tags, "upstream-unavailable")
1730
+ if active_lock is not None:
1731
+ append_unique(tags, "busy")
1732
+ if progress is not None and progress.any_stalled:
1733
+ append_unique(tags, "stalled")
1734
+ return tags
1735
+
1736
+
1737
+ def list_exception_tags(
1738
+ state: VerificationState | None,
1739
+ active_lock: dict | None,
1740
+ progress: ProgressSnapshot | None = None,
1741
+ ) -> list[str]:
1742
+ if state is None:
1743
+ tags = ["unverified"]
1744
+ if active_lock is not None:
1745
+ tags.append("busy")
1746
+ if progress is not None and progress.any_stalled:
1747
+ tags.append("stalled")
1748
+ return tags
1749
+ tags = list_state_tags(state, active_lock, progress)
1750
+ return [tag for tag in tags if tag != "clean"]
1751
+
1752
+
1753
+ def primary_state_tag(state) -> str:
1754
+ if state.status == "unavailable":
1755
+ return "upstream-unavailable"
1756
+ if state.status == "incomplete":
1757
+ if state_has_manifest_incomplete(state):
1758
+ return "manifest-incomplete"
1759
+ return "incomplete"
1760
+ if state.status == "dirty":
1761
+ if state.repair_paths:
1762
+ return "needs-repair"
1763
+ if any(str(issue) == "verification skipped" for issue in state.issues):
1764
+ return "unverified"
1765
+ return "dirty"
1766
+ if state.status == "in_progress":
1767
+ return "in-progress"
1768
+ return state.status or "unknown"
1769
+
1770
+
1771
+ def append_unique(values: list[str], value: str) -> None:
1772
+ if value not in values:
1773
+ values.append(value)
1774
+
1775
+
1776
+ def state_has_manifest_incomplete(state) -> bool:
1777
+ return state_has_cached_hash_missing(state) or any(".manifest missing" in str(issue) for issue in state.issues)
1778
+
1779
+
1780
+ def handle_verify(args, config: Config, *, hub=None) -> int:
1781
+ if args.all:
1782
+ failures = 0
1783
+ changed = 0
1784
+ repair_needed = 0
1785
+ cache_incomplete = 0
1786
+ for repo_id in list_model_ids(config):
1787
+ if args.max_age and should_skip_recent_clean(config, repo_id, args.repo_type, args.max_age):
1788
+ print(f"skipped recent clean verification: {repo_id}")
1789
+ continue
1790
+ try:
1791
+ rc = verify_one(config, repo_id, args, hub=hub)
1792
+ except ModelBusyError as exc:
1793
+ print(f"skipped busy: {repo_id} -> {exc.root} ({lock_label(exc.info)})")
1794
+ failures += 1
1795
+ continue
1796
+ state = read_verification_state(archive_path(config, repo_id, args.repo_type))
1797
+ if state is not None and state.upstream_status == "changed":
1798
+ changed += 1
1799
+ if rc != 0:
1800
+ failures += 1
1801
+ if state is not None and state.repair_paths:
1802
+ repair_needed += 1
1803
+ if state is not None and state_has_cached_hash_missing(state):
1804
+ cache_incomplete += 1
1805
+ if repair_needed:
1806
+ print("next: model-mirror repair --all")
1807
+ if cache_incomplete:
1808
+ print("cached verification incomplete: run full verification with model-mirror verify --all")
1809
+ if changed:
1810
+ print("update changed upstreams: model-mirror repair --all --update")
1811
+ return 1 if failures else 0
1812
+
1813
+ if not args.model:
1814
+ raise SystemExit("verify requires a model id unless --all is used")
1815
+ return verify_one(config, args.model, args, hub=hub)
1816
+
1817
+
1818
+ def handle_repair(args, config: Config, *, hub=None) -> int:
1819
+ if args.all and args.model:
1820
+ raise SystemExit("repair accepts a model id or --all, not both")
1821
+ if args.all:
1822
+ failures = 0
1823
+ for repo_id in list_model_ids(config):
1824
+ state = read_verification_state(archive_path(config, repo_id, args.repo_type))
1825
+ if state is not None and state.offline_only:
1826
+ print(f"skipped offline-only: {repo_id}; repair requires an upstream repository")
1827
+ continue
1828
+ try:
1829
+ rc = repair_one(config, repo_id, args, hub=hub)
1830
+ except ModelBusyError as exc:
1831
+ print(f"skipped busy: {repo_id} -> {exc.root} ({lock_label(exc.info)})")
1832
+ failures += 1
1833
+ continue
1834
+ if rc != 0:
1835
+ failures += 1
1836
+ return 1 if failures else 0
1837
+
1838
+ if not args.model:
1839
+ raise SystemExit("repair requires a model id unless --all is used")
1840
+ return repair_one(config, args.model, args, hub=hub)
1841
+
1842
+
1843
+ def handle_upgrade(args, config: Config) -> int:
1844
+ if args.all and args.model:
1845
+ raise SystemExit("upgrade accepts a model id or --all, not both")
1846
+ if not args.all and not args.model:
1847
+ raise SystemExit("upgrade requires a model id unless --all is used")
1848
+ repo_ids = list_repo_ids(config, args.repo_type) if args.all else [args.model]
1849
+ failures = 0
1850
+ for repo_id in repo_ids:
1851
+ root = archive_path(config, repo_id, args.repo_type)
1852
+ try:
1853
+ with ModelLock(root, "upgrade", repo_id, args.repo_type):
1854
+ snapshot = read_snapshot_plan(root)
1855
+ if snapshot is None:
1856
+ print(
1857
+ f"upgrade unavailable: {repo_id} has no pinned snapshot description; "
1858
+ f"run model-mirror verify {repo_id}"
1859
+ )
1860
+ failures += 1
1861
+ continue
1862
+ from .torrent_coverage import upgrade_coverage
1863
+
1864
+ result = upgrade_coverage(
1865
+ root,
1866
+ snapshot,
1867
+ dry_run=args.dry_run,
1868
+ on_progress=lambda rel, done, total: print_upgrade_progress(
1869
+ repo_id,
1870
+ rel,
1871
+ done,
1872
+ total,
1873
+ ),
1874
+ )
1875
+ except (OSError, ValueError, RuntimeError) as exc:
1876
+ print(f"upgrade failed: {repo_id} -> {exc}")
1877
+ failures += 1
1878
+ continue
1879
+ action = "would-hash" if args.dry_run else "hashed"
1880
+ print(
1881
+ f"{'upgrade dry-run' if args.dry_run else 'upgraded'}: {repo_id} "
1882
+ f"coverage={result.covered_files}/{result.total_files} "
1883
+ f"{action}_files={result.hashed_files} {action}_bytes={format_bytes(result.hashed_bytes)} "
1884
+ f"path={result.path}"
1885
+ )
1886
+ if not result.complete and not args.dry_run:
1887
+ failures += 1
1888
+ return 1 if failures else 0
1889
+
1890
+
1891
+ def print_upgrade_progress(repo_id: str, rel: str, done: int, total: int) -> None:
1892
+ if done == total:
1893
+ print(f"hashed: {repo_id}:{rel} {format_bytes(total)}")
1894
+
1895
+
1896
+ def handle_torrent(args, config: Config) -> int:
1897
+ if args.torrent_command is None:
1898
+ print("torrent requires a subcommand; run model-mirror help torrent")
1899
+ return 2
1900
+ if args.torrent_command == "serve":
1901
+ from .torrent_seed import serve
1902
+
1903
+ print(
1904
+ f"managed torrent backend: {'single reconciliation' if args.once else 'serving'} "
1905
+ f"archive={config.directory}"
1906
+ )
1907
+ try:
1908
+ serve(config, poll_seconds=args.poll_seconds, once=args.once)
1909
+ except (OSError, ValueError, RuntimeError) as exc:
1910
+ print(f"managed torrent backend failed: {exc}")
1911
+ return 1
1912
+ return 0
1913
+ if args.torrent_command in {"handoff", "import", "join"}:
1914
+ return handle_torrent_receive(args, config)
1915
+
1916
+ root = archive_path(config, args.model, args.repo_type)
1917
+ if args.torrent_command == "retire":
1918
+ return handle_torrent_retire(args, root)
1919
+ try:
1920
+ with ModelLock(root, f"torrent-{args.torrent_command}", args.model, args.repo_type):
1921
+ return handle_torrent_locked(args, root)
1922
+ except (OSError, ValueError, RuntimeError) as exc:
1923
+ print(f"torrent {args.torrent_command} failed: {args.model} -> {exc}")
1924
+ return 1
1925
+
1926
+
1927
+ def handle_torrent_locked(args, root: Path) -> int:
1928
+ from .torrent_publication import (
1929
+ create_publication,
1930
+ load_fenced_publication,
1931
+ set_seed_desired,
1932
+ )
1933
+
1934
+ if args.torrent_command in {"create", "publish"}:
1935
+ external = bool(args.external)
1936
+ result = create_publication(
1937
+ root,
1938
+ repo_id=args.model,
1939
+ repo_type=args.repo_type,
1940
+ desired_seed=args.torrent_command == "publish" and not external,
1941
+ client_mode="external" if external else "managed",
1942
+ )
1943
+ action = "created" if result.created else "reused"
1944
+ print(
1945
+ f"torrent {action}: {result.record.publication_id} "
1946
+ f"coverage_hashed_files={result.coverage_hashed_files} "
1947
+ f"coverage_hashed_bytes={format_bytes(result.coverage_hashed_bytes)}"
1948
+ )
1949
+ print_torrent_handoff(result.record, root)
1950
+ if args.torrent_command == "publish" and not external:
1951
+ print("desired seed: enabled; model-mirror torrent serve reconciles it now and after restart")
1952
+ elif external:
1953
+ print("client mode: external; model-mirror records the fence but does not manage client runtime")
1954
+ return 0
1955
+ if args.torrent_command == "show":
1956
+ fenced = load_fenced_publication(root)
1957
+ if fenced is None:
1958
+ print(f"unpublished: {args.model}")
1959
+ return 0
1960
+ print_torrent_status(fenced[0], root)
1961
+ return 0
1962
+ if args.torrent_command == "stop":
1963
+ record = set_seed_desired(root, desired=False)
1964
+ print(f"torrent stopping: {record.publication_id}; publication fence retained")
1965
+ return 0
1966
+ raise ValueError(f"unsupported torrent command: {args.torrent_command}")
1967
+
1968
+
1969
+ def handle_torrent_retire(args, root: Path) -> int:
1970
+ from .torrent_publication import (
1971
+ retire_publication,
1972
+ set_seed_desired,
1973
+ wait_for_maintenance_detach,
1974
+ )
1975
+
1976
+ try:
1977
+ with ModelLock(root, "torrent-retire-stop", args.model, args.repo_type):
1978
+ set_seed_desired(root, desired=False)
1979
+ wait_for_maintenance_detach(root)
1980
+ with ModelLock(root, "torrent-retire", args.model, args.repo_type):
1981
+ record = retire_publication(root)
1982
+ except (OSError, ValueError, RuntimeError) as exc:
1983
+ print(f"torrent retire failed: {args.model} -> {exc}")
1984
+ return 1
1985
+ print(
1986
+ f"torrent retired: {record.publication_id}; local update fence released. "
1987
+ "Already distributed torrent metadata cannot be revoked."
1988
+ )
1989
+ return 0
1990
+
1991
+
1992
+ def print_torrent_handoff(record, root: Path) -> None:
1993
+ print(f"torrent: {root / record.torrent_path}")
1994
+ print(f"recovery: {root / record.recovery_path}")
1995
+ print(f"magnet: {record.magnet_uri}")
1996
+ print(f"external client data location: {root.parent}")
1997
+
1998
+
1999
+ def handle_torrent_receive(args, config: Config) -> int:
2000
+ from .torrent_import import (
2001
+ external_handoff,
2002
+ import_external_payload,
2003
+ join_torrent,
2004
+ parse_publication_metainfo,
2005
+ )
2006
+
2007
+ try:
2008
+ if args.torrent_command == "handoff":
2009
+ metainfo = args.torrent_file.read_bytes()
2010
+ parsed = parse_publication_metainfo(metainfo)
2011
+ destination, command = external_handoff(config, parsed, args.torrent_file)
2012
+ print(f"publication: huggingface:{parsed.descriptor.repo_type}:{parsed.descriptor.repo_id}@{parsed.descriptor.resolved_commit}")
2013
+ print(f"external client data location: {destination}")
2014
+ print(f"after download: {command}")
2015
+ return 0
2016
+ if args.torrent_command == "import":
2017
+ result = import_external_payload(
2018
+ config,
2019
+ metainfo=args.torrent_file.read_bytes(),
2020
+ payload_root=args.payload_root,
2021
+ seed=args.seed,
2022
+ )
2023
+ else:
2024
+ result = join_torrent(
2025
+ config,
2026
+ args.source,
2027
+ seed=args.seed,
2028
+ metadata_timeout_seconds=args.metadata_timeout,
2029
+ on_progress=print_join_progress,
2030
+ )
2031
+ except (OSError, ValueError, RuntimeError) as exc:
2032
+ print(f"torrent {args.torrent_command} failed: {exc}")
2033
+ return 1
2034
+ print(
2035
+ f"torrent imported: {result.publication.publication_id} -> {result.path} "
2036
+ f"reread_files={result.reread_files} reread_bytes={format_bytes(result.reread_bytes)}"
2037
+ )
2038
+ print(
2039
+ "content=torrent-verified publication_trust=trusted-infohash "
2040
+ "upstream_provenance=not-upstream-verified upstream_availability=unknown"
2041
+ )
2042
+ print_torrent_handoff(result.publication, result.path)
2043
+ if result.publication.desired_seed:
2044
+ print("desired seed: enabled; model-mirror torrent serve will reconcile it")
2045
+ return 0
2046
+
2047
+
2048
+ def print_join_progress(status) -> None:
2049
+ print(
2050
+ f"torrent download: {status.name} {status.progress * 100:.1f}% "
2051
+ f"rate={format_bytes(int(status.download_payload_rate))}/s peers={status.num_peers}"
2052
+ )
2053
+
2054
+
2055
+ def print_torrent_status(record, root: Path) -> None:
2056
+ print("feature_stability: experimental")
2057
+ print(f"publication: {record.publication_id}")
2058
+ print(f"lifecycle: {record.lifecycle}")
2059
+ print(f"resolved_commit: {record.resolved_commit}")
2060
+ print(f"profile: {record.profile}")
2061
+ print(f"content_verification: {record.content_verification}")
2062
+ print(f"publication_trust: {record.publication_trust}")
2063
+ print(f"upstream_provenance: {record.upstream_provenance}")
2064
+ print(f"upstream_availability: {record.upstream_availability}")
2065
+ state = read_verification_state(root)
2066
+ print(
2067
+ f"update_available: "
2068
+ f"{str(state is not None and state.upstream_status == 'changed').lower()}"
2069
+ )
2070
+ print("archive_schema: model-mirror-snapshot/1")
2071
+ print(f"coverage_profile: {record.profile}")
2072
+ print(f"coverage_state: {torrent_coverage_status(root)}")
2073
+ print(f"client_mode: {record.client_mode}")
2074
+ print(f"desired_seed: {str(record.desired_seed).lower()}")
2075
+ print(f"observed_backend: {record.observed_backend}")
2076
+ if record.observed_detail:
2077
+ print(f"observed_detail: {record.observed_detail}")
2078
+ print(f"infohash_v1: {record.infohash_v1}")
2079
+ print(f"infohash_v2: {record.infohash_v2}")
2080
+ print_torrent_handoff(record, root)
2081
+
2082
+
2083
+ def state_has_cached_hash_missing(state) -> bool:
2084
+ return any(str(issue).startswith("cached_hash_missing:") for issue in state.issues)
2085
+
2086
+
2087
+ def state_has_upstream_unavailable(state) -> bool:
2088
+ return state.status == "unavailable" or any(
2089
+ str(issue).startswith("upstream unavailable:") for issue in state.issues
2090
+ )
2091
+
2092
+
2093
+ def repair_one(config: Config, repo_id: str, args, *, hub=None) -> int:
2094
+ selected_hub = hub or HuggingFaceHub(config)
2095
+ print_verification_age(config, repo_id, args.repo_type)
2096
+ if args.force_partial:
2097
+ print(
2098
+ "warning: --force-partial can leave the repository inconsistent when verification data is incomplete"
2099
+ )
2100
+ result = repair(
2101
+ config,
2102
+ repo_id,
2103
+ hub=selected_hub,
2104
+ repo_type=args.repo_type,
2105
+ update=args.update,
2106
+ force_partial=args.force_partial,
2107
+ )
2108
+ if result.status == "verify-required":
2109
+ print(f"run verify first: model-mirror verify {repo_id}")
2110
+ if result.status == "verification-incomplete":
2111
+ print(
2112
+ f"could not fully repair {repo_id}; missing verification data for some files. "
2113
+ f"Run full verify and repair again: model-mirror verify {repo_id} && model-mirror repair {repo_id}"
2114
+ )
2115
+ if result.status == "offline-only":
2116
+ print(
2117
+ f"cannot repair offline-only model {repo_id}; upstream link is disabled. "
2118
+ f"Run model-mirror online {repo_id} to re-enable Hub-backed repair."
2119
+ )
2120
+ print_repair_commit_notice(repo_id, result)
2121
+ print(f"{result.status}: {repo_id} -> {result.path}")
2122
+ return 0 if result.status in {"complete", "repaired", "updated"} else 1
2123
+
2124
+
2125
+ def verify_one(config: Config, repo_id: str, args, *, hub=None) -> int:
2126
+ selected_hub = hub or HuggingFaceHub(config)
2127
+ root = archive_path(config, repo_id, args.repo_type)
2128
+ with ModelLock(root, "verify", repo_id, args.repo_type):
2129
+ return verify_one_locked(config, repo_id, args, selected_hub, root)
2130
+
2131
+
2132
+ def verify_one_locked(config: Config, repo_id: str, args, selected_hub, root: Path) -> int:
2133
+ existing_state = read_verification_state(root)
2134
+ requested_revision = selected_revision_arg(args) or (
2135
+ existing_state.requested_revision if existing_state else config.revision
2136
+ )
2137
+
2138
+ if args.offline:
2139
+ return verify_one_offline(config, root, repo_id, args, existing_state, mode="offline")
2140
+ if existing_state is not None and existing_state.offline_only:
2141
+ return verify_one_offline(config, root, repo_id, args, existing_state, mode="offline-only")
2142
+
2143
+ try:
2144
+ upstream_snapshot = get_snapshot(selected_hub, repo_id, args.repo_type, requested_revision)
2145
+ except Exception as exc:
2146
+ return handle_upstream_unavailable(root, repo_id, args, existing_state, requested_revision, exc)
2147
+ resolved_commit = existing_state.resolved_commit if existing_state and existing_state.resolved_commit else upstream_snapshot.resolved_commit
2148
+ try:
2149
+ snapshot = upstream_snapshot if resolved_commit == upstream_snapshot.resolved_commit else get_snapshot(
2150
+ selected_hub, repo_id, args.repo_type, resolved_commit
2151
+ )
2152
+ except Exception as exc:
2153
+ return handle_upstream_unavailable(root, repo_id, args, existing_state, requested_revision, exc)
2154
+ metadata = snapshot.files
2155
+ checksum_result = None
2156
+ manifest_verified = False
2157
+ if not args.cached and config.checksum:
2158
+ if (root / MANIFEST).exists():
2159
+ checksum_result = verify_checksums(root, strict=args.strict)
2160
+ if checksum_result.ok:
2161
+ write_checksums(root, max_workers=config.checksum_workers)
2162
+ manifest_verified = True
2163
+ else:
2164
+ write_checksums(root, max_workers=config.checksum_workers)
2165
+ manifest_verified = True
2166
+ from_manifest = args.cached or manifest_verified
2167
+ result = verify_remote(
2168
+ root,
2169
+ metadata,
2170
+ cached=args.cached,
2171
+ from_manifest=from_manifest,
2172
+ strict=args.strict,
2173
+ )
2174
+ if checksum_result is not None:
2175
+ merge_checksum_result(result, checksum_result)
2176
+ if args.repo_type == "model":
2177
+ audit = audit_model(root, skip_transformers=True)
2178
+ else:
2179
+ audit = None
2180
+ state = state_from_results(
2181
+ repo_id,
2182
+ args.repo_type,
2183
+ requested_revision,
2184
+ result,
2185
+ audit,
2186
+ resolved_commit=resolved_commit,
2187
+ upstream_commit=upstream_snapshot.resolved_commit,
2188
+ )
2189
+ write_verification_state(root, state)
2190
+
2191
+ if state.status == "incomplete":
2192
+ print(f"cached verification incomplete: {repo_id}{upstream_change_suffix(state)}")
2193
+ print(f"run full verification: model-mirror verify {repo_id}")
2194
+ if state.upstream_status == "changed":
2195
+ print_update_next_step(repo_id)
2196
+ return 1
2197
+ if audit is not None and not audit.ok:
2198
+ print(f"verification failed: {repo_id}{upstream_change_suffix(state)}")
2199
+ print_verification_next_steps(repo_id, state)
2200
+ return 1
2201
+ if result.ok:
2202
+ mode = "cached" if args.cached else "full"
2203
+ print(f"verified ({mode}): {repo_id}{upstream_change_suffix(state)}")
2204
+ if state.upstream_status == "changed":
2205
+ print_update_next_step(repo_id)
2206
+ return 0
2207
+ print(f"verification failed: {repo_id}{upstream_change_suffix(state)}")
2208
+ print_verification_next_steps(repo_id, state)
2209
+ return 1
2210
+
2211
+
2212
+ def upstream_change_suffix(state) -> str:
2213
+ return " upstream=changed" if state.upstream_status == "changed" else ""
2214
+
2215
+
2216
+ def print_verification_next_steps(repo_id: str, state) -> None:
2217
+ if state.repair_paths:
2218
+ print(f"next: model-mirror repair {repo_id}")
2219
+ if state.upstream_status == "changed":
2220
+ print_update_next_step(repo_id)
2221
+
2222
+
2223
+ def print_update_next_step(repo_id: str) -> None:
2224
+ print(f"update changed upstream: model-mirror repair --update {repo_id}")
2225
+
2226
+
2227
+ def print_repair_commit_notice(repo_id: str, result) -> None:
2228
+ if result.upstream_status == "changed":
2229
+ print(
2230
+ f"upstream changed: {repo_id} local={result.resolved_commit} "
2231
+ f"upstream={result.upstream_commit} not_applied"
2232
+ )
2233
+
2234
+
2235
+ def verify_one_offline(config: Config, root: Path, repo_id: str, args, existing_state, *, mode: str) -> int:
2236
+ if existing_state is None:
2237
+ print(f"{mode} verification unavailable: {repo_id}")
2238
+ return 1
2239
+ if args.cached:
2240
+ print(f"verified ({mode} cached): {repo_id} state={existing_state.status}")
2241
+ return 0 if existing_state.clean else 1
2242
+ if not (root / MANIFEST).exists():
2243
+ state = local_incomplete_state(repo_id, args.repo_type, existing_state, f"{MANIFEST} missing")
2244
+ write_verification_state(root, state)
2245
+ print(f"{mode} verification incomplete: {repo_id} missing {MANIFEST}")
2246
+ return 1
2247
+ result = verify_checksums(root, strict=args.strict)
2248
+ remote_result = RemoteVerifyResult(
2249
+ missing=result.missing,
2250
+ hash_mismatches=result.failures,
2251
+ extras=result.extras,
2252
+ )
2253
+ state = state_from_results(
2254
+ repo_id,
2255
+ args.repo_type,
2256
+ existing_state.requested_revision,
2257
+ remote_result,
2258
+ resolved_commit=existing_state.resolved_commit,
2259
+ upstream_commit=existing_state.upstream_commit,
2260
+ offline_only=existing_state.offline_only,
2261
+ )
2262
+ write_verification_state(root, state)
2263
+ if result.ok:
2264
+ print(f"verified ({mode} full): {repo_id}")
2265
+ return 0
2266
+ print(f"{mode} verification failed: {repo_id}")
2267
+ if existing_state.offline_only:
2268
+ print(f"repair unavailable for offline-only model: {repo_id}")
2269
+ else:
2270
+ print_verification_next_steps(repo_id, state)
2271
+ return 1
2272
+
2273
+
2274
+ def local_incomplete_state(repo_id: str, repo_type: str, existing_state, issue: str) -> VerificationState:
2275
+ return VerificationState(
2276
+ status="incomplete",
2277
+ repo_id=repo_id,
2278
+ repo_type=repo_type,
2279
+ requested_revision=existing_state.requested_revision,
2280
+ resolved_commit=existing_state.resolved_commit,
2281
+ upstream_commit=existing_state.upstream_commit,
2282
+ upstream_status=existing_state.upstream_status,
2283
+ offline_only=existing_state.offline_only,
2284
+ repair_paths=[],
2285
+ issues=[issue],
2286
+ )
2287
+
2288
+
2289
+ def handle_upstream_unavailable(
2290
+ root: Path,
2291
+ repo_id: str,
2292
+ args,
2293
+ existing_state,
2294
+ requested_revision: str,
2295
+ exc: Exception,
2296
+ ) -> int:
2297
+ issue = f"upstream unavailable: {exc}"
2298
+ if existing_state is not None:
2299
+ issues = [item for item in existing_state.issues if not str(item).startswith("upstream unavailable:")]
2300
+ issues.append(issue)
2301
+ state = VerificationState(
2302
+ status=existing_state.status,
2303
+ repo_id=repo_id,
2304
+ repo_type=args.repo_type,
2305
+ requested_revision=requested_revision,
2306
+ resolved_commit=existing_state.resolved_commit,
2307
+ upstream_commit=existing_state.upstream_commit,
2308
+ upstream_status=existing_state.upstream_status,
2309
+ offline_only=existing_state.offline_only,
2310
+ repair_paths=existing_state.repair_paths,
2311
+ issues=issues,
2312
+ checked_at_utc=existing_state.checked_at_utc,
2313
+ )
2314
+ else:
2315
+ state = VerificationState(
2316
+ status="unavailable",
2317
+ repo_id=repo_id,
2318
+ repo_type=args.repo_type,
2319
+ requested_revision=requested_revision,
2320
+ issues=[issue],
2321
+ )
2322
+ write_verification_state(root, state)
2323
+ print(f"verification failed: {repo_id} upstream repository unavailable: {exc}")
2324
+ print(
2325
+ f"if the source repository is no longer available and you want to keep this local mirror, "
2326
+ f"run: model-mirror offline {repo_id}"
2327
+ )
2328
+ return 1
2329
+
2330
+
2331
+ def handle_offline_mode(args, config: Config, *, offline_only: bool) -> int:
2332
+ command = "offline" if offline_only else "online"
2333
+ root = archive_path(config, args.model, args.repo_type)
2334
+ with ModelLock(root, command, args.model, args.repo_type):
2335
+ state = read_verification_state(root)
2336
+ if state is None:
2337
+ print(f"verification state unavailable: {args.model}")
2338
+ print(f"run verify first: model-mirror verify {args.model}")
2339
+ return 1
2340
+ state.offline_only = offline_only
2341
+ if offline_only:
2342
+ state.issues = [item for item in state.issues if not str(item).startswith("upstream unavailable:")]
2343
+ if state.status == "unavailable":
2344
+ state.status = "incomplete"
2345
+ state.issues.append("local verification required")
2346
+ write_verification_state(root, state)
2347
+ if offline_only:
2348
+ print(f"offline-only enabled: {args.model}")
2349
+ else:
2350
+ print(f"offline-only disabled: {args.model}")
2351
+ return 0
2352
+
2353
+
2354
+ def list_model_ids(config: Config) -> list[str]:
2355
+ return list_repo_ids(config, "model")
2356
+
2357
+
2358
+ def list_repo_ids(config: Config, repo_type: str) -> list[str]:
2359
+ repos_root = Path(config.directory) / REPO_TYPE_DIRS[repo_type]
2360
+ if not repos_root.exists():
2361
+ return []
2362
+ result = []
2363
+ for owner in sorted(path for path in repos_root.iterdir() if path.is_dir()):
2364
+ for repo in sorted(path for path in owner.iterdir() if path.is_dir()):
2365
+ result.append(f"{owner.name}/{repo.name}")
2366
+ return result
2367
+
2368
+
2369
+ def parse_age(value: str) -> int:
2370
+ units = {"s": 1, "m": 60, "h": 3600, "d": 86400}
2371
+ value = value.strip().lower()
2372
+ if not value:
2373
+ raise ValueError("empty age")
2374
+ unit = value[-1]
2375
+ if unit in units:
2376
+ return int(value[:-1]) * units[unit]
2377
+ return int(value)
2378
+
2379
+
2380
+ def verification_age_seconds(checked_at_utc: str) -> int | None:
2381
+ if not checked_at_utc:
2382
+ return None
2383
+ try:
2384
+ checked = datetime.fromisoformat(checked_at_utc)
2385
+ except ValueError:
2386
+ return None
2387
+ if checked.tzinfo is None:
2388
+ checked = checked.replace(tzinfo=timezone.utc)
2389
+ return max(0, int((datetime.now(timezone.utc) - checked).total_seconds()))
2390
+
2391
+
2392
+ def verification_age_label(checked_at_utc: str) -> str:
2393
+ seconds = verification_age_seconds(checked_at_utc)
2394
+ return format_age_seconds(seconds)
2395
+
2396
+
2397
+ def format_age_seconds(seconds: int | None) -> str:
2398
+ if seconds is None:
2399
+ return "unknown"
2400
+ if seconds < 120:
2401
+ return f"{seconds}s"
2402
+ minutes = seconds // 60
2403
+ if minutes < 120:
2404
+ return f"{minutes}m"
2405
+ hours = minutes // 60
2406
+ if hours < 48:
2407
+ return f"{hours}h"
2408
+ return f"{hours // 24}d"
2409
+
2410
+
2411
+ def print_verification_age(config: Config, repo_id: str, repo_type: str) -> None:
2412
+ root = archive_path(config, repo_id, repo_type)
2413
+ state = read_verification_state(root)
2414
+ if state is None:
2415
+ print("verification age: unavailable")
2416
+ return
2417
+ age = verification_age_seconds(state.checked_at_utc)
2418
+ if age is None:
2419
+ path = verification_state_path(root)
2420
+ modified = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc)
2421
+ age = max(0, int((datetime.now(timezone.utc) - modified).total_seconds()))
2422
+ print(f"verification age: {format_age_seconds(age)}")
2423
+ if age is not None and age > 24 * 60 * 60:
2424
+ print("warning: verification is older than 24h; run verify again for fresh repair paths")
2425
+
2426
+
2427
+ def should_skip_recent_clean(config: Config, repo_id: str, repo_type: str, max_age: str) -> bool:
2428
+ state = read_verification_state(archive_path(config, repo_id, repo_type))
2429
+ if state is None or not state.clean or state_has_upstream_unavailable(state):
2430
+ return False
2431
+ age = verification_age_seconds(state.checked_at_utc)
2432
+ return age is not None and age <= parse_age(max_age)
2433
+
2434
+
2435
+ if __name__ == "__main__":
2436
+ raise SystemExit(main())