model-mirror-cli 0.2.0__py3-none-any.whl

model_mirror/torrent_seed.py

@@ -0,0 +1,228 @@
+from __future__ import annotations
+
+import hashlib
+import signal
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .config import Config, REPO_TYPE_DIRS
+from .lock import ModelBusyError, ModelLock
+from .torrent import TorrentPublicationError, load_libtorrent, verified_seed_params
+from .torrent_publication import (
+    PublicationRecord,
+    fence_path,
+    load_fenced_publication,
+    payload_fingerprints_match,
+    update_observed_backend,
+)
+
+
+@dataclass(slots=True)
+class ActiveSeed:
+    root: Path
+    record: PublicationRecord
+    handle: object
+
+
+class ManagedSeeder:
+    def __init__(self, *, libtorrent_module=None, session=None, listen_interfaces: str | None = None):
+        self.lt = libtorrent_module or load_libtorrent()
+        if session is None:
+            settings = {}
+            if listen_interfaces:
+                settings["listen_interfaces"] = listen_interfaces
+            session = self.lt.session(settings)
+        self.session = session
+        self.active: dict[str, ActiveSeed] = {}
+
+    def reconcile(self, config: Config) -> None:
+        discovered = {root: record for root, record in discover_publications(config)}
+        for publication_id, active in list(self.active.items()):
+            record = discovered.get(active.root)
+            if (
+                record is None
+                or record.publication_id != publication_id
+                or not record.desired_seed
+                or record.client_mode != "managed"
+                or record.lifecycle != "published"
+            ):
+                if record is None:
+                    self.session.remove_torrent(active.handle)
+                    del self.active[publication_id]
+                else:
+                    try:
+                        with ModelLock(
+                            active.root,
+                            "torrent-reconcile",
+                            record.repo_id,
+                            record.repo_type,
+                        ):
+                            current = load_fenced_publication(active.root)
+                            if current is not None and (
+                                current[0].desired_seed
+                                and current[0].client_mode == "managed"
+                                and current[0].lifecycle == "published"
+                            ):
+                                continue
+                            self.session.remove_torrent(active.handle)
+                            del self.active[publication_id]
+                            if current is not None:
+                                update_observed_backend(active.root, state="stopped")
+                    except ModelBusyError:
+                        continue
+
+        for root, record in discovered.items():
+            if (
+                not record.desired_seed
+                and record.client_mode == "managed"
+                and record.observed_backend == "stopping"
+            ):
+                self._set_observed(root, "stopped")
+                continue
+            if (
+                not record.desired_seed
+                or record.client_mode != "managed"
+                or record.lifecycle != "published"
+                or record.publication_id in self.active
+            ):
+                continue
+            try:
+                with ModelLock(root, "torrent-reconcile", record.repo_id, record.repo_type):
+                    current = load_fenced_publication(root)
+                    if current is None:
+                        continue
+                    record = current[0]
+                    if (
+                        not record.desired_seed
+                        or record.client_mode != "managed"
+                        or record.lifecycle != "published"
+                    ):
+                        continue
+                    matches, detail = payload_fingerprints_match(root, record)
+                    if not matches:
+                        update_observed_backend(
+                            root,
+                            state="unhealthy",
+                            detail=detail,
+                            lifecycle="unhealthy",
+                        )
+                        continue
+                    try:
+                        metainfo = read_verified_metainfo(root, record)
+                        params = verified_seed_params(metainfo, root, libtorrent_module=self.lt)
+                        handle = self.session.add_torrent(params)
+                    except Exception as exc:
+                        update_observed_backend(root, state="error", detail=str(exc))
+                        continue
+                    self.active[record.publication_id] = ActiveSeed(root, record, handle)
+                    update_observed_backend(root, state="seeding")
+            except ModelBusyError:
+                continue
+            except Exception as exc:
+                self._set_observed(root, "error", str(exc))
+
+    def close(self) -> None:
+        for publication_id, active in list(self.active.items()):
+            try:
+                with ModelLock(
+                    active.root,
+                    "torrent-reconcile",
+                    active.record.repo_id,
+                    active.record.repo_type,
+                ):
+                    self.session.remove_torrent(active.handle)
+                    update_observed_backend(
+                        active.root,
+                        state="stopped",
+                        detail="managed seeder stopped",
+                    )
+                    del self.active[publication_id]
+            except ModelBusyError:
+                continue
+
+    def _set_observed(
+        self,
+        root: Path,
+        state: str,
+        detail: str = "",
+        *,
+        lifecycle: str | None = None,
+    ) -> None:
+        fenced = load_fenced_publication(root)
+        if fenced is None:
+            return
+        record = fenced[0]
+        try:
+            with ModelLock(root, "torrent-reconcile", record.repo_id, record.repo_type):
+                update_observed_backend(root, state=state, detail=detail, lifecycle=lifecycle)
+        except ModelBusyError:
+            return
+
+
+def discover_publications(config: Config) -> list[tuple[Path, PublicationRecord]]:
+    result = []
+    archive_root = Path(config.directory)
+    for type_dir in REPO_TYPE_DIRS.values():
+        base = archive_root / type_dir
+        if not base.exists():
+            continue
+        for candidate in sorted(base.glob("*/*")):
+            if not fence_path(candidate).exists():
+                continue
+            try:
+                fenced = load_fenced_publication(candidate)
+            except (OSError, ValueError):
+                continue
+            if fenced is not None:
+                result.append((candidate, fenced[0]))
+    return result
+
+
+def read_verified_metainfo(root: Path, record: PublicationRecord) -> bytes:
+    relative = Path(record.torrent_path)
+    if not relative.parts or relative.is_absolute() or ".." in relative.parts:
+        raise TorrentPublicationError("unsafe torrent artifact path in publication record")
+    path = root / relative
+    try:
+        metainfo = path.read_bytes()
+    except OSError as exc:
+        raise TorrentPublicationError(f"torrent artifact is unavailable: {path}") from exc
+    if hashlib.sha256(metainfo).hexdigest() != record.metainfo_sha256:
+        raise TorrentPublicationError(f"torrent artifact digest mismatch: {path}")
+    return metainfo
+
+
+def serve(
+    config: Config,
+    *,
+    poll_seconds: float = 2.0,
+    once: bool = False,
+    libtorrent_module=None,
+    session=None,
+) -> None:
+    if poll_seconds <= 0:
+        raise ValueError("poll interval must be positive")
+    seeder = ManagedSeeder(libtorrent_module=libtorrent_module, session=session)
+    if once:
+        seeder.reconcile(config)
+        seeder.close()
+        return
+
+    stopped = False
+
+    def stop(_signum, _frame):
+        nonlocal stopped
+        stopped = True
+
+    previous = {}
+    for signum in (signal.SIGINT, signal.SIGTERM):
+        previous[signum] = signal.signal(signum, stop)
+    try:
+        while not stopped:
+            seeder.reconcile(config)
+            time.sleep(poll_seconds)
+    finally:
+        seeder.close()
+        for signum, handler in previous.items():
+            signal.signal(signum, handler)

model_mirror/verify.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .checksums import file_hashes, load_manifest, iter_payload_files, record_is_current
+
+
+@dataclass(slots=True)
+class RemoteVerifyResult:
+    files_checked: int = 0
+    hashes_checked: int = 0
+    missing: list[str] = field(default_factory=list)
+    size_mismatches: list[str] = field(default_factory=list)
+    hash_mismatches: list[str] = field(default_factory=list)
+    cached_hash_missing: list[str] = field(default_factory=list)
+    extras: list[str] = field(default_factory=list)
+
+    @property
+    def ok(self) -> bool:
+        return not (
+            self.missing
+            or self.size_mismatches
+            or self.hash_mismatches
+            or self.cached_hash_missing
+            or self.extras
+        )
+
+
+def metadata_path(item) -> str:
+    direct = getattr(item, "path", None)
+    if direct is not None:
+        return direct
+    return getattr(item, "rfilename")
+
+
+def metadata_size(item) -> int | None:
+    return getattr(item, "size", None)
+
+
+def metadata_lfs_sha256(item) -> str | None:
+    direct = getattr(item, "lfs_sha256", None)
+    if direct:
+        return direct
+    lfs = getattr(item, "lfs", None)
+    if lfs is None:
+        return None
+    return getattr(lfs, "sha256", None)
+
+
+def metadata_blob_id(item) -> str | None:
+    return getattr(item, "blob_id", None)
+
+
+def verify_remote(
+    root: Path,
+    metadata,
+    *,
+    cached: bool = False,
+    from_manifest: bool = False,
+    check_hashes: bool = True,
+    strict: bool = False,
+) -> RemoteVerifyResult:
+    result = RemoteVerifyResult()
+    manifest: dict[str, dict] = {}
+    if cached or from_manifest:
+        manifest = load_manifest(root)
+
+    expected_paths = set()
+    for item in metadata:
+        rel = metadata_path(item)
+        expected_paths.add(rel)
+        path = root / rel
+        if not path.exists():
+            result.missing.append(rel)
+            continue
+
+        result.files_checked += 1
+        stat = path.stat()
+        expected_size = metadata_size(item)
+        actual_size = stat.st_size
+        if expected_size is not None and actual_size != expected_size:
+            result.size_mismatches.append(rel)
+            continue
+
+        if not check_hashes:
+            continue
+
+        expected_lfs_hash = metadata_lfs_sha256(item)
+        if expected_lfs_hash is not None:
+            manifest_hash = current_manifest_hash(manifest, rel, stat.st_size, stat.st_mtime_ns, "sha256")
+            if manifest_hash is not None:
+                actual_hash = manifest_hash
+            elif cached:
+                result.cached_hash_missing.append(rel)
+                continue
+            else:
+                actual_hash = file_hashes(path).sha256
+            result.hashes_checked += 1
+            if actual_hash != expected_lfs_hash:
+                result.hash_mismatches.append(rel)
+            continue
+
+        expected_blob_id = metadata_blob_id(item)
+        if expected_blob_id is None:
+            continue
+
+        manifest_hash = current_manifest_hash(manifest, rel, stat.st_size, stat.st_mtime_ns, "git_blob_sha1")
+        if manifest_hash is not None:
+            actual_hash = manifest_hash
+        elif cached:
+            result.cached_hash_missing.append(rel)
+            continue
+        else:
+            actual_hash = file_hashes(path).git_blob_sha1
+        result.hashes_checked += 1
+        if actual_hash != expected_blob_id:
+            result.hash_mismatches.append(rel)
+
+    if strict:
+        result.extras = [
+            path.relative_to(root).as_posix()
+            for path in iter_payload_files(root)
+            if path.relative_to(root).as_posix() not in expected_paths
+        ]
+
+    return result
+
+
+def current_manifest_hash(
+    manifest: dict[str, dict],
+    rel: str,
+    size: int,
+    mtime_ns: int,
+    hash_key: str,
+) -> str | None:
+    row = manifest.get(rel)
+    if not record_is_current(row, size, mtime_ns):
+        return None
+    value = row.get(hash_key)
+    return str(value) if value else None
+
+
+def merge_checksum_result(remote_result, checksum_result) -> None:
+    for attr, values in (
+        ("missing", checksum_result.missing),
+        ("hash_mismatches", checksum_result.failures),
+        ("extras", checksum_result.extras),
+    ):
+        existing = getattr(remote_result, attr)
+        for value in values:
+            if value not in existing:
+                existing.append(value)