hfms-bridge 0.4.0__py3-none-any.whl

hfms_bridge/__init__.py

@@ -0,0 +1,6 @@
+"""Streaming repository transfer without storing payloads on disk."""
+
+from .transfer import Transfer, TransferOptions
+
+__all__ = ["Transfer", "TransferOptions"]
+__version__ = "0.4.0"

hfms_bridge/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

hfms_bridge/cli.py

@@ -0,0 +1,252 @@
+"""The hfms_bridge console entry point."""
+
+import argparse
+import json
+import logging
+import os
+import re
+import sqlite3
+from pathlib import Path
+
+import requests
+from filelock import Timeout
+
+from . import __version__
+from .http import TransferError
+from .transfer import Transfer, TransferOptions
+
+log = logging.getLogger("hfms-bridge")
+
+
+def byte_size(value):
+    match = re.fullmatch(r"([0-9]+)\s*(B|KiB|MiB|GiB|KB|MB|GB)?", value, re.IGNORECASE)
+    if not match:
+        raise argparse.ArgumentTypeError("Use bytes or an integer with KiB/MiB/GiB (e.g. 256MiB)")
+    unit = (match[2] or "B").upper()
+    multiplier = {
+        "B": 1,
+        "KIB": 1024,
+        "MIB": 1024**2,
+        "GIB": 1024**3,
+        "KB": 1000,
+        "MB": 1000**2,
+        "GB": 1000**3,
+    }[unit]
+    result = int(match[1]) * multiplier
+    if result <= 0:
+        raise argparse.ArgumentTypeError("Size must be positive")
+    return result
+
+
+def nonempty_token(value):
+    if not value.strip() or any(c in value for c in "\r\n"):
+        raise argparse.ArgumentTypeError("Token must be non-empty and contain no line breaks")
+    return value
+
+
+def build_parser():
+    parser = argparse.ArgumentParser(
+        prog="hfms-bridge",
+        description="Stream models and datasets between Hugging Face (HF) and ModelScope (MS) "
+        "without saving file contents to disk.",
+    )
+    parser.add_argument(
+        "--version", action="version", version=__version__, help="Show the package version and exit"
+    )
+    commands = parser.add_subparsers(
+        dest="direction",
+        required=True,
+        title="transfer directions",
+        help="Choose source and target",
+    )
+    for direction in ("hf2ms", "ms2hf"):
+        source, target = (
+            ("Hugging Face", "ModelScope")
+            if direction == "hf2ms"
+            else ("ModelScope", "Hugging Face")
+        )
+        command = commands.add_parser(
+            direction,
+            help=f"Transfer from {source} to {target}",
+            description=f"Stream models or datasets from {source} to {target}. "
+            "Run the same command again to continue an interrupted transfer.",
+            epilog="Quote glob patterns and place SOURCE DESTINATION before multi-value filters. "
+            "Sizes accept bytes or KiB/MiB/GiB and KB/MB/GB suffixes. "
+            "Process restarts resume at file granularity; unfinished files may restart from zero.",
+        )
+        _transfer_arguments(command, direction)
+    return parser
+
+
+def _transfer_arguments(parser, direction):
+    source, target = ("HF", "MS") if direction == "hf2ms" else ("MS", "HF")
+    parser.add_argument("source", help=f"{source} source owner/repository; must be accessible")
+    parser.add_argument(
+        "destination",
+        help=f"{target} destination owner/repository; created if missing, updated if it exists",
+    )
+    for hub, env in (("hf", "HF_TOKEN"), ("ms", "MODELSCOPE_API_TOKEN")):
+        is_target = (hub == "ms") == (direction == "hf2ms")
+        parser.add_argument(
+            f"--{hub}-token",
+            type=nonempty_token,
+            required=is_target,
+            default=None if is_target else os.getenv(env) or None,
+            help=f"Required {hub.upper()} token with destination write/create permission; "
+            "must be passed explicitly"
+            if is_target
+            else f"Optional {hub.upper()} source read token (default: {env} environment variable); "
+            "required for private or restricted sources",
+        )
+    parser.add_argument(
+        "--repo-type",
+        choices=("auto", "model", "dataset"),
+        default="auto",
+        help="Source repository type (default: auto). Auto probes source models and datasets, "
+        "uses the unique accessible match, and requires an explicit type if both exist.",
+    )
+    for hub, default in (("hf", "main"), ("ms", "master")):
+        is_target = (hub == "ms") == (direction == "hf2ms")
+        parser.add_argument(
+            f"--{hub}-revision",
+            default=default,
+            help=(
+                f"{hub.upper()} destination branch (default: {default}); custom branches must "
+                "already exist; new repositories start on the default branch"
+                if is_target
+                else f"{hub.upper()} source branch, tag or commit (default: {default}); "
+                "file versions are pinned for unfinished transfers"
+            ),
+        )
+    parser.add_argument(
+        "--visibility",
+        dest="visibility",
+        choices=("auto", "private", "public"),
+        default="auto",
+        help="New destination visibility (default: auto): public sources create public repos; "
+        "private, gated or unknown visibility creates private repos; existing repos are unchanged",
+    )
+    parser.add_argument(
+        "--cache-dir",
+        type=Path,
+        default=Path.home() / ".cache" / "hfms-bridge",
+        help="Metadata checkpoint directory (default: ~/.cache/hfms-bridge); "
+        "stores progress, not file contents",
+    )
+    parser.add_argument(
+        "--cache-max-size",
+        type=byte_size,
+        default=256 * 1024**2,
+        help="Metadata cache budget (default: 256MiB; minimum: 64KiB); evict oldest completed "
+        "tasks first, protect unfinished tasks, fail if protected state cannot fit",
+    )
+    parser.add_argument(
+        "--hf-endpoint",
+        default=os.getenv("HF_ENDPOINT", "https://huggingface.co"),
+        help="HF service base URL (default: HF_ENDPOINT environment variable or "
+        "https://huggingface.co); ms2hf routes hf-mirror.com to official HF for destination requests",
+    )
+    parser.add_argument(
+        "--ms-endpoint",
+        default=os.getenv("MODELSCOPE_ENDPOINT", "https://modelscope.cn"),
+        help="MS service base URL (default: MODELSCOPE_ENDPOINT environment variable or "
+        "https://modelscope.cn)",
+    )
+    parser.add_argument(
+        "--chunk-size",
+        type=byte_size,
+        default=1024**2,
+        help="Maximum source read buffer per active file (default: 1MiB); "
+        "does not control server upload part size",
+    )
+    parser.add_argument(
+        "--attempts",
+        type=int,
+        default=3,
+        help="Maximum attempts per operation, including the first (default: 3)",
+    )
+    parser.add_argument(
+        "--workers",
+        type=int,
+        default=4,
+        help="Concurrent file transfers (default: 4; use 1 for serial transfers)",
+    )
+    parser.add_argument(
+        "--progress",
+        choices=("auto", "on", "off"),
+        default="auto",
+        help="Overall file count and temporary per-file byte bars on stderr "
+        "(default: auto, terminals only); on forces bars, off uses ordinary logs",
+    )
+    parser.add_argument(
+        "--timeout",
+        type=int,
+        default=120,
+        help="Network connect/read timeout in seconds (default: 120); not a whole-file deadline",
+    )
+    parser.add_argument(
+        "--include",
+        action="extend",
+        nargs="+",
+        default=[],
+        help="Transfer paths matching any case-sensitive repository-relative glob; repeatable "
+        "(default: all files); quote patterns such as '*.safetensors'",
+    )
+    parser.add_argument(
+        "--exclude",
+        action="extend",
+        nargs="+",
+        default=[],
+        help="Exclude paths matching any case-sensitive repository-relative glob; repeatable "
+        "and takes priority over --include (default: none)",
+    )
+    parser.add_argument(
+        "--on-conflict",
+        choices=("overwrite", "error", "skip"),
+        default="overwrite",
+        help="Handle different or unverifiable existing content (default: overwrite): "
+        "overwrite updates it, error rejects conflicts before writes, skip leaves it unchanged; "
+        "identical files are always skipped",
+    )
+    parser.add_argument(
+        "--restart",
+        action="store_true",
+        help="Ignore local progress and resolve source versions again (default: resume); "
+        "identical remote files/blobs can still be reused; does not force re-upload",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Print a read-only transfer plan and JSON summary; no payload downloads, remote "
+        "writes or cache changes; missing hashes may leave content equality unknown",
+    )
+    return parser
+
+
+def main(argv=None):
+    args = vars(build_parser().parse_args(argv))
+    ms_token, hf_token = args.pop("ms_token"), args.pop("hf_token")
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+    try:
+        result = Transfer(TransferOptions(**args), ms_token=ms_token, hf_token=hf_token).run()
+    except KeyboardInterrupt:
+        log.info("Interrupted. Run the same command to resume.")
+        return 130
+    except Timeout:
+        log.error("Another process is using this task/cache; retry after it finishes")
+        return 1
+    except requests.RequestException:
+        log.error("Network request failed. Run the same command to resume.")
+        return 1
+    except sqlite3.Error:
+        log.error("Checkpoint database error; check cache permissions and available disk space")
+        return 1
+    except (TransferError, ValueError, OSError) as exc:
+        log.error("%s", exc)
+        log.info("Progress is saved. Run the same command to resume.")
+        return 1
+    if result.get("dry_run"):
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+    else:
+        log.info("Done: %s committed, %s skipped", result["committed"], result["skipped"])
+    return 0

hfms_bridge/http.py

@@ -0,0 +1,327 @@
+"""Explicit HTTP error handling and bounded, integrity-checked request bodies."""
+
+import hashlib
+import re
+from contextlib import contextmanager
+from urllib.parse import urljoin, urlparse
+
+import requests
+from urllib3.exceptions import HTTPError as UrllibError
+
+
+class TransferError(Exception):
+    pass
+
+
+class RetryableTransferError(TransferError):
+    """A response that can be retried without trusting incomplete metadata."""
+
+
+class HTTPError(TransferError):
+    def __init__(self, status, retry_after=None, context=None):
+        # Do not put signed URLs, tokens, or server-echoed bodies in logs.
+        label = "HTTP request" if not context else "HTTP " + context
+        super().__init__(f"{label} failed (status {status})")
+        self.status = status
+        self.retry_after = retry_after
+
+
+@contextmanager
+def error_stage(stage):
+    """Label failures without exposing request URLs, headers or exception bodies."""
+    try:
+        yield
+    except TransferError as exc:
+        if not getattr(exc, "stage", None):
+            exc.stage = stage
+            exc.args = (f"{stage}: {exc}",)
+        raise
+    except (requests.RequestException, UrllibError) as exc:
+        failure = RetryableTransferError(f"{stage}: network {type(exc).__name__}")
+        failure.stage = stage
+        raise failure from exc
+
+
+def check(response):
+    if not 200 <= response.status_code < 300:
+        raise HTTPError(
+            response.status_code, response.headers.get("Retry-After"), response_context(response)
+        )
+
+
+def response_context(response):
+    method = getattr(getattr(response, "request", None), "method", "request")
+    return f"{method} {urlparse(response.url or '').hostname or 'unknown host'}"
+
+
+def payload(response):
+    check(response)
+    body = response.json()
+    if isinstance(body, dict):
+        if str(body.get("Code")) in ("401", "403", "404", "409", "429", "500", "502", "503", "504"):
+            raise HTTPError(
+                int(body["Code"]), response.headers.get("Retry-After"), response_context(response)
+            )
+        if body.get("Success") is False or body.get("success") is False:
+            raise TransferError("API rejected the operation")
+        if body.get("Code") not in (None, 0, 200, "200"):
+            raise TransferError("API returned an unsuccessful business status")
+        return body.get("Data", body)
+    return body
+
+
+def origin(url):
+    p = urlparse(url)
+    return p.scheme, p.hostname, p.port or (443 if p.scheme == "https" else 80)
+
+
+def trusted_api_redirect(source, destination):
+    """Permit same-origin and explicit official aliases, never arbitrary subdomains."""
+    validate_url(destination)
+    if origin(source) == origin(destination):
+        return True
+    left, right = origin(source), origin(destination)
+    if left[0] != "https" or right[0] != "https" or left[2] != 443 or right[2] != 443:
+        return False
+    # The mirror may hand API requests back to official HF. Never allow the
+    # reverse direction to forward official-HF credentials to a third party.
+    if left[1] == "hf-mirror.com" and right[1] in ("huggingface.co", "www.huggingface.co"):
+        return True
+    aliases = (
+        {"modelscope.cn", "www.modelscope.cn"},
+        {"modelscope.ai", "www.modelscope.ai"},
+        {"huggingface.co", "www.huggingface.co", "hf.co"},
+    )
+    return any(left[1] in hosts and right[1] in hosts for hosts in aliases)
+
+
+def validate_url(url):
+    p = urlparse(url)
+    if p.username or p.password or not p.hostname or p.fragment:
+        raise ValueError("Invalid endpoint or transfer URL")
+    if p.scheme != "https" and not (
+        p.scheme == "http" and p.hostname in ("localhost", "127.0.0.1", "::1")
+    ):
+        raise ValueError("HTTPS is required except for local test servers")
+    return url.rstrip("/")
+
+
+class HTTP:
+    def __init__(self, timeout=120):
+        self.session = requests.Session()
+        self.session.headers["User-Agent"] = "hfms-bridge/0.4.0"
+        self.timeout = (15, timeout)
+
+    def close(self):
+        self.session.close()
+
+    def request(self, method, url, **kwargs):
+        validate_url(url)
+        # No automatic retries: consumed streaming bodies cannot be replayed.
+        return self.session.request(method, url, timeout=self.timeout, **kwargs)
+
+    def api_request(self, method, url, **kwargs):
+        """Follow API redirects for replayable metadata requests only.
+
+        Streamed PUT bodies deliberately do not use this method: replaying a
+        consumed stream could silently upload truncated content.
+        """
+        if "data" in kwargs and not isinstance(kwargs["data"], bytes):
+            raise ValueError("api_request does not accept streaming request bodies")
+        for value in (kwargs.get("files") or {}).values():
+            if not (
+                isinstance(value, tuple)
+                and len(value) == 2
+                and value[0] is None
+                and isinstance(value[1], str)
+            ):
+                raise ValueError("Only replayable text form fields are allowed")
+        kwargs.pop("allow_redirects", None)
+        visited = set()
+        for _ in range(10):
+            if url in visited:
+                raise TransferError("API redirect loop detected")
+            visited.add(url)
+            response = self.request(method, url, allow_redirects=False, **kwargs)
+            if response.status_code not in (301, 302, 303, 307, 308):
+                return response
+            status = response.status_code
+            location = response.headers.get("Location")
+            actual_url = response.url or url
+            response.close()
+            if not location:
+                raise TransferError("API redirect omitted Location")
+            if method.upper() not in ("GET", "HEAD") and status not in (307, 308):
+                raise TransferError("Refusing API redirect that could change the request method")
+            target = urljoin(actual_url, location)
+            if not trusted_api_redirect(actual_url, target):
+                raise TransferError(
+                    "Refusing API redirect to an untrusted origin; "
+                    "check --hf-endpoint / --ms-endpoint"
+                )
+            url = target
+            # Location is the complete redirect target; never append old query params twice.
+            kwargs.pop("params", None)
+        raise TransferError("Too many API redirects")
+
+    def download(self, url, token=None, offset=0, total=None):
+        headers = {"Accept-Encoding": "identity"}
+        if offset:
+            headers["Range"] = f"bytes={offset}-"
+        if token:
+            headers["Authorization"] = "Bearer " + token
+        # Handle redirects ourselves so cookies and auth cannot cross origins.
+        for _ in range(10):
+            response = self.request("GET", url, headers=headers, stream=True, allow_redirects=False)
+            if response.status_code in (301, 302, 303, 307, 308):
+                next_url = urljoin(url, response.headers.get("Location", ""))
+                response.close()
+                if origin(next_url) != origin(url):
+                    headers.pop("Authorization", None)
+                    self.session.cookies.clear()
+                url = next_url
+                continue
+            try:
+                if offset and response.status_code == 416:
+                    raise RetryableTransferError("Range response rejected the requested offset")
+                check(response)
+                if offset:
+                    match = re.fullmatch(
+                        r"bytes (\d+)-(\d+)/(\d+)",
+                        response.headers.get("Content-Range", ""),
+                    )
+                    if (
+                        response.status_code != 206
+                        or not match
+                        or tuple(map(int, match.groups())) != (offset, total - 1, total)
+                    ):
+                        raise RetryableTransferError("Range response has invalid status or range")
+                    length = response.headers.get("Content-Length")
+                    if length is not None and length != str(total - offset):
+                        raise RetryableTransferError("Range response has invalid length")
+                elif response.status_code != 200:
+                    raise TransferError("Expected a full-file HTTP 200 response")
+                if response.headers.get("Content-Encoding", "identity") != "identity":
+                    if offset:
+                        raise RetryableTransferError(
+                            "Range response ignored Accept-Encoding: identity"
+                        )
+                    raise TransferError("Server ignored Accept-Encoding: identity")
+                return response
+            except BaseException:
+                response.close()
+                raise
+        raise TransferError("Too many source redirects")
+
+
+class CheckedStream:
+    """Non-seekable body with a known length; read() is always bounded.
+
+    Check the digest BEFORE returning the last bytes to the uploader. This
+    prevents a corrupt source from completing an upload under the expected OID.
+    """
+
+    def __init__(self, raw, size, sha256, chunk_size=1024 * 1024, on_read=None, stage="HF read"):
+        self.stage = stage
+        self.raw, self.size, self.expected = raw, size, sha256
+        self.on_read = on_read
+        self.chunk_size = chunk_size
+        self.count = 0
+        self.hash = hashlib.sha256()
+        self.git_hash = hashlib.sha1(f"blob {size}\0".encode(), usedforsecurity=False)
+
+    def __len__(self):
+        return self.size
+
+    def read(self, size=-1):
+        with error_stage(self.stage):
+            return self._read(size)
+
+    def _read(self, size):
+        remaining = self.size - self.count
+        if not remaining:
+            return b""
+        amount = min(remaining, self.chunk_size, size if size >= 0 else self.chunk_size)
+        if amount == 0:
+            return b""
+        data = self._read_raw(amount)
+        if not data:
+            raise TransferError("Source ended before its declared size")
+        if len(data) > amount:
+            raise TransferError("Source exceeded requested read size")
+        self.count += len(data)
+        self.hash.update(data)
+        self.git_hash.update(data)
+        if self.count == self.size:
+            if self._read_raw(1):
+                raise TransferError("Source exceeded its declared size")
+            self.verify()
+        if self.on_read is not None:
+            self.on_read(len(data))
+        return data
+
+    def _read_raw(self, size):
+        try:
+            return self.raw.read(size)
+        except UrllibError as exc:
+            raise requests.ConnectionError("Source connection interrupted") from exc
+
+    def verify(self):
+        if self.count != self.size:
+            raise TransferError("Upload did not consume the complete source")
+        if self.expected and self.hash.hexdigest() != self.expected:
+            raise TransferError("Source SHA-256 mismatch")
+
+
+class ResumingReader:
+    """Reconnect the pinned source while keeping the destination PUT alive.
+
+    Only returned bytes count: a raw.read() that raises may have consumed partial
+    bytes internally, but those bytes must be fetched again. No payload is saved.
+    The retry budget covers the entire reader, not each successful chunk.
+    """
+
+    def __init__(self, opener, size, attempts, backoff, cancelled, on_resume, stage="HF read"):
+        self.stage = stage
+        self.opener, self.size = opener, size
+        self.attempts, self.backoff = attempts, backoff
+        self.cancelled, self.on_resume = cancelled, on_resume
+        self.response = None
+        self.offset = 0
+        self.reconnects = 0
+
+    def __enter__(self):
+        with error_stage(self.stage):
+            self.cancelled()
+            self.response = self.opener(0)
+        return self
+
+    def __exit__(self, *args):
+        if self.response is not None:
+            self.response.close()
+
+    def read(self, size):
+        with error_stage(self.stage):
+            while True:
+                self.cancelled()
+                try:
+                    if self.response is None:
+                        self.response = self.opener(self.offset)
+                    data = self.response.raw.read(size)
+                    if not data and self.offset < self.size:
+                        raise requests.ConnectionError("Source ended early")
+                    self.offset += len(data)
+                    return data
+                except (requests.RequestException, UrllibError) as exc:
+                    if self.response is not None:
+                        self.response.close()
+                        self.response = None
+                    # At EOF no remaining range exists; do not fabricate bytes.
+                    if self.reconnects >= self.attempts - 1 or self.offset >= self.size:
+                        raise RetryableTransferError(
+                            f"Source recovery exhausted ({type(exc).__name__}); "
+                            "restarting file if attempts remain"
+                        ) from exc
+                    self.reconnects += 1
+                    self.on_resume(self.offset, self.reconnects)
+                    self.backoff(min(2 ** (self.reconnects - 1), 30))

hfms_bridge/hubs/__init__.py

@@ -0,0 +1,5 @@
+from .common import File
+from .hf import HuggingFace
+from .ms import ModelScope
+
+__all__ = ["File", "HuggingFace", "ModelScope"]

hfms_bridge/hubs/common.py

@@ -0,0 +1,67 @@
+"""Shared file metadata and streamed LFS transport."""
+
+import re
+from dataclasses import dataclass
+from ..http import check, payload, error_stage
+
+INLINE_LIMIT = 1024 * 1024
+METADATA = {"README.MD", ".GITATTRIBUTES", "CONFIGURATION.JSON"}
+
+
+def repo_name(value):
+    if not re.fullmatch(r"[\w.-]+/[\w.-]+", value, flags=re.ASCII) or any(
+        part in (".", "..") for part in value.split("/")
+    ):
+        raise ValueError("Repository must be owner/name")
+    return value
+
+
+@dataclass(frozen=True)
+class File:
+    path: str
+    size: int
+    sha256: str = ""
+    revision: str = ""
+    lfs: bool = True
+
+    def __post_init__(self):
+        if (
+            not self.path
+            or self.path.startswith("/")
+            or "\\" in self.path
+            or any(p in ("", ".", "..") for p in self.path.split("/"))
+        ):
+            raise ValueError("Invalid repository file path")
+        if self.size < 0:
+            raise ValueError("Negative file size")
+        if self.sha256 and not re.fullmatch("[0-9a-f]{64}", self.sha256):
+            raise ValueError("Invalid SHA-256 metadata")
+
+
+class BlobTarget:
+    def upload(self, actions, stream, sha256, size):
+        action = actions["upload"]
+        headers = self._action_headers(action)
+        headers["Content-Length"] = str(size)
+        headers.pop("Transfer-Encoding", None)
+        # Redirect replay is unsafe for a non-seekable stream. Renegotiate on failure.
+        with self.blob_http.request(
+            "PUT", action["href"], data=stream, headers=headers, allow_redirects=False
+        ) as response:
+            check(response)
+            if response.content:
+                payload(response)
+        stream.verify()
+        if "verify" in actions:
+            action = actions["verify"]
+            with error_stage(f"{self.label} verify"):
+                with self.blob_http.request(
+                    "POST",
+                    action["href"],
+                    headers=self._action_headers(action),
+                    json={"oid": sha256, "size": size},
+                    allow_redirects=False,
+                ) as response:
+                    check(response)
+                    if response.content:
+                        payload(response)