modelscope-hub 0.0.0__tar.gz

modelscope_hub-0.0.0/PKG-INFO

@@ -0,0 +1,31 @@
+Metadata-Version: 2.4
+Name: modelscope-hub
+Version: 0.0.0
+Summary: An OpenAPI-first Python SDK for interacting with the ModelScope Hub platform.
+License: Apache-2.0
+Keywords: modelscope,hub,sdk,openapi,machine-learning
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.28
+Requires-Dist: tqdm>=4.64.0
+Requires-Dist: filelock>=3.9
+Requires-Dist: urllib3>=1.26
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-mock; extra == "dev"
+Requires-Dist: responses>=0.20; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+
+ModelScope hub
+

modelscope_hub-0.0.0/README.md

@@ -0,0 +1,2 @@
+ModelScope hub
+

modelscope_hub-0.0.0/pyproject.toml

@@ -0,0 +1,68 @@
+[project]
+name = "modelscope-hub"
+dynamic = ["version"]
+description = "An OpenAPI-first Python SDK for interacting with the ModelScope Hub platform."
+requires-python = ">=3.10"
+license = {text = "Apache-2.0"}
+readme = "README.md"
+keywords = ["modelscope", "hub", "sdk", "openapi", "machine-learning"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "requests>=2.28",
+    "tqdm>=4.64.0",
+    "filelock>=3.9",
+    "urllib3>=1.26",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "pytest-mock",
+    "responses>=0.20",
+    "ruff",
+    "mypy",
+]
+
+[project.scripts]
+modelscope = "modelscope_hub.cli.main:run_cmd"
+ms = "modelscope_hub.cli.main:run_cmd"
+
+[build-system]
+requires = ["setuptools>=68.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.dynamic]
+version = {attr = "modelscope_hub.version.__version__"}
+
+[tool.ruff]
+target-version = "py313"
+line-length = 120
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "W", "UP"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+    "remote: tests requiring remote API access (need .env credentials)",
+]
+addopts = "-v --tb=short"
+
+[tool.mypy]
+python_version = "3.13"
+strict = false
+warn_return_any = true

modelscope_hub-0.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

modelscope_hub-0.0.0/src/modelscope_hub/__init__.py

@@ -0,0 +1,71 @@
+"""ModelScope Hub SDK.
+
+An OpenAPI-first Python SDK for interacting with the ModelScope Hub platform.
+
+The public surface is intentionally small: most callers should construct a
+single :class:`HubApi` instance and call its methods. The data classes
+exported alongside it provide structured return types for type-checked code.
+"""
+
+from __future__ import annotations
+
+from .api import HubApi
+from .config import HubConfig, get_default_config, set_default_config
+from .constants import License, RepoType, Visibility
+from .errors import (
+    APIError,
+    AuthenticationError,
+    CacheError,
+    FileIntegrityError,
+    HubError,
+    NetworkError,
+    NotFoundError,
+    PermissionError,
+    RateLimitError,
+    ServerError,
+    ValidationError,
+)
+from .types import (
+    CacheInfo,
+    CachedRepoInfo,
+    CommitInfo,
+    FileInfo,
+    PagedResult,
+    RepoInfo,
+    UserInfo,
+)
+from .version import __version__
+
+__all__ = [
+    "__version__",
+    # Facade
+    "HubApi",
+    # Configuration
+    "HubConfig",
+    "get_default_config",
+    "set_default_config",
+    # Enums
+    "License",
+    "RepoType",
+    "Visibility",
+    # Data classes
+    "CacheInfo",
+    "CachedRepoInfo",
+    "CommitInfo",
+    "FileInfo",
+    "PagedResult",
+    "RepoInfo",
+    "UserInfo",
+    # Errors
+    "APIError",
+    "AuthenticationError",
+    "CacheError",
+    "FileIntegrityError",
+    "HubError",
+    "NetworkError",
+    "NotFoundError",
+    "PermissionError",
+    "RateLimitError",
+    "ServerError",
+    "ValidationError",
+]

modelscope_hub-0.0.0/src/modelscope_hub/_cache_manager.py

@@ -0,0 +1,190 @@
+"""Cache management utilities.
+
+Provides scanning and cleanup of the local blob/snapshot cache produced
+by :class:`~._download.DownloadManager`.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+from .config import get_default_config
+from .constants import RepoType
+from .errors import CacheError
+from .types import CacheInfo, CachedRepoInfo
+from .utils.logger import get_logger
+
+logger = get_logger("cache")
+
+# Repo types to scan by default
+_DEFAULT_SCAN_TYPES = [RepoType.MODEL, RepoType.DATASET, RepoType.STUDIO, RepoType.MCP]
+
+
+def scan_cache(cache_dir: Path | None = None) -> CacheInfo:
+    """Scan the local cache and return metadata about cached repositories.
+
+    Parameters
+    ----------
+    cache_dir:
+        Override for the cache directory. Defaults to the SDK config default.
+
+    Returns
+    -------
+    CacheInfo
+        Summary of all cached repositories, total size, etc.
+    """
+    config = get_default_config()
+    root = Path(cache_dir) if cache_dir else config.cache_dir
+
+    if not root.is_dir():
+        return CacheInfo(repos=[], total_size=0, cache_dir=str(root))
+
+    repos: list[CachedRepoInfo] = []
+    total_size = 0
+
+    for repo_type in _DEFAULT_SCAN_TYPES:
+        segment = f"{repo_type}s"
+        type_dir = root / segment
+        if not type_dir.is_dir():
+            continue
+
+        for repo_dir in type_dir.iterdir():
+            if not repo_dir.is_dir():
+                continue
+
+            # Compute size
+            size = _dir_size(repo_dir)
+            total_size += size
+
+            # Count files
+            nb_files = sum(1 for _ in repo_dir.rglob("*") if _.is_file())
+
+            # Determine revision from snapshot dirs
+            snapshots_dir = repo_dir / "snapshots"
+            revision = None
+            if snapshots_dir.is_dir():
+                revisions = [d.name for d in snapshots_dir.iterdir() if d.is_dir()]
+                revision = revisions[0] if len(revisions) == 1 else ",".join(revisions[:5])
+
+            # Last access time
+            try:
+                last_accessed_ts = repo_dir.stat().st_atime
+            except OSError:
+                last_accessed_ts = 0
+
+            # Decode repo_id from directory name (owner--name → owner/name)
+            repo_id = repo_dir.name.replace("--", "/")
+
+            repos.append(CachedRepoInfo(
+                repo_id=repo_id,
+                repo_type=repo_type,
+                revision=revision,
+                size_on_disk=size,
+                nb_files=nb_files,
+                last_accessed=last_accessed_ts if last_accessed_ts > 0 else None,
+                local_path=str(repo_dir),
+            ))
+
+    return CacheInfo(
+        repos=repos,
+        total_size=total_size,
+        cache_dir=str(root),
+    )
+
+
+def clear_cache(
+    cache_dir: Path | None = None,
+    repo_type: str | None = None,
+    repo_id: str | None = None,
+) -> int:
+    """Remove cached data from disk.
+
+    Parameters
+    ----------
+    cache_dir:
+        Override for the cache directory. Defaults to the SDK config default.
+    repo_type:
+        If given, only clear caches of this repo type.
+    repo_id:
+        If given, only clear the cache for this specific repository.
+        Must be used with ``repo_type``.
+
+    Returns
+    -------
+    int
+        Number of bytes freed.
+
+    Raises
+    ------
+    CacheError
+        On filesystem errors.
+    """
+    config = get_default_config()
+    root = Path(cache_dir) if cache_dir else config.cache_dir
+
+    # Guard against accidental nuke: passing only ``repo_id`` would otherwise
+    # silently fall through to the "clear everything" branch below.
+    if repo_id and not repo_type:
+        raise CacheError("repo_type is required when repo_id is specified")
+
+    if not root.is_dir():
+        logger.info("Cache directory does not exist: %s", root)
+        return 0
+
+    freed = 0
+
+    if repo_id and repo_type:
+        # Clear specific repo
+        segment = f"{repo_type}s" if not repo_type.endswith("s") else repo_type
+        safe_id = repo_id.replace("/", "--")
+        target = root / segment / safe_id
+        if target.is_dir():
+            freed = _dir_size(target)
+            _safe_rmtree(target)
+            logger.info("Cleared cache for %s/%s (%d bytes)", repo_type, repo_id, freed)
+    elif repo_type:
+        # Clear all repos of this type
+        segment = f"{repo_type}s" if not repo_type.endswith("s") else repo_type
+        type_dir = root / segment
+        if type_dir.is_dir():
+            freed = _dir_size(type_dir)
+            _safe_rmtree(type_dir)
+            logger.info("Cleared all %s caches (%d bytes)", repo_type, freed)
+    else:
+        # Clear everything
+        for repo_t in _DEFAULT_SCAN_TYPES:
+            segment = f"{repo_t}s"
+            type_dir = root / segment
+            if type_dir.is_dir():
+                freed += _dir_size(type_dir)
+                _safe_rmtree(type_dir)
+        logger.info("Cleared all caches (%d bytes)", freed)
+
+    return freed
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+def _dir_size(path: Path) -> int:
+    """Compute total size of all files under ``path`` recursively."""
+    total = 0
+    try:
+        for f in path.rglob("*"):
+            if f.is_file():
+                try:
+                    total += f.stat().st_size
+                except OSError:
+                    pass
+    except OSError:
+        pass
+    return total
+
+
+def _safe_rmtree(path: Path) -> None:
+    """Remove a directory tree, raising CacheError on failure."""
+    try:
+        shutil.rmtree(path)
+    except OSError as exc:
+        raise CacheError(f"Failed to remove {path}: {exc}") from exc

modelscope_hub-0.0.0/src/modelscope_hub/_download.py

@@ -0,0 +1,293 @@
+"""Internal file download implementation.
+
+Supports single-file and whole-repo (snapshot) downloads with:
+- HTTP Range-based resume
+- SHA256 integrity verification
+- tqdm progress display
+- Parallel downloads via ThreadPoolExecutor
+- Local snapshot cache directory management
+"""
+
+from __future__ import annotations
+
+import fnmatch
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from tqdm.auto import tqdm
+
+from .constants import DOWNLOAD_CHUNK_SIZE
+from .errors import FileIntegrityError, NetworkError
+from .utils.file_utils import compute_hash, ensure_dir
+from .utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from .config import HubConfig
+    from ._legacy_api import LegacyClient
+
+logger = get_logger("download")
+
+
+def _matches_patterns(path: str, patterns: list[str] | None) -> bool:
+    """Check if path matches any of the glob patterns."""
+    if not patterns:
+        return False
+    return any(fnmatch.fnmatch(path, pat) for pat in patterns)
+
+
+class DownloadManager:
+    """Internal file download implementation.
+
+    Dependencies are injected via constructor to keep this class testable.
+    """
+
+    def __init__(self, legacy_client: "LegacyClient", config: "HubConfig") -> None:
+        self._client = legacy_client
+        self._config = config
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+    def download_file(
+        self,
+        repo_id: str,
+        repo_type: str,
+        file_path: str,
+        revision: str = "master",
+        cache_dir: Path | None = None,
+        force: bool = False,
+    ) -> Path:
+        """Download a single file from a repository.
+
+        Uses the snapshot cache layout::
+
+            {cache_dir}/{type}s/{owner}--{name}/snapshots/{revision}/{file_path}
+
+        Parameters
+        ----------
+        repo_id:
+            Repository identifier (``owner/name``).
+        repo_type:
+            One of the :class:`~.constants.RepoType` values.
+        file_path:
+            Path within the repository.
+        revision:
+            Branch, tag, or commit hash.
+        cache_dir:
+            Override for the default cache directory.
+        force:
+            Re-download even if file exists in cache.
+
+        Returns
+        -------
+        Path
+            Absolute path to the downloaded (or cached) file on disk.
+        """
+        root = self._repo_cache_dir(repo_id, repo_type, cache_dir)
+        snapshot_dir = root / "snapshots" / revision
+        target = snapshot_dir / file_path
+
+        # Return cached file if valid and not forced
+        if not force and target.exists():
+            logger.debug("Cache hit: %s", target)
+            return target
+
+        # Ensure the snapshot directory exists before streaming into it.
+        ensure_dir(target.parent)
+
+        # Perform download with resume support
+        tmp_path = self._download_with_resume(repo_id, repo_type, file_path, revision, target)
+
+        return tmp_path
+
+    def download_repo(
+        self,
+        repo_id: str,
+        repo_type: str,
+        revision: str = "master",
+        cache_dir: Path | None = None,
+        allow_patterns: list[str] | None = None,
+        ignore_patterns: list[str] | None = None,
+        max_workers: int = 4,
+    ) -> Path:
+        """Download an entire repository (snapshot download).
+
+        Parameters
+        ----------
+        repo_id:
+            Repository identifier (``owner/name``).
+        repo_type:
+            One of the :class:`~.constants.RepoType` values.
+        revision:
+            Branch, tag, or commit hash.
+        cache_dir:
+            Override for the default cache directory.
+        allow_patterns:
+            Only files matching these globs will be downloaded.
+        ignore_patterns:
+            Files matching these globs will be skipped.
+        max_workers:
+            Number of parallel download threads.
+
+        Returns
+        -------
+        Path
+            Absolute path to the snapshot directory.
+        """
+        root = self._repo_cache_dir(repo_id, repo_type, cache_dir)
+        snapshot_dir = ensure_dir(root / "snapshots" / revision)
+
+        # Fetch file tree
+        files = self._client.list_repo_files(
+            repo_id=repo_id,
+            repo_type=repo_type,
+            revision=revision,
+            recursive=True,
+        )
+
+        # Filter files
+        file_paths: list[str] = []
+        for f in files:
+            path = f.get("Path") or f.get("path") or f.get("Name") or ""
+            ftype = f.get("Type") or f.get("type") or "blob"
+            if ftype == "tree":
+                continue
+            if not path:
+                continue
+            if allow_patterns and not _matches_patterns(path, allow_patterns):
+                continue
+            if ignore_patterns and _matches_patterns(path, ignore_patterns):
+                continue
+            file_paths.append(path)
+
+        if not file_paths:
+            logger.info("No files to download for %s@%s", repo_id, revision)
+            return snapshot_dir
+
+        logger.info("Downloading %d files from %s@%s", len(file_paths), repo_id, revision)
+
+        # Parallel download
+        errors: list[str] = []
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = {
+                executor.submit(
+                    self.download_file,
+                    repo_id=repo_id,
+                    repo_type=repo_type,
+                    file_path=fp,
+                    revision=revision,
+                    cache_dir=cache_dir,
+                ): fp
+                for fp in file_paths
+            }
+
+            with tqdm(total=len(file_paths), desc="Downloading", unit="file") as pbar:
+                for future in as_completed(futures):
+                    fp = futures[future]
+                    try:
+                        future.result()
+                    except Exception as exc:
+                        errors.append(f"{fp}: {exc}")
+                        logger.error("Failed to download %s: %s", fp, exc)
+                    finally:
+                        pbar.update(1)
+
+        if errors:
+            logger.warning("%d file(s) failed to download", len(errors))
+
+        return snapshot_dir
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+    def _repo_cache_dir(
+        self,
+        repo_id: str,
+        repo_type: str,
+        cache_dir: Path | None = None,
+    ) -> Path:
+        """Compute the cache directory for a given repo."""
+        base = cache_dir or self._config.cache_dir
+        segment = f"{repo_type}s" if not repo_type.endswith("s") else repo_type
+        # Encode repo_id: owner/name → owner--name for filesystem safety
+        safe_id = repo_id.replace("/", "--")
+        return ensure_dir(base / segment / safe_id)
+
+    def _download_with_resume(
+        self,
+        repo_id: str,
+        repo_type: str,
+        file_path: str,
+        revision: str,
+        target: Path,
+    ) -> Path:
+        """Download a file with HTTP Range resume support."""
+        # Use a temp file for partial downloads
+        tmp_path = target.with_suffix(target.suffix + ".incomplete")
+
+        existing_size = 0
+        if tmp_path.exists():
+            existing_size = tmp_path.stat().st_size
+
+        # Prepare headers for resume
+        extra_headers: dict[str, str] = {}
+        if existing_size > 0:
+            extra_headers["Range"] = f"bytes={existing_size}-"
+            logger.debug("Resuming download from byte %d", existing_size)
+
+        try:
+            resp = self._client.download_stream(
+                repo_id=repo_id,
+                repo_type=repo_type,
+                file_path=file_path,
+                revision=revision,
+                headers=extra_headers if extra_headers else None,
+            )
+        except Exception as exc:
+            raise NetworkError(f"Download failed for {file_path}: {exc}") from exc
+
+        # Determine total size
+        content_length = resp.headers.get("Content-Length")
+        total_size = int(content_length) if content_length else None
+        is_resumed = resp.status_code == 206
+
+        if is_resumed and total_size:
+            total_size += existing_size
+
+        # Write to temp file
+        mode = "ab" if is_resumed else "wb"
+        if not is_resumed:
+            existing_size = 0
+
+        with tqdm(
+            total=total_size,
+            initial=existing_size,
+            unit="B",
+            unit_scale=True,
+            desc=Path(file_path).name,
+            leave=False,
+        ) as pbar:
+            with open(tmp_path, mode) as fh:
+                for chunk in resp.iter_content(chunk_size=DOWNLOAD_CHUNK_SIZE):
+                    if chunk:
+                        fh.write(chunk)
+                        pbar.update(len(chunk))
+
+        # Move temp → final
+        tmp_path.replace(target)
+        logger.debug("Downloaded: %s", target)
+        return target
+
+    def verify_file(self, file_path: Path, expected_sha256: str) -> bool:
+        """Verify a downloaded file's SHA256 hash.
+
+        Raises :class:`~.errors.FileIntegrityError` on mismatch.
+        """
+        actual = compute_hash(file_path, "sha256")
+        if actual != expected_sha256:
+            raise FileIntegrityError(
+                f"Hash mismatch for {file_path.name}: "
+                f"expected {expected_sha256[:16]}..., got {actual[:16]}..."
+            )
+        return True