modeldownloaderutil 1.0.0__py3-none-any.whl

modeldownloaderutil/__init__.py

@@ -0,0 +1,10 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from .cache import cache_dir
+from .downloader import download_model
+
+__all__ = ["download_model", "cache_dir"]
+__version__ = "2.0.0"

modeldownloaderutil/_download.py

@@ -0,0 +1,46 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+
+import requests
+from tenacity import retry, stop_after_attempt, wait_exponential
+from tqdm import tqdm
+
+from .cache import commit_download, incomplete_path
+
+
+@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=8))
+def download_http(url: str, destination: Path) -> Path:
+    tmp = incomplete_path(destination)
+    tmp.parent.mkdir(parents=True, exist_ok=True)
+    with requests.get(url, stream=True, timeout=300) as response:
+        response.raise_for_status()
+        total = int(response.headers.get("Content-Length", 0)) or None
+        with open(tmp, "wb") as handle, tqdm(total=total, unit="B", unit_scale=True, unit_divisor=1024, desc=destination.name, disable=total is None) as bar:
+            for chunk in response.iter_content(chunk_size=1024 * 1024):
+                if chunk:
+                    handle.write(chunk)
+                    bar.update(len(chunk))
+    return commit_download(tmp, destination)
+
+
+def download_s3_object(client, bucket: str, key: str, destination: Path) -> Path:
+    tmp = incomplete_path(destination)
+    tmp.parent.mkdir(parents=True, exist_ok=True)
+    size = client.head_object(Bucket=bucket, Key=key).get("ContentLength")
+    with tqdm(total=size, unit="B", unit_scale=True, unit_divisor=1024, desc=destination.name, disable=not size) as bar:
+        client.download_file(bucket, key, str(tmp), Callback=lambda n: bar.update(n))
+    return commit_download(tmp, destination)
+
+
+def download_with_cache(destination: Path, fetch: Callable[[Path], Path], *, force: bool = False) -> Path:
+    if destination.exists() and not force:
+        return destination.resolve()
+    fetch(destination)
+    return destination.resolve()

modeldownloaderutil/cache.py

@@ -0,0 +1,83 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+from urllib.parse import urlparse
+
+from platformdirs import user_cache_dir
+
+APP_NAME = "model-downloader"
+APP_AUTHOR = "treeleaf"
+_SCHEME_ALIASES = {"minio": "s3", "rustfs": "s3"}
+
+
+def cache_dir() -> Path:
+    if cache := os.environ.get("MODEL_CACHE_DIR"):
+        return Path(cache).expanduser()
+    return Path(user_cache_dir(APP_NAME, appauthor=APP_AUTHOR))
+
+
+def cache_path(source: str) -> Path:
+    parsed = urlparse(source)
+    scheme = _SCHEME_ALIASES.get(parsed.scheme, parsed.scheme)
+    root = cache_dir()
+
+    if scheme in ("s3", "gs"):
+        key = parsed.path.lstrip("/")
+        if not parsed.netloc or not key:
+            raise ValueError(f"Invalid object URI: {source}")
+        return root / scheme / parsed.netloc / key
+
+    if scheme in ("http", "https"):
+        path = parsed.path.lstrip("/") or "download"
+        return root / "url" / _safe_segment(parsed.netloc) / path
+
+    if scheme == "git":
+        file_path = parsed.fragment
+        if not file_path:
+            raise ValueError(f"Invalid git URI (missing #<path>): {source}")
+        repo_url = f"{parsed.netloc}{parsed.path}" if parsed.netloc else parsed.path.lstrip("/")
+        if not repo_url:
+            raise ValueError(f"Invalid git URI: {source}")
+        return git_file_path(repo_url, file_path)
+
+    raise ValueError(f"Cannot derive cache path for: {source}")
+
+
+def git_repo_dir(repo_url: str) -> Path:
+    return cache_dir() / "git" / _safe_segment(repo_url)
+
+
+def git_file_path(repo_url: str, file_path: str) -> Path:
+    return git_repo_dir(repo_url) / file_path
+
+
+def incomplete_path(path: Path) -> Path:
+    return path.with_suffix(path.suffix + ".incomplete")
+
+
+def commit_download(tmp: Path, destination: Path) -> Path:
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    tmp.replace(destination)
+    return destination
+
+
+def _safe_segment(value: str) -> str:
+    return re.sub(r"[^a-zA-Z0-9._-]+", "_", value)
+
+
+def normalize_s3_uri(source: str) -> tuple[str, str | None]:
+    endpoint: str | None = None
+    if source.startswith("minio://"):
+        endpoint = os.environ.get("MINIO_ENDPOINT")
+        source = "s3://" + source[len("minio://") :]
+    elif source.startswith("rustfs://"):
+        endpoint = os.environ.get("RUSTFS_ENDPOINT")
+        source = "s3://" + source[len("rustfs://") :]
+    return source, endpoint

modeldownloaderutil/downloader.py

@@ -0,0 +1,32 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .env import load_env
+from .providers.base import ModelProvider
+from .providers.gcs import GCSProvider
+from .providers.git_lfs import GitLFSProvider
+from .providers.http import HttpProvider
+from .providers.local import LocalProvider
+from .providers.s3 import S3Provider
+
+_PROVIDERS: tuple[ModelProvider, ...] = (
+    LocalProvider(),
+    HttpProvider(),
+    S3Provider(),
+    GCSProvider(),
+    GitLFSProvider(),
+)
+
+
+def download_model(source: str, *, force_download: bool = False) -> Path:
+    load_env()
+    for provider in _PROVIDERS:
+        if provider.can_handle(source):
+            return provider.download(source, force=force_download)
+    raise ValueError(f"Unsupported source: {source}")

modeldownloaderutil/env.py

@@ -0,0 +1,33 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+_LOADED = False
+
+
+def load_env() -> None:
+    global _LOADED
+    if _LOADED:
+        return
+    from dotenv import load_dotenv
+
+    for path in _env_candidates():
+        load_dotenv(path, override=False)
+    _LOADED = True
+
+
+def _env_candidates() -> list[Path]:
+    seen: set[Path] = set()
+    out: list[Path] = []
+    for base in (Path.cwd(), Path(__file__).resolve().parents[1]):
+        for parent in (base, *base.parents):
+            env = (parent / ".env").resolve()
+            if env.is_file() and env not in seen:
+                seen.add(env)
+                out.append(env)
+    return out

modeldownloaderutil/providers/__init__.py

@@ -0,0 +1,5 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""

modeldownloaderutil/providers/base.py

@@ -0,0 +1,17 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+
+class ModelProvider(ABC):
+    @abstractmethod
+    def can_handle(self, source: str) -> bool: ...
+
+    @abstractmethod
+    def download(self, source: str, *, force: bool = False) -> Path: ...

modeldownloaderutil/providers/gcs.py

@@ -0,0 +1,36 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from urllib.parse import urlparse
+
+from google.cloud import storage
+
+from .._download import download_with_cache
+from ..cache import cache_path, commit_download, incomplete_path
+from .base import ModelProvider
+
+
+class GCSProvider(ModelProvider):
+    def can_handle(self, source: str) -> bool:
+        return source.startswith("gs://")
+
+    def download(self, source: str, *, force: bool = False) -> Path:
+        destination = cache_path(source)
+        parsed = urlparse(source)
+        bucket = parsed.netloc
+        blob_name = parsed.path.lstrip("/")
+        if not bucket or not blob_name:
+            raise ValueError(f"Invalid GCS URI: {source}")
+
+        def fetch(path: Path) -> Path:
+            tmp = incomplete_path(path)
+            tmp.parent.mkdir(parents=True, exist_ok=True)
+            storage.Client().bucket(bucket).blob(blob_name).download_to_filename(str(tmp))
+            return commit_download(tmp, path)
+
+        return download_with_cache(destination, fetch, force=force)

modeldownloaderutil/providers/git_lfs.py

@@ -0,0 +1,45 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+from ..cache import git_file_path, git_repo_dir
+from .base import ModelProvider
+
+
+class GitLFSProvider(ModelProvider):
+    def can_handle(self, source: str) -> bool:
+        return source.startswith("git+")
+
+    def download(self, source: str, *, force: bool = False) -> Path:
+        canonical = source[4:]
+        repo_url, _, file_path = canonical.partition("#")
+        if not repo_url or not file_path:
+            raise ValueError(f"Invalid git source (expected git+<url>#<path>): {source}")
+
+        target = git_file_path(repo_url, file_path)
+        repo_dir = git_repo_dir(repo_url)
+        if target.exists() and not force:
+            return target.resolve()
+
+        repo_dir.mkdir(parents=True, exist_ok=True)
+        if not (repo_dir / ".git").exists():
+            self._run(["git", "clone", "--filter=blob:none", "--no-checkout", repo_url, str(repo_dir)])
+            self._run(["git", "-C", str(repo_dir), "sparse-checkout", "init", "--no-cone"])
+
+        self._run(["git", "-C", str(repo_dir), "sparse-checkout", "set", file_path])
+        self._run(["git", "-C", str(repo_dir), "checkout", "HEAD"])
+        self._run(["git", "-C", str(repo_dir), "lfs", "pull", "--include", file_path])
+
+        if not target.exists():
+            raise FileNotFoundError(f"File not found after git LFS pull: {file_path} in {repo_url}")
+        return target.resolve()
+
+    @staticmethod
+    def _run(cmd: list[str]) -> None:
+        subprocess.run(cmd, check=True)

modeldownloaderutil/providers/http.py

@@ -0,0 +1,21 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .._download import download_http, download_with_cache
+from ..cache import cache_path
+from .base import ModelProvider
+
+
+class HttpProvider(ModelProvider):
+    def can_handle(self, source: str) -> bool:
+        return source.startswith(("http://", "https://"))
+
+    def download(self, source: str, *, force: bool = False) -> Path:
+        destination = cache_path(source)
+        return download_with_cache(destination, lambda path: download_http(source, path), force=force)

modeldownloaderutil/providers/local.py

@@ -0,0 +1,21 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .base import ModelProvider
+
+
+class LocalProvider(ModelProvider):
+    def can_handle(self, source: str) -> bool:
+        return Path(source).expanduser().exists()
+
+    def download(self, source: str, *, force: bool = False) -> Path:
+        path = Path(source).expanduser().resolve()
+        if not path.exists():
+            raise FileNotFoundError(source)
+        return path

modeldownloaderutil/providers/s3.py

@@ -0,0 +1,50 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from urllib.parse import urlparse
+
+import boto3
+from botocore.config import Config
+
+from .._download import download_s3_object, download_with_cache
+from ..cache import cache_path, normalize_s3_uri
+from .base import ModelProvider
+
+_S3_CONFIG = Config(signature_version="s3v4", s3={"addressing_style": "path"})
+_SCHEME_ENV = {"minio://": "MINIO_ENDPOINT", "rustfs://": "RUSTFS_ENDPOINT"}
+
+
+class S3Provider(ModelProvider):
+    def can_handle(self, source: str) -> bool:
+        return source.startswith(("s3://", "minio://", "rustfs://"))
+
+    def download(self, source: str, *, force: bool = False) -> Path:
+        scheme = source.split("://", 1)[0] + "://"
+        normalized, endpoint = normalize_s3_uri(source)
+        if scheme in _SCHEME_ENV and not endpoint:
+            env_name = _SCHEME_ENV[scheme]
+            raise ValueError(f"{env_name} is not set (check .env)")
+        if not os.environ.get("MODEL_ACCESS_KEY") or not os.environ.get("MODEL_SECRET_KEY"):
+            raise ValueError("MODEL_ACCESS_KEY and MODEL_SECRET_KEY are not set (check .env)")
+
+        destination = cache_path(normalized)
+        parsed = urlparse(normalized)
+        bucket = parsed.netloc
+        key = parsed.path.lstrip("/")
+        if not bucket or not key:
+            raise ValueError(f"Invalid S3 URI: {source}")
+
+        client = boto3.client(
+            "s3",
+            endpoint_url=endpoint,
+            aws_access_key_id=os.environ.get("MODEL_ACCESS_KEY"),
+            aws_secret_access_key=os.environ.get("MODEL_SECRET_KEY"),
+            config=_S3_CONFIG,
+        )
+        return download_with_cache(destination, lambda path: download_s3_object(client, bucket, key, path), force=force)

modeldownloaderutil-1.0.0.dist-info/METADATA

@@ -0,0 +1,109 @@
+Metadata-Version: 2.4
+Name: modeldownloaderutil
+Version: 1.0.0
+Summary: Download model files from local paths, HTTP, S3-compatible storage, GCS, and Git LFS.
+Author-email: Ashok Pant <ashok@treeleaf.ai>
+License: MIT
+Keywords: download,gcs,git-lfs,minio,model,rustfs,s3
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.11
+Requires-Dist: boto3>=1.43.22
+Requires-Dist: google-cloud-storage>=3.11.0
+Requires-Dist: platformdirs>=4.3.8
+Requires-Dist: python-dotenv>=1.1.0
+Requires-Dist: requests>=2.34.2
+Requires-Dist: tenacity>=9.1.4
+Requires-Dist: tqdm>=4.67.3
+Description-Content-Type: text/markdown
+
+# model-downloader-util
+
+Download or resolve model files from local paths, HTTP(S), S3-compatible storage (S3, MinIO, RustFS), Google Cloud Storage, and Git LFS.
+
+## Install
+
+```bash
+pip install modeldownloaderutil
+```
+
+From source:
+
+```bash
+uv sync
+```
+
+## Usage
+
+```python
+from modeldownloaderutil import download_model, cache_dir
+
+path = download_model("s3://my-bucket/models/weights.onnx")
+path = download_model("https://example.com/model.onnx", force_download=True)
+```
+
+## Supported sources
+
+| Scheme | Example |
+|--------|---------|
+| Local | `/path/to/model.onnx`, `~/models/x.onnx` |
+| HTTP(S) | `https://host/path/model.onnx` |
+| S3 | `s3://bucket/key` |
+| MinIO | `minio://bucket/key` (`MINIO_ENDPOINT`) |
+| RustFS | `rustfs://bucket/key` (`RUSTFS_ENDPOINT`) |
+| GCS | `gs://bucket/object` |
+| Git LFS | `git+https://github.com/org/repo.git#path/in/repo.onnx` |
+
+## Cache
+
+Default: `~/.cache/model-downloader` (override with `MODEL_CACHE_DIR`).
+
+```
+~/.cache/model-downloader/
+  s3/<bucket>/<key>
+  gs/<bucket>/<object>
+  url/<host>/<path>
+  git/<repo_slug>/<file_path>
+```
+
+## Environment
+
+Copy `.env.example` to `.env` in the project root (loaded automatically on `download_model`).
+
+| Variable | Purpose |
+|----------|---------|
+| `MODEL_CACHE_DIR` | Download cache root (`cache_dir()`) |
+| `RUSTFS_ENDPOINT` | Required for `rustfs://` URLs |
+| `MINIO_ENDPOINT` | Required for `minio://` URLs |
+| `MODEL_ACCESS_KEY` / `MODEL_SECRET_KEY` | S3-compatible credentials |
+
+## Development
+
+```bash
+make test
+make build
+```
+
+## Publish to PyPI
+
+Create an API token at [pypi.org](https://pypi.org/manage/account/token/) (scope: project `modeldownloaderutil` or entire account).
+
+```bash
+export UV_PUBLISH_TOKEN=pypi-xxxxxxxx
+make publish
+```
+
+Dry run on TestPyPI first:
+
+```bash
+export UV_PUBLISH_TOKEN=pypi-xxxxxxxx
+make publish-test
+pip install --index-url https://test.pypi.org/simple/ modeldownloaderutil
+```

modeldownloaderutil-1.0.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+modeldownloaderutil/__init__.py,sha256=RNdHa4RDSjtq_U7OZ8rInANM4saCO6VYHDe1UmsLyIA,228
+modeldownloaderutil/_download.py,sha256=D8Zbn9xujUkWsXZoi388O6Fet7bkwAtb-TK6wg1D31E,1885
+modeldownloaderutil/cache.py,sha256=m9x1gduXzDwXBaNpiivL1DM5SLWfSGyYsYYeGqVmvjo,2515
+modeldownloaderutil/downloader.py,sha256=zcdK6sSDdfriwb1MVIZ44v0WShvv_cmt7SQYtTJssHI,860
+modeldownloaderutil/env.py,sha256=Q6LV2Ym-CE7VKp6XYkS4D2gQzpNxkOL0Tw5FQN_Yflo,758
+modeldownloaderutil/providers/__init__.py,sha256=kV0qGZurx0FedRy1xoYoNqzgi8PIkM8jjL0sFuoALuU,95
+modeldownloaderutil/providers/base.py,sha256=KbhCD1NlyyKnbZnTT_k0mkZBsBDMOAiHOzhGU1ZOmkA,385
+modeldownloaderutil/providers/gcs.py,sha256=mho8ynJNXb6_jo5ikCJBo4VpnMqZtQIJLSEL0Caya6Y,1154
+modeldownloaderutil/providers/git_lfs.py,sha256=0_OCZSCTyBvwRCCBq1lg1nRBi8JQujfmD7d6fPzrkf8,1644
+modeldownloaderutil/providers/http.py,sha256=pJCeyJZewKydcDtldkhkuNCeimShqqYSYTQu0IAtBkI,635
+modeldownloaderutil/providers/local.py,sha256=Z53RE-0v5fcwRKawKmng1gHgxzx6oW8irMFT7QcMvVM,539
+modeldownloaderutil/providers/s3.py,sha256=7IaIRcOuFVYThsgxf9GjSMypoHntEDFWKq5UeH-W2Qw,1858
+modeldownloaderutil-1.0.0.dist-info/METADATA,sha256=YgpMe8D2p9XEk3-nEcQgMrvD3E7nUr3_vj1PCSpwyBQ,2934
+modeldownloaderutil-1.0.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+modeldownloaderutil-1.0.0.dist-info/RECORD,,

modeldownloaderutil-1.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any