modeldownloaderutil 1.0.0__tar.gz

modeldownloaderutil-1.0.0/.env.example

@@ -0,0 +1,5 @@
+MODEL_CACHE_DIR=~/.cache/model-downloader
+RUSTFS_ENDPOINT=http://localhost:9000
+MINIO_ENDPOINT=https://minio.example.com
+MODEL_ACCESS_KEY=your-access-key
+MODEL_SECRET_KEY=your-secret-key

modeldownloaderutil-1.0.0/.github/workflows/workflow.yml

@@ -0,0 +1,48 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --group dev
+
+      - name: Ruff
+        run: uv run ruff check modeldownloaderutil tests
+
+      - name: Pytest
+        run: uv run pytest
+
+  build:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Build package
+        run: uv build

modeldownloaderutil-1.0.0/.gitignore

@@ -0,0 +1,12 @@
+.venv
+*.pyc
+.idea
+*DS_Store
+__pycache__
+*.log
+*.sqlite3
+*.db
+*.env
+dist/
+build/
+*.egg-info/

modeldownloaderutil-1.0.0/Makefile

@@ -0,0 +1,26 @@
+.PHONY: install upgrade test clean build publish publish-test
+
+install:
+	uv sync --group dev
+
+upgrade:
+	uv sync --upgrade
+
+test:
+	uv run pytest
+
+clean:
+	rm -rf .venv dist build __pycache__ *.egg-info
+	find . -type d -name __pycache__ -exec rm -rf {} +
+	find . -type f -name "*.pyc" -delete
+
+build:
+	uv build
+
+publish: build
+	@test -n "$$UV_PUBLISH_TOKEN" || (echo "Set UV_PUBLISH_TOKEN (PyPI API token)" && exit 1)
+	uv publish
+
+publish-test: build
+	@test -n "$$UV_PUBLISH_TOKEN" || (echo "Set UV_PUBLISH_TOKEN (TestPyPI API token)" && exit 1)
+	uv publish --publish-url https://test.pypi.org/legacy/

modeldownloaderutil-1.0.0/PKG-INFO

@@ -0,0 +1,109 @@
+Metadata-Version: 2.4
+Name: modeldownloaderutil
+Version: 1.0.0
+Summary: Download model files from local paths, HTTP, S3-compatible storage, GCS, and Git LFS.
+Author-email: Ashok Pant <ashok@treeleaf.ai>
+License: MIT
+Keywords: download,gcs,git-lfs,minio,model,rustfs,s3
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.11
+Requires-Dist: boto3>=1.43.22
+Requires-Dist: google-cloud-storage>=3.11.0
+Requires-Dist: platformdirs>=4.3.8
+Requires-Dist: python-dotenv>=1.1.0
+Requires-Dist: requests>=2.34.2
+Requires-Dist: tenacity>=9.1.4
+Requires-Dist: tqdm>=4.67.3
+Description-Content-Type: text/markdown
+
+# model-downloader-util
+
+Download or resolve model files from local paths, HTTP(S), S3-compatible storage (S3, MinIO, RustFS), Google Cloud Storage, and Git LFS.
+
+## Install
+
+```bash
+pip install modeldownloaderutil
+```
+
+From source:
+
+```bash
+uv sync
+```
+
+## Usage
+
+```python
+from modeldownloaderutil import download_model, cache_dir
+
+path = download_model("s3://my-bucket/models/weights.onnx")
+path = download_model("https://example.com/model.onnx", force_download=True)
+```
+
+## Supported sources
+
+| Scheme | Example |
+|--------|---------|
+| Local | `/path/to/model.onnx`, `~/models/x.onnx` |
+| HTTP(S) | `https://host/path/model.onnx` |
+| S3 | `s3://bucket/key` |
+| MinIO | `minio://bucket/key` (`MINIO_ENDPOINT`) |
+| RustFS | `rustfs://bucket/key` (`RUSTFS_ENDPOINT`) |
+| GCS | `gs://bucket/object` |
+| Git LFS | `git+https://github.com/org/repo.git#path/in/repo.onnx` |
+
+## Cache
+
+Default: `~/.cache/model-downloader` (override with `MODEL_CACHE_DIR`).
+
+```
+~/.cache/model-downloader/
+  s3/<bucket>/<key>
+  gs/<bucket>/<object>
+  url/<host>/<path>
+  git/<repo_slug>/<file_path>
+```
+
+## Environment
+
+Copy `.env.example` to `.env` in the project root (loaded automatically on `download_model`).
+
+| Variable | Purpose |
+|----------|---------|
+| `MODEL_CACHE_DIR` | Download cache root (`cache_dir()`) |
+| `RUSTFS_ENDPOINT` | Required for `rustfs://` URLs |
+| `MINIO_ENDPOINT` | Required for `minio://` URLs |
+| `MODEL_ACCESS_KEY` / `MODEL_SECRET_KEY` | S3-compatible credentials |
+
+## Development
+
+```bash
+make test
+make build
+```
+
+## Publish to PyPI
+
+Create an API token at [pypi.org](https://pypi.org/manage/account/token/) (scope: project `modeldownloaderutil` or entire account).
+
+```bash
+export UV_PUBLISH_TOKEN=pypi-xxxxxxxx
+make publish
+```
+
+Dry run on TestPyPI first:
+
+```bash
+export UV_PUBLISH_TOKEN=pypi-xxxxxxxx
+make publish-test
+pip install --index-url https://test.pypi.org/simple/ modeldownloaderutil
+```

modeldownloaderutil-1.0.0/README.md

@@ -0,0 +1,83 @@
+# model-downloader-util
+
+Download or resolve model files from local paths, HTTP(S), S3-compatible storage (S3, MinIO, RustFS), Google Cloud Storage, and Git LFS.
+
+## Install
+
+```bash
+pip install modeldownloaderutil
+```
+
+From source:
+
+```bash
+uv sync
+```
+
+## Usage
+
+```python
+from modeldownloaderutil import download_model, cache_dir
+
+path = download_model("s3://my-bucket/models/weights.onnx")
+path = download_model("https://example.com/model.onnx", force_download=True)
+```
+
+## Supported sources
+
+| Scheme | Example |
+|--------|---------|
+| Local | `/path/to/model.onnx`, `~/models/x.onnx` |
+| HTTP(S) | `https://host/path/model.onnx` |
+| S3 | `s3://bucket/key` |
+| MinIO | `minio://bucket/key` (`MINIO_ENDPOINT`) |
+| RustFS | `rustfs://bucket/key` (`RUSTFS_ENDPOINT`) |
+| GCS | `gs://bucket/object` |
+| Git LFS | `git+https://github.com/org/repo.git#path/in/repo.onnx` |
+
+## Cache
+
+Default: `~/.cache/model-downloader` (override with `MODEL_CACHE_DIR`).
+
+```
+~/.cache/model-downloader/
+  s3/<bucket>/<key>
+  gs/<bucket>/<object>
+  url/<host>/<path>
+  git/<repo_slug>/<file_path>
+```
+
+## Environment
+
+Copy `.env.example` to `.env` in the project root (loaded automatically on `download_model`).
+
+| Variable | Purpose |
+|----------|---------|
+| `MODEL_CACHE_DIR` | Download cache root (`cache_dir()`) |
+| `RUSTFS_ENDPOINT` | Required for `rustfs://` URLs |
+| `MINIO_ENDPOINT` | Required for `minio://` URLs |
+| `MODEL_ACCESS_KEY` / `MODEL_SECRET_KEY` | S3-compatible credentials |
+
+## Development
+
+```bash
+make test
+make build
+```
+
+## Publish to PyPI
+
+Create an API token at [pypi.org](https://pypi.org/manage/account/token/) (scope: project `modeldownloaderutil` or entire account).
+
+```bash
+export UV_PUBLISH_TOKEN=pypi-xxxxxxxx
+make publish
+```
+
+Dry run on TestPyPI first:
+
+```bash
+export UV_PUBLISH_TOKEN=pypi-xxxxxxxx
+make publish-test
+pip install --index-url https://test.pypi.org/simple/ modeldownloaderutil
+```

modeldownloaderutil-1.0.0/examples/__init__.py

@@ -0,0 +1,5 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""

modeldownloaderutil-1.0.0/examples/downloader_example.py

@@ -0,0 +1,9 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from modeldownloaderutil import download_model
+
+if __name__ == "__main__":
+    print(download_model("rustfs://treeleaf-models/model_4_kps.onnx"))

modeldownloaderutil-1.0.0/modeldownloaderutil/__init__.py

@@ -0,0 +1,10 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from .cache import cache_dir
+from .downloader import download_model
+
+__all__ = ["download_model", "cache_dir"]
+__version__ = "2.0.0"

modeldownloaderutil-1.0.0/modeldownloaderutil/_download.py

@@ -0,0 +1,46 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+
+import requests
+from tenacity import retry, stop_after_attempt, wait_exponential
+from tqdm import tqdm
+
+from .cache import commit_download, incomplete_path
+
+
+@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=8))
+def download_http(url: str, destination: Path) -> Path:
+    tmp = incomplete_path(destination)
+    tmp.parent.mkdir(parents=True, exist_ok=True)
+    with requests.get(url, stream=True, timeout=300) as response:
+        response.raise_for_status()
+        total = int(response.headers.get("Content-Length", 0)) or None
+        with open(tmp, "wb") as handle, tqdm(total=total, unit="B", unit_scale=True, unit_divisor=1024, desc=destination.name, disable=total is None) as bar:
+            for chunk in response.iter_content(chunk_size=1024 * 1024):
+                if chunk:
+                    handle.write(chunk)
+                    bar.update(len(chunk))
+    return commit_download(tmp, destination)
+
+
+def download_s3_object(client, bucket: str, key: str, destination: Path) -> Path:
+    tmp = incomplete_path(destination)
+    tmp.parent.mkdir(parents=True, exist_ok=True)
+    size = client.head_object(Bucket=bucket, Key=key).get("ContentLength")
+    with tqdm(total=size, unit="B", unit_scale=True, unit_divisor=1024, desc=destination.name, disable=not size) as bar:
+        client.download_file(bucket, key, str(tmp), Callback=lambda n: bar.update(n))
+    return commit_download(tmp, destination)
+
+
+def download_with_cache(destination: Path, fetch: Callable[[Path], Path], *, force: bool = False) -> Path:
+    if destination.exists() and not force:
+        return destination.resolve()
+    fetch(destination)
+    return destination.resolve()

modeldownloaderutil-1.0.0/modeldownloaderutil/cache.py

@@ -0,0 +1,83 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+from urllib.parse import urlparse
+
+from platformdirs import user_cache_dir
+
+APP_NAME = "model-downloader"
+APP_AUTHOR = "treeleaf"
+_SCHEME_ALIASES = {"minio": "s3", "rustfs": "s3"}
+
+
+def cache_dir() -> Path:
+    if cache := os.environ.get("MODEL_CACHE_DIR"):
+        return Path(cache).expanduser()
+    return Path(user_cache_dir(APP_NAME, appauthor=APP_AUTHOR))
+
+
+def cache_path(source: str) -> Path:
+    parsed = urlparse(source)
+    scheme = _SCHEME_ALIASES.get(parsed.scheme, parsed.scheme)
+    root = cache_dir()
+
+    if scheme in ("s3", "gs"):
+        key = parsed.path.lstrip("/")
+        if not parsed.netloc or not key:
+            raise ValueError(f"Invalid object URI: {source}")
+        return root / scheme / parsed.netloc / key
+
+    if scheme in ("http", "https"):
+        path = parsed.path.lstrip("/") or "download"
+        return root / "url" / _safe_segment(parsed.netloc) / path
+
+    if scheme == "git":
+        file_path = parsed.fragment
+        if not file_path:
+            raise ValueError(f"Invalid git URI (missing #<path>): {source}")
+        repo_url = f"{parsed.netloc}{parsed.path}" if parsed.netloc else parsed.path.lstrip("/")
+        if not repo_url:
+            raise ValueError(f"Invalid git URI: {source}")
+        return git_file_path(repo_url, file_path)
+
+    raise ValueError(f"Cannot derive cache path for: {source}")
+
+
+def git_repo_dir(repo_url: str) -> Path:
+    return cache_dir() / "git" / _safe_segment(repo_url)
+
+
+def git_file_path(repo_url: str, file_path: str) -> Path:
+    return git_repo_dir(repo_url) / file_path
+
+
+def incomplete_path(path: Path) -> Path:
+    return path.with_suffix(path.suffix + ".incomplete")
+
+
+def commit_download(tmp: Path, destination: Path) -> Path:
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    tmp.replace(destination)
+    return destination
+
+
+def _safe_segment(value: str) -> str:
+    return re.sub(r"[^a-zA-Z0-9._-]+", "_", value)
+
+
+def normalize_s3_uri(source: str) -> tuple[str, str | None]:
+    endpoint: str | None = None
+    if source.startswith("minio://"):
+        endpoint = os.environ.get("MINIO_ENDPOINT")
+        source = "s3://" + source[len("minio://") :]
+    elif source.startswith("rustfs://"):
+        endpoint = os.environ.get("RUSTFS_ENDPOINT")
+        source = "s3://" + source[len("rustfs://") :]
+    return source, endpoint

modeldownloaderutil-1.0.0/modeldownloaderutil/downloader.py

@@ -0,0 +1,32 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .env import load_env
+from .providers.base import ModelProvider
+from .providers.gcs import GCSProvider
+from .providers.git_lfs import GitLFSProvider
+from .providers.http import HttpProvider
+from .providers.local import LocalProvider
+from .providers.s3 import S3Provider
+
+_PROVIDERS: tuple[ModelProvider, ...] = (
+    LocalProvider(),
+    HttpProvider(),
+    S3Provider(),
+    GCSProvider(),
+    GitLFSProvider(),
+)
+
+
+def download_model(source: str, *, force_download: bool = False) -> Path:
+    load_env()
+    for provider in _PROVIDERS:
+        if provider.can_handle(source):
+            return provider.download(source, force=force_download)
+    raise ValueError(f"Unsupported source: {source}")

modeldownloaderutil-1.0.0/modeldownloaderutil/env.py

@@ -0,0 +1,33 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+_LOADED = False
+
+
+def load_env() -> None:
+    global _LOADED
+    if _LOADED:
+        return
+    from dotenv import load_dotenv
+
+    for path in _env_candidates():
+        load_dotenv(path, override=False)
+    _LOADED = True
+
+
+def _env_candidates() -> list[Path]:
+    seen: set[Path] = set()
+    out: list[Path] = []
+    for base in (Path.cwd(), Path(__file__).resolve().parents[1]):
+        for parent in (base, *base.parents):
+            env = (parent / ".env").resolve()
+            if env.is_file() and env not in seen:
+                seen.add(env)
+                out.append(env)
+    return out

modeldownloaderutil-1.0.0/modeldownloaderutil/providers/__init__.py

@@ -0,0 +1,5 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""

modeldownloaderutil-1.0.0/modeldownloaderutil/providers/base.py

@@ -0,0 +1,17 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+
+class ModelProvider(ABC):
+    @abstractmethod
+    def can_handle(self, source: str) -> bool: ...
+
+    @abstractmethod
+    def download(self, source: str, *, force: bool = False) -> Path: ...

modeldownloaderutil-1.0.0/modeldownloaderutil/providers/gcs.py

@@ -0,0 +1,36 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from urllib.parse import urlparse
+
+from google.cloud import storage
+
+from .._download import download_with_cache
+from ..cache import cache_path, commit_download, incomplete_path
+from .base import ModelProvider
+
+
+class GCSProvider(ModelProvider):
+    def can_handle(self, source: str) -> bool:
+        return source.startswith("gs://")
+
+    def download(self, source: str, *, force: bool = False) -> Path:
+        destination = cache_path(source)
+        parsed = urlparse(source)
+        bucket = parsed.netloc
+        blob_name = parsed.path.lstrip("/")
+        if not bucket or not blob_name:
+            raise ValueError(f"Invalid GCS URI: {source}")
+
+        def fetch(path: Path) -> Path:
+            tmp = incomplete_path(path)
+            tmp.parent.mkdir(parents=True, exist_ok=True)
+            storage.Client().bucket(bucket).blob(blob_name).download_to_filename(str(tmp))
+            return commit_download(tmp, path)
+
+        return download_with_cache(destination, fetch, force=force)

modeldownloaderutil-1.0.0/modeldownloaderutil/providers/git_lfs.py

@@ -0,0 +1,45 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+from ..cache import git_file_path, git_repo_dir
+from .base import ModelProvider
+
+
+class GitLFSProvider(ModelProvider):
+    def can_handle(self, source: str) -> bool:
+        return source.startswith("git+")
+
+    def download(self, source: str, *, force: bool = False) -> Path:
+        canonical = source[4:]
+        repo_url, _, file_path = canonical.partition("#")
+        if not repo_url or not file_path:
+            raise ValueError(f"Invalid git source (expected git+<url>#<path>): {source}")
+
+        target = git_file_path(repo_url, file_path)
+        repo_dir = git_repo_dir(repo_url)
+        if target.exists() and not force:
+            return target.resolve()
+
+        repo_dir.mkdir(parents=True, exist_ok=True)
+        if not (repo_dir / ".git").exists():
+            self._run(["git", "clone", "--filter=blob:none", "--no-checkout", repo_url, str(repo_dir)])
+            self._run(["git", "-C", str(repo_dir), "sparse-checkout", "init", "--no-cone"])
+
+        self._run(["git", "-C", str(repo_dir), "sparse-checkout", "set", file_path])
+        self._run(["git", "-C", str(repo_dir), "checkout", "HEAD"])
+        self._run(["git", "-C", str(repo_dir), "lfs", "pull", "--include", file_path])
+
+        if not target.exists():
+            raise FileNotFoundError(f"File not found after git LFS pull: {file_path} in {repo_url}")
+        return target.resolve()
+
+    @staticmethod
+    def _run(cmd: list[str]) -> None:
+        subprocess.run(cmd, check=True)

modeldownloaderutil-1.0.0/modeldownloaderutil/providers/http.py

@@ -0,0 +1,21 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .._download import download_http, download_with_cache
+from ..cache import cache_path
+from .base import ModelProvider
+
+
+class HttpProvider(ModelProvider):
+    def can_handle(self, source: str) -> bool:
+        return source.startswith(("http://", "https://"))
+
+    def download(self, source: str, *, force: bool = False) -> Path:
+        destination = cache_path(source)
+        return download_with_cache(destination, lambda path: download_http(source, path), force=force)

modeldownloaderutil-1.0.0/modeldownloaderutil/providers/local.py

@@ -0,0 +1,21 @@
+"""
+-- Created by: Ashok Kumar Pant
+-- Email: asokpant@gmail.com
+-- Created on: 04/06/2026
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .base import ModelProvider
+
+
+class LocalProvider(ModelProvider):
+    def can_handle(self, source: str) -> bool:
+        return Path(source).expanduser().exists()
+
+    def download(self, source: str, *, force: bool = False) -> Path:
+        path = Path(source).expanduser().resolve()
+        if not path.exists():
+            raise FileNotFoundError(source)
+        return path