cached-hub 0.1.0__tar.gz

cached_hub-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,50 @@
+# Build and publish the package to PyPI when a GitHub release is created.
+# Uses PyPI trusted publishing (OIDC): no API token needed.
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+    permissions:
+      contents: write  # Required to update release notes
+      id-token: write  # Required for PyPI trusted publishing
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Needed for hatch-vcs to determine version
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Install hatch
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch
+
+      - name: Generate changelog for this release
+        uses: orhun/git-cliff-action@v4
+        with:
+          config: cliff.toml
+          args: --latest --strip header
+        env:
+          OUTPUT: release-notes.md
+
+      - name: Update release notes
+        uses: softprops/action-gh-release@v2
+        with:
+          body_path: release-notes.md
+
+      - name: Build python package
+        run: |
+          hatch build
+
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

cached_hub-0.1.0/.github/workflows/pytest.yml

@@ -0,0 +1,47 @@
+name: Python package
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Needed for hatch-vcs to determine version
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: |
+          uv sync --extra dev
+
+      - name: Lint with ruff
+        run: |
+          uv run ruff check cached_hub tests
+          uv run ruff format --check cached_hub tests
+
+      - name: Test with pytest
+        timeout-minutes: 10
+        run: |
+          uv run pytest

cached_hub-0.1.0/.gitignore

@@ -0,0 +1,16 @@
+dist/
+build/
+*.pyc
+__pycache__/
+*.egg-info/
+.venv/
+.uv/
+uv.lock
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.vscode/
+**/.DS_Store
+
+# Generated by hatch-vcs
+cached_hub/_version.py

cached_hub-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,36 @@
+default_install_hook_types:
+  - pre-commit
+  - commit-msg
+
+repos:
+- hooks:
+  - id: check-yaml
+  - id: check-toml
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+  repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v6.0.0
+- hooks:
+  - id: python-check-blanket-noqa
+  - id: python-no-eval
+  repo: https://github.com/pre-commit/pygrep-hooks
+  rev: v1.10.0
+- hooks:
+  - id: no-fixme-todo
+    name: No FIXME or TODO markers
+    entry: '(FIXME:|TODO:)'
+    language: pygrep
+    types: [python]
+  repo: local
+- hooks:
+  - id: ruff
+    args: [--fix]
+  - id: ruff-format
+  repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.14.10
+- hooks:
+  - id: conventional-pre-commit
+    stages:
+    - commit-msg
+  repo: https://github.com/compilerla/conventional-pre-commit
+  rev: v4.3.0

cached_hub-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

cached_hub-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Benjamin Piwowarski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cached_hub-0.1.0/PKG-INFO

@@ -0,0 +1,193 @@
+Metadata-Version: 2.5
+Name: cached-hub
+Version: 0.1.0
+Summary: Load HuggingFace models/datasets (and other course resources) from a shared local cache, falling back to the Hub; declare and pre-download them for classrooms
+Project-URL: Homepage, https://github.com/bpiwowar/cached-hub
+Project-URL: Repository, https://github.com/bpiwowar/cached-hub
+Author-email: Benjamin Piwowarski <benjamin@piwowarski.fr>
+License: MIT
+License-File: LICENSE
+Keywords: cache,datasets,huggingface,teaching,transformers
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Education
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Provides-Extra: datamaestro
+Requires-Dist: datamaestro>=1.5; extra == 'datamaestro'
+Provides-Extra: dev
+Requires-Dist: pre-commit>=3.5; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.8; extra == 'dev'
+Provides-Extra: hf
+Requires-Dist: datasets>=2.7; extra == 'hf'
+Requires-Dist: transformers>=4.30; extra == 'hf'
+Provides-Extra: pyterrier
+Requires-Dist: python-terrier>=0.10; extra == 'pyterrier'
+Description-Content-Type: text/markdown
+
+# cached-hub
+
+**Load HuggingFace models and datasets from a shared local cache, fall back to
+the Hub, and pre-populate that cache from a declarative list of what a course
+needs.**
+
+## Why
+
+In a classroom, every student pulling `gpt2`, `SmolLM2` and `imdb` from the Hub
+at the same minute is slow, fragile, and sometimes impossible (no home directory
+quota, shaky proxy, offline lab). The usual answer is a shared read-only
+directory pre-filled by the instructor. `cached-hub` makes that directory a
+first-class thing:
+
+- notebook code calls `load_hf_model("gpt2")` and gets the cached copy when it
+  exists, the Hub otherwise, with a one-line note saying which;
+- the instructor declares the resources once, and `cached-hub download` fills
+  the cache (each shared resource once, optional ones on demand);
+- an *enforce* mode turns any cache miss into an error, so you can check that
+  every notebook runs fully from the cache before the session.
+
+The package has **no required dependency**: `transformers`, `datasets`,
+`pyterrier` and `datamaestro` are imported only by the functions that use them.
+It was extracted from the Sorbonne Master MIND `master-mind` tool so that a
+course only needs this package (plus
+[jupytext-notebook-helper](https://github.com/bpiwowar/jupytext-notebook-helper)
+for building notebooks).
+
+## Install
+
+```sh
+pip install cached-hub            # loaders only (bring your own transformers/datasets)
+pip install "cached-hub[hf]"      # + transformers, datasets
+```
+
+## Supported libraries
+
+| Library                | cached-hub function                | Equivalent to                                   | Cached under `$CACHED_HUB_PATH`        | Pre-download with              |
+|------------------------|------------------------------------|-------------------------------------------------|----------------------------------------|--------------------------------|
+| transformers           | `load_hf_model(id, cls=AutoModel, **kw)`        | `cls.from_pretrained(id, **kw)`        | `huggingface/models/<id>/`             | `make_hf_model_resource`       |
+| transformers           | `load_hf_tokenizer(id, cls=AutoTokenizer, **kw)`| `cls.from_pretrained(id, **kw)`        | `huggingface/tokenizers/<id>/`         | `make_hf_tokenizer_resource`   |
+| transformers           | `load_hf_processor(id, cls=AutoProcessor, **kw)`| `cls.from_pretrained(id, **kw)`        | `huggingface/processors/<id>/`         | `make_hf_processor_resource`   |
+| datasets               | `load_hf_dataset(id, name=None, split=None, **kw)` | `datasets.load_dataset(id, name, split=split, **kw)` | `huggingface/datasets/<id>[-<name>]/<split>/` | `make_hf_dataset_resource` |
+| transformers           | `HFModel(id, tok_cls, model_cls, **kw)`         | lazy `.tokenizer` / `.model` via the two loaders above | as above                    | model + tokenizer resources    |
+| pyterrier / ir-datasets| (use `pt.get_dataset` directly)                 | `pt.get_dataset(id)`                   | pyterrier's own home (`PYTERRIER_HOME`, `IR_DATASETS_HOME`) | `make_pyterrier_dataset_resource` |
+| datamaestro            | (use `datamaestro.prepare_dataset` directly)    | `prepare_dataset(id)`                  | datamaestro's own store (`DATAMAESTRO_DIR`) | `make_datamaestro_resource` |
+
+The loaders differ from their equivalents in one way only: with
+`CACHED_HUB_PATH` set, they first look for the resource in the cache layout
+above, log where it came from, and on a miss forward to the equivalent call
+with `cache_dir=$CACHED_HUB_PATH/huggingface` added (unless you passed one), or
+raise `CacheMissError` in enforce mode. PyTerrier and datamaestro manage their
+own caches, so there is no loader for them: `cached-hub` only declares them as
+resources so that `cached-hub download` fetches everything a course needs in
+one go, and you keep calling those libraries as usual.
+
+## In notebooks
+
+```python
+from cached_hub import load_hf_model, load_hf_tokenizer, load_hf_dataset, HFModel
+from transformers import AutoModelForCausalLM
+
+tokenizer = load_hf_tokenizer("HuggingFaceTB/SmolLM2-1.7B-Instruct")
+model = load_hf_model("HuggingFaceTB/SmolLM2-1.7B-Instruct", AutoModelForCausalLM, device_map="auto")
+train = load_hf_dataset("imdb", split="train")
+sst2 = load_hf_dataset("glue", name="sst2")          # DatasetDict of the cached splits
+
+hf = HFModel("gpt2")            # lazy: nothing is loaded yet
+hf.tokenizer, hf.model          # AutoTokenizer / AutoModel, loaded on first access
+```
+
+Each loader checks the local cache first, then falls back to the Hub with a
+warning. Extra keyword arguments go to `from_pretrained` / `load_dataset`.
+
+## Configuration
+
+| Variable             | Effect                                                                 |
+|----------------------|------------------------------------------------------------------------|
+| `CACHED_HUB_PATH`    | Root of the shared cache. Unset: the library does nothing (see below). |
+| `CACHED_HUB_ENFORCE` | If set (any value), a cache miss raises `CacheMissError` instead of falling back. |
+
+Layout under the root (`org/name` becomes `org-name`):
+
+```
+$CACHED_HUB_PATH/huggingface/models/<id>/                 model.save_pretrained()
+$CACHED_HUB_PATH/huggingface/tokenizers/<id>/
+$CACHED_HUB_PATH/huggingface/processors/<id>/
+$CACHED_HUB_PATH/huggingface/datasets/<id>[-<name>]/<split>/   dataset.save_to_disk()
+$CACHED_HUB_PATH/huggingface/                             HF cache_dir used for fallbacks
+```
+
+A directory is used only when it contains the marker `.downloaded.ok`, written
+after a successful download, so a half-copied model is never picked up.
+
+**Without `CACHED_HUB_PATH`, `cached-hub` adds no caching of its own.**
+`load_hf_model("gpt2", cls, **kw)` is then exactly `cls.from_pretrained("gpt2", **kw)`,
+and `load_hf_dataset(...)` exactly `datasets.load_dataset(...)`: the usual
+HuggingFace cache (`~/.cache/huggingface`, `HF_HOME`) applies as it always does,
+and `cached-hub download` merely warms it. Notebooks can therefore import from
+`cached_hub` unconditionally and run unchanged on a laptop or on Colab; only the
+classroom machines set the variable.
+
+## Declaring and downloading resources
+
+A course lists what it needs as `{section: [resources]}`:
+
+```python
+# mycourse/resources.py
+from cached_hub import (
+    make_hf_model_resource, make_hf_tokenizer_resource, make_hf_processor_resource,
+    make_hf_dataset_resource, make_pyterrier_dataset_resource, make_datamaestro_resource,
+)
+
+RESOURCES = {
+    "practical1": [
+        make_hf_model_resource("gpt2", model_class="GPT2LMHeadModel"),
+        make_hf_tokenizer_resource("gpt2", tokenizer_class="GPT2Tokenizer"),
+        make_hf_dataset_resource("imdb", ["train", "test"]),
+    ],
+    "practical2": [
+        make_hf_model_resource("Qwen/Qwen2.5-7B-Instruct", model_class="AutoModelForCausalLM", optional=True),
+        make_pyterrier_dataset_resource("irds:lotte/technology/dev/search", "LoTTE technology"),
+    ],
+}
+```
+
+then, on the machine that hosts the cache:
+
+```sh
+export CACHED_HUB_PATH=/shared/cache
+cached-hub info
+cached-hub list     --from mycourse.resources:RESOURCES
+cached-hub download --from mycourse.resources:RESOURCES               # everything but optional
+cached-hub download --from mycourse.resources:RESOURCES --section practical2 --optional
+cached-hub download --from mycourse.resources:RESOURCES --key gpt2
+```
+
+`--from MODULE:ATTR` imports `MODULE` and reads `ATTR` from it: a
+`{section: [resources]}` mapping, or a zero-argument callable returning one
+(dotted attributes such as `plugin.Course.resources` are followed). No source
+scanning is involved; `RESOURCES` above is only a naming convention. The option
+can be repeated. Resources are identified by `(type, key)`, so a model shared
+by several practicals is downloaded once. `HF_HUB_OFFLINE` is lifted for the
+duration of a download.
+
+The same helpers are available from Python (`download_resources`,
+`select_resources`, `format_resources`, `merge_resources`), and any object with
+`resource_type`, `key`, `description`, `optional` and `download()` is a valid
+resource (`FunctionalResource` wraps a plain function).
+
+## Checking a cache before class
+
+```sh
+CACHED_HUB_PATH=/shared/cache CACHED_HUB_ENFORCE=1 python practical1.py
+```
+
+fails at the first resource that would have gone to the Hub.

cached_hub-0.1.0/README.md

@@ -0,0 +1,157 @@
+# cached-hub
+
+**Load HuggingFace models and datasets from a shared local cache, fall back to
+the Hub, and pre-populate that cache from a declarative list of what a course
+needs.**
+
+## Why
+
+In a classroom, every student pulling `gpt2`, `SmolLM2` and `imdb` from the Hub
+at the same minute is slow, fragile, and sometimes impossible (no home directory
+quota, shaky proxy, offline lab). The usual answer is a shared read-only
+directory pre-filled by the instructor. `cached-hub` makes that directory a
+first-class thing:
+
+- notebook code calls `load_hf_model("gpt2")` and gets the cached copy when it
+  exists, the Hub otherwise, with a one-line note saying which;
+- the instructor declares the resources once, and `cached-hub download` fills
+  the cache (each shared resource once, optional ones on demand);
+- an *enforce* mode turns any cache miss into an error, so you can check that
+  every notebook runs fully from the cache before the session.
+
+The package has **no required dependency**: `transformers`, `datasets`,
+`pyterrier` and `datamaestro` are imported only by the functions that use them.
+It was extracted from the Sorbonne Master MIND `master-mind` tool so that a
+course only needs this package (plus
+[jupytext-notebook-helper](https://github.com/bpiwowar/jupytext-notebook-helper)
+for building notebooks).
+
+## Install
+
+```sh
+pip install cached-hub            # loaders only (bring your own transformers/datasets)
+pip install "cached-hub[hf]"      # + transformers, datasets
+```
+
+## Supported libraries
+
+| Library                | cached-hub function                | Equivalent to                                   | Cached under `$CACHED_HUB_PATH`        | Pre-download with              |
+|------------------------|------------------------------------|-------------------------------------------------|----------------------------------------|--------------------------------|
+| transformers           | `load_hf_model(id, cls=AutoModel, **kw)`        | `cls.from_pretrained(id, **kw)`        | `huggingface/models/<id>/`             | `make_hf_model_resource`       |
+| transformers           | `load_hf_tokenizer(id, cls=AutoTokenizer, **kw)`| `cls.from_pretrained(id, **kw)`        | `huggingface/tokenizers/<id>/`         | `make_hf_tokenizer_resource`   |
+| transformers           | `load_hf_processor(id, cls=AutoProcessor, **kw)`| `cls.from_pretrained(id, **kw)`        | `huggingface/processors/<id>/`         | `make_hf_processor_resource`   |
+| datasets               | `load_hf_dataset(id, name=None, split=None, **kw)` | `datasets.load_dataset(id, name, split=split, **kw)` | `huggingface/datasets/<id>[-<name>]/<split>/` | `make_hf_dataset_resource` |
+| transformers           | `HFModel(id, tok_cls, model_cls, **kw)`         | lazy `.tokenizer` / `.model` via the two loaders above | as above                    | model + tokenizer resources    |
+| pyterrier / ir-datasets| (use `pt.get_dataset` directly)                 | `pt.get_dataset(id)`                   | pyterrier's own home (`PYTERRIER_HOME`, `IR_DATASETS_HOME`) | `make_pyterrier_dataset_resource` |
+| datamaestro            | (use `datamaestro.prepare_dataset` directly)    | `prepare_dataset(id)`                  | datamaestro's own store (`DATAMAESTRO_DIR`) | `make_datamaestro_resource` |
+
+The loaders differ from their equivalents in one way only: with
+`CACHED_HUB_PATH` set, they first look for the resource in the cache layout
+above, log where it came from, and on a miss forward to the equivalent call
+with `cache_dir=$CACHED_HUB_PATH/huggingface` added (unless you passed one), or
+raise `CacheMissError` in enforce mode. PyTerrier and datamaestro manage their
+own caches, so there is no loader for them: `cached-hub` only declares them as
+resources so that `cached-hub download` fetches everything a course needs in
+one go, and you keep calling those libraries as usual.
+
+## In notebooks
+
+```python
+from cached_hub import load_hf_model, load_hf_tokenizer, load_hf_dataset, HFModel
+from transformers import AutoModelForCausalLM
+
+tokenizer = load_hf_tokenizer("HuggingFaceTB/SmolLM2-1.7B-Instruct")
+model = load_hf_model("HuggingFaceTB/SmolLM2-1.7B-Instruct", AutoModelForCausalLM, device_map="auto")
+train = load_hf_dataset("imdb", split="train")
+sst2 = load_hf_dataset("glue", name="sst2")          # DatasetDict of the cached splits
+
+hf = HFModel("gpt2")            # lazy: nothing is loaded yet
+hf.tokenizer, hf.model          # AutoTokenizer / AutoModel, loaded on first access
+```
+
+Each loader checks the local cache first, then falls back to the Hub with a
+warning. Extra keyword arguments go to `from_pretrained` / `load_dataset`.
+
+## Configuration
+
+| Variable             | Effect                                                                 |
+|----------------------|------------------------------------------------------------------------|
+| `CACHED_HUB_PATH`    | Root of the shared cache. Unset: the library does nothing (see below). |
+| `CACHED_HUB_ENFORCE` | If set (any value), a cache miss raises `CacheMissError` instead of falling back. |
+
+Layout under the root (`org/name` becomes `org-name`):
+
+```
+$CACHED_HUB_PATH/huggingface/models/<id>/                 model.save_pretrained()
+$CACHED_HUB_PATH/huggingface/tokenizers/<id>/
+$CACHED_HUB_PATH/huggingface/processors/<id>/
+$CACHED_HUB_PATH/huggingface/datasets/<id>[-<name>]/<split>/   dataset.save_to_disk()
+$CACHED_HUB_PATH/huggingface/                             HF cache_dir used for fallbacks
+```
+
+A directory is used only when it contains the marker `.downloaded.ok`, written
+after a successful download, so a half-copied model is never picked up.
+
+**Without `CACHED_HUB_PATH`, `cached-hub` adds no caching of its own.**
+`load_hf_model("gpt2", cls, **kw)` is then exactly `cls.from_pretrained("gpt2", **kw)`,
+and `load_hf_dataset(...)` exactly `datasets.load_dataset(...)`: the usual
+HuggingFace cache (`~/.cache/huggingface`, `HF_HOME`) applies as it always does,
+and `cached-hub download` merely warms it. Notebooks can therefore import from
+`cached_hub` unconditionally and run unchanged on a laptop or on Colab; only the
+classroom machines set the variable.
+
+## Declaring and downloading resources
+
+A course lists what it needs as `{section: [resources]}`:
+
+```python
+# mycourse/resources.py
+from cached_hub import (
+    make_hf_model_resource, make_hf_tokenizer_resource, make_hf_processor_resource,
+    make_hf_dataset_resource, make_pyterrier_dataset_resource, make_datamaestro_resource,
+)
+
+RESOURCES = {
+    "practical1": [
+        make_hf_model_resource("gpt2", model_class="GPT2LMHeadModel"),
+        make_hf_tokenizer_resource("gpt2", tokenizer_class="GPT2Tokenizer"),
+        make_hf_dataset_resource("imdb", ["train", "test"]),
+    ],
+    "practical2": [
+        make_hf_model_resource("Qwen/Qwen2.5-7B-Instruct", model_class="AutoModelForCausalLM", optional=True),
+        make_pyterrier_dataset_resource("irds:lotte/technology/dev/search", "LoTTE technology"),
+    ],
+}
+```
+
+then, on the machine that hosts the cache:
+
+```sh
+export CACHED_HUB_PATH=/shared/cache
+cached-hub info
+cached-hub list     --from mycourse.resources:RESOURCES
+cached-hub download --from mycourse.resources:RESOURCES               # everything but optional
+cached-hub download --from mycourse.resources:RESOURCES --section practical2 --optional
+cached-hub download --from mycourse.resources:RESOURCES --key gpt2
+```
+
+`--from MODULE:ATTR` imports `MODULE` and reads `ATTR` from it: a
+`{section: [resources]}` mapping, or a zero-argument callable returning one
+(dotted attributes such as `plugin.Course.resources` are followed). No source
+scanning is involved; `RESOURCES` above is only a naming convention. The option
+can be repeated. Resources are identified by `(type, key)`, so a model shared
+by several practicals is downloaded once. `HF_HUB_OFFLINE` is lifted for the
+duration of a download.
+
+The same helpers are available from Python (`download_resources`,
+`select_resources`, `format_resources`, `merge_resources`), and any object with
+`resource_type`, `key`, `description`, `optional` and `download()` is a valid
+resource (`FunctionalResource` wraps a plain function).
+
+## Checking a cache before class
+
+```sh
+CACHED_HUB_PATH=/shared/cache CACHED_HUB_ENFORCE=1 python practical1.py
+```
+
+fails at the first resource that would have gone to the Hub.

cached_hub-0.1.0/cached_hub/__init__.py

@@ -0,0 +1,82 @@
+"""cached-hub: HuggingFace (and other) resources from a shared local cache.
+
+Notebook side::
+
+    from cached_hub import load_hf_model, load_hf_tokenizer, load_hf_dataset, HFModel
+
+Course side (declare what to pre-download)::
+
+    from cached_hub import make_hf_model_resource, make_hf_dataset_resource
+    RESOURCES = {"practical1": [make_hf_model_resource("gpt2"), ...]}
+
+then ``cached-hub download --from mycourse.resources:RESOURCES``.
+
+Configure the cache root with ``CACHED_HUB_PATH`` (see :mod:`cached_hub.config`).
+"""
+
+from .config import (
+    ENV_ENFORCE,
+    ENV_PATH,
+    CacheMissError,
+    get_cache_path,
+    is_enforce_mode,
+)
+from .datamaestro import make_datamaestro_resource
+from .hf import (
+    HFModel,
+    load_hf_dataset,
+    load_hf_model,
+    load_hf_processor,
+    load_hf_tokenizer,
+    make_hf_dataset_resource,
+    make_hf_model_resource,
+    make_hf_processor_resource,
+    make_hf_tokenizer_resource,
+)
+from .pyterrier import make_pyterrier_dataset_resource
+from .resources import (
+    DownloadableResource,
+    FunctionalResource,
+    Resource,
+    Resources,
+    download_resources,
+    format_resources,
+    merge_resources,
+    select_resources,
+)
+
+try:
+    from ._version import __version__
+except ImportError:  # pragma: no cover - not installed from a build
+    __version__ = "0.0.0+unknown"
+
+__all__ = [
+    "__version__",
+    # config
+    "ENV_PATH",
+    "ENV_ENFORCE",
+    "CacheMissError",
+    "get_cache_path",
+    "is_enforce_mode",
+    # loading
+    "HFModel",
+    "load_hf_dataset",
+    "load_hf_model",
+    "load_hf_processor",
+    "load_hf_tokenizer",
+    # resources
+    "Resource",
+    "DownloadableResource",
+    "FunctionalResource",
+    "Resources",
+    "download_resources",
+    "format_resources",
+    "merge_resources",
+    "select_resources",
+    "make_hf_dataset_resource",
+    "make_hf_model_resource",
+    "make_hf_processor_resource",
+    "make_hf_tokenizer_resource",
+    "make_pyterrier_dataset_resource",
+    "make_datamaestro_resource",
+]

cached_hub-0.1.0/cached_hub/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

cached_hub-0.1.0/cached_hub/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None