cosmochains 0.0.2__tar.gz

cosmochains-0.0.2/PKG-INFO

@@ -0,0 +1,47 @@
+Metadata-Version: 2.3
+Name: cosmochains
+Version: 0.0.2
+Summary: Cloud-based MCMC chain repository manager
+Author: Rodrigo Calderon
+Author-email: Rodrigo Calderon <calderon.cosmology@gmail.com>
+Requires-Dist: numpy>=1.26
+Requires-Dist: pydantic>=2.7
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: typer>=0.12
+Requires-Dist: rich>=13.0
+Requires-Dist: shellingham>=1.5
+Requires-Dist: tomli-w>=1.0
+Requires-Dist: huggingface-hub>=0.23 ; extra == 'all-backends'
+Requires-Dist: python-dotenv>=1.0 ; extra == 'all-backends'
+Requires-Dist: h5py>=3.10 ; extra == 'all-backends'
+Requires-Dist: pygithub>=2.1 ; extra == 'all-backends'
+Requires-Dist: google-api-python-client>=2.100 ; extra == 'all-backends'
+Requires-Dist: google-auth>=2.20 ; extra == 'all-backends'
+Requires-Dist: dropbox>=12.0 ; extra == 'all-backends'
+Requires-Dist: pytest>=8.0 ; extra == 'dev'
+Requires-Dist: pytest-mock>=3.14 ; extra == 'dev'
+Requires-Dist: mkdocs-material>=9.5 ; extra == 'docs'
+Requires-Dist: mkdocstrings[python]>=0.25 ; extra == 'docs'
+Requires-Dist: dropbox>=12.0 ; extra == 'dropbox'
+Requires-Dist: getdist>=1.4 ; extra == 'getdist'
+Requires-Dist: pygithub>=2.1 ; extra == 'github'
+Requires-Dist: google-api-python-client>=2.100 ; extra == 'google-drive'
+Requires-Dist: google-auth>=2.20 ; extra == 'google-drive'
+Requires-Dist: h5py>=3.10 ; extra == 'hdf5'
+Requires-Dist: huggingface-hub>=0.23 ; extra == 'hf'
+Requires-Dist: python-dotenv>=1.0 ; extra == 'hf'
+Requires-Dist: h5py>=3.10 ; extra == 'hf'
+Requires-Dist: zarr>=2.18 ; extra == 'zarr'
+Requires-Python: >=3.12
+Provides-Extra: all-backends
+Provides-Extra: dev
+Provides-Extra: docs
+Provides-Extra: dropbox
+Provides-Extra: getdist
+Provides-Extra: github
+Provides-Extra: google-drive
+Provides-Extra: hdf5
+Provides-Extra: hf
+Provides-Extra: zarr
+Description-Content-Type: text/markdown
+

cosmochains-0.0.2/pyproject.toml

@@ -0,0 +1,73 @@
+[project]
+name = "cosmochains"
+version = "0.0.2"
+description = "Cloud-based MCMC chain repository manager"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "numpy>=1.26",
+    "pydantic>=2.7",
+    "pyyaml>=6.0",
+    "typer>=0.12",
+    "rich>=13.0",
+    "shellingham>=1.5",
+    "tomli-w>=1.0",
+]
+
+[[project.authors]]
+name = "Rodrigo Calderon"
+email = "calderon.cosmology@gmail.com"
+
+[project.optional-dependencies]
+hf = [
+    "huggingface-hub>=0.23",
+    "python-dotenv>=1.0",
+    "h5py>=3.10",
+]
+hdf5 = ["h5py>=3.10"]
+zarr = ["zarr>=2.18"]
+getdist = ["getdist>=1.4"]
+github = ["PyGithub>=2.1"]
+google-drive = [
+    "google-api-python-client>=2.100",
+    "google-auth>=2.20",
+]
+dropbox = ["dropbox>=12.0"]
+all-backends = [
+    "huggingface-hub>=0.23",
+    "python-dotenv>=1.0",
+    "h5py>=3.10",
+    "PyGithub>=2.1",
+    "google-api-python-client>=2.100",
+    "google-auth>=2.20",
+    "dropbox>=12.0",
+]
+dev = [
+    "pytest>=8.0",
+    "pytest-mock>=3.14",
+]
+docs = [
+    "mkdocs-material>=9.5",
+    "mkdocstrings[python]>=0.25",
+]
+
+[project.scripts]
+cosmochains = "cosmochains.cli.app:main"
+
+[build-system]
+requires = ["uv_build>=0.9.9,<0.10.0"]
+build-backend = "uv_build"
+
+[tool.pytest.ini_options]
+markers = [
+    "integration: requires credentials and network access",
+    "integration_github: GitHub integration tests",
+    "integration_googledrive: Google Drive integration tests",
+    "integration_dropbox: Dropbox integration tests",
+]
+
+[dependency-groups]
+dev = [
+    "jupyter>=1.1.1",
+    "nbconvert>=7.17.1",
+]

cosmochains-0.0.2/pyproject.toml.orig

@@ -0,0 +1,59 @@
+[project]
+name = "cosmochains"
+version = "0.0.2"
+description = "Cloud-based MCMC chain repository manager"
+readme = "README.md"
+authors = [
+    { name = "Rodrigo Calderon", email = "calderon.cosmology@gmail.com" }
+]
+requires-python = ">=3.12"
+dependencies = [
+    "numpy>=1.26",
+    "pydantic>=2.7",
+    "pyyaml>=6.0",
+    "typer>=0.12",
+    "rich>=13.0",
+    "shellingham>=1.5",
+    "tomli-w>=1.0",
+]
+
+[project.optional-dependencies]
+hf = ["huggingface-hub>=0.23", "python-dotenv>=1.0", "h5py>=3.10"]
+hdf5 = ["h5py>=3.10"]
+zarr = ["zarr>=2.18"]
+getdist = ["getdist>=1.4"]
+github = ["PyGithub>=2.1"]
+google-drive = ["google-api-python-client>=2.100", "google-auth>=2.20"]
+dropbox = ["dropbox>=12.0"]
+all-backends = [
+    "huggingface-hub>=0.23",
+    "python-dotenv>=1.0",
+    "h5py>=3.10",
+    "PyGithub>=2.1",
+    "google-api-python-client>=2.100",
+    "google-auth>=2.20",
+    "dropbox>=12.0",
+]
+dev = ["pytest>=8.0", "pytest-mock>=3.14"]
+docs = ["mkdocs-material>=9.5", "mkdocstrings[python]>=0.25"]
+
+[project.scripts]
+cosmochains = "cosmochains.cli.app:main"
+
+[build-system]
+requires = ["uv_build>=0.9.9,<0.10.0"]
+build-backend = "uv_build"
+
+[tool.pytest.ini_options]
+markers = [
+    "integration: requires credentials and network access",
+    "integration_github: GitHub integration tests",
+    "integration_googledrive: Google Drive integration tests",
+    "integration_dropbox: Dropbox integration tests",
+]
+
+[dependency-groups]
+dev = [
+    "jupyter>=1.1.1",
+    "nbconvert>=7.17.1",
+]

cosmochains-0.0.2/src/cosmochains/__init__.py

@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+import concurrent.futures
+from pathlib import Path
+
+import numpy as np
+
+from .backends.base import ChainBackend
+from .catalog import resolve_name
+from .config import get_default_backend
+from .local import read
+from .models import Chain, ChainArrays, ChainMetadata
+from .readers import get_reader, read_from_zip
+from .stats import Stats, marginal_stats
+
+
+def _backend(backend: ChainBackend | None) -> ChainBackend:
+    return backend if backend is not None else get_default_backend()
+
+
+def load_chain(
+    name: str | list[str],
+    backend: ChainBackend | None = None,
+) -> Chain | dict[str, Chain]:
+    """Download and return one or more chains by name.
+
+    When *name* is a list the chains are fetched concurrently and returned as
+    a ``{name: Chain}`` mapping. When *name* is a string a single
+    :class:`~cosmochains.models.Chain` is returned.
+
+    Args:
+        name: Chain name or list of chain names as registered in the catalog.
+        backend: Backend instance to use. Falls back to
+            :func:`~cosmochains.config.get_default_backend` when ``None``.
+
+    Returns:
+        A single :class:`~cosmochains.models.Chain` or a ``dict[str, Chain]``
+        when *name* is a list.
+
+    Raises:
+        KeyError: If *name* is not found in the catalog.
+        RuntimeError: If no backend is configured and environment variables are
+            missing.
+
+    Example:
+        >>> import os
+        >>> os.environ["COSMOCHAINS_REPO"] = "myorg/my-chains"
+        >>> from cosmochains import load_chain
+        >>> chain = load_chain("planck2018_TTTEEE")
+        >>> chain.samples.shape
+        (5000, 6)
+    """
+    b = _backend(backend)
+    if isinstance(name, list):
+        with concurrent.futures.ThreadPoolExecutor() as pool:
+            futures = {n: pool.submit(_load_one, n, b) for n in name}
+        return {n: f.result() for n, f in futures.items()}
+    return _load_one(name, b)
+
+
+def _load_one(name: str, backend: ChainBackend) -> Chain:
+    path = backend.download(name)
+    catalog = backend.get_catalog()
+    entry = resolve_name(name, catalog)
+    fmt = entry.get("format", "hdf5")
+    if fmt in ("hdf5", "h5"):
+        from .readers.hdf5 import read_hdf5
+
+        return read_hdf5(path)
+    arrays = _read_arrays(path, fmt)
+    metadata = ChainMetadata.model_validate(backend.download_metadata(name))
+    return arrays.to_chain(metadata)
+
+
+def _read_arrays(path: Path, fmt: str) -> ChainArrays:
+    if fmt in ("getdist", "zarr"):
+        return read_from_zip(path, fmt)
+    return get_reader(path).read(path)
+
+
+def list_chains(backend: ChainBackend | None = None) -> list[str]:
+    """Return the names of all chains registered in the catalog.
+
+    Args:
+        backend: Backend instance to use. Falls back to
+            :func:`~cosmochains.config.get_default_backend` when ``None``.
+
+    Returns:
+        Sorted list of chain name strings.
+    """
+    return _backend(backend).list_chains()
+
+
+def get_metadata(name: str, backend: ChainBackend | None = None) -> ChainMetadata:
+    """Fetch and return the metadata for a single chain without loading its arrays.
+
+    Uses the lightweight :meth:`~cosmochains.backends.base.ChainBackend.download_metadata`
+    path — no chain file is downloaded.
+
+    Args:
+        name: Chain name as registered in the catalog.
+        backend: Backend instance to use. Falls back to
+            :func:`~cosmochains.config.get_default_backend` when ``None``.
+
+    Returns:
+        :class:`~cosmochains.models.ChainMetadata` for the requested chain.
+    """
+    b = _backend(backend)
+    return ChainMetadata.model_validate(b.download_metadata(name))
+
+
+def load_samples(name: str, backend: ChainBackend | None = None) -> np.ndarray:
+    """Download a chain and return its sample array.
+
+    Args:
+        name: Chain name as registered in the catalog.
+        backend: Backend instance to use. Falls back to
+            :func:`~cosmochains.config.get_default_backend` when ``None``.
+
+    Returns:
+        NumPy array of shape ``(n_samples, n_params)``.
+    """
+    return load_chain(name, backend=backend).samples
+
+
+def load_weights(name: str, backend: ChainBackend | None = None) -> np.ndarray:
+    """Download a chain and return its importance weights.
+
+    Args:
+        name: Chain name as registered in the catalog.
+        backend: Backend instance to use. Falls back to
+            :func:`~cosmochains.config.get_default_backend` when ``None``.
+
+    Returns:
+        NumPy array of shape ``(n_samples,)``.
+    """
+    return load_chain(name, backend=backend).weights
+
+
+__all__ = [
+    "read",
+    "marginal_stats",
+    "Stats",
+    "load_chain",
+    "list_chains",
+    "get_metadata",
+    "load_samples",
+    "load_weights",
+    "Chain",
+    "ChainMetadata",
+    "ChainBackend",
+]

cosmochains-0.0.2/src/cosmochains/backends/__init__.py

@@ -0,0 +1,12 @@
+from .base import ChainBackend
+from .huggingface import HuggingFaceBackend
+from .huggingface_bucket import HuggingFaceBucketBackend
+
+__all__ = ["ChainBackend", "HuggingFaceBackend", "HuggingFaceBucketBackend"]
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .dropbox import DropboxBackend
+    from .github import GitHubBackend
+    from .googledrive import GoogleDriveBackend

cosmochains-0.0.2/src/cosmochains/backends/base.py

@@ -0,0 +1,108 @@
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class ChainBackend(Protocol):
+    """Structural protocol that all storage backends must satisfy.
+
+    Implementations are duck-typed: any class that provides all methods below
+    is a valid backend regardless of inheritance.
+
+    Built-in implementations:
+
+    - :class:`~cosmochains.backends.huggingface.HuggingFaceBackend` —
+      HuggingFace dataset repository (Git-backed, atomic uploads).
+    - :class:`~cosmochains.backends.huggingface_bucket.HuggingFaceBucketBackend`
+      — HuggingFace S3-compatible bucket (non-atomic, no versioning).
+    - :class:`~cosmochains.backends.github.GitHubBackend` —
+      dedicated branch of a GitHub repository (atomic uploads via Git Data API;
+      100 MB per-file limit). Requires ``cosmochains[github]``.
+    - :class:`~cosmochains.backends.googledrive.GoogleDriveBackend` —
+      Google Drive folder (non-atomic; deleted files go to trash). Requires
+      ``cosmochains[google-drive]``.
+    - :class:`~cosmochains.backends.dropbox.DropboxBackend` —
+      Dropbox folder path (non-atomic; chunked upload for files > 150 MB).
+      Requires ``cosmochains[dropbox]``.
+
+    Select a backend at runtime via the ``COSMOCHAINS_BACKEND`` environment
+    variable or the ``--backend`` CLI flag. Auth tokens can be set with the
+    generic ``COSMOCHAINS_TOKEN`` variable or backend-specific variables
+    (``HF_TOKEN``, ``GITHUB_TOKEN``, ``GOOGLE_APPLICATION_CREDENTIALS``,
+    ``DROPBOX_REFRESH_TOKEN`` + key/secret).
+    """
+
+    def download(self, name: str) -> Path:
+        """Download a chain file and return its local cache path.
+
+        Args:
+            name: Chain name as registered in the catalog.
+
+        Returns:
+            :class:`pathlib.Path` to the cached local file.
+        """
+        ...
+
+    def upload(self, name: str, local_path: Path, metadata: dict) -> None:
+        """Upload a chain file and its metadata YAML to the remote store.
+
+        Args:
+            name: Chain name to register in the remote catalog.
+            local_path: Local file to upload.
+            metadata: Metadata dict that will be serialized as YAML alongside
+                the chain file.
+        """
+        ...
+
+    def download_metadata(self, name: str) -> dict:
+        """Fetch and return the metadata dict for *name*.
+
+        Args:
+            name: Chain name as registered in the catalog.
+
+        Returns:
+            Raw metadata dict loaded from the YAML sidecar file.
+        """
+        ...
+
+    def list_chains(self) -> list[str]:
+        """Return the names of all concrete chains in the remote catalog.
+
+        Returns:
+            List of chain name strings (aliases excluded).
+        """
+        ...
+
+    def get_catalog(self) -> dict:
+        """Fetch and return the full catalog from the remote store.
+
+        Returns:
+            Catalog dict; empty dict when the catalog does not yet exist.
+        """
+        ...
+
+    def update_catalog(self, catalog: dict) -> None:
+        """Write *catalog* back to the remote store, replacing the current version.
+
+        Args:
+            catalog: Complete catalog dict to persist.
+        """
+        ...
+
+    def delete(self, name: str) -> None:
+        """Delete a chain's files and metadata from the remote store.
+
+        Also removes any catalog aliases that point to *name*.
+
+        Args:
+            name: Chain name as registered in the catalog.
+        """
+        ...
+
+    def create_repo(self, *, private: bool = False) -> None:
+        """Create the remote repository or bucket if it does not already exist.
+
+        Args:
+            private: If ``True``, create a private repository/bucket.
+        """
+        ...

cosmochains-0.0.2/src/cosmochains/backends/cache.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+
+def cache_path(backend_key: str, repo_id: str, relative: str) -> Path:
+    """Return a local cache path for a remote file, creating parent dirs as needed.
+
+    Args:
+        backend_key: Short identifier for the backend (e.g. ``"github"``).
+        repo_id: Repository / folder identifier used to namespace the cache.
+        relative: Relative file path within the remote store
+            (e.g. ``"chains/run1/chain.hdf5"``).
+
+    Returns:
+        Absolute :class:`pathlib.Path` inside
+        ``~/.cache/cosmochains/<backend_key>/<hash>/``.
+        Parent directories are created automatically.
+    """
+    safe = hashlib.sha256(repo_id.encode()).hexdigest()[:16]
+    p = Path.home() / ".cache" / "cosmochains" / backend_key / safe / relative
+    p.parent.mkdir(parents=True, exist_ok=True)
+    return p