dynamical-catalog 0.2.0__py3-none-any.whl

dynamical_catalog/__init__.py

@@ -0,0 +1,87 @@
+"""dynamical_catalog - Load dynamical.org weather datasets in one line."""
+
+from __future__ import annotations
+
+from importlib.metadata import version
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import xarray as xr
+    from zarr.abc.store import Store
+
+from dynamical_catalog._catalog import Catalog
+from dynamical_catalog._stac import clear_cache, load_catalog, set_identifier
+
+__version__ = version("dynamical-catalog")
+
+catalog = Catalog(load_catalog)
+
+
+def identify(identifier: str) -> None:
+    """Set your identity for STAC catalog requests.
+
+    The identifier (typically an email or company name) is included in
+    the User-Agent header: ``dynamical-catalog/0.1.0 (identifier)``.
+
+    Args:
+        identifier: Email or company name, e.g. "marshall@dynamical.org".
+    """
+    set_identifier(identifier)
+
+
+def get_store(dataset_id: str, engine: str = "zarr") -> Store:
+    """Get a zarr Store for a dynamical.org dataset.
+
+    Args:
+        dataset_id: Dataset identifier (e.g. "noaa-gfs-forecast").
+            Underscores are also accepted (e.g. "noaa_gfs_forecast").
+        engine: "zarr" (default) or "icechunk".
+
+    Returns:
+        zarr.abc.Store
+    """
+    from dynamical_catalog._open import _get_store
+
+    return _get_store(_resolve(dataset_id), engine=engine)
+
+
+def open(dataset_id: str, engine: str = "zarr", **kwargs: Any) -> xr.Dataset:
+    """Open a dynamical.org dataset by ID.
+
+    Args:
+        dataset_id: Dataset identifier (e.g. "noaa-gfs-forecast").
+            Underscores are also accepted (e.g. "noaa_gfs_forecast").
+        engine: "zarr" (default) or "icechunk".
+        **kwargs: Passed through to xr.open_zarr().
+
+    Returns:
+        xarray.Dataset
+    """
+    from dynamical_catalog._open import _open_dataset
+
+    return _open_dataset(_resolve(dataset_id), engine=engine, **kwargs)
+
+
+def list() -> list[str]:  # type: ignore[valid-type]
+    """List available dataset IDs."""
+    return sorted(load_catalog().keys())
+
+
+def _resolve(dataset_id: str) -> dict[str, Any]:
+    datasets = load_catalog()
+    normalized_id = dataset_id.replace("_", "-")
+    if normalized_id not in datasets:
+        available = ", ".join(sorted(datasets.keys()))
+        raise ValueError(f"Unknown dataset {dataset_id!r}. Available: {available}")
+    return datasets[normalized_id]
+
+
+__all__ = [
+    "open",
+    "get_store",
+    "list",
+    "identify",
+    "catalog",
+    "clear_cache",
+    "__version__",
+]

dynamical_catalog/_catalog.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from dynamical_catalog._dataset_entry import DatasetEntry
+
+
+class Catalog:
+    """Provides attribute-style access to dynamical.org datasets.
+
+    Supports tab-completion in IPython/Jupyter::
+
+        >>> import dynamical_catalog
+        >>> dynamical_catalog.catalog.<TAB>
+        >>> dynamical_catalog.catalog.noaa_gfs_forecast.open()
+    """
+
+    def __init__(self, loader: Callable[[], dict[str, dict[str, Any]]]) -> None:
+        self._loader = loader
+        self._entries: dict[str, DatasetEntry] = {}
+
+    @property
+    def _datasets(self) -> dict[str, dict[str, Any]]:
+        return self._loader()
+
+    def __getattr__(self, name: str) -> DatasetEntry:
+        if name.startswith("_"):
+            raise AttributeError(name)
+        dataset_id = name.replace("_", "-")
+        datasets = self._datasets
+        if dataset_id not in datasets:
+            available = ", ".join(self._dataset_attr_names())
+            raise AttributeError(f"No dataset named {name!r}. Available: {available}")
+        if dataset_id not in self._entries:
+            self._entries[dataset_id] = DatasetEntry(datasets[dataset_id])
+        return self._entries[dataset_id]
+
+    def __dir__(self) -> list[str]:
+        return list(self._dataset_attr_names()) + list(super().__dir__())
+
+    def _dataset_attr_names(self) -> list[str]:
+        return [did.replace("-", "_") for did in self._datasets]
+
+    def __repr__(self) -> str:
+        names = self._dataset_attr_names()
+        return f"Catalog({len(names)} datasets: {', '.join(names)})"

dynamical_catalog/_dataset_entry.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import xarray as xr
+    from zarr.abc.store import Store
+
+
+class DatasetEntry:
+    """A single dataset from the dynamical.org catalog."""
+
+    def __init__(self, data: dict[str, Any]) -> None:
+        self._data = data
+
+    @property
+    def id(self) -> str:
+        return self._data["id"]
+
+    @property
+    def name(self) -> str:
+        return self._data["name"]
+
+    @property
+    def description(self) -> str:
+        return self._data["description"]
+
+    @property
+    def zarr_url(self) -> str:
+        return self._data["zarr_url"]
+
+    @property
+    def icechunk_config(self) -> dict[str, str] | None:
+        return self._data.get("icechunk")
+
+    @property
+    def status(self) -> str:
+        return self._data["status"]
+
+    def get_store(self, engine: str = "zarr") -> Store:
+        """Get a zarr Store for this dataset.
+
+        Args:
+            engine: "zarr" (default) or "icechunk".
+
+        Returns:
+            zarr.abc.Store
+        """
+        from dynamical_catalog._open import _get_store
+
+        return _get_store(self._data, engine=engine)
+
+    def open(self, engine: str = "zarr", **kwargs: Any) -> xr.Dataset:
+        """Open this dataset as an xarray Dataset.
+
+        Args:
+            engine: "zarr" (default) or "icechunk".
+            **kwargs: Passed through to xr.open_zarr().
+
+        Returns:
+            xarray.Dataset
+        """
+        from dynamical_catalog._open import _open_dataset
+
+        return _open_dataset(self._data, engine=engine, **kwargs)
+
+    def __repr__(self) -> str:
+        return f"DatasetEntry({self.id!r}, {self.name!r})"

dynamical_catalog/_open.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import icechunk
+import xarray as xr
+import zarr.storage
+
+if TYPE_CHECKING:
+    from zarr.abc.store import Store
+
+
+def _get_store(
+    dataset_data: dict[str, Any],
+    engine: str = "zarr",
+) -> Store:
+    if engine == "zarr":
+        return zarr.storage.FsspecStore.from_url(dataset_data["zarr_url"])
+    if engine == "icechunk":
+        return _icechunk_store(dataset_data)
+    raise ValueError(f"Unknown engine {engine!r}. Use 'zarr' or 'icechunk'.")
+
+
+def _open_dataset(
+    dataset_data: dict[str, Any],
+    engine: str = "zarr",
+    **kwargs: Any,
+) -> xr.Dataset:
+    store = _get_store(dataset_data, engine=engine)
+    return xr.open_zarr(store, **kwargs)
+
+
+def _icechunk_store(dataset_data: dict[str, Any]) -> Store:
+    config = dataset_data.get("icechunk")
+    if config is None:
+        raise ValueError(
+            f"Dataset {dataset_data['id']!r} does not have icechunk configuration."
+        )
+
+    storage = icechunk.s3_storage(
+        bucket=config["bucket"],
+        prefix=config["prefix"],
+        region=config["region"],
+        anonymous=True,
+    )
+    repo = icechunk.Repository.open(storage)
+    session = repo.readonly_session("main")
+    return session.store

dynamical_catalog/_stac.py

@@ -0,0 +1,108 @@
+"""Fetch and parse the dynamical.org STAC catalog."""
+
+from __future__ import annotations
+
+import concurrent.futures
+import json
+import threading
+import urllib.error
+import urllib.request
+from typing import Any
+from urllib.parse import urljoin
+
+STAC_CATALOG_URL = "https://dynamical.org/stac/catalog.json"
+
+_TIMEOUT_SECONDS = 30
+_lock = threading.Lock()
+_datasets: dict[str, dict[str, Any]] | None = None
+_identifier: str | None = None
+
+
+def set_identifier(identifier: str) -> None:
+    global _identifier
+    _identifier = identifier
+
+
+def _user_agent() -> str:
+    from dynamical_catalog import __version__
+
+    ua = f"dynamical-catalog/{__version__}"
+    if _identifier:
+        ua += f" ({_identifier})"
+    return ua
+
+
+def _fetch_json(url: str) -> Any:
+    req = urllib.request.Request(url, headers={"User-Agent": _user_agent()})
+    try:
+        with urllib.request.urlopen(req, timeout=_TIMEOUT_SECONDS) as resp:
+            return json.loads(resp.read())
+    except urllib.error.URLError as e:
+        raise RuntimeError(
+            f"Failed to fetch dynamical.org STAC catalog from {url}: {e}"
+        ) from e
+
+
+def _parse_collection(collection: dict[str, Any]) -> dict[str, Any]:
+    """Extract the dataset config we need from a STAC Collection."""
+    assets = collection.get("assets", {})
+    zarr_asset = assets.get("zarr")
+    if zarr_asset is None:
+        raise ValueError(
+            f"STAC Collection {collection.get('id', '?')} is missing a 'zarr' asset"
+        )
+
+    result: dict[str, Any] = {
+        "id": collection["id"],
+        "name": collection.get("title", collection["id"]),
+        "description": collection.get("description", ""),
+        "status": "live",
+        "zarr_url": zarr_asset["href"],
+    }
+
+    icechunk_asset = assets.get("icechunk")
+    if icechunk_asset is not None:
+        storage = icechunk_asset.get("icechunk:storage", {})
+        result["icechunk"] = {
+            "bucket": storage["bucket"],
+            "prefix": storage["prefix"],
+            "region": storage["region"],
+        }
+
+    return result
+
+
+def load_catalog() -> dict[str, dict[str, Any]]:
+    """Fetch the STAC catalog and all child collections.
+
+    Results are cached in-process after the first call.
+    Child collections are fetched in parallel for faster startup.
+    """
+    global _datasets
+    if _datasets is not None:
+        return _datasets
+
+    with _lock:
+        if _datasets is not None:
+            return _datasets
+
+        catalog = _fetch_json(STAC_CATALOG_URL)
+        child_links = [link for link in catalog["links"] if link["rel"] == "child"]
+        urls = [urljoin(STAC_CATALOG_URL, link["href"]) for link in child_links]
+
+        with concurrent.futures.ThreadPoolExecutor() as pool:
+            collections = pool.map(_fetch_json, urls)
+
+        datasets: dict[str, dict[str, Any]] = {}
+        for collection in collections:
+            parsed = _parse_collection(collection)
+            datasets[parsed["id"]] = parsed
+
+        _datasets = datasets
+        return _datasets
+
+
+def clear_cache() -> None:
+    """Clear the cached catalog data, forcing a fresh fetch on next access."""
+    global _datasets
+    _datasets = None

dynamical_catalog-0.2.0.dist-info/METADATA

@@ -0,0 +1,65 @@
+Metadata-Version: 2.4
+Name: dynamical-catalog
+Version: 0.2.0
+Summary: Load dynamical.org weather datasets in one line
+Project-URL: Homepage, https://dynamical.org
+Project-URL: Repository, https://github.com/dynamical-org/dynamical-catalog
+Project-URL: Catalog, https://dynamical.org/catalog/
+Project-URL: Issues, https://github.com/dynamical-org/dynamical-catalog/issues
+Author-email: "dynamical.org" <feedback@dynamical.org>
+License-Expression: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
+Requires-Python: >=3.11
+Requires-Dist: aiohttp
+Requires-Dist: fsspec
+Requires-Dist: icechunk>=1.0.0
+Requires-Dist: xarray>=2025.1.2
+Requires-Dist: zarr>=3.0.8
+Provides-Extra: dev
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-mock; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# dynamical-catalog
+
+Load [dynamical.org](https://dynamical.org) weather datasets in one line.
+
+## Install
+
+```bash
+pip install dynamical-catalog
+```
+
+## Usage
+
+```python
+import dynamical_catalog
+
+# Optional: let us know who you are so we can improve the catalog!
+dynamical_catalog.identify("you@example.com")
+
+# Open a dataset as xarray (zarr v3)
+ds = dynamical_catalog.open("noaa-gfs-forecast")
+
+# Open via icechunk
+ds = dynamical_catalog.open("noaa-gfs-forecast", engine="icechunk")
+
+# Get the underlying zarr store
+store = dynamical_catalog.get_store("noaa-gfs-forecast")
+
+# Tab-completable catalog
+ds = dynamical_catalog.catalog.noaa_gfs_forecast.open()
+
+# List all available datasets
+dynamical_catalog.list()
+```

dynamical_catalog-0.2.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+dynamical_catalog/__init__.py,sha256=jB7lotnHdS9Uq1kM12Oqmz823V_3WyOcd5LG-7DAE0Y,2445
+dynamical_catalog/_catalog.py,sha256=aUv0F-VW776YsYVFxe51R95YgGey3x6aSVP7pRDiziA,1626
+dynamical_catalog/_dataset_entry.py,sha256=fGA-V38GUsfX_d405wjZ9uEGQKCsqoV4Q8CL9nOIX7o,1686
+dynamical_catalog/_open.py,sha256=iUMyxHqfkboFy1yR_eXbUZYkeM_Ibp98D7VNs3ACtAU,1265
+dynamical_catalog/_stac.py,sha256=F7E_mSHGxe8AQcROs3aCN7t7xY0zIqO3NHtVo-7MbCo,3179
+dynamical_catalog/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dynamical_catalog-0.2.0.dist-info/METADATA,sha256=D8R8vTYSvV-zgYuG9mc7V7HVxl_2CDwJsXGydL08_A4,2006
+dynamical_catalog-0.2.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+dynamical_catalog-0.2.0.dist-info/RECORD,,

dynamical_catalog-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any