dclimate-client-py 0.1.1__tar.gz

dclimate_client_py-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 dClimate
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dclimate_client_py-0.1.1/PKG-INFO

@@ -0,0 +1,183 @@
+Metadata-Version: 2.1
+Name: dclimate-client-py
+Version: 0.1.1
+Summary: Python client library for accessing dClimate weather and climate data
+Keywords: climate,weather,data,dclimate,meteorology,climate-data
+Author-Email: Phil Van Meppelen Scheppink <phil@dclimate.net>, Evan Schechter <evan@arbol.io>, Chris Rossi <chris@christophermrossi.com>, Robert Banick <robert.banick@arbol.io>
+License: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Project-URL: Homepage, https://dclimate.net/
+Requires-Python: >=3.12
+Requires-Dist: aiobotocore
+Requires-Dist: xarray>=2025.3.0
+Requires-Dist: rioxarray
+Requires-Dist: zarr>=3.0.8
+Requires-Dist: numpy>=2.1.3
+Requires-Dist: py_hamt>=3.2.0
+Requires-Dist: requests
+Requires-Dist: pyarrow
+Requires-Dist: geopandas
+Requires-Dist: pandas
+Requires-Dist: s3fs
+Requires-Dist: shapely
+Requires-Dist: scipy
+Requires-Dist: pycryptodome>=3.21.0
+Requires-Dist: pytest-asyncio>=1.3.0
+Provides-Extra: testing
+Requires-Dist: pytest; extra == "testing"
+Requires-Dist: pytest-cov; extra == "testing"
+Requires-Dist: pytest-mock; extra == "testing"
+Provides-Extra: dev
+Requires-Dist: pre-commit>=4.1.0; extra == "dev"
+Requires-Dist: ruff>=0.9.5; extra == "dev"
+Description-Content-Type: text/markdown
+
+<p align="center">
+<a href="https://dclimate.net/" target="_blank" rel="noopener noreferrer">
+<img width="50%" src="https://user-images.githubusercontent.com/41392423/173133333-79ef15d0-6671-4be3-ac97-457344e9e958.svg" alt="dClimate logo">
+</a>
+</p>
+
+# dClimate-Client-Py
+[![codecov](https://codecov.io/gh/dClimate/dClimate-Zarr-Client/graph/badge.svg?token=AovaMO6DX5)](https://codecov.io/gh/dClimate/dClimate-Zarr-Client)
+
+Retrieve dClimate GIS zarr datasets stored on IPFS
+
+Uses [py-hamt](https://github.com/dClimate/py-hamt) to access Zarr data structures stored efficiently on IPFS.
+
+Filtering and aggregation are packaged into convenience functions optimized for flexibility and performance.
+
+> **Looking for JavaScript?** Check out our [JavaScript client](https://www.npmjs.com/package/@dclimate/dclimate-client-js) for Node.js and browser environments.
+
+## Usage
+
+```python
+from datetime import datetime
+import dclimate_client_py as client
+from dclimate_client_py import dClimateClient
+
+# --- Recommended: Using dClimateClient (async context manager) ---
+
+async def main():
+    # The client manages IPFS connections automatically
+    # No need to import or configure KuboCAS directly!
+    async with dClimateClient() as dclimate:
+        # Load datasets by name from the internal catalog
+        # For datasets with multiple variants, you must specify which variant
+        dataset = await dclimate.load_dataset(
+            dataset="2m_temperature",
+            collection="era5",
+            variant="finalized",  # Required for multi-variant datasets
+            return_xarray=False   # Returns GeotemporalData wrapper (default)
+        )
+
+        # Apply queries using the GeotemporalData interface
+        dataset_filtered = dataset.point(latitude=40.875, longitude=-104.875)
+        dataset_filtered = dataset_filtered.time_range(
+            datetime(2023, 1, 1),
+            datetime(2023, 1, 5)
+        )
+        data_dict = dataset_filtered.as_dict()
+        print(data_dict['data'])
+
+# Custom IPFS endpoints (optional)
+async def main_custom_ipfs():
+    async with dClimateClient(
+        gateway_base_url="https://ipfs.io",
+        rpc_base_url="http://localhost:5001"
+    ) as dclimate:
+        dataset = await dclimate.load_dataset(
+            dataset="2m_temperature",
+            collection="era5",
+            variant="finalized"
+        )
+        # Query dataset...
+
+# Get raw xarray.Dataset directly
+async def main_xarray():
+    async with dClimateClient() as dclimate:
+        xr_dataset = await dclimate.load_dataset(
+            dataset="2m_temperature",
+            collection="era5",
+            variant="finalized",
+            return_xarray=True  # Returns xarray.Dataset
+        )
+        print(xr_dataset)
+
+# List available datasets in the catalog (synchronous helper)
+catalog = client.list_dataset_catalog()
+for collection in catalog:
+    print(f"Collection: {collection['collection']}")
+    for dataset in collection['datasets']:
+        print(f"  Dataset: {dataset['dataset']}")
+        for variant in dataset['variants']:
+            print(f"    Variant: {variant['variant']}")
+
+```
+
+> More examples can be found at [dClimate Jupyter Notebooks](https://github.com/dClimate/jupyter-notebooks/tree/main/notebooks). To run your own IPFS gateway follow the instructions for [installing ipfs](https://docs.ipfs.tech/install/command-line/#install-official-binary-distributions). For additional assistance find us on [Discord](https://discord.com/invite/bYWVdNDMpe ), if you are an organization or business reach out to us at community at dclimate dot net.
+
+## Create and activate a virtual environment:
+
+``` shell
+uv venv .venv
+source .venv/bin/activate  # macOS/Linux
+.\.venv\Scripts\activate   # Windows
+```
+
+## Install Dependencies
+
+```shell
+uv sync --extra dev --extra testing
+```
+
+## Run tests for your local environment
+```shell
+uv run pytest tests/
+```
+
+## Use Coverage
+
+```shell
+uv run pytest --cov=dclimate_client_py tests/ --cov-report=xml
+```
+
+## Environment requirements
+
+- Optionally you can run your own IPFS Server to host your own datasets or connect to others.
+
+
+## File breakdown:
+
+### client.py
+
+Entrypoint to code, contains `geo_temporal_query`, which combines all possible subsetting
+and aggregation logic in a single function. Can output the data as either a `dict`
+or `bytes` representing an `xarray` dataset.
+
+---
+
+### dclimate_zarr_errors.py
+
+Various exceptions to be raised for bad or invalid user input.
+
+---
+
+### geo_utils.py
+
+Functions to manipulate `xarray` datasets. Contains polygon, rectangle, circle and point spatial
+subsetting options, as well as temporal subsetting. Also allows for both spatial and temporal
+aggregations.
+
+---
+
+### ipfs_retrieval.py
+
+Functions for resolving IPNS names, traversing the dClimate STAC catalog stored on IPFS, and loading Zarr datasets using `py-hamt`. Handles interaction with IPFS gateways and RPC endpoints.

dclimate_client_py-0.1.1/README.md

@@ -0,0 +1,142 @@
+<p align="center">
+<a href="https://dclimate.net/" target="_blank" rel="noopener noreferrer">
+<img width="50%" src="https://user-images.githubusercontent.com/41392423/173133333-79ef15d0-6671-4be3-ac97-457344e9e958.svg" alt="dClimate logo">
+</a>
+</p>
+
+# dClimate-Client-Py
+[![codecov](https://codecov.io/gh/dClimate/dClimate-Zarr-Client/graph/badge.svg?token=AovaMO6DX5)](https://codecov.io/gh/dClimate/dClimate-Zarr-Client)
+
+Retrieve dClimate GIS zarr datasets stored on IPFS
+
+Uses [py-hamt](https://github.com/dClimate/py-hamt) to access Zarr data structures stored efficiently on IPFS.
+
+Filtering and aggregation are packaged into convenience functions optimized for flexibility and performance.
+
+> **Looking for JavaScript?** Check out our [JavaScript client](https://www.npmjs.com/package/@dclimate/dclimate-client-js) for Node.js and browser environments.
+
+## Usage
+
+```python
+from datetime import datetime
+import dclimate_client_py as client
+from dclimate_client_py import dClimateClient
+
+# --- Recommended: Using dClimateClient (async context manager) ---
+
+async def main():
+    # The client manages IPFS connections automatically
+    # No need to import or configure KuboCAS directly!
+    async with dClimateClient() as dclimate:
+        # Load datasets by name from the internal catalog
+        # For datasets with multiple variants, you must specify which variant
+        dataset = await dclimate.load_dataset(
+            dataset="2m_temperature",
+            collection="era5",
+            variant="finalized",  # Required for multi-variant datasets
+            return_xarray=False   # Returns GeotemporalData wrapper (default)
+        )
+
+        # Apply queries using the GeotemporalData interface
+        dataset_filtered = dataset.point(latitude=40.875, longitude=-104.875)
+        dataset_filtered = dataset_filtered.time_range(
+            datetime(2023, 1, 1),
+            datetime(2023, 1, 5)
+        )
+        data_dict = dataset_filtered.as_dict()
+        print(data_dict['data'])
+
+# Custom IPFS endpoints (optional)
+async def main_custom_ipfs():
+    async with dClimateClient(
+        gateway_base_url="https://ipfs.io",
+        rpc_base_url="http://localhost:5001"
+    ) as dclimate:
+        dataset = await dclimate.load_dataset(
+            dataset="2m_temperature",
+            collection="era5",
+            variant="finalized"
+        )
+        # Query dataset...
+
+# Get raw xarray.Dataset directly
+async def main_xarray():
+    async with dClimateClient() as dclimate:
+        xr_dataset = await dclimate.load_dataset(
+            dataset="2m_temperature",
+            collection="era5",
+            variant="finalized",
+            return_xarray=True  # Returns xarray.Dataset
+        )
+        print(xr_dataset)
+
+# List available datasets in the catalog (synchronous helper)
+catalog = client.list_dataset_catalog()
+for collection in catalog:
+    print(f"Collection: {collection['collection']}")
+    for dataset in collection['datasets']:
+        print(f"  Dataset: {dataset['dataset']}")
+        for variant in dataset['variants']:
+            print(f"    Variant: {variant['variant']}")
+
+```
+
+> More examples can be found at [dClimate Jupyter Notebooks](https://github.com/dClimate/jupyter-notebooks/tree/main/notebooks). To run your own IPFS gateway follow the instructions for [installing ipfs](https://docs.ipfs.tech/install/command-line/#install-official-binary-distributions). For additional assistance find us on [Discord](https://discord.com/invite/bYWVdNDMpe ), if you are an organization or business reach out to us at community at dclimate dot net.
+
+## Create and activate a virtual environment:
+
+``` shell
+uv venv .venv
+source .venv/bin/activate  # macOS/Linux
+.\.venv\Scripts\activate   # Windows
+```
+
+## Install Dependencies
+
+```shell
+uv sync --extra dev --extra testing
+```
+
+## Run tests for your local environment
+```shell
+uv run pytest tests/
+```
+
+## Use Coverage
+
+```shell
+uv run pytest --cov=dclimate_client_py tests/ --cov-report=xml
+```
+
+## Environment requirements
+
+- Optionally you can run your own IPFS Server to host your own datasets or connect to others.
+
+
+## File breakdown:
+
+### client.py
+
+Entrypoint to code, contains `geo_temporal_query`, which combines all possible subsetting
+and aggregation logic in a single function. Can output the data as either a `dict`
+or `bytes` representing an `xarray` dataset.
+
+---
+
+### dclimate_zarr_errors.py
+
+Various exceptions to be raised for bad or invalid user input.
+
+---
+
+### geo_utils.py
+
+Functions to manipulate `xarray` datasets. Contains polygon, rectangle, circle and point spatial
+subsetting options, as well as temporal subsetting. Also allows for both spatial and temporal
+aggregations.
+
+---
+
+### ipfs_retrieval.py
+
+Functions for resolving IPNS names, traversing the dClimate STAC catalog stored on IPFS, and loading Zarr datasets using `py-hamt`. Handles interaction with IPFS gateways and RPC endpoints.

dclimate_client_py-0.1.1/dclimate_client_py/.pytest_cache/.gitignore

@@ -0,0 +1,2 @@
+# Created by pytest automatically.
+*

dclimate_client_py-0.1.1/dclimate_client_py/.pytest_cache/CACHEDIR.TAG

@@ -0,0 +1,4 @@
+Signature: 8a477f597d28d172789f06886806bc55
+# This file is a cache directory tag created by pytest.
+# For information about cache directory tags, see:
+#	https://bford.info/cachedir/spec.html

dclimate_client_py-0.1.1/dclimate_client_py/.pytest_cache/README.md

@@ -0,0 +1,8 @@
+# pytest cache directory #
+
+This directory contains data from the pytest's cache plugin,
+which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
+
+**Do not** commit this to version control.
+
+See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.

dclimate_client_py-0.1.1/dclimate_client_py/.pytest_cache/v/cache/nodeids

@@ -0,0 +1 @@
+[]

dclimate_client_py-0.1.1/dclimate_client_py/.pytest_cache/v/cache/stepwise

@@ -0,0 +1 @@
+[]

dclimate_client_py-0.1.1/dclimate_client_py/__init__.py

@@ -0,0 +1,32 @@
+# public API
+from .client import (
+    load_s3,
+    geo_temporal_query,
+)
+from .dclimate_client import dClimateClient
+from .geotemporal_data import GeotemporalData
+from .encryption_codec import (
+    EncryptionCodec,
+)
+from .datasets import (
+    list_dataset_catalog,
+    fetch_cid_from_url,
+    DatasetCatalog,
+    CatalogCollection,
+    CatalogDataset,
+    DatasetVariantConfig,
+)
+
+__all__ = [
+    "dClimateClient",
+    "load_s3",
+    "geo_temporal_query",
+    "list_dataset_catalog",
+    "fetch_cid_from_url",
+    "GeotemporalData",
+    "EncryptionCodec",
+    "DatasetCatalog",
+    "CatalogCollection",
+    "CatalogDataset",
+    "DatasetVariantConfig",
+]

dclimate_client_py-0.1.1/dclimate_client_py/cids.json

@@ -0,0 +1 @@
+{"cpc-precip-conus": "bafyr4iff6vuvpkb4zexkx7exoafsjvfnno4ofidyc37gsto2aaxixg5zia", "cpc-precip-global": "bafyr4ihe35o55qz47y2dbau6ikvfkbrthqn57pdhem5jdxhvqf5u7qlrbm", "chirps-final-p05": "bafyr4ic2n63fkxdvwgh2uzealcrbeubncivcqaptttfrr4vp5ecbe5e65a"}

dclimate_client_py-0.1.1/dclimate_client_py/client.py

@@ -0,0 +1,203 @@
+"""
+Functions that will map to endpoints in the flask app
+"""
+
+import datetime
+import typing
+import xarray as xr
+
+from .dclimate_zarr_errors import (
+    ConflictingGeoRequestError,
+    ConflictingAggregationRequestError,
+    InvalidExportFormatError,
+    InvalidSelectionError,
+)
+from .geotemporal_data import GeotemporalData, DEFAULT_POINT_LIMIT
+from .s3_retrieval import get_dataset_from_s3
+from .ipfs_retrieval import (
+    _get_dataset_by_ipfs_cid,
+)
+from .datasets import (
+    resolve_dataset_source,
+    get_concatenable_variants,
+    find_dataset_by_name,
+    find_collection_by_name,
+    fetch_cid_from_url,
+    DATASET_CATALOG_INTERNAL,
+    DatasetCatalog,
+)
+from .concatenate import concatenate_datasets
+
+def load_s3(
+    dataset_name: str,
+    bucket_name: str,
+) -> GeotemporalData:
+    """
+    Load a Geotemporal dataset from an S3 bucket.
+
+    Parameters
+    ----------
+
+    dataset_name: str
+        The name of the dataset in the bucket.
+    bucket_name: str
+        S3 bucket name where the dataset is going to be fetched
+    """
+    ds = get_dataset_from_s3(dataset_name, bucket_name)
+    return GeotemporalData(ds, dataset_name=dataset_name)
+
+
+def geo_temporal_query(
+    dataset_name: str,
+    source: typing.Literal["s3"] = "s3",
+    bucket_name: str = None,
+    var_name: str = None,
+    gateway_uri_stem: str | None = None,
+    rpc_uri_stem: str | None = None,
+    forecast_reference_time: str = None,
+    point_kwargs: dict = None,
+    circle_kwargs: dict = None,
+    rectangle_kwargs: dict = None,
+    polygon_kwargs: dict = None,
+    multiple_points_kwargs: dict = None,
+    spatial_agg_kwargs: dict = None,
+    temporal_agg_kwargs: dict = None,
+    rolling_agg_kwargs: dict = None,
+    time_range: typing.Optional[typing.List[datetime.datetime]] = None,
+    # as_of: typing.Optional[datetime.datetime] = None, # Removed as_of
+    point_limit: int = DEFAULT_POINT_LIMIT,
+    output_format: str = "array",
+) -> typing.Union[dict, bytes]:
+    """Filter an XArray dataset
+
+    Filter an XArray dataset by specified spatial and/or temporal bounds and aggregate
+    according to spatial and/or temporal logic, if desired. Before aggregating check
+    that the filtered data fits within specified point and area maximums to avoid
+    computationally expensive retrieval and processing operations. When bounds or
+    aggregation logic are not provided, pass the dataset along untouched.
+
+    Return either a numpy array of data values or a NetCDF file.
+
+    Only one of point, circle, rectangle, or polygon kwargs may be provided. Only one of
+    temporal or rolling aggregation kwargs may be provided, although they can be chained
+    with spatial aggregations if desired.
+
+    Args:
+        dataset_name (str): Name used to identify the dataset within the STAC catalog (for IPFS)
+                            or the dataset name in the bucket (for S3).
+        source: (typing.Literal["ipfs", "s3"]): how to pull data. Defaults to "ipfs".
+        bucket_name (str): S3 bucket name where the datasets are going to be fetched (required if source="s3").
+        var_name (str, optional): Specific data variable to use within the dataset.
+        gateway_uri_stem (str | None, optional): Custom IPFS HTTP Gateway URI stem for IPFS source.
+        rpc_uri_stem (str | None, optional): Custom IPFS RPC API URI stem for IPFS source.
+        forecast_reference_time (str): Isoformatted string representing the desire date
+            to return all available forecasts for
+        circle_kwargs (dict, optional): a dictionary of parameters relevant to a
+            circular query
+        rectangle_kwargs (dict, optional): a dictionary of parameters relevant to a
+            rectangular query
+        polygon_kwargs (dict, optional): a dictionary of parameters relevant to a
+            polygonal query
+        multiple_points_kwargs (dict, optional): Parameters for querying multiple specific points.
+        point_kwargs (dict, optional): Parameters for querying a single point.
+        spatial_agg_kwargs (dict, optional): a dictionary of parameters relevant to a
+            spatial aggregation operation
+        temporal_agg_kwargs (dict, optional): a dictionary of parameters relevant to a
+            temporal aggregation operation
+        rolling_agg_kwargs (dict, optional): a dictionary of parameters relevant to a
+            rolling aggregation operation
+        time_range (typing.Optional[typing.List[datetime.datetime]], optional):
+            time range in which to subset data.
+            Defaults to None.
+        # REMOVED  as_of (typing.Optional[datetime.datetime], optional):
+        #     pull in most recent data created before this time. If None, just get most
+        #     recent. Defaults to None.
+        point_limit (int, optional): maximum number of data points user can request.
+            Defaults to DEFAULT_POINT_LIMIT.
+        output_format (str, optional): Current supported formats are `array` and
+            `netcdf`. Defaults to "array", which provides a dict of data
+            values and coordinates.
+
+    Returns:
+        typing.Union[dict, bytes]: Output data as dict (default) or NetCDF bytes.
+    """
+    # Check for incompatible request parameters
+    if (
+        len(
+            [
+                kwarg_dict
+                for kwarg_dict in [
+                    circle_kwargs,
+                    rectangle_kwargs,
+                    polygon_kwargs,
+                    multiple_points_kwargs,
+                    point_kwargs,
+                ]
+                if kwarg_dict is not None
+            ]
+        )
+        > 1
+    ):
+        raise ConflictingGeoRequestError(
+            "User requested more than one type of geographic query, but only one can "
+            "be submitted at a time"
+        )
+    if spatial_agg_kwargs and point_kwargs:
+        raise ConflictingGeoRequestError(
+            "User requested spatial aggregation methods on a single point, "
+            "but these are mutually exclusive parameters. Only one may be requested at "
+            "a time."
+        )
+    if temporal_agg_kwargs and rolling_agg_kwargs:
+        raise ConflictingAggregationRequestError(
+            "User requested both rolling and temporal aggregation, but these are "
+            "mutually exclusive operations. Only one may be requested at a time."
+        )
+    if output_format not in ["array", "netcdf"]:
+        raise InvalidExportFormatError(
+            "User requested an invalid export format. Only 'array' or 'netcdf' "
+            "permitted."
+        )
+
+    # Set defaults to avoid Nones accidentally passed by users causing a TypeError
+    if not point_limit:
+        point_limit = DEFAULT_POINT_LIMIT
+
+    # Load the dataset based on the source
+    if source == "s3":
+        if not bucket_name:
+            raise ValueError("bucket_name is required when source is 's3'")
+        data = load_s3(dataset_name, bucket_name)
+    else:
+        raise ValueError(
+            "Invalid source specified. Must be 's3'. "
+            "IPFS source is deprecated - use dClimateClient instead."
+        )
+
+    # If specific variable is requested, use that
+    if var_name is not None:
+        data = data.use(var_name)
+
+    # Filter data down temporally, then spatially, and check that the size of resulting
+    # dataset fits within the limit. While a user can get the entire DS by providing no
+    # filters, this will almost certainly cause the size checks to fail
+
+    data = data.query(
+        forecast_reference_time=forecast_reference_time,
+        point_kwargs=point_kwargs,
+        circle_kwargs=circle_kwargs,
+        rectangle_kwargs=rectangle_kwargs,
+        polygon_kwargs=polygon_kwargs,
+        multiple_points_kwargs=multiple_points_kwargs,
+        spatial_agg_kwargs=spatial_agg_kwargs,
+        temporal_agg_kwargs=temporal_agg_kwargs,
+        rolling_agg_kwargs=rolling_agg_kwargs,
+        time_range=time_range,
+        point_limit=point_limit,
+    )
+
+    # Export
+    if output_format == "netcdf":
+        return data.to_netcdf()
+    else:  # "array"
+        return data.as_dict()