dclimate-client-py 0.1.1__py3-none-any.whl

dclimate_client_py/.pytest_cache/.gitignore

@@ -0,0 +1,2 @@
+# Created by pytest automatically.
+*

dclimate_client_py/.pytest_cache/CACHEDIR.TAG

@@ -0,0 +1,4 @@
+Signature: 8a477f597d28d172789f06886806bc55
+# This file is a cache directory tag created by pytest.
+# For information about cache directory tags, see:
+#	https://bford.info/cachedir/spec.html

dclimate_client_py/.pytest_cache/README.md

@@ -0,0 +1,8 @@
+# pytest cache directory #
+
+This directory contains data from the pytest's cache plugin,
+which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
+
+**Do not** commit this to version control.
+
+See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.

dclimate_client_py/.pytest_cache/v/cache/nodeids

@@ -0,0 +1 @@
+[]

dclimate_client_py/.pytest_cache/v/cache/stepwise

@@ -0,0 +1 @@
+[]

dclimate_client_py/__init__.py

@@ -0,0 +1,32 @@
+# public API
+from .client import (
+    load_s3,
+    geo_temporal_query,
+)
+from .dclimate_client import dClimateClient
+from .geotemporal_data import GeotemporalData
+from .encryption_codec import (
+    EncryptionCodec,
+)
+from .datasets import (
+    list_dataset_catalog,
+    fetch_cid_from_url,
+    DatasetCatalog,
+    CatalogCollection,
+    CatalogDataset,
+    DatasetVariantConfig,
+)
+
+__all__ = [
+    "dClimateClient",
+    "load_s3",
+    "geo_temporal_query",
+    "list_dataset_catalog",
+    "fetch_cid_from_url",
+    "GeotemporalData",
+    "EncryptionCodec",
+    "DatasetCatalog",
+    "CatalogCollection",
+    "CatalogDataset",
+    "DatasetVariantConfig",
+]

dclimate_client_py/cids.json

@@ -0,0 +1 @@
+{"cpc-precip-conus": "bafyr4iff6vuvpkb4zexkx7exoafsjvfnno4ofidyc37gsto2aaxixg5zia", "cpc-precip-global": "bafyr4ihe35o55qz47y2dbau6ikvfkbrthqn57pdhem5jdxhvqf5u7qlrbm", "chirps-final-p05": "bafyr4ic2n63fkxdvwgh2uzealcrbeubncivcqaptttfrr4vp5ecbe5e65a"}

dclimate_client_py/client.py

@@ -0,0 +1,203 @@
+"""
+Functions that will map to endpoints in the flask app
+"""
+
+import datetime
+import typing
+import xarray as xr
+
+from .dclimate_zarr_errors import (
+    ConflictingGeoRequestError,
+    ConflictingAggregationRequestError,
+    InvalidExportFormatError,
+    InvalidSelectionError,
+)
+from .geotemporal_data import GeotemporalData, DEFAULT_POINT_LIMIT
+from .s3_retrieval import get_dataset_from_s3
+from .ipfs_retrieval import (
+    _get_dataset_by_ipfs_cid,
+)
+from .datasets import (
+    resolve_dataset_source,
+    get_concatenable_variants,
+    find_dataset_by_name,
+    find_collection_by_name,
+    fetch_cid_from_url,
+    DATASET_CATALOG_INTERNAL,
+    DatasetCatalog,
+)
+from .concatenate import concatenate_datasets
+
+def load_s3(
+    dataset_name: str,
+    bucket_name: str,
+) -> GeotemporalData:
+    """
+    Load a Geotemporal dataset from an S3 bucket.
+
+    Parameters
+    ----------
+
+    dataset_name: str
+        The name of the dataset in the bucket.
+    bucket_name: str
+        S3 bucket name where the dataset is going to be fetched
+    """
+    ds = get_dataset_from_s3(dataset_name, bucket_name)
+    return GeotemporalData(ds, dataset_name=dataset_name)
+
+
+def geo_temporal_query(
+    dataset_name: str,
+    source: typing.Literal["s3"] = "s3",
+    bucket_name: str = None,
+    var_name: str = None,
+    gateway_uri_stem: str | None = None,
+    rpc_uri_stem: str | None = None,
+    forecast_reference_time: str = None,
+    point_kwargs: dict = None,
+    circle_kwargs: dict = None,
+    rectangle_kwargs: dict = None,
+    polygon_kwargs: dict = None,
+    multiple_points_kwargs: dict = None,
+    spatial_agg_kwargs: dict = None,
+    temporal_agg_kwargs: dict = None,
+    rolling_agg_kwargs: dict = None,
+    time_range: typing.Optional[typing.List[datetime.datetime]] = None,
+    # as_of: typing.Optional[datetime.datetime] = None, # Removed as_of
+    point_limit: int = DEFAULT_POINT_LIMIT,
+    output_format: str = "array",
+) -> typing.Union[dict, bytes]:
+    """Filter an XArray dataset
+
+    Filter an XArray dataset by specified spatial and/or temporal bounds and aggregate
+    according to spatial and/or temporal logic, if desired. Before aggregating check
+    that the filtered data fits within specified point and area maximums to avoid
+    computationally expensive retrieval and processing operations. When bounds or
+    aggregation logic are not provided, pass the dataset along untouched.
+
+    Return either a numpy array of data values or a NetCDF file.
+
+    Only one of point, circle, rectangle, or polygon kwargs may be provided. Only one of
+    temporal or rolling aggregation kwargs may be provided, although they can be chained
+    with spatial aggregations if desired.
+
+    Args:
+        dataset_name (str): Name used to identify the dataset within the STAC catalog (for IPFS)
+                            or the dataset name in the bucket (for S3).
+        source: (typing.Literal["ipfs", "s3"]): how to pull data. Defaults to "ipfs".
+        bucket_name (str): S3 bucket name where the datasets are going to be fetched (required if source="s3").
+        var_name (str, optional): Specific data variable to use within the dataset.
+        gateway_uri_stem (str | None, optional): Custom IPFS HTTP Gateway URI stem for IPFS source.
+        rpc_uri_stem (str | None, optional): Custom IPFS RPC API URI stem for IPFS source.
+        forecast_reference_time (str): Isoformatted string representing the desire date
+            to return all available forecasts for
+        circle_kwargs (dict, optional): a dictionary of parameters relevant to a
+            circular query
+        rectangle_kwargs (dict, optional): a dictionary of parameters relevant to a
+            rectangular query
+        polygon_kwargs (dict, optional): a dictionary of parameters relevant to a
+            polygonal query
+        multiple_points_kwargs (dict, optional): Parameters for querying multiple specific points.
+        point_kwargs (dict, optional): Parameters for querying a single point.
+        spatial_agg_kwargs (dict, optional): a dictionary of parameters relevant to a
+            spatial aggregation operation
+        temporal_agg_kwargs (dict, optional): a dictionary of parameters relevant to a
+            temporal aggregation operation
+        rolling_agg_kwargs (dict, optional): a dictionary of parameters relevant to a
+            rolling aggregation operation
+        time_range (typing.Optional[typing.List[datetime.datetime]], optional):
+            time range in which to subset data.
+            Defaults to None.
+        # REMOVED  as_of (typing.Optional[datetime.datetime], optional):
+        #     pull in most recent data created before this time. If None, just get most
+        #     recent. Defaults to None.
+        point_limit (int, optional): maximum number of data points user can request.
+            Defaults to DEFAULT_POINT_LIMIT.
+        output_format (str, optional): Current supported formats are `array` and
+            `netcdf`. Defaults to "array", which provides a dict of data
+            values and coordinates.
+
+    Returns:
+        typing.Union[dict, bytes]: Output data as dict (default) or NetCDF bytes.
+    """
+    # Check for incompatible request parameters
+    if (
+        len(
+            [
+                kwarg_dict
+                for kwarg_dict in [
+                    circle_kwargs,
+                    rectangle_kwargs,
+                    polygon_kwargs,
+                    multiple_points_kwargs,
+                    point_kwargs,
+                ]
+                if kwarg_dict is not None
+            ]
+        )
+        > 1
+    ):
+        raise ConflictingGeoRequestError(
+            "User requested more than one type of geographic query, but only one can "
+            "be submitted at a time"
+        )
+    if spatial_agg_kwargs and point_kwargs:
+        raise ConflictingGeoRequestError(
+            "User requested spatial aggregation methods on a single point, "
+            "but these are mutually exclusive parameters. Only one may be requested at "
+            "a time."
+        )
+    if temporal_agg_kwargs and rolling_agg_kwargs:
+        raise ConflictingAggregationRequestError(
+            "User requested both rolling and temporal aggregation, but these are "
+            "mutually exclusive operations. Only one may be requested at a time."
+        )
+    if output_format not in ["array", "netcdf"]:
+        raise InvalidExportFormatError(
+            "User requested an invalid export format. Only 'array' or 'netcdf' "
+            "permitted."
+        )
+
+    # Set defaults to avoid Nones accidentally passed by users causing a TypeError
+    if not point_limit:
+        point_limit = DEFAULT_POINT_LIMIT
+
+    # Load the dataset based on the source
+    if source == "s3":
+        if not bucket_name:
+            raise ValueError("bucket_name is required when source is 's3'")
+        data = load_s3(dataset_name, bucket_name)
+    else:
+        raise ValueError(
+            "Invalid source specified. Must be 's3'. "
+            "IPFS source is deprecated - use dClimateClient instead."
+        )
+
+    # If specific variable is requested, use that
+    if var_name is not None:
+        data = data.use(var_name)
+
+    # Filter data down temporally, then spatially, and check that the size of resulting
+    # dataset fits within the limit. While a user can get the entire DS by providing no
+    # filters, this will almost certainly cause the size checks to fail
+
+    data = data.query(
+        forecast_reference_time=forecast_reference_time,
+        point_kwargs=point_kwargs,
+        circle_kwargs=circle_kwargs,
+        rectangle_kwargs=rectangle_kwargs,
+        polygon_kwargs=polygon_kwargs,
+        multiple_points_kwargs=multiple_points_kwargs,
+        spatial_agg_kwargs=spatial_agg_kwargs,
+        temporal_agg_kwargs=temporal_agg_kwargs,
+        rolling_agg_kwargs=rolling_agg_kwargs,
+        time_range=time_range,
+        point_limit=point_limit,
+    )
+
+    # Export
+    if output_format == "netcdf":
+        return data.to_netcdf()
+    else:  # "array"
+        return data.as_dict()

dclimate_client_py/concatenate.py

@@ -0,0 +1,207 @@
+"""
+Concatenation utilities for combining dataset variants.
+
+Implements smart concatenation logic similar to dclimate-client-js,
+avoiding duplicate data by finding split indices where new data begins.
+"""
+
+import logging
+import typing
+from datetime import datetime
+import numpy as np
+import xarray as xr
+
+from .dclimate_zarr_errors import NoDataFoundError
+
+logger = logging.getLogger(__name__)
+
+
+def find_split_index(
+    combined_coords: typing.Any,
+    next_coords: typing.Any,
+    last_coord_value: typing.Any,
+) -> int:
+    """
+    Find the index in next_coords where values are strictly greater than last_coord_value.
+
+    This prevents duplicate data when concatenating datasets.
+
+    Args:
+        combined_coords: Coordinates from the combined dataset (for reference)
+        next_coords: Coordinates from the next variant to concatenate
+        last_coord_value: The last coordinate value from the combined dataset
+
+    Returns:
+        Index in next_coords where to start slicing (first value > last_coord_value)
+
+    Raises:
+        NoDataFoundError: If no new data found in next variant
+    """
+    # Convert coordinates to numpy array for easier comparison
+    next_array = np.array(next_coords)
+
+    # Handle different data types
+    if isinstance(last_coord_value, (datetime, np.datetime64)):
+        # Convert to numpy datetime64 for comparison
+        if isinstance(last_coord_value, datetime):
+            last_coord_value = np.datetime64(last_coord_value)
+
+        # Find where next coords are strictly after last coord
+        mask = next_array > last_coord_value
+        indices = np.where(mask)[0]
+
+        if len(indices) == 0:
+            raise NoDataFoundError(
+                f"No new data found in next variant after {last_coord_value}"
+            )
+
+        return int(indices[0])
+
+    elif isinstance(last_coord_value, (int, float, np.number)):
+        # Numeric comparison
+        mask = next_array > last_coord_value
+        indices = np.where(mask)[0]
+
+        if len(indices) == 0:
+            raise NoDataFoundError(
+                f"No new data found in next variant after {last_coord_value}"
+            )
+
+        return int(indices[0])
+
+    elif isinstance(last_coord_value, str):
+        # String comparison (lexicographic)
+        mask = next_array > last_coord_value
+        indices = np.where(mask)[0]
+
+        if len(indices) == 0:
+            raise NoDataFoundError(
+                f"No new data found in next variant after '{last_coord_value}'"
+            )
+
+        return int(indices[0])
+
+    else:
+        # For other types, try direct comparison
+        try:
+            mask = next_array > last_coord_value
+            indices = np.where(mask)[0]
+
+            if len(indices) == 0:
+                raise NoDataFoundError(
+                    f"No new data found in next variant after {last_coord_value}"
+                )
+
+            return int(indices[0])
+        except Exception as e:
+            raise TypeError(
+                f"Cannot compare coordinate type {type(last_coord_value).__name__}: {e}"
+            )
+
+
+async def concatenate_datasets(
+    datasets: typing.List[xr.Dataset],
+    dimension: str = "time",
+) -> xr.Dataset:
+    """
+    Concatenate multiple xarray datasets along a dimension, avoiding duplicate data.
+
+    Implements smart concatenation logic:
+    1. Start with first dataset (highest priority)
+    2. For each subsequent dataset:
+       - Find the last coordinate value in combined dataset
+       - Find split index in next dataset where coords > last coord
+       - Slice next dataset to only include new data
+       - Concatenate sliced dataset
+
+    Args:
+        datasets: List of xarray datasets to concatenate (in priority order)
+        dimension: Dimension to concatenate along (default: "time")
+
+    Returns:
+        Concatenated xarray dataset
+
+    Raises:
+        ValueError: If datasets list is empty or dimension not found
+        NoDataFoundError: If a variant contains no new data
+    """
+    if not datasets:
+        raise ValueError("Cannot concatenate empty list of datasets")
+
+    if len(datasets) == 1:
+        logger.info("Only one dataset provided, returning it without concatenation")
+        return datasets[0]
+
+    # Check that all datasets have the dimension
+    for i, ds in enumerate(datasets):
+        if dimension not in ds.dims:
+            raise ValueError(
+                f"Dataset {i} does not have dimension '{dimension}'. "
+                f"Available dimensions: {list(ds.dims.keys())}"
+            )
+
+    logger.info(f"Concatenating {len(datasets)} datasets along '{dimension}' dimension")
+
+    # Start with the first dataset (highest priority)
+    combined = datasets[0]
+    logger.debug(
+        f"Starting with dataset 1/{len(datasets)}, "
+        f"{dimension} range: {combined[dimension].values[0]} to {combined[dimension].values[-1]}"
+    )
+
+    # Concatenate each subsequent dataset
+    for i, next_ds in enumerate(datasets[1:], start=2):
+        # Get the last coordinate value from combined dataset
+        last_coord_value = combined[dimension].values[-1]
+
+        # Get coordinates from next dataset
+        next_coords = next_ds[dimension].values
+
+        logger.debug(
+            f"Processing dataset {i}/{len(datasets)}, "
+            f"{dimension} range: {next_coords[0]} to {next_coords[-1]}"
+        )
+
+        # Find where to split the next dataset
+        try:
+            split_index = find_split_index(
+                combined[dimension].values,
+                next_coords,
+                last_coord_value,
+            )
+
+            logger.debug(
+                f"Found split index {split_index} (out of {len(next_coords)} coords) "
+                f"for dataset {i}, new data starts at: {next_coords[split_index]}"
+            )
+
+            # Slice the next dataset to only include new data
+            sliced_next = next_ds.isel({dimension: slice(split_index, None)})
+
+            logger.debug(
+                f"Sliced dataset {i} to {len(sliced_next[dimension])} new coords"
+            )
+
+            # Concatenate with combined dataset
+            new_combined = xr.concat(
+                [combined, sliced_next],
+                dim=dimension,
+            )
+            logger.debug(
+                f"After concatenating dataset {i}, total {dimension} coords: {len(combined[dimension])}"
+            )
+
+        except NoDataFoundError as e:
+            logger.warning(
+                f"Skipping dataset {i} as it contains no new data: {e}"
+            )
+            # Continue to next dataset
+            continue
+
+    logger.info(
+        f"Concatenation complete. Final dataset has {len(combined[dimension])} "
+        f"{dimension} coordinates ranging from {combined[dimension].values[0]} "
+        f"to {combined[dimension].values[-1]}"
+    )
+
+    return combined