cecil 0.0.24__py3-none-any.whl → 0.0.30__py3-none-any.whl

cecil/client.py

@@ -9,10 +9,12 @@ from pydantic import BaseModel
 from requests import auth
 from cryptography.hazmat.primitives import serialization
 from typing import Dict, List, Optional
+from warnings import warn
 
 from .errors import (
     Error,
     _handle_bad_request,
+    _handle_method_not_allowed,
     _handle_not_found,
     _handle_too_many_requests,
     _handle_unprocessable_entity,
@@ -34,9 +36,12 @@ from .models import (
     User,
     UserCreate,
     DataRequestMetadata,
+    DataRequestParquetFiles,
+    DataRequestLoadXarray,
 )
 from .version import __version__
 from .xarray import load_xarray
+from .xarray import load_xarray_v2
 
 
 class Client:
@@ -87,9 +92,27 @@ class Client:
         metadata = DataRequestMetadata(**res)
         return load_xarray(metadata)
 
+    def load_xarray_v2(self, data_request_id: str) -> xarray.Dataset:
+        res = self._get(url=f"/v0/data-requests/{data_request_id}/load-xarray")
+        load_xarray_info = DataRequestLoadXarray(**res)
+        return load_xarray_v2(load_xarray_info)
+
+    def load_dataframe(self, data_request_id: str) -> pd.DataFrame:
+        res = self._get(url=f"/v0/data-requests/{data_request_id}/parquet-files")
+        metadata = DataRequestParquetFiles(**res)
+        df = pd.concat((pd.read_parquet(f) for f in metadata.files))
+        return df[
+            [col for col in df.columns if col not in ("organisation_id", "created_at")]
+        ]
+
     def create_transformation(
         self, data_request_id: str, crs: str, spatial_resolution: float
     ) -> Transformation:
+        warn(
+            "create_transformation() is deprecated, refer to https://github.com/cecilearth/examples",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         res = self._post(
             url="/v0/transformations",
             model=TransformationCreate(
@@ -101,14 +124,32 @@ class Client:
         return Transformation(**res)
 
     def get_transformation(self, id: str) -> Transformation:
+        warn(
+            "get_transformation() is deprecated.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         res = self._get(url=f"/v0/transformations/{id}")
         return Transformation(**res)
 
     def list_transformations(self) -> List[Transformation]:
+        warn(
+            "list_transformations() is deprecated.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         res = self._get(url="/v0/transformations")
         return [Transformation(**record) for record in res["records"]]
 
     def query(self, sql: str) -> pd.DataFrame:
+        warn(
+            "query() is deprecated, use load_xarray() or load_dataframe() instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self._query(sql)
+
+    def _query(self, sql: str) -> pd.DataFrame:
         if self._snowflake_user_creds is None:
             res = self._get(url="/v0/snowflake-user-credentials")
             self._snowflake_user_creds = SnowflakeUserCredentials(**res)
@@ -212,6 +253,8 @@ class Client:
                     raise Error("unauthorised")
                 case 404:
                     _handle_not_found(err.response)
+                case 405:
+                    _handle_method_not_allowed(err.response)
                 case 422:
                     _handle_unprocessable_entity(err.response)
                 case 429:

cecil/errors.py

@@ -36,6 +36,17 @@ def _handle_bad_request(response):
     raise Error("bad request", details)
 
 
+def _handle_method_not_allowed(response):
+    if not _is_json(response.text):
+        raise Error("method not allowed")
+
+    details = {}
+    for key, value in response.json().items():
+        details[_format_json_key(key)] = value
+
+    raise Error("method not allowed", details)
+
+
 def _handle_not_found(response):
     if not _is_json(response.text):
         raise Error("resource not found")

cecil/models.py

@@ -135,3 +135,29 @@ class DataRequestMetadata(BaseModel):
     aoi_id: str
     data_request_id: str
     files: List[File]
+
+
+class Bucket(BaseModel):
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+    name: str
+    prefix: str
+    access_key_id: str
+    secret_access_key: str
+    session_token: str
+    expiration: datetime.datetime
+
+
+class DataRequestLoadXarray(BaseModel):
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+    provider_name: str
+    dataset_id: str
+    dataset_name: str
+    dataset_crs: str
+    aoi_id: str
+    data_request_id: str
+    bucket: Bucket
+
+
+class DataRequestParquetFiles(BaseModel):
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+    files: List[str]

cecil/version.py

@@ -1 +1 @@
-__version__ = "0.0.24"
+__version__ = "0.0.30"

cecil/xarray.py

@@ -1,14 +1,17 @@
 import os
+import re
+import time
+
+import boto3
 import rioxarray
 import xarray
 
 from datetime import datetime
 
 from .errors import Error
-from .models import DataRequestMetadata
+from .models import DataRequestMetadata, DataRequestLoadXarray, Bucket
 
-os.environ["GDAL_NUM_THREADS"] = "1"
-os.environ["GDAL_DISABLE_READDIR_ON_OPEN"] = "FALSE"
+os.environ["GDAL_DISABLE_READDIR_ON_OPEN"] = "TRUE"
 
 
 def align_pixel_grids(time_series):
@@ -21,29 +24,50 @@ def align_pixel_grids(time_series):
         try:
             aligned_da = da.rio.reproject_match(reference_da)
             aligned_series.append(aligned_da)
-        except Exception as e:
+        except Exception:
             raise Error
-            continue
 
     return aligned_series
 
 
+def retry_with_exponential_backoff(
+    func, retries, start_delay, multiplier, *args, **kwargs
+):
+    delay = start_delay
+    for attempt in range(1, retries + 1):
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            if attempt == retries:
+                raise e
+            time.sleep(delay)
+            delay *= multiplier
+    return None
+
+
+def load_file(url: str):
+    return rioxarray.open_rasterio(
+        url,
+        chunks={"x": 2000, "y": 2000},
+    )
+
+
 def load_xarray(metadata: DataRequestMetadata) -> xarray.Dataset:
     data_vars = {}
 
     for f in metadata.files:
-        dataset = rioxarray.open_rasterio(
-            f.url,
-            chunks={"x": 2000, "y": 2000},
-        )
+        try:
+            dataset = retry_with_exponential_backoff(load_file, 5, 1, 2, f.url)
+        except Exception as e:
+            raise ValueError(f"failed to load file: {e}")
 
         for b in f.bands:
             band = dataset.sel(band=b.number, drop=True)
 
             if b.time and b.time_pattern:
-                time = datetime.strptime(b.time, b.time_pattern)
+                t = datetime.strptime(b.time, b.time_pattern)
                 band = band.expand_dims("time")
-                band = band.assign_coords(time=[time])
+                band = band.assign_coords(time=[t])
 
             band.name = b.variable_name
 
@@ -72,3 +96,70 @@ def load_xarray(metadata: DataRequestMetadata) -> xarray.Dataset:
             "data_request_id": metadata.data_request_id,
         },
     )
+
+
+def load_xarray_v2(load_xarray_info: DataRequestLoadXarray) -> xarray.Dataset:
+    data_vars = {}
+
+    keys = _get_xarray_keys(load_xarray_info.bucket)
+    for key in keys:
+        timestamp_pattern = re.compile(r"\d{4}/\d{2}/\d{2}/\d{2}/\d{2}/\d{2}")
+        timestamp_str = timestamp_pattern.search(key).group()
+
+        variable_name = key.split("/")[14]
+        filename = f"s3://{load_xarray_info.bucket.name}/{key}"
+        dataset = rioxarray.open_rasterio(filename, chunks={"x": 2000, "y": 2000})
+        band = dataset.sel(band=1, drop=True)
+        band.name = variable_name
+
+        # Dataset without time information
+        if timestamp_str != "0000/00/00/00/00/00":
+            time = datetime.strptime(timestamp_str, "%Y/%m/%d/%H/%M/%S")
+            band = band.expand_dims("time")
+            band = band.assign_coords(time=[time])
+
+        if variable_name not in data_vars:
+            data_vars[variable_name] = []
+
+        data_vars[variable_name].append(band)
+
+    for variable_name, time_series in data_vars.items():
+        if "time" in time_series[0].dims:
+            data_vars[variable_name] = xarray.concat(
+                time_series, dim="time", join="exact"
+            )
+        else:
+            data_vars[variable_name] = time_series[0]
+
+    return xarray.Dataset(
+        data_vars=data_vars,
+        attrs={
+            "provider_name": load_xarray_info.provider_name,
+            "dataset_id": load_xarray_info.dataset_id,
+            "dataset_name": load_xarray_info.dataset_name,
+            "dataset_crs": load_xarray_info.dataset_crs,
+            "aoi_id": load_xarray_info.aoi_id,
+            "data_request_id": load_xarray_info.data_request_id,
+        },
+    )
+
+
+def _get_xarray_keys(bucket: Bucket) -> list[str]:
+    os.environ["AWS_ACCESS_KEY_ID"] = bucket.access_key_id
+    os.environ["AWS_SECRET_ACCESS_KEY"] = bucket.secret_access_key
+    os.environ["AWS_SESSION_TOKEN"] = bucket.session_token
+
+    s3_client = boto3.client("s3")
+
+    paginator = s3_client.get_paginator("list_objects_v2")
+    page_iterator = paginator.paginate(
+        Bucket=bucket.name,
+        Prefix=bucket.prefix,
+    )
+
+    keys = []
+    for page in page_iterator:
+        for obj in page.get("Contents", []):
+            keys.append(obj["Key"])
+
+    return keys

{cecil-0.0.24.dist-info → cecil-0.0.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cecil
-Version: 0.0.24
+Version: 0.0.30
 Summary: Python SDK for Cecil Earth
 License-Expression: MIT
 License-File: LICENSE.txt
@@ -8,10 +8,13 @@ Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.8
-Requires-Dist: pydantic
-Requires-Dist: requests
-Requires-Dist: snowflake-connector-python[pandas]
+Requires-Python: >=3.10
+Requires-Dist: dask==2025.9.1
+Requires-Dist: pydantic<3.0.0,>=2.11.9
+Requires-Dist: requests<3.0.0,>=2.32.5
+Requires-Dist: rioxarray==0.19.0
+Requires-Dist: snowflake-connector-python[pandas]<4.0.0,>=3.17.4
+Requires-Dist: xarray==2025.6.1
 Description-Content-Type: text/markdown
 
 # Cecil SDK

cecil-0.0.30.dist-info/RECORD

@@ -0,0 +1,10 @@
+cecil/__init__.py,sha256=AEcRl73BDSAQe6W0d1PDD87IEcumARtREl7dCVa_YQY,86
+cecil/client.py,sha256=mhe7l133-uy7hpRWjcb2s4DD4wsnxPqJLoEuBKMut5I,9501
+cecil/errors.py,sha256=EnyYvFfU_JWYTTRax58bdwOndri2f-HzbqyzxtoV8uo,2100
+cecil/models.py,sha256=lI4UulUv-J0Qh4zrm1UBuqS96CDewyL6sGWRP4AQEQs,4163
+cecil/version.py,sha256=8ZeepqkW4DvpVeNm92mx0tIzgvVevS4NKWkTXXHuXNY,23
+cecil/xarray.py,sha256=K3IRfTkdWAJVUK0LgZLjztpeqhC4QptkrdNg9WYoIVk,5024
+cecil-0.0.30.dist-info/METADATA,sha256=-G1QZ40hNHT_Nz8p5G9PFpqZTTnZoYeW6eEslNdlUGw,2800
+cecil-0.0.30.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cecil-0.0.30.dist-info/licenses/LICENSE.txt,sha256=mUexcmfYx3bG1VIzAdQTOf_NzStYw6-QkKVdUY_d4i4,1066
+cecil-0.0.30.dist-info/RECORD,,

cecil-0.0.24.dist-info/RECORD

@@ -1,10 +0,0 @@
-cecil/__init__.py,sha256=AEcRl73BDSAQe6W0d1PDD87IEcumARtREl7dCVa_YQY,86
-cecil/client.py,sha256=KyxrMSZpo1R46r6LIy8vSv5YxONsr8xiM4B0W4AGC_o,7923
-cecil/errors.py,sha256=ZNiSTYH2MgNZ7tNIgV07-Ge3KtmdncfzWiBi9yjURGs,1818
-cecil/models.py,sha256=3W892XywxLIZL6TDCHM8_PVRHzR48bJXT5kTV7UU_bY,3509
-cecil/version.py,sha256=ZQhXtFuXIGDwpCAUqfQWo3IQMf-8Lz9t5nYhJnOYBdI,23
-cecil/xarray.py,sha256=CvqfJ7NBMLLA4jf73ek2ZtV3sj8xOII__5S0_l3KXYI,2161
-cecil-0.0.24.dist-info/METADATA,sha256=rWLY1mb0eD52cUBkiofgC2ojdVVtjJ9twSo8KoQWqx4,2659
-cecil-0.0.24.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cecil-0.0.24.dist-info/licenses/LICENSE.txt,sha256=mUexcmfYx3bG1VIzAdQTOf_NzStYw6-QkKVdUY_d4i4,1066
-cecil-0.0.24.dist-info/RECORD,,