cecil 0.0.21__py3-none-any.whl → 0.0.30__py3-none-any.whl

cecil/client.py

@@ -1,16 +1,20 @@
 import os
-from typing import Dict, List
 
 import pandas as pd
 import requests
 import snowflake.connector
+import xarray
+
 from pydantic import BaseModel
 from requests import auth
 from cryptography.hazmat.primitives import serialization
+from typing import Dict, List, Optional
+from warnings import warn
 
 from .errors import (
     Error,
     _handle_bad_request,
+    _handle_method_not_allowed,
     _handle_not_found,
     _handle_too_many_requests,
     _handle_unprocessable_entity,
@@ -31,8 +35,13 @@ from .models import (
     TransformationCreate,
     User,
     UserCreate,
+    DataRequestMetadata,
+    DataRequestParquetFiles,
+    DataRequestLoadXarray,
 )
 from .version import __version__
+from .xarray import load_xarray
+from .xarray import load_xarray_v2
 
 
 class Client:
@@ -43,9 +52,12 @@ class Client:
         )
         self._snowflake_user_creds = None
 
-    def create_aoi(self, name: str, geometry: Dict) -> AOI:
+    def create_aoi(self, geometry: Dict, external_ref: Optional[str] = None) -> AOI:
         # TODO: validate geometry
-        res = self._post(url="/v0/aois", model=AOICreate(name=name, geometry=geometry))
+        res = self._post(
+            url="/v0/aois",
+            model=AOICreate(geometry=geometry, external_ref=external_ref),
+        )
         return AOI(**res)
 
     def get_aoi(self, id: str) -> AOI:
@@ -56,10 +68,14 @@ class Client:
         res = self._get(url="/v0/aois")
         return [AOIRecord(**record) for record in res["records"]]
 
-    def create_data_request(self, aoi_id: str, dataset_id: str) -> DataRequest:
+    def create_data_request(
+        self, aoi_id: str, dataset_id: str, external_ref: Optional[str] = None
+    ) -> DataRequest:
         res = self._post(
             url="/v0/data-requests",
-            model=DataRequestCreate(aoi_id=aoi_id, dataset_id=dataset_id),
+            model=DataRequestCreate(
+                aoi_id=aoi_id, dataset_id=dataset_id, external_ref=external_ref
+            ),
         )
         return DataRequest(**res)
 
@@ -71,9 +87,32 @@ class Client:
         res = self._get(url="/v0/data-requests")
         return [DataRequest(**record) for record in res["records"]]
 
+    def load_xarray(self, data_request_id: str) -> xarray.Dataset:
+        res = self._get(url=f"/v0/data-requests/{data_request_id}/metadata")
+        metadata = DataRequestMetadata(**res)
+        return load_xarray(metadata)
+
+    def load_xarray_v2(self, data_request_id: str) -> xarray.Dataset:
+        res = self._get(url=f"/v0/data-requests/{data_request_id}/load-xarray")
+        load_xarray_info = DataRequestLoadXarray(**res)
+        return load_xarray_v2(load_xarray_info)
+
+    def load_dataframe(self, data_request_id: str) -> pd.DataFrame:
+        res = self._get(url=f"/v0/data-requests/{data_request_id}/parquet-files")
+        metadata = DataRequestParquetFiles(**res)
+        df = pd.concat((pd.read_parquet(f) for f in metadata.files))
+        return df[
+            [col for col in df.columns if col not in ("organisation_id", "created_at")]
+        ]
+
     def create_transformation(
         self, data_request_id: str, crs: str, spatial_resolution: float
     ) -> Transformation:
+        warn(
+            "create_transformation() is deprecated, refer to https://github.com/cecilearth/examples",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         res = self._post(
             url="/v0/transformations",
             model=TransformationCreate(
@@ -85,14 +124,32 @@ class Client:
         return Transformation(**res)
 
     def get_transformation(self, id: str) -> Transformation:
+        warn(
+            "get_transformation() is deprecated.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         res = self._get(url=f"/v0/transformations/{id}")
         return Transformation(**res)
 
     def list_transformations(self) -> List[Transformation]:
+        warn(
+            "list_transformations() is deprecated.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         res = self._get(url="/v0/transformations")
         return [Transformation(**record) for record in res["records"]]
 
     def query(self, sql: str) -> pd.DataFrame:
+        warn(
+            "query() is deprecated, use load_xarray() or load_dataframe() instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self._query(sql)
+
+    def _query(self, sql: str) -> pd.DataFrame:
         if self._snowflake_user_creds is None:
             res = self._get(url="/v0/snowflake-user-credentials")
             self._snowflake_user_creds = SnowflakeUserCredentials(**res)
@@ -196,6 +253,8 @@ class Client:
                     raise Error("unauthorised")
                 case 404:
                     _handle_not_found(err.response)
+                case 405:
+                    _handle_method_not_allowed(err.response)
                 case 422:
                     _handle_unprocessable_entity(err.response)
                 case 429:

cecil/errors.py

@@ -36,6 +36,17 @@ def _handle_bad_request(response):
     raise Error("bad request", details)
 
 
+def _handle_method_not_allowed(response):
+    if not _is_json(response.text):
+        raise Error("method not allowed")
+
+    details = {}
+    for key, value in response.json().items():
+        details[_format_json_key(key)] = value
+
+    raise Error("method not allowed", details)
+
+
 def _handle_not_found(response):
     if not _is_json(response.text):
         raise Error("resource not found")

cecil/models.py

@@ -1,5 +1,5 @@
 import datetime
-from typing import Dict, Optional
+from typing import Dict, Optional, List
 
 from pydantic import BaseModel, ConfigDict, SecretStr
 from pydantic.alias_generators import to_camel
@@ -8,7 +8,7 @@ from pydantic.alias_generators import to_camel
 class AOI(BaseModel):
     model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
     id: str
-    name: str
+    external_ref: Optional[str]
     geometry: Dict
     hectares: float
     created_at: datetime.datetime
@@ -18,7 +18,7 @@ class AOI(BaseModel):
 class AOIRecord(BaseModel):
     model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
     id: str
-    name: str
+    external_ref: Optional[str]
     hectares: float
     created_at: datetime.datetime
     created_by: str
@@ -26,8 +26,8 @@ class AOIRecord(BaseModel):
 
 class AOICreate(BaseModel):
     model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
-    name: str
     geometry: Dict
+    external_ref: Optional[str]
 
 
 class DataRequest(BaseModel):
@@ -35,6 +35,7 @@ class DataRequest(BaseModel):
     id: str
     aoi_id: str
     dataset_id: str
+    external_ref: Optional[str]
     created_at: datetime.datetime
     created_by: str
 
@@ -43,6 +44,7 @@ class DataRequestCreate(BaseModel):
     model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
     aoi_id: str
     dataset_id: str
+    external_ref: Optional[str]
 
 
 class OrganisationSettings(BaseModel):
@@ -108,3 +110,54 @@ class UserCreate(BaseModel):
     first_name: str
     last_name: str
     email: str
+
+
+class Band(BaseModel):
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+    variable_name: str
+    time: str
+    time_pattern: str
+    number: int
+
+
+class File(BaseModel):
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+    url: str
+    bands: List[Band]
+
+
+class DataRequestMetadata(BaseModel):
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+    provider_name: str
+    dataset_id: str
+    dataset_name: str
+    dataset_crs: str
+    aoi_id: str
+    data_request_id: str
+    files: List[File]
+
+
+class Bucket(BaseModel):
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+    name: str
+    prefix: str
+    access_key_id: str
+    secret_access_key: str
+    session_token: str
+    expiration: datetime.datetime
+
+
+class DataRequestLoadXarray(BaseModel):
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+    provider_name: str
+    dataset_id: str
+    dataset_name: str
+    dataset_crs: str
+    aoi_id: str
+    data_request_id: str
+    bucket: Bucket
+
+
+class DataRequestParquetFiles(BaseModel):
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+    files: List[str]

cecil/version.py

@@ -1 +1 @@
-__version__ = "0.0.21"
+__version__ = "0.0.30"

cecil/xarray.py

@@ -0,0 +1,165 @@
+import os
+import re
+import time
+
+import boto3
+import rioxarray
+import xarray
+
+from datetime import datetime
+
+from .errors import Error
+from .models import DataRequestMetadata, DataRequestLoadXarray, Bucket
+
+os.environ["GDAL_DISABLE_READDIR_ON_OPEN"] = "TRUE"
+
+
+def align_pixel_grids(time_series):
+    # Use the first timestep as reference
+    reference_da = time_series[0]
+    aligned_series = [reference_da]
+
+    # Align all other timesteps to the reference grid
+    for i, da in enumerate(time_series[1:], 1):
+        try:
+            aligned_da = da.rio.reproject_match(reference_da)
+            aligned_series.append(aligned_da)
+        except Exception:
+            raise Error
+
+    return aligned_series
+
+
+def retry_with_exponential_backoff(
+    func, retries, start_delay, multiplier, *args, **kwargs
+):
+    delay = start_delay
+    for attempt in range(1, retries + 1):
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            if attempt == retries:
+                raise e
+            time.sleep(delay)
+            delay *= multiplier
+    return None
+
+
+def load_file(url: str):
+    return rioxarray.open_rasterio(
+        url,
+        chunks={"x": 2000, "y": 2000},
+    )
+
+
+def load_xarray(metadata: DataRequestMetadata) -> xarray.Dataset:
+    data_vars = {}
+
+    for f in metadata.files:
+        try:
+            dataset = retry_with_exponential_backoff(load_file, 5, 1, 2, f.url)
+        except Exception as e:
+            raise ValueError(f"failed to load file: {e}")
+
+        for b in f.bands:
+            band = dataset.sel(band=b.number, drop=True)
+
+            if b.time and b.time_pattern:
+                t = datetime.strptime(b.time, b.time_pattern)
+                band = band.expand_dims("time")
+                band = band.assign_coords(time=[t])
+
+            band.name = b.variable_name
+
+            if b.variable_name not in data_vars:
+                data_vars[b.variable_name] = []
+
+            data_vars[b.variable_name].append(band)
+
+    for variable_name, time_series in data_vars.items():
+        if "time" in time_series[0].dims:
+            # time_series = align_pixel_grids(time_series)
+            data_vars[variable_name] = xarray.concat(
+                time_series, dim="time", join="exact"
+            )
+        else:
+            data_vars[variable_name] = time_series[0]
+
+    return xarray.Dataset(
+        data_vars=data_vars,
+        attrs={
+            "provider_name": metadata.provider_name,
+            "dataset_id": metadata.dataset_id,
+            "dataset_name": metadata.dataset_name,
+            "dataset_crs": metadata.dataset_crs,
+            "aoi_id": metadata.aoi_id,
+            "data_request_id": metadata.data_request_id,
+        },
+    )
+
+
+def load_xarray_v2(load_xarray_info: DataRequestLoadXarray) -> xarray.Dataset:
+    data_vars = {}
+
+    keys = _get_xarray_keys(load_xarray_info.bucket)
+    for key in keys:
+        timestamp_pattern = re.compile(r"\d{4}/\d{2}/\d{2}/\d{2}/\d{2}/\d{2}")
+        timestamp_str = timestamp_pattern.search(key).group()
+
+        variable_name = key.split("/")[14]
+        filename = f"s3://{load_xarray_info.bucket.name}/{key}"
+        dataset = rioxarray.open_rasterio(filename, chunks={"x": 2000, "y": 2000})
+        band = dataset.sel(band=1, drop=True)
+        band.name = variable_name
+
+        # Dataset without time information
+        if timestamp_str != "0000/00/00/00/00/00":
+            time = datetime.strptime(timestamp_str, "%Y/%m/%d/%H/%M/%S")
+            band = band.expand_dims("time")
+            band = band.assign_coords(time=[time])
+
+        if variable_name not in data_vars:
+            data_vars[variable_name] = []
+
+        data_vars[variable_name].append(band)
+
+    for variable_name, time_series in data_vars.items():
+        if "time" in time_series[0].dims:
+            data_vars[variable_name] = xarray.concat(
+                time_series, dim="time", join="exact"
+            )
+        else:
+            data_vars[variable_name] = time_series[0]
+
+    return xarray.Dataset(
+        data_vars=data_vars,
+        attrs={
+            "provider_name": load_xarray_info.provider_name,
+            "dataset_id": load_xarray_info.dataset_id,
+            "dataset_name": load_xarray_info.dataset_name,
+            "dataset_crs": load_xarray_info.dataset_crs,
+            "aoi_id": load_xarray_info.aoi_id,
+            "data_request_id": load_xarray_info.data_request_id,
+        },
+    )
+
+
+def _get_xarray_keys(bucket: Bucket) -> list[str]:
+    os.environ["AWS_ACCESS_KEY_ID"] = bucket.access_key_id
+    os.environ["AWS_SECRET_ACCESS_KEY"] = bucket.secret_access_key
+    os.environ["AWS_SESSION_TOKEN"] = bucket.session_token
+
+    s3_client = boto3.client("s3")
+
+    paginator = s3_client.get_paginator("list_objects_v2")
+    page_iterator = paginator.paginate(
+        Bucket=bucket.name,
+        Prefix=bucket.prefix,
+    )
+
+    keys = []
+    for page in page_iterator:
+        for obj in page.get("Contents", []):
+            keys.append(obj["Key"])
+
+    return keys

{cecil-0.0.21.dist-info → cecil-0.0.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cecil
-Version: 0.0.21
+Version: 0.0.30
 Summary: Python SDK for Cecil Earth
 License-Expression: MIT
 License-File: LICENSE.txt
@@ -8,10 +8,13 @@ Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.8
-Requires-Dist: pydantic
-Requires-Dist: requests
-Requires-Dist: snowflake-connector-python[pandas]
+Requires-Python: >=3.10
+Requires-Dist: dask==2025.9.1
+Requires-Dist: pydantic<3.0.0,>=2.11.9
+Requires-Dist: requests<3.0.0,>=2.32.5
+Requires-Dist: rioxarray==0.19.0
+Requires-Dist: snowflake-connector-python[pandas]<4.0.0,>=3.17.4
+Requires-Dist: xarray==2025.6.1
 Description-Content-Type: text/markdown
 
 # Cecil SDK

cecil-0.0.30.dist-info/RECORD

@@ -0,0 +1,10 @@
+cecil/__init__.py,sha256=AEcRl73BDSAQe6W0d1PDD87IEcumARtREl7dCVa_YQY,86
+cecil/client.py,sha256=mhe7l133-uy7hpRWjcb2s4DD4wsnxPqJLoEuBKMut5I,9501
+cecil/errors.py,sha256=EnyYvFfU_JWYTTRax58bdwOndri2f-HzbqyzxtoV8uo,2100
+cecil/models.py,sha256=lI4UulUv-J0Qh4zrm1UBuqS96CDewyL6sGWRP4AQEQs,4163
+cecil/version.py,sha256=8ZeepqkW4DvpVeNm92mx0tIzgvVevS4NKWkTXXHuXNY,23
+cecil/xarray.py,sha256=K3IRfTkdWAJVUK0LgZLjztpeqhC4QptkrdNg9WYoIVk,5024
+cecil-0.0.30.dist-info/METADATA,sha256=-G1QZ40hNHT_Nz8p5G9PFpqZTTnZoYeW6eEslNdlUGw,2800
+cecil-0.0.30.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cecil-0.0.30.dist-info/licenses/LICENSE.txt,sha256=mUexcmfYx3bG1VIzAdQTOf_NzStYw6-QkKVdUY_d4i4,1066
+cecil-0.0.30.dist-info/RECORD,,

cecil-0.0.21.dist-info/RECORD

@@ -1,9 +0,0 @@
-cecil/__init__.py,sha256=AEcRl73BDSAQe6W0d1PDD87IEcumARtREl7dCVa_YQY,86
-cecil/client.py,sha256=TXcTdNUF8ER8gs8KQBzHIsoQPqDLj3npFeF0PWEi4EI,7430
-cecil/errors.py,sha256=ZNiSTYH2MgNZ7tNIgV07-Ge3KtmdncfzWiBi9yjURGs,1818
-cecil/models.py,sha256=GpW1pT9NBKS5y4Os0pW8UR3MO9kVJe1r8jOK-MjADLA,2799
-cecil/version.py,sha256=PsqtE_T084MVsMv47JyTQ3DK2CRZJ3Kd9Q_vnw02oZk,23
-cecil-0.0.21.dist-info/METADATA,sha256=2Q9DsNWUMtimtqNDdDqbssyFGT33DgWaVjrYHrkLmrs,2659
-cecil-0.0.21.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cecil-0.0.21.dist-info/licenses/LICENSE.txt,sha256=mUexcmfYx3bG1VIzAdQTOf_NzStYw6-QkKVdUY_d4i4,1066
-cecil-0.0.21.dist-info/RECORD,,