openeo-gfmap 0.1.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

openeo_gfmap/features/feature_extractor.py

@@ -1,6 +1,7 @@
 """Feature extractor functionalities. Such as a base class to assist the
 implementation of feature extractors of a UDF.
 """
+
 import functools
 import inspect
 import logging
@@ -32,6 +33,8 @@ class FeatureExtractor(ABC):
     """
 
     def __init__(self) -> None:
+        self._epsg = None
+
         logging.basicConfig(level=logging.INFO)
         self.logger = logging.getLogger(self.__class__.__name__)
 
@@ -88,6 +91,10 @@ class FeatureExtractor(ABC):
         """Returns the EPSG code of the datacube."""
         return self._epsg
 
+    @epsg.setter
+    def epsg(self, value: int):
+        self._epsg = value
+
     def dependencies(self) -> list:
         """Returns the additional dependencies such as wheels or zip files.
         Dependencies should be returned as a list of string, which will set-up at the top of the
@@ -204,6 +211,7 @@ class PatchFeatureExtractor(FeatureExtractor):
         arr.loc[dict(bands=s1_bands_to_select)] = data_to_rescale
         return arr
 
+    # TODO to remove the fixed transpose as it contributes to unclear code.
     def _execute(self, cube: XarrayDataCube, parameters: dict) -> XarrayDataCube:
         arr = cube.get_array().transpose("bands", "t", "y", "x")
         arr = self._common_preparations(arr, parameters)
@@ -371,6 +379,7 @@ def apply_feature_extractor_local(
         )
 
     feature_extractor = feature_extractor_class()
+    feature_extractor._parameters = parameters
     output_labels = feature_extractor.output_labels()
     dependencies = feature_extractor.dependencies()
 

openeo_gfmap/fetching/__init__.py

@@ -7,15 +7,27 @@ component.
 
 import logging
 
-from .fetching import CollectionFetcher, FetchType
-from .s1 import build_sentinel1_grd_extractor
-from .s2 import build_sentinel2_l2a_extractor
-
 _log = logging.getLogger(__name__)
+_log.setLevel(logging.INFO)
+
+ch = logging.StreamHandler()
+ch.setLevel(logging.INFO)
+
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+ch.setFormatter(formatter)
+
+_log.addHandler(ch)
 
 __all__ = [
     "build_sentinel2_l2a_extractor",
     "CollectionFetcher",
     "FetchType",
     "build_sentinel1_grd_extractor",
+    "build_generic_extractor",
+    "build_generic_extractor_stac",
 ]
+
+from .fetching import CollectionFetcher, FetchType  # noqa: E402
+from .generic import build_generic_extractor, build_generic_extractor_stac  # noqa: E402
+from .s1 import build_sentinel1_grd_extractor  # noqa: E402
+from .s2 import build_sentinel2_l2a_extractor  # noqa: E402

openeo_gfmap/fetching/commons.py

@@ -108,6 +108,7 @@ def _load_collection_hybrid(
     return cube
 
 
+# TODO; deprecated?
 def _load_collection(
     connection: openeo.Connection,
     bands: list,

openeo_gfmap/fetching/generic.py

@@ -1,11 +1,10 @@
 """ Generic extraction of features, supporting VITO backend.
 """
 
-from functools import partial
-from typing import Callable
+from typing import Callable, Optional
 
 import openeo
-from geojson import GeoJSON
+from openeo.rest import OpenEoApiError
 
 from openeo_gfmap.backend import Backend, BackendContext
 from openeo_gfmap.fetching import CollectionFetcher, FetchType, _log
@@ -29,15 +28,32 @@ BASE_WEATHER_MAPPING = {
     "vapour-pressure": "AGERA5-VAPOUR",
     "wind-speed": "AGERA5-WIND",
 }
+AGERA5_STAC_MAPPING = {
+    "dewpoint_temperature_mean": "AGERA5-DEWTEMP",
+    "total_precipitation": "AGERA5-PRECIP",
+    "solar_radiation_flux": "AGERA5-SOLRAD",
+    "2m_temperature_max": "AGERA5-TMAX",
+    "2m_temperature_mean": "AGERA5-TMEAN",
+    "2m_temperature_min": "AGERA5-TMIN",
+    "vapour_pressure": "AGERA5-VAPOUR",
+    "wind_speed": "AGERA5-WIND",
+}
+KNOWN_UNTEMPORAL_COLLECTIONS = ["COPERNICUS_30"]
+
+AGERA5_TERRASCOPE_STAC = "https://stac.openeo.vito.be/collections/agera5_daily"
+
 
+def _get_generic_fetcher(
+    collection_name: str, fetch_type: FetchType, backend: Backend, is_stac: bool
+) -> Callable:
+    band_mapping: Optional[dict] = None
 
-def _get_generic_fetcher(collection_name: str, fetch_type: FetchType) -> Callable:
     if collection_name == "COPERNICUS_30":
-        BASE_MAPPING = BASE_DEM_MAPPING
+        band_mapping = BASE_DEM_MAPPING
     elif collection_name == "AGERA5":
-        BASE_MAPPING = BASE_WEATHER_MAPPING
-    else:
-        raise Exception("Please choose a valid collection.")
+        band_mapping = BASE_WEATHER_MAPPING
+    elif is_stac and (AGERA5_TERRASCOPE_STAC in collection_name):
+        band_mapping = AGERA5_STAC_MAPPING
 
     def generic_default_fetcher(
         connection: openeo.Connection,
@@ -46,43 +62,58 @@ def _get_generic_fetcher(collection_name: str, fetch_type: FetchType) -> Callabl
         bands: list,
         **params,
     ) -> openeo.DataCube:
-        bands = convert_band_names(bands, BASE_MAPPING)
+        if band_mapping is not None:
+            bands = convert_band_names(bands, band_mapping)
 
-        if (collection_name == "COPERNICUS_30") and (temporal_extent is not None):
+        if (collection_name in KNOWN_UNTEMPORAL_COLLECTIONS) and (
+            temporal_extent is not None
+        ):
             _log.warning(
-                "User set-up non None temporal extent for DEM collection. Ignoring it."
+                "Ignoring the temporal extent provided by the user as the collection %s is known to be untemporal.",
+                collection_name,
             )
             temporal_extent = None
 
-        cube = _load_collection(
-            connection,
-            bands,
-            collection_name,
-            spatial_extent,
-            temporal_extent,
-            fetch_type,
-            **params,
-        )
+        try:
+            cube = _load_collection(
+                connection,
+                bands,
+                collection_name,
+                spatial_extent,
+                temporal_extent,
+                fetch_type,
+                is_stac=is_stac,
+                **params,
+            )
+        except OpenEoApiError as e:
+            if "CollectionNotFound" in str(e):
+                raise ValueError(
+                    f"Collection {collection_name} not found in the selected backend {backend.value}."
+                ) from e
+            raise e
 
-        # Apply if the collection is a GeoJSON Feature collection
-        if isinstance(spatial_extent, GeoJSON):
-            cube = cube.filter_spatial(spatial_extent)
+        # # Apply if the collection is a GeoJSON Feature collection
+        # if isinstance(spatial_extent, GeoJSON):
+        #     cube = cube.filter_spatial(spatial_extent)
 
         return cube
 
     return generic_default_fetcher
 
 
-def _get_generic_processor(collection_name: str, fetch_type: FetchType) -> Callable:
+def _get_generic_processor(
+    collection_name: str, fetch_type: FetchType, is_stac: bool
+) -> Callable:
     """Builds the preprocessing function from the collection name as it stored
     in the target backend.
     """
+    band_mapping: Optional[dict] = None
     if collection_name == "COPERNICUS_30":
-        BASE_MAPPING = BASE_DEM_MAPPING
+        band_mapping = BASE_DEM_MAPPING
     elif collection_name == "AGERA5":
-        BASE_MAPPING = BASE_WEATHER_MAPPING
-    else:
-        raise Exception("Please choose a valid collection.")
+        band_mapping = BASE_WEATHER_MAPPING
+    elif is_stac and (AGERA5_TERRASCOPE_STAC in collection_name):
+        band_mapping = AGERA5_STAC_MAPPING
 
     def generic_default_processor(cube: openeo.DataCube, **params):
         """Default collection preprocessing method for generic datasets.
@@ -100,51 +131,14 @@ def _get_generic_processor(collection_name: str, fetch_type: FetchType) -> Calla
         if collection_name == "COPERNICUS_30":
             cube = cube.min_time()
 
-        cube = rename_bands(cube, BASE_MAPPING)
+        if band_mapping is not None:
+            cube = rename_bands(cube, band_mapping)
 
         return cube
 
     return generic_default_processor
 
 
-OTHER_BACKEND_MAP = {
-    "AGERA5": {
-        Backend.TERRASCOPE: {
-            "fetch": partial(_get_generic_fetcher, collection_name="AGERA5"),
-            "preprocessor": partial(_get_generic_processor, collection_name="AGERA5"),
-        },
-        Backend.CDSE: {
-            "fetch": partial(_get_generic_fetcher, collection_name="AGERA5"),
-            "preprocessor": partial(_get_generic_processor, collection_name="AGERA5"),
-        },
-        Backend.FED: {
-            "fetch": partial(_get_generic_fetcher, collection_name="AGERA5"),
-            "preprocessor": partial(_get_generic_processor, collection_name="AGERA5"),
-        },
-    },
-    "COPERNICUS_30": {
-        Backend.TERRASCOPE: {
-            "fetch": partial(_get_generic_fetcher, collection_name="COPERNICUS_30"),
-            "preprocessor": partial(
-                _get_generic_processor, collection_name="COPERNICUS_30"
-            ),
-        },
-        Backend.CDSE: {
-            "fetch": partial(_get_generic_fetcher, collection_name="COPERNICUS_30"),
-            "preprocessor": partial(
-                _get_generic_processor, collection_name="COPERNICUS_30"
-            ),
-        },
-        Backend.FED: {
-            "fetch": partial(_get_generic_fetcher, collection_name="COPERNICUS_30"),
-            "preprocessor": partial(
-                _get_generic_processor, collection_name="COPERNICUS_30"
-            ),
-        },
-    },
-}
-
-
 def build_generic_extractor(
     backend_context: BackendContext,
     bands: list,
@@ -152,14 +146,28 @@ def build_generic_extractor(
     collection_name: str,
     **params,
 ) -> CollectionFetcher:
-    """Creates a generic extractor adapted to the given backend. Currently only tested with VITO backend"""
-    backend_functions = OTHER_BACKEND_MAP.get(collection_name).get(
-        backend_context.backend
+    """Creates a generic extractor adapted to the given backend. Provides band mappings for known
+    collections, such as AGERA5 available on Terrascope/FED and Copernicus 30m DEM in all backends.
+    """
+    fetcher = _get_generic_fetcher(
+        collection_name, fetch_type, backend_context.backend, False
     )
+    preprocessor = _get_generic_processor(collection_name, fetch_type, False)
 
-    fetcher, preprocessor = (
-        backend_functions["fetch"](fetch_type=fetch_type),
-        backend_functions["preprocessor"](fetch_type=fetch_type),
+    return CollectionFetcher(backend_context, bands, fetcher, preprocessor, **params)
+
+
+def build_generic_extractor_stac(
+    backend_context: BackendContext,
+    bands: list,
+    fetch_type: FetchType,
+    collection_url: str,
+    **params,
+) -> CollectionFetcher:
+    """Creates a generic extractor adapted to the given backend. Currently only tested with VITO backend"""
+    fetcher = _get_generic_fetcher(
+        collection_url, fetch_type, backend_context.backend, True
     )
+    preprocessor = _get_generic_processor(collection_url, fetch_type, True)
 
     return CollectionFetcher(backend_context, bands, fetcher, preprocessor, **params)

openeo_gfmap/fetching/s1.py

@@ -67,8 +67,6 @@ def _get_s1_grd_default_fetcher(
         """
         bands = convert_band_names(bands, BASE_SENTINEL1_GRD_MAPPING)
 
-        load_collection_parameters = params.get("load_collection", {})
-
         cube = _load_collection(
             connection,
             bands,
@@ -76,7 +74,7 @@ def _get_s1_grd_default_fetcher(
             spatial_extent,
             temporal_extent,
             fetch_type,
-            **load_collection_parameters,
+            **params,
         )
 
         if fetch_type is not FetchType.POINT and isinstance(spatial_extent, GeoJSON):

openeo_gfmap/fetching/s2.py

@@ -113,6 +113,7 @@ def _get_s2_l2a_default_fetcher(
     return s2_l2a_fetch_default
 
 
+# TODO deprecated?
 def _get_s2_l2a_element84_fetcher(
     collection_name: str, fetch_type: FetchType
 ) -> Callable:

openeo_gfmap/inference/model_inference.py

@@ -1,6 +1,7 @@
 """Inference functionalities. Such as a base class to assist the implementation
 of inference models on an UDF.
 """
+
 import functools
 import inspect
 import logging
@@ -74,8 +75,9 @@ class ModelInference(ABC):
 
         return abs_path
 
+    @classmethod
     @functools.lru_cache(maxsize=6)
-    def load_ort_session(self, model_url: str):
+    def load_ort_session(cls, model_url: str):
         """Loads an onnx session from a publicly available URL. The URL must be a direct
         download link to the ONNX session file.
         The `lru_cache` decorator avoids loading multiple time the model within the same worker.
@@ -180,7 +182,7 @@ class ONNXModelInference(ModelInference):
             raise ValueError("The model_url must be defined in the parameters.")
 
         # Load the model and the input_name parameters
-        session = self.load_ort_session(self._parameters.get("model_url"))
+        session = ModelInference.load_ort_session(self._parameters.get("model_url"))
 
         input_name = self._parameters.get("input_name")
         if input_name is None:
@@ -329,6 +331,7 @@ def apply_model_inference_local(
         )
 
     model_inference = model_inference_class()
+    model_inference._parameters = parameters
     output_labels = model_inference.output_labels()
     dependencies = model_inference.dependencies()