openeo-gfmap 0.1.0__py3-none-any.whl

openeo_gfmap/fetching/commons.py

@@ -0,0 +1,213 @@
+"""
+Common internal operations within collection extraction logic, such as reprojection.
+"""
+
+from functools import partial
+from typing import Dict, Optional, Sequence, Union
+
+import openeo
+from geojson import GeoJSON
+from openeo.api.process import Parameter
+from openeo.rest.connection import InputDate
+from pyproj.crs import CRS
+from pyproj.exceptions import CRSError
+
+from openeo_gfmap.spatial import BoundingBoxExtent, SpatialContext
+from openeo_gfmap.temporal import TemporalContext
+
+from .fetching import FetchType
+
+
+def convert_band_names(desired_bands: list, band_dict: dict) -> list:
+    """Renames the desired bands to the band names of the collection specified
+    in the backend.
+
+    Parameters
+    ----------
+    desired_bands: list
+        List of bands that are desired by the user, written in the OpenEO-GFMAP
+        harmonized names convention.
+    band_dict: dict
+        Dictionnary mapping for a backend the collection band names to the
+        OpenEO-GFMAP harmonized names. This dictionnary will be reversed to be
+        used within this function.
+    Returns
+    -------
+    backend_band_list: list
+        List of band names within the backend collection names.
+    """
+    # Reverse the dictionarry
+    band_dict = {v: k for k, v in band_dict.items()}
+    return [band_dict[band] for band in desired_bands]
+
+
+def resample_reproject(
+    datacube: openeo.DataCube,
+    resolution: float,
+    epsg_code: Optional[Union[str, int]] = None,
+    method: str = "near",
+) -> openeo.DataCube:
+    """Reprojects the given datacube to the target epsg code, if the provided
+    epsg code is not None. Also performs checks on the give code to check
+    its validity.
+    """
+    if epsg_code is not None:
+        # Checks that the code is valid
+        try:
+            CRS.from_epsg(int(epsg_code))
+        except (CRSError, ValueError) as exc:
+            raise ValueError(
+                f"Specified target_crs: {epsg_code} is not a valid " "EPSG code."
+            ) from exc
+        return datacube.resample_spatial(
+            resolution=resolution, projection=epsg_code, method=method
+        )
+    return datacube.resample_spatial(resolution=resolution, method=method)
+
+
+def rename_bands(datacube: openeo.DataCube, mapping: dict) -> openeo.DataCube:
+    """Rename the bands from the given mapping scheme"""
+
+    # Filter out bands that are not part of the datacube
+    def filter_condition(band_name, _):
+        return band_name in datacube.metadata.band_names
+
+    mapping = {k: v for k, v in mapping.items() if filter_condition(k, v)}
+
+    return datacube.rename_labels(
+        dimension="bands", target=list(mapping.values()), source=list(mapping.keys())
+    )
+
+
+def _load_collection_hybrid(
+    connection: openeo.Connection,
+    is_stac: bool,
+    collection_id_or_url: str,
+    bands: list,
+    spatial_extent: Union[Dict[str, float], Parameter, None] = None,
+    temporal_extent: Union[Sequence[InputDate], Parameter, str, None] = None,
+    properties: Optional[dict] = None,
+):
+    """Wrapper around the load_collection, or load_stac method of the openeo connection."""
+    if not is_stac:
+        return connection.load_collection(
+            collection_id=collection_id_or_url,
+            spatial_extent=spatial_extent,
+            temporal_extent=temporal_extent,
+            bands=bands,
+            properties=properties,
+        )
+    cube = connection.load_stac(
+        url=collection_id_or_url,
+        spatial_extent=spatial_extent,
+        temporal_extent=temporal_extent,
+        bands=bands,
+        properties=properties,
+    )
+    cube = cube.rename_labels(dimension="bands", target=bands)
+    return cube
+
+
+def _load_collection(
+    connection: openeo.Connection,
+    bands: list,
+    collection_name: str,
+    spatial_extent: SpatialContext,
+    temporal_extent: Optional[TemporalContext],
+    fetch_type: FetchType,
+    is_stac: bool = False,
+    **params,
+):
+    """Loads a collection from the openeo backend, acting differently depending
+    on the fetch type.
+    """
+    load_collection_parameters = params.get("load_collection", {})
+    load_collection_method = partial(
+        _load_collection_hybrid, is_stac=is_stac, collection_id_or_url=collection_name
+    )
+
+    if (
+        temporal_extent is not None
+    ):  # Can be ignored for intemporal collections such as DEM
+        temporal_extent = [temporal_extent.start_date, temporal_extent.end_date]
+
+    if fetch_type == FetchType.TILE:
+        assert isinstance(
+            spatial_extent, BoundingBoxExtent
+        ), "Please provide only a bounding box for tile based fetching."
+        spatial_extent = dict(spatial_extent)
+        cube = load_collection_method(
+            connection=connection,
+            bands=bands,
+            spatial_extent=spatial_extent,
+            temporal_extent=temporal_extent,
+            properties=load_collection_parameters,
+        )
+    elif fetch_type == FetchType.POINT:
+        assert isinstance(
+            spatial_extent, GeoJSON
+        ), "Please provide only a GeoJSON FeatureCollection for point based fetching."
+        assert (
+            spatial_extent["type"] == "FeatureCollection"
+        ), "Please provide a FeatureCollection type of GeoJSON"
+        cube = load_collection_method(
+            connection=connection,
+            bands=bands,
+            spatial_extent=spatial_extent,
+            temporal_extent=temporal_extent,
+            properties=load_collection_parameters,
+        )
+    elif fetch_type == FetchType.POLYGON:
+        if isinstance(spatial_extent, GeoJSON):
+            assert (
+                spatial_extent["type"] == "FeatureCollection"
+            ), "Please provide a FeatureCollection type of GeoJSON"
+        elif isinstance(spatial_extent, str):
+            assert spatial_extent.startswith("https://") or spatial_extent.startswith(
+                "http://"
+            ), "Please provide a valid URL or a path to a GeoJSON file."
+        else:
+            raise ValueError(
+                "Please provide a valid URL to a GeoParquet or GeoJSON file."
+            )
+        cube = load_collection_method(
+            connection=connection,
+            bands=bands,
+            temporal_extent=temporal_extent,
+            properties=load_collection_parameters,
+        )
+
+    # Adding the process graph updates for experimental features
+    if params.get("update_arguments") is not None:
+        cube.result_node().update_arguments(**params["update_arguments"])
+
+    # Peforming pre-mask optimization
+    pre_mask = params.get("pre_mask", None)
+    if pre_mask is not None:
+        assert isinstance(pre_mask, openeo.DataCube), (
+            f"The 'pre_mask' parameter must be an openeo datacube, " f"got {pre_mask}."
+        )
+        cube = cube.mask(pre_mask)
+
+    # Merges additional bands continuing the operations.
+    pre_merge_cube = params.get("pre_merge", None)
+    if pre_merge_cube is not None:
+        assert isinstance(pre_merge_cube, openeo.DataCube), (
+            f"The 'pre_merge' parameter value must be an openeo datacube, "
+            f"got {pre_merge_cube}."
+        )
+        if pre_mask is not None:
+            pre_merge_cube = pre_merge_cube.mask(pre_mask)
+        cube = cube.merge_cubes(pre_merge_cube)
+
+    if fetch_type == FetchType.POLYGON:
+        if isinstance(spatial_extent, str):
+            geometry = connection.load_url(
+                spatial_extent,
+                format="Parquet" if ".parquet" in spatial_extent else "GeoJSON",
+            )
+            cube = cube.filter_spatial(geometry)
+        else:
+            cube = cube.filter_spatial(spatial_extent)
+
+    return cube

openeo_gfmap/fetching/fetching.py

@@ -0,0 +1,98 @@
+""" Main file for extractions and pre-processing of data through OpenEO
+"""
+
+from enum import Enum
+from typing import Callable
+
+import openeo
+
+from openeo_gfmap import BackendContext
+from openeo_gfmap.spatial import SpatialContext
+from openeo_gfmap.temporal import TemporalContext
+
+
+class FetchType(Enum):
+    """Enumerates the different types of extraction. There are three types of
+    enumerations.
+
+    * TILE: Tile based extractions, getting the data for a dense part. The
+    output of such fetching process in a dense DataCube.
+    * POINT: Point based extractions. From a datasets of polygons, gets sparse
+    extractions and performs spatial aggregation on the selected polygons. The
+    output of such fetching process is a VectorCube, that can be used to get
+    a pandas.DataFrame
+    * POLYGON: Patch based extractions, returning a VectorCube of sparsed
+    patches. This can be retrieved as multiple NetCDF files from one job.
+    """
+
+    TILE = "tile"
+    POINT = "point"
+    POLYGON = "polygon"
+
+
+class CollectionFetcher:
+    """Base class to fetch a particular collection.
+
+    Parameters
+    ----------
+    backend_context: BackendContext
+        Information about the backend in use, useful in certain cases.
+    bands: list
+        List of band names to load from that collection.
+    collection_fetch: Callable
+        Function defining how to fetch a collection for a specific backend,
+        the function accepts the following parameters: connection,
+        spatial extent, temporal extent, bands and additional parameters.
+    collection_preprocessing: Callable
+        Function defining how to harmonize the data of a collection in a
+        backend. For example, this function could rename the bands as they
+        can be different for every backend/collection (SENTINEL2_L2A or
+        SENTINEL2_L2A_SENTINELHUB). Accepts the following parameters:
+        datacube (of pre-fetched collection) and additional parameters.
+    colection_params: dict
+        Additional parameters encoded within a dictionnary that will be
+        passed in the fetch and preprocessing function.
+    """
+
+    def __init__(
+        self,
+        backend_context: BackendContext,
+        bands: list,
+        collection_fetch: Callable,
+        collection_preprocessing: Callable,
+        **collection_params,
+    ):
+        self.backend_contect = backend_context
+        self.bands = bands
+        self.fetcher = collection_fetch
+        self.processing = collection_preprocessing
+        self.params = collection_params
+
+    def get_cube(
+        self,
+        connection: openeo.Connection,
+        spatial_context: SpatialContext,
+        temporal_context: TemporalContext,
+    ) -> openeo.DataCube:
+        """Retrieve a data cube from the given spatial and temporal context.
+
+        Parameters
+        ----------
+        connection: openeo.Connection
+            A connection to an OpenEO backend. The backend provided must be the
+            same as the one this extractor class is configured for.
+        spatial_extent: SpatialContext
+            Either a GeoJSON collection on which spatial filtering will be
+            applied or a bounding box with an EPSG code. If a bounding box is
+            provided, no filtering is applied and the entirety of the data is
+            fetched for that region.
+        temporal_extent: TemporalContext
+            The begin and end date of the extraction.
+        """
+        collection_data = self.fetcher(
+            connection, spatial_context, temporal_context, self.bands, **self.params
+        )
+
+        preprocessed_data = self.processing(collection_data, **self.params)
+
+        return preprocessed_data

openeo_gfmap/fetching/generic.py

@@ -0,0 +1,165 @@
+""" Generic extraction of features, supporting VITO backend.
+"""
+
+from functools import partial
+from typing import Callable
+
+import openeo
+from geojson import GeoJSON
+
+from openeo_gfmap.backend import Backend, BackendContext
+from openeo_gfmap.fetching import CollectionFetcher, FetchType, _log
+from openeo_gfmap.fetching.commons import (
+    _load_collection,
+    convert_band_names,
+    rename_bands,
+    resample_reproject,
+)
+from openeo_gfmap.spatial import SpatialContext
+from openeo_gfmap.temporal import TemporalContext
+
+BASE_DEM_MAPPING = {"DEM": "COP-DEM"}
+BASE_WEATHER_MAPPING = {
+    "dewpoint-temperature": "AGERA5-DEWTEMP",
+    "precipitation-flux": "AGERA5-PRECIP",
+    "solar-radiation-flux": "AGERA5-SOLRAD",
+    "temperature-max": "AGERA5-TMAX",
+    "temperature-mean": "AGERA5-TMEAN",
+    "temperature-min": "AGERA5-TMIN",
+    "vapour-pressure": "AGERA5-VAPOUR",
+    "wind-speed": "AGERA5-WIND",
+}
+
+
+def _get_generic_fetcher(collection_name: str, fetch_type: FetchType) -> Callable:
+    if collection_name == "COPERNICUS_30":
+        BASE_MAPPING = BASE_DEM_MAPPING
+    elif collection_name == "AGERA5":
+        BASE_MAPPING = BASE_WEATHER_MAPPING
+    else:
+        raise Exception("Please choose a valid collection.")
+
+    def generic_default_fetcher(
+        connection: openeo.Connection,
+        spatial_extent: SpatialContext,
+        temporal_extent: TemporalContext,
+        bands: list,
+        **params,
+    ) -> openeo.DataCube:
+        bands = convert_band_names(bands, BASE_MAPPING)
+
+        if (collection_name == "COPERNICUS_30") and (temporal_extent is not None):
+            _log.warning(
+                "User set-up non None temporal extent for DEM collection. Ignoring it."
+            )
+            temporal_extent = None
+
+        cube = _load_collection(
+            connection,
+            bands,
+            collection_name,
+            spatial_extent,
+            temporal_extent,
+            fetch_type,
+            **params,
+        )
+
+        # Apply if the collection is a GeoJSON Feature collection
+        if isinstance(spatial_extent, GeoJSON):
+            cube = cube.filter_spatial(spatial_extent)
+
+        return cube
+
+    return generic_default_fetcher
+
+
+def _get_generic_processor(collection_name: str, fetch_type: FetchType) -> Callable:
+    """Builds the preprocessing function from the collection name as it stored
+    in the target backend.
+    """
+    if collection_name == "COPERNICUS_30":
+        BASE_MAPPING = BASE_DEM_MAPPING
+    elif collection_name == "AGERA5":
+        BASE_MAPPING = BASE_WEATHER_MAPPING
+    else:
+        raise Exception("Please choose a valid collection.")
+
+    def generic_default_processor(cube: openeo.DataCube, **params):
+        """Default collection preprocessing method for generic datasets.
+        This method renames bands and removes the time dimension in case the
+        requested dataset is DEM
+        """
+        if params.get("target_resolution", None) is not None:
+            cube = resample_reproject(
+                cube,
+                params.get("target_resolution", 10.0),
+                params.get("target_crs", None),
+                method=params.get("resampling_method", "near"),
+            )
+
+        if collection_name == "COPERNICUS_30":
+            cube = cube.min_time()
+
+        cube = rename_bands(cube, BASE_MAPPING)
+
+        return cube
+
+    return generic_default_processor
+
+
+OTHER_BACKEND_MAP = {
+    "AGERA5": {
+        Backend.TERRASCOPE: {
+            "fetch": partial(_get_generic_fetcher, collection_name="AGERA5"),
+            "preprocessor": partial(_get_generic_processor, collection_name="AGERA5"),
+        },
+        Backend.CDSE: {
+            "fetch": partial(_get_generic_fetcher, collection_name="AGERA5"),
+            "preprocessor": partial(_get_generic_processor, collection_name="AGERA5"),
+        },
+        Backend.FED: {
+            "fetch": partial(_get_generic_fetcher, collection_name="AGERA5"),
+            "preprocessor": partial(_get_generic_processor, collection_name="AGERA5"),
+        },
+    },
+    "COPERNICUS_30": {
+        Backend.TERRASCOPE: {
+            "fetch": partial(_get_generic_fetcher, collection_name="COPERNICUS_30"),
+            "preprocessor": partial(
+                _get_generic_processor, collection_name="COPERNICUS_30"
+            ),
+        },
+        Backend.CDSE: {
+            "fetch": partial(_get_generic_fetcher, collection_name="COPERNICUS_30"),
+            "preprocessor": partial(
+                _get_generic_processor, collection_name="COPERNICUS_30"
+            ),
+        },
+        Backend.FED: {
+            "fetch": partial(_get_generic_fetcher, collection_name="COPERNICUS_30"),
+            "preprocessor": partial(
+                _get_generic_processor, collection_name="COPERNICUS_30"
+            ),
+        },
+    },
+}
+
+
+def build_generic_extractor(
+    backend_context: BackendContext,
+    bands: list,
+    fetch_type: FetchType,
+    collection_name: str,
+    **params,
+) -> CollectionFetcher:
+    """Creates a generic extractor adapted to the given backend. Currently only tested with VITO backend"""
+    backend_functions = OTHER_BACKEND_MAP.get(collection_name).get(
+        backend_context.backend
+    )
+
+    fetcher, preprocessor = (
+        backend_functions["fetch"](fetch_type=fetch_type),
+        backend_functions["preprocessor"](fetch_type=fetch_type),
+    )
+
+    return CollectionFetcher(backend_context, bands, fetcher, preprocessor, **params)

openeo_gfmap/fetching/meteo.py

@@ -0,0 +1,126 @@
+"""Meteo data fetchers."""
+
+from functools import partial
+
+import openeo
+from geojson import GeoJSON
+
+from openeo_gfmap import (
+    Backend,
+    BackendContext,
+    FetchType,
+    SpatialContext,
+    TemporalContext,
+)
+from openeo_gfmap.fetching import CollectionFetcher
+from openeo_gfmap.fetching.commons import convert_band_names
+from openeo_gfmap.fetching.generic import (
+    _get_generic_fetcher,
+    _get_generic_processor,
+    _load_collection,
+)
+
+WEATHER_MAPPING_TERRASCOPE = {
+    "dewpoint-temperature": "AGERA5-DEWTEMP",
+    "precipitation-flux": "AGERA5-PRECIP",
+    "solar-radiation-flux": "AGERA5-SOLRAD",
+    "temperature-max": "AGERA5-TMAX",
+    "temperature-mean": "AGERA5-TMEAN",
+    "temperature-min": "AGERA5-TMIN",
+    "vapour-pressure": "AGERA5-VAPOUR",
+    "wind-speed": "AGERA5-WIND",
+}
+
+WEATHER_MAPPING_STAC = {
+    "dewpoint_temperature_mean": "AGERA5-DEWTEMP",
+    "total_precipitation": "AGERA5-PRECIP",
+    "solar_radiataion_flux": "AGERA5-SOLRAD",
+    "2m_temperature_max": "AGERA5-TMAX",
+    "2m_temperature_mean": "AGERA5-TMEAN",
+    "2m_temperature_min": "AGERA5-TMIN",
+    "vapour_pressure": "AGERA5-VAPOUR",
+    "wind_speed": "AGERA5-WIND",
+}
+
+
+def stac_fetcher(
+    connection: openeo.Connection,
+    spatial_extent: SpatialContext,
+    temporal_extent: TemporalContext,
+    bands: list,
+    fetch_type: FetchType,
+    **params,
+) -> openeo.DataCube:
+    bands = convert_band_names(bands, WEATHER_MAPPING_STAC)
+
+    cube = _load_collection(
+        connection,
+        bands,
+        "https://stac.openeo.vito.be/collections/agera5_daily",
+        spatial_extent,
+        temporal_extent,
+        fetch_type,
+        is_stac=True,
+        **params,
+    )
+
+    if isinstance(spatial_extent, GeoJSON) and fetch_type == FetchType.POLYGON:
+        cube = cube.filter_spatial(spatial_extent)
+
+    return cube
+
+
+METEO_BACKEND_MAP = {
+    Backend.TERRASCOPE: {
+        "fetch": partial(
+            _get_generic_fetcher,
+            collection_name="AGERA5",
+            band_mapping=WEATHER_MAPPING_TERRASCOPE,
+        ),
+        "preprocessor": partial(
+            _get_generic_processor,
+            collection_name="AGERA5",
+            band_mapping=WEATHER_MAPPING_TERRASCOPE,
+        ),
+    },
+    Backend.CDSE: {
+        "fetch": stac_fetcher,
+        "preprocessor": partial(
+            _get_generic_processor,
+            collection_name="AGERA5",
+            band_mapping=WEATHER_MAPPING_STAC,
+        ),
+    },
+    Backend.CDSE_STAGING: {
+        "fetch": stac_fetcher,
+        "preprocessor": partial(
+            _get_generic_processor,
+            collection_name="AGERA5",
+            band_mapping=WEATHER_MAPPING_STAC,
+        ),
+    },
+    Backend.FED: {
+        "fetch": stac_fetcher,
+        "preprocessor": partial(
+            _get_generic_processor,
+            collection_name="AGERA5",
+            band_mapping=WEATHER_MAPPING_STAC,
+        ),
+    },
+}
+
+
+def build_meteo_extractor(
+    backend_context: BackendContext,
+    bands: list,
+    fetch_type: FetchType,
+    **params,
+) -> CollectionFetcher:
+    backend_functions = METEO_BACKEND_MAP.get(backend_context.backend)
+
+    fetcher, preprocessor = (
+        partial(backend_functions["fetch"], fetch_type=fetch_type),
+        backend_functions["preprocessor"](fetch_type=fetch_type),
+    )
+
+    return CollectionFetcher(backend_context, bands, fetcher, preprocessor, **params)