openeo-gfmap 0.1.0__py3-none-any.whl

openeo_gfmap/spatial.py

@@ -0,0 +1,53 @@
+""" Definitions of spatial context, either point-based or spatial"""
+
+from dataclasses import dataclass
+from typing import Union
+
+from geojson import GeoJSON
+from shapely.geometry import Polygon, box
+
+
+@dataclass
+class BoundingBoxExtent:
+    """Definition of a bounding box as accepted by OpenEO
+
+    Contains the minx, miny, maxx, maxy coordinates expressed as west, south
+    east, north. The EPSG is also defined.
+    """
+
+    west: float
+    south: float
+    east: float
+    north: float
+    epsg: int = 4326
+
+    def __dict__(self):
+        return {
+            "west": self.west,
+            "south": self.south,
+            "east": self.east,
+            "north": self.north,
+            "crs": f"EPSG:{self.epsg}",
+            "srs": f"EPSG:{self.epsg}",
+        }
+
+    def __iter__(self):
+        return iter(
+            [
+                ("west", self.west),
+                ("south", self.south),
+                ("east", self.east),
+                ("north", self.north),
+                ("crs", f"EPSG:{self.epsg}"),
+                ("srs", f"EPSG:{self.epsg}"),
+            ]
+        )
+
+    def to_geometry(self) -> Polygon:
+        return box(self.west, self.south, self.east, self.north)
+
+    def to_geojson(self) -> GeoJSON:
+        return self.to_geometry().__geo_interface__
+
+
+SpatialContext = Union[GeoJSON, BoundingBoxExtent, str]

openeo_gfmap/stac/__init__.py

@@ -0,0 +1,2 @@
+"""Definitions of the constants in the STAC collection
+"""

openeo_gfmap/stac/constants.py

@@ -0,0 +1,51 @@
+"""
+Constants in the STAC collection generated after a series of batch jobs
+"""
+
+import pystac
+
+LICENSE = "CC-BY-4.0"
+LICENSE_LINK = pystac.Link(
+    rel="license",
+    target="https://spdx.org/licenses/CC-BY-4.0.html",
+    media_type=pystac.MediaType.HTML,
+    title="Creative Commons Attribution 4.0 International License",
+)
+STAC_EXTENSIONS = [
+    "https://stac-extensions.github.io/eo/v1.1.0/schema.json",
+    "https://stac-extensions.github.io/file/v2.1.0/schema.json",
+    "https://stac-extensions.github.io/processing/v1.1.0/schema.json",
+    "https://stac-extensions.github.io/projection/v1.1.0/schema.json",
+]
+CONSTELLATION = {
+    "sentinel2": ["sentinel-2"],
+    "sentinel1": ["sentinel-1"],
+}
+
+PLATFORM = {
+    "sentinel2": ["sentinel-2a", "sentinel-2b"],
+    "sentinel1": ["sentinel-1a", "sentinel-1b"],
+}
+
+INSTRUMENTS = {"sentinel2": ["msi"], "sentinel1": ["c-sar"]}
+
+GSD = {"sentinel2": [10, 20, 60], "sentinel1": [10]}
+
+SUMMARIES = {
+    "sentinel2": pystac.summaries.Summaries(
+        {
+            "constellation": CONSTELLATION["sentinel2"],
+            "platform": PLATFORM["sentinel2"],
+            "instruments": INSTRUMENTS["sentinel2"],
+            "gsd": GSD["sentinel2"],
+        }
+    ),
+    "sentinel1": pystac.summaries.Summaries(
+        {
+            "constellation": CONSTELLATION["sentinel1"],
+            "platform": PLATFORM["sentinel1"],
+            "instruments": INSTRUMENTS["sentinel1"],
+            "gsd": GSD["sentinel1"],
+        }
+    ),
+}

openeo_gfmap/temporal.py

@@ -0,0 +1,22 @@
+""" Definitions of temporal context"""
+
+from dataclasses import dataclass
+from datetime import datetime
+
+
+@dataclass
+class TemporalContext:
+    """Temporal context is defined by a `start_date` and `end_date` values.
+
+    The value must be encoded on a YYYY-mm-dd format, e.g. 2020-01-01
+    """
+
+    start_date: str
+    end_date: str
+
+    def to_datetime(self):
+        """Converts the temporal context to a tuple of datetime objects."""
+        return (
+            datetime.strptime(self.start_date, "%Y-%m-%d"),
+            datetime.strptime(self.end_date, "%Y-%m-%d"),
+        )

openeo_gfmap/utils/__init__.py

@@ -0,0 +1,23 @@
+"""This sub-module contains utilitary function and tools for OpenEO-GFMap"""
+
+from openeo_gfmap.utils.build_df import load_json
+from openeo_gfmap.utils.intervals import quintad_intervals
+from openeo_gfmap.utils.netcdf import update_nc_attributes
+from openeo_gfmap.utils.tile_processing import (
+    array_bounds,
+    arrays_cosine_similarity,
+    normalize_array,
+    select_optical_bands,
+    select_sar_bands,
+)
+
+__all__ = [
+    "load_json",
+    "normalize_array",
+    "select_optical_bands",
+    "array_bounds",
+    "select_sar_bands",
+    "arrays_cosine_similarity",
+    "quintad_intervals",
+    "update_nc_attributes",
+]

openeo_gfmap/utils/build_df.py

@@ -0,0 +1,48 @@
+"""Utilities to build a `pandas.DataFrame` from the output of a VectorCube
+based job. Usefull to collect the output of point based extraction.
+"""
+
+from pathlib import Path
+
+import pandas as pd
+
+VECTORCUBE_TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S%z"
+TIMESTAMP_FORMAT = "%Y-%m-%d"
+
+
+def load_json(input_file: Path, bands: list) -> pd.DataFrame:
+    """Reads a json file and outputs it as a proper pandas dataframe.
+
+    Parameters
+    ----------
+    input_file: PathLike
+        The path of the JSON file to read.
+    bands: list
+        The name of the bands that will be used in the columns names. The band
+        names must be the same as the vector cube that resulted into the parsed
+        JSON file.
+    Returns
+    -------
+    df: pd.DataFrame
+        A `pandas.DataFrame` containing a combination of the band names and the
+        timestamps as column names.
+        For example, the Sentinel-2 green band on the 1st October 2020 is will
+        have the column name `S2-L2A-B02:2020-10-01`
+    """
+
+    df = pd.read_json(input_file)
+
+    target_timestamps = list(
+        map(lambda date: date.strftime(TIMESTAMP_FORMAT), df.columns.to_pydatetime())
+    )
+
+    df = df.rename(dict(zip(df.columns, target_timestamps)), axis=1)
+
+    expanded_df = pd.DataFrame()
+    for col in df.columns:
+        expanded_col = pd.DataFrame(
+            df[col].to_list(), columns=[f"{feature}:{col}" for feature in bands]
+        )
+        expanded_df = pd.concat([expanded_df, expanded_col], axis=1)
+
+    return expanded_df

openeo_gfmap/utils/catalogue.py

@@ -0,0 +1,248 @@
+"""Functionalities to interract with product catalogues."""
+
+import requests
+from geojson import GeoJSON
+from pyproj.crs import CRS
+from rasterio.warp import transform_bounds
+from shapely import unary_union
+from shapely.geometry import box, shape
+
+from openeo_gfmap import (
+    Backend,
+    BackendContext,
+    BoundingBoxExtent,
+    SpatialContext,
+    TemporalContext,
+)
+
+
+class UncoveredS1Exception(Exception):
+    """Exception raised when there is no product available to fully cover spatially a given
+    spatio-temporal context for the Sentinel-1 collection."""
+
+    pass
+
+
+def _parse_cdse_products(response: dict):
+    """Parses the geometry of products from the CDSE catalogue."""
+    geoemetries = []
+    products = response["features"]
+
+    for product in products:
+        geoemetries.append(shape(product["geometry"]))
+    return geoemetries
+
+
+def _query_cdse_catalogue(
+    collection: str,
+    bounds: list,
+    temporal_extent: TemporalContext,
+    **additional_parameters: dict,
+) -> dict:
+    minx, miny, maxx, maxy = bounds
+
+    # The date format should be YYYY-MM-DD
+    start_date = f"{temporal_extent.start_date}T00:00:00Z"
+    end_date = f"{temporal_extent.end_date}T00:00:00Z"
+
+    url = (
+        f"https://catalogue.dataspace.copernicus.eu/resto/api/collections/"
+        f"{collection}/search.json?box={minx},{miny},{maxx},{maxy}"
+        f"&sortParam=startDate&maxRecords=100"
+        f"&dataset=ESA-DATASET&startDate={start_date}&completionDate={end_date}"
+    )
+    for key, value in additional_parameters.items():
+        url += f"&{key}={value}"
+
+    response = requests.get(url)
+
+    if response.status_code != 200:
+        raise Exception(
+            f"Cannot check S1 catalogue on CDSE: Request to {url} failed with "
+            f"status code {response.status_code}"
+        )
+
+    return response.json()
+
+
+def _check_cdse_catalogue(
+    collection: str,
+    bounds: list,
+    temporal_extent: TemporalContext,
+    **additional_parameters: dict,
+) -> bool:
+    """Checks if there is at least one product available in the
+    given spatio-temporal context for a collection in the CDSE catalogue,
+    as there might be issues in the API that sometimes returns empty results
+    for a valid query.
+
+    Parameters
+    ----------
+    collection : str
+        The collection name to be checked. (For example: Sentinel1 or Sentinel2)
+    spatial_extent : SpatialContext
+        The spatial extent to be checked, it will check within its bounding box.
+    temporal_extent : TemporalContext
+        The temporal period to be checked.
+    additional_parameters : Optional[dict], optional
+        Additional parameters to be passed to the catalogue, by default empty.
+        Parameters (key, value) will be passed as "&key=value" in the query,
+        for example: {"sortOrder": "ascending"} will be passed as "&ascendingOrder=True"
+
+    Returns
+    -------
+    True if there is at least one product, False otherwise.
+    """
+    body = _query_cdse_catalogue(
+        collection, bounds, temporal_extent, **additional_parameters
+    )
+
+    grd_tiles = list(
+        filter(
+            lambda feature: feature["properties"]["productType"].contains("GRD"),
+            body["features"],
+        )
+    )
+
+    return len(grd_tiles) > 0
+
+
+def s1_area_per_orbitstate(
+    backend: BackendContext,
+    spatial_extent: SpatialContext,
+    temporal_extent: TemporalContext,
+) -> dict:
+    """Evaluates for both the ascending and descending state orbits the area of interesection
+    between the given spatio-temporal context and the products available in the backend's
+    catalogue.
+
+    Parameters
+    ----------
+    backend : BackendContext
+        The backend to be within, as each backend might use different catalogues.
+    spatial_extent : SpatialContext
+        The spatial extent to be checked, it will check within its bounding box.
+    temporal_extent : TemporalContext
+        The temporal period to be checked.
+
+    Returns
+    ------
+    dict
+        Keys containing the orbit state and values containing the total area of intersection in
+        km^2
+    """
+    if isinstance(spatial_extent, GeoJSON):
+        # Transform geojson into shapely geometry and compute bounds
+        bounds = shape(spatial_extent).bounds
+        epsg = 4362
+    elif isinstance(spatial_extent, BoundingBoxExtent):
+        bounds = [
+            spatial_extent.west,
+            spatial_extent.south,
+            spatial_extent.east,
+            spatial_extent.north,
+        ]
+        epsg = spatial_extent.epsg
+    else:
+        raise ValueError(
+            "Provided spatial extent is not a valid GeoJSON or SpatialContext object."
+        )
+    # Warp the bounds if  the epsg is different from 4326
+    if epsg != 4326:
+        bounds = transform_bounds(CRS.from_epsg(epsg), CRS.from_epsg(4326), *bounds)
+
+    # Queries the products in the catalogues
+    if backend.backend in [Backend.CDSE, Backend.CDSE_STAGING, Backend.FED]:
+        ascending_products = _parse_cdse_products(
+            _query_cdse_catalogue(
+                "Sentinel1", bounds, temporal_extent, orbitDirection="ASCENDING"
+            )
+        )
+        descending_products = _parse_cdse_products(
+            _query_cdse_catalogue(
+                "Sentinel1",
+                bounds,
+                temporal_extent,
+                orbitDirection="DESCENDING",
+            )
+        )
+    else:
+        raise NotImplementedError(
+            f"This feature is not supported for backend: {backend.backend}."
+        )
+
+    # Builds the shape of the spatial extent and computes the area
+    spatial_extent = box(*bounds)
+
+    # Computes if there is the full overlap for each of those states
+    union_ascending = unary_union(ascending_products)
+    union_descending = unary_union(descending_products)
+
+    ascending_covers = union_ascending.contains(spatial_extent)
+    descending_covers = union_descending.contains(spatial_extent)
+
+    # Computes the area of intersection
+    return {
+        "ASCENDING": {
+            "full_overlap": ascending_covers,
+            "area": sum(
+                product.intersection(spatial_extent).area
+                for product in ascending_products
+            ),
+        },
+        "DESCENDING": {
+            "full_overlap": descending_covers,
+            "area": sum(
+                product.intersection(spatial_extent).area
+                for product in descending_products
+            ),
+        },
+    }
+
+
+def select_S1_orbitstate(
+    backend: BackendContext,
+    spatial_extent: SpatialContext,
+    temporal_extent: TemporalContext,
+) -> str:
+    """Selects the orbit state that covers the most area of the given spatio-temporal context
+    for the Sentinel-1 collection.
+
+    Parameters
+    ----------
+    backend : BackendContext
+        The backend to be within, as each backend might use different catalogues.
+    spatial_extent : SpatialContext
+        The spatial extent to be checked, it will check within its bounding box.
+    temporal_extent : TemporalContext
+        The temporal period to be checked.
+
+    Returns
+    ------
+    str
+        The orbit state that covers the most area of the given spatio-temporal context
+    """
+
+    # Queries the products in the catalogues
+    areas = s1_area_per_orbitstate(backend, spatial_extent, temporal_extent)
+
+    ascending_overlap = areas["ASCENDING"]["full_overlap"]
+    descending_overlap = areas["DESCENDING"]["full_overlap"]
+
+    if ascending_overlap and not descending_overlap:
+        return "ASCENDING"
+    elif descending_overlap and not ascending_overlap:
+        return "DESCENDING"
+    elif ascending_overlap and descending_overlap:
+        ascending_cover_area = areas["ASCENDING"]["area"]
+        descending_cover_area = areas["DESCENDING"]["area"]
+
+        # Selects the orbit state that covers the most area
+        if ascending_cover_area > descending_cover_area:
+            return "ASCENDING"
+        else:
+            return "DESCENDING"
+
+    raise UncoveredS1Exception(
+        "No product available to fully cover the given spatio-temporal context."
+    )

openeo_gfmap/utils/intervals.py

@@ -0,0 +1,64 @@
+"""Utilitary function for intervals, useful for temporal aggregation
+methods.
+"""
+
+from datetime import timedelta
+
+from openeo_gfmap import TemporalContext
+
+
+def quintad_intervals(temporal_extent: TemporalContext) -> list:
+    """Returns a list of tuples (start_date, end_date) of quintad intervals
+    from the input temporal extent. Quintad intervals are intervals of
+    generally 5 days, that never overlap two months.
+
+    All months are divided in 6 quintads, where the 6th quintad might
+    contain 6 days for months of 31 days.
+    For the month of February, the 6th quintad is only of three days, or
+    four days for the leap year.
+    """
+    start_date, end_date = temporal_extent.to_datetime()
+    quintads = []
+
+    current_date = start_date
+
+    # Compute the offset of the first day on the start of the last quintad
+    if start_date.day != 1:
+        offset = (start_date - timedelta(days=1)).day % 5
+        current_date = current_date - timedelta(days=offset)
+    else:
+        offset = 0
+
+    while current_date <= end_date:
+        # Get the last day of the current month
+        last_day = current_date.replace(day=28) + timedelta(days=4)
+        last_day = last_day - timedelta(days=last_day.day)
+
+        # Get the last day of the current quintad
+        last_quintad = current_date + timedelta(days=4)
+
+        # Add a day if the day is the 30th and there is the 31th in the current month
+        if last_quintad.day == 30 and last_day.day == 31:
+            last_quintad = last_quintad + timedelta(days=1)
+
+        # If the last quintad is after the last day of the month, then
+        # set it to the last day of the month
+        if last_quintad > last_day:
+            last_quintad = last_day
+        # In the case the last quintad is after the end date, then set it to the end date
+        elif last_quintad > end_date:
+            last_quintad = end_date
+
+        quintads.append((current_date, last_quintad))
+
+        # Set the current date to the next quintad
+        current_date = last_quintad + timedelta(days=1)
+
+    # Fixing the offset issue for intervals starting in the middle of a quintad
+    quintads[0] = (quintads[0][0] + timedelta(days=offset), quintads[0][1])
+
+    # Returns to string with the YYYY-mm-dd format
+    return [
+        (start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d"))
+        for start_date, end_date in quintads
+    ]

openeo_gfmap/utils/netcdf.py

@@ -0,0 +1,25 @@
+"""Utilities to edit and update netCDF files.
+"""
+
+from pathlib import Path
+from typing import Union
+
+from netCDF4 import Dataset
+
+
+def update_nc_attributes(path: Union[str, Path], attributes: dict):
+    """
+    Update attributes of a NetCDF file.
+
+    Parameters:
+        path (str): Path to the NetCDF file.
+        attributes (dict): Dictionary containing attributes to update.
+                                Keys are attribute names, values are attribute values.
+    """
+
+    with Dataset(path, "r+") as nc:
+        for name, value in attributes.items():
+            if name in nc.ncattrs():
+                setattr(nc, name, value)
+            else:
+                nc.setncattr(name, value)

openeo_gfmap/utils/tile_processing.py

@@ -0,0 +1,64 @@
+"""Utilitaries to process data tiles."""
+
+import numpy as np
+import xarray as xr
+
+
+def normalize_array(inarr: xr.DataArray, percentile: float = 0.99) -> xr.DataArray:
+    """Performs normalization between 0.0 and 1.0 using the given
+    percentile.
+    """
+    quantile_value = inarr.quantile(percentile, dim=["x", "y", "t"])
+    minimum = inarr.min(dim=["x", "y", "t"])
+
+    inarr = (inarr - minimum) / (quantile_value - minimum)
+
+    # Perform clipping on values that are higher than the computed quantile
+    return inarr.where(inarr < 1.0, 1.0)
+
+
+def select_optical_bands(inarr: xr.DataArray) -> xr.DataArray:
+    """Filters and keep only the optical bands for a given array."""
+    return inarr.sel(
+        bands=[
+            band
+            for band in inarr.coords["bands"].to_numpy()
+            if band.startswith("S2-L2A-B")
+        ]
+    )
+
+
+def select_sar_bands(inarr: xr.DataArray) -> xr.DataArray:
+    """Filters and keep only the SAR bands for a given array."""
+    return inarr.sel(
+        bands=[
+            band
+            for band in inarr.coords["bands"].to_numpy()
+            if band in ["S1-SIGMA0-VV", "S1-SIGMA0-VH", "S1-SIGMA0-HH", "S1-SIGMA0-HV"]
+        ]
+    )
+
+
+def array_bounds(inarr: xr.DataArray) -> tuple:
+    """Returns the 4 bounds values for the x and y coordinates of the tile"""
+    return (
+        inarr.coords["x"].min().item(),
+        inarr.coords["y"].min().item(),
+        inarr.coords["x"].max().item(),
+        inarr.coords["y"].max().item(),
+    )
+
+
+def arrays_cosine_similarity(
+    first_array: xr.DataArray, second_array: xr.DataArray
+) -> float:
+    """Returns a similarity score based on normalized cosine distance. The
+    input arrays must have similar ranges to obtain a valid score.
+    1.0 represents the best score (same tiles), while 0.0 is the worst score.
+    """
+    dot_product = np.sum(first_array * second_array)
+    first_norm = np.linalg.norm(first_array)
+    second_norm = np.linalg.norm(second_array)
+    similarity = (dot_product / (first_norm * second_norm)).item()
+
+    return similarity

openeo_gfmap-0.1.0.dist-info/METADATA

@@ -0,0 +1,57 @@
+Metadata-Version: 2.3
+Name: openeo_gfmap
+Version: 0.1.0
+Summary: OpenEO General Framework for Mapping
+Project-URL: Homepage, https://github.com/Open-EO/openeo-gfmap
+Project-URL: Bug Tracker, https://github.com/Open-EO/openeo-gfmap/issues
+Author: Darius Couchard, Vincent Verelst, Kristof Van Tricht, Stefaan Lippens, Jeroen Degerickx
+License-File: LICENSE
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Requires-Dist: cftime
+Requires-Dist: fastparquet
+Requires-Dist: geojson>=3.0.0
+Requires-Dist: geopandas
+Requires-Dist: h3
+Requires-Dist: h5netcdf>=1.2.0
+Requires-Dist: netcdf4
+Requires-Dist: numpy<2.0.0
+Requires-Dist: onnxruntime
+Requires-Dist: openeo
+Requires-Dist: pyarrow
+Requires-Dist: rasterio
+Requires-Dist: scipy
+Provides-Extra: dev
+Requires-Dist: matplotlib>=3.3.0; extra == 'dev'
+Requires-Dist: pre-commit; extra == 'dev'
+Requires-Dist: pytest-depends; extra == 'dev'
+Requires-Dist: pytest-timeout>=2.2.0; extra == 'dev'
+Requires-Dist: pytest>=7.4.0; extra == 'dev'
+Requires-Dist: rioxarray>=0.13.0; extra == 'dev'
+Requires-Dist: xarray>=2022.3.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# OpenEO General Framework for Mapping
+
+openEO GFMap aims to simplify for its users the development of mapping applications through Remote Sensing data by leveraging the power of [OpenEO](https://openeo.org/). This framework is developed primarily for Crop Type mapping and Land Cover Classification, but other applications such as regression tasks can be applied.
+
+## How is it used?
+
+In order to be used, the user has to specify which kind of input data it expects (satellite, meteo, DEM, ...) and which mode of classification it expects (point based mapping or polygon based). The user specifies then two <i>user defined files (UDF)</i>, one for extracting features from the pre-processed data and the other for performing classification through a model.
+
+The Frameworks provides assistance in extraction of training data as well as inference phase, and makes sure that both training data and inference data are processed the same way before passing through the model. The user is responsible for the machine learning related details, and for the training phase itself.
+
+<p align="center">
+    <img src="./workflow.png">
+</p>
+
+## Framework core principles
+
+1.	<b>Backend agnostic</b>: The users shouldn’t have to take care of backend related configurations. The use of OpenEO can vary depending on the backend that is currently in use (for example, the name of data collections). The framework takes care of those differences, while the users only specify the backend they desire to use.
+
+2. <b> Data consistent</b>: providing a common pipeline for training and for inference. The best way of making sure data is processed the same way during the construction of a training dataset than during inference, is to re-use as much as code as possible. The users should be able to extract and preprocess training data with the same configuration. OpenEO leaves the possibility to perform data extraction on sparse points/polygons or directly on dense datacubes. This leaves the possibility to implement a framework that could do both tile-based inference and pixel-based or parcel-based data extraction/preprocessing using the same code.
+
+3. <b>Easy and Collaborative</b>: Pre-implementing common preprocessing/postprocessing routines. Many operations, such a compositing or linear interpolation, are very common within Remote Sensing applications and should be already implemented in the framework. This will avoid code duplication among the personal code of the framework’s users and encourage collaboration for improvement and optimization of existing techniques.
+
+4.	<b>Performant</b>: Leverage OpenEO processes as much as possible for preprocessing. In the cropclass and worldcereal projects, preprocessing is performed with a combination of OpenEO processes (masking, compositing, linear interpolation) and the implementation of a Feature Extractor within an UDF (computing indices, percentiles). Ideally, OpenEO features should be used as much as possible, while the features extractor should be as simple as possible, only implementing what is currently not possible within OpenEO.