openeo-gfmap 0.1.0__py3-none-any.whl

openeo_gfmap/preprocessing/cloudmasking.py

@@ -0,0 +1,268 @@
+"""Different cloud masking strategies for an OpenEO datacubes."""
+
+from pathlib import Path
+from typing import Union
+
+import openeo
+from openeo.processes import if_, is_nan
+
+SCL_HARMONIZED_NAME: str = "S2-L2A-SCL"
+BAPSCORE_HARMONIZED_NAME: str = "S2-L2A-BAPSCORE"
+
+
+def mask_scl_dilation(cube: openeo.DataCube, **params: dict) -> openeo.DataCube:
+    """Creates a mask from the SCL, dilates it and applies the mask to the optical
+    bands of the datacube. The other bands such as DEM, SAR and METEO will not
+    be affected by the mask.
+    """
+    # Asserts if the SCL layer exists
+    assert (
+        SCL_HARMONIZED_NAME in cube.metadata.band_names
+    ), f"The SCL band ({SCL_HARMONIZED_NAME}) is not present in the datacube."
+
+    kernel1_size = params.get("kernel1_size", 17)
+    kernel2_size = params.get("kernel2_size", 3)
+    erosion_kernel_size = params.get("erosion_kernel_size", 3)
+
+    # TODO adapt the dilation size given the mask size in meters
+    # TODO check how to get the spatial resolution from the cube metadata
+
+    # Only applies the filtering to the optical part of the cube
+    optical_cube = cube.filter_bands(
+        bands=list(filter(lambda band: band.startswith("S2"), cube.metadata.band_names))
+    )
+
+    nonoptical_cube = cube.filter_bands(
+        bands=list(
+            filter(lambda band: not band.startswith("S2"), cube.metadata.band_names)
+        )
+    )
+
+    optical_cube = optical_cube.process(
+        "mask_scl_dilation",
+        data=optical_cube,
+        scl_band_name=SCL_HARMONIZED_NAME,
+        kernel1_size=kernel1_size,
+        kernel2_size=kernel2_size,
+        mask1_values=[2, 4, 5, 6, 7],
+        mask2_values=[3, 8, 9, 10, 11],
+        erosion_kernel_size=erosion_kernel_size,
+    )
+
+    if len(nonoptical_cube.metadata.band_names) == 0:
+        return optical_cube
+
+    return optical_cube.merge_cubes(nonoptical_cube)
+
+
+def get_bap_score(cube: openeo.DataCube, **params: dict) -> openeo.DataCube:
+    """Calculates the Best Available Pixel (BAP) score for the given datacube,
+    computed from the SCL layer.
+
+    The BAP score is calculated via using a UDF, which gives a lot of
+    flexibility on the calculation methodology. The BAP score is a weighted
+    average of three scores:
+    * Distance-to-Cloud Score: Pixels that are clouds are given score 0.
+      Pixels that are moren than 50 pixels - calculated with the Manhattan
+      distance measure - away from a cloud pixel are given score 1. The pixels
+      in between are given a score versus distance-to-cloud that follows a
+      Gaussian shape.
+    * Coverage Score: Per date, the percentage of all pixels that are classified
+      as a cloud over the entire spatial extent is calculated. The Coverage
+      Score is then equal to 1 - the cloud percentage.
+    * Date Score: In order to favor pixels that are observed in the middle of a
+      month, a date score is calculated, which follows a Gaussian shape. I.e.
+      the largest scores are given for days in the middle of the month, the
+      lowest scores are given for days at the beginning and end of the month.
+
+    The final BAP score is a weighted average of the three aforementioned
+    scores. The weights are 1, 0.5 and 0.8 for the Distance-to-Cloud, Coverage
+    and Date Score respectively.
+
+    Parameters
+    ----------
+    cube : openeo.DataCube
+        The datacube to compute the BAP score from, only the SCL band is used.
+    params : dict
+        Addtional parameters to add to this routine.
+        * `apply_scl_dilation`: Whether to apply dilation to the SCL mask before computing the BAP.
+        * `kernel1_size`: The size of the first kernel used for the dilation of the SCL mask.
+        * `kernel2_size`: The size of the second kernel used for the dilation of the SCL mask.
+        * `erosion_kernel_size`: The size of the kernel used for the erosion of the SCL mask.
+
+    Returns
+    -------
+    openeo.DataCube
+        A 4D datacube containing the BAP score as name 'S2-L2A-BAPSCORE'.
+    """
+    udf_path = Path(__file__).parent / "udf_score.py"
+
+    # Select the SCL band
+    scl_cube = cube.filter_bands([SCL_HARMONIZED_NAME])
+
+    if params.get("apply_scl_dilation", False):
+        kernel1_size = params.get("kernel1_size", 17)
+        kernel2_size = params.get("kernel2_size", 3)
+        erosion_kernel_size = params.get("erosion_kernel_size", 3)
+
+        scl_cube = scl_cube.process(
+            "to_scl_dilation_mask",
+            data=scl_cube,
+            scl_band_name=SCL_HARMONIZED_NAME,
+            kernel1_size=kernel1_size,
+            kernel2_size=kernel2_size,
+            mask1_values=[2, 4, 5, 6, 7],
+            mask2_values=[3, 8, 9, 10, 11],
+            erosion_kernel_size=erosion_kernel_size,
+        )
+
+    # Replace NaN to 0 to avoid issues in the UDF
+    scl_cube = scl_cube.apply(lambda x: if_(is_nan(x), 0, x))
+
+    score = scl_cube.apply_neighborhood(
+        process=openeo.UDF.from_file(str(udf_path)),
+        size=[
+            {"dimension": "x", "unit": "px", "value": 256},
+            {"dimension": "y", "unit": "px", "value": 256},
+        ],
+        overlap=[
+            {"dimension": "x", "unit": "px", "value": 16},
+            {"dimension": "y", "unit": "px", "value": 16},
+        ],
+    )
+
+    score = score.rename_labels("bands", [BAPSCORE_HARMONIZED_NAME])
+
+    # Merge the score to the scl cube
+    return score
+
+
+def get_bap_mask(cube: openeo.DataCube, period: Union[str, list], **params: dict):
+    """Computes the bap score and masks the optical bands of the datacube using
+    the best scores for each pixel on a given time period. This method both
+    performs cloud masking but also a type of compositing.
+
+    The BAP mask is computed using the method `get_bap_score`, from which the
+    maximum argument is taken on every pixel on the given period. This will
+    therefore return an array that for each optical observation, will return
+    if the pixel must be loaded or not, allowing for high cost optimization.
+
+    Parameters
+    ----------
+    cube : openeo.DataCube
+        The datacube to be processed.
+    period : Union[str, list]
+        A string or a list of dates (in YYYY-mm-dd format) to be used as the
+        temporal period to compute the BAP score.
+    params : dict
+        Additionals parameters, not used yet.
+    Returns
+    -------
+    openeo.DataCube
+        The datacube with the BAP mask applied.
+    """
+    # Checks if the S2-L2A-SCL band is present in the datacube
+    assert (
+        SCL_HARMONIZED_NAME in cube.metadata.band_names
+    ), f"The {SCL_HARMONIZED_NAME} band is not present in the datacube."
+
+    bap_score = get_bap_score(cube, **params)
+
+    if isinstance(period, str):
+
+        def max_score_selection(score):
+            max_score = score.max()
+            return score.array_apply(lambda x: x != max_score)
+
+        rank_mask = bap_score.apply_neighborhood(
+            max_score_selection,
+            size=[
+                {"dimension": "x", "unit": "px", "value": 1},
+                {"dimension": "y", "unit": "px", "value": 1},
+                {"dimension": "t", "value": period},
+            ],
+            overlap=[],
+        )
+    elif isinstance(period, list):
+        udf_path = Path(__file__).parent / "udf_rank.py"
+        rank_mask = bap_score.apply_neighborhood(
+            process=openeo.UDF.from_file(str(udf_path), context={"intervals": period}),
+            size=[
+                {"dimension": "x", "unit": "px", "value": 256},
+                {"dimension": "y", "unit": "px", "value": 256},
+            ],
+            overlap=[],
+        )
+    else:
+        raise ValueError(
+            f"'period' must be a string or a list of dates (in YYYY-mm-dd format), got {period}."
+        )
+
+    return rank_mask.rename_labels("bands", ["S2-L2A-BAPMASK"])
+
+
+def bap_masking(cube: openeo.DataCube, period: Union[str, list], **params: dict):
+    """Computes the bap mask as described in `get_bap_mask` and applies it to
+    the optical part of the cube.
+
+    Parameters
+    ----------
+    cube : openeo.DataCube
+        The datacube to be processed.
+    period : Union[str, list]
+        A string or a list of dates (in YYYY-mm-dd format) to be used as the
+        temporal period to compute the BAP score.
+    params : dict
+        Additionals parameters, not used yet.
+    Returns
+    -------
+    openeo.DataCube
+        The datacube with the BAP mask applied.
+    """
+    optical_cube = cube.filter_bands(
+        bands=list(filter(lambda band: band.startswith("S2"), cube.metadata.band_names))
+    )
+
+    nonoptical_cube = cube.filter_bands(
+        bands=list(
+            filter(lambda band: not band.startswith("S2"), cube.metadata.band_names)
+        )
+    )
+
+    rank_mask = get_bap_mask(optical_cube, period, **params)
+
+    optical_cube = optical_cube.mask(rank_mask.resample_cube_spatial(cube))
+
+    # Do not merge if bands are empty!
+    if len(nonoptical_cube.metadata.band_names) == 0:
+        return optical_cube
+
+    return optical_cube.merge_cubes(nonoptical_cube)
+
+
+def cloudmask_percentage(
+    cube: openeo.DataCube, percentage: float = 0.95
+) -> openeo.DataCube:
+    """Compute a cloud mask array, that either fully covers an observation or is empty.
+    It computes the percentage of HIGH_CLOUD_PROBABILITY pixels in the SCL mask. If the percentage
+    is higher than the given threshold, the mask will be covering the observation, otherwise False.
+    """
+    non_scl_cube = cube.filter_bands(
+        bands=list(filter(lambda band: "SCL" not in band, cube.metadata.band_names))
+    )
+
+    scl_cube = cube.filter_bands(["SCL"])
+
+    cloud_mask = scl_cube.apply_neighborhood(
+        process=openeo.UDF.from_file("udf_mask.py", context={}),
+        size=[
+            {"dimension": "x", "unit": "px", "value": 1024},
+            {"dimension": "y", "unit": "px", "value": 1024},
+            {"dimension": "t", "value": 1},
+        ],
+        overlap=[],
+    )
+
+    non_scl_cube = non_scl_cube.mask(cloud_mask.resample_cube_spatial(cube))
+
+    return non_scl_cube.merge_cubes(scl_cube)

openeo_gfmap/preprocessing/compositing.py

@@ -0,0 +1,74 @@
+"""Temporal compositing, or temporal aggregation, is a method to increase the
+quality of data within timesteps by reducing the temporal resolution of a time
+series of satellite images.
+"""
+
+from typing import Union
+
+import openeo
+
+
+def median_compositing(
+    cube: openeo.DataCube, period: Union[str, list]
+) -> openeo.DataCube:
+    """Perfrom median compositing on the given datacube."""
+    if isinstance(period, str):
+        return cube.aggregate_temporal_period(
+            period=period, reducer="median", dimension="t"
+        )
+    elif isinstance(period, list):
+        return cube.aggregate_temporal(
+            intervals=period, reducer="median", dimension="t"
+        )
+
+
+def mean_compositing(
+    cube: openeo.DataCube, period: Union[str, list]
+) -> openeo.DataCube:
+    """Perfrom mean compositing on the given datacube."""
+    if isinstance(period, str):
+        return cube.aggregate_temporal_period(
+            period=period, reducer="mean", dimension="t"
+        )
+    elif isinstance(period, list):
+        return cube.aggregate_temporal(intervals=period, reducer="mean", dimension="t")
+
+
+def sum_compositing(cube: openeo.DataCube, period: Union[str, list]) -> openeo.DataCube:
+    """Perform sum compositing on the given datacube."""
+    if isinstance(period, str):
+        return cube.aggregate_temporal_period(
+            period=period, reducer="sum", dimension="t"
+        )
+    elif isinstance(period, list):
+        return cube.aggregate_temporal(intervals=period, reducer="sum", dimension="t")
+
+
+def max_ndvi_compositing(cube: openeo.DataCube, period: str) -> openeo.DataCube:
+    """Perform compositing by selecting the observation with the highest NDVI value over the
+    given compositing window."""
+
+    def max_ndvi_selection(ndvi: openeo.DataCube):
+        max_ndvi = ndvi.max()
+        return ndvi.array_apply(lambda x: x != max_ndvi)
+
+    if isinstance(period, str):
+        ndvi = cube.ndvi(nir="S2-L2A-B08", red="S2-L2A-B04")
+
+        rank_mask = ndvi.apply_neighborhood(
+            max_ndvi_selection,
+            size=[
+                {"dimension": "x", "unit": "px", "value": 1},
+                {"dimension": "y", "unit": "px", "value": 1},
+                {"dimension": "t", "value": period},
+            ],
+            overlap=[],
+        )
+
+        cube = cube.mask(mask=rank_mask).aggregate_temporal_period(period, "first")
+
+    else:
+        raise ValueError(
+            "Custom temporal intervals are not yet supported for max NDVI compositing."
+        )
+    return cube

openeo_gfmap/preprocessing/interpolation.py

@@ -0,0 +1,12 @@
+"""Utilities to perform interpolation on missing values using the temporal
+dimension.
+"""
+
+import openeo
+
+
+def linear_interpolation(
+    cube: openeo.DataCube,
+) -> openeo.DataCube:
+    """Perform linear interpolation on the given datacube."""
+    return cube.apply_dimension(dimension="t", process="array_interpolate_linear")

openeo_gfmap/preprocessing/sar.py

@@ -0,0 +1,64 @@
+"""Routines to pre-process sar signals."""
+
+import openeo
+from openeo.processes import array_create, if_, is_nodata, power
+
+from openeo_gfmap import Backend, BackendContext
+
+
+def compress_backscatter_uint16(
+    backend_context: BackendContext, cube: openeo.DataCube
+) -> openeo.DataCube:
+    """
+    Scaling the bands from float32 power values to uint16 for memory optimization. The scaling
+    casts the values from power to decibels and applies a linear scaling from 0 to 65534.
+
+    The resulting datacube has a uint16 memory representation which makes an optimization
+    before passing through any UDFs.
+
+    Parameters
+    ----------
+    backend_context : BackendContext
+        The backend context to fetch the backend name.
+    cube : openeo.DataCube
+        The datacube to compress the backscatter values.
+    Returns
+    -------
+    openeo.DataCube
+        The datacube with the backscatter values compressed to uint16.
+    """
+    backend = backend_context.backend
+
+    # Additional check related to problematic values present in creodias collections.
+    # https://github.com/Open-EO/openeo-geopyspark-driver/issues/293
+    if backend in [Backend.CDSE, Backend.CDSE_STAGING, Backend.FED]:
+        cube = cube.apply_dimension(
+            dimension="bands",
+            process=lambda x: array_create(
+                [
+                    if_(
+                        is_nodata(x[0]),
+                        1,
+                        power(base=10, p=(10.0 * x[0].log(base=10) + 83.0) / 20.0),
+                    ),
+                    if_(
+                        is_nodata(x[1]),
+                        1,
+                        power(base=10, p=(10.0 * x[1].log(base=10) + 83.0) / 20.0),
+                    ),
+                ]
+            ),
+        )
+    else:
+        cube = cube.apply_dimension(
+            dimension="bands",
+            process=lambda x: array_create(
+                [
+                    power(base=10, p=(10.0 * x[0].log(base=10) + 83.0) / 20.0),
+                    power(base=10, p=(10.0 * x[1].log(base=10) + 83.0) / 20.0),
+                ]
+            ),
+        )
+
+    # Change the data type to uint16 for optimization purposes
+    return cube.linear_scale_range(1, 65534, 1, 65534)

openeo_gfmap/preprocessing/scaling.py

@@ -0,0 +1,65 @@
+"""Scaling and compressing methods for datacubes."""
+
+import openeo
+
+
+def _compress(
+    cube: openeo.DataCube,
+    min_val: int,
+    max_val: int,
+    alpha: float,
+    beta: float,
+):
+    if (
+        alpha != 1.0 or beta != 0.0
+    ):  # Avoid adding a node in the computing graph if scaling is not necessary
+        cube = (cube * alpha) + beta
+
+    return cube.linear_scale_range(min_val, max_val, min_val, max_val)
+
+
+def compress_uint16(
+    cube: openeo.DataCube, alpha: float = 1.0, beta: float = 0.0
+) -> openeo.DataCube:
+    """Scales the data linearly using the formula `output = (input * a) + b` and compresses values
+    from float32 to uint16 for memory optimization.
+
+    Parameters
+    ----------
+    cube : openeo.DataCube
+        The input datacube to compress, only meteo data should be present.
+    alpha : float, optional (default=1.0)
+        The scaling factor. Values in the input datacube are multiplied by this coefficient.
+    beta : float, optional (default=0.0)
+        The offset. Values in the input datacube are added by this value.
+
+    Returns
+    -------
+    cube : openeo.DataCube
+        The datacube with the data linearly scaled and compressed to uint16 and rescaled frome.
+    """
+    return _compress(cube, 0, 65534, alpha, beta)
+
+
+def compress_uint8(
+    cube: openeo.DataCube, alpha: float = 1.0, beta: float = 0.0
+) -> openeo.DataCube:
+    """
+    Scales the data linearly using the formula `output = (input * a) + b` and compresses values
+    from float32 to uint8 for memory optimization.
+
+    Parameters
+    ----------
+    cube : openeo.DataCube
+        The input datacube to compress, only meteo data should be present.
+    alpha : float, optional (default=1.0)
+        The scaling factor. Values in the input datacube are multiplied by this coefficient.
+    beta : float, optional (default=0.0)
+        The offset. Values in the input datacube are added by this value.
+
+    Returns
+    -------
+    cube : openeo.DataCube
+        The datacube with the data linearly scaled and compressed to uint8 and rescaled frome.
+    """
+    return _compress(cube, 0, 253, alpha, beta)

openeo_gfmap/preprocessing/udf_cldmask.py

@@ -0,0 +1,36 @@
+import numpy as np
+import xarray as xr
+from openeo.udf import XarrayDataCube
+
+
+def apply_datacube(cube: XarrayDataCube, context: dict) -> XarrayDataCube:
+    """
+    Computes a cloud mask covering a full observation or nothing depending on the percentage of
+    high probability cloud pixels. If the amount of before mentioned pixels is higher than 95%,
+    then returns a mask covering the whole observation, otherwise returns an empty mask.
+    """
+    array = cube.get_array().transpose("t", "bands", "y", "x")
+
+    output_array = np.zeros(
+        shape=(array.shape[0], 1, array.shape[2], array.shape[3]), dtype=np.uint8
+    )
+
+    for i in range(array.shape[0]):
+        high_proba_count = ((array[i] == 9) * 1).sum()
+        high_proba_percentage = high_proba_count / (array.shape[2] * array.shape[3])
+
+        if high_proba_percentage > 0.95:
+            output_array[i] = 1
+
+    output_array = xr.DataArray(
+        output_array,
+        dims=["t", "bands", "y", "x"],
+        coords={
+            "t": array.t,
+            "bands": ["mask"],
+            "y": array.y,
+            "x": array.x,
+        },
+    )
+
+    return XarrayDataCube(output_array)

openeo_gfmap/preprocessing/udf_rank.py

@@ -0,0 +1,37 @@
+import numpy as np
+import xarray as xr
+from openeo.udf import XarrayDataCube
+
+
+def apply_datacube(cube: XarrayDataCube, context: dict) -> XarrayDataCube:
+    """For a cube having the BAP score, and a given period of list of intervals,
+    create a binary mask that for each pixel, is True if the BAP score is the
+    best within the given interval. The output has the same dimensions as the
+    input, but only has binary values.
+
+    This UDF do not support yet the implementation of string periods such as
+    "month", "dekad", etc...
+    """
+    # First check if the period is defined in the context
+    intervals = context.get("intervals", None)
+    array = cube.get_array().transpose("t", "bands", "y", "x")
+
+    bap_score = array.sel(bands=["S2-L2A-BAPSCORE"])
+
+    def select_maximum(score: xr.DataArray):
+        max_score = score.max(dim="t")
+        return score == max_score
+
+    if isinstance(intervals, str):
+        raise NotImplementedError(
+            "Period as string is not implemented yet, please provide a list of interval tuples."
+        )
+    elif isinstance(intervals, list):
+        # Convert YYYY-mm-dd to datetime64 objects
+        time_bins = [np.datetime64(interval[0]) for interval in intervals]
+
+        rank_mask = bap_score.groupby_bins("t", bins=time_bins).map(select_maximum)
+    else:
+        raise ValueError("Period is not defined in the UDF. Cannot run it.")
+
+    return XarrayDataCube(rank_mask)

openeo_gfmap/preprocessing/udf_score.py

@@ -0,0 +1,103 @@
+import math
+
+import numpy as np
+import xarray as xr
+from openeo.udf import XarrayDataCube
+from scipy.ndimage import distance_transform_cdt
+from skimage.morphology import binary_erosion, footprints
+
+
+def apply_datacube(cube: XarrayDataCube, context: dict) -> XarrayDataCube:
+    cube_array: xr.DataArray = cube.get_array()
+    cube_array = cube_array.transpose("t", "bands", "y", "x")
+
+    clouds = np.logical_or(
+        np.logical_and(cube_array < 11, cube_array >= 8), cube_array == 3
+    ).isel(bands=0)
+
+    weights = [1, 0.8, 0.5]
+
+    # Calculate the Day Of Year score
+    times = cube_array.t.dt.day.values  # returns day of the month for each date
+    sigma = 5
+    mu = 15
+    score_doy = (
+        1
+        / (sigma * math.sqrt(2 * math.pi))
+        * np.exp(-0.5 * ((times - mu) / sigma) ** 2)
+    )
+    score_doy = np.broadcast_to(
+        score_doy[:, np.newaxis, np.newaxis],
+        [cube_array.sizes["t"], cube_array.sizes["y"], cube_array.sizes["x"]],
+    )
+
+    # Calculate the Distance To Cloud score
+    # Erode
+    # Source: https://github.com/dzanaga/satio-pc/blob/e5fc46c0c14bba77e01dca409cf431e7ef22c077/src/satio_pc/preprocessing/clouds.py#L127
+    e = footprints.disk(3)
+
+    # Define a function to apply binary erosion
+    def erode(image, selem):
+        return ~binary_erosion(image, selem)
+
+    # Use apply_ufunc to apply the erosion operation
+    eroded = xr.apply_ufunc(
+        erode,  # function to apply
+        clouds,  # input DataArray
+        input_core_dims=[["y", "x"]],  # dimensions over which to apply function
+        output_core_dims=[["y", "x"]],  # dimensions of the output
+        vectorize=True,  # vectorize the function over non-core dimensions
+        dask="parallelized",  # enable dask parallelization
+        output_dtypes=[np.int32],  # data type of the output
+        kwargs={"selem": e},  # additional keyword arguments to pass to erode
+    )
+
+    # Distance to cloud = dilation
+    d_min = 0
+    d_req = 50
+    d = xr.apply_ufunc(
+        distance_transform_cdt,
+        eroded,
+        input_core_dims=[["y", "x"]],
+        output_core_dims=[["y", "x"]],
+        vectorize=True,
+        dask="parallelized",
+        output_dtypes=[np.int32],
+    )
+    d = xr.where(d == -1, d_req, d)
+    score_clouds = 1 / (1 + np.exp(-0.2 * (np.minimum(d, d_req) - (d_req - d_min) / 2)))
+
+    # Calculate the Coverage score
+    score_cov = 1 - clouds.sum(dim="x").sum(dim="y") / (
+        cube_array.sizes["x"] * cube_array.sizes["y"]
+    )
+    score_cov = np.broadcast_to(
+        score_cov.values[:, np.newaxis, np.newaxis],
+        [cube_array.sizes["t"], cube_array.sizes["y"], cube_array.sizes["x"]],
+    )
+
+    # Final score is weighted average
+    score = (
+        weights[0] * score_clouds + weights[1] * score_doy + weights[2] * score_cov
+    ) / sum(weights)
+    score = np.where(cube_array.values[:, 0, :, :] == 0, 0, score)
+
+    score_da = xr.DataArray(
+        score,
+        coords={
+            "t": cube_array.coords["t"],
+            "y": cube_array.coords["y"],
+            "x": cube_array.coords["x"],
+        },
+        dims=["t", "y", "x"],
+    )
+
+    score_da = score_da.expand_dims(
+        dim={
+            "bands": cube_array.coords["bands"],
+        },
+    )
+
+    score_da = score_da.transpose("t", "bands", "y", "x")
+
+    return XarrayDataCube(score_da)