openeo-gfmap 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

openeo_gfmap/manager/job_manager.py

@@ -17,9 +17,6 @@ from pystac import CatalogType
 from openeo_gfmap.manager import _log
 from openeo_gfmap.stac import constants
 
-# Lock to use when writing to the STAC collection
-_stac_lock = Lock()
-
 
 def retry_on_exception(max_retries: int, delay_s: int = 180):
     """Decorator to retry a function if an exception occurs.
@@ -132,6 +129,7 @@ class GFMAPJobManager(MultiBackendJobManager):
         self._catalogue_cache = output_dir / "catalogue_cache.bin"
 
         self.stac = stac
+        self.lock = Lock()
         self.stac_enabled = stac_enabled
         self.collection_id = collection_id
         self.collection_description = collection_description
@@ -258,7 +256,7 @@ class GFMAPJobManager(MultiBackendJobManager):
                     "Resuming postprocessing of job %s, queueing on_job_finished...",
                     row.id,
                 )
-                future = self._executor.submit(self.on_job_done, job, row, _stac_lock)
+                future = self._executor.submit(self.on_job_done, job, row)
                 future.add_done_callback(
                     partial(
                         done_callback,
@@ -327,7 +325,7 @@ class GFMAPJobManager(MultiBackendJobManager):
                     "Job %s finished successfully, queueing on_job_done...", job.job_id
                 )
                 job_status = "postprocessing"
-                future = self._executor.submit(self.on_job_done, job, row, _stac_lock)
+                future = self._executor.submit(self.on_job_done, job, row)
                 # Future will setup the status to finished when the job is done
                 future.add_done_callback(
                     partial(
@@ -416,37 +414,40 @@ class GFMAPJobManager(MultiBackendJobManager):
             )
 
     @retry_on_exception(max_retries=2, delay_s=30)
-    def on_job_done(
-        self, job: BatchJob, row: pd.Series, lock: Lock
-    ):  # pylint: disable=arguments-differ
+    def on_job_done(self, job: BatchJob, row: pd.Series):
         """Method called when a job finishes successfully. It will first download the results of
         the job and then call the `post_job_action` method.
         """
 
         job_products = {}
-        for idx, asset in enumerate(job.get_results().get_assets()):
+        job_results = job.get_results()
+        asset_ids = [a.name for a in job_results.get_assets()]
+        for idx, asset_id in enumerate(asset_ids):
             try:
+                asset = job_results.get_asset(asset_id)
                 _log.debug(
                     "Generating output path for asset %s from job %s...",
-                    asset.name,
+                    asset_id,
                     job.job_id,
                 )
-                output_path = self._output_path_gen(self._output_dir, idx, row)
+                output_path = self._output_path_gen(
+                    self._output_dir, idx, row, asset_id
+                )
                 # Make the output path
                 output_path.parent.mkdir(parents=True, exist_ok=True)
                 asset.download(output_path)
                 # Add to the list of downloaded products
-                job_products[f"{job.job_id}_{asset.name}"] = [output_path]
+                job_products[f"{job.job_id}_{asset_id}"] = [output_path]
                 _log.debug(
                     "Downloaded %s from job %s -> %s",
-                    asset.name,
+                    asset_id,
                     job.job_id,
                     output_path,
                 )
             except Exception as e:
                 _log.exception(
                     "Error downloading asset %s from job %s:\n%s",
-                    asset.name,
+                    asset_id,
                     job.job_id,
                     e,
                 )
@@ -491,7 +492,7 @@ class GFMAPJobManager(MultiBackendJobManager):
         _log.info("Adding %s items to the STAC collection...", len(job_items))
 
         if self.stac_enabled:
-            with lock:
+            with self.lock:
                 self._update_stac(job.job_id, job_items)
 
         _log.info("Job %s and post job action finished successfully.", job.job_id)

openeo_gfmap/manager/job_splitters.py

@@ -8,6 +8,7 @@ from typing import List
 import geopandas as gpd
 import h3
 import requests
+import s2sphere
 
 from openeo_gfmap.manager import _log
 
@@ -16,11 +17,11 @@ def load_s2_grid(web_mercator: bool = False) -> gpd.GeoDataFrame:
     """Returns a geo data frame from the S2 grid."""
     # Builds the path where the geodataframe should be
     if not web_mercator:
-        gdf_path = Path.home() / ".openeo-gfmap" / "s2grid_bounds_4326_v2.geoparquet"
-        url = "https://artifactory.vgt.vito.be/artifactory/auxdata-public/gfmap/s2grid_bounds_4326_v2.geoparquet"
+        gdf_path = Path.home() / ".openeo-gfmap" / "s2grid_voronoi_4326.parquet"
+        url = "https://artifactory.vgt.vito.be/artifactory/auxdata-public/gfmap/s2grid_voronoi_4326.parquet"
     else:
-        gdf_path = Path.home() / ".openeo-gfmap" / "s2grid_bounds_3857_v2.geoparquet"
-        url = "https://artifactory.vgt.vito.be/artifactory/auxdata-public/gfmap/s2grid_bounds_3857_v2.geoparquet"
+        gdf_path = Path.home() / ".openeo-gfmap" / "s2grid_voronoi_3857.parquet"
+        url = "https://artifactory.vgt.vito.be/artifactory/auxdata-public/gfmap/s2grid_voronoi_3857.parquet"
 
     if not gdf_path.exists():
         _log.info("S2 grid not found, downloading it from artifactory.")
@@ -40,6 +41,38 @@ def load_s2_grid(web_mercator: bool = False) -> gpd.GeoDataFrame:
     return gpd.read_parquet(gdf_path)
 
 
+def load_s2_grid_centroids(web_mercator: bool = False) -> gpd.GeoDataFrame:
+    """Returns a geo data frame from the S2 grid centroids."""
+    # Builds the path where the geodataframe should be
+    if not web_mercator:
+        gdf_path = (
+            Path.home() / ".openeo-gfmap" / "s2grid_bounds_4326_centroids.geoparquet"
+        )
+        url = "https://artifactory.vgt.vito.be/artifactory/auxdata-public/gfmap/s2grid_bounds_4326_centroids.geoparquet"
+    else:
+        gdf_path = (
+            Path.home() / ".openeo-gfmap" / "s2grid_bounds_3857_centroids.geoparquet"
+        )
+        url = "https://artifactory.vgt.vito.be/artifactory/auxdata-public/gfmap/s2grid_bounds_3857_centroids.geoparquet"
+
+    if not gdf_path.exists():
+        _log.info("S2 grid centroids not found, downloading it from artifactory.")
+        # Downloads the file from the artifactory URL
+        gdf_path.parent.mkdir(exist_ok=True)
+        response = requests.get(
+            url,
+            timeout=180,  # 3mins
+        )
+        if response.status_code != 200:
+            raise ValueError(
+                "Failed to download the S2 grid centroids from the artifactory. "
+                f"Status code: {response.status_code}"
+            )
+        with open(gdf_path, "wb") as f:
+            f.write(response.content)
+    return gpd.read_parquet(gdf_path)
+
+
 def _resplit_group(
     polygons: gpd.GeoDataFrame, max_points: int
 ) -> List[gpd.GeoDataFrame]:
@@ -79,14 +112,12 @@ def split_job_s2grid(
 
     polygons["centroid"] = polygons.geometry.centroid
 
-    # Dataset containing all the S2 tiles, find the nearest S2 tile for each point
     s2_grid = load_s2_grid(web_mercator)
-    s2_grid["geometry"] = s2_grid.geometry.centroid
-
-    s2_grid = s2_grid[s2_grid.cdse_valid]
 
-    polygons = gpd.sjoin_nearest(
-        polygons.set_geometry("centroid"), s2_grid[["tile", "geometry"]]
+    polygons = gpd.sjoin(
+        polygons.set_geometry("centroid"),
+        s2_grid[["tile", "geometry"]],
+        predicate="intersects",
     ).drop(columns=["index_right", "centroid"])
 
     polygons = polygons.set_geometry("geometry").to_crs(original_crs)
@@ -113,7 +144,7 @@ def append_h3_index(
     geom_col = geom_col.to_crs(epsg=4326)
 
     polygons["h3index"] = geom_col.apply(
-        lambda pt: h3.geo_to_h3(pt.y, pt.x, grid_resolution)
+        lambda pt: h3.latlng_to_cell(pt.y, pt.x, grid_resolution)
     )
     return polygons
 
@@ -165,3 +196,96 @@ def split_job_hex(
             split_datasets.append(sub_gdf.reset_index(drop=True))
 
     return split_datasets
+
+
+def split_job_s2sphere(
+    gdf: gpd.GeoDataFrame, max_points=500, start_level=8
+) -> List[gpd.GeoDataFrame]:
+    """
+    EXPERIMENTAL
+    Split a GeoDataFrame into multiple groups based on the S2geometry cell ID of each geometry.
+
+    S2geometry is a library that provides a way to index and query spatial data. This function splits
+    the GeoDataFrame into groups based on the S2 cell ID of each geometry, based on it's centroid.
+
+    If a cell contains more points than max_points, it will be recursively split into
+    smaller cells until each cell contains at most max_points points.
+
+    More information on S2geometry can be found at https://s2geometry.io/
+    An overview of the S2 cell hierarchy can be found at https://s2geometry.io/resources/s2cell_statistics.html
+
+    :param gdf: GeoDataFrame containing points to split
+    :param max_points: Maximum number of points per group
+    :param start_level: Starting S2 cell level
+    :return: List of GeoDataFrames containing the split groups
+    """
+
+    if "geometry" not in gdf.columns:
+        raise ValueError("The GeoDataFrame must contain a 'geometry' column.")
+
+    if gdf.crs is None:
+        raise ValueError("The GeoDataFrame must contain a CRS")
+
+    # Store the original CRS of the GeoDataFrame and reproject to EPSG:3857
+    original_crs = gdf.crs
+    gdf = gdf.to_crs(epsg=3857)
+
+    # Add a centroid column to the GeoDataFrame and convert it to EPSG:4326
+    gdf["centroid"] = gdf.geometry.centroid
+
+    # Reproject the GeoDataFrame to its orginial CRS
+    gdf = gdf.to_crs(original_crs)
+
+    # Set the GeoDataFrame's geometry to the centroid column and reproject to EPSG:4326
+    gdf = gdf.set_geometry("centroid")
+    gdf = gdf.to_crs(epsg=4326)
+
+    # Create a dictionary to store points by their S2 cell ID
+    cell_dict = {}
+
+    # Iterate over each point in the GeoDataFrame
+    for idx, row in gdf.iterrows():
+        # Get the S2 cell ID for the point at a given level
+        cell_id = _get_s2cell_id(row.centroid, start_level)
+
+        if cell_id not in cell_dict:
+            cell_dict[cell_id] = []
+
+        cell_dict[cell_id].append(row)
+
+    result_groups = []
+
+    # Function to recursively split cells if they contain more points than max_points
+    def _split_s2cell(cell_id, points, current_level=start_level):
+        if len(points) <= max_points:
+            if len(points) > 0:
+                points = gpd.GeoDataFrame(
+                    points, crs=original_crs, geometry="geometry"
+                ).drop(columns=["centroid"])
+                points["s2sphere_cell_id"] = cell_id
+                points["s2sphere_cell_level"] = current_level
+                result_groups.append(gpd.GeoDataFrame(points))
+        else:
+            children = s2sphere.CellId(cell_id).children()
+            child_cells = {child.id(): [] for child in children}
+
+            for point in points:
+                child_cell_id = _get_s2cell_id(point.centroid, current_level + 1)
+                child_cells[child_cell_id].append(point)
+
+            for child_cell_id, child_points in child_cells.items():
+                _split_s2cell(child_cell_id, child_points, current_level + 1)
+
+    # Split cells that contain more points than max_points
+    for cell_id, points in cell_dict.items():
+        _split_s2cell(cell_id, points)
+
+    return result_groups
+
+
+def _get_s2cell_id(point, level):
+    lat, lon = point.y, point.x
+    cell_id = s2sphere.CellId.from_lat_lng(
+        s2sphere.LatLng.from_degrees(lat, lon)
+    ).parent(level)
+    return cell_id.id()

openeo_gfmap/preprocessing/sar.py

@@ -1,9 +1,9 @@
 """Routines to pre-process sar signals."""
 
 import openeo
-from openeo.processes import array_create, if_, is_nodata, power
+from openeo.processes import array_create, power
 
-from openeo_gfmap import Backend, BackendContext
+from openeo_gfmap import BackendContext
 
 
 def compress_backscatter_uint16(
@@ -27,38 +27,17 @@ def compress_backscatter_uint16(
     openeo.DataCube
         The datacube with the backscatter values compressed to uint16.
     """
-    backend = backend_context.backend
 
-    # Additional check related to problematic values present in creodias collections.
-    # https://github.com/Open-EO/openeo-geopyspark-driver/issues/293
-    if backend in [Backend.CDSE, Backend.CDSE_STAGING, Backend.FED]:
-        cube = cube.apply_dimension(
-            dimension="bands",
-            process=lambda x: array_create(
-                [
-                    if_(
-                        is_nodata(x[0]),
-                        1,
-                        power(base=10, p=(10.0 * x[0].log(base=10) + 83.0) / 20.0),
-                    ),
-                    if_(
-                        is_nodata(x[1]),
-                        1,
-                        power(base=10, p=(10.0 * x[1].log(base=10) + 83.0) / 20.0),
-                    ),
-                ]
-            ),
-        )
-    else:
-        cube = cube.apply_dimension(
-            dimension="bands",
-            process=lambda x: array_create(
-                [
-                    power(base=10, p=(10.0 * x[0].log(base=10) + 83.0) / 20.0),
-                    power(base=10, p=(10.0 * x[1].log(base=10) + 83.0) / 20.0),
-                ]
-            ),
-        )
+    # Apply rescaling of power values in a logarithmic way
+    cube = cube.apply_dimension(
+        dimension="bands",
+        process=lambda x: array_create(
+            [
+                power(base=10, p=(10.0 * x[0].log(base=10) + 83.0) / 20.0),
+                power(base=10, p=(10.0 * x[1].log(base=10) + 83.0) / 20.0),
+            ]
+        ),
+    )
 
     # Change the data type to uint16 for optimization purposes
     return cube.linear_scale_range(1, 65534, 1, 65534)

openeo_gfmap/utils/catalogue.py

@@ -8,7 +8,7 @@ import requests
 from pyproj.crs import CRS
 from rasterio.warp import transform_bounds
 from requests import adapters
-from shapely.geometry import box, shape
+from shapely.geometry import Point, box, shape
 from shapely.ops import unary_union
 
 from openeo_gfmap import (
@@ -204,8 +204,14 @@ def s1_area_per_orbitstate_vvvh(
         shapely_geometries = [
             shape(feature["geometry"]) for feature in spatial_extent["features"]
         ]
-        geometry = unary_union(shapely_geometries)
-        bounds = geometry.bounds
+        if len(shapely_geometries) == 1 and isinstance(shapely_geometries[0], Point):
+            point = shapely_geometries[0]
+            buffer_size = 0.0001
+            buffered_geometry = point.buffer(buffer_size)
+            bounds = buffered_geometry.bounds
+        else:
+            geometry = unary_union(shapely_geometries)
+            bounds = geometry.bounds
         epsg = 4326
     elif isinstance(spatial_extent, BoundingBoxExtent):
         bounds = [

{openeo_gfmap-0.2.0.dist-info → openeo_gfmap-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: openeo_gfmap
-Version: 0.2.0
+Version: 0.4.0
 Summary: OpenEO General Framework for Mapping
 Project-URL: Homepage, https://github.com/Open-EO/openeo-gfmap
 Project-URL: Bug Tracker, https://github.com/Open-EO/openeo-gfmap/issues
@@ -13,14 +13,15 @@ Requires-Dist: cftime
 Requires-Dist: fastparquet
 Requires-Dist: geojson>=3.0.0
 Requires-Dist: geopandas
-Requires-Dist: h3
+Requires-Dist: h3==4.1.0
 Requires-Dist: h5netcdf>=1.2.0
 Requires-Dist: netcdf4
 Requires-Dist: numpy<2.0.0
 Requires-Dist: onnxruntime
-Requires-Dist: openeo
+Requires-Dist: openeo<=0.35
 Requires-Dist: pyarrow
 Requires-Dist: rasterio
+Requires-Dist: s2sphere==0.2.*
 Requires-Dist: scipy
 Provides-Extra: dev
 Requires-Dist: matplotlib>=3.3.0; extra == 'dev'

{openeo_gfmap-0.2.0.dist-info → openeo_gfmap-0.4.0.dist-info}/RECORD

@@ -14,13 +14,13 @@ openeo_gfmap/fetching/s2.py,sha256=ytjrZiZIwXxrdiky2V0bAKLBU9Dpaa5b2XsHvI6jl1M,7
 openeo_gfmap/inference/__init__.py,sha256=M6NnKGYCpHNYmRL9OkHi5GmfCtWoJ0wCNR6VXRuDgjE,165
 openeo_gfmap/inference/model_inference.py,sha256=0qPUgrjI1hy5ZnyGwuuvvw5oxnMGdgvvu9Go6-e9LZQ,12550
 openeo_gfmap/manager/__init__.py,sha256=2bckkPiDQBgoBWD9spk1BKXy2UGkWKe50A3HmIwmqrA,795
-openeo_gfmap/manager/job_manager.py,sha256=7NwpU6kvrtqtvhtDjymbR4AKySp8E2KyCfBJFqrd2Ns,27165
-openeo_gfmap/manager/job_splitters.py,sha256=PD4DsZr34MEMMVMHMUiKyFvdgKYyQpL-Lohpak8g_vU,5947
+openeo_gfmap/manager/job_manager.py,sha256=-MZJBfF_wV94FejoYbFPNviEQx3jLmJXb6XLeHg7egE,27221
+openeo_gfmap/manager/job_splitters.py,sha256=kkGxgiudY3LrA40Ro_9q2eFW_Pjdm5a5VaNOXk2w5qo,10694
 openeo_gfmap/preprocessing/__init__.py,sha256=-kJAy_WY4o8oqziRozcUuXtuGIM0IOvTCF6agTUgRWA,619
 openeo_gfmap/preprocessing/cloudmasking.py,sha256=d280H5fByjNbCVZHjPn_dUatNI-ejphu4A75sUVoRqo,10029
 openeo_gfmap/preprocessing/compositing.py,sha256=Jp9Ku5JpU7TJ4DYGc6YuqMeP1Ip7zns7NguC17BtFyA,2526
 openeo_gfmap/preprocessing/interpolation.py,sha256=VVD483NoC0KYUSh28XaBNZzNaybQjdyCN8rXyXy2W9E,327
-openeo_gfmap/preprocessing/sar.py,sha256=mYERSzjNJa92GriSCGmDI4lQLxFxRPeMUU3Yebs6NI0,2238
+openeo_gfmap/preprocessing/sar.py,sha256=XtOIlBrX3o2DlrGHBksHYWzOdzhkZMQT2rp5LxDbdMQ,1391
 openeo_gfmap/preprocessing/scaling.py,sha256=oUNhykVC41Je3E_men_-PikAKNwYhYbwN9J1_Ru8Zi4,2121
 openeo_gfmap/preprocessing/udf_cldmask.py,sha256=WqqFLBK5rIQPkb_dlgUWWSzicsPtVSthaIef40FHKJA,1162
 openeo_gfmap/preprocessing/udf_rank.py,sha256=n2gSIY2ZHVVr9wJx1Bs2HtmvScAkz2NqhjxUM-iIKM0,1438
@@ -29,12 +29,12 @@ openeo_gfmap/stac/__init__.py,sha256=kVMJ9hrN4MjcRCOgRDCj5TfAWRXe0GHu2gJQjG-dS4Y
 openeo_gfmap/stac/constants.py,sha256=O1bcijRBj6YRqR_aAcYO5JzJg7mdzhzUSm4vKnxMbtQ,1485
 openeo_gfmap/utils/__init__.py,sha256=UDwkWUwsnV6ZLXeaJKOCos-MDG2ZaIFyg8s0IiRVtng,997
 openeo_gfmap/utils/build_df.py,sha256=OPmD_Onkl9ybYIiLxmU_GmanP8xD71F1ZybJc7xQmns,1515
-openeo_gfmap/utils/catalogue.py,sha256=-dG07sJ6mXOk2RguiOPcrWdCo5z8BhNmXN0ae7j3kr0,13348
+openeo_gfmap/utils/catalogue.py,sha256=sgiXbRfywY77HSdQnnJ9eKoadc4TKQVa-uuaRadgAOw,13642
 openeo_gfmap/utils/intervals.py,sha256=V6l3ofww50fN_pvWC4NuGQ2ZsyGdhAlTZTiRcC0foVE,2395
 openeo_gfmap/utils/netcdf.py,sha256=KkAAxnq-ZCMjDMd82638noYwxqNpMsnpiU04Q-qX26A,698
 openeo_gfmap/utils/split_stac.py,sha256=asjT0jx6ic8GJFqqAisaWxOvQ_suSRv4sxyFOyHFvpI,3895
 openeo_gfmap/utils/tile_processing.py,sha256=QZ9bi5tPmyTVyyNvFZgd26s5dSnMl1grTKq2veK1C90,2068
-openeo_gfmap-0.2.0.dist-info/METADATA,sha256=aSf8KibRnGJl6B8lRH1nT1RYWpR9TnMaKHy6Y3qIE-s,4278
-openeo_gfmap-0.2.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-openeo_gfmap-0.2.0.dist-info/licenses/LICENSE,sha256=aUuGpjieWiscTNtyLcSaeVsJ4pb6J9c4wUq1bR0e4t4,11349
-openeo_gfmap-0.2.0.dist-info/RECORD,,
+openeo_gfmap-0.4.0.dist-info/METADATA,sha256=65GVaP6sG-Gnemf3HlqyRtREBlujPgBHScIAGSk8dpI,4322
+openeo_gfmap-0.4.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+openeo_gfmap-0.4.0.dist-info/licenses/LICENSE,sha256=aUuGpjieWiscTNtyLcSaeVsJ4pb6J9c4wUq1bR0e4t4,11349
+openeo_gfmap-0.4.0.dist-info/RECORD,,

{openeo_gfmap-0.2.0.dist-info → openeo_gfmap-0.4.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.25.0
+Generator: hatchling 1.27.0
 Root-Is-Purelib: true
 Tag: py3-none-any