eotdl 2023.7.19.post4__py3-none-any.whl → 2023.9.14__py3-none-any.whl

eotdl/curation/stac/extensions.py

@@ -2,11 +2,24 @@
 Module for STAC extensions objects
 """
 
+import rasterio
+import json
 import pystac
+
+from os.path import join, dirname
 from pystac.extensions.sar import SarExtension
 from pystac.extensions.sar import FrequencyBand, Polarization
 from pystac.extensions.eo import Band, EOExtension
-from typing import Union
+from pystac.extensions.label import (LabelClasses, LabelExtension, SummariesLabelExtension)
+from pystac.extensions.raster import RasterExtension, RasterBand
+from pystac.extensions.projection import ProjectionExtension
+from typing import Union, List, Optional, Dict
+
+import pandas as pd
+from typing import List
+
+
+SUPPORTED_EXTENSIONS = ('eo', 'sar', 'proj', 'raster')
 
 
 class STACExtensionObject:
@@ -15,12 +28,14 @@ class STACExtensionObject:
         self.properties = dict()
 
     def add_extension_to_object(
-        self, obj: Union[pystac.Item, pystac.Asset]
+        self, obj: Union[pystac.Item, pystac.Asset],
+        obj_info: Optional[pd.DataFrame] = None
     ) -> Union[pystac.Item, pystac.Asset]:
         """
         Add the extension to the given object
 
         :param obj: object to add the extension
+        :param obj_info: object info from the STACDataFrame
         """
         pass
 
@@ -28,22 +43,31 @@ class STACExtensionObject:
 class SarExtensionObject(STACExtensionObject):
     def __init__(self) -> None:
         super().__init__()
-        pass
+        self.polarizations = [Polarization.VV, Polarization.VH]
+        self.polarizations_dict = {"VV": Polarization.VV, "VH": Polarization.VH}
 
     def add_extension_to_object(
-        self, obj: Union[pystac.Item, pystac.Asset]
+        self, obj: Union[pystac.Item, pystac.Asset],
+        obj_info: Optional[pd.DataFrame] = None
     ) -> Union[pystac.Item, pystac.Asset]:
         """
         Add the extension to the given object
 
         :param obj: object to add the extension
+        :param obj_info: object info from the STACDataFrame
         """
+        # Add SAR extension to the item
         sar_ext = SarExtension.ext(obj, add_if_missing=True)
-        if isinstance(obj, pystac.Item):
-            polarizations = [Polarization.VV, Polarization.VH]
-        elif isinstance(obj, pystac.Asset):
-            polarizations_dict = {"VV": Polarization.VV, "VH": Polarization.VH}
-            polarizations = [polarizations_dict[obj.title]]
+        if isinstance(obj, pystac.Item) or (
+            isinstance(obj, pystac.Asset)
+            and obj.title not in self.polarizations_dict.keys()
+        ):
+            polarizations = self.polarizations
+        elif (
+            isinstance(obj, pystac.Asset)
+            and obj.title in self.polarizations_dict.keys()
+        ):
+            polarizations = [self.polarizations_dict[obj.title]]
         sar_ext.apply(
             instrument_mode="EW",
             polarizations=polarizations,
@@ -57,93 +81,102 @@ class SarExtensionObject(STACExtensionObject):
 class EOS2ExtensionObject(STACExtensionObject):
     def __init__(self) -> None:
         super().__init__()
-        self.bands = [
-            Band.create(
-                name="Aerosols",
+        self.bands_dict = {
+            "B01": Band.create(
+                name="B01",
                 description="Coastal aerosol, 442.7 nm (S2A), 442.3 nm (S2B)",
                 common_name="coastal",
             ),
-            Band.create(
-                name="Blue",
+            "B02": Band.create(
+                name="B02",
                 description="Blue, 492.4 nm (S2A), 492.1 nm (S2B)",
                 common_name="blue",
             ),
-            Band.create(
-                name="Green",
+            "B03": Band.create(
+                name="B03",
                 description="Green, 559.8 nm (S2A), 559.0 nm (S2B)",
                 common_name="green",
             ),
-            Band.create(
-                name="Red",
+            "B04": Band.create(
+                name="B04",
                 description="Red, 664.6 nm (S2A), 665.0 nm (S2B)",
                 common_name="red",
             ),
-            Band.create(
-                name="Red edge 1",
+            "B05": Band.create(
+                name="B05",
                 description="Vegetation red edge, 704.1 nm (S2A), 703.8 nm (S2B)",
                 common_name="rededge",
             ),
-            Band.create(
-                name="Red edge 2",
+            "B06": Band.create(
+                name="B06",
                 description="Vegetation red edge, 740.5 nm (S2A), 739.1 nm (S2B)",
                 common_name="rededge",
             ),
-            Band.create(
-                name="Red edge 3",
+            "B07": Band.create(
+                name="B07",
                 description="Vegetation red edge, 782.8 nm (S2A), 779.7 nm (S2B)",
                 common_name="rededge",
             ),
-            Band.create(
-                name="NIR",
+            "B08": Band.create(
+                name="B08",
                 description="NIR, 832.8 nm (S2A), 833.0 nm (S2B)",
                 common_name="nir",
             ),
-            Band.create(
-                name="Red edge 4",
+            "B08a": Band.create(
+                name="B08a",
                 description="Narrow NIR, 864.7 nm (S2A), 864.0 nm (S2B)",
                 common_name="nir08",
             ),
-            Band.create(
-                name="Water vapour",
+            "B09": Band.create(
+                name="B09",
                 description="Water vapour, 945.1 nm (S2A), 943.2 nm (S2B)",
                 common_name="nir09",
             ),
-            Band.create(
-                name="Cirrus",
+            "B10": Band.create(
+                name="B10",
                 description="SWIR – Cirrus, 1373.5 nm (S2A), 1376.9 nm (S2B)",
                 common_name="cirrus",
             ),
-            Band.create(
-                name="SWIR1",
+            "B11": Band.create(
+                name="B11",
                 description="SWIR, 1613.7 nm (S2A), 1610.4 nm (S2B)",
                 common_name="swir16",
             ),
-            Band.create(
-                name="SWIR2",
+            "B12": Band.create(
+                name="B12",
                 description="SWIR, 2202.4 nm (S2A), 2185.7 nm (S2B)",
                 common_name="swir22",
             ),
-        ]
+        }
 
     def add_extension_to_object(
-        self, obj: Union[pystac.Item, pystac.Asset]
+        self, obj: Union[pystac.Item, pystac.Asset],
+        obj_info: pd.DataFrame
     ) -> Union[pystac.Item, pystac.Asset]:
         """
         Add the extension to the given object
 
         :param obj: object to add the extension
+        :param obj_info: object info from the STACDataFrame
         """
-        if isinstance(obj, pystac.Asset):
-            return
         # Add EO extension
         eo_ext = EOExtension.ext(obj, add_if_missing=True)
-        # Add the existing bands from the rasters assets list
-        eo_ext.apply(bands=self.bands)
         # Add common metadata
-        obj.common_metadata.constellation = "Sentinel-2"
-        obj.common_metadata.platform = "Sentinel-2"
-        obj.common_metadata.instruments = ["Sentinel-2"]
-        obj.common_metadata.gsd = 10  # TODO Where to obtain it?
+        if isinstance(obj, pystac.Item) or (
+            isinstance(obj, pystac.Asset) and obj.title not in self.bands_dict.keys()
+        ):
+            obj.common_metadata.constellation = "Sentinel-2"
+            obj.common_metadata.platform = "Sentinel-2"
+            obj.common_metadata.instruments = ["Sentinel-2"]
+            obj.common_metadata.gsd = 10
+            # Add bands
+            bands = obj_info["bands"].values
+            bands = bands[0] if bands else None
+            bands_list = [self.bands_dict[band] for band in bands] if bands else None
+            eo_ext.apply(bands=bands_list)
+
+        elif isinstance(obj, pystac.Asset):
+            eo_ext.apply(bands=[self.bands_dict[obj.title]])
 
         return obj
 
@@ -158,8 +191,224 @@ class DEMExtensionObject(STACExtensionObject):
         super().__init__()
 
 
+class LabelExtensionObject(STACExtensionObject):
+    def __init__(self) -> None:
+        super().__init__()
+
+    @classmethod
+    def add_extension_to_item(
+        self,
+        obj: pystac.Item,
+        label_names: List[str],
+        label_classes: List[str],
+        label_properties: list,
+        label_description: str,
+        label_methods: list,
+        label_tasks: List[str],
+        label_type: str
+    ) -> Union[pystac.Item, pystac.Asset]:
+        """
+        Add the extension to the given object
+
+        :param obj: object to add the extension
+        :param label_names: list of label names
+        :param label_classes: list of label classes of the item
+        :param label_classes_list: list of all possible label classes
+        :param label_properties: list of label properties
+        :param label_description: label description
+        :param label_methods: list of labeling methods
+        :param label_tasks: list of label tasks
+        :param label_type: label type
+
+        :return: the item with the label extension
+        """
+        label_item = pystac.Item(id=obj.id,
+                                 geometry=obj.geometry,
+                                 bbox=obj.bbox,
+                                 properties=dict(),
+                                 datetime=obj.datetime
+                                 )
+        
+        # Add the label extension to the item
+        LabelExtension.add_to(label_item)
+
+        # Access the label extension
+        label_ext = LabelExtension.ext(label_item)
+
+        # Add the label classes
+        for name, classes in zip(label_names, label_classes):
+            label_classes = LabelClasses.create(
+                name=name,
+                classes=classes,
+            )
+            label_ext.label_classes = [label_classes]
+
+        # TODO kwargs
+        # Add the label properties
+        label_ext.label_properties = label_properties
+        # Add the label description
+        label_ext.label_description = label_description
+        # Add the label methods
+        label_ext.label_methods = label_methods
+        # Add the label type
+        label_ext.label_type = label_type
+        # Add the label tasks
+        label_ext.label_tasks = label_tasks
+        # Add the source
+        label_ext.add_source(obj)
+
+        return label_item
+    
+    @classmethod
+    def add_geojson_to_items(self, 
+                             collection: pystac.Collection,
+                             df: pd.DataFrame
+                             ) -> None:
+        """
+        """
+        for item in collection.get_all_items():
+            label_type = item.properties['label:type']
+            file_name = 'vector_labels' if label_type == 'vector' else 'raster_labels'
+            geojson_path = join(dirname(item.get_self_href()), f'{file_name}.geojson')
+
+            properties = {'roles': ['labels', f'labels-{label_type}']}
+
+            # TODO depending on the tasks, there must be extra fields
+            # TODO https://github.com/stac-extensions/label#assets
+            tasks = item.properties['label:tasks']
+            if 'tile_regression' in tasks:
+                pass
+            elif any(task in tasks for task in ('tile_classification', 'object_detection', 'segmentation')):
+                pass
+
+            label_ext = LabelExtension.ext(item)
+            label_ext.add_geojson_labels(href=geojson_path, 
+                                         title='Label', 
+                                         properties=properties)
+            item.make_asset_hrefs_relative()
+            
+            item_id = item.id
+            geometry = item.geometry
+            labels = [df[df['id'] == item_id]['label'].values[0]]
+            # There is data like DEM data that does not have datetime but start and end datetime
+            datetime = item.datetime.isoformat() if item.datetime else (item.properties.start_datetime.isoformat(),
+                                                                        item.properties.end_datetime.isoformat())
+            labels_properties = dict(zip(item.properties['label:properties'], labels))
+            labels_properties['datetime'] = datetime
+
+            geojson = {
+                "type": "FeatureCollection",
+                "features": [
+                    {
+                        "type": "Feature",
+                        "geometry": geometry,
+                        "properties": labels_properties,
+                    }
+                ],
+            }
+
+            with open(geojson_path, "w") as f:
+                json.dump(geojson, f)
+
+    @classmethod
+    def add_extension_to_collection(
+            self,
+            obj: pystac.Collection,
+            label_names: List[str],
+            label_classes: List[Union[list, tuple]],
+            label_type: str
+    ) -> None:
+        """
+        Add the label extension to the given collection
+
+        :param obj: object to add the extension
+        :param label_names: list of label names
+        :param label_classes: list of label classes
+        :param label_type: label type
+        """
+        LabelExtension.add_to(obj)
+
+        # Add the label extension to the collection
+        label_ext = SummariesLabelExtension(obj)
+
+        # Add the label classes
+        for name, classes in zip(label_names, label_classes):
+            label_classes = LabelClasses.create(
+                name=name,
+                classes=classes,
+            )
+            label_ext.label_classes = [label_classes]
+
+        # Add the label type
+        label_ext.label_type = label_type
+
+
+class RasterExtensionObject(STACExtensionObject):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def add_extension_to_object(
+        self, obj: Union[pystac.Item, pystac.Asset],
+        obj_info: Optional[pd.DataFrame] = None
+    ) -> Union[pystac.Item, pystac.Asset]:
+        """
+        Add the extension to the given object
+
+        :param obj: object to add the extension
+        :param obj_info: object info from the STACDataFrame
+        """
+        if not isinstance(obj, pystac.Asset):
+            return obj
+        else:
+            raster_ext = RasterExtension.ext(obj, add_if_missing=True)
+            src = rasterio.open(obj.href)
+            bands = list()
+            for band in src.indexes:
+                bands.append(RasterBand.create(
+                    nodata=src.nodatavals[band - 1],
+                    data_type=src.dtypes[band - 1],
+                    spatial_resolution=src.res) if src.nodatavals else RasterBand.create(
+                        data_type=src.dtypes[band - 1],
+                        spatial_resolution=src.res))
+            raster_ext.apply(bands=bands)
+                
+        return obj
+
+
+class ProjExtensionObject(STACExtensionObject):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def add_extension_to_object(
+        self, obj: Union[pystac.Item, pystac.Asset],
+        obj_info: pd.DataFrame
+    ) -> Union[pystac.Item, pystac.Asset]:
+        """
+        Add the extension to the given object
+
+        :param obj: object to add the extension
+        :param obj_info: object info from the STACDataFrame
+        """
+        # Add raster extension to the item
+        if isinstance(obj, pystac.Asset):
+            return obj
+        elif isinstance(obj, pystac.Item):
+            proj_ext = ProjectionExtension.ext(obj, add_if_missing=True)
+            ds = rasterio.open(obj_info['image'].values[0])
+            # Assume all the bands have the same projection
+            proj_ext.apply(
+                epsg=ds.crs.to_epsg(),
+                transform=ds.transform,
+                shape=ds.shape,
+                )
+
+        return obj
+
+
 type_stac_extensions_dict = {
     "sar": SarExtensionObject(),
     "eo": EOS2ExtensionObject(),
     "dem": DEMExtensionObject(),
+    "raster": RasterExtensionObject(),
+    "proj": ProjExtensionObject()
 }

eotdl/curation/stac/extent.py

@@ -0,0 +1,130 @@
+'''
+Module for STAC extent
+'''
+
+import pystac
+from datetime import datetime
+import rasterio
+import json
+
+from glob import glob
+from os.path import dirname
+
+from typing import List
+
+
+def get_dem_temporal_interval() -> pystac.TemporalExtent:
+    """
+    Get a temporal interval for DEM data
+    """
+    min_date = datetime.strptime('2011-01-01', '%Y-%m-%d')
+    max_date = datetime.strptime('2015-01-07', '%Y-%m-%d')
+
+    return pystac.TemporalExtent([(min_date, max_date)])
+    
+def get_unknow_temporal_interval() -> pystac.TemporalExtent:
+    """
+    Get an unknown temporal interval
+    """
+    min_date = datetime.strptime('2000-01-01', '%Y-%m-%d')
+    max_date = datetime.strptime('2023-12-31', '%Y-%m-%d')
+
+    return pystac.TemporalExtent([(min_date, max_date)])
+    
+def get_unknow_extent() -> pystac.Extent:
+    """
+    """
+    return pystac.Extent(spatial=pystac.SpatialExtent([[0, 0, 0, 0]]),
+                         temporal=pystac.TemporalExtent([(datetime.strptime('2000-01-01', '%Y-%m-%d'), 
+                                                          datetime.strptime('2023-12-31', '%Y-%m-%d')
+                                                          )]))
+
+
+def get_collection_extent(rasters: List[str]) -> pystac.Extent:
+    """
+    Get the extent of a collection
+    
+    :param rasters: list of rasters
+    """
+    # Get the spatial extent of the collection
+    spatial_extent = get_collection_spatial_extent(rasters)
+    # Get the temporal interval of the collection
+    temporal_interval = get_collection_temporal_interval(rasters)
+    # Create the Extent object
+    extent = pystac.Extent(spatial=spatial_extent, temporal=temporal_interval)
+
+    return extent
+    
+def get_collection_spatial_extent(rasters: List[str]) -> pystac.SpatialExtent:
+    """
+    Get the spatial extent of a collection
+
+    :param path: path to the directory
+    """
+    # Get the bounding boxes of all the given rasters
+    bboxes = list()
+    for raster in rasters:
+        with rasterio.open(raster) as ds:
+            bounds = ds.bounds
+            dst_crs = 'EPSG:4326'
+            try:
+                left, bottom, right, top = rasterio.warp.transform_bounds(ds.crs, dst_crs, *bounds)
+                bbox = [left, bottom, right, top]
+            except rasterio.errors.CRSError:
+                spatial_extent = pystac.SpatialExtent([[0, 0, 0, 0]])
+                return spatial_extent
+            bboxes.append(bbox)
+    # Get the minimum and maximum values of the bounding boxes
+    try:
+        left = min([bbox[0] for bbox in bboxes])
+        bottom = min([bbox[1] for bbox in bboxes])
+        right = max([bbox[2] for bbox in bboxes])
+        top = max([bbox[3] for bbox in bboxes])
+        spatial_extent = pystac.SpatialExtent([[left, bottom, right, top]])
+    except ValueError:
+        spatial_extent = pystac.SpatialExtent([[0, 0, 0, 0]])
+    finally:
+        return spatial_extent
+
+def get_collection_temporal_interval(rasters: List[str]) -> pystac.TemporalExtent:
+    """
+    Get the temporal interval of a collection
+
+    :param path: path to the directory
+    """
+    # Get all the metadata.json files in the directory of all the given rasters
+    metadata_json_files = list()
+    for raster in rasters:
+        metadata_json_files += glob(f'{dirname(raster)}/*.json', recursive=True)
+
+    if not metadata_json_files:
+        return get_unknow_temporal_interval()   # If there is no metadata, set a generic temporal interval
+    
+    # Get the temporal interval of every metadata.json file and the type of the data
+    data_types = list()
+    temporal_intervals = list()
+    for metadata_json_file in metadata_json_files:
+        with open(metadata_json_file, 'r') as f:
+            metadata = json.load(f)
+        # Append the temporal interval to the list as a datetime object
+        temporal_intervals.append(metadata['date-adquired']) if metadata['date-adquired'] else None
+        # Append the data type to the list
+        data_types.append(metadata['type']) if metadata['type'] else None
+        
+    if temporal_intervals:
+        try:
+            # Get the minimum and maximum values of the temporal intervals
+            min_date = min([datetime.strptime(interval, '%Y-%m-%d') for interval in temporal_intervals])
+            max_date = max([datetime.strptime(interval, '%Y-%m-%d') for interval in temporal_intervals])
+        except ValueError:
+            min_date = datetime.strptime('2000-01-01', '%Y-%m-%d')
+            max_date = datetime.strptime('2023-12-31', '%Y-%m-%d')
+        finally:
+            # Create the temporal interval
+            return pystac.TemporalExtent([(min_date, max_date)])
+    else:
+        # Check if the collection is composed by DEM data. If not, set a generic temporal interval
+        if set(data_types) == {'dem'} or set(data_types) == {'DEM'} or set(data_types) == {'dem', 'DEM'}:
+            return get_dem_temporal_interval()
+        else:
+            return get_unknow_temporal_interval()