eotdl 2023.11.2.post5__py3-none-any.whl → 2023.11.3.post2__py3-none-any.whl

eotdl/curation/stac/extent.py

@@ -1,16 +1,14 @@
-'''
+"""
 Module for STAC extent
-'''
+"""
 
-import pystac
 from datetime import datetime
-import rasterio
-import json
-
-from glob import glob
-from os.path import dirname
 from typing import List
 
+import pystac
+
+import rasterio
+
 from ...tools import get_item_metadata
 
 
@@ -18,33 +16,43 @@ def get_dem_temporal_interval() -> pystac.TemporalExtent:
     """
     Get a temporal interval for DEM data
     """
-    min_date = datetime.strptime('2011-01-01', '%Y-%m-%d')
-    max_date = datetime.strptime('2015-01-07', '%Y-%m-%d')
+    min_date = datetime.strptime("2011-01-01", "%Y-%m-%d")
+    max_date = datetime.strptime("2015-01-07", "%Y-%m-%d")
 
     return pystac.TemporalExtent([(min_date, max_date)])
-    
+
+
 def get_unknow_temporal_interval() -> pystac.TemporalExtent:
     """
     Get an unknown temporal interval
     """
-    min_date = datetime.strptime('2000-01-01', '%Y-%m-%d')
-    max_date = datetime.strptime('2023-12-31', '%Y-%m-%d')
+    min_date = datetime.strptime("2000-01-01", "%Y-%m-%d")
+    max_date = datetime.strptime("2023-12-31", "%Y-%m-%d")
 
     return pystac.TemporalExtent([(min_date, max_date)])
-    
+
+
 def get_unknow_extent() -> pystac.Extent:
     """
+    Get an unknown extent
     """
-    return pystac.Extent(spatial=pystac.SpatialExtent([[0, 0, 0, 0]]),
-                         temporal=pystac.TemporalExtent([(datetime.strptime('2000-01-01', '%Y-%m-%d'), 
-                                                          datetime.strptime('2023-12-31', '%Y-%m-%d')
-                                                          )]))
+    return pystac.Extent(
+        spatial=pystac.SpatialExtent([[0, 0, 0, 0]]),
+        temporal=pystac.TemporalExtent(
+            [
+                (
+                    datetime.strptime("2000-01-01", "%Y-%m-%d"),
+                    datetime.strptime("2023-12-31", "%Y-%m-%d"),
+                )
+            ]
+        ),
+    )
 
 
 def get_collection_extent(rasters: List[str]) -> pystac.Extent:
     """
     Get the extent of a collection
-    
+
     :param rasters: list of rasters
     """
     # Get the spatial extent of the collection
@@ -55,7 +63,8 @@ def get_collection_extent(rasters: List[str]) -> pystac.Extent:
     extent = pystac.Extent(spatial=spatial_extent, temporal=temporal_interval)
 
     return extent
-    
+
+
 def get_collection_spatial_extent(rasters: List[str]) -> pystac.SpatialExtent:
     """
     Get the spatial extent of a collection
@@ -63,13 +72,15 @@ def get_collection_spatial_extent(rasters: List[str]) -> pystac.SpatialExtent:
     :param path: path to the directory
     """
     # Get the bounding boxes of all the given rasters
-    bboxes = list()
+    bboxes = []
     for raster in rasters:
         with rasterio.open(raster) as ds:
             bounds = ds.bounds
-            dst_crs = 'EPSG:4326'
+            dst_crs = "EPSG:4326"
             try:
-                left, bottom, right, top = rasterio.warp.transform_bounds(ds.crs, dst_crs, *bounds)
+                left, bottom, right, top = rasterio.warp.transform_bounds(
+                    ds.crs, dst_crs, *bounds
+                )
                 bbox = [left, bottom, right, top]
             except rasterio.errors.CRSError:
                 spatial_extent = pystac.SpatialExtent([[0, 0, 0, 0]])
@@ -77,16 +88,17 @@ def get_collection_spatial_extent(rasters: List[str]) -> pystac.SpatialExtent:
             bboxes.append(bbox)
     # Get the minimum and maximum values of the bounding boxes
     try:
-        left = min([bbox[0] for bbox in bboxes])
-        bottom = min([bbox[1] for bbox in bboxes])
-        right = max([bbox[2] for bbox in bboxes])
-        top = max([bbox[3] for bbox in bboxes])
+        left = min(bbox[0] for bbox in bboxes)
+        bottom = min(bbox[1] for bbox in bboxes)
+        right = max(bbox[2] for bbox in bboxes)
+        top = max(bbox[3] for bbox in bboxes)
         spatial_extent = pystac.SpatialExtent([[left, bottom, right, top]])
     except ValueError:
         spatial_extent = pystac.SpatialExtent([[0, 0, 0, 0]])
     finally:
         return spatial_extent
 
+
 def get_collection_temporal_interval(rasters: List[str]) -> pystac.TemporalExtent:
     """
     Get the temporal interval of a collection
@@ -94,38 +106,53 @@ def get_collection_temporal_interval(rasters: List[str]) -> pystac.TemporalExten
     :param path: path to the directory
     """
     # Get all the metadata.json files in the directory of all the given rasters
-    metadata_jsons = list()
+    metadata_jsons = []
     for raster in rasters:
         metadata_json = get_item_metadata(raster)
         if metadata_json:
             metadata_jsons.append(metadata_json)
 
     if not metadata_jsons:
-        return get_unknow_temporal_interval()   # If there is no metadata, set a generic temporal interval
-    
+        return (
+            get_unknow_temporal_interval()
+        )  # If there is no metadata, set a generic temporal interval
+
     # Get the temporal interval of every metadata.json file and the type of the data
-    data_types = list()
-    temporal_intervals = list()
+    data_types = []
+    temporal_intervals = []
     for metadata in metadata_jsons:
         # Append the temporal interval to the list as a datetime object
-        temporal_intervals.append(metadata['acquisition-date']) if metadata['acquisition-date'] else None
+        temporal_intervals.append(metadata["acquisition-date"]) if metadata[
+            "acquisition-date"
+        ] else None
         # Append the data type to the list
-        data_types.append(metadata['type']) if metadata['type'] else None
-        
+        data_types.append(metadata["type"]) if metadata["type"] else None
+
     if temporal_intervals:
+        min_date, max_date = None, None
         try:
             # Get the minimum and maximum values of the temporal intervals
-            min_date = min([datetime.strptime(interval, '%Y-%m-%d') for interval in temporal_intervals])
-            max_date = max([datetime.strptime(interval, '%Y-%m-%d') for interval in temporal_intervals])
+            min_date = min(
+                datetime.strptime(interval, "%Y-%m-%d")
+                for interval in temporal_intervals
+            )
+            max_date = max(
+                datetime.strptime(interval, "%Y-%m-%d")
+                for interval in temporal_intervals
+            )
         except ValueError:
-            min_date = datetime.strptime('2000-01-01', '%Y-%m-%d')
-            max_date = datetime.strptime('2023-12-31', '%Y-%m-%d')
+            min_date = datetime.strptime("2000-01-01", "%Y-%m-%d")
+            max_date = datetime.strptime("2023-12-31", "%Y-%m-%d")
         finally:
             # Create the temporal interval
             return pystac.TemporalExtent([(min_date, max_date)])
     else:
         # Check if the collection is composed by DEM data. If not, set a generic temporal interval
-        if set(data_types) == {'dem'} or set(data_types) == {'DEM'} or set(data_types) == {'dem', 'DEM'}:
+        if (
+            set(data_types) == {"dem"}
+            or set(data_types) == {"DEM"}
+            or set(data_types) == {"dem", "DEM"}
+        ):
             return get_dem_temporal_interval()
         else:
             return get_unknow_temporal_interval()

eotdl/curation/stac/parsers.py

@@ -1,26 +1,31 @@
-'''
+"""
 Module for STAC parsers
-'''
+"""
 
 from os.path import dirname, basename
 
 
 class STACIdParser:
-    
+    """
+    STAC ID parser base class
+    """
     def get_item_id(self, raster_path: str):
         """
         Get the ID of the STAC Item from the given raster path
 
         :param raster_path: path to the raster file
         """
-        pass
+        return
 
 
 class StructuredParser(STACIdParser):
+    """
+    Structured STAC ID parser
+    """
 
     def __init__(self) -> None:
         super().__init__()
-    
+
     def get_item_id(self, raster_path: str):
         """
         Get the ID of the STAC Item from the given raster path.
@@ -30,16 +35,19 @@ class StructuredParser(STACIdParser):
         :param raster_path: path to the raster file
         """
         tiff_dir_path = dirname(raster_path)
-        id = tiff_dir_path.split('/')[-1]
+        item_id = tiff_dir_path.split("/")[-1]
 
-        return id
+        return item_id
 
 
 class UnestructuredParser(STACIdParser):
+    """
+    Unstructured STAC ID parser
+    """
 
     def __init__(self) -> None:
         super().__init__()
-    
+
     def get_item_id(self, raster_path: str):
         """
         Get the ID of the STAC Item from the given raster path.
@@ -48,6 +56,6 @@ class UnestructuredParser(STACIdParser):
 
         :param raster_path: path to the raster file
         """
-        id = basename(raster_path).split('.')[0]
+        item_id = basename(raster_path).split(".")[0]
 
-        return id
+        return item_id

eotdl/curation/stac/stac.py

@@ -1,41 +1,39 @@
 """
-Module for generating STAC metadata 
+Module for generating STAC metadata
 """
 
-import traceback
-from typing import Union
-import pandas as pd
-import pystac
-from tqdm import tqdm
-
+import random
+from datetime import datetime
+from typing import Union, Optional
 from os.path import join, basename, dirname
-from shutil import rmtree
 
+import pandas as pd
+import pystac
 import rasterio
-import random
-from rasterio.warp import transform_bounds
-from typing import Union, List
-
-from datetime import datetime
+from tqdm import tqdm
 from shapely.geometry import Polygon, mapping
-from glob import glob
-from typing import Union, Optional
 
 from .parsers import STACIdParser, StructuredParser
 from .assets import STACAssetGenerator
 from .dataframe_labeling import LabelingStrategy, UnlabeledStrategy
-from ...tools import (format_time_acquired, 
-                    cut_images, 
-                    get_item_metadata,
-                    get_all_images_in_path)
-from .extensions import (type_stac_extensions_dict, 
-                         SUPPORTED_EXTENSIONS, 
-                         LabelExtensionObject)
-from .extent import (get_unknow_extent, 
-                     get_collection_extent)
+from ...tools import (
+    format_time_acquired,
+    cut_images,
+    get_item_metadata,
+    get_all_images_in_path,
+)
+from .extensions import (
+    type_stac_extensions_dict,
+    SUPPORTED_EXTENSIONS,
+)
+from .extent import get_collection_extent
 
 
 class STACGenerator:
+    """
+    STAC generator class
+    """
+
     def __init__(
         self,
         image_format: str = "tiff",
@@ -63,7 +61,7 @@ class STACGenerator:
 
     def generate_stac_metadata(
         self,
-        id: str,
+        stac_id: str,
         description: str,
         stac_dataframe: pd.DataFrame = None,
         output_folder: str = "stac",
@@ -84,8 +82,8 @@ class STACGenerator:
             raise ValueError("No STAC dataframe provided")
 
         # Create an empty catalog
-        catalog = pystac.Catalog(id=id, description=description, **kwargs)
-        
+        catalog = pystac.Catalog(id=stac_id, description=description, **kwargs)
+
         # Add the collections to the catalog
         collections = self._stac_dataframe.collection.unique()
         for collection_path in collections:
@@ -97,7 +95,9 @@ class STACGenerator:
         # Check there have been generate all the items from the images
         items_count = 0
         for collection in catalog.get_children():
-            items = list(set([item.id for item in collection.get_items(recursive=True)]))
+            items = list(
+                set([item.id for item in collection.get_items(recursive=True)])
+            )
             items_count += len(items)
         if len(self._stac_dataframe) != items_count:
             raise pystac.STACError(
@@ -117,13 +117,14 @@ class STACGenerator:
             print(f"Catalog validation error: {e}")
             return
 
-    def get_stac_dataframe(self, 
-                           path: str, 
-                           collections: Optional[Union[str, dict]]='source',
-                           bands: Optional[dict]=None, 
-                           extensions: Optional[dict]=None,
-                           sample: Optional[int]=None
-                           ) -> pd.DataFrame:
+    def get_stac_dataframe(
+        self,
+        path: str,
+        collections: Optional[Union[str, dict]] = "source",
+        bands: Optional[dict] = None,
+        extensions: Optional[dict] = None,
+        sample: Optional[int] = None,
+    ) -> pd.DataFrame:
         """
         Get a dataframe with the STAC metadata of a given directory containing the assets to generate metadata
 
@@ -134,16 +135,20 @@ class STACGenerator:
         """
         images = get_all_images_in_path(path, self._image_format)
         if len(images) == 0:
-            raise ValueError("No images found in the given path with the given extension. Please check the path and the extension")
-        
-        if self._assets_generator.type == 'Extracted':
+            raise ValueError(
+                "No images found in the given path with the given extension. Please check the path and the extension"
+            )
+
+        if self._assets_generator.type == "Extracted":
             images = cut_images(images)
 
         if sample:
             try:
                 images = random.sample(images, sample)
             except ValueError:
-                raise ValueError(f"Sample size must be smaller than the number of images ({len(images)}). May be there are no images found in the given path with the given extension")
+                raise ValueError(
+                    f"Sample size must be smaller than the number of images ({len(images)}). May be there are no images found in the given path with the given extension"
+                )
 
         labels, ixs = self._labeling_strategy.get_images_labels(images)
         bands_values = self._get_items_list_from_dict(labels, bands)
@@ -152,22 +157,32 @@ class STACGenerator:
         if collections == "source":
             # List of path with the same value repeated as many times as the number of images
             collections_values = [join(path, "source") for i in range(len(images))]
-        elif collections == '*':
-            collections_values = [join(path, basename(dirname(image))) for image in images]
+        elif collections == "*":
+            collections_values = [
+                join(path, basename(dirname(image))) for image in images
+            ]
         else:
             try:
-                collections_values = [join(path, value) for value in self._get_items_list_from_dict(labels, collections)]
+                collections_values = [
+                    join(path, value)
+                    for value in self._get_items_list_from_dict(labels, collections)
+                ]
             except TypeError:
-                raise pystac.STACError('There is an error generating the collections. Please check the collections dictionary')
-
-        df = pd.DataFrame({'image': images, 
-                           'label': labels, 
-                           'ix': ixs, 
-                           'collection': collections_values, 
-                           'extensions': extensions_values, 
-                           'bands': bands_values
-                           })
-        
+                raise pystac.STACError(
+                    "There is an error generating the collections. Please check the collections dictionary"
+                )
+
+        df = pd.DataFrame(
+            {
+                "image": images,
+                "label": labels,
+                "ix": ixs,
+                "collection": collections_values,
+                "extensions": extensions_values,
+                "bands": bands_values,
+            }
+        )
+
         self._stac_dataframe = df
 
         return df
@@ -182,7 +197,7 @@ class STACGenerator:
         if not items:
             # Create list of None with the same length as the labels list
             return [None for _ in labels]
-        items_list = list()
+        items_list = []
         for label in labels:
             if label in items.keys():
                 items_list.append(items[label])
@@ -219,7 +234,7 @@ class STACGenerator:
         # Return the collection
         return collection
 
-    def create_stac_item(self, raster_path: str, kwargs: dict = {}) -> pystac.Item:
+    def create_stac_item(self, raster_path: str) -> pystac.Item:
         """
         Create a STAC item from a directory containing the raster files and the metadata.json file
 
@@ -250,12 +265,16 @@ class STACGenerator:
         )
 
         # Initialize pySTAC item parameters
-        params = dict()
-        params["properties"] = dict()
+        params = {}
+        params["properties"] = {}
 
         # Obtain the date acquired
         start_time, end_time = None, None
-        if metadata and metadata["acquisition-date"] and metadata["type"] not in ('dem', 'DEM'):
+        if (
+            metadata
+            and metadata["acquisition-date"]
+            and metadata["type"] not in ("dem", "DEM")
+        ):
             time_acquired = format_time_acquired(metadata["acquisition-date"])
         else:
             # Check if the type of the data is DEM
@@ -270,15 +289,15 @@ class STACGenerator:
                 time_acquired = datetime.strptime("2000-01-01", "%Y-%m-%d")
 
         # Obtain the item ID. The approach depends on the item parser
-        id = self._item_parser.get_item_id(raster_path)
+        item_id = self._item_parser.get_item_id(raster_path)
         # Add the item ID to the dataframe, to be able to get it later
         self._stac_dataframe.loc[
             self._stac_dataframe["image"] == raster_path, "id"
-        ] = id
-        
+        ] = item_id
+
         # Instantiate pystac item
         item = pystac.Item(
-            id=id, geometry=geom, bbox=bbox, datetime=time_acquired, **params
+            id=item_id, geometry=geom, bbox=bbox, datetime=time_acquired, **params
         )
 
         # Get the item info, from the raster path
@@ -319,6 +338,6 @@ class STACGenerator:
                             else:
                                 extension_obj = self._extensions_dict[extension]
                                 extension_obj.add_extension_to_object(asset, item_info)
-        item.set_self_href(join(dirname(raster_path), f"{id}.json"))
+        item.set_self_href(join(dirname(raster_path), f"{item_id}.json"))
         item.make_asset_hrefs_relative()
         return item

eotdl/datasets/__init__.py

@@ -1,3 +1,3 @@
 from .retrieve import retrieve_datasets  # , retrieve_dataset, list_datasets
 from .ingest import ingest_dataset
-from .download import download_dataset  # , download_file_url
+from .download import download_dataset, download_file_url

eotdl/datasets/download.py

@@ -4,8 +4,8 @@ from tqdm import tqdm
 
 from ..auth import with_auth
 from .retrieve import retrieve_dataset, retrieve_dataset_files
-from ..shared import calculate_checksum
-from ..repos import FilesAPIRepo
+from ..repos import FilesAPIRepo, DatasetsAPIRepo
+from ..curation.stac import STACDataFrame
 
 
 @with_auth
@@ -13,7 +13,7 @@ def download_dataset(
     dataset_name,
     version=None,
     path=None,
-    logger=None,
+    logger=print,
     assets=False,
     force=False,
     verbose=False,
@@ -45,20 +45,6 @@ def download_dataset(
     if dataset["quality"] == 0:
         if file:
             raise NotImplementedError("Downloading a specific file is not implemented")
-            # files = [f for f in dataset["files"] if f["name"] == file]
-            # if not files:
-            #     raise Exception(f"File {file} not found")
-            # if len(files) > 1:
-            #     raise Exception(f"Multiple files with name {file} found")
-            # dst_path = download(
-            #     dataset,
-            #     dataset["id"],
-            #     file,
-            #     files[0]["checksum"],
-            #     download_path,
-            #     user,
-            # )
-            # return Outputs(dst_path=dst_path)
         dataset_files = retrieve_dataset_files(dataset["id"], version)
         repo = FilesAPIRepo()
         for file in tqdm(dataset_files, disable=verbose, unit="file", position=0):
@@ -76,44 +62,45 @@ def download_dataset(
             # if calculate_checksum(dst_path) != checksum:
             #     logger(f"Checksum for {file} does not match")
             if verbose:
-                logger(f"Done")
-        return download_path
+                logger("Done")
     else:
-        raise NotImplementedError("Downloading a STAC dataset is not implemented")
-    #     logger("Downloading STAC metadata...")
-    #     gdf, error = repo.download_stac(
-    #         dataset["id"],
-    #         user["id_token"],
-    #     )
-    #     if error:
-    #         raise Exception(error)
-    #     df = STACDataFrame(gdf)
-    #     # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
-    #     path = path
-    #     if path is None:
-    #         path = download_base_path + "/" + dataset["name"]
-    #     df.to_stac(path)
-    #     # download assets
-    #     if assets:
-    #         logger("Downloading assets...")
-    #         df = df.dropna(subset=["assets"])
-    #         for row in tqdm(df.iterrows(), total=len(df)):
-    #             id = row[1]["stac_id"]
-    #             # print(row[1]["links"])
-    #             for k, v in row[1]["assets"].items():
-    #                 href = v["href"]
-    #                 repo.download_file_url(
-    #                     href, f"{path}/assets/{id}", user["id_token"]
-    #                 )
-    #     else:
-    #         logger("To download assets, set assets=True or -a in the CLI.")
-    #     return Outputs(dst_path=path)
+        # raise NotImplementedError("Downloading a STAC dataset is not implemented")
+        if verbose:
+            logger("Downloading STAC metadata...")
+        repo = DatasetsAPIRepo()
+        gdf, error = repo.download_stac(
+            dataset["id"],
+            user["id_token"],
+        )
+        if error:
+            raise Exception(error)
+        df = STACDataFrame(gdf)
+        # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
+        df.to_stac(download_path)
+        # download assets
+        if assets:
+            if verbose:
+                logger("Downloading assets...")
+            repo = FilesAPIRepo()
+            df = df.dropna(subset=["assets"])
+            for row in tqdm(df.iterrows(), total=len(df)):
+                for k, v in row[1]["assets"].items():
+                    href = v["href"]
+                    _, filename = href.split("/download/")
+                    # will overwrite assets with same name :(
+                    repo.download_file_url(
+                        href, filename, f"{download_path}/assets", user["id_token"]
+                    )
+        else:
+            if verbose:
+                logger("To download assets, set assets=True or -a in the CLI.")
+    return download_path
 
 
-# @with_auth
-# def download_file_url(url, path, progress=True, logger=None, user=None):
-#     api_repo = APIRepo()
-#     download = DownloadFileURL(api_repo, logger, progress)
-#     inputs = DownloadFileURL.Inputs(url=url, path=path, user=user)
-#     outputs = download(inputs)
-#     return outputs.dst_path
+@with_auth
+def download_file_url(url, path, progress=True, logger=print, user=None):
+    repo = FilesAPIRepo()
+    _, filename = url.split("/download/")
+    return repo.download_file_url(
+        url, filename, f"{path}/assets", user["id_token"], progress
+    )