eotdl 2023.11.2.post5__py3-none-any.whl → 2023.11.3.post2__py3-none-any.whl

eotdl/tools/geo_utils.py

@@ -1,9 +1,15 @@
+"""
+Geo Utils
+"""
+
+import tarfile
+from typing import Union
+from statistics import mean
+
 import geopandas as gpd
 import rasterio
 import rasterio.warp
-import tarfile
 
-from typing import Union
 from shapely import geometry
 from shapely.geometry import box, Polygon, shape
 from pyproj import Transformer
@@ -12,9 +18,12 @@ from pandas import isna
 
 
 def is_bounding_box(bbox: list) -> bool:
+    """
+    Check if the given bounding box is a bounding box and is valid
+    """
     if not isinstance(bbox, (list, tuple)) or len(bbox) != 4:
         return False
-    
+
     for value in bbox:
         if not isinstance(value, (int, float)):
             return False
@@ -27,13 +36,19 @@ def is_bounding_box(bbox: list) -> bool:
 
 
 def compute_image_size(bounding_box, parameters):
+    """
+    Compute the image size from the bounding box and the resolution
+    """
     bbox = BBox(bbox=bounding_box, crs=CRS.WGS84)
     bbox_size = bbox_to_dimensions(bbox, resolution=parameters.RESOLUTION)
 
-    return bbox, bbox_size  
+    return bbox, bbox_size
 
 
 def get_image_bbox(raster: Union[tarfile.ExFileObject, str]):
+    """
+    Get the bounding box of a raster
+    """
     with rasterio.open(raster) as src:
         bounds = src.bounds
         dst_crs = "EPSG:4326"
@@ -45,6 +60,9 @@ def get_image_bbox(raster: Union[tarfile.ExFileObject, str]):
 
 
 def get_image_resolution(raster: Union[tarfile.ExFileObject, str]):
+    """
+    Get the resolution of a raster
+    """
     with rasterio.open(raster) as src:
         resolution = src.res
     return resolution
@@ -52,13 +70,14 @@ def get_image_resolution(raster: Union[tarfile.ExFileObject, str]):
 
 def bbox_to_coordinates(bounding_box: list) -> list:
     """
+    Convert a bounding box to a list of polygon coordinates
     """
     polygon_coordinates = [
         (bounding_box[0], bounding_box[1]),  # bottom left
         (bounding_box[0], bounding_box[3]),  # top left
         (bounding_box[2], bounding_box[3]),  # top right
         (bounding_box[2], bounding_box[1]),  # bottom right
-        (bounding_box[0], bounding_box[1])   # back to bottom left
+        (bounding_box[0], bounding_box[1]),  # back to bottom left
     ]
 
     return polygon_coordinates
@@ -66,22 +85,24 @@ def bbox_to_coordinates(bounding_box: list) -> list:
 
 def bbox_to_polygon(bounding_box: list) -> Polygon:
     """
+    Convert a bounding box to a shapely polygon
     """
     polygon = box(bounding_box[0], bounding_box[1], bounding_box[2], bounding_box[3])
 
     return polygon
 
 
-from_4326_transformer = Transformer.from_crs('EPSG:4326', 'EPSG:3857')
-from_3857_transformer = Transformer.from_crs('EPSG:3857', 'EPSG:4326')
+from_4326_transformer = Transformer.from_crs("EPSG:4326", "EPSG:3857")
+from_3857_transformer = Transformer.from_crs("EPSG:3857", "EPSG:4326")
 
 
-def bbox_from_centroid(x: Union[int, float], 
-                       y: Union[int, float],
-                       pixel_size: Union[int, float],
-                       width: Union[int, float],
-                       height: Union[int, float]
-                       ) -> list:
+def bbox_from_centroid(
+    x: Union[int, float],
+    y: Union[int, float],
+    pixel_size: Union[int, float],
+    width: Union[int, float],
+    height: Union[int, float],
+) -> list:
     """
     Generate a bounding box from a centroid, pixel size and image dimensions.
 
@@ -122,9 +143,7 @@ def bbox_from_centroid(x: Union[int, float],
     return [min_y, min_x, max_y, max_x]
 
 
-def generate_bounding_box(geom: geometry.point.Point, 
-                          differences: list
-                          ) -> list:
+def generate_bounding_box(geom: geometry.point.Point, differences: list) -> list:
     """
     Generate the bounding box of a given point using the difference
     between the maximum and mininum coordinates of the bounding box
@@ -137,12 +156,14 @@ def generate_bounding_box(geom: geometry.point.Point,
     """
     long_diff, lat_diff = differences[0], differences[1]
     lon, lat = geom.x, geom.y
-    
-    bbox = (lon - (long_diff/2), 
-            lat - (lat_diff/2), 
-            lon + (long_diff/2), 
-            lat + (lat_diff/2))
-    
+
+    bbox = (
+        lon - (long_diff / 2),
+        lat - (lat_diff / 2),
+        lon + (long_diff / 2),
+        lat + (lat_diff / 2),
+    )
+
     # Round the coordinates to 6 decimals
     bounding_box = [round(i, 6) for i in bbox]
 
@@ -151,17 +172,18 @@ def generate_bounding_box(geom: geometry.point.Point,
 
 def calculate_average_coordinates_distance(bounding_box_by_location: dict) -> list:
     """
-    Calculate the mean distance between maximum and minixum longitude and latitude of the bounding boxes
-    from the existing locations. This is intended to use these mean distance to generate the bounding 
+    Calculate the mean distance between maximum and minixum longitude
+    and latitude of the bounding boxes from the existing locations.
+    This is intended to use these mean distance to generate the bounding
     boxes of the new locations given a centroid.
 
-    :param bounding_box_by_location: dictionary with format location_id : bounding_box for the existing
-            locations in the sen12floods dataset.
-    :return mean_long_diff, mean_lat_diff: mean longitude and latitude difference in the bounding boxes
+    :param bounding_box_by_location: dictionary with format
+    location_id : bounding_box for the existing locations in
+    the sen12floods dataset.
+    :return mean_long_diff, mean_lat_diff: mean longitude
+    and latitude difference in the bounding boxes
     """
-    from statistics import mean
-
-    long_diff_list, lat_diff_list = list(), list()
+    long_diff_list, lat_diff_list = [], []
 
     for bbox in bounding_box_by_location.values():
         long_diff = bbox[2] - bbox[0]
@@ -175,28 +197,36 @@ def calculate_average_coordinates_distance(bounding_box_by_location: dict) -> li
     return mean_long_diff, mean_lat_diff
 
 
-def generate_new_locations_bounding_boxes(gdf: gpd.GeoDataFrame,
-                                          mean_differences: list,
-                                          latest_id: int
-                                          ) -> dict:
+def generate_new_locations_bounding_boxes(
+    gdf: gpd.GeoDataFrame, mean_differences: list, latest_id: int
+) -> dict:
     """
-    Generate the bounding box of every new location, using the mean difference between the maximum and
-    minimum calculated longitude and latitude. This function also returns the time interval which we
-    want to request from Sentinel Hub Services.
-
-    :param gdf: GeoDataFrame wiht the new locations that are going to be added to the dataset
-    :param mean_differences: list with the longitude and latitude mean differences, which are going to be used
-            to generate the bounding boxes.
-    :return: bbox_by_new_location: dict with format {<location_id>: {'bounding_box': list(), 'time_interval': list()}, ... }
-            that contains the bounding box and time interval of the imagery for each location
+    Generate the bounding box of every new location, using
+    the mean difference between the maximum and minimum calculated
+    longitude and latitude. This function also returns the time
+    interval which we want to request from Sentinel Hub Services.
+
+    :param gdf: GeoDataFrame wiht the new locations that
+    are going to be added to the dataset
+    :param mean_differences: list with the longitude
+    and latitude mean differences, which are going to be used to generate
+    the bounding boxes.
+    :return: bbox_by_new_location: dict with format {<location_id>:
+    {'bounding_box': list(), 'time_interval': list()}, ... }
+    that contains the bounding box and time interval of the imagery for each location
     """
-    bbox_by_new_location = dict()
+    bbox_by_new_location = {}
 
-    for i, row in gdf.iterrows():
+    for _, row in gdf.iterrows():
         new_location_id = str(latest_id + 1)
-        time_interval = row['Began'].strftime("%Y-%m-%d"), row['Ended'].strftime("%Y-%m-%d")
-        bbox = generate_bounding_box(row['geometry'], mean_differences)
-        bbox_by_new_location[new_location_id] = {'bounding_box': bbox, 'time_interval': time_interval}
+        time_interval = row["Began"].strftime("%Y-%m-%d"), row["Ended"].strftime(
+            "%Y-%m-%d"
+        )
+        bbox = generate_bounding_box(row["geometry"], mean_differences)
+        bbox_by_new_location[new_location_id] = {
+            "bounding_box": bbox,
+            "time_interval": time_interval,
+        }
         latest_id += 1
 
     return bbox_by_new_location

eotdl/tools/metadata.py

@@ -1,6 +1,6 @@
-'''
+"""
 Metadata utilities for STAC
-'''
+"""
 
 import json
 
@@ -20,25 +20,27 @@ def get_item_metadata(raster_path: str) -> str:
     raster_dir_path = dirname(raster_path)
     # Get the metadata JSON file
     # Check if there is a metadata.json file in the directory
-    if 'metadata.json' in listdir(raster_dir_path):
-        metadata_json = join(raster_dir_path, 'metadata.json')
+    if "metadata.json" in listdir(raster_dir_path):
+        metadata_json = join(raster_dir_path, "metadata.json")
     else:
         # If there is no metadata.json file in the directory, check if there is
         # a json file with the same name as the raster file
         base = splitext(raster_path)[0]
-        metadata_json = base + '.json'
+        metadata_json = base + ".json"
         if not exists(metadata_json):
             # If there is no metadata file in the directory, return None
             return None
-    
+
     # Open the metadata file and return it
-    with open(metadata_json, 'r') as f:
+    with open(metadata_json, "r", encoding="utf-8") as f:
         metadata = json.load(f)
-    
+
     return metadata
 
 
-def remove_raster_metadata(folder: str, metadata_file: Optional[str] = 'metadata.json') -> None:
+def remove_raster_metadata(
+    folder: str, metadata_file: Optional[str] = "metadata.json"
+) -> None:
     """
     Remove metadata.json file from a folder
 
@@ -48,5 +50,5 @@ def remove_raster_metadata(folder: str, metadata_file: Optional[str] = 'metadata
     # Search for all the metadata files in the folder
     metadata_files = glob(join(folder, "**", metadata_file), recursive=True)
     # Remove all the metadata files
-    for metadata_file in metadata_files:
-        remove(metadata_file)
+    for metadata in metadata_files:
+        remove(metadata)

eotdl/tools/paths.py

@@ -1,6 +1,6 @@
-'''
+"""
 Paths utils
-'''
+"""
 
 from os.path import dirname
 from typing import Union, Optional
@@ -28,24 +28,24 @@ def cut_images(images_list: Union[list, tuple]) -> list:
 
     :return: list of unique directories
     """
-    dirnames = list()
-    images = list()
+    dirnames = []
+    images = []
 
     for image in images_list:
-        dir = dirname(image)
-        if dir not in dirnames:
-            dirnames.append(dir)
+        directory = dirname(image)
+        if directory not in dirnames:
+            dirnames.append(directory)
             images.append(image)
 
     return images
 
 
-def get_all_images_in_path(path: str, image_format: Optional[str] = 'tif') -> list:
-        """
-        Get all the images in a directory
+def get_all_images_in_path(path: str, image_format: Optional[str] = "tif") -> list:
+    """
+    Get all the images in a directory
 
-        :param path: path to the directory
+    :param path: path to the directory
 
-        :return: list of images
-        """
-        return glob(str(path) + f'/**/*.{image_format}', recursive=True)
+    :return: list of images
+    """
+    return glob(str(path) + f"/**/*.{image_format}", recursive=True)

eotdl/tools/stac.py

@@ -2,21 +2,22 @@
 Module for data engineering with STAC elements
 """
 
-import geopandas as gpd
-import pystac
-
 from os.path import dirname, join, abspath
 from os import makedirs
 from json import dumps
 from typing import Union, Optional
-from tqdm import tqdm
-from traceback import print_exc
 from shutil import rmtree
+from traceback import print_exc
+
+import geopandas as gpd
+import pystac
+
+from tqdm import tqdm
 
 
 def stac_items_to_gdf(items: pystac.ItemCollection) -> gpd.GeoDataFrame:
     """
-    Get a GeoDataFrame from a given pystac.ItemCollection. 
+    Get a GeoDataFrame from a given pystac.ItemCollection.
 
     :param: items: A pystac.ItemCollection
     :return: GeoDataframe from the given ItemCollection
@@ -29,12 +30,12 @@ def stac_items_to_gdf(items: pystac.ItemCollection) -> gpd.GeoDataFrame:
         if f not in features:
             # Add all the keys in the properties dict as columns in the GeoDataFrame
             for k, v in f.items():
-                if k not in f['properties'] and k != 'geometry':
-                    f['properties'][k] = v
-            if 'scene_id' in f['properties']:
-                f['properties']['scene_id'] = f['id'].split('_')[3]
-            features.append(f)       
-    
+                if k not in f["properties"] and k != "geometry":
+                    f["properties"][k] = v
+            if "scene_id" in f["properties"]:
+                f["properties"]["scene_id"] = f["id"].split("_")[3]
+            features.append(f)
+
     return gpd.GeoDataFrame.from_features(features)
 
 
@@ -65,9 +66,10 @@ def get_all_children(obj: pystac.STACObject) -> list:
     return children
 
 
-def make_links_relative_to_path(path: str,
-                                catalog: Union[pystac.Catalog, str],
-                                ) -> pystac.Catalog:
+def make_links_relative_to_path(
+    path: str,
+    catalog: Union[pystac.Catalog, str],
+) -> pystac.Catalog:
     """
     Makes all asset HREFs and links in the STAC catalog relative to a given path
     """
@@ -76,9 +78,9 @@ def make_links_relative_to_path(path: str,
     path = abspath(path)
 
     # Create a temporary catalog in the destination path to set as root
-    future_path = join(path, 'catalog.json')
+    future_path = join(path, "catalog.json")
     makedirs(path, exist_ok=True)
-    with open(future_path, 'w') as f:
+    with open(future_path, "w", encoding="utf-8") as f:
         f.write(dumps(catalog.to_dict(), indent=4))
     temp_catalog = pystac.Catalog.from_file(future_path)
 
@@ -88,7 +90,7 @@ def make_links_relative_to_path(path: str,
     for collection in catalog.get_children():
         # Create new collection
         new_collection = collection.clone()
-        new_collection.set_self_href(join(path, collection.id, f"collection.json"))
+        new_collection.set_self_href(join(path, collection.id, "collection.json"))
         new_collection.set_root(catalog)
         new_collection.set_parent(catalog)
         # Remove old collection and add new one to catalog
@@ -97,7 +99,9 @@ def make_links_relative_to_path(path: str,
         for item in collection.get_all_items():
             # Create new item from old collection and add it to the new collection
             new_item = item.clone()
-            new_item.set_self_href(join(path, collection.id, item.id, f"{item.id}.json"))
+            new_item.set_self_href(
+                join(path, collection.id, item.id, f"{item.id}.json")
+            )
             new_item.set_parent(collection)
             new_item.set_root(catalog)
             new_item.make_asset_hrefs_relative()
@@ -108,12 +112,13 @@ def make_links_relative_to_path(path: str,
     return catalog
 
 
-def merge_stac_catalogs(catalog_1: Union[pystac.Catalog, str],
-                        catalog_2: Union[pystac.Catalog, str],
-                        destination: Optional[str] = None,
-                        keep_extensions: Optional[bool] = False,
-                        catalog_type: Optional[pystac.CatalogType] = pystac.CatalogType.SELF_CONTAINED
-                        ) -> None:
+def merge_stac_catalogs(
+    catalog_1: Union[pystac.Catalog, str],
+    catalog_2: Union[pystac.Catalog, str],
+    destination: Optional[str] = None,
+    keep_extensions: Optional[bool] = False,
+    catalog_type: Optional[pystac.CatalogType] = pystac.CatalogType.SELF_CONTAINED,
+) -> None:
     """
     Merge two STAC catalogs, keeping the properties, collection and items of both catalogs
     """
@@ -122,10 +127,10 @@ def merge_stac_catalogs(catalog_1: Union[pystac.Catalog, str],
     if isinstance(catalog_2, str):
         catalog_2 = pystac.Catalog.from_file(catalog_2)
 
-    for col1 in tqdm(catalog_1.get_children(), desc='Merging catalogs...'):
+    for col1 in tqdm(catalog_1.get_children(), desc="Merging catalogs..."):
         # Check if the collection exists in catalog_2
         col2 = catalog_2.get_child(col1.id)
-        if col2 is None:
+        if not col2:
             # If it does not exist, add it
             col1_ = col1.clone()
             catalog_2.add_child(col1)
@@ -157,10 +162,10 @@ def merge_stac_catalogs(catalog_1: Union[pystac.Catalog, str],
     try:
         print("Validating and saving...")
         catalog_2.validate()
-        rmtree(destination) if not destination else None # Remove the old catalog and replace it with the new one
-        catalog_2.normalize_and_save(root_href=destination, 
-                                     catalog_type=catalog_type
-                                     )
+        rmtree(
+            destination
+        ) if not destination else None  # Remove the old catalog and replace it with the new one
+        catalog_2.normalize_and_save(root_href=destination, catalog_type=catalog_type)
         print("Success!")
     except pystac.STACValidationError:
         # Return full callback

eotdl/tools/time_utils.py

@@ -1,14 +1,22 @@
-import geopandas as gpd
-import pandas as pd
+"""
+Time utils
+"""
 
 from datetime import datetime, timedelta
-from typing import Union, Optional, List
+from typing import Union, Optional
+from dateutil import parser
+
+import geopandas as gpd
+import pandas as pd
 
 
 def is_time_interval(time_interval: list) -> bool:
+    """
+    Check if is time interval and is valid
+    """
     if not isinstance(time_interval, (list, tuple)) or len(time_interval) != 2:
         return False
-    
+
     for value in time_interval:
         if not isinstance(value, str):
             return False
@@ -19,6 +27,9 @@ def is_time_interval(time_interval: list) -> bool:
 
 
 def is_valid_date(date_str: str) -> bool:
+    """
+    Check if a date is valid
+    """
     try:
         datetime.strptime(date_str, "%Y-%m-%d")
         return True
@@ -26,24 +37,29 @@ def is_valid_date(date_str: str) -> bool:
         return False
 
 
-def get_first_last_dates(dataframe: Union[pd.DataFrame, gpd.GeoDataFrame], dates_column: Optional[str] = 'datetime'):
+def get_first_last_dates(
+    dataframe: Union[pd.DataFrame, gpd.GeoDataFrame],
+    dates_column: Optional[str] = "datetime",
+):
     """
+    Get first and last dates from a dataframe
     """
     dataframe[dates_column] = dataframe[dates_column].apply(lambda x: sorted(x))
-    dataframe['first_date'] = dataframe['dates_list'].apply(lambda x: x[0])
-    dataframe['last_date'] = dataframe['dates_list'].apply(lambda x: x[-1])
-    dataframe = dataframe.sort_values(by=['first_date', 'last_date'])
+    dataframe["first_date"] = dataframe["dates_list"].apply(lambda x: x[0])
+    dataframe["last_date"] = dataframe["dates_list"].apply(lambda x: x[-1])
+    dataframe = dataframe.sort_values(by=["first_date", "last_date"])
     # Sort by sequence id
-    dataframe = dataframe.sort_values(by=['location_id'])
+    dataframe = dataframe.sort_values(by=["location_id"])
     # Convert first_date and last_date to datetime, in format YYYY-MM-DD
-    dataframe['first_date'] = pd.to_datetime(dataframe['first_date'], format='%Y-%m-%d')
-    dataframe['last_date'] = pd.to_datetime(dataframe['last_date'], format='%Y-%m-%d')
+    dataframe["first_date"] = pd.to_datetime(dataframe["first_date"], format="%Y-%m-%d")
+    dataframe["last_date"] = pd.to_datetime(dataframe["last_date"], format="%Y-%m-%d")
 
     return dataframe
 
 
 def create_time_slots(start_date: datetime, end_date: datetime, n_chunks: int):
     """
+    Create time slots from start date to end date, with n_chunks
     """
     if isinstance(start_date, str):
         start_date = datetime.strptime(start_date, "%Y-%m-%d")
@@ -56,16 +72,19 @@ def create_time_slots(start_date: datetime, end_date: datetime, n_chunks: int):
     return slots
 
 
-def expand_time_interval(time_interval: Union[list, tuple], format: str='%Y-%m-%dT%H:%M:%S.%fZ') -> list:
+def expand_time_interval(
+    time_interval: Union[list, tuple], time_format: str = "%Y-%m-%dT%H:%M:%S.%fZ"
+) -> list:
     """
+    Expand time interval to get more data
     """
     start_date = time_interval[0]
     end_date = time_interval[1]
 
     if isinstance(start_date, str):
-        start_date = datetime.datetime.strptime(start_date, format)
+        start_date = datetime.datetime.strptime(start_date, time_format)
     if isinstance(end_date, str):
-        end_date = datetime.datetime.strptime(end_date, format)
+        end_date = datetime.datetime.strptime(end_date, time_format)
 
     # Add one day to start date and remove one day to end date
     new_start_date = start_date - datetime.timedelta(days=1)
@@ -79,15 +98,22 @@ def expand_time_interval(time_interval: Union[list, tuple], format: str='%Y-%m-%
 
 
 def prepare_time_interval(date):
+    """
+    Prepare time interval to request data
+    """
     if isinstance(date, str):
         date = datetime.strptime(date, "%Y-%m-%d")
     elif isinstance(date, tuple):
-        if not is_time_interval(date):
-            raise ValueError('The time interval must be a range of two dates, with format YYYY-MM-DD or a datetime object')
-        else:
+        if is_time_interval(date):
             return date
+        else:
+            raise ValueError(
+                "The time interval must be a range of two dates, with format YYYY-MM-DD or a datetime object"
+            )
     elif not isinstance(date, datetime):
-        raise ValueError('The date must be a string with format YYYY-MM-DD or a datetime object')
+        raise ValueError(
+            "The date must be a string with format YYYY-MM-DD or a datetime object"
+        )
 
     date_day_before = date - timedelta(days=1)
     date_next_day = date + timedelta(days=1)
@@ -98,10 +124,11 @@ def prepare_time_interval(date):
     return (date_day_before_str, date_next_day_str)
 
 
-def get_day_between(from_date: Union[datetime, str], 
-                    to_date: Union[datetime, str]
-                    ) -> str:
+def get_day_between(
+    from_date: Union[datetime, str], to_date: Union[datetime, str]
+) -> str:
     """
+    Get the day between two dates
     """
     if isinstance(from_date, str):
         from_date = datetime.strptime(from_date, "%Y-%m-%dT%H:%M:%SZ")
@@ -109,8 +136,8 @@ def get_day_between(from_date: Union[datetime, str],
         to_date = datetime.strptime(to_date, "%Y-%m-%dT%H:%M:%SZ")
 
     date_between = from_date + timedelta(days=1)
-    date_between = date_between.strftime("%Y-%m-%d") 
-    
+    date_between = date_between.strftime("%Y-%m-%d")
+
     return date_between
 
 
@@ -120,8 +147,8 @@ def format_time_acquired(dt: Union[str, datetime]) -> str:
 
     :param dt: date time to format
     """
-    from dateutil import parser
-    
     dt_str = parser.parse(dt).strftime("%Y-%m-%dT%H:%M:%S.%f")
+    # convert to datetime object
+    dt = datetime.strptime(dt_str, "%Y-%m-%dT%H:%M:%S.%f")
 
-    return dt_str
+    return dt