eotdl 2023.7.19.post3__py3-none-any.whl → 2023.9.14__py3-none-any.whl

eotdl/commands/datasets.py

@@ -4,8 +4,7 @@ from pathlib import Path
 from ..datasets import (
     retrieve_datasets,
     download_dataset,
-    ingest_folder,
-    ingest_stac,
+    ingest_dataset,
 )
 
 app = typer.Typer()
@@ -14,31 +13,21 @@ app = typer.Typer()
 @app.command()
 def ingest(
     path: Path,
-    f: bool = typer.Option(False, "--f", help="Force ingest even if file exists"),
-    d: bool = typer.Option(False, "--d", help="Delete files not in the dataset"),
+    f: bool = typer.Option(
+        False, "--force", "-f", help="Force ingest even if file exists"
+    ),
+    d: bool = typer.Option(
+        False, "--delete", "-d", help="Delete files not in the dataset"
+    ),
 ):
-    """
-    Ingest a dataset
-
-    path: Path to folder with the dataset
-    """
     try:
-        if not path.is_dir():
-            typer.echo("Path must be a folder")
-            return
-        if "catalog.json" in [f.name for f in path.iterdir()]:
-            ingest_stac(str(path) + "/catalog.json", typer.echo)
-        else:
-            ingest_folder(path, f, d, typer.echo)
+        ingest_dataset(path, f, d, typer.echo)
     except Exception as e:
         typer.echo(e)
 
 
 @app.command()
 def list():
-    """
-    List all datasets and files
-    """
     datasets = retrieve_datasets()
     typer.echo(datasets)
 
@@ -46,18 +35,15 @@ def list():
 @app.command()
 def get(
     dataset: str,
-    path: str = None,
-    file: str = None,
+    path: Path = typer.Option(None, "--path", "-p", help="Download to a specific path"),
+    file: bool = typer.Option(None, "--file", "-f", help="Download a specific file"),
+    assets: bool = typer.Option(False, "--assets", "-a", help="Download assets"),
+    force: bool = typer.Option(
+        False, "--force", "-f", help="Force download even if file exists"
+    ),
 ):
-    """
-    Download a dataset
-
-    dataset: Name of the dataset
-    file: Name of the file to download (optional, if not provided, the whole dataset will be downloaded)
-    path: Path to download the dataset to (optional, if not provided, the dataset will be downloaded to ~/.eotdl/datasets)
-    """
     try:
-        dst_path = download_dataset(dataset, file, path, typer.echo)
+        dst_path = download_dataset(dataset, file, path, typer.echo, assets, force)
         typer.echo(f"Data available at {dst_path}")
     except Exception as e:
         typer.echo(e)

eotdl/curation/__init__.py

@@ -1,6 +1,6 @@
-# from .stac.dataframe import STACDataFrame  # , read_stac
+from .stac.dataframe import STACDataFrame  # , read_stac
 
-# from .stac.stac import STACGenerator
-# from .formatters import SHFolderFormatter
-# from .stac.utils import format_time_acquired
-# from .stac.parsers import STACIdParser, StructuredParser, UnestructuredParser
+from .stac.stac import STACGenerator, merge_stac_catalogs
+from .formatters import SHFolderFormatter
+from .stac.utils import format_time_acquired
+from .stac.parsers import STACIdParser, StructuredParser, UnestructuredParser

eotdl/curation/formatters.py

@@ -2,8 +2,6 @@
 Module for formatter classes
 """
 
-import rasterio
-
 from os.path import join, exists
 from os import listdir, mkdir
 from glob import glob

eotdl/curation/metadata.py

@@ -6,13 +6,16 @@ import datetime
 import json
 import rasterio
 from rasterio.warp import transform_bounds
+from typing import Union, Optional
 
+from os import remove
+from glob import glob
 from os.path import dirname, join, exists
 
 
 def generate_raster_metadata(raster_path: str,
                              output_folder: str,
-                             date_adquired: str|datetime.datetime
+                             date_adquired: Union[str, datetime.datetime]
                              ) -> None:
     """
     Generate metadata.json file for a raster file
@@ -23,8 +26,10 @@ def generate_raster_metadata(raster_path: str,
     """
     with rasterio.open(raster_path) as ds:
         bounds = ds.bounds
-        dst_crs = 'EPSG:4326'  # EPSG identifier for WGS84 coordinate system used by the geojson format
-        left, bottom, right, top = rasterio.warp.transform_bounds(ds.crs, dst_crs, *bounds)
+        dst_crs = "EPSG:4326"  # EPSG identifier for WGS84 coordinate system used by the geojson format
+        left, bottom, right, top = rasterio.warp.transform_bounds(
+            ds.crs, dst_crs, *bounds
+        )
         bbox = [left, bottom, right, top]
 
     # Get raster directory path to get the request.json file
@@ -32,12 +37,32 @@ def generate_raster_metadata(raster_path: str,
 
     # Read the request.json file and get the request data type
     if exists(raster_dir_path):
-        with open(join(raster_dir_path, 'request.json'), 'r') as f:
+        with open(join(raster_dir_path, "request.json"), "r") as f:
             request = json.load(f)
-            request_data_type = request['request']['payload']['input']['data'][0]['type']
+            request_data_type = request["request"]["payload"]["input"]["data"][0][
+                "type"
+            ]
 
-    metadata_path = join(output_folder, 'metadata.json')
-    metadata = {'date-adquired': date_adquired, 'bounding-box': bbox, 'type': request_data_type}
-    
-    with open(metadata_path, 'w') as f:
+    metadata_path = join(output_folder, "metadata.json")
+    metadata = {
+        "date-adquired": date_adquired,
+        "bounding-box": bbox,
+        "type": request_data_type,
+    }
+
+    with open(metadata_path, "w") as f:
         json.dump(metadata, f)
+
+
+def remove_raster_metadata(folder: str, metadata_file: Optional[str] = 'metadata.json') -> None:
+    """
+    Remove metadata.json file from a folder
+
+    :param folder: folder path
+    :param metadata_file: metadata file name
+    """
+    # Search for all the metadata files in the folder
+    metadata_files = glob(join(folder, "**", metadata_file), recursive=True)
+    # Remove all the metadata files
+    for metadata_file in metadata_files:
+        remove(metadata_file)

eotdl/curation/stac/assets.py

@@ -0,0 +1,127 @@
+'''
+Module for STAC Asset Generators
+'''
+
+from os import remove, listdir
+from os.path import dirname, join, basename
+from ..metadata import remove_raster_metadata
+from pathlib import Path
+
+import pandas as pd
+import rasterio
+import pystac
+
+
+MEDIA_TYPES_DICT = {
+    'tif': pystac.MediaType.GEOTIFF,
+    'tiff': pystac.MediaType.GEOTIFF,
+    'png': pystac.MediaType.PNG,
+    'jpg': pystac.MediaType.JPEG,
+    'jpeg': pystac.MediaType.JPEG,
+}
+
+
+class STACAssetGenerator:
+
+    type = 'None'
+    
+    def __init__(self):
+        pass
+
+    @classmethod
+    def extract_assets(self, obj_info: pd.DataFrame):
+        """
+        Extract the assets from the raster file
+
+        :param raster_path: path to the raster file
+        """
+        # If there is no bands, create a single band asset from the file, assuming thats a singleband raster
+        raster_path = obj_info["image"].values[0]
+        title = basename(raster_path).split('.')[0]
+        # Get the file extension
+        raster_format = raster_path.split('.')[-1]
+        asset = pystac.Asset(href=Path(raster_path).as_posix(), 
+                             title=title, 
+                             media_type=MEDIA_TYPES_DICT[raster_format], 
+                             roles=['data'])
+
+        return [asset]
+
+
+class BandsAssetGenerator(STACAssetGenerator):
+
+    type = 'Bands'
+
+    def __init__(self) -> None:
+        super().__init__()
+    
+    def extract_assets(self, obj_info: pd.DataFrame):
+        """
+        Extract the assets from the raster file from the bands column
+
+        :param raster_path: path to the raster file
+        """
+        asset_list = []
+        # File path
+        raster_path = obj_info["image"].values[0]
+        # Bands
+        bands = obj_info["bands"].values
+        bands = bands[0] if bands else None
+
+        if bands:
+            with rasterio.open(raster_path, 'r') as raster:
+                if isinstance(bands, str):
+                    bands = [bands]
+                for band in bands:
+                    i = bands.index(band)
+                    raster_format = raster_path.split('.')[-1]   # Will be used later to save the bands files
+                    try:
+                        single_band = raster.read(i + 1)
+                    except IndexError:
+                        single_band = raster.read(1)
+                    band_name = f'{band}.{raster_format}'
+                    output_band = join(dirname(raster_path), band_name)
+                    # Copy the metadata
+                    metadata = raster.meta.copy()
+                    metadata.update({"count": 1})
+                    # Write the band to the output folder
+                    with rasterio.open(output_band, "w", **metadata) as dest:
+                        dest.write(single_band, 1)
+                    # Instantiate pystac asset and append it to the list
+                    asset_list.append(pystac.Asset(href=output_band, 
+                                                   title=band, 
+                                                   media_type=MEDIA_TYPES_DICT[raster_format]))
+
+            # Remove the original raster file and its metadata
+            remove(raster_path)
+            remove_raster_metadata(dirname(raster_path))
+
+            return asset_list
+
+
+class ExtractedAssets(STACAssetGenerator):
+
+    type = 'Extracted'
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def extract_assets(self, obj_info: pd.DataFrame):
+        """
+        Get all the files with the same extension as the image file as assets
+        """
+        asset_list = []
+        # File path
+        raster_path = obj_info["image"].values[0]
+        raster_dir = dirname(raster_path)
+        # Get the files with the same extension as the image file
+        files = [f for f in listdir(raster_dir) if f.endswith(raster_path.split('.')[-1])]
+        # Instantiate pystac asset and append it to the list
+        for file in files:
+            # Get the file extension
+            raster_format = file.split('.')[-1]
+            asset_list.append(pystac.Asset(href=join(raster_dir, file), 
+                                           title=basename(file), 
+                                           media_type=MEDIA_TYPES_DICT[raster_format]))
+
+        return asset_list

eotdl/curation/stac/dataframe.py

@@ -9,9 +9,11 @@ import json
 from geomet import wkt
 from os.path import join
 from os import makedirs
-from typing import Union
+from typing import Union, Optional
+
 from math import isnan
 from .utils import convert_df_geom_to_shape, get_all_children
+from pathlib import Path
 
 
 class STACDataFrame(gpd.GeoDataFrame):
@@ -19,9 +21,11 @@ class STACDataFrame(gpd.GeoDataFrame):
         super().__init__(*args, **kwargs)
 
     @classmethod
-    def from_stac_file(self, stac_file):
+    def from_stac_file(self, stac_file: pystac.STACObject):
         """
         Create a STACDataFrame from a STAC file
+
+        :param stac_file: STAC file
         """
         return read_stac(stac_file)
 
@@ -120,7 +124,7 @@ class STACDataFrame(gpd.GeoDataFrame):
 
 def read_stac(
     stac_file: Union[pystac.Catalog, pystac.Collection, str],
-    geometry_column: str = "geometry",
+    geometry_column: Optional[str] = "geometry",
 ) -> STACDataFrame:
     """
     Read a STAC file and return a STACDataFrame
@@ -128,7 +132,7 @@ def read_stac(
     :param stac_file: STAC file to read
     :param geometry_column: name of the geometry column
     """
-    if isinstance(stac_file, str):
+    if isinstance(stac_file, str) or isinstance(stac_file, Path):
         stac_file = pystac.read_file(stac_file)
     children = get_all_children(stac_file)