eotdl 2025.2.10__py3-none-any.whl → 2025.4.2__py3-none-any.whl

eotdl/access/sentinelhub/parameters.py

@@ -2,15 +2,27 @@
 Utils
 """
 
-from sentinelhub import DataCollection, MosaickingOrder
-
+from sentinelhub import DataCollection, MosaickingOrder, MimeType
 from .evalscripts import EvalScripts
 
 
+class OUTPUT_FORMAT:
+    TIFF = MimeType.TIFF
+    JPG = MimeType.JPG
+    PNG = MimeType.PNG
+
+
 class SHParameters:
     """
     Sentinel Hub Parameters base class
     """
+
+    MAX_CLOUD_COVERAGE: float = None
+    FIELDS = None
+    MOSAICKING_ORDER = MosaickingOrder.MOST_RECENT
+    EVALSCRIPT = None
+    OUTPUT_FORMAT = MimeType.TIFF
+
     def __init__(self):
         pass
 
@@ -19,8 +31,8 @@ class SHS2L2AParameters(SHParameters):
     """
     Sentinel-2-L2A parameters
     """
+
     DATA_COLLECTION = DataCollection.SENTINEL2_L2A
-    RESOLUTION = 10
     MOSAICKING_ORDER = MosaickingOrder.LEAST_CC
     EVALSCRIPT = EvalScripts.SENTINEL_2_L2A
     FIELDS = {
@@ -28,14 +40,17 @@ class SHS2L2AParameters(SHParameters):
         "exclude": [],
     }
     FILTER = None
+    RESOLUTION = 10
+    BASE_URL = "https://services.sentinel-hub.com"
+    CLOUD_COVERAGE = True
 
 
 class SHS2L1CParameters(SHParameters):
     """
     Sentinel-2-L1C parameters
     """
+
     DATA_COLLECTION = DataCollection.SENTINEL2_L1C
-    RESOLUTION = 10
     MOSAICKING_ORDER = MosaickingOrder.LEAST_CC
     EVALSCRIPT = EvalScripts.SENTINEL_2_L1C
     FIELDS = {
@@ -43,47 +58,110 @@ class SHS2L1CParameters(SHParameters):
         "exclude": [],
     }
     FILTER = None
+    RESOLUTION = 10
+    BASE_URL = "https://services.sentinel-hub.com"
+    CLOUD_COVERAGE = True
+
 
 class SHS1Parameters(SHParameters):
     """
     Sentinel-1 parameters
     """
+
     DATA_COLLECTION = DataCollection.SENTINEL1
-    RESOLUTION = 3
     EVALSCRIPT = EvalScripts.SENTINEL_1
-    MOSAICKING_ORDER = None
+    MOSAICKING_ORDER = MosaickingOrder.MOST_RECENT
     FIELDS = {
-            "include": [
-                "id",
-                "properties.datetime",
-                "sar:instrument_mode",
-                "s1:polarization",
-                "sat:orbit_state",
-                "s1:resolution",
-                "s1:timeliness",
-            ],
-            "exclude": [],
-        }
+        "include": [
+            "id",
+            "properties.datetime",
+            "sar:instrument_mode",
+            "s1:polarization",
+            "sat:orbit_state",
+            "s1:resolution",
+            "s1:timeliness",
+        ],
+        "exclude": [],
+    }
     FILTER = None
+    RESOLUTION = 3
+    BASE_URL = "https://services.sentinel-hub.com"
+    CLOUD_COVERAGE = False
 
 
 class SHDEMParameters(SHParameters):
     """
     Copernicus DEM parameters
     """
+
     DATA_COLLECTION = DataCollection.DEM_COPERNICUS_30
-    RESOLUTION = 3
     MOSAICKING_ORDER = None
     EVALSCRIPT = EvalScripts.DEM
+    FIELDS = None
+    FILTER = None
+    RESOLUTION = 3
+    BASE_URL = "https://services.sentinel-hub.com"
+    CLOUD_COVERAGE = False
+
+
+class SHHarmonizedLandsatSentinel(SHParameters):
+    """
+    Harmonized Landsat Sentinel parameters
+    """
+
+    DATA_COLLECTION = DataCollection.HARMONIZED_LANDSAT_SENTINEL
+    MOSAICKING_ORDER = MosaickingOrder.LEAST_CC
+    EVALSCRIPT = EvalScripts.HLS_TRUE_COLOR
+    FIELDS = None
     FILTER = None
+    RESOLUTION = 10
+    BASE_URL = "https://services-uswest2.sentinel-hub.com"
+    CLOUD_COVERAGE = True
+
+
+class SHLandsatOTL2(SHParameters):
+    """
+    Landsat 8-9 Collection 2 imagery processed to level 2
+    """
+
+    DATA_COLLECTION = DataCollection.LANDSAT_OT_L2
+    MOSAICKING_ORDER = MosaickingOrder.LEAST_CC
+    EVALSCRIPT = EvalScripts.LANDSAT_OT_L2_TRUE_COLOR
     FIELDS = None
+    FILTER = None
+    RESOLUTION = 10
+    BASE_URL = "https://services-uswest2.sentinel-hub.com"
+    CLOUD_COVERAGE = True
+
 
+class DATA_COLLECTION_ID:
+    SENTINEL_1_GRD = DataCollection.SENTINEL1.api_id
+    SENTINEL_2_L1C = DataCollection.SENTINEL2_L1C.api_id
+    SENTINEL_2_L2A = DataCollection.SENTINEL2_L2A.api_id
+    DEM = DataCollection.DEM_COPERNICUS_30.api_id
+    HLS = DataCollection.HARMONIZED_LANDSAT_SENTINEL.api_id
+    LANDSAT_OT_L2 = DataCollection.LANDSAT_OT_L2.api_id
 
-SUPPORTED_SENSORS = ("sentinel-1-grd", "sentinel-2-l1c", "sentinel-2-l2a", "dem")
+
+SUPPORTED_COLLECTION_IDS = [
+    value
+    for name, value in DATA_COLLECTION_ID.__dict__.items()
+    if not name.startswith("__")
+]
 
 SH_PARAMETERS_DICT = {
-    "sentinel-1-grd": SHS1Parameters,
-    "sentinel-2-l1c": SHS2L1CParameters,
-    "sentinel-2-l2a": SHS2L2AParameters,
-    "dem": SHDEMParameters,
+    DATA_COLLECTION_ID.SENTINEL_1_GRD: SHS1Parameters,
+    DATA_COLLECTION_ID.SENTINEL_2_L1C: SHS2L1CParameters,
+    DATA_COLLECTION_ID.SENTINEL_2_L2A: SHS2L2AParameters,
+    DATA_COLLECTION_ID.DEM: SHDEMParameters,
+    DATA_COLLECTION_ID.HLS: SHHarmonizedLandsatSentinel,
+    DATA_COLLECTION_ID.LANDSAT_OT_L2: SHLandsatOTL2,
 }
+
+
+def get_default_parameters(collection_id: str) -> SHParameters:
+    return SH_PARAMETERS_DICT[collection_id]()
+
+
+def supports_cloud_coverage(collection_id: str):
+    return SH_PARAMETERS_DICT[collection_id]().CLOUD_COVERAGE

eotdl/access/sentinelhub/utils.py

@@ -5,22 +5,23 @@ Utils for Sentinel Hub access
 import json
 
 from os import makedirs
-from datetime import datetime
-from typing import Union, Optional
+from datetime import datetime, timedelta
+from typing import Union, Optional, Iterable, List
 from glob import glob
 from shutil import copyfile, rmtree
 
-from .parameters import SUPPORTED_SENSORS
+from .parameters import SUPPORTED_COLLECTION_IDS, SHParameters, OUTPUT_FORMAT
 from ...tools.geo_utils import is_bounding_box, get_image_bbox
 from ...tools.time_utils import is_time_interval, get_day_between
 
 
 def evaluate_sentinel_parameters(
-    sensor: str,
     time_interval: Union[str, datetime],
     bounding_box: list,
+    collection_id: Optional[str] = None,
     output: Optional[str] = None,
     output_needed: Optional[bool] = True,
+    parameters: Optional[SHParameters] = None,
 ) -> None:
     """
     Evaluate parameters for Sentinel Hub access
@@ -28,10 +29,20 @@ def evaluate_sentinel_parameters(
     if output_needed:
         if not output:
             raise ValueError("Output path must be specified.")
-    if sensor not in SUPPORTED_SENSORS:
-        raise ValueError(
-            f"Sensor {sensor} is not supported. Supported sensors are: {SUPPORTED_SENSORS}"
-        )
+        if parameters and not parameters.OUTPUT_FORMAT:
+            raise ValueError("Output format must be specified.")
+    if collection_id:
+        if collection_id not in SUPPORTED_COLLECTION_IDS:
+            raise ValueError(
+                f"Collection id {collection_id} is not supported. Supported collections ids are: {SUPPORTED_COLLECTION_IDS}"
+            )
+    else:
+        if not (
+            parameters
+            and hasattr(parameters, "DATA_COLLECTION")
+            and hasattr(parameters.DATA_COLLECTION, "api_id")
+        ):
+            raise ValueError(f"Data collection is not defined properly.")
     if not time_interval:
         raise ValueError("Time interval must be specified.")
     else:
@@ -46,24 +57,34 @@ def evaluate_sentinel_parameters(
             raise ValueError(
                 "Bounding box must be a list or tuple with four elements in format (lon_min, lat_min, lon_max, lat_max)."
             )
+    if parameters and parameters.MAX_CLOUD_COVERAGE:
+        if not isinstance(parameters.MAX_CLOUD_COVERAGE, (int, float)) or (
+            parameters.MAX_CLOUD_COVERAGE < 0 or parameters.MAX_CLOUD_COVERAGE > 100
+        ):
+            raise ValueError("Max cloud coverage must be a number between 0 and 100.")
 
 
 def imagery_from_tmp_to_dir(
     output_dir: str,
+    bounding_box: List[Union[int, float]],
     tmp_dir: Optional[str],
     name: Optional[str] = None,
     bulk: Optional[bool] = False,
+    output_format: Optional[str] = OUTPUT_FORMAT.TIFF,
 ) -> None:
     """
     Copy imagery from tmp to output dir
     """
-    downloaded_files = glob(f"{tmp_dir}/**/response.tiff")
+    format = output_format
+    downloaded_files = glob(f"{tmp_dir}/**/response." + format)
+
     if len(downloaded_files) == 0:
         return
     makedirs(output_dir, exist_ok=True)
     for downloaded_file in downloaded_files:
-        request_json = downloaded_file.replace("response.tiff", "request.json")
-        metadata = generate_raster_metadata(downloaded_file, request_json)
+        request_json = downloaded_file.replace("response." + format, "request.json")
+        metadata = generate_raster_metadata(request_json, bounding_box)
+
         if name and not bulk:
             output_filename = name
         elif name and bulk:
@@ -73,17 +94,16 @@ def imagery_from_tmp_to_dir(
                 output_filename = f"{metadata['type']}_{metadata['acquisition-date']}"
             else:
                 output_filename = metadata["type"]
-        copyfile(downloaded_file, f"{output_dir}/{output_filename}.tif")
+        copyfile(downloaded_file, f"{output_dir}/{output_filename}." + format)
         with open(f"{output_dir}/{output_filename}.json", "w", encoding="utf-8") as f:
             json.dump(metadata, f)
     rmtree(tmp_dir)
 
 
-def generate_raster_metadata(raster: str, request_json: str) -> None:
+def generate_raster_metadata(request_json: str, bounding_box) -> None:
     """
     Generate metadata for raster
     """
-    bbox = get_image_bbox(raster)
     with open(request_json, "r", encoding="utf-8") as f:
         json_content = json.load(f)
 
@@ -98,8 +118,27 @@ def generate_raster_metadata(raster: str, request_json: str) -> None:
 
     metadata = {
         "acquisition-date": acquisition_date,
-        "bounding-box": bbox,
+        "bounding-box": bounding_box,
         "type": sensor_type,
     }
 
     return metadata
+
+
+def filter_times(
+    timestamps: Iterable[datetime], time_difference: timedelta
+) -> list[datetime]:
+    """
+    Filters out timestamps within time_difference, preserving only the oldest timestamp.
+    """
+    timestamps = sorted(set(timestamps))
+
+    filtered_timestamps: list[datetime] = []
+    for current_timestamp in timestamps:
+        if (
+            not filtered_timestamps
+            or current_timestamp - filtered_timestamps[-1] > time_difference
+        ):
+            filtered_timestamps.append(current_timestamp)
+
+    return filtered_timestamps

eotdl/cli.py

@@ -1,7 +1,7 @@
 import typer
 import os
 
-from .commands import auth, datasets, models
+from .commands import auth, datasets, models, stac
 from .repos import APIRepo
 from . import __version__
 
@@ -10,7 +10,7 @@ app = typer.Typer(help="Welcome to EOTDL. Learn more at https://www.eotdl.com/")
 app.add_typer(auth.app, name="auth")
 app.add_typer(datasets.app, name="datasets")
 app.add_typer(models.app, name="models")
-
+app.add_typer(stac.app, name="stac")
 
 @app.command()
 def version():

eotdl/commands/datasets.py

@@ -4,39 +4,11 @@ from pathlib import Path
 from ..datasets import (
     retrieve_datasets,
     ingest_dataset,
-    download_dataset,
+    stage_dataset,
 )
 
 app = typer.Typer(help="Explore, ingest and download training datasets.")
 
-
-@app.command()
-def list(
-    name: str = typer.Option(
-        None, "--name", "-n", help="Filter the returned datasets by name"
-    ),
-    limit: int = typer.Option(
-        None, "--limit", "-l", help="Limit the number of returned results"
-    ),
-):
-    """
-    Retrieve a list with all the datasets in the EOTDL.
-
-    If using --name, it will filter the results by name. If no name is provided, it will return all the datasets.\n
-    If using --limit, it will limit the number of results. If no limit is provided, it will return all the datasets.
-    \n\n
-    Examples\n
-    --------\n
-    $ eotdl datasets list\n
-    $ eotdl datasets list --name YourModel --limit 5
-    """
-    try:
-        datasets = retrieve_datasets(name, limit)
-        typer.echo(datasets)
-    except Exception as e:
-        typer.echo(e)
-
-
 @app.command()
 def ingest(
     path: Path = typer.Option(
@@ -62,7 +34,7 @@ def ingest(
     ),
 ):
     """
-    Ingest a dataset to the EOTDL.
+    Ingest a dataset to the EOTDL.asdf
 
     This command ingests the dataset to the EOTDL. The dataset must be a folder with the dataset files,
     and at least a README.md file (and a catalog.json file for Q1+). If these files are missing, the ingestion
@@ -90,7 +62,32 @@ def ingest(
         ingest_dataset(path, verbose, typer.echo, foce_metadata_update, sync_metadata)
     except Exception as e:
         typer.echo(e)
+        
+@app.command()
+def list(
+    name: str = typer.Option(
+        None, "--name", "-n", help="Filter the returned datasets by name"
+    ),
+    limit: int = typer.Option(
+        None, "--limit", "-l", help="Limit the number of returned results"
+    ),
+):
+    """
+    Retrieve a list with all the datasets in the EOTDL.
 
+    If using --name, it will filter the results by name. If no name is provided, it will return all the datasets.\n
+    If using --limit, it will limit the number of results. If no limit is provided, it will return all the datasets.
+    \n\n
+    Examples\n
+    --------\n
+    $ eotdl datasets list\n
+    $ eotdl datasets list --name YourModel --limit 5
+    """
+    try:
+        datasets = retrieve_datasets(name, limit)
+        typer.echo(datasets)
+    except Exception as e:
+        typer.echo(e)
 
 @app.command()
 def get(
@@ -130,7 +127,7 @@ def get(
     $ eotdl dataset get YourDataset --path /path/to/download --file dataset.zip --version 1 --assets True --force True --verbose True
     """
     try:
-        dst_path = download_dataset(
+        dst_path = stage_dataset(
             dataset,
             version,
             path,

eotdl/commands/models.py

@@ -4,39 +4,11 @@ from pathlib import Path
 from ..models import (
     retrieve_models,
     ingest_model,
-    download_model,
+    stage_model,
 )
 
 app = typer.Typer(help="Explore, ingest and download ML models.")
 
-
-@app.command()
-def list(
-    name: str = typer.Option(
-        None, "--name", "-n", help="Filter the returned models by name"
-    ),
-    limit: int = typer.Option(
-        None, "--limit", "-l", help="Limit the number of returned results"
-    ),
-):
-    """
-    Retrieve a list with all the models in the EOTDL.
-
-    If using --name, it will filter the results by name. If no name is provided, it will return all the models.\n
-    If using --limit, it will limit the number of results. If no limit is provided, it will return all the models.
-    \n\n
-    Examples\n
-    --------\n
-    $ eotdl models list\n
-    $ eotdl models list --name YourModel --limit 5
-    """
-    try:
-        models = retrieve_models(name, limit)
-        typer.echo(models)
-    except Exception as e:
-        typer.echo(e)
-
-
 @app.command()
 def ingest(
     path: Path = typer.Option(..., "--path", "-p", help="Path to the model to ingest"),
@@ -88,6 +60,31 @@ def ingest(
     except Exception as e:
         typer.echo(e)
 
+@app.command()
+def list(
+    name: str = typer.Option(
+        None, "--name", "-n", help="Filter the returned models by name"
+    ),
+    limit: int = typer.Option(
+        None, "--limit", "-l", help="Limit the number of returned results"
+    ),
+):
+    """
+    Retrieve a list with all the models in the EOTDL.
+
+    If using --name, it will filter the results by name. If no name is provided, it will return all the models.\n
+    If using --limit, it will limit the number of results. If no limit is provided, it will return all the models.
+    \n\n
+    Examples\n
+    --------\n
+    $ eotdl models list\n
+    $ eotdl models list --name YourModel --limit 5
+    """
+    try:
+        models = retrieve_models(name, limit)
+        typer.echo(models)
+    except Exception as e:
+        typer.echo(e)
 
 @app.command()
 def get(
@@ -127,7 +124,7 @@ def get(
     $ eotdl models get YourModel --path /path/to/download --file model.zip --version 1 --assets True --force True --verbose True
     """
     try:
-        dst_path = download_model(
+        dst_path = stage_model(
             model, version, path, typer.echo, assets, force, verbose
         )
         typer.echo(f"Data available at {dst_path}")

eotdl/commands/stac.py

@@ -0,0 +1,57 @@
+import typer
+from typing import Optional
+
+from ..curation.stac.api import api_status, search_stac_columns, retrieve_stac_collections, retrieve_stac_collection, retrieve_stac_items, retrieve_stac_item, search_stac_items
+
+app = typer.Typer(help="EOTDL STAC API")
+
+@app.command()
+def status():
+    try:
+        data = api_status()
+        typer.echo(data)
+    except Exception as e:
+        typer.echo(e)
+        raise typer.Abort()
+
+@app.command()
+def collections():
+    try:
+        data = retrieve_stac_collections()
+        typer.echo(data)
+    except Exception as e:
+        typer.echo(e)
+        raise typer.Abort()
+    
+@app.command()
+def collection(collection_id: str):
+    try:
+        data = retrieve_stac_collection(collection_id)
+        typer.echo(data)
+    except Exception as e:
+        typer.echo(e)
+        raise typer.Abort()
+    
+@app.command()
+def items(collection_id: str):
+    try:
+        data = retrieve_stac_items(collection_id)
+        typer.echo(data)
+    except Exception as e:
+        typer.echo(e)
+
+@app.command()
+def item(collection_id: str, item_id: str):
+    try:
+        data = retrieve_stac_item(collection_id, item_id)
+        typer.echo(data)
+    except Exception as e:
+        typer.echo(e)
+
+@app.command()
+def search(collection_id: str, query: Optional[str] = None):
+    try:
+        data = search_stac_items(collection_id, query)
+        typer.echo(data)
+    except Exception as e:
+        typer.echo(e)

eotdl/curation/__init__.py

@@ -1,8 +0,0 @@
-"""
-Curation module
-"""
-
-from .stac.dataframe import STACDataFrame  # , read_stac
-from .stac.stac import STACGenerator
-from .stac.parsers import STACIdParser, StructuredParser, UnestructuredParser
-from .stac.dataframe_labeling import UnlabeledStrategy, LabeledStrategy

eotdl/curation/stac/__init__.py

@@ -1,8 +1 @@
-"""
-STAC module
-"""
-
-# from .stac import STACGenerator
-# from .utils import format_time_acquired
-# from .parsers import STACIdParser, StructuredParser, UnestructuredParser
-from .dataframe import STACDataFrame, read_stac
+from .stac import create_stac_catalog

eotdl/curation/stac/api.py

@@ -0,0 +1,58 @@
+import json
+
+from ...repos import STACAPIRepo
+
+def api_status():
+    repo = STACAPIRepo()
+    data, error = repo.status()
+    if error:
+        raise Exception(error)
+    return data
+
+def retrieve_stac_collections():
+    repo = STACAPIRepo()
+    data, error = repo.collections()
+    if error:
+        raise Exception(error)
+    return data
+
+def retrieve_stac_collection(collection_id):
+    repo = STACAPIRepo()
+    data, error = repo.collection(collection_id)
+    if error:
+        raise Exception(error)
+    return data
+
+def retrieve_stac_items(collection_id):
+    repo = STACAPIRepo()
+    data, error = repo.items(collection_id)
+    if error:
+        raise Exception(error)
+    return data
+
+def retrieve_stac_item(collection_id, item_id):
+    repo = STACAPIRepo()
+    data, error = repo.item(collection_id, item_id)
+    if error:
+        raise Exception(error)
+    return data
+
+def search_stac_items(collection_id, query = None):
+    repo = STACAPIRepo()
+    if query is None:
+        data, error = repo.search_columns(collection_id)
+        if error:
+            raise Exception(error)
+        return data
+    data, error = repo.search(collection_id, str(query))
+    if error:
+        raise Exception(error)
+    return json.loads(data)
+    
+
+def search_stac_columns(collection_id):
+    repo = STACAPIRepo()
+    data, error = repo.search_columns(collection_id)
+    if error:
+        raise Exception(error)
+    return data