eotdl 2024.3.15__tar.gz → 2024.5.2__tar.gz

{eotdl-2024.3.15 → eotdl-2024.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eotdl
-Version: 2024.3.15
+Version: 2024.5.2
 Summary: Earth Observation Training Data Lab
 License: MIT
 Author: EarthPulse
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: black (>=23.10.1,<24.0.0)
 Requires-Dist: geomet (>=1.0.0,<2.0.0)
 Requires-Dist: geopandas (>=0.13.2,<0.14.0)

eotdl-2024.5.2/eotdl/__init__.py

@@ -0,0 +1 @@
+__version__ = "2024.05.02"

{eotdl-2024.3.15 → eotdl-2024.5.2}/eotdl/access/download.py

@@ -3,7 +3,7 @@ Download imagery
 """
 
 from datetime import datetime
-from typing import Union, List
+from typing import Union, List, Optional
 
 from .sentinelhub import (
     SHClient,
@@ -19,6 +19,7 @@ def download_sentinel_imagery(
     time_interval: Union[str, datetime, List[Union[str, datetime]]],
     bounding_box: List[Union[int, float]],
     sensor: str,
+    name: Optional[str] = None,
 ) -> None:
     """
     Download Sentinel imagery
@@ -28,9 +29,21 @@ def download_sentinel_imagery(
     client = SHClient()
     parameters = SH_PARAMETERS_DICT[sensor]()
 
-    request = client.request_data(time_interval, bounding_box, parameters)
-    client.download_data(request)
-    imagery_from_tmp_to_dir(output)
+    results = search_sentinel_imagery(time_interval, bounding_box, sensor)
+    timestamps = [date.strftime("%Y-%m-%d") for date in results.get_timestamps()]
+
+    requests_list = []
+    for date in timestamps:
+        requests_list.append(client.request_data(date, bounding_box, parameters))
+    if len(requests_list) == 0:
+        print(f"No images found for {sensor} in the specified time: {time_interval}")
+        return
+    elif len(requests_list) <= 2:
+        bulk = False
+    else:
+        bulk = True
+    client.download_data(requests_list)
+    imagery_from_tmp_to_dir(output, name=name, bulk=bulk)
 
 
 def search_and_download_sentinel_imagery(
@@ -42,16 +55,9 @@ def search_and_download_sentinel_imagery(
     """
     Search and download Sentinel imagery
     """
-    evaluate_sentinel_parameters(sensor, time_interval, bounding_box, output)
-
-    client = SHClient()
-    parameters = SH_PARAMETERS_DICT[sensor]()
-
-    results = search_sentinel_imagery(time_interval, bounding_box, sensor)
-    timestamps = [date.strftime("%Y-%m-%d") for date in results.get_timestamps()]
+    from warnings import warn
 
-    requests_list = []
-    for date in timestamps:
-        requests_list.append(client.request_data(date, bounding_box, parameters))
-    client.download_data(requests_list)
-    imagery_from_tmp_to_dir(output)
+    warn(
+        "The function `search_and_download_sentinel_imagery` has been deprecated and will be removed in future updates. Please use download_satellite_imagery instead."
+    )
+    download_sentinel_imagery(output, time_interval, bounding_box, sensor)

{eotdl-2024.3.15 → eotdl-2024.5.2}/eotdl/access/sentinelhub/utils.py

@@ -49,7 +49,10 @@ def evaluate_sentinel_parameters(
 
 
 def imagery_from_tmp_to_dir(
-    output_dir: str, tmp_dir: Optional[str] = "/tmp/sentinelhub"
+    output_dir: str,
+    tmp_dir: Optional[str] = "/tmp/sentinelhub",
+    name: Optional[str] = None,
+    bulk: Optional[bool] = False,
 ) -> None:
     """
     Copy imagery from tmp to output dir
@@ -63,10 +66,15 @@ def imagery_from_tmp_to_dir(
     for downloaded_file in downloaded_files:
         request_json = downloaded_file.replace("response.tiff", "request.json")
         metadata = generate_raster_metadata(downloaded_file, request_json)
-        if metadata["acquisition-date"]:
-            output_filename = f"{metadata['type']}_{metadata['acquisition-date']}"
+        if name and not bulk:
+            output_filename = name
+        elif name and bulk:
+            output_filename = f"{name}_{metadata['acquisition-date']}"
         else:
-            output_filename = metadata["type"]
+            if metadata["acquisition-date"]:
+                output_filename = f"{metadata['type']}_{metadata['acquisition-date']}"
+            else:
+                output_filename = metadata["type"]
 
         copyfile(downloaded_file, f"{output_dir}/{output_filename}.tif")
         with open(f"{output_dir}/{output_filename}.json", "w", encoding="utf-8") as f:

{eotdl-2024.3.15 → eotdl-2024.5.2}/eotdl/curation/stac/dataframe.py

@@ -157,9 +157,7 @@ def read_stac(
     """
     if isinstance(stac_file, (str, Path)):
         stac_file = pystac.read_file(stac_file)  # we assume this is always a catalog
-    print(stac_file)
     stac_file.make_all_asset_hrefs_absolute()
-    print("ie")
     children = get_all_children(stac_file)
 
     # Convert Dataframe to STACDataFrame

{eotdl-2024.3.15 → eotdl-2024.5.2}/eotdl/datasets/__init__.py

@@ -1,3 +1,3 @@
-from .retrieve import retrieve_datasets  # , retrieve_dataset, list_datasets
+from .retrieve import retrieve_datasets, retrieve_dataset, retrieve_dataset_files
 from .ingest import ingest_dataset
 from .download import download_dataset, download_file_url

{eotdl-2024.3.15 → eotdl-2024.5.2}/eotdl/datasets/download.py

@@ -60,11 +60,10 @@ def download_dataset(
                 file_version,
                 progress=True,
             )
-            # if calculate_checksum(dst_path) != checksum:
-            #     logger(f"Checksum for {file} does not match")
-
+            if verbose:
+                logger("Generating README.md ...")
+            generate_metadata(download_path, dataset)
     else:
-        # raise NotImplementedError("Downloading a STAC dataset is not implemented")
         if verbose:
             logger("Downloading STAC metadata...")
         repo = DatasetsAPIRepo()
@@ -92,11 +91,7 @@ def download_dataset(
                         href, filename, f"{download_path}/assets", user
                     )
         else:
-            if verbose:
-                logger("To download assets, set assets=True or -a in the CLI.")
-    if verbose:
-        logger("Generating README.md ...")
-    generate_metadata(download_path, dataset)
+            logger("To download assets, set assets=True or -a in the CLI.")
     if verbose:
         logger("Done")
     return download_path

{eotdl-2024.3.15 → eotdl-2024.5.2}/eotdl/datasets/ingest.py

@@ -68,6 +68,7 @@ def ingest_folder(
         metadata = Metadata(**metadata)
         content = None
     except Exception as e:
+        # print(str(e))
         raise Exception("Error loading metadata")
     # retrieve dataset (create if doesn't exist)
     dataset = retrieve_dataset(metadata, user)

{eotdl-2024.3.15 → eotdl-2024.5.2}/eotdl/files/__init__.py

@@ -1 +1,2 @@
 from .ingest import ingest_files, create_new_version
+from .list_files import list_files

eotdl-2024.5.2/eotdl/files/list_files.py

@@ -0,0 +1,13 @@
+from ..datasets import retrieve_dataset, retrieve_dataset_files
+from ..models import retrieve_model, retrieve_model_files
+
+def list_files(dataset_or_model_name, version=1):
+	try:
+		dataset = retrieve_dataset(dataset_or_model_name)
+		return retrieve_dataset_files(dataset['id'], version)
+	except Exception as e:
+		try:
+			model = retrieve_model(dataset_or_model_name)
+			return retrieve_model_files(model['id'], version)
+		except Exception as e:
+			raise Exception(f"Dataset or model {dataset_or_model_name} not found.")

eotdl-2024.5.2/eotdl/models/__init__.py

@@ -0,0 +1,3 @@
+from .retrieve import retrieve_models, retrieve_model, retrieve_model_files
+from .ingest import ingest_model
+from .download import download_model

{eotdl-2024.3.15 → eotdl-2024.5.2}/eotdl/repos/FilesAPIRepo.py

@@ -2,6 +2,7 @@ import requests
 import os
 from tqdm import tqdm
 import hashlib
+from io import BytesIO
 
 from ..repos import APIRepo
 
@@ -189,3 +190,11 @@ class FilesAPIRepo(APIRepo):
             headers=self.generate_headers(user),
         )
         return self.format_response(r)
+    
+    def get_file_stream(self, dataset_id, filename, user, version=None):
+        url = self.url + f"datasets/{dataset_id}/download/{filename}"
+        if version is not None:
+            url += "?version=" + str(version)
+        headers = self.generate_headers(user)
+        response = requests.get(url, headers=headers, stream=True)
+        return BytesIO(response.content)

{eotdl-2024.3.15 → eotdl-2024.5.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "eotdl"
-version = "2024.03.15"
+version = "2024.05.02"
 description = "Earth Observation Training Data Lab"
 authors = ["EarthPulse <it@earthpulse.es>"]
 license = "MIT"

eotdl-2024.3.15/eotdl/__init__.py

@@ -1 +0,0 @@
-__version__ = "2024.03.15"

eotdl-2024.3.15/eotdl/models/__init__.py

@@ -1,3 +0,0 @@
-from .retrieve import retrieve_models
-from .ingest import ingest_model
-from .download import download_model