eotdl 2024.3.15__py3-none-any.whl → 2024.5.2__py3-none-any.whl

eotdl/__init__.py

@@ -1 +1 @@
-__version__ = "2024.03.15"
+__version__ = "2024.05.02"

eotdl/access/download.py

@@ -3,7 +3,7 @@ Download imagery
 """
 
 from datetime import datetime
-from typing import Union, List
+from typing import Union, List, Optional
 
 from .sentinelhub import (
     SHClient,
@@ -19,6 +19,7 @@ def download_sentinel_imagery(
     time_interval: Union[str, datetime, List[Union[str, datetime]]],
     bounding_box: List[Union[int, float]],
     sensor: str,
+    name: Optional[str] = None,
 ) -> None:
     """
     Download Sentinel imagery
@@ -28,9 +29,21 @@ def download_sentinel_imagery(
     client = SHClient()
     parameters = SH_PARAMETERS_DICT[sensor]()
 
-    request = client.request_data(time_interval, bounding_box, parameters)
-    client.download_data(request)
-    imagery_from_tmp_to_dir(output)
+    results = search_sentinel_imagery(time_interval, bounding_box, sensor)
+    timestamps = [date.strftime("%Y-%m-%d") for date in results.get_timestamps()]
+
+    requests_list = []
+    for date in timestamps:
+        requests_list.append(client.request_data(date, bounding_box, parameters))
+    if len(requests_list) == 0:
+        print(f"No images found for {sensor} in the specified time: {time_interval}")
+        return
+    elif len(requests_list) <= 2:
+        bulk = False
+    else:
+        bulk = True
+    client.download_data(requests_list)
+    imagery_from_tmp_to_dir(output, name=name, bulk=bulk)
 
 
 def search_and_download_sentinel_imagery(
@@ -42,16 +55,9 @@ def search_and_download_sentinel_imagery(
     """
     Search and download Sentinel imagery
     """
-    evaluate_sentinel_parameters(sensor, time_interval, bounding_box, output)
-
-    client = SHClient()
-    parameters = SH_PARAMETERS_DICT[sensor]()
-
-    results = search_sentinel_imagery(time_interval, bounding_box, sensor)
-    timestamps = [date.strftime("%Y-%m-%d") for date in results.get_timestamps()]
+    from warnings import warn
 
-    requests_list = []
-    for date in timestamps:
-        requests_list.append(client.request_data(date, bounding_box, parameters))
-    client.download_data(requests_list)
-    imagery_from_tmp_to_dir(output)
+    warn(
+        "The function `search_and_download_sentinel_imagery` has been deprecated and will be removed in future updates. Please use download_satellite_imagery instead."
+    )
+    download_sentinel_imagery(output, time_interval, bounding_box, sensor)

eotdl/access/sentinelhub/utils.py

@@ -49,7 +49,10 @@ def evaluate_sentinel_parameters(
 
 
 def imagery_from_tmp_to_dir(
-    output_dir: str, tmp_dir: Optional[str] = "/tmp/sentinelhub"
+    output_dir: str,
+    tmp_dir: Optional[str] = "/tmp/sentinelhub",
+    name: Optional[str] = None,
+    bulk: Optional[bool] = False,
 ) -> None:
     """
     Copy imagery from tmp to output dir
@@ -63,10 +66,15 @@ def imagery_from_tmp_to_dir(
     for downloaded_file in downloaded_files:
         request_json = downloaded_file.replace("response.tiff", "request.json")
         metadata = generate_raster_metadata(downloaded_file, request_json)
-        if metadata["acquisition-date"]:
-            output_filename = f"{metadata['type']}_{metadata['acquisition-date']}"
+        if name and not bulk:
+            output_filename = name
+        elif name and bulk:
+            output_filename = f"{name}_{metadata['acquisition-date']}"
         else:
-            output_filename = metadata["type"]
+            if metadata["acquisition-date"]:
+                output_filename = f"{metadata['type']}_{metadata['acquisition-date']}"
+            else:
+                output_filename = metadata["type"]
 
         copyfile(downloaded_file, f"{output_dir}/{output_filename}.tif")
         with open(f"{output_dir}/{output_filename}.json", "w", encoding="utf-8") as f:

eotdl/curation/stac/dataframe.py

@@ -157,9 +157,7 @@ def read_stac(
     """
     if isinstance(stac_file, (str, Path)):
         stac_file = pystac.read_file(stac_file)  # we assume this is always a catalog
-    print(stac_file)
     stac_file.make_all_asset_hrefs_absolute()
-    print("ie")
     children = get_all_children(stac_file)
 
     # Convert Dataframe to STACDataFrame

eotdl/datasets/__init__.py

@@ -1,3 +1,3 @@
-from .retrieve import retrieve_datasets  # , retrieve_dataset, list_datasets
+from .retrieve import retrieve_datasets, retrieve_dataset, retrieve_dataset_files
 from .ingest import ingest_dataset
 from .download import download_dataset, download_file_url

eotdl/datasets/download.py

@@ -60,11 +60,10 @@ def download_dataset(
                 file_version,
                 progress=True,
             )
-            # if calculate_checksum(dst_path) != checksum:
-            #     logger(f"Checksum for {file} does not match")
-
+            if verbose:
+                logger("Generating README.md ...")
+            generate_metadata(download_path, dataset)
     else:
-        # raise NotImplementedError("Downloading a STAC dataset is not implemented")
         if verbose:
             logger("Downloading STAC metadata...")
         repo = DatasetsAPIRepo()
@@ -92,11 +91,7 @@ def download_dataset(
                         href, filename, f"{download_path}/assets", user
                     )
         else:
-            if verbose:
-                logger("To download assets, set assets=True or -a in the CLI.")
-    if verbose:
-        logger("Generating README.md ...")
-    generate_metadata(download_path, dataset)
+            logger("To download assets, set assets=True or -a in the CLI.")
     if verbose:
         logger("Done")
     return download_path

eotdl/datasets/ingest.py

@@ -68,6 +68,7 @@ def ingest_folder(
         metadata = Metadata(**metadata)
         content = None
     except Exception as e:
+        # print(str(e))
         raise Exception("Error loading metadata")
     # retrieve dataset (create if doesn't exist)
     dataset = retrieve_dataset(metadata, user)

eotdl/files/__init__.py

@@ -1 +1,2 @@
 from .ingest import ingest_files, create_new_version
+from .list_files import list_files

eotdl/files/list_files.py

@@ -0,0 +1,13 @@
+from ..datasets import retrieve_dataset, retrieve_dataset_files
+from ..models import retrieve_model, retrieve_model_files
+
+def list_files(dataset_or_model_name, version=1):
+	try:
+		dataset = retrieve_dataset(dataset_or_model_name)
+		return retrieve_dataset_files(dataset['id'], version)
+	except Exception as e:
+		try:
+			model = retrieve_model(dataset_or_model_name)
+			return retrieve_model_files(model['id'], version)
+		except Exception as e:
+			raise Exception(f"Dataset or model {dataset_or_model_name} not found.")

eotdl/models/__init__.py

@@ -1,3 +1,3 @@
-from .retrieve import retrieve_models
+from .retrieve import retrieve_models, retrieve_model, retrieve_model_files
 from .ingest import ingest_model
 from .download import download_model

eotdl/repos/FilesAPIRepo.py

@@ -2,6 +2,7 @@ import requests
 import os
 from tqdm import tqdm
 import hashlib
+from io import BytesIO
 
 from ..repos import APIRepo
 
@@ -189,3 +190,11 @@ class FilesAPIRepo(APIRepo):
             headers=self.generate_headers(user),
         )
         return self.format_response(r)
+    
+    def get_file_stream(self, dataset_id, filename, user, version=None):
+        url = self.url + f"datasets/{dataset_id}/download/{filename}"
+        if version is not None:
+            url += "?version=" + str(version)
+        headers = self.generate_headers(user)
+        response = requests.get(url, headers=headers, stream=True)
+        return BytesIO(response.content)

{eotdl-2024.3.15.dist-info → eotdl-2024.5.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eotdl
-Version: 2024.3.15
+Version: 2024.5.2
 Summary: Earth Observation Training Data Lab
 License: MIT
 Author: EarthPulse
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: black (>=23.10.1,<24.0.0)
 Requires-Dist: geomet (>=1.0.0,<2.0.0)
 Requires-Dist: geopandas (>=0.13.2,<0.14.0)

{eotdl-2024.3.15.dist-info → eotdl-2024.5.2.dist-info}/RECORD

@@ -1,16 +1,16 @@
-eotdl/__init__.py,sha256=BroqyJEAr6BLg-im4pXz9vPffuGIwZS2xlW6fZKbEHs,27
+eotdl/__init__.py,sha256=xrNIo8ojW99vbCIfmq4FSOCpbcqkPd11zXtBDaGUxXA,27
 eotdl/access/__init__.py,sha256=jbyjD7BRGJURlTNmtcbBBhw3Xk4EiZvkqmEykM-bJ1k,231
 eotdl/access/airbus/__init__.py,sha256=G_kkRS9eFjXbQ-aehmTLXeAxh7zpAxz_rgB7J_w0NRg,107
 eotdl/access/airbus/client.py,sha256=zjfgB_NTsCCIszoQesYkyLJgheKg-eTh28vbleXYxfw,12018
 eotdl/access/airbus/parameters.py,sha256=Z8XIrxG5wAOuOoH-fkdKfdNMEMLFp6PaxJN7v4MefMI,1009
 eotdl/access/airbus/utils.py,sha256=oh_N1Rn4fhcvUgNPpH2QzVvpe4bA0gqRgNguzRVqUps,652
-eotdl/access/download.py,sha256=3LxfGbiZoPO2ReefKtscwnF5cMQdyLBfOnBDQA5xnlw,1568
+eotdl/access/download.py,sha256=DgemJKafNOlCUVW8OxpSP4br9ij5F1iSrSD-x0B5qFU,1845
 eotdl/access/search.py,sha256=sO2hml6JLK30DncNUqTWq16zy9LvRiWC6wtt5bNRzrI,633
 eotdl/access/sentinelhub/__init__.py,sha256=YpvaUBTRXM26WrXipo51ZUBCDv9WjRIdT8l1Pklpt_M,238
 eotdl/access/sentinelhub/client.py,sha256=g40avqlUpIa-WLjD7tK8CL8_SohBA2v3m8NZ0KbIFxc,4098
 eotdl/access/sentinelhub/evalscripts.py,sha256=m6cnZ6ryXHgdH2B7RDVSlDHXWfvKi7HMGkTHXEcJsTw,4142
 eotdl/access/sentinelhub/parameters.py,sha256=Ni3Lqx1bLVcMzgmnuayDS00BLDm7SuM1FExVdGafopI,2061
-eotdl/access/sentinelhub/utils.py,sha256=AUI3M_UX84lVdiSzOuZNiCLpHEP-sXJa97vEzAAZ-cM,3295
+eotdl/access/sentinelhub/utils.py,sha256=X9Q1YvErBdMsRKszXyaOaG6ZMvPdM2Nl_0SH-dWSFo0,3560
 eotdl/auth/__init__.py,sha256=OuGNfJQ-8Kymn4zIywlHQfImEO8DJMJIwOwTQm-u_dc,99
 eotdl/auth/auth.py,sha256=EjbVFREA2H0sjFJhVqjFZrwjKPzxRJ2x83MTjizpRBs,2029
 eotdl/auth/errors.py,sha256=E1lv3Igk--J-SOgNH18i8Xx9bXrrMyBSHKt_CAUmGPo,308
@@ -24,7 +24,7 @@ eotdl/commands/models.py,sha256=fxS0HJCkhorUpH9mwGMwbWxvQzbeRFXpwpZHtlfuZus,4873
 eotdl/curation/__init__.py,sha256=Qdp1cRR_wUYjnpyzGfyRFnL3X7aKtmgZQS8NNzPCc1s,269
 eotdl/curation/stac/__init__.py,sha256=BGBDvW-IqcxhrxvLap2Ocj0nINYaqjgjzFuAipRcoa4,220
 eotdl/curation/stac/assets.py,sha256=ay3JO6iEANMqTAe40sF7QYeEY574LbrhyanqSlVUITc,3347
-eotdl/curation/stac/dataframe.py,sha256=HZ_WrADbsoffOVlJDTJ8RL114mAaYzzOxUJjrXCSpVg,5540
+eotdl/curation/stac/dataframe.py,sha256=w0CXK4tAegbXQgnXbak2QbLfWgC6SQXpR2g-snoimAM,5503
 eotdl/curation/stac/dataframe_bck.py,sha256=PwAwol7kll0xYtlkhLeQ_Sc5TBQ85cVd6eyzdfTMJnE,8493
 eotdl/curation/stac/dataframe_labeling.py,sha256=F22-4gpF9zFuCLqRva2XAyArOmGdrgGxXbgo9d54BFE,1520
 eotdl/curation/stac/extensions/__init__.py,sha256=NSzKe14Iyr8Pm2AVg8RHxddtBD2so45--BRJmJd8bTs,629
@@ -42,15 +42,16 @@ eotdl/curation/stac/extensions/sar.py,sha256=Akw3_5brY_x2yU094nSLbv0E6M9jnAeUC0V
 eotdl/curation/stac/extent.py,sha256=Jb3K4v59eu_h5t429r0762o0zG_LA50iEE-abWNL0e0,5108
 eotdl/curation/stac/parsers.py,sha256=H5IukLA61mpLojeuhWNQdiZk2eiYHAfpJBFxmjdGDso,1529
 eotdl/curation/stac/stac.py,sha256=9GdiB1mV8iyREvXZoJmNJKQGawWyK3h9Eu0hJX5u6O4,13180
-eotdl/datasets/__init__.py,sha256=oFSUD0OLvUS4_iwu_V1iAQ_VAML-8RWDbxgtuFdU5ZA,170
-eotdl/datasets/download.py,sha256=IVajcyU9f0zESNEOQWRjSnPnF_mAzSECBlQm9QJOXVs,3880
-eotdl/datasets/ingest.py,sha256=3yK8KxtTw-eadNJizQtuwCXZPFPxIFdwC4oeSAPb8WA,5782
+eotdl/datasets/__init__.py,sha256=xNbkUjqI1mf0TGjy52tpZtr5DnWNX5EVgyM04P4sU3Y,175
+eotdl/datasets/download.py,sha256=JSOi7OKiENYQ3TRVj2WsC1CPYD22VlC9W5kYHPfx8Vs,3667
+eotdl/datasets/ingest.py,sha256=Ukiah6TapD1oBE0AxUTaEFf-k3hAMRrKr4cXVlVW-_Y,5806
 eotdl/datasets/metadata.py,sha256=qonblTDGr4IZvJaiIc2rh7qwP76whEA72DorSVqZxd4,1457
 eotdl/datasets/retrieve.py,sha256=DJz5K1bCLizg9YNwBnhHMFzcxMXar2socYkFONdSL4c,1041
 eotdl/datasets/update.py,sha256=x-rpfxnavn9X-7QYkFMGtbn1b3bKmAZydOeS7Tjr5AQ,386
-eotdl/files/__init__.py,sha256=wStggK9L_Ni3vdZkKtKmKVdAaZRVuiMbsjYBCrJ8rxs,53
+eotdl/files/__init__.py,sha256=2zfAxgLHmU_jWq_3emnfPXsX-R20gSt-yZX0bPa9h0g,87
 eotdl/files/ingest.py,sha256=dgjZfd-ACCKradDo2B02CPahwEhFtWvnKvTm372K5eo,6185
-eotdl/models/__init__.py,sha256=fe1VfnlOxlfviphkkzaY4q8mkm0QxcdJxB3peScBZYk,108
+eotdl/files/list_files.py,sha256=k4OgdbQ7u6tUEE9nJZGXw5s5HtvG0ZApOVTy0KbfTqs,519
+eotdl/models/__init__.py,sha256=-PvGWG0iSRNBqeFWpxol12dYw-QodXjR81n-JX3x6zI,146
 eotdl/models/download.py,sha256=uXWUslaBkJ222BfctbO0eKNcw6Unx1-Pa9Yv5RHXJIo,4119
 eotdl/models/ingest.py,sha256=KmBFUS1pkUYdegfi0lL1sd__lj9GMoJ7jxvnVXfY8Ts,3381
 eotdl/models/metadata.py,sha256=S5bpIB4e2pivDnReszJKC3bYBZcaHu-KMYOc3AwHbQ4,1443
@@ -60,7 +61,7 @@ eotdl/repos/APIRepo.py,sha256=dJNdX3atBpug0FZNako7uyom25iccNPQrnoIRTIZEXE,791
 eotdl/repos/AuthAPIRepo.py,sha256=vYCqFawe3xUm2cx4SqVXCvzl8J_sr9rs_MkipYC0bXE,957
 eotdl/repos/AuthRepo.py,sha256=jpzzhINCcDZHRCyrPDsp49h17IlXp2HvX3BB3f5cnb4,1154
 eotdl/repos/DatasetsAPIRepo.py,sha256=rKqSe-UjJOlD0Kbypu4Gs5kx2mzUD7TY05gv2vgUTv4,2660
-eotdl/repos/FilesAPIRepo.py,sha256=vHyEePwNERInSo2d3-USgtx-QkXqA3Jzgo5AtuWrAHU,6768
+eotdl/repos/FilesAPIRepo.py,sha256=cS6CFTkSYIXcefHEeLt7I69_EyyO-jhoAwQ7UWYlLkc,7171
 eotdl/repos/ModelsAPIRepo.py,sha256=7-0c-3OCTCwKUZ-Z7PWtbRusUxpTU7m2H5jIAcFoJ9M,1677
 eotdl/repos/__init__.py,sha256=WvX5TP49k7yYb5dWWNjv5kzbdluO3dJ4LqjQxRIOUVc,222
 eotdl/shared/__init__.py,sha256=mF7doJC8Z5eTPmB01UQvPivThZac32DRY33T6qshXfg,41
@@ -72,7 +73,7 @@ eotdl/tools/paths.py,sha256=yWhOtVxX4NxrDrrBX2fuye5N1mAqrxXFy_eA7dffd84,1152
 eotdl/tools/stac.py,sha256=ovXdrPm4Sn9AAJmrP88WnxDmq2Ut-xPoscjphxz3Iyo,5763
 eotdl/tools/time_utils.py,sha256=qJ3-rk1I7ne722SLfAP6-59kahQ0vLQqIf9VpOi0Kpg,4691
 eotdl/tools/tools.py,sha256=Tl4_v2ejkQo_zyZek8oofJwoYcdVosdOwW1C0lvWaNM,6354
-eotdl-2024.3.15.dist-info/METADATA,sha256=5Ujw3YnNC-Nd_e5yEKVYCgWDeO0bPTnKe5LN0oXH4zI,4052
-eotdl-2024.3.15.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-eotdl-2024.3.15.dist-info/entry_points.txt,sha256=s6sfxUfRrSX2IP2UbrzTFTvRCtLgw3_OKcHlOKf_5F8,39
-eotdl-2024.3.15.dist-info/RECORD,,
+eotdl-2024.5.2.dist-info/METADATA,sha256=O_RtJWIg7MvaJ3PeX4ZPIpQHuxUdSSZvdtK6qWb8ciY,4102
+eotdl-2024.5.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+eotdl-2024.5.2.dist-info/entry_points.txt,sha256=s6sfxUfRrSX2IP2UbrzTFTvRCtLgw3_OKcHlOKf_5F8,39
+eotdl-2024.5.2.dist-info/RECORD,,

{eotdl-2024.3.15.dist-info → eotdl-2024.5.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.7.0
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any