eotdl 2023.11.3.post4__py3-none-any.whl → 2024.2.15__py3-none-any.whl

eotdl/repos/DatasetsAPIRepo.py

@@ -20,11 +20,11 @@ class DatasetsAPIRepo(APIRepo):
         response = requests.get(url)
         return self.format_response(response)
 
-    def create_dataset(self, metadata, id_token):
+    def create_dataset(self, metadata, user):
         response = requests.post(
             self.url + "datasets",
             json=metadata,
-            headers={"Authorization": "Bearer " + id_token},
+            headers=self.generate_headers(user),
         )
         return self.format_response(response)
 
@@ -32,94 +32,33 @@ class DatasetsAPIRepo(APIRepo):
         response = requests.get(self.url + "datasets?name=" + name)
         return self.format_response(response)
 
-    def create_version(self, dataset_id, id_token):
+    def create_version(self, dataset_id, user):
         response = requests.post(
             self.url + "datasets/version/" + dataset_id,
-            headers={"Authorization": "Bearer " + id_token},
+            headers=self.generate_headers(user),
         )
         return self.format_response(response)
 
-    def create_stac_dataset(self, name, id_token):
+    def create_stac_dataset(self, name, user):
         response = requests.post(
             self.url + "datasets/stac",
             json={"name": name},
-            headers={"Authorization": "Bearer " + id_token},
+            headers=self.generate_headers(user),
         )
         return self.format_response(response)
 
-    def ingest_stac(self, stac_json, dataset_id, id_token):
+    def ingest_stac(self, stac_json, dataset_id, user):
         response = requests.put(
             self.url + f"datasets/stac/{dataset_id}",
             json={"stac": stac_json},
-            headers={"Authorization": "Bearer " + id_token},
+            headers=self.generate_headers(user),
         )
         return self.format_response(response)
 
-    def download_stac(self, dataset_id, id_token):
+    def download_stac(self, dataset_id, user):
         url = self.url + "datasets/" + dataset_id + "/download"
-        headers = {"Authorization": "Bearer " + id_token}
+        headers = self.generate_headers(user)
         response = requests.get(url, headers=headers)
         if response.status_code != 200:
             return None, response.json()["detail"]
         return gpd.GeoDataFrame.from_features(response.json()["features"]), None
-
-    # def update_dataset(self, name, path, id_token, checksum):
-    #     # check that dataset exists
-    #     data, error = self.retrieve_dataset(name)
-    #     if error:
-    #         return None, error
-    #     # first call to get upload id
-    #     dataset_id = data["id"]
-    #     url = self.url + f"datasets/chunk/{dataset_id}?checksum={checksum}"
-    #     response = requests.get(url, headers={"Authorization": "Bearer " + id_token})
-    #     if response.status_code != 200:
-    #         return None, response.json()["detail"]
-    #     data = response.json()
-    #     _, upload_id, parts = data["dataset_id"], data["upload_id"], data["parts"]
-    #     # assert dataset_id is None
-    #     content_path = os.path.abspath(path)
-    #     content_size = os.stat(content_path).st_size
-    #     url = self.url + "datasets/chunk"
-    #     chunk_size = 1024 * 1024 * 100  # 100 MiB
-    #     total_chunks = content_size // chunk_size
-    #     headers = {
-    #         "Authorization": "Bearer " + id_token,
-    #         "Upload-Id": upload_id,
-    #         "Dataset-Id": dataset_id,
-    #     }
-    #     # upload chunks sequentially
-    #     pbar = tqdm(
-    #         self.read_in_chunks(open(content_path, "rb"), chunk_size),
-    #         total=total_chunks,
-    #     )
-    #     index = 0
-    #     for chunk in pbar:
-    #         offset = index + len(chunk)
-    #         part = index // chunk_size + 1
-    #         index = offset
-    #         if part not in parts:
-    #             headers["Part-Number"] = str(part)
-    #             file = {"file": chunk}
-    #             r = requests.post(url, files=file, headers=headers)
-    #             if r.status_code != 200:
-    #                 return None, r.json()["detail"]
-    #         pbar.set_description(
-    #             "{:.2f}/{:.2f} MB".format(
-    #                 offset / 1024 / 1024, content_size / 1024 / 1024
-    #             )
-    #         )
-    #     pbar.close()
-    #     # complete upload
-    #     url = self.url + "datasets/complete"
-    #     r = requests.post(
-    #         url,
-    #         json={"checksum": checksum},
-    #         headers={
-    #             "Authorization": "Bearer " + id_token,
-    #             "Upload-Id": upload_id,
-    #             "Dataset-Id": dataset_id,
-    #         },
-    #     )
-    #     if r.status_code != 200:
-    #         return None, r.json()["detail"]
-    #     return r.json(), None

eotdl/repos/FilesAPIRepo.py

@@ -15,7 +15,7 @@ class FilesAPIRepo(APIRepo):
         batch,  # ziped batch of files
         checksums,
         dataset_or_model_id,
-        id_token,
+        user,
         endpoint,
         version=None,
     ):
@@ -26,7 +26,7 @@ class FilesAPIRepo(APIRepo):
             url,
             files={"batch": ("batch.zip", batch)},
             data={"checksums": checksums},
-            headers={"Authorization": "Bearer " + id_token},
+            headers=self.generate_headers(user),
         )
         return self.format_response(reponse)
 
@@ -35,7 +35,7 @@ class FilesAPIRepo(APIRepo):
         batch,
         dataset_or_model_id,
         version,
-        id_token,
+        user,
         endpoint,
     ):
         reponse = requests.post(
@@ -44,13 +44,14 @@ class FilesAPIRepo(APIRepo):
                 "filenames": [f["path"] for f in batch],
                 "checksums": [f["checksum"] for f in batch],
             },
-            headers={"Authorization": "Bearer " + id_token},
+            headers=self.generate_headers(user),
         )
         return self.format_response(reponse)
 
     def ingest_file(
-        self, file, dataset_or_model_id, id_token, checksum, endpoint, version=None
+        self, file, dataset_or_model_id, user, checksum, endpoint, version=None
     ):
+        # TODO: ingest file URL
         url = self.url + f"{endpoint}/{dataset_or_model_id}"
         if version is not None:
             url += "?version=" + str(version)
@@ -58,7 +59,7 @@ class FilesAPIRepo(APIRepo):
             url,
             files={"file": open(file, "rb")},
             data={"checksum": checksum},
-            headers={"Authorization": "Bearer " + id_token},
+            headers=self.generate_headers(user),
         )
         return self.format_response(reponse)
 
@@ -73,7 +74,7 @@ class FilesAPIRepo(APIRepo):
         self,
         dataset_or_model_id,
         file_name,
-        id_token,
+        user,
         path,
         file_version,
         endpoint="datasets",
@@ -82,10 +83,10 @@ class FilesAPIRepo(APIRepo):
         url = self.url + f"{endpoint}/{dataset_or_model_id}/download/{file_name}"
         if file_version is not None:
             url += "?version=" + str(file_version)
-        return self.download_file_url(url, file_name, path, id_token, progress=progress)
+        return self.download_file_url(url, file_name, path, user, progress=progress)
 
-    def download_file_url(self, url, filename, path, id_token, progress=False):
-        headers = {"Authorization": "Bearer " + id_token}
+    def download_file_url(self, url, filename, path, user, progress=False):
+        headers = self.generate_headers(user)
         path = f"{path}/{filename}"
         for i in range(1, len(path.split("/")) - 1):
             # print("/".join(path.split("/")[: i + 1]))
@@ -113,23 +114,13 @@ class FilesAPIRepo(APIRepo):
                 progress_bar.close()
             return path
 
-    # def ingest_file_url(self, file, dataset, id_token):
-    #     reponse = requests.post(
-    #         self.url + f"datasets/{dataset}/url",
-    #         json={"url": file},
-    #         headers={"Authorization": "Bearer " + id_token},
-    #     )
-    #     if reponse.status_code != 200:
-    #         return None, reponse.json()["detail"]
-    #     return reponse.json(), None
-
     def prepare_large_upload(
-        self, filename, dataset_or_model_id, checksum, id_token, endpoint
+        self, filename, dataset_or_model_id, checksum, user, endpoint
     ):
         response = requests.post(
             self.url + f"{endpoint}/{dataset_or_model_id}/uploadId",
             json={"filname": filename, "checksum": checksum},
-            headers={"Authorization": "Bearer " + id_token},
+            headers=self.generate_headers(user),
         )
         if response.status_code != 200:
             raise Exception(response.json()["detail"])
@@ -157,7 +148,7 @@ class FilesAPIRepo(APIRepo):
             yield data
 
     def ingest_large_file(
-        self, file_path, files_size, upload_id, id_token, parts, endpoint
+        self, file_path, files_size, upload_id, user, parts, endpoint
     ):
         print(endpoint)
         # content_path = os.path.abspath(file)
@@ -180,7 +171,7 @@ class FilesAPIRepo(APIRepo):
                     f"{self.url}{endpoint}/chunk/{upload_id}",
                     files={"file": chunk},
                     data={"part_number": part, "checksum": checksum},
-                    headers={"Authorization": "Bearer " + id_token},
+                    headers=self.generate_headers(user),
                 )
                 if response.status_code != 200:
                     raise Exception(response.json()["detail"])
@@ -192,18 +183,9 @@ class FilesAPIRepo(APIRepo):
         pbar.close()
         return
 
-    def complete_upload(self, id_token, upload_id, version, endpoint):
+    def complete_upload(self, user, upload_id, version, endpoint):
         r = requests.post(
             f"{self.url}{endpoint}/complete/{upload_id}?version={version}",
-            headers={"Authorization": "Bearer " + id_token},
+            headers=self.generate_headers(user),
         )
         return self.format_response(r)
-
-    # def delete_file(self, dataset_id, file_name, id_token):
-    #     response = requests.delete(
-    #         self.url + "datasets/" + dataset_id + "/file/" + file_name,
-    #         headers={"Authorization": "Bearer " + id_token},
-    #     )
-    #     if response.status_code != 200:
-    #         return None, response.json()["detail"]
-    #     return response.json(), None

eotdl/repos/ModelsAPIRepo.py

@@ -19,11 +19,11 @@ class ModelsAPIRepo(APIRepo):
         response = requests.get(url)
         return self.format_response(response)
 
-    def create_model(self, metadata, id_token):
+    def create_model(self, metadata, user):
         response = requests.post(
             self.url + "models",
             json=metadata,
-            headers={"Authorization": "Bearer " + id_token},
+            headers=self.generate_headers(user),
         )
         return self.format_response(response)
 
@@ -31,9 +31,9 @@ class ModelsAPIRepo(APIRepo):
         response = requests.get(self.url + "models?name=" + name)
         return self.format_response(response)
 
-    def create_version(self, model_id, id_token):
+    def create_version(self, model_id, user):
         response = requests.post(
             self.url + "models/version/" + model_id,
-            headers={"Authorization": "Bearer " + id_token},
+            headers=self.generate_headers(user),
         )
         return self.format_response(response)

eotdl/tools/tools.py

@@ -15,7 +15,7 @@ from shapely.geometry import box
 from .geo_utils import get_image_bbox
 
 
-def get_images_by_location(gdf: gpd.GeoDataFrame) -> pd.DataFrame:
+def get_images_by_location(gdf: gpd.GeoDataFrame, column: str) -> pd.DataFrame:
     """
     Generate a GeoDataFrame with the available images for each location in the dataset.
 
@@ -27,7 +27,7 @@ def get_images_by_location(gdf: gpd.GeoDataFrame) -> pd.DataFrame:
                 - images_count: the count of available images of each location.
                 - images_dates: list with the dates of the available images of each location.
     """
-    uniques_location_id = gdf["scene_id"].unique()  # List of unique location ids
+    uniques_location_id = gdf[column].unique()  # List of unique location ids
     uniques_location_id.sort()
 
     images_count_list, images_dates_list = [], []
@@ -35,13 +35,13 @@ def get_images_by_location(gdf: gpd.GeoDataFrame) -> pd.DataFrame:
     # Iterate the unique location ids, count the number of images per location and generate
     # a list with the dates of every image in a location
     for location_id in uniques_location_id:
-        dates = gdf[gdf["scene_id"] == location_id]["datetime"]
+        dates = gdf[gdf[column] == location_id]["datetime"]
         images_count_list.append(dates.count())
         images_dates_list.append(dates.tolist())
 
     images_dates_list.sort()  # Sort the list of dates
     data = {
-        "scene_id": uniques_location_id,
+        column: uniques_location_id,
         "dates_count": images_count_list,
         "dates_list": images_dates_list,
     }

{eotdl-2023.11.3.post4.dist-info → eotdl-2024.2.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eotdl
-Version: 2023.11.3.post4
+Version: 2024.2.15
 Summary: Earth Observation Training Data Lab
 License: MIT
 Author: EarthPulse
@@ -29,7 +29,7 @@ Description-Content-Type: text/markdown
 
 <p align="center">
   <a href="https://www.eotdl.com/">
-    <img src="eotdl.png" alt="EOTDL" />
+    <img src="https://raw.githubusercontent.com/earthpulse/eotdl/main/eotdl/eotdl.png" alt="EOTDL" style="width: 75%;"/>
   </a>
 </p>
 
@@ -40,9 +40,6 @@ Description-Content-Type: text/markdown
     <a href="https://pypi.python.org/pypi/eotdl">
         <img src="https://img.shields.io/pypi/v/eotdl.svg" alt="NPM Version" />
     </a>
-    <a href="https://discord.eotdl.io">
-        <img src="https://img.shields.io/discord/1024232224486326362?label=Discord" alt="EOTDL on Discord" />
-    </a>
 </p>
 
 This is the main library and CLI for the **Earth Observation Training Data Lab** (EOTDL), a complete environment that allows you, among other things, to:

{eotdl-2023.11.3.post4.dist-info → eotdl-2024.2.15.dist-info}/RECORD

@@ -1,4 +1,4 @@
-eotdl/__init__.py,sha256=gSrwUZr2mBINM6QBGDwtTjXz1i4NeI31XLvjsQFYqk0,29
+eotdl/__init__.py,sha256=WKZfCvNYtuas-KdIFXdxyFqYY3MyW7MCPuaqs7lEG1E,27
 eotdl/access/__init__.py,sha256=jbyjD7BRGJURlTNmtcbBBhw3Xk4EiZvkqmEykM-bJ1k,231
 eotdl/access/airbus/__init__.py,sha256=G_kkRS9eFjXbQ-aehmTLXeAxh7zpAxz_rgB7J_w0NRg,107
 eotdl/access/airbus/client.py,sha256=zjfgB_NTsCCIszoQesYkyLJgheKg-eTh28vbleXYxfw,12018
@@ -8,11 +8,11 @@ eotdl/access/download.py,sha256=3LxfGbiZoPO2ReefKtscwnF5cMQdyLBfOnBDQA5xnlw,1568
 eotdl/access/search.py,sha256=sO2hml6JLK30DncNUqTWq16zy9LvRiWC6wtt5bNRzrI,633
 eotdl/access/sentinelhub/__init__.py,sha256=YpvaUBTRXM26WrXipo51ZUBCDv9WjRIdT8l1Pklpt_M,238
 eotdl/access/sentinelhub/client.py,sha256=g40avqlUpIa-WLjD7tK8CL8_SohBA2v3m8NZ0KbIFxc,4098
-eotdl/access/sentinelhub/evalscripts.py,sha256=uuLC6Km7IPssvM1xiIlLkx--TA1Ash159M4jov_I1Y4,4315
-eotdl/access/sentinelhub/parameters.py,sha256=kZSVdq85UiZjIRDe81TJ8IpOSCeVIOlbDWA5u8Ylg8Q,2050
-eotdl/access/sentinelhub/utils.py,sha256=XSDJD6Etg4Cg5D9dsXPbZnC09L4kWrQxTspGqY8LHs8,3305
+eotdl/access/sentinelhub/evalscripts.py,sha256=m6cnZ6ryXHgdH2B7RDVSlDHXWfvKi7HMGkTHXEcJsTw,4142
+eotdl/access/sentinelhub/parameters.py,sha256=Ni3Lqx1bLVcMzgmnuayDS00BLDm7SuM1FExVdGafopI,2061
+eotdl/access/sentinelhub/utils.py,sha256=AUI3M_UX84lVdiSzOuZNiCLpHEP-sXJa97vEzAAZ-cM,3295
 eotdl/auth/__init__.py,sha256=OuGNfJQ-8Kymn4zIywlHQfImEO8DJMJIwOwTQm-u_dc,99
-eotdl/auth/auth.py,sha256=voxwxTERghLQdqrBSyjZrgvKrcF01aWUTc0-vxLFFgY,1601
+eotdl/auth/auth.py,sha256=EjbVFREA2H0sjFJhVqjFZrwjKPzxRJ2x83MTjizpRBs,2029
 eotdl/auth/errors.py,sha256=E1lv3Igk--J-SOgNH18i8Xx9bXrrMyBSHKt_CAUmGPo,308
 eotdl/auth/is_logged.py,sha256=QREuhkoDnarZoUZwCxVCNoESGb_Yukh0lJo1pXvrV9Q,115
 eotdl/auth/logout.py,sha256=P_Sp6WmVvnG3R9V1L9541KNyHFko9DtQPqAKD2vaguw,161
@@ -23,8 +23,8 @@ eotdl/commands/datasets.py,sha256=XeDq-tJec-EmliocIXHPkg_1wBwCR2DJqxBxRbwjSGg,44
 eotdl/commands/models.py,sha256=7JPzjgxrnLLuj2c_D6RS2pc-m54MhmYIpGcob8pRJ_Q,4362
 eotdl/curation/__init__.py,sha256=Qdp1cRR_wUYjnpyzGfyRFnL3X7aKtmgZQS8NNzPCc1s,269
 eotdl/curation/stac/__init__.py,sha256=BGBDvW-IqcxhrxvLap2Ocj0nINYaqjgjzFuAipRcoa4,220
-eotdl/curation/stac/assets.py,sha256=SuoUxbDkI-lqv89XpaE7YiyUkgC1ZWKRxi5r7dEGmGI,4510
-eotdl/curation/stac/dataframe.py,sha256=sks5CcJAYaOUsnJYI30mbnXfkCvxmiLtsQPCnsUpDBI,5371
+eotdl/curation/stac/assets.py,sha256=ay3JO6iEANMqTAe40sF7QYeEY574LbrhyanqSlVUITc,3347
+eotdl/curation/stac/dataframe.py,sha256=HZ_WrADbsoffOVlJDTJ8RL114mAaYzzOxUJjrXCSpVg,5540
 eotdl/curation/stac/dataframe_bck.py,sha256=PwAwol7kll0xYtlkhLeQ_Sc5TBQ85cVd6eyzdfTMJnE,8493
 eotdl/curation/stac/dataframe_labeling.py,sha256=F22-4gpF9zFuCLqRva2XAyArOmGdrgGxXbgo9d54BFE,1520
 eotdl/curation/stac/extensions/__init__.py,sha256=NSzKe14Iyr8Pm2AVg8RHxddtBD2so45--BRJmJd8bTs,629
@@ -34,7 +34,7 @@ eotdl/curation/stac/extensions/eo.py,sha256=cT4RrbyoimIuuOxNRLkamhZgHpDbj4z_Ziat
 eotdl/curation/stac/extensions/label/__init__.py,sha256=R6xLkgJaZHoMh5BhpmueupWdM9NWKvmaRurum-ryU_s,159
 eotdl/curation/stac/extensions/label/base.py,sha256=5xbniQWjzXkrQxxXp8v9QQxFQdRYnaFPBd5_in1QXUw,4069
 eotdl/curation/stac/extensions/label/image_name_labeler.py,sha256=bivZN-qEtIXE6ehqwPKRsJO9RVckZ1lK2BG8ifUbaA8,8074
-eotdl/curation/stac/extensions/label/scaneo.py,sha256=LPrebS3TUsKjy8tnV7yG_T3xI4iKlv5lg5YOfjpu3CQ,8721
+eotdl/curation/stac/extensions/label/scaneo.py,sha256=uUzEqEwdnKYZx-qx9o6d8HcXp1VPy8eYWc9kM0sjiyA,8787
 eotdl/curation/stac/extensions/ml_dataset.py,sha256=SjOX_EvVRhwIUc4iQLw6YX_bG3VTqlRd9SpmpNCnRak,21429
 eotdl/curation/stac/extensions/projection.py,sha256=ussVIwr_wOOhn07OmpAWY4qqbeAmYUxKjbE8onrAy7o,1236
 eotdl/curation/stac/extensions/raster.py,sha256=o5U_1ow8BsgwZXpSQYwQIvMJldhyn7xoGoJmUANTJTE,1540
@@ -43,31 +43,23 @@ eotdl/curation/stac/extent.py,sha256=Jb3K4v59eu_h5t429r0762o0zG_LA50iEE-abWNL0e0
 eotdl/curation/stac/parsers.py,sha256=H5IukLA61mpLojeuhWNQdiZk2eiYHAfpJBFxmjdGDso,1529
 eotdl/curation/stac/stac.py,sha256=9GdiB1mV8iyREvXZoJmNJKQGawWyK3h9Eu0hJX5u6O4,13180
 eotdl/datasets/__init__.py,sha256=oFSUD0OLvUS4_iwu_V1iAQ_VAML-8RWDbxgtuFdU5ZA,170
-eotdl/datasets/download.py,sha256=tgWCGhRfy8EtH_6Q6Hn8CHaKvIkPZ03iCW5_Aotrkok,3812
-eotdl/datasets/ingest.py,sha256=Z-PFfSp1tO3z7cusADPrniOdbNpYqIPYpP74vpWrqJ8,4020
+eotdl/datasets/download.py,sha256=WM9XJqKQmsUcqnM2S9fKOHGlaoJ3FVePIQhobckHZrw,3750
+eotdl/datasets/ingest.py,sha256=1XA5wwtx1VwEY7dTqb9dH3cBMlrRYuMTAuZt9reRMeo,3903
 eotdl/datasets/metadata.py,sha256=L23_EziGVSDJ-WZbYUYNN22GrgbCdMGxwJhgK9uzW0U,390
 eotdl/datasets/retrieve.py,sha256=DJz5K1bCLizg9YNwBnhHMFzcxMXar2socYkFONdSL4c,1041
-eotdl/datasets/usecases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-eotdl/datasets/usecases/datasets/DownloadFile.py,sha256=PoP0Dl2LBshKzbgArgxwxIVs-KT2IsfZKi0qSyiBsoI,936
-eotdl/datasets/usecases/datasets/DownloadFileURL.py,sha256=6OcNuCys45MXu9-7GtES0zg2QmY02EAkj_P3cVKVUvo,603
-eotdl/datasets/usecases/datasets/IngestDataset.py,sha256=d2H5nPXsEj-LhZpWGwNDSPs9uYNXRO2V07xsTFygQDc,953
-eotdl/datasets/usecases/datasets/IngestLargeDataset.py,sha256=yRl4eqDCNPcmbU0rrtooWDq1OPpi88rSFSl8C4RE7oM,1424
-eotdl/datasets/usecases/datasets/IngestLargeDatasetParallel.py,sha256=G6uVmpZLkn6lqUVQkjw0ne1xlBcJKHoJik4xLnlqd6o,1612
-eotdl/datasets/usecases/datasets/IngestSTAC.py,sha256=QAMLSkMFlc-ic0JWaIdAf0SXPkYcawy-RJqfvDqTD7A,3075
-eotdl/datasets/usecases/datasets/__init__.py,sha256=Tx4ISHtcjbu4KUVgjTac_pjpT0vsN-QpkIwtyx8xUYg,383
 eotdl/files/__init__.py,sha256=wStggK9L_Ni3vdZkKtKmKVdAaZRVuiMbsjYBCrJ8rxs,53
-eotdl/files/ingest.py,sha256=EIElSanwBgGUBniEBDkBf1SG-0mPkx8o0Bd1-Y1l6oA,6015
+eotdl/files/ingest.py,sha256=ObLOsf2fuhRRHj5WSby6SDqoakTDozelOS0sTCN7BkQ,5943
 eotdl/models/__init__.py,sha256=fe1VfnlOxlfviphkkzaY4q8mkm0QxcdJxB3peScBZYk,108
-eotdl/models/download.py,sha256=NaKpi2nB4tpVnLWaHanprnaD1JDhAj6hA9-SkN1kiJA,4339
-eotdl/models/ingest.py,sha256=Zxv9oL5VOqPXgJgaOROt7v2twRPqzajllpP5ZgyNf3Q,1519
+eotdl/models/download.py,sha256=ClE1V-gIpo7aZY8Q-DIaHL4Cpx5wHvrH6deTuNZclrQ,4017
+eotdl/models/ingest.py,sha256=1NZe60Fl6-nu7SCFgenzFnKlglHCfANh50_GjUl3-is,1507
 eotdl/models/metadata.py,sha256=L23_EziGVSDJ-WZbYUYNN22GrgbCdMGxwJhgK9uzW0U,390
 eotdl/models/retrieve.py,sha256=-Ij7dT4J1p7MW4n13OlPB9OW4tBaBXPwk9dW8IuCZPc,664
-eotdl/repos/APIRepo.py,sha256=tjvtr96d1fhnKEdGDmc90NrwBW7YVCtdsKLLtV82SBE,402
-eotdl/repos/AuthAPIRepo.py,sha256=0ovz92NMOvQsOZOJ_shetO8wsS_80mBf-tDKVfwttH0,760
-eotdl/repos/AuthRepo.py,sha256=6AWOdWgXKAVYJ2pB3Fj6X2KABoH2u-tpUyhEkNMWxX0,1001
-eotdl/repos/DatasetsAPIRepo.py,sha256=2YYOIG_9Yx4D8H7L-HIGdMiIYA6QscCQ0yaJ5jKRLGg,4667
-eotdl/repos/FilesAPIRepo.py,sha256=luln1RI8YviPLxkI-ZKlXn0y_eAYGKZdPVwOyz4vlV4,7626
-eotdl/repos/ModelsAPIRepo.py,sha256=nYBaAmGlD4H7XMpeHMwx7GYacmIoWqfhH9PZ4sRfp90,1201
+eotdl/repos/APIRepo.py,sha256=_czH_z7dg9iZvwi0U4lC99TaShqXNUNfqYGbG4gTsaA,662
+eotdl/repos/AuthAPIRepo.py,sha256=vYCqFawe3xUm2cx4SqVXCvzl8J_sr9rs_MkipYC0bXE,957
+eotdl/repos/AuthRepo.py,sha256=jpzzhINCcDZHRCyrPDsp49h17IlXp2HvX3BB3f5cnb4,1154
+eotdl/repos/DatasetsAPIRepo.py,sha256=wyJbgzSx0PNJo5BAwaQiPj8HV9bNAr-7gIZzNKlAUt0,2144
+eotdl/repos/FilesAPIRepo.py,sha256=vHyEePwNERInSo2d3-USgtx-QkXqA3Jzgo5AtuWrAHU,6768
+eotdl/repos/ModelsAPIRepo.py,sha256=FgtXqgdPSjVwRXY-YTUT6aeRH8oDX9KvioQd1Ov7Nh0,1169
 eotdl/repos/__init__.py,sha256=WvX5TP49k7yYb5dWWNjv5kzbdluO3dJ4LqjQxRIOUVc,222
 eotdl/shared/__init__.py,sha256=mF7doJC8Z5eTPmB01UQvPivThZac32DRY33T6qshXfg,41
 eotdl/shared/checksum.py,sha256=4IB6N9jRO0chMDNJzpdnFDhC9wcFF9bO5oHq2HodcHw,479
@@ -77,8 +69,8 @@ eotdl/tools/metadata.py,sha256=RvNmoMdfEKoo-DzhEAqL-f9ZCjIe_bsdHQwACMk6w1E,1664
 eotdl/tools/paths.py,sha256=yWhOtVxX4NxrDrrBX2fuye5N1mAqrxXFy_eA7dffd84,1152
 eotdl/tools/stac.py,sha256=ovXdrPm4Sn9AAJmrP88WnxDmq2Ut-xPoscjphxz3Iyo,5763
 eotdl/tools/time_utils.py,sha256=qJ3-rk1I7ne722SLfAP6-59kahQ0vLQqIf9VpOi0Kpg,4691
-eotdl/tools/tools.py,sha256=ESmvh-gCyWHmXsOdqtB70EmdRK_Vzap6kVOuYmlp5gw,6353
-eotdl-2023.11.3.post4.dist-info/METADATA,sha256=bjG_M5-N0wC3pWBJ64UBXdiMtZYQDUti1Czqe6VK-Yc,3998
-eotdl-2023.11.3.post4.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-eotdl-2023.11.3.post4.dist-info/entry_points.txt,sha256=s6sfxUfRrSX2IP2UbrzTFTvRCtLgw3_OKcHlOKf_5F8,39
-eotdl-2023.11.3.post4.dist-info/RECORD,,
+eotdl/tools/tools.py,sha256=Tl4_v2ejkQo_zyZek8oofJwoYcdVosdOwW1C0lvWaNM,6354
+eotdl-2024.2.15.dist-info/METADATA,sha256=PvicntB6mwav_KhBYF5YGmq1z7JPu3OQvkQLficNK84,3914
+eotdl-2024.2.15.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+eotdl-2024.2.15.dist-info/entry_points.txt,sha256=s6sfxUfRrSX2IP2UbrzTFTvRCtLgw3_OKcHlOKf_5F8,39
+eotdl-2024.2.15.dist-info/RECORD,,

eotdl/datasets/usecases/datasets/DownloadFile.py

@@ -1,30 +0,0 @@
-from pydantic import BaseModel
-from ....utils import calculate_checksum
-
-
-class DownloadFile:
-    def __init__(self, repo, retrieve_dataset, logger):
-        self.repo = repo
-        self.retrieve_dataset = retrieve_dataset
-        self.logger = logger if logger else print
-
-    class Inputs(BaseModel):
-        dataset: str
-        file: str
-        path: str = None
-        user: dict
-        checksum: str
-
-    class Outputs(BaseModel):
-        dst_path: str
-
-    def __call__(self, inputs: Inputs) -> Outputs:
-        dataset = self.retrieve_dataset(inputs.dataset)
-        dst_path = self.repo.download_file(
-            inputs.dataset, inputs.file, inputs.user["id_token"], inputs.path
-        )
-        checksum = calculate_checksum(dst_path)
-        self.logger(f"Checksum: {checksum}")
-        if dataset["checksum"] != checksum:
-            self.logger("Checksums do not match")
-        return self.Outputs(dst_path=dst_path)

eotdl/datasets/usecases/datasets/DownloadFileURL.py

@@ -1,22 +0,0 @@
-from pydantic import BaseModel
-
-
-class DownloadFileURL:
-    def __init__(self, repo, logger, progress=True):
-        self.repo = repo
-        self.logger = logger if logger else print
-        self.progress = progress
-
-    class Inputs(BaseModel):
-        url: str
-        path: str = None
-        user: dict
-
-    class Outputs(BaseModel):
-        dst_path: str
-
-    def __call__(self, inputs: Inputs) -> Outputs:
-        dst_path = self.repo.download_file_url(
-            inputs.url, inputs.path, inputs.user["id_token"], progress=self.progress
-        )
-        return self.Outputs(dst_path=dst_path)

eotdl/datasets/usecases/datasets/IngestDataset.py

@@ -1,30 +0,0 @@
-from pydantic import BaseModel
-
-
-class IngestDataset:
-    def __init__(self, repo, logger):
-        self.repo = repo
-        self.logger = logger if logger else print
-
-    class Inputs(BaseModel):
-        name: str
-        description: str
-        path: str = None
-        user: dict
-
-    class Outputs(BaseModel):
-        dataset: dict
-
-    def __call__(self, inputs: Inputs) -> Outputs:
-        # allow only zip files
-        if not inputs.path.endswith(".zip"):
-            raise Exception("Only zip files are allowed")
-        self.logger("Ingesting dataset...")
-        data, error = self.repo.ingest_dataset(
-            inputs.name, inputs.description, inputs.path, inputs.user["id_token"]
-        )
-        # response = self.repo.ingest_large_dataset(inputs.name, inputs.description, inputs.path, inputs.user['id_token'])
-        if error:
-            raise Exception(error)
-        self.logger("Done")
-        return self.Outputs(dataset=data)

eotdl/datasets/usecases/datasets/IngestLargeDataset.py

@@ -1,43 +0,0 @@
-from pydantic import BaseModel
-from ....utils import calculate_checksum
-
-
-class IngestLargeDataset:
-    def __init__(self, repo, logger):
-        self.repo = repo
-        self.logger = logger if logger else print
-
-    class Inputs(BaseModel):
-        name: str
-        path: str = None
-        user: dict
-
-    class Outputs(BaseModel):
-        dataset: dict
-
-    def __call__(self, inputs: Inputs) -> Outputs:
-        data, error = self.repo.retrieve_dataset(inputs.name)
-        if data:
-            raise Exception("Dataset already exists")
-        # allow only zip files
-        if not inputs.path.endswith(".zip"):
-            raise Exception("Only zip files are allowed")
-        self.logger("Computing checksum...")
-        checksum = calculate_checksum(inputs.path)
-        self.logger(checksum)
-        self.logger("Ingesting dataset...")
-        id_token = inputs.user["id_token"]
-        dataset_id, upload_id, parts = self.repo.prepare_large_upload(
-            inputs.name, id_token, checksum
-        )
-        self.repo.ingest_large_dataset(
-            inputs.path, upload_id, dataset_id, id_token, parts
-        )
-        self.logger("\nCompleting upload...")
-        data, error = self.repo.complete_upload(
-            inputs.name, id_token, upload_id, dataset_id, checksum
-        )
-        if error:
-            raise Exception(error)
-        self.logger("Done")
-        return self.Outputs(dataset=data)

eotdl/datasets/usecases/datasets/IngestLargeDatasetParallel.py

@@ -1,51 +0,0 @@
-from pydantic import BaseModel
-from eotdl.eotdl.datasets.utils import calculate_checksum
-
-
-class IngestLargeDatasetParallel:
-    def __init__(self, repo, logger):
-        self.repo = repo
-        self.logger = logger
-
-    class Inputs(BaseModel):
-        name: str
-        path: str = None
-        user: dict
-        threads: int = 0
-
-    class Outputs(BaseModel):
-        dataset: dict
-
-    def __call__(self, inputs: Inputs) -> Outputs:
-        data, error = self.repo.retrieve_dataset(inputs.name)
-        if data:
-            raise Exception("Dataset already exists")
-        # allow only zip files
-        if not inputs.path.endswith(".zip"):
-            raise Exception("Only zip files are allowed")
-        self.logger("Computing checksum...")
-        checksum = calculate_checksum(
-            inputs.path
-        )  # should do this at chunk level, before and after
-        self.logger(checksum)
-        self.logger("Ingesting dataset...")
-        id_token = inputs.user["id_token"]
-        dataset_id, upload_id, parts = self.repo.prepare_large_upload(
-            inputs.name, id_token, checksum
-        )
-        self.repo.ingest_large_dataset_parallel(
-            inputs.path,
-            upload_id,
-            dataset_id,
-            id_token,
-            parts,
-            inputs.threads,
-        )
-        self.logger("\nCompleting upload...")
-        data, error = self.repo.complete_upload(
-            inputs.name, id_token, upload_id, dataset_id, checksum
-        )
-        if error:
-            raise Exception(error)
-        self.logger("Done")
-        return self.Outputs(dataset=data)