eotdl 2025.2.10__py3-none-any.whl → 2025.4.2__py3-none-any.whl

eotdl/repos/FilesAPIRepo.py

@@ -11,190 +11,283 @@ class FilesAPIRepo(APIRepo):
     def __init__(self, url=None):
         super().__init__(url)
 
-    def ingest_files_batch(
-        self,
-        batch,  # ziped batch of files
-        checksums,
-        dataset_or_model_id,
-        user,
-        endpoint,
-        version=None,
-    ):
-        url = self.url + f"{endpoint}/{dataset_or_model_id}/batch"
-        if version is not None:
-            url += "?version=" + str(version)
-        reponse = requests.post(
-            url,
-            files={"batch": ("batch.zip", batch)},
-            data={"checksums": checksums},
-            headers=self.generate_headers(user),
-        )
-        return self.format_response(reponse)
-
-    def add_files_batch_to_version(
-        self,
-        batch,
-        dataset_or_model_id,
-        version,
-        user,
-        endpoint,
-    ):
-        reponse = requests.post(
-            self.url + f"{endpoint}/{dataset_or_model_id}/files?version={str(version)}",
-            data={
-                "filenames": [f["path"] for f in batch],
-                "checksums": [f["checksum"] for f in batch],
-            },
-            headers=self.generate_headers(user),
-        )
-        return self.format_response(reponse)
-
     def ingest_file(
-        self, file, dataset_or_model_id, user, checksum, endpoint, version=None
+        self, file_path_or_bytes, file_name, dataset_or_model_id, user, endpoint, version=None
     ):
-        # TODO: ingest file URL
         url = self.url + f"{endpoint}/{dataset_or_model_id}"
         if version is not None:
             url += "?version=" + str(version)
+        # get a presigned url to upload the file directly to the bucket
         reponse = requests.post(
             url,
-            files={"file": open(file, "rb")},
-            data={"checksum": checksum},
+            json={
+                "file_name": file_name,
+                # "file_size": files_size,
+                # "checksum": checksum
+            },
             headers=self.generate_headers(user),
         )
-        return self.format_response(reponse)
+        data, error = self.format_response(reponse)
+        if error:
+            raise Exception(error)
+        # ingest the file
+        error = None
+        try:
+            presigned_url = data["presigned_url"]
+            if isinstance(file_path_or_bytes, (str, bytes)):
+                if isinstance(file_path_or_bytes, str):
+                    # Handle file path
+                    with open(file_path_or_bytes, 'rb') as f:
+                        file_data = f.read()
+                else:
+                    # Handle bytes directly
+                    file_data = file_path_or_bytes
+                # Send file data to presigned URL
+                response = requests.put(presigned_url, data=file_data)
+                response.raise_for_status()
+            else:
+                raise TypeError("file_path_or_bytes must be either a file path string or bytes")
+        except Exception as e:
+            error = str(e)
+        return data, error
 
-    def retrieve_files(self, dataset_or_model_id, endpoint, version=None):
-        url = f"{self.url}{endpoint}/{dataset_or_model_id}/files"
-        if version is not None:
-            url += "?version=" + str(version)
-        response = requests.get(url)
-        return self.format_response(response)
-
-    def download_file(
+    def stage_file(
         self,
         dataset_or_model_id,
         file_name,
         user,
         path,
-        file_version,
         endpoint="datasets",
         progress=False,
     ):
-        url = self.url + f"{endpoint}/{dataset_or_model_id}/download/{file_name}"
-        if file_version is not None:
-            url += "?version=" + str(file_version)
-        return self.download_file_url(url, file_name, path, user, progress=progress)
-
-    def download_file_url(self, url, filename, path, user, progress=False):
-        headers = self.generate_headers(user)
-        path = f"{path}/{filename}"
-        for i in range(1, len(path.split("/")) - 1):
-            # print("/".join(path.split("/")[: i + 1]))
-            os.makedirs("/".join(path.split("/")[: i + 1]), exist_ok=True)
-        with requests.get(url, headers=headers, stream=True) as r:
-            r.raise_for_status()
-            total_size = int(r.headers.get("content-length", 0))
-            block_size = 1024 * 1024 * 10
-            progress = progress and total_size > 1024 * 1024 * 16
-            if progress:
-                progress_bar = tqdm(
-                    total=total_size,
-                    unit="iB",
-                    unit_scale=True,
-                    unit_divisor=1024,
-                    position=1,
-                )
-            with open(path, "wb") as f:
-                for chunk in r.iter_content(block_size):
-                    if progress:
-                        progress_bar.update(len(chunk))
-                    if chunk:
-                        f.write(chunk)
-            if progress:
-                progress_bar.close()
-            return path
-
-    def prepare_large_upload(
-        self, filename, dataset_or_model_id, checksum, user, endpoint
-    ):
-        response = requests.post(
-            self.url + f"{endpoint}/{dataset_or_model_id}/uploadId",
-            json={"filname": filename, "checksum": checksum},
-            headers=self.generate_headers(user),
-        )
-        if response.status_code != 200:
-            raise Exception(response.json()["detail"])
-        data = response.json()
-        upload_id, parts = (
-            data["upload_id"],
-            data["parts"] if "parts" in data else [],
-        )
-        return upload_id, parts
-
-    def get_chunk_size(self, content_size):
-        # adapt chunk size to content size to avoid S3 limits (10000 parts, 500MB per part, 5TB per object)
-        chunk_size = 1024 * 1024 * 10  # 10 MB (up to 100 GB, 10000 parts)
-        if content_size >= 1024 * 1024 * 1024 * 100:  # 100 GB
-            chunk_size = 1024 * 1024 * 100  # 100 MB (up to 1 TB, 10000 parts)
-        elif content_size >= 1024 * 1024 * 1024 * 1000:  # 1 TB
-            chunk_size = 1024 * 1024 * 500  # 0.5 GB (up to 5 TB, 10000 parts)
-        return chunk_size
-
-    def read_in_chunks(self, file_object, CHUNK_SIZE):
-        while True:
-            data = file_object.read(CHUNK_SIZE)
-            if not data:
-                break
-            yield data
-
-    def ingest_large_file(
-        self, file_path, files_size, upload_id, user, parts, endpoint
+        url = self.url + f"{endpoint}/{dataset_or_model_id}/stage/{file_name}"
+        # if file_version is not None:
+        #     url += "?version=" + str(file_version)
+        return self.stage_file_url(url, path, user)
+
+
+    def stage_file_url(
+        self,
+        url,
+        path,
+        user,
     ):
-        print(endpoint)
-        # content_path = os.path.abspath(file)
-        # content_size = os.stat(content_path).st_size
-        chunk_size = self.get_chunk_size(files_size)
-        total_chunks = files_size // chunk_size
-        # upload chunks sequentially
-        pbar = tqdm(
-            self.read_in_chunks(open(file_path, "rb"), chunk_size),
-            total=total_chunks,
-        )
-        index = 0
-        for chunk in pbar:
-            part = index // chunk_size + 1
-            offset = index + len(chunk)
-            index = offset
-            if part not in parts:
-                checksum = hashlib.md5(chunk).hexdigest()
-                response = requests.post(
-                    f"{self.url}{endpoint}/chunk/{upload_id}",
-                    files={"file": chunk},
-                    data={"part_number": part, "checksum": checksum},
-                    headers=self.generate_headers(user),
-                )
-                if response.status_code != 200:
-                    raise Exception(response.json()["detail"])
-            pbar.set_description(
-                "{:.2f}/{:.2f} MB".format(
-                    offset / 1024 / 1024, files_size / 1024 / 1024
-                )
-            )
-        pbar.close()
-        return
-
-    def complete_upload(self, user, upload_id, version, endpoint):
-        r = requests.post(
-            f"{self.url}{endpoint}/complete/{upload_id}?version={version}",
-            headers=self.generate_headers(user),
-        )
-        return self.format_response(r)
+        if '/stage/' in url:  # asset is in EOTDL (can do better...)
+            file_name = url.split("/stage/")[-1] 
+            reponse = requests.get(url, headers=self.generate_headers(user))
+            data, error = self.format_response(reponse)
+            if error:
+                raise Exception(error)
+            presigned_url = data["presigned_url"]
+        else:
+            file_name = url.split("//")[-1]
+            presigned_url = url
+        file_path = f"{path}/{file_name}"
+        for i in range(1, len(file_path.split("/")) - 1):
+            os.makedirs("/".join(file_path.split("/")[: i + 1]), exist_ok=True)
+        try:
+            response = requests.get(presigned_url)
+            response.raise_for_status()  # This will raise an HTTPError for 4XX and 5XX status codes
+            with open(file_path, 'wb') as f:
+                f.write(response.content)
+        except requests.exceptions.HTTPError as e:
+            raise Exception(f"Failed to stage file: {str(e)}")
+        except Exception as e:
+            raise Exception(f"Unexpected error while staging file: {str(e)}")
+        return file_path
     
-    def get_file_stream(self, dataset_id, filename, user, version=None):
-        url = self.url + f"datasets/{dataset_id}/download/{filename}"
-        if version is not None:
-            url += "?version=" + str(version)
-        headers = self.generate_headers(user)
-        response = requests.get(url, headers=headers, stream=True)
-        return BytesIO(response.content)
+    def generate_presigned_url(self, filename, dataset_or_model_id, user, endpoint="datasets"):
+        url = f"{self.url}{endpoint}/{dataset_or_model_id}/stage/{filename}"
+        reponse = requests.get(url, headers=self.generate_headers(user))
+        data, error = self.format_response(reponse)
+        if error:
+            # print("ERROR generate_presigned_url", error)
+            return None
+        return data["presigned_url"]
+
+        # can we download large files?
+
+        # with requests.get(presigned_url, headers=headers, stream=True) as r:
+        #     r.raise_for_status()
+        #     total_size = int(r.headers.get("content-length", 0))
+        #     block_size = 1024 * 1024 * 10
+        #     progress = progress and total_size > 1024 * 1024 * 16
+        #     if progress:
+        #         progress_bar = tqdm(
+        #             total=total_size,
+        #             unit="iB",
+        #             unit_scale=True,
+        #             unit_divisor=1024,
+        #             position=1,
+        #         )
+        #     with open(path, "wb") as f:
+        #         for chunk in r.iter_content(block_size):
+        #             if progress:
+        #                 progress_bar.update(len(chunk))
+        #             if chunk:
+        #                 f.write(chunk)
+        #     if progress:
+        #         progress_bar.close()
+        #     return path
+
+
+
+    # def ingest_files_batch(
+    #     self,
+    #     batch,  # ziped batch of files
+    #     checksums,
+    #     dataset_or_model_id,
+    #     user,
+    #     endpoint,
+    #     version=None,
+    # ):
+    #     url = self.url + f"{endpoint}/{dataset_or_model_id}/batch"
+    #     if version is not None:
+    #         url += "?version=" + str(version)
+    #     reponse = requests.post(
+    #         url,
+    #         files={"batch": ("batch.zip", batch)},
+    #         data={"checksums": checksums},
+    #         headers=self.generate_headers(user),
+    #     )
+    #     return self.format_response(reponse)
+
+    # def add_files_batch_to_version(
+    #     self,
+    #     batch,
+    #     dataset_or_model_id,
+    #     version,
+    #     user,
+    #     endpoint,
+    # ):
+    #     reponse = requests.post(
+    #         self.url + f"{endpoint}/{dataset_or_model_id}/files?version={str(version)}",
+    #         data={
+    #             "filenames": [f["path"] for f in batch],
+    #             "checksums": [f["checksum"] for f in batch],
+    #         },
+    #         headers=self.generate_headers(user),
+    #     )
+    #     return self.format_response(reponse)
+
+    # def retrieve_files(self, dataset_or_model_id, endpoint, version=None):
+    #     url = f"{self.url}{endpoint}/{dataset_or_model_id}/files"
+    #     if version is not None:
+    #         url += "?version=" + str(version)
+    #     response = requests.get(url)
+    #     return self.format_response(response)
+
+
+
+    # def download_file_url(self, url, filename, path, user, progress=False):
+    #     headers = self.generate_headers(user)
+    #     path = f"{path}/{filename}"
+    #     for i in range(1, len(path.split("/")) - 1):
+    #         # print("/".join(path.split("/")[: i + 1]))
+    #         os.makedirs("/".join(path.split("/")[: i + 1]), exist_ok=True)
+    #     with requests.get(url, headers=headers, stream=True) as r:
+    #         r.raise_for_status()
+    #         total_size = int(r.headers.get("content-length", 0))
+    #         block_size = 1024 * 1024 * 10
+    #         progress = progress and total_size > 1024 * 1024 * 16
+    #         if progress:
+    #             progress_bar = tqdm(
+    #                 total=total_size,
+    #                 unit="iB",
+    #                 unit_scale=True,
+    #                 unit_divisor=1024,
+    #                 position=1,
+    #             )
+    #         with open(path, "wb") as f:
+    #             for chunk in r.iter_content(block_size):
+    #                 if progress:
+    #                     progress_bar.update(len(chunk))
+    #                 if chunk:
+    #                     f.write(chunk)
+    #         if progress:
+    #             progress_bar.close()
+    #         return path
+
+    # def prepare_large_upload(
+    #     self, filename, dataset_or_model_id, checksum, user, endpoint
+    # ):
+    #     response = requests.post(
+    #         self.url + f"{endpoint}/{dataset_or_model_id}/uploadId",
+    #         json={"filname": filename, "checksum": checksum},
+    #         headers=self.generate_headers(user),
+    #     )
+    #     if response.status_code != 200:
+    #         raise Exception(response.json()["detail"])
+    #     data = response.json()
+    #     upload_id, parts = (
+    #         data["upload_id"],
+    #         data["parts"] if "parts" in data else [],
+    #     )
+    #     return upload_id, parts
+
+    # def get_chunk_size(self, content_size):
+    #     # adapt chunk size to content size to avoid S3 limits (10000 parts, 500MB per part, 5TB per object)
+    #     chunk_size = 1024 * 1024 * 10  # 10 MB (up to 100 GB, 10000 parts)
+    #     if content_size >= 1024 * 1024 * 1024 * 100:  # 100 GB
+    #         chunk_size = 1024 * 1024 * 100  # 100 MB (up to 1 TB, 10000 parts)
+    #     elif content_size >= 1024 * 1024 * 1024 * 1000:  # 1 TB
+    #         chunk_size = 1024 * 1024 * 500  # 0.5 GB (up to 5 TB, 10000 parts)
+    #     return chunk_size
+
+    # def read_in_chunks(self, file_object, CHUNK_SIZE):
+    #     while True:
+    #         data = file_object.read(CHUNK_SIZE)
+    #         if not data:
+    #             break
+    #         yield data
+
+    # def ingest_large_file(
+    #     self, file_path, files_size, upload_id, user, parts, endpoint
+    # ):
+    #     print(endpoint)
+    #     # content_path = os.path.abspath(file)
+    #     # content_size = os.stat(content_path).st_size
+    #     chunk_size = self.get_chunk_size(files_size)
+    #     total_chunks = files_size // chunk_size
+    #     # upload chunks sequentially
+    #     pbar = tqdm(
+    #         self.read_in_chunks(open(file_path, "rb"), chunk_size),
+    #         total=total_chunks,
+    #     )
+    #     index = 0
+    #     for chunk in pbar:
+    #         part = index // chunk_size + 1
+    #         offset = index + len(chunk)
+    #         index = offset
+    #         if part not in parts:
+    #             checksum = hashlib.md5(chunk).hexdigest()
+    #             response = requests.post(
+    #                 f"{self.url}{endpoint}/chunk/{upload_id}",
+    #                 files={"file": chunk},
+    #                 data={"part_number": part, "checksum": checksum},
+    #                 headers=self.generate_headers(user),
+    #             )
+    #             if response.status_code != 200:
+    #                 raise Exception(response.json()["detail"])
+    #         pbar.set_description(
+    #             "{:.2f}/{:.2f} MB".format(
+    #                 offset / 1024 / 1024, files_size / 1024 / 1024
+    #             )
+    #         )
+    #     pbar.close()
+    #     return
+
+    # def complete_upload(self, user, upload_id, version, endpoint):
+    #     r = requests.post(
+    #         f"{self.url}{endpoint}/complete/{upload_id}?version={version}",
+    #         headers=self.generate_headers(user),
+    #     )
+    #     return self.format_response(r)
+    
+    # def get_file_stream(self, dataset_id, filename, user, version=None):
+    #     url = self.url + f"datasets/{dataset_id}/download/{filename}"
+    #     if version is not None:
+    #         url += "?version=" + str(version)
+    #     headers = self.generate_headers(user)
+    #     response = requests.get(url, headers=headers, stream=True)
+    #     return BytesIO(response.content)

eotdl/repos/ModelsAPIRepo.py

@@ -20,6 +20,10 @@ class ModelsAPIRepo(APIRepo):
         response = requests.get(url)
         return self.format_response(response)
 
+    def retrieve_model(self, name):
+        response = requests.get(self.url + "models?name=" + name)
+        return self.format_response(response)
+    
     def create_model(self, metadata, user):
         response = requests.post(
             self.url + "models",
@@ -28,53 +32,57 @@ class ModelsAPIRepo(APIRepo):
         )
         return self.format_response(response)
 
-    def retrieve_model(self, name):
-        response = requests.get(self.url + "models?name=" + name)
-        return self.format_response(response)
-
-    def create_version(self, model_id, user):
+    def complete_ingestion(self, model_id, version, size, user):
         response = requests.post(
-            self.url + "models/version/" + model_id,
+            self.url + "models/complete/" + model_id,
+            json={"version": version, "size": size},
             headers=self.generate_headers(user),
         )
         return self.format_response(response)
 
-    def update_model(
-        self, model_id, authors, source, license, thumbnail, content, user
-    ):
-        response = requests.put(
-            self.url + f"models/{model_id}",
-            json={
-                "authors": authors,
-                "source": source,
-                "license": license,
-                "thumbnail": thumbnail,
-                "description": content,
-            },
-            headers=self.generate_headers(user),
-        )
-        return self.format_response(response)
+    # def create_version(self, model_id, user):
+    #     response = requests.post(
+    #         self.url + "models/version/" + model_id,
+    #         headers=self.generate_headers(user),
+    #     )
+    #     return self.format_response(response)
 
-    def create_stac_model(self, name, user):
-        response = requests.post(
-            self.url + "models/stac",
-            json={"name": name},
-            headers=self.generate_headers(user),
-        )
-        return self.format_response(response)
+    # def update_model(
+    #     self, model_id, authors, source, license, thumbnail, content, user
+    # ):
+    #     response = requests.put(
+    #         self.url + f"models/{model_id}",
+    #         json={
+    #             "authors": authors,
+    #             "source": source,
+    #             "license": license,
+    #             "thumbnail": thumbnail,
+    #             "description": content,
+    #         },
+    #         headers=self.generate_headers(user),
+    #     )
+    #     return self.format_response(response)
 
-    def ingest_stac(self, stac_json, model_id, user):
-        response = requests.put(
-            self.url + f"models/stac/{model_id}",
-            json={"stac": stac_json},
-            headers=self.generate_headers(user),
-        )
-        return self.format_response(response)
+    # def create_stac_model(self, name, user):
+    #     response = requests.post(
+    #         self.url + "models/stac",
+    #         json={"name": name},
+    #         headers=self.generate_headers(user),
+    #     )
+    #     return self.format_response(response)
+
+    # def ingest_stac(self, stac_json, model_id, user):
+    #     response = requests.put(
+    #         self.url + f"models/stac/{model_id}",
+    #         json={"stac": stac_json},
+    #         headers=self.generate_headers(user),
+    #     )
+    #     return self.format_response(response)
 
-    def download_stac(self, model_id, user):
-        url = self.url + "models/" + model_id + "/download"
-        headers = self.generate_headers(user)
-        response = requests.get(url, headers=headers)
-        if response.status_code != 200:
-            return None, response.json()["detail"]
-        return gpd.GeoDataFrame.from_features(response.json()["features"]), None
+    # def download_stac(self, model_id, user):
+    #     url = self.url + "models/" + model_id + "/download"
+    #     headers = self.generate_headers(user)
+    #     response = requests.get(url, headers=headers)
+    #     if response.status_code != 200:
+    #         return None, response.json()["detail"]
+    #     return gpd.GeoDataFrame.from_features(response.json()["features"]), None

eotdl/repos/STACAPIRepo.py

@@ -0,0 +1,40 @@
+import requests
+import geopandas as gpd
+
+from ..repos import APIRepo
+
+
+class STACAPIRepo(APIRepo):
+    def __init__(self, url=None):
+        super().__init__(url)
+
+    def status(self):
+        response = requests.get(self.url + "stac")
+        return self.format_response(response)
+    
+    def collections(self):
+        response = requests.get(self.url + "stac/collections")
+        return self.format_response(response)
+
+    def collection(self, collection_id):
+        response = requests.get(self.url + f"stac/collections/{collection_id}")
+        return self.format_response(response)
+
+    def items(self, collection_id):
+        response = requests.get(self.url + f"stac/collections/{collection_id}/items")
+        return self.format_response(response)
+
+    def item(self, collection_id, item_id):
+        response = requests.get(self.url + f"stac/collections/{collection_id}/items/{item_id}")
+        return self.format_response(response)
+    
+    def search(self, collection_id, query):
+        body = {"collection_id": collection_id}
+        if query is not None:
+            body["query"] = query
+        response = requests.post(self.url + f"stac/search", json=body)
+        return self.format_response(response)
+    
+    def search_columns(self, collection_id):
+        response = requests.get(self.url + f"stac/search?collection={collection_id}")
+        return self.format_response(response)

eotdl/repos/__init__.py

@@ -4,3 +4,4 @@ from .AuthAPIRepo import AuthAPIRepo
 from .DatasetsAPIRepo import DatasetsAPIRepo
 from .FilesAPIRepo import FilesAPIRepo
 from .ModelsAPIRepo import ModelsAPIRepo
+from .STACAPIRepo import STACAPIRepo

eotdl/tools/time_utils.py

@@ -131,11 +131,11 @@ def get_day_between(
     Get the day between two dates
     """
     if isinstance(from_date, str):
-        from_date = datetime.strptime(from_date, "%Y-%m-%dT%H:%M:%SZ")
+        from_date = format_time_acquired(from_date)
     if isinstance(to_date, str):
-        to_date = datetime.strptime(to_date, "%Y-%m-%dT%H:%M:%SZ")
+        to_date = format_time_acquired(to_date)
 
-    date_between = from_date + timedelta(days=1)
+    date_between = from_date + (to_date - from_date) / 2
     date_between = date_between.strftime("%Y-%m-%d")
 
     return date_between

{eotdl-2025.2.10.dist-info → eotdl-2025.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eotdl
-Version: 2025.2.10
+Version: 2025.4.2
 Summary: Earth Observation Training Data Lab
 Author-email: earthpulse <it@earthpulse.es>
 License-Expression: MIT