eotdl 2023.6.14.post10__py3-none-any.whl → 2023.7.19__py3-none-any.whl

eotdl/src/repos/APIRepo.py

@@ -6,11 +6,13 @@ from concurrent.futures import ThreadPoolExecutor
 import time
 import multiprocessing
 import hashlib
+import geopandas as gpd
 
 
 class APIRepo:
     def __init__(self, url=os.getenv("EOTDL_API_URL", "https://api.eotdl.com/")):
         self.url = url
+        # print(self.url)
 
     def login(self):
         return requests.get(self.url + "auth/login")
@@ -22,6 +24,16 @@ class APIRepo:
         response = requests.get(self.url + "auth/logout")
         return response.json()["logout_url"]
 
+    def create_dataset(self, metadata, id_token):
+        response = requests.post(
+            self.url + "datasets",
+            json=metadata,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if response.status_code == 200:
+            return response.json(), None
+        return None, response.json()["detail"]
+
     def retrieve_datasets(self):
         return requests.get(self.url + "datasets").json()
 
@@ -31,12 +43,10 @@ class APIRepo:
             return response.json(), None
         return None, response.json()["detail"]
 
-    def download_dataset(self, dataset_id, id_token, path):
-        url = self.url + "datasets/" + dataset_id + "/download"
+    def download_file(self, dataset, dataset_id, file, id_token, path):
+        url = self.url + "datasets/" + dataset_id + "/download/" + file
         headers = {"Authorization": "Bearer " + id_token}
-        if path is None:
-            path = str(Path.home()) + "/.eotdl/datasets"
-            os.makedirs(path, exist_ok=True)
+        path = f"{path}/{file}"
         with requests.get(url, headers=headers, stream=True) as r:
             r.raise_for_status()
             total_size = int(r.headers.get("content-length", 0))
@@ -44,10 +54,6 @@ class APIRepo:
             progress_bar = tqdm(
                 total=total_size, unit="iB", unit_scale=True, unit_divisor=1024
             )
-            filename = r.headers.get("content-disposition").split("filename=")[1][1:-1]
-            path = f"{path}/{filename}"
-            if os.path.exists(path):
-                raise Exception("File already exists")
             with open(path, "wb") as f:
                 for chunk in r.iter_content(block_size):
                     progress_bar.update(len(chunk))
@@ -56,6 +62,27 @@ class APIRepo:
             progress_bar.close()
             return path
 
+    def ingest_file(self, file, dataset_id, id_token, checksum=None):
+        reponse = requests.post(
+            self.url + "datasets/" + dataset_id,
+            files={"file": open(file, "rb")},
+            data={"checksum": checksum} if checksum else None,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if reponse.status_code != 200:
+            return None, reponse.json()["detail"]
+        return reponse.json(), None
+
+    def ingest_file_url(self, file, dataset, id_token):
+        reponse = requests.post(
+            self.url + "datasets/url",
+            json={"dataset": dataset, "url": file},
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if reponse.status_code != 200:
+            return None, reponse.json()["detail"]
+        return reponse.json(), None
+
     def read_in_chunks(self, file_object, CHUNK_SIZE):
         while True:
             data = file_object.read(CHUNK_SIZE)
@@ -63,18 +90,21 @@ class APIRepo:
                 break
             yield data
 
-    def prepare_large_upload(self, name, id_token, checksum):
-        url = self.url + "datasets/chunk?name=" + name + "&checksum=" + checksum
-        response = requests.get(url, headers={"Authorization": "Bearer " + id_token})
+    def prepare_large_upload(self, file, dataset_id, checksum, id_token):
+        filename = Path(file).name
+        response = requests.post(
+            self.url + f"datasets/{dataset_id}/uploadId",
+            json={"name": filename, "checksum": checksum},
+            headers={"Authorization": "Bearer " + id_token},
+        )
         if response.status_code != 200:
             raise Exception(response.json()["detail"])
         data = response.json()
-        dataset_id, upload_id, parts = (
-            data["dataset_id"],
+        upload_id, parts = (
             data["upload_id"],
             data["parts"] if "parts" in data else [],
         )
-        return dataset_id, upload_id, parts
+        return upload_id, parts
 
     def get_chunk_size(self, content_size):
         # adapt chunk size to content size to avoid S3 limits (10000 parts, 500MB per part, 5TB per object)
@@ -85,37 +115,31 @@ class APIRepo:
             chunk_size = 1024 * 1024 * 500  # 0.5 GB (up to 5 TB, 10000 parts)
         return chunk_size
 
-    def ingest_large_dataset(self, path, upload_id, dataset_id, id_token, parts):
-        content_path = os.path.abspath(path)
+    def ingest_large_dataset(self, file, upload_id, id_token, parts):
+        content_path = os.path.abspath(file)
         content_size = os.stat(content_path).st_size
         chunk_size = self.get_chunk_size(content_size)
         total_chunks = content_size // chunk_size
-        url = self.url + "datasets/chunk"
-        headers = {
-            "Authorization": "Bearer " + id_token,
-            "Upload-Id": upload_id,
-            "Dataset-Id": dataset_id,
-        }
         # upload chunks sequentially
         pbar = tqdm(
             self.read_in_chunks(open(content_path, "rb"), chunk_size),
             total=total_chunks,
         )
         index = 0
-        parts_checkusms = []
         for chunk in pbar:
             part = index // chunk_size + 1
             offset = index + len(chunk)
             index = offset
             if part not in parts:
-                headers["Part-Number"] = str(part)
                 checksum = hashlib.md5(chunk).hexdigest()
-                parts_checkusms.append(checksum)
-                headers["Checksum"] = checksum
-                file = {"file": chunk}
-                r = requests.post(url, files=file, headers=headers)
-                if r.status_code != 200:
-                    return None, r.json()["detail"]
+                response = requests.post(
+                    self.url + "datasets/chunk/" + upload_id,
+                    files={"file": chunk},
+                    data={"part_number": part, "checksum": checksum},
+                    headers={"Authorization": "Bearer " + id_token},
+                )
+                if response.status_code != 200:
+                    raise Exception(response.json()["detail"])
             pbar.set_description(
                 "{:.2f}/{:.2f} MB".format(
                     offset / 1024 / 1024, content_size / 1024 / 1024
@@ -124,16 +148,10 @@ class APIRepo:
         pbar.close()
         return
 
-    def complete_upload(self, name, id_token, upload_id, dataset_id, checksum):
-        url = self.url + "datasets/complete"
+    def complete_upload(self, id_token, upload_id):
         r = requests.post(
-            url,
-            json={"name": name, "checksum": checksum},
-            headers={
-                "Authorization": "Bearer " + id_token,
-                "Upload-Id": upload_id,
-                "Dataset-Id": dataset_id,
-            },
+            self.url + "datasets/complete/" + upload_id,
+            headers={"Authorization": "Bearer " + id_token},
         )
         if r.status_code != 200:
             return None, r.json()["detail"]
@@ -200,63 +218,29 @@ class APIRepo:
             return None, r.json()["detail"]
         return r.json(), None
 
-    def ingest_large_dataset_parallel(
-        self,
-        path,
-        upload_id,
-        dataset_id,
-        id_token,
-        parts,
-        threads,
-    ):
-        # Create thread pool executor
-        max_workers = threads if threads > 0 else multiprocessing.cpu_count()
-        executor = ThreadPoolExecutor(max_workers=max_workers)
-
-        # Divide file into chunks and create tasks for each chunk
-        offset = 0
-        tasks = []
-        content_path = os.path.abspath(path)
-        content_size = os.stat(content_path).st_size
-        chunk_size = self.get_chunk_size(content_size)
-        total_chunks = content_size // chunk_size
-        while offset < content_size:
-            chunk_end = min(offset + chunk_size, content_size)
-            part = str(offset // chunk_size + 1)
-            if part not in parts:
-                tasks.append((offset, chunk_end, part))
-            offset = chunk_end
-
-        # Define the function that will upload each chunk
-        def upload_chunk(start, end, part):
-            # print(f"Uploading chunk {start} - {end}", part)
-            with open(content_path, "rb") as f:
-                f.seek(start)
-                chunk = f.read(end - start)
-            checksum = hashlib.md5(chunk).hexdigest()
-            response = requests.post(
-                self.url + "datasets/chunk",
-                files={"file": chunk},
-                headers={
-                    "Authorization": "Bearer " + id_token,
-                    "Upload-Id": upload_id,
-                    "Dataset-Id": dataset_id,
-                    "Checksum": checksum,
-                    "Part-Number": str(part),
-                },
-            )
-            if response.status_code != 200:
-                print(f"Failed to upload chunk {start} - {end}")
-            return response
+    def delete_file(self, dataset_id, file_name, id_token):
+        response = requests.delete(
+            self.url + "datasets/" + dataset_id + "/file/" + file_name,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if response.status_code != 200:
+            return None, response.json()["detail"]
+        return response.json(), None
 
-        # Submit each task to the executor
-        with tqdm(total=total_chunks) as pbar:
-            futures = []
-            for task in tasks:
-                future = executor.submit(upload_chunk, *task)
-                future.add_done_callback(lambda p: pbar.update())
-                futures.append(future)
+    def ingest_stac(self, stac_json, dataset, id_token):
+        reponse = requests.post(
+            self.url + "datasets/stac",
+            json={"dataset": dataset, "stac": stac_json},
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if reponse.status_code != 200:
+            return None, reponse.json()["detail"]
+        return reponse.json(), None
 
-            # Wait for all tasks to complete
-            for future in futures:
-                future.result()
+    def download_stac(self, dataset_id, id_token):
+        url = self.url + "datasets/" + dataset_id + "/download"
+        headers = {"Authorization": "Bearer " + id_token}
+        response = requests.get(url, headers=headers)
+        if response.status_code != 200:
+            return None, response.json()["detail"]
+        return gpd.GeoDataFrame.from_features(response.json()["features"]), None

eotdl/src/usecases/datasets/DownloadDataset.py

@@ -1,27 +1,79 @@
 from pydantic import BaseModel
 from ....src.utils import calculate_checksum
+from ....curation.stac import STACDataFrame
+from pathlib import Path
+import os
 
 
 class DownloadDataset:
-    def __init__(self, repo, logger):
+    def __init__(self, repo, retrieve_dataset, logger):
         self.repo = repo
+        self.retrieve_dataset = retrieve_dataset
         self.logger = logger if logger else print
 
     class Inputs(BaseModel):
         dataset: str
+        file: str = None
         path: str = None
         user: dict
-        checksum: str
 
     class Outputs(BaseModel):
         dst_path: str
 
-    def __call__(self, inputs: Inputs) -> Outputs:
-        dst_path = self.repo.download_dataset(
-            inputs.dataset, inputs.user["id_token"], inputs.path
+    def download(self, dataset, dataset_id, file, checksum, path, user):
+        self.logger(f"Downloading {file}")
+        dst_path = self.repo.download_file(
+            dataset, dataset_id, file, user["id_token"], path
         )
-        checksum = calculate_checksum(dst_path)
-        self.logger(f"Checksum: {checksum}")
-        if inputs.checksum != checksum:
-            self.logger("Checksums do not match")
-        return self.Outputs(dst_path=dst_path)
+        if calculate_checksum(dst_path) != checksum:
+            self.logger(f"Checksum for {file} does not match")
+        self.logger(f"Done")
+        return dst_path
+
+    def __call__(self, inputs: Inputs) -> Outputs:
+        dataset = self.retrieve_dataset(inputs.dataset)
+        if inputs.path is None:
+            download_path = str(Path.home()) + "/.eotdl/datasets/" + inputs.dataset
+        else:
+            download_path = inputs.path + "/" + inputs.dataset
+        os.makedirs(download_path, exist_ok=True)
+        if dataset["quality"] == 0:
+            if inputs.file:
+                files = [f for f in dataset["files"] if f["name"] == inputs.file]
+                if not files:
+                    raise Exception(f"File {inputs.file} not found")
+                if len(files) > 1:
+                    raise Exception(f"Multiple files with name {inputs.file} found")
+                dst_path = self.download(
+                    inputs.dataset,
+                    dataset["id"],
+                    inputs.file,
+                    files[0]["checksum"],
+                    download_path,
+                    inputs.user,
+                )
+                return self.Outputs(dst_path=dst_path)
+            for file in dataset["files"]:
+                dst_path = self.download(
+                    inputs.dataset,
+                    dataset["id"],
+                    file["name"],
+                    file["checksum"],
+                    download_path,
+                    inputs.user,
+                )
+            return self.Outputs(dst_path="/".join(dst_path.split("/")[:-1]))
+        else:
+            gdf, error = self.repo.download_stac(
+                dataset["id"],
+                inputs.user["id_token"],
+            )
+            if error:
+                raise Exception(error)
+            df = STACDataFrame(gdf)
+            # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
+            path = inputs.path
+            if path is None:
+                path = str(Path.home()) + "/.eotdl/datasets/" + dataset["name"]
+            df.to_stac(path)
+            return self.Outputs(dst_path=path)

eotdl/src/usecases/datasets/DownloadFile.py

@@ -0,0 +1,30 @@
+from pydantic import BaseModel
+from ....src.utils import calculate_checksum
+
+
+class DownloadFile:
+    def __init__(self, repo, retrieve_dataset, logger):
+        self.repo = repo
+        self.retrieve_dataset = retrieve_dataset
+        self.logger = logger if logger else print
+
+    class Inputs(BaseModel):
+        dataset: str
+        file: str
+        path: str = None
+        user: dict
+        checksum: str
+
+    class Outputs(BaseModel):
+        dst_path: str
+
+    def __call__(self, inputs: Inputs) -> Outputs:
+        dataset = self.retrieve_dataset(inputs.dataset)
+        dst_path = self.repo.download_file(
+            inputs.dataset, inputs.file, inputs.user["id_token"], inputs.path
+        )
+        checksum = calculate_checksum(dst_path)
+        self.logger(f"Checksum: {checksum}")
+        if dataset["checksum"] != checksum:
+            self.logger("Checksums do not match")
+        return self.Outputs(dst_path=dst_path)

eotdl/src/usecases/datasets/IngestFile.py

@@ -0,0 +1,60 @@
+from pydantic import BaseModel
+import os
+import typing
+
+from ....src.utils import calculate_checksum
+
+
+class IngestFile:
+    def __init__(self, repo, allowed_extensions, logger):
+        self.repo = repo
+        self.allowed_extensions = allowed_extensions
+        self.logger = logger if logger else print
+
+    class Inputs(BaseModel):
+        file: typing.Any
+        dataset_id: str
+        user: dict
+
+    class Outputs(BaseModel):
+        data: dict
+
+    def __call__(self, inputs: Inputs) -> Outputs:
+        # validate file extension
+        extension = os.path.splitext(inputs.file)[1]
+        if extension not in self.allowed_extensions:
+            raise Exception(
+                f"Only {', '.join(self.allowed_extensions)} files are allowed"
+            )
+        id_token = inputs.user["id_token"]
+        self.logger(f"Uploading file {inputs.file}...")
+        # if inputs.file.startswith("http://") or inputs.file.startswith("https://"):
+        #     data, error = self.repo.ingest_file_url(
+        #         inputs.file, inputs.metadata.name, id_token
+        #     )
+        # else:
+        self.logger("Computing checksum...")
+        checksum = calculate_checksum(inputs.file)
+        self.logger(checksum)
+        self.logger("Ingesting file...")
+        filesize = os.path.getsize(inputs.file)
+        # ingest small file
+        if filesize < 1024 * 1024 * 16:  # 16 MB
+            data, error = self.repo.ingest_file(
+                inputs.file, inputs.dataset_id, id_token, checksum
+            )
+            if error:
+                raise Exception(error)
+            self.logger("Done")
+            return self.Outputs(data=data)
+        # ingest large file
+        upload_id, parts = self.repo.prepare_large_upload(
+            inputs.file, inputs.dataset_id, checksum, id_token
+        )
+        self.repo.ingest_large_dataset(inputs.file, upload_id, id_token, parts)
+        self.logger("\nCompleting upload...")
+        data, error = self.repo.complete_upload(id_token, upload_id)
+        if error:
+            raise Exception(error)
+        self.logger("Done")
+        return self.Outputs(data=data)

eotdl/src/usecases/datasets/IngestFolder.py

@@ -0,0 +1,98 @@
+from pydantic import BaseModel
+import os
+from pathlib import Path
+import yaml
+from ...models import Metadata
+
+
+class IngestFolder:
+    def __init__(self, repo, ingest_file, allowed_extensions, logger):
+        self.repo = repo
+        self.ingest_file = ingest_file
+        self.allowed_extensions = allowed_extensions
+        self.logger = logger if logger else print
+
+    class Inputs(BaseModel):
+        folder: Path
+        user: dict
+        force: bool = False
+        delete: bool = False
+
+    class Outputs(BaseModel):
+        dataset: dict
+
+    def __call__(self, inputs: Inputs) -> Outputs:
+        # validate folder
+        self.logger("Uploading directory (only files, not recursive)")
+        items = list(inputs.folder.glob("*"))
+        filtered_items = [item for item in items if item.is_file()]
+        filtered_items = [
+            item for item in filtered_items if item.suffix in self.allowed_extensions
+        ]
+        if len(filtered_items) == 0:
+            raise Exception("No files found in directory")
+        if len(filtered_items) > 10:
+            raise Exception("Too many files in directory, limited to 10")
+        if "metadata.yml" not in [item.name for item in filtered_items]:
+            raise Exception("metadata.yml not found in directory")
+        # load metadata
+        metadata = yaml.safe_load(
+            open(inputs.folder.joinpath("metadata.yml"), "r").read()
+        )
+        metadata = Metadata(**metadata)
+        # remove metadata.yml from files
+        filtered_items = [
+            item for item in filtered_items if item.name != "metadata.yml"
+        ]
+        # create dataset
+        data, error = self.repo.create_dataset(metadata.dict(), inputs.user["id_token"])
+        # dataset may already exists, but if user is owner continue ingesting files
+        current_files = []
+        if error:
+            data, error2 = self.repo.retrieve_dataset(metadata.name)
+            if error2:
+                raise Exception(error)
+            if data["uid"] != inputs.user["sub"]:
+                raise Exception("Dataset already exists.")
+            data["dataset_id"] = data["id"]
+            current_files = [item["name"] for item in data["files"]]
+            if len(current_files) > 0 and not inputs.force:
+                self.logger(
+                    "The following files already exist and will not be uploaded (use --f to force re-upload):"
+                )
+                for item in current_files:
+                    self.logger(f"{item}")
+            # TODO: delete current_files that are not in filtered_items if --delete
+            hanged_files = [
+                file
+                for file in current_files
+                if file not in [item.name for item in filtered_items]
+            ]
+            if len(hanged_files) > 0:
+                self.logger(
+                    "The following files are no longer in your dataset (use --d to delete):"
+                )
+                for item in hanged_files:
+                    self.logger(f"{item}")
+                    if inputs.delete:
+                        self.logger(f"Deleting file {item}...")
+                        _, error = self.repo.delete_file(
+                            data["dataset_id"], item, inputs.user["id_token"]
+                        )
+                        if error:
+                            self.logger(error)
+                        else:
+                            self.logger("Done")
+            filtered_items = [
+                item for item in filtered_items if item.name not in current_files
+            ]
+        dataset_id = data["dataset_id"]
+        # upload files
+        if len(filtered_items) == 0:
+            raise Exception("No files to upload")
+        self.logger("The following files will be uploaded:")
+        for item in filtered_items:
+            self.logger(f"{item.name}")
+        for item in filtered_items:
+            data = self.ingest_file(item, dataset_id, logger=self.logger)
+        return self.Outputs(dataset=data)

eotdl/src/usecases/datasets/IngestSTAC.py

@@ -0,0 +1,42 @@
+from pydantic import BaseModel
+from ....curation.stac import STACDataFrame
+import json
+
+
+class IngestSTAC:
+    def __init__(self, repo, ingest_file, allowed_extensions):
+        self.repo = repo
+        self.ingest_file = ingest_file
+        self.allowed_extensions = allowed_extensions
+
+    class Inputs(BaseModel):
+        stac_catalog: str
+        dataset: str
+        user: dict
+
+    class Outputs(BaseModel):
+        dataset: dict
+
+    def __call__(self, inputs: Inputs) -> Outputs:
+        # load the STAC catalog as a STACsetFrame
+        df = STACDataFrame.from_stac_file(inputs.stac_catalog)
+        # upload all assets to EOTDL
+        for row in df.dropna(subset=["assets"]).iterrows():
+            # for asset in df.assets.dropna().values[:10]:
+            try:
+                for k, v in row[1]["assets"].items():
+                    data = self.ingest_file(
+                        v["href"],
+                        inputs.dataset,
+                        allowed_extensions=self.allowed_extensions + [".tif", ".tiff"],
+                    )
+                    file_url = f"{self.repo.url}datasets/{data['dataset_id']}/download/{data['file_name']}"
+                    df.loc[row[0], "assets"][k]["href"] = file_url
+            except Exception as e:
+                break
+        data, error = self.repo.ingest_stac(
+            json.loads(df.to_json()), inputs.dataset, inputs.user["id_token"]
+        )
+        if error:
+            raise Exception(error)
+        return self.Outputs(dataset=data)

eotdl/src/usecases/datasets/RetrieveDatasets.py

@@ -1,7 +1,8 @@
 from pydantic import BaseModel
 from typing import List
 
-class RetrieveDatasets():
+
+class RetrieveDatasets:
     def __init__(self, repo):
         self.repo = repo
 
@@ -9,9 +10,9 @@ class RetrieveDatasets():
         pass
 
     class Outputs(BaseModel):
-        datasets: List[str]
+        datasets: dict
 
     def __call__(self, inputs: Inputs) -> Outputs:
         data = self.repo.retrieve_datasets()
-        datasets = [d['name'] for d in data]
-        return self.Outputs(datasets=datasets)
+        datasets = {d["name"]: [f["name"] for f in d["files"]] for d in data}
+        return self.Outputs(datasets=datasets)

eotdl/src/usecases/datasets/__init__.py

@@ -3,4 +3,6 @@ from .IngestDataset import IngestDataset
 from .IngestLargeDataset import IngestLargeDataset
 from .RetrieveDataset import RetrieveDataset
 from .RetrieveDatasets import RetrieveDatasets
-from .UpdateDataset import UpdateDataset
+from .IngestFile import IngestFile
+from .IngestFolder import IngestFolder
+from .IngestSTAC import IngestSTAC

eotdl/src/utils.py

@@ -1,17 +1,17 @@
 import hashlib
 
 
-def calculate_checksum(file_path):
-    hasher = hashlib.md5()
-    with open(file_path, "rb") as f:
-        for chunk in iter(lambda: f.read(4096), b""):
-            hasher.update(chunk)
-    return hasher.hexdigest()
+# def calculate_checksum(file_path):
+#     hasher = hashlib.md5()
+#     with open(file_path, "rb") as f:
+#         for chunk in iter(lambda: f.read(4096), b""):
+#             hasher.update(chunk)
+#     return hasher.hexdigest()
 
 
-# def calculate_checksum(file_path):
-#     sha1_hash = hashlib.sha1()
-#     with open(file_path, "rb") as file:
-#         for chunk in iter(lambda: file.read(4096), b""):
-#             sha1_hash.update(chunk)
-#     return sha1_hash.hexdigest()
+def calculate_checksum(file_path):
+    sha1_hash = hashlib.sha1()
+    with open(file_path, "rb") as file:
+        for chunk in iter(lambda: file.read(4096), b""):
+            sha1_hash.update(chunk)
+    return sha1_hash.hexdigest()

{eotdl-2023.6.14.post10.dist-info → eotdl-2023.7.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eotdl
-Version: 2023.6.14.post10
+Version: 2023.7.19
 Summary: Earth Observation Training Data Lab
 License: MIT
 Author: EarthPulse