eotdl 2023.6.27__tar.gz → 2023.7.19__tar.gz

{eotdl-2023.6.27 → eotdl-2023.7.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eotdl
-Version: 2023.6.27
+Version: 2023.7.19
 Summary: Earth Observation Training Data Lab
 License: MIT
 Author: EarthPulse

{eotdl-2023.6.27 → eotdl-2023.7.19}/eotdl/cli.py

@@ -6,6 +6,5 @@ app = typer.Typer()
 app.add_typer(auth.app, name="auth")
 app.add_typer(datasets.app, name="datasets")
 
-
 if __name__ == "__main__":
     app()

{eotdl-2023.6.27 → eotdl-2023.7.19}/eotdl/commands/datasets.py

@@ -4,26 +4,32 @@ from pathlib import Path
 from ..datasets import (
     retrieve_datasets,
     download_dataset,
-    ingest_file,
     ingest_folder,
+    ingest_stac,
 )
 
 app = typer.Typer()
 
 
 @app.command()
-def ingest(path: Path, dataset: str):
+def ingest(
+    path: Path,
+    f: bool = typer.Option(False, "--f", help="Force ingest even if file exists"),
+    d: bool = typer.Option(False, "--d", help="Delete files not in the dataset"),
+):
     """
-    Ingest a file
+    Ingest a dataset
 
-    path: Path to folder with data (limited to 10 files, not recursive!)
-    dataset: Name of the dataset
+    path: Path to folder with the dataset
     """
     try:
-        if path.is_dir():
-            ingest_folder(path, dataset, typer.echo)
+        if not path.is_dir():
+            typer.echo("Path must be a folder")
+            return
+        if "catalog.json" in [f.name for f in path.iterdir()]:
+            ingest_stac(str(path) + "/catalog.json", typer.echo)
         else:
-            ingest_file(path, dataset, typer.echo)
+            ingest_folder(path, f, d, typer.echo)
     except Exception as e:
         typer.echo(e)
 
@@ -38,7 +44,11 @@ def list():
 
 
 @app.command()
-def get(dataset: str, file: str = None, path: str = None):
+def get(
+    dataset: str,
+    path: str = None,
+    file: str = None,
+):
     """
     Download a dataset
 

{eotdl-2023.6.27 → eotdl-2023.7.19}/eotdl/curation/stac/__init__.py

@@ -1,4 +1,4 @@
 # from .stac import STACGenerator
 # from .utils import format_time_acquired
 # from .parsers import STACIdParser, StructuredParser, UnestructuredParser
-# from .dataframe import STACDataFrame, read_stac
+from .dataframe import STACDataFrame, read_stac

eotdl-2023.7.19/eotdl/curation/stac/dataframe.py

@@ -0,0 +1,144 @@
+"""
+Module for the STAC dataframe
+"""
+
+import pandas as pd
+import geopandas as gpd
+import pystac
+import json
+from geomet import wkt
+from os.path import join
+from os import makedirs
+from typing import Union
+from math import isnan
+from .utils import convert_df_geom_to_shape, get_all_children
+
+
+class STACDataFrame(gpd.GeoDataFrame):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def from_stac_file(self, stac_file):
+        """
+        Create a STACDataFrame from a STAC file
+        """
+        return read_stac(stac_file)
+
+    def to_stac(self, path):
+        """
+        Create a STAC catalog and children from a STACDataFrame
+        """
+        df = self.copy()
+
+        if "id" in df.columns and "stac_id" in df.columns:
+            id_column = "stac_id"
+            stac_id_exists = True
+        else:
+            id_column = "id"
+            stac_id_exists = False
+
+        # First, create the catalog and its folder, if exists
+        catalog_df = df[df["type"] == "Catalog"]
+
+        if catalog_df.empty:
+            makedirs(path, exist_ok=True)
+        else:
+            for index, row in catalog_df.iterrows():
+                root_output_folder = path + "/" + row[id_column]
+                makedirs(root_output_folder, exist_ok=True)
+                row_json = row.to_dict()
+
+                # Curate the json row
+                row_json = self.curate_json_row(row_json, stac_id_exists)
+
+                with open(join(root_output_folder, f"catalog.json"), "w") as f:
+                    json.dump(row_json, f)
+
+        # Second, create the collections and their folders, if exist
+        collections = dict()
+        collections_df = df[df["type"] == "Collection"]
+        for index, row in collections_df.iterrows():
+            stac_output_folder = join(root_output_folder, row[id_column])
+            collections[row[id_column]] = stac_output_folder
+            makedirs(stac_output_folder, exist_ok=True)
+            row_json = row.to_dict()
+
+            # Curate the json row
+            row_json = self.curate_json_row(row_json, stac_id_exists)
+
+            with open(join(stac_output_folder, f"collection.json"), "w") as f:
+                json.dump(row_json, f)
+
+        # Then, create the items and their folders, if exist
+        features_df = df[df["type"] == "Feature"]
+        for index, row in features_df.iterrows():
+            collection = row["collection"]
+            stac_output_folder = join(collections[collection], row[id_column])
+
+            # Convert the geometry from WKT back to geojson
+            row["geometry"] = row["geometry"].wkt
+            row["geometry"] = wkt.loads(row["geometry"])
+            makedirs(stac_output_folder, exist_ok=True)
+            row_json = row.to_dict()
+
+            # Curate the json row
+            row_json = self.curate_json_row(row_json, stac_id_exists)
+
+            with open(join(stac_output_folder, f'{row_json["id"]}.json'), "w") as f:
+                json.dump(row_json, f)
+
+    def curate_json_row(self, row: dict, stac_id_exists: bool) -> dict:
+        """
+        Curate the json row of a STACDataFrame, in order to generate a valid STAC file
+
+        :param row: row of a STACDataFrame
+        :param stac_id_exists: if the stac_id column exists
+        """
+        keys_to_remove = list()
+
+        # Remove the created_at and modified_at columns, if the STACDataFrame comes from GeoDB
+        for i in "created_at", "modified_at":
+            if i in row.keys():
+                keys_to_remove.append(i)
+
+        # Rename the stac_id column to id, to avoid conflicts with the id column
+        if stac_id_exists:
+            row["id"] = row["stac_id"]
+            del row["stac_id"]
+
+        # Remove the NaN values and empty strings
+        for k, v in row.items():
+            if (isinstance(v, float) and isnan(v)) or v == "":
+                keys_to_remove.append(k)
+        for key in keys_to_remove:
+            del row[key]
+        del row["geometry"]
+
+        return row
+
+
+def read_stac(
+    stac_file: Union[pystac.Catalog, pystac.Collection, str],
+    geometry_column: str = "geometry",
+) -> STACDataFrame:
+    """
+    Read a STAC file and return a STACDataFrame
+
+    :param stac_file: STAC file to read
+    :param geometry_column: name of the geometry column
+    """
+    if isinstance(stac_file, str):
+        stac_file = pystac.read_file(stac_file)
+    children = get_all_children(stac_file)
+
+    # Convert Dataframe to STACDataFrame
+    dataframe = pd.DataFrame(children)
+    dataframe[geometry_column] = dataframe.apply(convert_df_geom_to_shape, axis=1)
+    stac_dataframe = STACDataFrame(
+        dataframe,
+        crs="EPSG:4326",
+        geometry=gpd.GeoSeries.from_wkt(dataframe[geometry_column]),
+    )
+
+    return stac_dataframe

{eotdl-2023.6.27 → eotdl-2023.7.19}/eotdl/datasets/__init__.py

@@ -1,3 +1,3 @@
-from .ingest import ingest_file, ingest_folder
+from .ingest import ingest_file, ingest_folder, ingest_q1, ingest_stac
 from .download import download_dataset
 from .retrieve import retrieve_datasets, retrieve_dataset, list_datasets

eotdl-2023.7.19/eotdl/datasets/ingest.py

@@ -0,0 +1,52 @@
+import os
+
+from ..src.repos import APIRepo
+from ..src.usecases.datasets import IngestFile, IngestFolder, IngestSTAC
+from ..auth import with_auth
+
+
+allowed_extensions = [
+    ".zip",
+    ".tar",
+    ".tar.gz",
+    ".csv",
+    ".txt",
+    ".json",
+    ".pdf",
+    ".md",
+    ".yml",
+]
+
+
+def ingest_q1(dataset, stac_catalog):
+    print("hola")
+    return
+
+
+@with_auth
+def ingest_file(
+    file, dataset_id, logger=None, allowed_extensions=allowed_extensions, user=None
+):
+    api_repo = APIRepo()
+    ingest = IngestFile(api_repo, allowed_extensions, logger)
+    inputs = ingest.Inputs(file=file, dataset_id=dataset_id, user=user)
+    outputs = ingest(inputs)
+    return outputs.data
+
+
+@with_auth
+def ingest_folder(folder, force, delete, logger=None, user=None):
+    api_repo = APIRepo()
+    ingest = IngestFolder(api_repo, ingest_file, allowed_extensions, logger)
+    inputs = ingest.Inputs(folder=folder, user=user, force=force, delete=delete)
+    outputs = ingest(inputs)
+    return outputs.dataset
+
+
+@with_auth
+def ingest_stac(stac_catalog, dataset, logger=None, user=None):
+    api_repo = APIRepo()
+    ingest = IngestSTAC(api_repo, ingest_file, allowed_extensions)
+    inputs = ingest.Inputs(stac_catalog=stac_catalog, dataset=dataset, user=user)
+    outputs = ingest(inputs)
+    return outputs.dataset

eotdl-2023.7.19/eotdl/src/models/__init__.py

@@ -0,0 +1 @@
+from .metadata import Metadata

eotdl-2023.7.19/eotdl/src/models/metadata.py

@@ -0,0 +1,16 @@
+from pydantic import BaseModel, validator
+from typing import List
+
+
+class Metadata(BaseModel):
+    authors: List[str]
+    license: str
+    source: str
+    name: str
+
+    # validate source is a URL
+    @validator("source")
+    def source_is_url(cls, v):
+        if not v.startswith("http") and not v.startswith("https"):
+            raise ValueError("source must be a URL")
+        return v

{eotdl-2023.6.27 → eotdl-2023.7.19}/eotdl/src/repos/APIRepo.py

@@ -6,11 +6,13 @@ from concurrent.futures import ThreadPoolExecutor
 import time
 import multiprocessing
 import hashlib
+import geopandas as gpd
 
 
 class APIRepo:
     def __init__(self, url=os.getenv("EOTDL_API_URL", "https://api.eotdl.com/")):
         self.url = url
+        # print(self.url)
 
     def login(self):
         return requests.get(self.url + "auth/login")
@@ -22,6 +24,16 @@ class APIRepo:
         response = requests.get(self.url + "auth/logout")
         return response.json()["logout_url"]
 
+    def create_dataset(self, metadata, id_token):
+        response = requests.post(
+            self.url + "datasets",
+            json=metadata,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if response.status_code == 200:
+            return response.json(), None
+        return None, response.json()["detail"]
+
     def retrieve_datasets(self):
         return requests.get(self.url + "datasets").json()
 
@@ -34,12 +46,7 @@ class APIRepo:
     def download_file(self, dataset, dataset_id, file, id_token, path):
         url = self.url + "datasets/" + dataset_id + "/download/" + file
         headers = {"Authorization": "Bearer " + id_token}
-        if path is None:
-            path = str(Path.home()) + "/.eotdl/datasets/" + dataset
-            os.makedirs(path, exist_ok=True)
         path = f"{path}/{file}"
-        # if os.path.exists(path):
-        #     raise Exception("File already exists")
         with requests.get(url, headers=headers, stream=True) as r:
             r.raise_for_status()
             total_size = int(r.headers.get("content-length", 0))
@@ -55,14 +62,21 @@ class APIRepo:
             progress_bar.close()
             return path
 
-    def ingest_file(self, file, dataset, id_token, checksum):
+    def ingest_file(self, file, dataset_id, id_token, checksum=None):
         reponse = requests.post(
-            self.url + "datasets",
+            self.url + "datasets/" + dataset_id,
             files={"file": open(file, "rb")},
-            data={
-                "dataset": dataset,
-                "checksum": checksum,
-            },
+            data={"checksum": checksum} if checksum else None,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if reponse.status_code != 200:
+            return None, reponse.json()["detail"]
+        return reponse.json(), None
+
+    def ingest_file_url(self, file, dataset, id_token):
+        reponse = requests.post(
+            self.url + "datasets/url",
+            json={"dataset": dataset, "url": file},
             headers={"Authorization": "Bearer " + id_token},
         )
         if reponse.status_code != 200:
@@ -76,11 +90,11 @@ class APIRepo:
                 break
             yield data
 
-    def prepare_large_upload(self, file, dataset, checksum, id_token):
+    def prepare_large_upload(self, file, dataset_id, checksum, id_token):
         filename = Path(file).name
         response = requests.post(
-            self.url + "datasets/uploadId",
-            json={"name": filename, "checksum": checksum, "dataset": dataset},
+            self.url + f"datasets/{dataset_id}/uploadId",
+            json={"name": filename, "checksum": checksum},
             headers={"Authorization": "Bearer " + id_token},
         )
         if response.status_code != 200:
@@ -204,63 +218,29 @@ class APIRepo:
             return None, r.json()["detail"]
         return r.json(), None
 
-    def ingest_large_dataset_parallel(
-        self,
-        path,
-        upload_id,
-        dataset_id,
-        id_token,
-        parts,
-        threads,
-    ):
-        # Create thread pool executor
-        max_workers = threads if threads > 0 else multiprocessing.cpu_count()
-        executor = ThreadPoolExecutor(max_workers=max_workers)
-
-        # Divide file into chunks and create tasks for each chunk
-        offset = 0
-        tasks = []
-        content_path = os.path.abspath(path)
-        content_size = os.stat(content_path).st_size
-        chunk_size = self.get_chunk_size(content_size)
-        total_chunks = content_size // chunk_size
-        while offset < content_size:
-            chunk_end = min(offset + chunk_size, content_size)
-            part = str(offset // chunk_size + 1)
-            if part not in parts:
-                tasks.append((offset, chunk_end, part))
-            offset = chunk_end
-
-        # Define the function that will upload each chunk
-        def upload_chunk(start, end, part):
-            # print(f"Uploading chunk {start} - {end}", part)
-            with open(content_path, "rb") as f:
-                f.seek(start)
-                chunk = f.read(end - start)
-            checksum = hashlib.md5(chunk).hexdigest()
-            response = requests.post(
-                self.url + "datasets/chunk",
-                files={"file": chunk},
-                headers={
-                    "Authorization": "Bearer " + id_token,
-                    "Upload-Id": upload_id,
-                    "Dataset-Id": dataset_id,
-                    "Checksum": checksum,
-                    "Part-Number": str(part),
-                },
-            )
-            if response.status_code != 200:
-                print(f"Failed to upload chunk {start} - {end}")
-            return response
+    def delete_file(self, dataset_id, file_name, id_token):
+        response = requests.delete(
+            self.url + "datasets/" + dataset_id + "/file/" + file_name,
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if response.status_code != 200:
+            return None, response.json()["detail"]
+        return response.json(), None
 
-        # Submit each task to the executor
-        with tqdm(total=total_chunks) as pbar:
-            futures = []
-            for task in tasks:
-                future = executor.submit(upload_chunk, *task)
-                future.add_done_callback(lambda p: pbar.update())
-                futures.append(future)
+    def ingest_stac(self, stac_json, dataset, id_token):
+        reponse = requests.post(
+            self.url + "datasets/stac",
+            json={"dataset": dataset, "stac": stac_json},
+            headers={"Authorization": "Bearer " + id_token},
+        )
+        if reponse.status_code != 200:
+            return None, reponse.json()["detail"]
+        return reponse.json(), None
 
-            # Wait for all tasks to complete
-            for future in futures:
-                future.result()
+    def download_stac(self, dataset_id, id_token):
+        url = self.url + "datasets/" + dataset_id + "/download"
+        headers = {"Authorization": "Bearer " + id_token}
+        response = requests.get(url, headers=headers)
+        if response.status_code != 200:
+            return None, response.json()["detail"]
+        return gpd.GeoDataFrame.from_features(response.json()["features"]), None

eotdl-2023.7.19/eotdl/src/usecases/datasets/DownloadDataset.py

@@ -0,0 +1,79 @@
+from pydantic import BaseModel
+from ....src.utils import calculate_checksum
+from ....curation.stac import STACDataFrame
+from pathlib import Path
+import os
+
+
+class DownloadDataset:
+    def __init__(self, repo, retrieve_dataset, logger):
+        self.repo = repo
+        self.retrieve_dataset = retrieve_dataset
+        self.logger = logger if logger else print
+
+    class Inputs(BaseModel):
+        dataset: str
+        file: str = None
+        path: str = None
+        user: dict
+
+    class Outputs(BaseModel):
+        dst_path: str
+
+    def download(self, dataset, dataset_id, file, checksum, path, user):
+        self.logger(f"Downloading {file}")
+        dst_path = self.repo.download_file(
+            dataset, dataset_id, file, user["id_token"], path
+        )
+        if calculate_checksum(dst_path) != checksum:
+            self.logger(f"Checksum for {file} does not match")
+        self.logger(f"Done")
+        return dst_path
+
+    def __call__(self, inputs: Inputs) -> Outputs:
+        dataset = self.retrieve_dataset(inputs.dataset)
+        if inputs.path is None:
+            download_path = str(Path.home()) + "/.eotdl/datasets/" + inputs.dataset
+        else:
+            download_path = inputs.path + "/" + inputs.dataset
+        os.makedirs(download_path, exist_ok=True)
+        if dataset["quality"] == 0:
+            if inputs.file:
+                files = [f for f in dataset["files"] if f["name"] == inputs.file]
+                if not files:
+                    raise Exception(f"File {inputs.file} not found")
+                if len(files) > 1:
+                    raise Exception(f"Multiple files with name {inputs.file} found")
+                dst_path = self.download(
+                    inputs.dataset,
+                    dataset["id"],
+                    inputs.file,
+                    files[0]["checksum"],
+                    download_path,
+                    inputs.user,
+                )
+                return self.Outputs(dst_path=dst_path)
+            for file in dataset["files"]:
+                dst_path = self.download(
+                    inputs.dataset,
+                    dataset["id"],
+                    file["name"],
+                    file["checksum"],
+                    download_path,
+                    inputs.user,
+                )
+            return self.Outputs(dst_path="/".join(dst_path.split("/")[:-1]))
+        else:
+            gdf, error = self.repo.download_stac(
+                dataset["id"],
+                inputs.user["id_token"],
+            )
+            if error:
+                raise Exception(error)
+            df = STACDataFrame(gdf)
+            # df.geometry = df.geometry.apply(lambda x: Polygon() if x is None else x)
+            path = inputs.path
+            if path is None:
+                path = str(Path.home()) + "/.eotdl/datasets/" + dataset["name"]
+            df.to_stac(path)
+            return self.Outputs(dst_path=path)

{eotdl-2023.6.27 → eotdl-2023.7.19}/eotdl/src/usecases/datasets/IngestFile.py

@@ -13,11 +13,11 @@ class IngestFile:
 
     class Inputs(BaseModel):
         file: typing.Any
-        dataset: str = None
+        dataset_id: str
         user: dict
 
     class Outputs(BaseModel):
-        dataset: dict
+        data: dict
 
     def __call__(self, inputs: Inputs) -> Outputs:
         # validate file extension
@@ -26,31 +26,35 @@ class IngestFile:
             raise Exception(
                 f"Only {', '.join(self.allowed_extensions)} files are allowed"
             )
+        id_token = inputs.user["id_token"]
         self.logger(f"Uploading file {inputs.file}...")
+        # if inputs.file.startswith("http://") or inputs.file.startswith("https://"):
+        #     data, error = self.repo.ingest_file_url(
+        #         inputs.file, inputs.metadata.name, id_token
+        #     )
+        # else:
         self.logger("Computing checksum...")
         checksum = calculate_checksum(inputs.file)
         self.logger(checksum)
-        self.logger("Ingesting dataset...")
-        id_token = inputs.user["id_token"]
+        self.logger("Ingesting file...")
         filesize = os.path.getsize(inputs.file)
         # ingest small file
         if filesize < 1024 * 1024 * 16:  # 16 MB
             data, error = self.repo.ingest_file(
-                inputs.file, inputs.dataset, id_token, checksum
+                inputs.file, inputs.dataset_id, id_token, checksum
             )
             if error:
                 raise Exception(error)
             self.logger("Done")
-            return self.Outputs(dataset=data)
+            return self.Outputs(data=data)
         # ingest large file
         upload_id, parts = self.repo.prepare_large_upload(
-            inputs.file, inputs.dataset, checksum, id_token
+            inputs.file, inputs.dataset_id, checksum, id_token
         )
-        print(upload_id, parts)
         self.repo.ingest_large_dataset(inputs.file, upload_id, id_token, parts)
         self.logger("\nCompleting upload...")
         data, error = self.repo.complete_upload(id_token, upload_id)
         if error:
             raise Exception(error)
         self.logger("Done")
-        return self.Outputs(dataset=data)
+        return self.Outputs(data=data)

eotdl-2023.7.19/eotdl/src/usecases/datasets/IngestFolder.py

@@ -0,0 +1,98 @@
+from pydantic import BaseModel
+import os
+from pathlib import Path
+import yaml
+from ...models import Metadata
+
+
+class IngestFolder:
+    def __init__(self, repo, ingest_file, allowed_extensions, logger):
+        self.repo = repo
+        self.ingest_file = ingest_file
+        self.allowed_extensions = allowed_extensions
+        self.logger = logger if logger else print
+
+    class Inputs(BaseModel):
+        folder: Path
+        user: dict
+        force: bool = False
+        delete: bool = False
+
+    class Outputs(BaseModel):
+        dataset: dict
+
+    def __call__(self, inputs: Inputs) -> Outputs:
+        # validate folder
+        self.logger("Uploading directory (only files, not recursive)")
+        items = list(inputs.folder.glob("*"))
+        filtered_items = [item for item in items if item.is_file()]
+        filtered_items = [
+            item for item in filtered_items if item.suffix in self.allowed_extensions
+        ]
+        if len(filtered_items) == 0:
+            raise Exception("No files found in directory")
+        if len(filtered_items) > 10:
+            raise Exception("Too many files in directory, limited to 10")
+        if "metadata.yml" not in [item.name for item in filtered_items]:
+            raise Exception("metadata.yml not found in directory")
+        # load metadata
+        metadata = yaml.safe_load(
+            open(inputs.folder.joinpath("metadata.yml"), "r").read()
+        )
+        metadata = Metadata(**metadata)
+        # remove metadata.yml from files
+        filtered_items = [
+            item for item in filtered_items if item.name != "metadata.yml"
+        ]
+        # create dataset
+        data, error = self.repo.create_dataset(metadata.dict(), inputs.user["id_token"])
+        # dataset may already exists, but if user is owner continue ingesting files
+        current_files = []
+        if error:
+            data, error2 = self.repo.retrieve_dataset(metadata.name)
+            if error2:
+                raise Exception(error)
+            if data["uid"] != inputs.user["sub"]:
+                raise Exception("Dataset already exists.")
+            data["dataset_id"] = data["id"]
+            current_files = [item["name"] for item in data["files"]]
+            if len(current_files) > 0 and not inputs.force:
+                self.logger(
+                    "The following files already exist and will not be uploaded (use --f to force re-upload):"
+                )
+                for item in current_files:
+                    self.logger(f"{item}")
+            # TODO: delete current_files that are not in filtered_items if --delete
+            hanged_files = [
+                file
+                for file in current_files
+                if file not in [item.name for item in filtered_items]
+            ]
+            if len(hanged_files) > 0:
+                self.logger(
+                    "The following files are no longer in your dataset (use --d to delete):"
+                )
+                for item in hanged_files:
+                    self.logger(f"{item}")
+                    if inputs.delete:
+                        self.logger(f"Deleting file {item}...")
+                        _, error = self.repo.delete_file(
+                            data["dataset_id"], item, inputs.user["id_token"]
+                        )
+                        if error:
+                            self.logger(error)
+                        else:
+                            self.logger("Done")
+            filtered_items = [
+                item for item in filtered_items if item.name not in current_files
+            ]
+        dataset_id = data["dataset_id"]
+        # upload files
+        if len(filtered_items) == 0:
+            raise Exception("No files to upload")
+        self.logger("The following files will be uploaded:")
+        for item in filtered_items:
+            self.logger(f"{item.name}")
+        for item in filtered_items:
+            data = self.ingest_file(item, dataset_id, logger=self.logger)
+        return self.Outputs(dataset=data)

eotdl-2023.7.19/eotdl/src/usecases/datasets/IngestSTAC.py

@@ -0,0 +1,42 @@
+from pydantic import BaseModel
+from ....curation.stac import STACDataFrame
+import json
+
+
+class IngestSTAC:
+    def __init__(self, repo, ingest_file, allowed_extensions):
+        self.repo = repo
+        self.ingest_file = ingest_file
+        self.allowed_extensions = allowed_extensions
+
+    class Inputs(BaseModel):
+        stac_catalog: str
+        dataset: str
+        user: dict
+
+    class Outputs(BaseModel):
+        dataset: dict
+
+    def __call__(self, inputs: Inputs) -> Outputs:
+        # load the STAC catalog as a STACsetFrame
+        df = STACDataFrame.from_stac_file(inputs.stac_catalog)
+        # upload all assets to EOTDL
+        for row in df.dropna(subset=["assets"]).iterrows():
+            # for asset in df.assets.dropna().values[:10]:
+            try:
+                for k, v in row[1]["assets"].items():
+                    data = self.ingest_file(
+                        v["href"],
+                        inputs.dataset,
+                        allowed_extensions=self.allowed_extensions + [".tif", ".tiff"],
+                    )
+                    file_url = f"{self.repo.url}datasets/{data['dataset_id']}/download/{data['file_name']}"
+                    df.loc[row[0], "assets"][k]["href"] = file_url
+            except Exception as e:
+                break
+        data, error = self.repo.ingest_stac(
+            json.loads(df.to_json()), inputs.dataset, inputs.user["id_token"]
+        )
+        if error:
+            raise Exception(error)
+        return self.Outputs(dataset=data)

{eotdl-2023.6.27 → eotdl-2023.7.19}/eotdl/src/usecases/datasets/__init__.py

@@ -5,3 +5,4 @@ from .RetrieveDataset import RetrieveDataset
 from .RetrieveDatasets import RetrieveDatasets
 from .IngestFile import IngestFile
 from .IngestFolder import IngestFolder
+from .IngestSTAC import IngestSTAC

{eotdl-2023.6.27 → eotdl-2023.7.19}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "eotdl"
-version = "2023.06.27"
+version = "2023.07.19"
 description = "Earth Observation Training Data Lab"
 authors = ["EarthPulse <it@earthpulse.es>"]
 license = "MIT"

{eotdl-2023.6.27 → eotdl-2023.7.19}/setup.py

@@ -12,6 +12,7 @@ packages = \
  'eotdl.datasets',
  'eotdl.src',
  'eotdl.src.errors',
+ 'eotdl.src.models',
  'eotdl.src.repos',
  'eotdl.src.usecases',
  'eotdl.src.usecases.auth',
@@ -34,7 +35,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'eotdl',
-    'version': '2023.6.27',
+    'version': '2023.7.19',
     'description': 'Earth Observation Training Data Lab',
     'long_description': '# eotdl \n\nThis is the main library and CLI for EOTDL.\n\n',
     'author': 'EarthPulse',

eotdl-2023.6.27/eotdl/datasets/ingest.py

@@ -1,61 +0,0 @@
-from ..src.repos import APIRepo
-from ..src.usecases.datasets import IngestFile, IngestFolder
-from ..auth import with_auth
-
-allowed_extensions = [
-    ".zip",
-    ".tar",
-    ".tar.gz",
-    ".csv",
-    ".txt",
-    ".json",
-    ".pdf",
-    ".md",
-]
-
-
-@with_auth
-def ingest_file(file, dataset, logger=None, user=None):
-    api_repo = APIRepo()
-    ingest = IngestFile(api_repo, allowed_extensions, logger)
-    inputs = ingest.Inputs(file=file, dataset=dataset, user=user)
-    outputs = ingest(inputs)
-    return outputs.dataset
-
-
-@with_auth
-def ingest_folder(folder, dataset, logger=None, user=None):
-    api_repo = APIRepo()
-    ingest = IngestFolder(api_repo, ingest_file, allowed_extensions, logger)
-    inputs = ingest.Inputs(folder=folder, dataset=dataset, user=user)
-    outputs = ingest(inputs)
-    return outputs.dataset
-
-
-# @with_auth
-# def ingest_dataset(name, description, path, logger=None, user=None):
-#     api_repo = APIRepo()
-#     ingest = IngestDataset(
-#         api_repo,
-#     )
-#     inputs = ingest.Inputs(name=name, description=description, path=path, user=user)
-#     outputs = ingest(inputs)
-#     return outputs.dataset
-
-
-# @with_auth
-# def ingest_large_dataset(name, path, logger=None, user=None):
-#     api_repo = APIRepo()
-#     ingest = IngestLargeDataset(api_repo, logger)
-#     inputs = ingest.Inputs(name=name, path=path, user=user)
-#     outputs = ingest(inputs)
-#     return outputs.dataset
-
-
-# def ingest_q0(dataset, path):
-#     return ingest_large_dataset(dataset, path)
-
-
-# def ingest_q1(dataset, stac_catalog):
-#     print("holas")
-#     return

eotdl-2023.6.27/eotdl/src/usecases/datasets/DownloadDataset.py

@@ -1,56 +0,0 @@
-from pydantic import BaseModel
-from ....src.utils import calculate_checksum
-
-
-class DownloadDataset:
-    def __init__(self, repo, retrieve_dataset, logger):
-        self.repo = repo
-        self.retrieve_dataset = retrieve_dataset
-        self.logger = logger if logger else print
-
-    class Inputs(BaseModel):
-        dataset: str
-        file: str = None
-        path: str = None
-        user: dict
-
-    class Outputs(BaseModel):
-        dst_path: str
-
-    def download(self, dataset, dataset_id, file, checksum, path, user):
-        self.logger(f"Downloading {file}")
-        dst_path = self.repo.download_file(
-            dataset, dataset_id, file, user["id_token"], path
-        )
-        if calculate_checksum(dst_path) != checksum:
-            self.logger(f"Checksum for {file} does not match")
-        self.logger(f"Done")
-        return dst_path
-
-    def __call__(self, inputs: Inputs) -> Outputs:
-        dataset = self.retrieve_dataset(inputs.dataset)
-        if inputs.file:
-            files = [f for f in dataset["files"] if f["name"] == inputs.file]
-            if not files:
-                raise Exception(f"File {inputs.file} not found")
-            if len(files) > 1:
-                raise Exception(f"Multiple files with name {inputs.file} found")
-            dst_path = self.download(
-                inputs.dataset,
-                dataset["id"],
-                inputs.file,
-                files[0]["checksum"],
-                inputs.path,
-                inputs.user,
-            )
-            return self.Outputs(dst_path=dst_path)
-        for file in dataset["files"]:
-            dst_path = self.download(
-                inputs.dataset,
-                dataset["id"],
-                file["name"],
-                file["checksum"],
-                inputs.path,
-                inputs.user,
-            )
-        return self.Outputs(dst_path="/".join(dst_path.split("/")[:-1]))

eotdl-2023.6.27/eotdl/src/usecases/datasets/IngestFolder.py

@@ -1,37 +0,0 @@
-from pydantic import BaseModel
-import os
-from pathlib import Path
-
-
-class IngestFolder:
-    def __init__(self, repo, ingest_file, allowed_extensions, logger):
-        self.repo = repo
-        self.ingest_file = ingest_file
-        self.allowed_extensions = allowed_extensions
-        self.logger = logger if logger else print
-
-    class Inputs(BaseModel):
-        folder: Path
-        dataset: str = None
-        user: dict
-
-    class Outputs(BaseModel):
-        dataset: dict
-
-    def __call__(self, inputs: Inputs) -> Outputs:
-        self.logger("Uploading directory (only files, not recursive)")
-        items = list(inputs.folder.glob("*"))
-        filtered_items = [item for item in items if item.is_file()]
-        filtered_items = [
-            item for item in filtered_items if item.suffix in self.allowed_extensions
-        ]
-        if len(filtered_items) == 0:
-            raise Exception("No files found in directory")
-        if len(filtered_items) > 10:
-            raise Exception("Too many files in directory, limited to 10")
-        self.logger("The following files will be uploaded:")
-        for item in filtered_items:
-            self.logger(f"{item.name}")
-        for item in filtered_items:
-            data = self.ingest_file(item, inputs.dataset, logger=self.logger)
-        return self.Outputs(dataset=data)